(defpackage #:search (:use #:cl #:app-constants #:archive #:cl-json #:local-time #:utils) (:import-from #:dexador #:request) (:export #:build-keywords-string #:build-payload #:delete-entry #:find-entry #:get-id #:get-keywords #:submit-entry #:set-filter-attributes #:delete-all-entries #:create-dump #:update-ranking-rules #:repopulate-database #:delete-index)) (in-package #:search) ;; Explains the "~{~A~^,~}" in the format call below. ;; https://stackoverflow.com/questions/8830888/whats-the-canonical-way-to-join-strings-in-a-list (defun build-keywords-string (id) "Gets the keywords for `ID' in meilisearch DB and formats into a string. The string should look something like: 'art,blog post,testing,whatever'. One thing to note is meilisearch uses the comma to separate and create tokens of the string. So, the user can have spaces in their keywords but they cannot separate the keywords with a space. 'art blog post' is classed as three tokens and 'art,blog post' is classes as two." (format nil "~{~A~^,~}" (search:get-keywords (search:find-entry id)))) (defun build-payload (id-value title-value relative-path-value thumbnail-path-value publish-month-value publish-year-value keywords-value) "Creates a JSON object which reflects the schema in the meilsearch database. Note: The JSON object to encoded as a string." (cl-json:encode-json-to-string `((("id" . ,id-value) ("title" . ,title-value) ("relative-path" . ,relative-path-value) ("thumbnail-path" . ,thumbnail-path-value) ("year" . ,publish-year-value) ("month" . ,publish-month-value) ("keywords" . ,(cl-ppcre:split "," keywords-value)))))) (defun build-search-url (path) "Constructs the URL to connect to the meilisearch instance (beta or prod.) The function will check to see which environment the current instance of this site is running in and use the beta or prod. URL's to connect to meilisearch. `PATH' is the relative path which this function will concatenate onto the end of the base URL." (if (ritherdon-archive.config:developmentp) (concatenate 'string "http://localhost:7700" path) (utils:build-url-root) (concatenate 'string "https://www.nera.com" path))) (defun delete-entry (id) "Deletes and entry from the meilisearch database based on its `ID'. This does not affect this website's (nera) database -- only the meilisearch one." (dexador:request (build-search-url (format nil "/indexes/nera/documents/~a" id)); (get-id (find-entry id)))) :method :delete :use-connection-pool nil :headers `(("Authorization" . ,(meilisearch-api-key))) :verbose nil)) (defun documents-total () "Gets the total number of documents in the meilisearch database." (rest (second (second (assoc :indexes (cl-json:decode-json-from-string (dexador:request (build-search-url "/stats") :method :get :use-connection-pool nil :headers `(("Content-Type" . "application/json") ("Authorization" . ,(meilisearch-api-key))) :verbose nil))))))) (defun find-entry (title) "Finds the entry in the meilisearch database by its `TITLE'." (cl-json:decode-json-from-string (dexador:request (build-search-url "/indexes/nera/search") :method :post :use-connection-pool nil :headers `(("Content-Type" . "application/json") ("Authorization" . ,(meilisearch-api-key))) :content (format nil "{ \"q\": \"~a\", \"limit\": 1 }" title) :verbose nil))) (defun get-id (payload) "Gets the id from the JSON `PAYLOAD', make sure limit is set to 1." (rest (third (second (first payload))))) (defun get-keywords (payload) "Get the keywords from the JSON `PAYLOAD'." (rest (first (last (last (second (first payload))))))) (defun meilisearch-api-key () "Returns either the beta or prod. API key for meilisearch. The API key is determined based on which environment this website is currently running in." (if (ritherdon-archive.config:developmentp) "Bearer meilisearch-beta-key" "Bearer meilisearch-production-key-nera")) (defun submit-entry (payload) "Adds a new article to the meilisearch database." (dexador:request ;;"http://127.0.0.1:7700/indexes/nera/documents" (build-search-url "/indexes/nera/documents") :method :post :use-connection-pool nil :headers `(("Content-Type" . "application/json") ("Authorization" . ,(meilisearch-api-key))) :content payload :verbose nil)) (defun set-filter-attributes () "Sets the filter attributes in the Meilisearch database. These values are hard-coded into the system because they are based on what Nic has requested. She would like the filtering to consist on years and months." (utils:run-bash-command (format nil "curl -X PATCH \'~a\' -H \'Authorization: ~a\' -H \'Content-Type: application/json\' --data-binary \'{ \"filterableAttributes\": [ \"year\", \"month\", \"keywords\" ]}\'" (build-search-url "/indexes/nera/settings") (meilisearch-api-key)))) (defun delete-all-entries () "Deletes all the archive entries in the Meilisearch database -- not the DB." (utils:run-bash-command (format nil "curl -X DELETE \'~a\'" (build-search-url "/indexes/nera/documents")))) (defun create-dump () "Creates a dump of the Meilisearch database." (utils:run-bash-command (format nil "curl -X POST \'~a\'" (build-search-url "/dumps")))) (defun update-ranking-rules () "Updates the way Meilisearch ranks and orders the search results. The main intention for this function is to show the latest entries into the database first (when no search term is entered by the user)." (utils:run-bash-command (format nil "curl -X PATCH \'~a\' -H \'Authorization: ~a\' -H \'Content-Type: application/json\' --data-binary \'[ \"words\", \"typo\", \"proximity\", \"attribute\", \"sort\", \"exactness\",\"rank:asc\", \"year:desc\" ]\'" (build-search-url "/indexes/nera/settings") (meilisearch-api-key)))) (defun repopulate-database (archive-entries) "Empties the Meilisearch database and populates it with `ARCHIVE-ENTRIES'." (delete-all-entries) (loop for entry in archive-entries do (submit-entry (build-payload (archive::search-id-of entry) (archive::title-of entry) (format nil "view/archive/~a" (archive::slug-of entry)) (format nil "storage/thumb/archive/~a" (archive::slug-of entry)) (archive::month-of entry) (archive::year-of entry) (archive::keywords-of entry))))) (defun delete-index (index-name) "Deletes `INDEX-NAME' in Meilisearch DB, doesn't need to be this project. Because Meilisearch is a seperate service running alongside this website, it can host searchable databases for other projects on the system. `INDEX-NAME' refers to those other databases. I have not hard-coded this project's database name into this function out of convenience. I can call this function from SLIME/SLY and clean-up my Meilisearch dev. instance. It, also, helps if I've made botched this project's DB with an incorrect name and need to quickly delete it." (utils:run-bash-command (format nil "curl -X DELETE \'~a\'" (build-search-url (format nil "/indexes/~a" index-name)))))