|
|
|
@ -89,30 +89,35 @@ wrong listings. |
|
|
|
Need to come back to this and finish it off. Left it in a mess because it's the |
|
|
|
Need to come back to this and finish it off. Left it in a mess because it's the |
|
|
|
end of the day (at time of commit) and need to get some sleep. |
|
|
|
end of the day (at time of commit) and need to get some sleep. |
|
|
|
|
|
|
|
|
|
|
|
#+begin_src lisp :results silent |
|
|
|
#+begin_src lisp :results output raw |
|
|
|
(with-open-file (out-stream |
|
|
|
(let ((filepath #P"working-data/2024-02-24-ideal-flatmate-manc.csv")) |
|
|
|
#P"working-data/2024-02-24-ideal-flatmate-manc.csv" |
|
|
|
(with-open-file (out-stream |
|
|
|
:direction :output |
|
|
|
filepath |
|
|
|
:if-exists :supersede) |
|
|
|
:direction :output |
|
|
|
(let ((row-id 0)) |
|
|
|
:if-exists :supersede) |
|
|
|
(format out-stream "ROW-ID,RAW-LISTING-INFO,RENT,LOCATION,URL~%") |
|
|
|
(let ((row-id 0)) |
|
|
|
(loop for file-path |
|
|
|
(format out-stream "ROW-ID,RAW-LISTING-INFO,RENT,LOCATION,URL~%") |
|
|
|
in (directory #P"raw-data/external/2024-02-24_ideal-flatmate-manc-listings/*.html") |
|
|
|
(loop for file-path |
|
|
|
do (with-open-file (in-stream file-path) |
|
|
|
in (directory #P"raw-data/external/2024-02-24_ideal-flatmate-manc-listings/*.html") |
|
|
|
(let* ((doc (plump:parse in-stream)) |
|
|
|
do (with-open-file (in-stream file-path) |
|
|
|
(listing (lquery:$ doc ".card-infos-left" (text))) |
|
|
|
(let* ((doc (plump:parse in-stream)) |
|
|
|
(cleaned-price (first (cl-ppcre:all-matches-as-strings "\\d+" (first (str:split "room" (aref listing 0)))))) |
|
|
|
(listing (lquery:$ doc ".card-infos-left" (text))) |
|
|
|
(cleaned-location (str:replace-all "," "" (second (str:split "room" (aref listing 0))))) |
|
|
|
(cleaned-price (first (cl-ppcre:all-matches-as-strings "\\d+" (first (str:split "room" (aref listing 0)))))) |
|
|
|
(url (lquery:$ doc "a" (attr "href")))) |
|
|
|
(cleaned-location (str:replace-all "," "" (second (str:split "room" (aref listing 0))))) |
|
|
|
(format out-stream "~d,~s,~d,~a,~a~%" |
|
|
|
(url (lquery:$ doc "a" (attr "href")))) |
|
|
|
row-id |
|
|
|
(format out-stream "~d,~s,~d,~a,~a~%" |
|
|
|
(aref listing 0) |
|
|
|
row-id |
|
|
|
cleaned-price |
|
|
|
(aref listing 0) |
|
|
|
cleaned-location |
|
|
|
cleaned-price |
|
|
|
(aref url 0)))) |
|
|
|
cleaned-location |
|
|
|
(incf row-id)))) |
|
|
|
(aref url 0)))) |
|
|
|
|
|
|
|
(incf row-id)))) |
|
|
|
|
|
|
|
(format t "[[file:./working-data/2024-02-24-ideal-flatmate-manc.csv]]")) |
|
|
|
#+end_src |
|
|
|
#+end_src |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#+RESULTS: |
|
|
|
|
|
|
|
[[file:./working-data/2024-02-24-ideal-flatmate-manc.csv]] |
|
|
|
|
|
|
|
|
|
|
|
The amount of listings on Ideal Flatmate is tiny compared to the others. I can |
|
|
|
The amount of listings on Ideal Flatmate is tiny compared to the others. I can |
|
|
|
basically show all them here. |
|
|
|
basically show all them here. |
|
|
|
|
|
|
|
|
|
|
|
|
