diff --git a/open-rent-manchester.org b/open-rent-manchester.org new file mode 100644 index 0000000..e2d96f6 --- /dev/null +++ b/open-rent-manchester.org @@ -0,0 +1,299 @@ +#+options: ':nil *:t -:t ::t <:t H:3 \n:nil ^:t arch:headline author:t +#+options: broken-links:nil c:nil creator:nil d:(not "LOGBOOK") date:t e:t +#+options: email:nil expand-links:t f:t inline:t num:t p:nil pri:nil prop:nil +#+options: stat:t tags:t tasks:t tex:t timestamp:t title:t toc:t todo:t |:t +#+title: Open Rent Manchester +#+date: \today +#+author: Craig Oates +#+email: craig@craigoates.net +#+language: en +#+select_tags: export +#+exclude_tags: noexport +#+creator: Emacs 29.1.90 (Org mode 9.7-pre) +#+cite_export: + +* Setup Common Lisp Environment + +I’ve copied the following code block over from other files. Run it if this is +your first file open in the session. + +#+begin_src lisp :session :results silent + (ql:quickload :com.inuoe.jzon) ; JSON parser. + (ql:quickload :dexador) ; HTTP requests. + (ql:quickload :plump) ; HTML/XML parser. + (ql:quickload :lquery) ; HTML/DOM manipulation. + (ql:quickload :lparallel) ; Parallel programming. + (ql:quickload :cl-ppcre) ; RegEx. library. + (ql:quickload :plot/vega) ; Vega plotting library. + (ql:quickload :lisp-stat) ; Stat's library. + (ql:quickload :data-frame) ; Data frame library eqv. to Python's Numpy. + (ql:quickload :str) ; String library, expands on 'string' library. +#+end_src + +* Gather Open Rent Data (Manually) + +- [[https://www.openrent.co.uk][Open Rent]] + +The site has an infinite-scroller feature, which means I can’t use ~curl~ to +grab a load of pages. With that said, the list isn’t as big as say +[[file:./spare-room-manchester.org][Spare Room (Manc.)]], so I can just save the page from within the browser and +parse that instead. + +The filters I used: + +- Date: 2024-03-02 +- Location: Manchester +- Price Range: £400–£1500 pcm +- Proximity: 15 km +- No student listings +- Only monthly rents listed (pcm) + +The website says it’s showing 120 of 741 properties found – with the filters +applied. + +The data is saved in the =raw-data/external/2024-03-02-open-rent-manc/= +directory. This data will not be committed to the repositories commit history, +as is the standard I set for other websites. + +* Clean Up and Parse Data + +Going to separate out the listing into their own HTML files and then build a CSV +file of that data. This is standard practice I've established in other +(Manchester based) data exploration files. + +#+begin_src shell :results silent + mkdir raw-data/external/2024-03-02-open-rent-manc-listings/ +#+end_src + +#+begin_src lisp :results silent + (let ((counter 0)) + (loop for file-path + in (directory #P"raw-data/external/2024-03-02-open-rent-manc/*.html") + do (with-open-file (in-stream file-path) + (let* ((doc (plump:parse in-stream)) + ;; .listing-info + (listings (lquery:$ doc "a.pli.clearfix" (serialize)))) + (loop for item across listings + do (let ((out-path + (merge-pathnames + #P"raw-data/external/2024-03-02-open-rent-manc-listings/" + (format nil "listing-~a.html" (write-to-string counter))))) + (with-open-file (out-stream + out-path + :direction :output + :if-exists :supersede) + (format out-stream "~a~%" item)) + (incf counter))))))) +#+end_src + +The site has a ‘Let Agreed’ section, which is of no use to me here. They are +listing which have already been taken. So, I will need to remove them from the +listings. + +#+begin_src shell :results output + #!/bin/bash + + target_string="Let Agreed" + directory="raw-data/external/2024-03-02-open-rent-manc-listings" + + # Loop over all files in the directory + for file in "$directory"/* + do + # If the target string is found in the file, delete the file + if grep -q "$target_string" "$file" + then + rm "$file" + echo "Deleted file: $file" + fi + done +#+end_src + +#+RESULTS: +: Deleted file: raw-data/external/2024-03-02-open-rent-manc-listings/listing-114.html +: Deleted file: raw-data/external/2024-03-02-open-rent-manc-listings/listing-115.html +: Deleted file: raw-data/external/2024-03-02-open-rent-manc-listings/listing-116.html +: Deleted file: raw-data/external/2024-03-02-open-rent-manc-listings/listing-117.html +: Deleted file: raw-data/external/2024-03-02-open-rent-manc-listings/listing-118.html +: Deleted file: raw-data/external/2024-03-02-open-rent-manc-listings/listing-119.html + +Deleting those files leaves, + +#+begin_src shell :results output + COUNT=$(ls -1 "raw-data/external/2024-03-02-open-rent-manc-listings/" | wc -l) + echo "There are $COUNT files left in the directory." +#+end_src + +#+RESULTS: +: There are 114 files left in the directory. + + +The 114 files left and the 6 files deleted bring the total amount of listings up +to 120, which is what the website stated is was displaying. So, all looks good +and ready to move forward. + +** Create CSV of Listings + +#+begin_src lisp :results output raw + (let ((row-id 0) + (out-file #P"working-data/2024-03-02-open-rent-manc.csv")) + (with-open-file (out-stream + out-file + :direction :output + :if-exists :supersede) + (format out-stream "ROW-ID,PRICE,LOCATION,URL~%") + (loop for input + in (directory #P"raw-data/external/2024-03-02-open-rent-manc-listings/*.html") + do (with-open-file (in-stream input) + (let* ((doc (plump:parse in-stream)) + (price (lquery:$ doc ".pl-title" (text))) + ;; e.g. Transform '£1,250 per month' to '1250'. + (cleaned-price + (first + (cl-ppcre:all-matches-as-strings + "\\d+" + (cl-ppcre:regex-replace-all + "[\\s\\n]+" + (format nil "~s" (str:replace-all "," "" (aref price 0))) + " ")))) + (location (lquery:$ doc ".listing-title" (text))) + (link (lquery:$ doc "a" (attr :href)))) + (format out-stream "~d,~d,~a,~a~%" + row-id + cleaned-price + (str:replace-all "," " " (aref location 0)) + (aref link 0)))) + (incf row-id))) + (format t "[[file:./~a]]" out-file)) +#+end_src + +#+RESULTS: +[[file:./working-data/2024-03-02-open-rent-manc.csv]] + +There is nothing which stands out in the CSV file which looks like it needs +looking at manually. Going to start exploring the data. + +* Explore CSV Data for Open Rent (2024-03-02 + +#+begin_src lisp :session + (lisp-stat:defdf *or-manc* + (lisp-stat:read-csv #P"working-data/2024-03-02-open-rent-manc.csv")) +#+end_src + +#+RESULTS: +: # + +The number entries listed in the file is a bit to big for here, so going to just +show the head of the file. + +#+begin_src shell :results output raw + head -n 11 working-data/2024-03-02-open-rent-manc.csv | csvlook +#+end_src + +#+RESULTS: +| ROW-ID | PRICE | LOCATION | URL | +|--------+-------+------------------------------------------------------------------+------------------------------------| +| 0 | 400 | Room in a Shared House Manor Road M32 | https://www.openrent.co.uk/1862286 | +| 1 | 450 | Room in a Shared Flat Near Uni Oxford Road Piccadilly Stat M12 | https://www.openrent.co.uk/1999077 | +| 2 | 500 | Room in a Shared House Whiteway Street M9 | https://www.openrent.co.uk/2001232 | +| 3 | 1,000 | Room in a Shared House Ashfield Road M13 | https://www.openrent.co.uk/1976038 | +| 4 | 1,050 | Room in a Shared House Adelphi Wharf 1 M3 | https://www.openrent.co.uk/1916001 | +| 5 | 1,100 | 1 Bed Flat Teal Close WA14 | https://www.openrent.co.uk/2000362 | +| 6 | 1,150 | 1 Bed Flat Green Quarter M3 | https://www.openrent.co.uk/256714 | +| 7 | 1,150 | Room in a Shared Flat Water Street M3 | https://www.openrent.co.uk/1997609 | +| 8 | 1,200 | Room in a Shared House Noor Gardens M16 | https://www.openrent.co.uk/1967416 | +| 9 | 1,200 | 2 Bed Flat Swinton M28 | https://www.openrent.co.uk/1968257 | + +#+begin_src lisp :session :results file + (vega:defplot or-montly-manc + `(:title "Rent Rates (Bills Inc.) for Manchester on Open Rent (02/03/2024)" + :description "A breakdown of the rent rates for listings on Open Rent, in Manchester (02/03/2024)" + :data ,*or-manc* + :width 1900 + :height 1000 + :layer #((:mark (:type :bar) + :encoding (:x (:field :row-id :title "Assigned Id." :type :nominal) + :y (:field :price :title "Monthly Rent (£)" :type :quantitative) + :tooltip (:field :price))) + (:mark (:type :rule :color "darkorange" :size 3) + :encoding (:y (:field :price :type :quantitative :aggregate :average) + :tooltip (:field :price :type :quantitative :aggregate :average)))))) + (vega:write-html or-montly-manc "renders/2024-03-02-open-rent-manc.html") +#+end_src + +#+RESULTS: +[[file:renders/2024-03-02-open-rent-manc.html]] + +#+begin_src shell :results silent + mv ~/Downloads/visualization.png ./renders/2024-03-02-open-rent-manc.png +#+end_src + +[[file:./renders/2024-03-02-open-rent-manc.png]] + +#+begin_src lisp :session :results output code raw + (format t "- Mean Rent: £ ~a~%" (float (lisp-stat:mean *or-manc*:price))) + (format t "- Min. Rent: £ ~d~%" (reduce #'min *or-manc*:price)) + (format t "- Max. Rent: £ ~d" (reduce #'max *or-manc*:price)) +#+end_src + +#+RESULTS: +- Mean Rent: £ 735.4123 +- Min. Rent: £ 400 +- Max. Rent: £ 1350 + +#+begin_src calc :results ouput + 735.41 * 12 +#+end_src + +#+RESULTS: +: 8824.92 + +* Summary of Open Rent + +Based on the average rent for Open Rent, I would need to make around £9,000/yr +to cover my living costs. This does not include travel, food, clothing, +socialising or Income Tax payments. + +Most listings, if not all, are advertised as a house share. This puts Open Rent +more inline with [[file:./spare-room-manchester.org][Spare Room]] than the others. + +At this point, I've established a practice of add £5,000 to the annual average +rent total. This is to get a baseline idea of how bare-bones my income needs to +be, for me to survive. + +#+begin_src calc :results output + 8824.92 + 5000 +#+end_src + +#+RESULTS: +: 13824.92 + +Using £13824.92 as the starting point (see [[file:./uk-wage-tax.org][UK Wage and Tax Rated]], + +#+begin_src lisp :results output raw + (let* ((earning-target 13824.94) + (p-allow 12570) + (taxable-income (- earning-target p-allow)) + (tax-to-pay (* taxable-income 0.2)) + (total (- earning-target tax-to-pay))) + (format t "- Annual Target Salary: £~a~%" earning-target) + (format t "- Part of Salary which is Taxable: £~a~%" taxable-income) + (format t "- Tax to Pay: £~a~%" tax-to-pay) + (format t "- Salary After Tax: £~a~%" total)) +#+end_src + +#+RESULTS: +- Annual Target Salary: £13824.94 +- Part of Salary which is Taxable: £1254.9404 +- Tax to Pay: £250.98808 +- Salary After Tax: £13573.952 + +| Time Span | Value After Tax (£) | Mean Rent (£) | +|-----------------------+---------------------+---------------| +| Annually | 13573.95 | 735.41 | +| Monthly (Before Rent) | 1131.1625 | | +| Monthly (After Rent) | 395.7525 | | +| Weekly (After Rent) | 98.938125 | | +| Daily (After Rent) | 14.134018 | | +#+TBLFM: @3$2=@-1/12::@4$2=@-1-@-2$+1::@5$2=@-1/4::@6$2=@-1/7 + +So, a spending limit of £14.13/day.