Merge branch 'unstable' into stable

app/coordinators/art_coordinator.py

@@ -0,0 +1,47 @@
+import requests
+from services import art_services, data_services, logging_services
+
+def update_data(arguments):
+    directory = arguments.target
+    v_setting = arguments.verbose
+    v_out = logging_services.log # Partial function -- for brevity.
+    save = data_services.store_json # (P.F.) For easier reading.
+    
+    v_out(v_setting, "Beginning to update Art data...")
+    
+    try:
+        raw_art_data = data_services.get_json(
+            "https://api.craigoates.net/api/1.0/Artwork")
+        v_out(v_setting, "Data from API retrived.")
+        
+        save(art_services.get_creation_date_totals(raw_art_data),
+             f"{directory}/art_creation_dates.json")
+        v_out(v_setting, "Art creation dates processed.")
+        
+        save(art_services.get_db_column_totals(raw_art_data, "category"),
+             f"{directory}/art_category.json")
+        v_out(v_setting, "Art categories processed.")
+        
+        save(art_services.get_db_column_totals(raw_art_data, "medium"),
+             f"{directory}/art_medium.json")
+        v_out(v_setting, "Art medium(s) totals processed.")
+
+        save(art_services.get_dimension_totals
+             (raw_art_data, "dimensions", "width"),
+             f"{directory}/art_width.json")
+        v_out(v_setting, "Art width totals processed.")
+    
+        save(art_services.get_dimension_totals
+             (raw_art_data, "dimensions", "height"),
+             f"{directory}/art_height.json")
+        v_out(v_setting, "Art height totals processed.")
+
+        save(art_services.get_dimension_totals
+             (raw_art_data, "dimensions", "depth"),
+             f"{directory}/art_depth.json")
+        v_out(v_setting, "Art depth totals processed.")
+
+        v_out(v_setting, "Completed updating Art data.")
+             
+    except Exception:
+        print("ERROR: [art_coordinator] Unable to update Art data.")

app/main.py

@@ -0,0 +1,11 @@
+from services import parser_services
+from coordinators import art_coordinator
+
+def main():
+    args = parser_services.create_args()
+    art_coordinator.update_data(args)
+    # update_software_data(args) # Future update.
+    # update_article_data(args) # Future update.
+
+if __name__ == "__main__":
+    main()

app/services/art_services.py

@@ -0,0 +1,92 @@
+from datetime import datetime
+
+'''
+Note: Hard-Coding "Months" and "Days" Sets
+======================================================================
+I have hardcoded the "months" and "days" sets because they are
+fixed values -- unless something monumental happens scientifically
+or politically. On top of that, this makes the graphs easier to
+read because they are in chronological order. This is not
+guaranteed if the "keys" for "months" and "sets" are formed from
+the data-object this function receives.
+Unfortunately, I cannot do the same for years. That will continue
+to grow as the years roll on through here -- unless something
+monumental happens scientifically or politically.
+This code is intended to be used in graphs -- in co-data project.
+'''
+def get_creation_date_totals(data):
+    years = {}
+    
+    months = {"1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0,
+              "7": 0, "8": 0, "9": 0,"10": 0, "11": 0, "12": 0}
+
+    days = {"1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0,
+            "7": 0, "8": 0, "9": 0,"10": 0, "11": 0, "12": 0,
+            "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0,
+            "19": 0, "20": 0, "21": 0,"22": 0, "23": 0, "24": 0,
+            "25": 0, "26": 0, "27": 0, "28": 0, "29": 0, "30": 0,
+            "31": 0}
+
+    for item in data:
+        ft = datetime.fromisoformat(item["dateCreated"])
+
+        if str(ft.year) in years:
+            years[str(ft.year)] += 1
+        else:
+            years[str(ft.year)] = 1
+            
+        if str(ft.month) in months:
+            months[str(ft.month)] += 1
+        else:
+            months[str(ft.month)] = 1
+            
+        if str(ft.day) in days:
+            days[str(ft.day)] += 1
+        else:
+            days[str(ft.day)] = 1
+
+    return [years, months, days]
+
+def get_category_totals(data):
+    categories = {}
+    for item in data:
+        cat = item["category"]
+        '''
+        The join and split is because the data returned from the A.P.I.
+        call contains a lot of white spaces. This just cleans it up.
+        The white space was, also, making the chart render incorrectly.
+        '''
+        cat = ''.join(cat.split())
+        if cat in categories:
+            total = categories.get(cat)
+            categories[cat] = total + 1
+        else:
+           categories[cat] = 1
+    return categories
+
+def get_db_column_totals(data, column_name):
+    column_data = {}
+    for item in data:
+        col = item[column_name]
+        col = " ".join(col.split())
+        # print(col)
+        if col in column_data:
+            total = column_data.get(col)
+            column_data[col] = total + 1
+        else:
+            column_data[col] = 1
+    return column_data
+
+def get_dimension_totals(data, column_name, dimension):
+    dimensions = {}
+    for item in data:
+        dim = item[column_name]
+        distance = dim[dimension]["value"]["distance"]
+        if distance is not None:
+            w = str(distance)
+            if w not in dimensions:
+                dimensions[str(distance)] = 1
+            else:
+                total = dimensions.get(str(distance))
+                dimensions[str(distance)] = total + 1
+    return dimensions

app/services/charting_services.py

@@ -0,0 +1,73 @@
+from bokeh.plotting import figure, output_file, show
+from bokeh.embed import components
+from bokeh.io import show, output_file
+from bokeh.models import ColumnDataSource
+from bokeh.palettes import Blues256, Category20c
+from bokeh.plotting import figure
+from bokeh.transform import factor_cmap, cumsum
+
+def build_creation_date_chart(data, title):
+    units = list(data.keys())
+    totals = list(data.values())
+
+    if title == "By Year":
+        units.reverse()
+        totals.reverse()
+        
+    source = ColumnDataSource(data=dict(units=units, totals=totals))
+
+    chart = figure(x_range=units, plot_height=350, plot_width=800,
+               toolbar_location=None, title=title)
+
+    chart.vbar(x='units', top='totals', width=0.9, source=source,
+           legend_field="units",line_color='white',
+           fill_color=factor_cmap('units', palette=Blues256, factors=units))
+    
+    chart.title.text_font_size = '18pt'
+    chart.title.text_font = "intended, sans-serif"
+    chart.title.text_color = "#424242"
+    chart.outline_line_color = "#424242"
+    chart.xgrid.grid_line_color = None
+    chart.ygrid.grid_line_color = "#424242"
+    chart.y_range.start = 0
+    chart.y_range.end = (max(totals) + 10)
+    chart.background_fill_color = "#f5f5f6"
+    chart.border_fill_color = "#f5f5f6"
+    chart.legend.visible = False
+
+    script, div = components(chart)
+
+    return [script, div]
+
+def build_sorted_chart(data, title):
+    d = sorted(data.items())
+    totals = list()
+    units = list()
+    for i in d:
+        units.append(i[0])
+        totals.append(i[1])
+        
+    source = ColumnDataSource(data=dict(units=units, totals=totals))
+
+    chart = figure(x_range=units, plot_height=350, plot_width=800,
+               toolbar_location=None, title=title)
+
+    chart.vbar(x='units', top='totals', width=0.9, source=source,
+           legend_field="units",line_color='white',
+           fill_color=factor_cmap('units', palette=Blues256, factors=units))
+    
+    chart.title.text_font_size = '18pt'
+    chart.title.text_font = "intended, sans-serif"
+    chart.title.text_color = "#424242"
+    chart.outline_line_color = "#424242"
+    chart.xgrid.grid_line_color = None
+    chart.ygrid.grid_line_color = "#424242"
+    chart.y_range.start = 0
+    chart.y_range.end = (max(totals) + 10)
+    chart.background_fill_color = "#f5f5f6"
+    chart.border_fill_color = "#f5f5f6"
+    chart.legend.visible = False
+
+    script, div = components(chart)
+
+    return [script, div]

app/services/data_services.py

@@ -0,0 +1,21 @@
+import requests
+import json
+
+def get_data(url):
+    return requests.get(url)
+
+def get_json(url):
+    return requests.get(url).json()
+
+def store_json(data, file_name):
+    with open (file_name, "w") as outfile:
+        json.dump(data, outfile, indent = 4)
+
+def load_json(file_name):
+    with open(file_name, "r") as infile:
+        data = json.load(infile)
+    return data
+
+def store_txt(data, file_name):
+    with open(file_name, "w") as outfile:
+        outfile.write(data)

app/services/logging_services.py

@@ -0,0 +1,6 @@
+# This is for outputting the program's status when the verbose switch
+# is used.
+
+def log(log_output, message):
+    if log_output is True:
+        print(message)

app/services/parser_services.py

@@ -0,0 +1,21 @@
+import argparse
+import os
+
+def dir_path(string):
+    if os.path.isdir(string):
+        return string
+    else:
+        raise NotADirectoryError(string)
+    
+def create_args():
+    parser = argparse.ArgumentParser(
+        "Parses the coblob database and transforms it. " +
+        "This is mostly for the the benefit of the co-data project. " +
+        "It, also, requires access to the co-api project, via the internet.")
+    parser.add_argument("-t", "--target", type = dir_path, required = True,
+                        help = "the location you would like the data to be stored at.")
+    parser.add_argument("-v", "--verbose", action = "store_true",
+                        help = "provides detailed output when program is running.")
+    
+    args = parser.parse_args()
+    return args

requirements.txt

@@ -0,0 +1,18 @@
+bokeh==2.0.2
+certifi==2020.4.5.1
+chardet==3.0.4
+idna==2.9
+Jinja2==2.11.2
+MarkupSafe==1.1.1
+numpy==1.18.3
+packaging==20.3
+Pillow==7.1.1
+pkg-resources==0.0.0
+pyparsing==2.4.7
+python-dateutil==2.8.1
+PyYAML==5.3.1
+requests==2.23.0
+six==1.14.0
+tornado==6.0.4
+typing-extensions==3.7.4.2
+urllib3==1.25.9