From eb9cd4aba4daa9bc45ae4ff1ac1e9dd634e345c6 Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 9 Jan 2022 06:58:06 +0100 Subject: [PATCH] Reimplemented caching of yearly and weekly stats --- maloja/database/__init__.py | 38 ++++++----- maloja/database/cached.py | 69 +++++++++++++++++++ maloja/utilities/__init__.py | 1 - maloja/utilities/maintenance.py | 114 -------------------------------- 4 files changed, 91 insertions(+), 131 deletions(-) create mode 100644 maloja/database/cached.py delete mode 100644 maloja/utilities/maintenance.py diff --git a/maloja/database/__init__.py b/maloja/database/__init__.py index 4585868..4e4ab2d 100644 --- a/maloja/database/__init__.py +++ b/maloja/database/__init__.py @@ -10,6 +10,7 @@ from ..thirdparty import proxy_scrobble_all from ..globalconf import data_dir, malojaconfig, apikeystore #db from . import sqldb +from . import cached # doreah toolkit from doreah.logging import log @@ -42,9 +43,9 @@ import urllib dbstatus = { - "healthy":False, + "healthy":False, # we can access the db "rebuildinprogress":False, - "complete":False + "complete":False # information is complete } class DatabaseNotBuilt(HTTPError): def __init__(self): @@ -235,19 +236,17 @@ def artist_info(artist): c = [e for e in alltimecharts if e["artist"] == artist][0] others = sqldb.get_associated_artists(artist) position = c["rank"] - performance_weekly = get_performance(artist=artist,step="week")[:-1] #current week doesn't count - performance_yearly = get_performance(artist=artist,step="year")[:-1] #current year doesn't count return { "artist":artist, "scrobbles":scrobbles, "position":position, "associated":others, "medals":{ - "gold":[e['range'] for e in performance_yearly if e['rank'] == 1], - "silver":[e['range'] for e in performance_yearly if e['rank'] == 2], - "bronze":[e['range'] for e in performance_yearly if e['rank'] == 3] + "gold": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['gold']], + "silver": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['silver']], + "bronze": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['bronze']], }, - "topweeks":len([e for e in performance_weekly if e['rank'] == 1]) + "topweeks":len([e for e in cached.weekly_topartists if e == artist]) } except: # if the artist isnt in the charts, they are not being credited and we @@ -276,21 +275,18 @@ def track_info(track): elif scrobbles >= threshold_platinum: cert = "platinum" elif scrobbles >= threshold_gold: cert = "gold" - performance_weekly = get_performance(track=track,step="week")[:-1] #current week doesn't count - performance_yearly = get_performance(track=track,step="year")[:-1] #current year doesn't count - return { "track":track, "scrobbles":scrobbles, "position":position, "medals":{ - "gold":[e['range'] for e in performance_yearly if e['rank'] == 1], - "silver":[e['range'] for e in performance_yearly if e['rank'] == 2], - "bronze":[e['range'] for e in performance_yearly if e['rank'] == 3] + "gold": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['gold']], + "silver": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['silver']], + "bronze": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['bronze']], }, "certification":cert, - "topweeks":len([e for e in performance_weekly if e['rank'] == 1]) + "topweeks":len([e for e in cached.weekly_toptracks if e == track]) } @@ -452,18 +448,28 @@ def get_predefined_rulesets(): def start_db(): + # Upgrade database from .. import upgrade upgrade.upgrade_db(sqldb.add_scrobbles) + # Load temporary tables from . import associated associated.load_associated_rules() dbstatus['healthy'] = True - dbstatus['complete'] = True + # inform time module about begin of scrobbling firstscrobble = sqldb.get_scrobbles()[0] register_scrobbletime(firstscrobble['time']) + # create cached information + cached.update_medals() + cached.update_weekly() + + dbstatus['complete'] = True + + + diff --git a/maloja/database/cached.py b/maloja/database/cached.py new file mode 100644 index 0000000..4aea767 --- /dev/null +++ b/maloja/database/cached.py @@ -0,0 +1,69 @@ +# for information that is not authorative, but should be saved anyway because it +# changes infrequently and DB access is expensive + +from doreah.regular import yearly, daily +from .. import database +from .. import malojatime as mjt + + + +medals_artists = { + # year: {'gold':[],'silver':[],'bronze':[]} +} +medals_tracks = { + # year: {'gold':[],'silver':[],'bronze':[]} +} + +weekly_topartists = [] +weekly_toptracks = [] + +@yearly +def update_medals(): + + global medals_artists, medals_tracks + medals_artists.clear() + medals_tracks.clear() + + for year in mjt.ranges(step="year"): + if year == mjt.thisyear(): break + + charts_artists = database.get_charts_artists(timerange=year) + charts_tracks = database.get_charts_tracks(timerange=year) + + entry_artists = {'gold':[],'silver':[],'bronze':[]} + entry_tracks = {'gold':[],'silver':[],'bronze':[]} + medals_artists[year.desc()] = entry_artists + medals_tracks[year.desc()] = entry_tracks + + for entry in charts_artists: + if entry['rank'] == 1: entry_artists['gold'].append(entry['artist']) + elif entry['rank'] == 2: entry_artists['silver'].append(entry['artist']) + elif entry['rank'] == 3: entry_artists['bronze'].append(entry['artist']) + else: break + for entry in charts_tracks: + if entry['rank'] == 1: entry_tracks['gold'].append(entry['track']) + elif entry['rank'] == 2: entry_tracks['silver'].append(entry['track']) + elif entry['rank'] == 3: entry_tracks['bronze'].append(entry['track']) + else: break + + + +@daily +def update_weekly(): + + global weekly_topartists, weekly_toptracks + weekly_topartists.clear() + weekly_toptracks.clear() + + for week in mjt.ranges(step="week"): + if week == mjt.thisweek(): break + + charts_artists = database.get_charts_artists(timerange=week) + charts_tracks = database.get_charts_tracks(timerange=week) + + for entry in charts_artists: + if entry['rank'] == 1: weekly_topartists.append(entry['artist']) + else: break + for entry in charts_tracks: + if entry['rank'] == 1: weekly_toptracks.append(entry['track']) + else: break diff --git a/maloja/utilities/__init__.py b/maloja/utilities/__init__.py index 5fc1c9a..20b2204 100644 --- a/maloja/utilities/__init__.py +++ b/maloja/utilities/__init__.py @@ -1,3 +1,2 @@ from .images import * -from .maintenance import * from .utils import * diff --git a/maloja/utilities/maintenance.py b/maloja/utilities/maintenance.py deleted file mode 100644 index b8df8f4..0000000 --- a/maloja/utilities/maintenance.py +++ /dev/null @@ -1,114 +0,0 @@ -from ..__pkginfo__ import VERSION -from ..malojatime import ranges, thisweek, thisyear -from ..globalconf import malojaconfig - -from doreah.regular import yearly, daily -from doreah.logging import log - -import datetime -import json -import urllib -import itertools - - - -get_track = lambda x:(frozenset(x["track"]["artists"]),x["track"]["title"]) -get_artist = lambda x:x["artist"] - -def group_by_attribute(sequence,attribute): - grouped = itertools.groupby(sequence,key=lambda x:x[attribute]) - for attrvalue,members in grouped: - yield attrvalue,list(members) - -def collect_rankings(chart,identify,collection,iteration=None,count=True): - grouped = group_by_attribute(chart,"rank") - for rank, members in grouped: - if not count and rank not in rankmedals: break - if count and rank != 1: break - - for m in members: - # get the actual object that we're counting - entity = identify(m) - - # count no1 spots - if count: - collection[entity] = collection.setdefault(entity,0) + 1 - - # collect instances of top3 spots - else: - medal = rankmedals[rank] - collection.setdefault(entity,{}).setdefault(medal,[]).append(iteration) - - -rankmedals = { - 1:'gold', - 2:'silver', - 3:'bronze' -} - -@yearly -def update_medals(): - - - from ..database import MEDALS_ARTISTS, MEDALS_TRACKS, STAMPS, get_charts_artists, get_charts_tracks - - - MEDALS_ARTISTS.clear() - MEDALS_TRACKS.clear() - - for year in ranges(step="year"): - if year == thisyear(): break - - charts_artists = get_charts_artists(timerange=year) - charts_tracks = get_charts_tracks(timerange=year) - - collect_rankings(charts_artists,get_artist,MEDALS_ARTISTS,iteration=year,count=False) - collect_rankings(charts_tracks,get_track,MEDALS_TRACKS,iteration=year,count=False) - - -@daily -def update_weekly(): - - from ..database import WEEKLY_TOPTRACKS, WEEKLY_TOPARTISTS, get_charts_artists, get_charts_tracks - - - WEEKLY_TOPARTISTS.clear() - WEEKLY_TOPTRACKS.clear() - - for week in ranges(step="week"): - if week == thisweek(): break - - charts_artists = get_charts_artists(timerange=week) - charts_tracks = get_charts_tracks(timerange=week) - - collect_rankings(charts_artists,get_artist,WEEKLY_TOPARTISTS) - collect_rankings(charts_tracks,get_track,WEEKLY_TOPTRACKS) - - -@daily -def send_stats(): - if malojaconfig["SEND_STATS"]: - - log("Sending daily stats report...") - - from ..database import ARTISTS, TRACKS, SCROBBLES - - keys = { - "url":"https://myrcella.krateng.ch/malojastats", - "method":"POST", - "headers":{"Content-Type": "application/json"}, - "data":json.dumps({ - "name":malojaconfig["NAME"], - "url":malojaconfig["PUBLIC_URL"], - "version":VERSION, - "artists":len(ARTISTS), - "tracks":len(TRACKS), - "scrobbles":len(SCROBBLES) - }).encode("utf-8") - } - try: - req = urllib.request.Request(**keys) - response = urllib.request.urlopen(req) - log("Sent daily report!") - except: - log("Could not send daily report!")