From 0dd6cd9dd55b28277ef0b58c54b9bf6001912fd1 Mon Sep 17 00:00:00 2001 From: krateng Date: Tue, 4 Jan 2022 07:55:07 +0100 Subject: [PATCH] I promise this is temporary code! --- maloja/database.py | 14 +++--- maloja/db/convert.py | 40 ---------------- maloja/db/sqldb.py | 108 +++++++++++++++++++++++++++++++++++++++++-- maloja/upgrade.py | 2 + 4 files changed, 112 insertions(+), 52 deletions(-) diff --git a/maloja/database.py b/maloja/database.py index 7a5ea3f..5bcc712 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -9,7 +9,7 @@ from .malojauri import uri_to_internal, internal_to_uri, compose_querystring from .thirdparty import proxy_scrobble_all from .globalconf import data_dir, malojaconfig, apikeystore #db -from .db.sqldb import * +from .db import sqldb # doreah toolkit from doreah.logging import log @@ -545,7 +545,7 @@ def get_predefined_rulesets(): def start_db(): from . import upgrade - upgrade.upgrade_db(add_scrobbles) + upgrade.upgrade_db(sqldb.add_scrobbles) dbstatus['healthy'] = True dbstatus['complete'] = True @@ -735,22 +735,20 @@ def db_query_full(artist=None,artists=None,title=None,track=None,timerange=None, #artist = None if artists is not None and title is not None: - return get_scrobbles_of_track(track={"artists":artists,"title":title},since=since,to=to) + return sqldb.get_scrobbles_of_track(track={"artists":artists,"title":title},since=since,to=to) if artist is not None: - return get_scrobbles_of_artist(artist=artist,since=since,to=to) + return sqldb.get_scrobbles_of_artist(artist=artist,since=since,to=to) - return get_scrobbles(since=since,to=to) + return sqldb.get_scrobbles(since=since,to=to) - # pointless to check for artist when track is checked because every track has a fixed set of artists, but it's more elegant this way - # Queries that... well... aggregate def db_aggregate_full(by=None,since=None,to=None,within=None,timerange=None,artist=None): if not dbstatus['healthy']: raise DatabaseNotBuilt() - (since, to) = time_stamps(since=since,to=to,within=within,range=timerange) + (since, to) = time_stamps(range=timerange) if isinstance(artist, str): artist = ARTISTS.index(artist) diff --git a/maloja/db/convert.py b/maloja/db/convert.py index be9c42b..e69de29 100644 --- a/maloja/db/convert.py +++ b/maloja/db/convert.py @@ -1,40 +0,0 @@ -#### ATTENTION ALL ADVENTURERS -#### THIS IS WHAT A SCROBBLE DICT WILL LOOK LIKE FROM NOW ON -#### THIS IS THE SINGLE CANONICAL SOURCE OF TRUTH -#### STOP MAKING DIFFERENT LITTLE DICTS IN EVERY SINGLE FUNCTION -#### THIS IS THE SCHEMA THAT WILL DEFINITELY 100% STAY LIKE THIS AND NOT -#### RANDOMLY GET CHANGED TWO VERSIONS LATER -#### HERE WE GO -# -# { -# "time":int, -# "track":{ -# "artists":list, -# "title":string, -# "album":{ -# "name":string, -# "artists":list -# }, -# "length":None -# }, -# "duration":int, -# "origin":string -# } - - - -def scrobble_db_to_dict(resultrow): - return { - "time":resultrow.timestamp, - "track":track_db_to_dict(resultrow.track), - "duration":resultrow.duration, - "origin":resultrow.origin - } - -def track_db_to_dict(resultrow): - return { - "artists":[], - "title":resultrow.title, - "album":{}, - "length":resultrow.length - } diff --git a/maloja/db/sqldb.py b/maloja/db/sqldb.py index 2aa67af..26517ec 100644 --- a/maloja/db/sqldb.py +++ b/maloja/db/sqldb.py @@ -6,6 +6,8 @@ from ..globalconf import data_dir +##### DB Technical + DB = {} @@ -42,9 +44,76 @@ DB['trackartists'] = sql.Table( meta.create_all(engine) +##### DB <-> Dict translations + +## ATTENTION ALL ADVENTURERS +## this is what a scrobble dict will look like from now on +## this is the single canonical source of truth +## stop making different little dicts in every single function +## this is the schema that will definitely 100% stay like this and not +## randomly get changed two versions later +## here we go +# +# { +# "time":int, +# "track":{ +# "artists":list, +# "title":string, +# "album":{ +# "name":string, +# "artists":list +# }, +# "length":None +# }, +# "duration":int, +# "origin":string, +# "extra":{string-keyed mapping for all flags with the scrobble} +# } +def scrobble_db_to_dict(row): + + + return { + "time":row.timestamp, + "track":get_track(row.track_id), + "duration":row.duration, + "origin":row.origin + } + +def track_db_to_dict(row): + return { + "title":row.title, + "length":row.length + } + +def artist_db_to_dict(row): + return row.name + +def scrobble_dict_to_db(info): + return { + "rawscrobble":json.dumps(info), + "timestamp":info['time'], + "origin":info['origin'], + "duration":info['duration'], + "extra":info['extra'], + "track_id":get_track_id(info['track']) + } + +def track_dict_to_db(info): + return { + "title":info['title'], + "title_normalized":normalize_name(info['title']), + "length":info['length'] + } + +def artist_dict_to_db(info): + return { + "name": info, + "name_normalized":normalize_name(info) + } + @@ -60,7 +129,7 @@ def add_scrobbles(scrobbleslist): rawscrobble=json.dumps(s), timestamp=s['time'], origin=s['origin'], - duration=s['duration'] or -1, + duration=s['duration'], track_id=get_track_id(s['track']) ) for s in scrobbleslist ] @@ -180,20 +249,51 @@ def get_scrobbles_of_track(track,since,to): def get_scrobbles(since,to): - - artist_id = get_artist_id(artist) + print(since,to) with engine.begin() as conn: op = DB['scrobbles'].select().where( DB['scrobbles'].c.timestamp<=to, DB['scrobbles'].c.timestamp>=since, ) + print(str(op)) result = conn.execute(op).all() - print(result) + result = [scrobble_db_to_dict(row) for row in result] + return result + +def get_track(id): + with engine.begin() as conn: + op = DB['tracks'].select().where( + DB['tracks'].c.id==id + ) + result = conn.execute(op).all() + + trackinfo = result[0] + + + with engine.begin() as conn: + op = DB['trackartists'].select().where( + DB['trackartists'].c.track_id==id + ) + result = conn.execute(op).all() + + artists = [get_artist(row.artist_id) for row in result] + + result = track_db_to_dict(trackinfo) + result['artists'] = artists return result +def get_artist(id): + with engine.begin() as conn: + op = DB['artists'].select().where( + DB['artists'].c.id==id + ) + result = conn.execute(op).all() + + artistinfo = result[0] + return artist_db_to_dict(artistinfo) diff --git a/maloja/upgrade.py b/maloja/upgrade.py index b43b4f2..7029c88 100644 --- a/maloja/upgrade.py +++ b/maloja/upgrade.py @@ -27,6 +27,7 @@ def upgrade_db(callback_add_scrobbles): print(col['yellow']("Upgrading v2 Database to v3 Database. This could take a while...")) oldfolder = os.path.join(dir_settings['state'],"scrobbles") newfolder = os.path.join(dir_settings['state'],".oldscrobbles") + os.makedirs(newfolder,exist_ok=True) if os.path.exists(oldfolder): scrobblefiles = os.listdir(oldfolder) for sf in scrobblefiles: @@ -63,3 +64,4 @@ def upgrade_db(callback_add_scrobbles): }) callback_add_scrobbles(scrobblelist) os.rename(os.path.join(oldfolder,sf),os.path.join(newfolder,sf)) + print(col['yellow']("Done!"))