From c8ed894efb381abf5b16ad55775f4a93703edff8 Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 27 Mar 2022 05:31:15 +0200 Subject: [PATCH] Updated tasks to new database architecture --- maloja/proccontrol/tasks/generate.py | 20 +++-- maloja/proccontrol/tasks/import_scrobbles.py | 84 ++++++++++---------- maloja/utilities/images.py | 3 - 3 files changed, 57 insertions(+), 50 deletions(-) diff --git a/maloja/proccontrol/tasks/generate.py b/maloja/proccontrol/tasks/generate.py index 7268089..0184897 100644 --- a/maloja/proccontrol/tasks/generate.py +++ b/maloja/proccontrol/tasks/generate.py @@ -1,6 +1,8 @@ import random import datetime +from ...database.sqldb import add_scrobbles + artists = [ "Chou Tzuyu","Jennie Kim","Kim Seolhyun","Nancy McDonie","Park Junghwa","Hirai Momo","Rosé Park","Laura Brehm","HyunA", @@ -66,14 +68,22 @@ def generate_track(): def generate(): - # TODO - pass + scrobbles = [] for _ in range(200): track = generate_track() + print("Generated",track) for _ in range(random.randint(1, 50)): timestamp = random.randint(1, int(datetime.datetime.now().timestamp())) - entry = "\t".join([str(timestamp),"␟".join(track['artists']),track['title'],"-"]) - fd.write(entry) - fd.write("\n") + scrobbles.append({ + "time":timestamp, + "track":{ + "artists":track['artists'], + "title":track['title'] + }, + "duration":None, + "origin":"generated" + }) + + add_scrobbles(scrobbles) diff --git a/maloja/proccontrol/tasks/import_scrobbles.py b/maloja/proccontrol/tasks/import_scrobbles.py index 4645f2e..630f326 100644 --- a/maloja/proccontrol/tasks/import_scrobbles.py +++ b/maloja/proccontrol/tasks/import_scrobbles.py @@ -5,6 +5,8 @@ import json, csv from ...cleanup import * from doreah.io import col, ask from ...globalconf import data_dir + +from ...database.sqldb import add_scrobbles #from ...utilities import * @@ -23,60 +25,58 @@ def import_scrobbles(fromfile): ext = fromfile.split('.')[-1].lower() if ext == 'csv': - type = "Last.fm" - outputf = data_dir['scrobbles']("lastfmimport.tsv") + import_type = "Last.fm" importfunc = parse_lastfm elif ext == 'json': - type = "Spotify" - outputf = data_dir['scrobbles']("spotifyimport.tsv") + import_type = "Spotify" importfunc = parse_spotify - print(f"Parsing {col['yellow'](fromfile)} as {col['cyan'](type)} export") + print(f"Parsing {col['yellow'](fromfile)} as {col['cyan'](import_type)} export") - if os.path.exists(outputf): - overwrite = ask("Already imported data. Overwrite?",default=False) - if not overwrite: return + success = 0 + failed = 0 + timestamps = set() + scrobblebuffer = [] - with open(outputf,"w") as outputfd: - success = 0 - failed = 0 - timestamps = set() - for scrobble in importfunc(fromfile): - if scrobble is None: - failed += 1 - else: - success += 1 + for scrobble in importfunc(fromfile): + if scrobble is None: + failed += 1 + else: + success += 1 - ## We prevent double timestamps in the database creation, so we - ## technically don't need them in the files - ## however since the conversion to maloja is a one-time thing, - ## we should take any effort to make the file as good as possible - while scrobble['timestamp'] in timestamps: - scrobble['timestamp'] += 1 - timestamps.add(scrobble['timestamp']) + # prevent duplicate timestamps within one import file + while scrobble['timestamp'] in timestamps: + scrobble['timestamp'] += 1 + timestamps.add(scrobble['timestamp']) + # clean up + (scrobble['artists'],scrobble['title']) = c.fullclean(scrobble['artists'],scrobble['title']) - # Format fields for tsv - scrobble['timestamp'] = str(scrobble['timestamp']) - scrobble['duration'] = str(scrobble['duration']) if scrobble['duration'] is not None else '-' - (artists,scrobble['title']) = c.fullclean(scrobble['artiststr'],scrobble['title']) - scrobble['artiststr'] = "␟".join(artists) + scrobblebuffer.append({ + "time":scrobble['timestamp'], + "track":{ + "artists":scrobble['artists'], + "title":scrobble['title'], + "album":{ + "name":scrobble['album'], + "artists":scrobble['artists'] + }, + "length":None + }, + "duration":scrobble['duration'], + "origin":"import:" + import_type, + "extra":{} + }) - outputline = "\t".join([ - scrobble['timestamp'], - scrobble['artiststr'], - scrobble['title'], - scrobble['album'], - scrobble['duration'] - ]) - outputfd.write(outputline + '\n') - - if success % 100 == 0: - print(f"Imported {success} scrobbles...") + if success % 1000 == 0: + print(f"Imported {success} scrobbles...") + add_scrobbles(scrobblebuffer) + scrobblebuffer = [] + add_scrobbles(scrobblebuffer) print("Successfully imported",success,"scrobbles!") if failed > 0: print(col['red'](str(failed) + " Errors!")) @@ -95,7 +95,7 @@ def parse_spotify(inputf): try: yield { 'title':entry['master_metadata_track_name'], - 'artiststr': entry['master_metadata_album_artist_name'], + 'artists': entry['master_metadata_album_artist_name'], 'album': entry['master_metadata_album_album_name'], 'timestamp': int(datetime.datetime.strptime( entry['ts'].replace('Z','+0000',), @@ -124,7 +124,7 @@ def parse_lastfm(inputf): try: yield { 'title': row[2], - 'artiststr': row[0], + 'artists': row[0], 'album': row[1], 'timestamp': int(datetime.datetime.strptime( row[3] + '+0000', diff --git a/maloja/utilities/images.py b/maloja/utilities/images.py index 6ce71d9..b9f63b7 100644 --- a/maloja/utilities/images.py +++ b/maloja/utilities/images.py @@ -128,7 +128,6 @@ def resolve_track_image(track_id): return result track = database.sqldb.get_track(track_id) - print(track) # local image if malojaconfig["USE_LOCAL_IMAGES"]: @@ -156,7 +155,6 @@ def resolve_artist_image(artist_id): return result artist = database.sqldb.get_artist(artist_id) - print(artist) # local image if malojaconfig["USE_LOCAL_IMAGES"]: @@ -275,7 +273,6 @@ def set_image(b64,**keys): else: entity = keys['artist'] id = database.sqldb.get_artist_id(entity) - print('id is',id) log("Trying to set image, b64 string: " + str(b64[:30] + "..."),module="debug")