From de625cc3c25b0dce4142c659472f3790ab408f99 Mon Sep 17 00:00:00 2001 From: krateng Date: Thu, 21 Apr 2022 17:46:25 +0200 Subject: [PATCH 1/5] Fixed missing output when running in Docker, fix GH-117 --- Containerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Containerfile b/Containerfile index 22b0b72..6584bf1 100644 --- a/Containerfile +++ b/Containerfile @@ -43,6 +43,8 @@ RUN pip install /usr/src/app ENV MALOJA_SKIP_SETUP=yes ENV MALOJA_HOST=0.0.0.0 +ENV PYTHONUNBUFFERED=1 + EXPOSE 42010 # use exec form for better signal handling https://docs.docker.com/engine/reference/builder/#entrypoint ENTRYPOINT ["maloja", "run"] From df6bbebe31960258bbe3a7ba1ef44debe5e487a6 Mon Sep 17 00:00:00 2001 From: krateng Date: Thu, 21 Apr 2022 17:51:16 +0200 Subject: [PATCH 2/5] Removed now unnecessary default override for container --- Containerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Containerfile b/Containerfile index 6584bf1..8e11da8 100644 --- a/Containerfile +++ b/Containerfile @@ -39,10 +39,9 @@ COPY . . RUN pip install /usr/src/app -# Docker-specific configuration and default to IPv4 +# Docker-specific configuration +# defaulting to IPv4 is no longer necessary (default host is dual stack) ENV MALOJA_SKIP_SETUP=yes -ENV MALOJA_HOST=0.0.0.0 - ENV PYTHONUNBUFFERED=1 EXPOSE 42010 From 28163348faed13beef54d8796d7e0670ce0c2037 Mon Sep 17 00:00:00 2001 From: krateng Date: Thu, 21 Apr 2022 21:15:53 +0200 Subject: [PATCH 3/5] Fixed importing with direct filename, fix GH-124 --- maloja/proccontrol/tasks/import_scrobbles.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/maloja/proccontrol/tasks/import_scrobbles.py b/maloja/proccontrol/tasks/import_scrobbles.py index c9d98a6..ca82078 100644 --- a/maloja/proccontrol/tasks/import_scrobbles.py +++ b/maloja/proccontrol/tasks/import_scrobbles.py @@ -117,7 +117,8 @@ def import_scrobbles(inputf): return result def parse_spotify_lite(inputf): - inputfolder = os.path.dirname(inputf) + pth = os.path + inputfolder = pth.relpath(pth.dirname(pth.abspath(inputf))) filenames = re.compile(r'StreamingHistory[0-9]+\.json') inputfiles = [os.path.join(inputfolder,f) for f in os.listdir(inputfolder) if filenames.match(f)] @@ -161,8 +162,8 @@ def parse_spotify_lite(inputf): def parse_spotify_full(inputf): - - inputfolder = os.path.dirname(inputf) + pth = os.path + inputfolder = pth.relpath(pth.dirname(pth.abspath(inputf))) filenames = re.compile(r'endsong_[0-9]+\.json') inputfiles = [os.path.join(inputfolder,f) for f in os.listdir(inputfolder) if filenames.match(f)] From 646c290a373ef668a67c8e8b08b7974bf15309fa Mon Sep 17 00:00:00 2001 From: krateng Date: Thu, 21 Apr 2022 21:17:55 +0200 Subject: [PATCH 4/5] Added separate output for importing zero files --- maloja/proccontrol/tasks/import_scrobbles.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/maloja/proccontrol/tasks/import_scrobbles.py b/maloja/proccontrol/tasks/import_scrobbles.py index ca82078..41c76c7 100644 --- a/maloja/proccontrol/tasks/import_scrobbles.py +++ b/maloja/proccontrol/tasks/import_scrobbles.py @@ -122,6 +122,10 @@ def parse_spotify_lite(inputf): filenames = re.compile(r'StreamingHistory[0-9]+\.json') inputfiles = [os.path.join(inputfolder,f) for f in os.listdir(inputfolder) if filenames.match(f)] + if len(inputfiles) == 0: + print("No files found!") + return + if inputfiles != [inputf]: print("Spotify files should all be imported together to identify duplicates across the whole dataset.") if not ask("Import " + ", ".join(col['yellow'](i) for i in inputfiles) + "?",default=True): @@ -167,6 +171,10 @@ def parse_spotify_full(inputf): filenames = re.compile(r'endsong_[0-9]+\.json') inputfiles = [os.path.join(inputfolder,f) for f in os.listdir(inputfolder) if filenames.match(f)] + if len(inputfiles) == 0: + print("No files found!") + return + if inputfiles != [inputf]: print("Spotify files should all be imported together to identify duplicates across the whole dataset.") if not ask("Import " + ", ".join(col['yellow'](i) for i in inputfiles) + "?",default=True): From 6e4e62755df3e3d7438c704a32c7ed3f127f592c Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 22 Apr 2022 06:01:59 +0200 Subject: [PATCH 5/5] Fixed global database lock release during scrobble creation, GH-126 --- dev/releases/3.0.yml | 3 ++ maloja/database/sqldb.py | 95 ++++++++++++++++++++-------------------- 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/dev/releases/3.0.yml b/dev/releases/3.0.yml index 3cb6289..6ac393d 100644 --- a/dev/releases/3.0.yml +++ b/dev/releases/3.0.yml @@ -36,3 +36,6 @@ minor_release_name: "Yeonhee" 3.0.6: notes: - "[Bugfix] Better parsing of featuring artists" + - "[Bugfix] Fixed buffered output in Docker" + - "[Bugfix] Fixed importing a Spotify file without path" + - "[Bugfix] No longer releasing database lock during scrobble creation" diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py index 1623d57..e0954d9 100644 --- a/maloja/database/sqldb.py +++ b/maloja/database/sqldb.py @@ -157,8 +157,8 @@ def connection_provider(func): ### DB -> DICT -def scrobbles_db_to_dict(rows,include_internal=False): - tracks = get_tracks_map(set(row.track_id for row in rows)) +def scrobbles_db_to_dict(rows,include_internal=False,dbconn=None): + tracks = get_tracks_map(set(row.track_id for row in rows),dbconn=dbconn) return [ { **{ @@ -176,11 +176,11 @@ def scrobbles_db_to_dict(rows,include_internal=False): for row in rows ] -def scrobble_db_to_dict(row): - return scrobbles_db_to_dict([row])[0] +def scrobble_db_to_dict(row,dbconn=None): + return scrobbles_db_to_dict([row],dbconn=dbconn)[0] -def tracks_db_to_dict(rows): - artists = get_artists_of_tracks(set(row.id for row in rows)) +def tracks_db_to_dict(rows,dbconn=None): + artists = get_artists_of_tracks(set(row.id for row in rows),dbconn=dbconn) return [ { "artists":artists[row.id], @@ -191,41 +191,41 @@ def tracks_db_to_dict(rows): for row in rows ] -def track_db_to_dict(row): - return tracks_db_to_dict([row])[0] +def track_db_to_dict(row,dbconn=None): + return tracks_db_to_dict([row],dbconn=dbconn)[0] -def artists_db_to_dict(rows): +def artists_db_to_dict(rows,dbconn=None): return [ row.name for row in rows ] -def artist_db_to_dict(row): - return artists_db_to_dict([row])[0] +def artist_db_to_dict(row,dbconn=None): + return artists_db_to_dict([row],dbconn=dbconn)[0] ### DICT -> DB -def scrobble_dict_to_db(info): +def scrobble_dict_to_db(info,dbconn=None): return { "timestamp":info['time'], "origin":info['origin'], "duration":info['duration'], - "track_id":get_track_id(info['track']), + "track_id":get_track_id(info['track'],dbconn=dbconn), "extra":json.dumps(info.get('extra',{})), "rawscrobble":json.dumps(info.get('rawscrobble',{})) } -def track_dict_to_db(info): +def track_dict_to_db(info,dbconn=None): return { "title":info['title'], "title_normalized":normalize_name(info['title']), "length":info.get('length') } -def artist_dict_to_db(info): +def artist_dict_to_db(info,dbconn=None): return { "name": info, "name_normalized":normalize_name(info) @@ -249,7 +249,7 @@ def add_scrobbles(scrobbleslist,dbconn=None): ops = [ DB['scrobbles'].insert().values( - **scrobble_dict_to_db(s) + **scrobble_dict_to_db(s,dbconn=dbconn) ) for s in scrobbleslist ] @@ -283,7 +283,7 @@ def delete_scrobble(scrobble_id,dbconn=None): @connection_provider def get_track_id(trackdict,dbconn=None): ntitle = normalize_name(trackdict['title']) - artist_ids = [get_artist_id(a) for a in trackdict['artists']] + artist_ids = [get_artist_id(a,dbconn=dbconn) for a in trackdict['artists']] artist_ids = list(set(artist_ids)) @@ -313,7 +313,7 @@ def get_track_id(trackdict,dbconn=None): op = DB['tracks'].insert().values( - **track_dict_to_db(trackdict) + **track_dict_to_db(trackdict,dbconn=dbconn) ) result = dbconn.execute(op) track_id = result.inserted_primary_key[0] @@ -366,7 +366,7 @@ def get_scrobbles_of_artist(artist,since=None,to=None,resolve_references=True,db if since is None: since=0 if to is None: to=now() - artist_id = get_artist_id(artist) + artist_id = get_artist_id(artist,dbconn=dbconn) jointable = sql.join(DB['scrobbles'],DB['trackartists'],DB['scrobbles'].c.track_id == DB['trackartists'].c.track_id) @@ -378,7 +378,7 @@ def get_scrobbles_of_artist(artist,since=None,to=None,resolve_references=True,db result = dbconn.execute(op).all() if resolve_references: - result = scrobbles_db_to_dict(result) + result = scrobbles_db_to_dict(result,dbconn=dbconn) #result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for row in result] return result @@ -389,7 +389,7 @@ def get_scrobbles_of_track(track,since=None,to=None,resolve_references=True,dbco if since is None: since=0 if to is None: to=now() - track_id = get_track_id(track) + track_id = get_track_id(track,dbconn=dbconn) op = DB['scrobbles'].select().where( DB['scrobbles'].c.timestamp<=to, @@ -417,7 +417,7 @@ def get_scrobbles(since=None,to=None,resolve_references=True,dbconn=None): result = dbconn.execute(op).all() if resolve_references: - result = scrobbles_db_to_dict(result) + result = scrobbles_db_to_dict(result,dbconn=dbconn) #result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for i,row in enumerate(result) if i