mirror/maloja
1
0
Fork 0
mirror of https://github.com/krateng/maloja.git synced 2023-08-10 21:12:55 +03:00
Code Issues Projects Releases Wiki Activity

Fixed global database lock release during scrobble creation, GH-126

Browse Source
This commit is contained in:
krateng 2022-04-22 06:01:59 +02:00
parent 646c290a37
commit 6e4e62755d
2 changed files with 50 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dev/releases/3.0.yml
View File

@ -36,3 +36,6 @@ minor_release_name: "Yeonhee"
3.0.6: 3.0.6:
notes: notes:
- "[Bugfix] Better parsing of featuring artists" - "[Bugfix] Better parsing of featuring artists"
- "[Bugfix] Fixed buffered output in Docker"
- "[Bugfix] Fixed importing a Spotify file without path"
- "[Bugfix] No longer releasing database lock during scrobble creation"

81
maloja/database/sqldb.py
View File

@ -157,8 +157,8 @@ def connection_provider(func):
### DB -> DICT ### DB -> DICT
def scrobbles_db_to_dict(rows,include_internal=False): def scrobbles_db_to_dict(rows,include_internal=False,dbconn=None):
tracks = get_tracks_map(set(row.track_id for row in rows)) tracks = get_tracks_map(set(row.track_id for row in rows),dbconn=dbconn)
return [ return [
{ {
**{ **{
@ -176,11 +176,11 @@ def scrobbles_db_to_dict(rows,include_internal=False):
for row in rows for row in rows
] ]
def scrobble_db_to_dict(row): def scrobble_db_to_dict(row,dbconn=None):
return scrobbles_db_to_dict([row])[0] return scrobbles_db_to_dict([row],dbconn=dbconn)[0]
def tracks_db_to_dict(rows): def tracks_db_to_dict(rows,dbconn=None):
artists = get_artists_of_tracks(set(row.id for row in rows)) artists = get_artists_of_tracks(set(row.id for row in rows),dbconn=dbconn)
return [ return [
{ {
"artists":artists[row.id], "artists":artists[row.id],
@ -191,41 +191,41 @@ def tracks_db_to_dict(rows):
for row in rows for row in rows
] ]
def track_db_to_dict(row): def track_db_to_dict(row,dbconn=None):
return tracks_db_to_dict([row])[0] return tracks_db_to_dict([row],dbconn=dbconn)[0]
def artists_db_to_dict(rows): def artists_db_to_dict(rows,dbconn=None):
return [ return [
row.name row.name
for row in rows for row in rows
] ]
def artist_db_to_dict(row): def artist_db_to_dict(row,dbconn=None):
return artists_db_to_dict([row])[0] return artists_db_to_dict([row],dbconn=dbconn)[0]
### DICT -> DB ### DICT -> DB
def scrobble_dict_to_db(info): def scrobble_dict_to_db(info,dbconn=None):
return { return {
"timestamp":info['time'], "timestamp":info['time'],
"origin":info['origin'], "origin":info['origin'],
"duration":info['duration'], "duration":info['duration'],
"track_id":get_track_id(info['track']), "track_id":get_track_id(info['track'],dbconn=dbconn),
"extra":json.dumps(info.get('extra',{})), "extra":json.dumps(info.get('extra',{})),
"rawscrobble":json.dumps(info.get('rawscrobble',{})) "rawscrobble":json.dumps(info.get('rawscrobble',{}))
} }
def track_dict_to_db(info): def track_dict_to_db(info,dbconn=None):
return { return {
"title":info['title'], "title":info['title'],
"title_normalized":normalize_name(info['title']), "title_normalized":normalize_name(info['title']),
"length":info.get('length') "length":info.get('length')
} }
def artist_dict_to_db(info): def artist_dict_to_db(info,dbconn=None):
return { return {
"name": info, "name": info,
"name_normalized":normalize_name(info) "name_normalized":normalize_name(info)
@ -249,7 +249,7 @@ def add_scrobbles(scrobbleslist,dbconn=None):
ops = [ ops = [
DB['scrobbles'].insert().values( DB['scrobbles'].insert().values(
**scrobble_dict_to_db(s) **scrobble_dict_to_db(s,dbconn=dbconn)
) for s in scrobbleslist ) for s in scrobbleslist
] ]
@ -283,7 +283,7 @@ def delete_scrobble(scrobble_id,dbconn=None):
@connection_provider @connection_provider
def get_track_id(trackdict,dbconn=None): def get_track_id(trackdict,dbconn=None):
ntitle = normalize_name(trackdict['title']) ntitle = normalize_name(trackdict['title'])
artist_ids = [get_artist_id(a) for a in trackdict['artists']] artist_ids = [get_artist_id(a,dbconn=dbconn) for a in trackdict['artists']]
artist_ids = list(set(artist_ids)) artist_ids = list(set(artist_ids))
@ -313,7 +313,7 @@ def get_track_id(trackdict,dbconn=None):
op = DB['tracks'].insert().values( op = DB['tracks'].insert().values(
**track_dict_to_db(trackdict) **track_dict_to_db(trackdict,dbconn=dbconn)
) )
result = dbconn.execute(op) result = dbconn.execute(op)
track_id = result.inserted_primary_key[0] track_id = result.inserted_primary_key[0]
@ -366,7 +366,7 @@ def get_scrobbles_of_artist(artist,since=None,to=None,resolve_references=True,db
if since is None: since=0 if since is None: since=0
if to is None: to=now() if to is None: to=now()
artist_id = get_artist_id(artist) artist_id = get_artist_id(artist,dbconn=dbconn)
jointable = sql.join(DB['scrobbles'],DB['trackartists'],DB['scrobbles'].c.track_id == DB['trackartists'].c.track_id) jointable = sql.join(DB['scrobbles'],DB['trackartists'],DB['scrobbles'].c.track_id == DB['trackartists'].c.track_id)
@ -378,7 +378,7 @@ def get_scrobbles_of_artist(artist,since=None,to=None,resolve_references=True,db
result = dbconn.execute(op).all() result = dbconn.execute(op).all()
if resolve_references: if resolve_references:
result = scrobbles_db_to_dict(result) result = scrobbles_db_to_dict(result,dbconn=dbconn)
#result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for row in result] #result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for row in result]
return result return result
@ -389,7 +389,7 @@ def get_scrobbles_of_track(track,since=None,to=None,resolve_references=True,dbco
if since is None: since=0 if since is None: since=0
if to is None: to=now() if to is None: to=now()
track_id = get_track_id(track) track_id = get_track_id(track,dbconn=dbconn)
op = DB['scrobbles'].select().where( op = DB['scrobbles'].select().where(
DB['scrobbles'].c.timestamp<=to, DB['scrobbles'].c.timestamp<=to,
@ -417,7 +417,7 @@ def get_scrobbles(since=None,to=None,resolve_references=True,dbconn=None):
result = dbconn.execute(op).all() result = dbconn.execute(op).all()
if resolve_references: if resolve_references:
result = scrobbles_db_to_dict(result) result = scrobbles_db_to_dict(result,dbconn=dbconn)
#result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for i,row in enumerate(result) if i<max] #result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for i,row in enumerate(result) if i<max]
return result return result
@ -447,7 +447,7 @@ def get_artists_of_track(track_id,resolve_references=True,dbconn=None):
) )
result = dbconn.execute(op).all() result = dbconn.execute(op).all()
artists = [get_artist(row.artist_id) if resolve_references else row.artist_id for row in result] artists = [get_artist(row.artist_id,dbconn=dbconn) if resolve_references else row.artist_id for row in result]
return artists return artists
@ -455,14 +455,14 @@ def get_artists_of_track(track_id,resolve_references=True,dbconn=None):
@connection_provider @connection_provider
def get_tracks_of_artist(artist,dbconn=None): def get_tracks_of_artist(artist,dbconn=None):
artist_id = get_artist_id(artist) artist_id = get_artist_id(artist,dbconn=dbconn)
op = sql.join(DB['tracks'],DB['trackartists']).select().where( op = sql.join(DB['tracks'],DB['trackartists']).select().where(
DB['trackartists'].c.artist_id==artist_id DB['trackartists'].c.artist_id==artist_id
) )
result = dbconn.execute(op).all() result = dbconn.execute(op).all()
return tracks_db_to_dict(result) return tracks_db_to_dict(result,dbconn=dbconn)
@cached_wrapper @cached_wrapper
@connection_provider @connection_provider
@ -471,7 +471,7 @@ def get_artists(dbconn=None):
op = DB['artists'].select() op = DB['artists'].select()
result = dbconn.execute(op).all() result = dbconn.execute(op).all()
return artists_db_to_dict(result) return artists_db_to_dict(result,dbconn=dbconn)
@cached_wrapper @cached_wrapper
@connection_provider @connection_provider
@ -480,7 +480,7 @@ def get_tracks(dbconn=None):
op = DB['tracks'].select() op = DB['tracks'].select()
result = dbconn.execute(op).all() result = dbconn.execute(op).all()
return tracks_db_to_dict(result) return tracks_db_to_dict(result,dbconn=dbconn)
### functions that count rows for parameters ### functions that count rows for parameters
@ -516,7 +516,7 @@ def count_scrobbles_by_artist(since,to,dbconn=None):
counts = [row.count for row in result] counts = [row.count for row in result]
artists = get_artists_map([row.artist_id for row in result]) artists = get_artists_map([row.artist_id for row in result],dbconn=dbconn)
result = [{'scrobbles':row.count,'artist':artists[row.artist_id]} for row in result] result = [{'scrobbles':row.count,'artist':artists[row.artist_id]} for row in result]
result = rank(result,key='scrobbles') result = rank(result,key='scrobbles')
return result return result
@ -537,7 +537,7 @@ def count_scrobbles_by_track(since,to,dbconn=None):
counts = [row.count for row in result] counts = [row.count for row in result]
tracks = get_tracks_map([row.track_id for row in result]) tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn)
result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result] result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result]
result = rank(result,key='scrobbles') result = rank(result,key='scrobbles')
return result return result
@ -546,7 +546,7 @@ def count_scrobbles_by_track(since,to,dbconn=None):
@connection_provider @connection_provider
def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None): def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None):
artist_id = get_artist_id(artist) artist_id = get_artist_id(artist,dbconn=dbconn)
jointable = sql.join( jointable = sql.join(
DB['scrobbles'], DB['scrobbles'],
@ -566,7 +566,7 @@ def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None):
counts = [row.count for row in result] counts = [row.count for row in result]
tracks = get_tracks_map([row.track_id for row in result]) tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn)
result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result] result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result]
result = rank(result,key='scrobbles') result = rank(result,key='scrobbles')
return result return result
@ -586,7 +586,7 @@ def get_artists_of_tracks(track_ids,dbconn=None):
artists = {} artists = {}
for row in result: for row in result:
artists.setdefault(row.track_id,[]).append(artist_db_to_dict(row)) artists.setdefault(row.track_id,[]).append(artist_db_to_dict(row,dbconn=dbconn))
return artists return artists
@ -601,7 +601,7 @@ def get_tracks_map(track_ids,dbconn=None):
tracks = {} tracks = {}
result = list(result) result = list(result)
# this will get a list of artistdicts in the correct order of our rows # this will get a list of artistdicts in the correct order of our rows
trackdicts = tracks_db_to_dict(result) trackdicts = tracks_db_to_dict(result,dbconn=dbconn)
for row,trackdict in zip(result,trackdicts): for row,trackdict in zip(result,trackdicts):
tracks[row.id] = trackdict tracks[row.id] = trackdict
return tracks return tracks
@ -618,7 +618,7 @@ def get_artists_map(artist_ids,dbconn=None):
artists = {} artists = {}
result = list(result) result = list(result)
# this will get a list of artistdicts in the correct order of our rows # this will get a list of artistdicts in the correct order of our rows
artistdicts = artists_db_to_dict(result) artistdicts = artists_db_to_dict(result,dbconn=dbconn)
for row,artistdict in zip(result,artistdicts): for row,artistdict in zip(result,artistdicts):
artists[row.id] = artistdict artists[row.id] = artistdict
return artists return artists
@ -629,7 +629,7 @@ def get_artists_map(artist_ids,dbconn=None):
@cached_wrapper @cached_wrapper
@connection_provider @connection_provider
def get_associated_artists(*artists,dbconn=None): def get_associated_artists(*artists,dbconn=None):
artist_ids = [get_artist_id(a) for a in artists] artist_ids = [get_artist_id(a,dbconn=dbconn) for a in artists]
jointable = sql.join( jointable = sql.join(
DB['associated_artists'], DB['associated_artists'],
@ -642,13 +642,13 @@ def get_associated_artists(*artists,dbconn=None):
) )
result = dbconn.execute(op).all() result = dbconn.execute(op).all()
artists = artists_db_to_dict(result) artists = artists_db_to_dict(result,dbconn=dbconn)
return artists return artists
@cached_wrapper @cached_wrapper
@connection_provider @connection_provider
def get_credited_artists(*artists,dbconn=None): def get_credited_artists(*artists,dbconn=None):
artist_ids = [get_artist_id(a) for a in artists] artist_ids = [get_artist_id(a,dbconn=dbconn) for a in artists]
jointable = sql.join( jointable = sql.join(
DB['associated_artists'], DB['associated_artists'],
@ -662,7 +662,7 @@ def get_credited_artists(*artists,dbconn=None):
) )
result = dbconn.execute(op).all() result = dbconn.execute(op).all()
artists = artists_db_to_dict(result) artists = artists_db_to_dict(result,dbconn=dbconn)
return artists return artists
@ -677,7 +677,7 @@ def get_track(id,dbconn=None):
result = dbconn.execute(op).all() result = dbconn.execute(op).all()
trackinfo = result[0] trackinfo = result[0]
return track_db_to_dict(trackinfo) return track_db_to_dict(trackinfo,dbconn=dbconn)
@cached_wrapper @cached_wrapper
@connection_provider @connection_provider
@ -688,7 +688,7 @@ def get_artist(id,dbconn=None):
result = dbconn.execute(op).all() result = dbconn.execute(op).all()
artistinfo = result[0] artistinfo = result[0]
return artist_db_to_dict(artistinfo) return artist_db_to_dict(artistinfo,dbconn=dbconn)
@ -746,7 +746,6 @@ def renormalize_names():
if norm_actual != norm_target: if norm_actual != norm_target:
log(f"{name} should be normalized to {norm_target}, but is instead {norm_actual}, fixing...") log(f"{name} should be normalized to {norm_target}, but is instead {norm_actual}, fixing...")
with engine.begin() as conn:
rows = conn.execute(DB['artists'].update().where(DB['artists'].c.id == id).values(name_normalized=norm_target)) rows = conn.execute(DB['artists'].update().where(DB['artists'].c.id == id).values(name_normalized=norm_target))