Fixed global database lock release during scrobble creation, GH-126

dev/releases/3.0.yml

@@ -36,3 +36,6 @@ minor_release_name: "Yeonhee"
 3.0.6:
     notes:
     - "[Bugfix] Better parsing of featuring artists"
+    - "[Bugfix] Fixed buffered output in Docker"
+    - "[Bugfix] Fixed importing a Spotify file without path"
+    - "[Bugfix] No longer releasing database lock during scrobble creation"

maloja/database/sqldb.py

@@ -157,8 +157,8 @@ def connection_provider(func):
 
 
 ### DB -> DICT
-def scrobbles_db_to_dict(rows,include_internal=False):
+def scrobbles_db_to_dict(rows,include_internal=False,dbconn=None):
-	tracks = get_tracks_map(set(row.track_id for row in rows))
+	tracks = get_tracks_map(set(row.track_id for row in rows),dbconn=dbconn)
 	return [
 		{
 			**{
@@ -176,11 +176,11 @@ def scrobbles_db_to_dict(rows,include_internal=False):
 		for row in rows
 	]
 
-def scrobble_db_to_dict(row):
+def scrobble_db_to_dict(row,dbconn=None):
-	return scrobbles_db_to_dict([row])[0]
+	return scrobbles_db_to_dict([row],dbconn=dbconn)[0]
 
-def tracks_db_to_dict(rows):
+def tracks_db_to_dict(rows,dbconn=None):
-	artists = get_artists_of_tracks(set(row.id for row in rows))
+	artists = get_artists_of_tracks(set(row.id for row in rows),dbconn=dbconn)
 	return [
 		{
 			"artists":artists[row.id],
@@ -191,41 +191,41 @@ def tracks_db_to_dict(rows):
 		for row in rows
 	]
 
-def track_db_to_dict(row):
+def track_db_to_dict(row,dbconn=None):
-	return tracks_db_to_dict([row])[0]
+	return tracks_db_to_dict([row],dbconn=dbconn)[0]
 
-def artists_db_to_dict(rows):
+def artists_db_to_dict(rows,dbconn=None):
 	return [
 		row.name
 		for row in rows
 	]
 
-def artist_db_to_dict(row):
+def artist_db_to_dict(row,dbconn=None):
-	return artists_db_to_dict([row])[0]
+	return artists_db_to_dict([row],dbconn=dbconn)[0]
 
 
 
 
 ### DICT -> DB
 
-def scrobble_dict_to_db(info):
+def scrobble_dict_to_db(info,dbconn=None):
 	return {
 		"timestamp":info['time'],
 		"origin":info['origin'],
 		"duration":info['duration'],
-		"track_id":get_track_id(info['track']),
+		"track_id":get_track_id(info['track'],dbconn=dbconn),
 		"extra":json.dumps(info.get('extra',{})),
 		"rawscrobble":json.dumps(info.get('rawscrobble',{}))
 	}
 
-def track_dict_to_db(info):
+def track_dict_to_db(info,dbconn=None):
 	return {
 		"title":info['title'],
 		"title_normalized":normalize_name(info['title']),
 		"length":info.get('length')
 	}
 
-def artist_dict_to_db(info):
+def artist_dict_to_db(info,dbconn=None):
 	return {
 		"name": info,
 		"name_normalized":normalize_name(info)
@@ -249,7 +249,7 @@ def add_scrobbles(scrobbleslist,dbconn=None):
 
 		ops = [
 			DB['scrobbles'].insert().values(
-				**scrobble_dict_to_db(s)
+				**scrobble_dict_to_db(s,dbconn=dbconn)
 			) for s in scrobbleslist
 		]
 
@@ -283,7 +283,7 @@ def delete_scrobble(scrobble_id,dbconn=None):
 @connection_provider
 def get_track_id(trackdict,dbconn=None):
 	ntitle = normalize_name(trackdict['title'])
-	artist_ids = [get_artist_id(a) for a in trackdict['artists']]
+	artist_ids = [get_artist_id(a,dbconn=dbconn) for a in trackdict['artists']]
 	artist_ids = list(set(artist_ids))
 
 
@@ -313,7 +313,7 @@ def get_track_id(trackdict,dbconn=None):
 
 
 	op = DB['tracks'].insert().values(
-		**track_dict_to_db(trackdict)
+		**track_dict_to_db(trackdict,dbconn=dbconn)
 	)
 	result = dbconn.execute(op)
 	track_id = result.inserted_primary_key[0]
@@ -366,7 +366,7 @@ def get_scrobbles_of_artist(artist,since=None,to=None,resolve_references=True,db
 	if since is None: since=0
 	if to is None: to=now()
 
-	artist_id = get_artist_id(artist)
+	artist_id = get_artist_id(artist,dbconn=dbconn)
 
 	jointable = sql.join(DB['scrobbles'],DB['trackartists'],DB['scrobbles'].c.track_id == DB['trackartists'].c.track_id)
 
@@ -378,7 +378,7 @@ def get_scrobbles_of_artist(artist,since=None,to=None,resolve_references=True,db
 	result = dbconn.execute(op).all()
 
 	if resolve_references:
-		result = scrobbles_db_to_dict(result)
+		result = scrobbles_db_to_dict(result,dbconn=dbconn)
 	#result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for row in result]
 	return result
 
@@ -389,7 +389,7 @@ def get_scrobbles_of_track(track,since=None,to=None,resolve_references=True,dbco
 	if since is None: since=0
 	if to is None: to=now()
 
-	track_id = get_track_id(track)
+	track_id = get_track_id(track,dbconn=dbconn)
 
 	op = DB['scrobbles'].select().where(
 		DB['scrobbles'].c.timestamp<=to,
@@ -417,7 +417,7 @@ def get_scrobbles(since=None,to=None,resolve_references=True,dbconn=None):
 	result = dbconn.execute(op).all()
 
 	if resolve_references:
-		result = scrobbles_db_to_dict(result)
+		result = scrobbles_db_to_dict(result,dbconn=dbconn)
 	#result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for i,row in enumerate(result) if i<max]
 	return result
 
@@ -447,7 +447,7 @@ def get_artists_of_track(track_id,resolve_references=True,dbconn=None):
 	)
 	result = dbconn.execute(op).all()
 
-	artists = [get_artist(row.artist_id) if resolve_references else row.artist_id for row in result]
+	artists = [get_artist(row.artist_id,dbconn=dbconn) if resolve_references else row.artist_id for row in result]
 	return artists
 
 
@@ -455,14 +455,14 @@ def get_artists_of_track(track_id,resolve_references=True,dbconn=None):
 @connection_provider
 def get_tracks_of_artist(artist,dbconn=None):
 
-	artist_id = get_artist_id(artist)
+	artist_id = get_artist_id(artist,dbconn=dbconn)
 
 	op = sql.join(DB['tracks'],DB['trackartists']).select().where(
 		DB['trackartists'].c.artist_id==artist_id
 	)
 	result = dbconn.execute(op).all()
 
-	return tracks_db_to_dict(result)
+	return tracks_db_to_dict(result,dbconn=dbconn)
 
 @cached_wrapper
 @connection_provider
@@ -471,7 +471,7 @@ def get_artists(dbconn=None):
 	op = DB['artists'].select()
 	result = dbconn.execute(op).all()
 
-	return artists_db_to_dict(result)
+	return artists_db_to_dict(result,dbconn=dbconn)
 
 @cached_wrapper
 @connection_provider
@@ -480,7 +480,7 @@ def get_tracks(dbconn=None):
 	op = DB['tracks'].select()
 	result = dbconn.execute(op).all()
 
-	return tracks_db_to_dict(result)
+	return tracks_db_to_dict(result,dbconn=dbconn)
 
 ### functions that count rows for parameters
 
@@ -516,7 +516,7 @@ def count_scrobbles_by_artist(since,to,dbconn=None):
 
 
 	counts = [row.count for row in result]
-	artists = get_artists_map([row.artist_id for row in result])
+	artists = get_artists_map([row.artist_id for row in result],dbconn=dbconn)
 	result = [{'scrobbles':row.count,'artist':artists[row.artist_id]} for row in result]
 	result = rank(result,key='scrobbles')
 	return result
@@ -537,7 +537,7 @@ def count_scrobbles_by_track(since,to,dbconn=None):
 
 
 	counts = [row.count for row in result]
-	tracks = get_tracks_map([row.track_id for row in result])
+	tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn)
 	result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result]
 	result = rank(result,key='scrobbles')
 	return result
@@ -546,7 +546,7 @@ def count_scrobbles_by_track(since,to,dbconn=None):
 @connection_provider
 def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None):
 
-	artist_id = get_artist_id(artist)
+	artist_id = get_artist_id(artist,dbconn=dbconn)
 
 	jointable = sql.join(
 		DB['scrobbles'],
@@ -566,7 +566,7 @@ def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None):
 
 
 	counts = [row.count for row in result]
-	tracks = get_tracks_map([row.track_id for row in result])
+	tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn)
 	result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result]
 	result = rank(result,key='scrobbles')
 	return result
@@ -586,7 +586,7 @@ def get_artists_of_tracks(track_ids,dbconn=None):
 
 	artists = {}
 	for row in result:
-		artists.setdefault(row.track_id,[]).append(artist_db_to_dict(row))
+		artists.setdefault(row.track_id,[]).append(artist_db_to_dict(row,dbconn=dbconn))
 	return artists
 
 
@@ -601,7 +601,7 @@ def get_tracks_map(track_ids,dbconn=None):
 	tracks = {}
 	result = list(result)
 	# this will get a list of artistdicts in the correct order of our rows
-	trackdicts = tracks_db_to_dict(result)
+	trackdicts = tracks_db_to_dict(result,dbconn=dbconn)
 	for row,trackdict in zip(result,trackdicts):
 		tracks[row.id] = trackdict
 	return tracks
@@ -618,7 +618,7 @@ def get_artists_map(artist_ids,dbconn=None):
 	artists = {}
 	result = list(result)
 	# this will get a list of artistdicts in the correct order of our rows
-	artistdicts = artists_db_to_dict(result)
+	artistdicts = artists_db_to_dict(result,dbconn=dbconn)
 	for row,artistdict in zip(result,artistdicts):
 		artists[row.id] = artistdict
 	return artists
@@ -629,7 +629,7 @@ def get_artists_map(artist_ids,dbconn=None):
 @cached_wrapper
 @connection_provider
 def get_associated_artists(*artists,dbconn=None):
-	artist_ids = [get_artist_id(a) for a in artists]
+	artist_ids = [get_artist_id(a,dbconn=dbconn) for a in artists]
 
 	jointable = sql.join(
 		DB['associated_artists'],
@@ -642,13 +642,13 @@ def get_associated_artists(*artists,dbconn=None):
 	)
 	result = dbconn.execute(op).all()
 
-	artists = artists_db_to_dict(result)
+	artists = artists_db_to_dict(result,dbconn=dbconn)
 	return artists
 
 @cached_wrapper
 @connection_provider
 def get_credited_artists(*artists,dbconn=None):
-	artist_ids = [get_artist_id(a) for a in artists]
+	artist_ids = [get_artist_id(a,dbconn=dbconn) for a in artists]
 
 	jointable = sql.join(
 		DB['associated_artists'],
@@ -662,7 +662,7 @@ def get_credited_artists(*artists,dbconn=None):
 	)
 	result = dbconn.execute(op).all()
 
-	artists = artists_db_to_dict(result)
+	artists = artists_db_to_dict(result,dbconn=dbconn)
 	return artists
 
 
@@ -677,7 +677,7 @@ def get_track(id,dbconn=None):
 	result = dbconn.execute(op).all()
 
 	trackinfo = result[0]
-	return track_db_to_dict(trackinfo)
+	return track_db_to_dict(trackinfo,dbconn=dbconn)
 
 @cached_wrapper
 @connection_provider
@@ -688,7 +688,7 @@ def get_artist(id,dbconn=None):
 	result = dbconn.execute(op).all()
 
 	artistinfo = result[0]
-	return artist_db_to_dict(artistinfo)
+	return artist_db_to_dict(artistinfo,dbconn=dbconn)
 
 
 
@@ -738,15 +738,14 @@ def renormalize_names():
 		with engine.begin() as conn:
 			rows = conn.execute(DB['artists'].select()).all()
 
-		for row in rows:
+			for row in rows:
-			id = row.id
+				id = row.id
-			name = row.name
+				name = row.name
-			norm_actual = row.name_normalized
+				norm_actual = row.name_normalized
-			norm_target = normalize_name(name)
+				norm_target = normalize_name(name)
-			if norm_actual != norm_target:
+				if norm_actual != norm_target:
-				log(f"{name} should be normalized to {norm_target}, but is instead {norm_actual}, fixing...")
+					log(f"{name} should be normalized to {norm_target}, but is instead {norm_actual}, fixing...")
 
-				with engine.begin() as conn:
 					rows = conn.execute(DB['artists'].update().where(DB['artists'].c.id == id).values(name_normalized=norm_target))