Merge branch 'feature-albums' into next_minor_version

maloja/__main__.py

@@ -148,7 +148,7 @@ def print_info():
 		print("Could not determine dependency versions.")
 	print()
 
-@mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images'],shield=True)
+@mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images','prefer_existing'],shield=True)
 def main(*args,**kwargs):
 
 	actions = {
@@ -166,7 +166,7 @@ def main(*args,**kwargs):
 		"generate":generate.generate_scrobbles,	# maloja generate 400
 		"export":tasks.export,					# maloja export
 		"apidebug":apidebug.run,				# maloja apidebug
-		"parsealbums":tasks.parse_albums,		# maloja parsealbums
+		"parsealbums":tasks.parse_albums,		# maloja parsealbums --strategy majority
 		# aux
 		"info":print_info
 	}

maloja/database/__init__.py

@@ -2,14 +2,14 @@
 from bottle import request, response, FormsDict
 
 
-# we're running an auxiliary task that doesn't require all the random background
+# decorator that makes sure this function is only run in normal operation,
-# nonsense to be fired up
+# not when we run a task that needs to access the database
-# this is temporary
+def no_aux_mode(func):
-# FIX YO DAMN ARCHITECTURE ALREADY
+	def wrapper(*args,**kwargs):
-AUX_MODE = False
+		from ..pkg_global import conf
-def set_aux_mode():
+		if conf.AUX_MODE: return
-	global AUX_MODE
+		return func(*args,**kwargs)
-	AUX_MODE = True
+	return wrapper
 
 
 # rest of the project
@@ -619,6 +619,7 @@ def start_db():
 	# Upgrade database
 	from .. import upgrade
 	upgrade.upgrade_db(sqldb.add_scrobbles)
+	upgrade.parse_old_albums()
 
 	# Load temporary tables
 	from . import associated

maloja/database/dbcache.py

@@ -10,7 +10,7 @@ from doreah.regular import runhourly
 from doreah.logging import log
 
 from ..pkg_global.conf import malojaconfig
-
+from . import no_aux_mode
 
 
 if malojaconfig['USE_GLOBAL_CACHE']:
@@ -21,15 +21,12 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 
 
 	@runhourly
+	@no_aux_mode
 	def maintenance():
-		from . import AUX_MODE
-		if AUX_MODE: return
 		print_stats()
 		trim_cache()
 
 	def print_stats():
-		from . import AUX_MODE
-		if AUX_MODE: return
 		for name,c in (('Cache',cache),('Entity Cache',entitycache)):
 			hits, misses = c.get_stats()
 			log(f"{name}: Size: {len(c)} | Hits: {hits}/{hits+misses} | Estimated Memory: {human_readable_size(c)}")
@@ -37,8 +34,6 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 
 
 	def cached_wrapper(inner_func):
-		from . import AUX_MODE
-		if AUX_MODE: return inner_func
 
 		def outer_func(*args,**kwargs):
 
@@ -63,8 +58,6 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 	# we don't want a new cache entry for every single combination, but keep a common
 	# cache that's aware of what we're calling
 	def cached_wrapper_individual(inner_func):
-		from . import AUX_MODE
-		if AUX_MODE: return
 
 		def outer_func(set_arg,**kwargs):
 			if 'dbconn' in kwargs:
@@ -88,9 +81,8 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 
 		return outer_func
 
+	@no_aux_mode
 	def invalidate_caches(scrobbletime=None):
-		from . import AUX_MODE
-		if AUX_MODE: return
 
 		cleared, kept = 0, 0
 		for k in cache.keys():
@@ -102,16 +94,11 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 				kept += 1
 		log(f"Invalidated {cleared} of {cleared+kept} DB cache entries")
 
-
+	@no_aux_mode
 	def invalidate_entity_cache():
-		from . import AUX_MODE
-		if AUX_MODE: return
 		entitycache.clear()
 
-
 	def trim_cache():
-		from . import AUX_MODE
-		if AUX_MODE: return
 		ramprct = psutil.virtual_memory().percent
 		if ramprct > malojaconfig["DB_MAX_MEMORY"]:
 			log(f"{ramprct}% RAM usage, clearing cache!")

maloja/database/exceptions.py

@@ -16,7 +16,7 @@ class DatabaseNotBuilt(HTTPError):
 	def __init__(self):
 		super().__init__(
 			status=503,
-			body="The Maloja Database is being upgraded to Version 3. This could take quite a long time! (~ 2-5 minutes per 10 000 scrobbles)",
+			body="The Maloja Database is being upgraded to support new Maloja features. This could take a while.",
 			headers={"Retry-After":120}
 		)
 

maloja/database/sqldb.py

@@ -1,4 +1,5 @@
 import sqlalchemy as sql
+from sqlalchemy.dialects.sqlite import insert as sqliteinsert
 import json
 import unicodedata
 import math
@@ -8,6 +9,7 @@ from threading import Lock
 from ..pkg_global.conf import data_dir
 from .dbcache import cached_wrapper, cached_wrapper_individual, invalidate_caches, invalidate_entity_cache
 from . import exceptions as exc
+from . import no_aux_mode
 
 from doreah.logging import log
 from doreah.regular import runhourly, runmonthly
@@ -19,6 +21,13 @@ from doreah.regular import runhourly, runmonthly
 
 DBTABLES = {
 	# name - type - foreign key - kwargs
+	'_maloja':{
+		'columns':[
+			("key",                 sql.String,                                   {'primary_key':True}),
+			("value",               sql.String,                                   {})
+		],
+		'extraargs':(),'extrakwargs':{}
+	},
 	'scrobbles':{
 		'columns':[
 			("timestamp",           sql.Integer,                                  {'primary_key':True}),
@@ -150,6 +159,29 @@ def connection_provider(func):
 	wrapper.__innerfunc__ = func
 	return wrapper
 
+@connection_provider
+def get_maloja_info(keys,dbconn=None):
+	op = DB['_maloja'].select().where(
+		DB['_maloja'].c.key.in_(keys)
+	)
+	result = dbconn.execute(op).all()
+
+	info = {}
+	for row in result:
+		info[row.key] = row.value
+	return info
+
+@connection_provider
+def set_maloja_info(info,dbconn=None):
+	for k in info:
+		op = sqliteinsert(DB['_maloja']).values(
+			key=k, value=info[k]
+		).on_conflict_do_update(
+			index_elements=['key'],
+			set_={'value':info[k]}
+		)
+		dbconn.execute(op)
+
 ##### DB <-> Dict translations
 
 ## ATTENTION ALL ADVENTURERS
@@ -451,14 +483,11 @@ def get_artist_id(artistname,create_new=True,dbconn=None):
 
 @cached_wrapper
 @connection_provider
-def get_album_id(albumdict,create_new=True,dbconn=None):
+def get_album_id(albumdict,create_new=True,ignore_albumartists=False,dbconn=None):
 	ntitle = normalize_name(albumdict['albumtitle'])
 	artist_ids = [get_artist_id(a,dbconn=dbconn) for a in albumdict.get('artists') or []]
 	artist_ids = list(set(artist_ids))
 
-
-
-
 	op = DB['albums'].select(
 #		DB['albums'].c.id
 	).where(
@@ -466,20 +495,23 @@ def get_album_id(albumdict,create_new=True,dbconn=None):
 	)
 	result = dbconn.execute(op).all()
 	for row in result:
-		# check if the artists are the same
+		if ignore_albumartists:
-		foundtrackartists = []
-
-		op = DB['albumartists'].select(
-#			DB['albumartists'].c.artist_id
-		).where(
-			DB['albumartists'].c.album_id==row.id
-		)
-		result = dbconn.execute(op).all()
-		match_artist_ids = [r.artist_id for r in result]
-		#print("required artists",artist_ids,"this match",match_artist_ids)
-		if set(artist_ids) == set(match_artist_ids):
-			#print("ID for",albumdict['title'],"was",row[0])
 			return row.id
+		else:
+			# check if the artists are the same
+			foundtrackartists = []
+
+			op = DB['albumartists'].select(
+	#			DB['albumartists'].c.artist_id
+			).where(
+				DB['albumartists'].c.album_id==row.id
+			)
+			result = dbconn.execute(op).all()
+			match_artist_ids = [r.artist_id for r in result]
+			#print("required artists",artist_ids,"this match",match_artist_ids)
+			if set(artist_ids) == set(match_artist_ids):
+				#print("ID for",albumdict['title'],"was",row[0])
+				return row.id
 
 	if not create_new: return None
 
@@ -1361,11 +1393,9 @@ def search_album(searchterm,dbconn=None):
 
 @runhourly
 @connection_provider
+@no_aux_mode
 def clean_db(dbconn=None):
 
-	from . import AUX_MODE
-	if AUX_MODE: return
-
 	with SCROBBLE_LOCK:
 		log(f"Database Cleanup...")
 
@@ -1410,11 +1440,9 @@ def clean_db(dbconn=None):
 
 
 @runmonthly
+@no_aux_mode
 def renormalize_names():
 
-	from . import AUX_MODE
-	if AUX_MODE: return
-
 	with SCROBBLE_LOCK:
 		with engine.begin() as conn:
 			rows = conn.execute(DB['artists'].select()).all()
@@ -1573,7 +1601,7 @@ def guess_albums(track_ids=None,replace=False,dbconn=None):
 				}}
 				if len(artists) == 0:
 					# for albums without artist, assume track artist
-					res[track_id]["guess_artists"] = True
+					res[track_id]["guess_artists"] = []
 			else:
 				res[track_id] = {"assigned":False,"reason":"Not enough data"}
 
@@ -1582,7 +1610,7 @@ def guess_albums(track_ids=None,replace=False,dbconn=None):
 
 
 
-	missing_artists = [track_id for track_id in res if res[track_id].get("guess_artists")]
+	missing_artists = [track_id for track_id in res if "guess_artists" in res[track_id]]
 
 	#we're pointlessly getting the albumartist names here even though the IDs would be enough
 	#but it's better for function separation I guess
@@ -1599,10 +1627,7 @@ def guess_albums(track_ids=None,replace=False,dbconn=None):
 	result = dbconn.execute(op).all()
 
 	for row in result:
-		res[row.track_id]["assigned"]["artists"].append(row.name)
+		res[row.track_id]["guess_artists"].append(row.name)
-	for track_id in res:
-		if res[track_id].get("guess_artists"):
-			del res[track_id]["guess_artists"]
 
 	return res
 

maloja/images.py

@@ -134,6 +134,14 @@ resolve_semaphore = BoundedSemaphore(8)
 
 def resolve_track_image(track_id):
 
+	if malojaconfig["USE_ALBUM_ARTWORK_FOR_TRACKS"]:
+		track = database.sqldb.get_track(track_id)
+		if "album" in track:
+			album_id = database.sqldb.get_album_id(track["album"])
+			albumart = resolve_album_image(album_id)
+			if albumart:
+				return albumart
+
 	with resolve_semaphore:
 		# check cache
 		result = get_image_from_cache(track_id,'tracks')

maloja/pkg_global/conf.py

@@ -6,6 +6,8 @@ from doreah.configuration import types as tp
 from ..__pkginfo__ import VERSION
 
 
+# this mode specifies whether we run some auxiliary task instead of the main server
+AUX_MODE = True
 
 
 # if DATA_DIRECTORY is specified, this is the directory to use for EVERYTHING, no matter what
@@ -194,6 +196,7 @@ malojaconfig = Configuration(
 			"album_showcase":(tp.Boolean(),										"Display Album Showcase",		True,		"Display a graphical album showcase for artist overview pages instead of a chart list"),
 			"display_art_icons":(tp.Boolean(),									"Display Album/Artist Icons",	True),
 			"default_album_artist":(tp.String(),								"Default Albumartist",			"Various Artists"),
+			"use_album_artwork_for_tracks":(tp.Boolean(),						"Use Album Artwork for tracks",	True),
 			"discourage_cpu_heavy_stats":(tp.Boolean(),							"Discourage CPU-heavy stats",	False,					"Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!"),
 			"use_local_images":(tp.Boolean(),									"Use Local Images",				True),
 			#"local_image_rotate":(tp.Integer(),									"Local Image Rotate",			3600),
@@ -300,15 +303,6 @@ data_dir = {
 
 
 
-### write down the last ran version
-with open(pthj(dir_settings['state'],".lastmalojaversion"),"w") as filed:
-	filed.write(VERSION)
-	filed.write("\n")
-
-
-
-
-
 ### DOREAH CONFIGURATION
 
 from doreah import config
@@ -334,7 +328,8 @@ config(
 
 custom_css_files = [f for f in os.listdir(data_dir['css']()) if f.lower().endswith('.css')]
 
-
+from ..database.sqldb import set_maloja_info
+set_maloja_info({'last_run_version':VERSION})
 
 # what the fuck did i just write
 # this spaghetti file is proudly sponsored by the rice crackers i'm eating at the

maloja/proccontrol/tasks/import_scrobbles.py

@@ -21,9 +21,6 @@ outputs = {
 
 def import_scrobbles(inputf):
 
-	from ...database import set_aux_mode
-	set_aux_mode()
-
 	from ...database.sqldb import add_scrobbles
 
 	result = {

maloja/proccontrol/tasks/parse_albums.py

@@ -1,23 +1,108 @@
+from doreah.io import col
+
+def parse_albums(strategy=None,prefer_existing=False):
+
+	if strategy not in ("track","none","all","majority","most"):
+		print("""
+Please specify your album parsing strategy:
+
+    --strategy           Specify what strategy to use when the scrobble contains
+                         no information about album artists.
+                         track      Take the track artists. This can lead to
+                                    separate albums being created for compilation
+                                    albums or albums that have collaboration tracks.
+                         none       Merge all albums with the same name and assign
+                                    'Various Artists' as the album artist.
+                         all        Merge all albums with the same name and assign
+                                    every artist that appears on the album as an album
+                                    artist.
+                         majority   Merge all albums with the same name and assign
+                                    artists that appear in at least half the tracks
+                                    of the album as album artists. [RECOMMENDED]
+                         most       Merge all albums with the same name and assign
+                                    the artist that appears most on the album as album
+                                    artist.
+    --prefer_existing    If an album with the same name already exists, use it
+                         without further examination of track artists.
+		""")
+		return
 
 
 
-def parse_albums(replace=False):
-
-	from ...database import set_aux_mode
-	set_aux_mode()
-
 	from ...database.sqldb import guess_albums, get_album_id, add_track_to_album
 
 	print("Parsing album information...")
-	result = guess_albums(replace=replace)
+	result = guess_albums()
 
 	result = {track_id:result[track_id] for track_id in result if result[track_id]["assigned"]}
-	print("Adding",len(result),"tracks to albums...")
+	print("Found",col['yellow'](len(result)),"Tracks to assign albums to")
+
+	result_authorative = {track_id:result[track_id] for track_id in result if result[track_id]["assigned"]["artists"]}
+	result_guesswork = {track_id:result[track_id] for track_id in result if not result[track_id]["assigned"]["artists"]}
+
 	i = 0
-	for track_id in result:
+
-		album_id = get_album_id(result[track_id]["assigned"])
+	def countup(i):
-		add_track_to_album(track_id,album_id)
+		i+=1
-		i += 1
 		if (i % 100) == 0:
-			print(i,"of",len(result))
+			print(f"Added album information for {i} of {len(result)} tracks...")
-	print("Done!")
+		return i
+
+	for track_id in result_authorative:
+		albuminfo = result[track_id]['assigned']
+		album_id = get_album_id(albuminfo)
+		add_track_to_album(track_id,album_id)
+		i=countup(i)
+
+	albums = {}
+	for track_id in result_guesswork:
+		albuminfo = result[track_id]['assigned']
+
+		# check if already exists
+		if prefer_existing:
+			album_id = get_album_id(albuminfo,ignore_albumartists=True,create_new=False)
+			if album_id:
+				add_track_to_album(track_id,album_id)
+				i=countup(i)
+				continue
+
+		if strategy == 'track':
+			albuminfo['artists'] = result[track_id]['guess_artists']
+			album_id = get_album_id(albuminfo)
+			add_track_to_album(track_id,album_id)
+			i=countup(i)
+			continue
+
+		if strategy == 'none':
+			albuminfo['artists'] = []
+			album_id = get_album_id(albuminfo)
+			add_track_to_album(track_id,album_id)
+			i=countup(i)
+			continue
+
+		if strategy in ['all','majority','most']:
+			cleantitle = albuminfo['albumtitle'].lower()
+			albums.setdefault(cleantitle,{'track_ids':[],'artists':{},'title':albuminfo['albumtitle']})
+			albums[cleantitle]['track_ids'].append(track_id)
+			for a in result[track_id]['guess_artists']:
+				albums[cleantitle]['artists'].setdefault(a,0)
+				albums[cleantitle]['artists'][a] += 1
+
+
+	for cleantitle in albums:
+		artistoptions = albums[cleantitle]['artists']
+		track_ids = albums[cleantitle]['track_ids']
+		realtitle = albums[cleantitle]['title']
+		if strategy == 'all':
+			artists = [a for a in artistoptions]
+		elif strategy == 'majority':
+			artists = [a for a in artistoptions if artistoptions[a] >= (len(track_ids) / 2)]
+		elif strategy == 'most':
+			artists = [max(artistoptions,key=artistoptions.get)]
+
+		for track_id in track_ids:
+			album_id = get_album_id({'albumtitle':realtitle,'artists':artists})
+			add_track_to_album(track_id,album_id)
+			i=countup(i)
+
+	print(col['lawngreen']("Done!"))

maloja/server.py

@@ -22,6 +22,7 @@ from .database.jinjaview import JinjaDBConnection
 from .images import resolve_track_image, resolve_artist_image, resolve_album_image
 from .malojauri import uri_to_internal, remove_identical
 from .pkg_global.conf import malojaconfig, data_dir
+from .pkg_global import conf
 from .jinjaenv.context import jinja_environment
 from .apis import init_apis, apikeystore
 
@@ -285,6 +286,8 @@ logging.getLogger().addHandler(WaitressLogHandler())
 
 
 def run_server():
+	conf.AUX_MODE = False
+
 	log("Starting up Maloja server...")
 
 	## start database

maloja/upgrade.py

@@ -11,6 +11,9 @@ from .pkg_global.conf import data_dir, dir_settings
 from .apis import _apikeys
 
 
+from .database.sqldb import get_maloja_info, set_maloja_info
+
+
 # Dealing with old style tsv files - these should be phased out everywhere
 def read_tsvs(path,types):
 	result = []
@@ -40,7 +43,7 @@ def upgrade_apikeys():
 		except Exception:
 			pass
 
-
+# v2 to v3 iupgrade
 def upgrade_db(callback_add_scrobbles):
 
 	oldfolder = os.path.join(dir_settings['state'],"scrobbles")
@@ -88,3 +91,13 @@ def upgrade_db(callback_add_scrobbles):
 				callback_add_scrobbles(scrobblelist)
 				os.rename(os.path.join(oldfolder,sf),os.path.join(newfolder,sf))
 			log("Done!",color='yellow')
+
+
+# 3.2 album support
+def parse_old_albums():
+	setting_name = "db_upgrade_albums"
+	if get_maloja_info([setting_name]).get(setting_name):
+		pass
+	else:
+		pass
+		#set_maloja_info({setting_name:True})

maloja/web/jinja/artist.jinja

@@ -90,7 +90,11 @@
 </table>
 
 
-{% if info["isalbumartist"] %}
+{% set albums_info = dbc.get_albums_artist_appears_on(filterkeys,limitkeys) %}
+{% set ownalbums = albums_info.own_albums %}
+{% set otheralbums = albums_info.appears_on %}
+
+{% if ownalbums or otheralbums %}
 
 {% if settings['ALBUM_SHOWCASE'] %}
 	<h2><a href='{{ mlj_uri.create_uri("/charts_albums",filterkeys) }}'>Albums</a></h2>