mirror/maloja
1
0
Fork 0
mirror of https://github.com/krateng/maloja.git synced 2023-08-10 21:12:55 +03:00
Code Issues Projects Releases Wiki Activity

Merge branch 'feature-albums' into next_minor_version

Browse Source
This commit is contained in:
krateng 2023-04-01 04:19:33 +02:00
commit 88c5d1da00
12 changed files with 204 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
maloja/__main__.py
View File

@ -148,7 +148,7 @@ def print_info():
print("Could not determine dependency versions.")
print()
@mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images'],shield=True)
@mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images','prefer_existing'],shield=True)
def main(*args,**kwargs):
actions = {
@ -166,7 +166,7 @@ def main(*args,**kwargs):
"generate":generate.generate_scrobbles, # maloja generate 400
"export":tasks.export, # maloja export
"apidebug":apidebug.run, # maloja apidebug
"parsealbums":tasks.parse_albums, # maloja parsealbums
"parsealbums":tasks.parse_albums, # maloja parsealbums --strategy majority
# aux
"info":print_info
}

19
maloja/database/__init__.py
View File

@ -2,16 +2,16 @@
from bottle import request, response, FormsDict
# we're running an auxiliary task that doesn't require all the random background
# nonsense to be fired up
# this is temporary
# FIX YO DAMN ARCHITECTURE ALREADY
AUX_MODE = False
def set_aux_mode():
global AUX_MODE
AUX_MODE = True
# decorator that makes sure this function is only run in normal operation,
# not when we run a task that needs to access the database
def no_aux_mode(func):
def wrapper(*args,**kwargs):
from ..pkg_global import conf
if conf.AUX_MODE: return
return func(*args,**kwargs)
return wrapper
# rest of the project
from ..cleanup import CleanerAgent
from .. import images
@ -619,6 +619,7 @@ def start_db():
# Upgrade database
from .. import upgrade
upgrade.upgrade_db(sqldb.add_scrobbles)
upgrade.parse_old_albums()
# Load temporary tables
from . import associated

21
maloja/database/dbcache.py
View File

@ -10,7 +10,7 @@ from doreah.regular import runhourly
from doreah.logging import log
from ..pkg_global.conf import malojaconfig
from . import no_aux_mode
if malojaconfig['USE_GLOBAL_CACHE']:
@ -21,15 +21,12 @@ if malojaconfig['USE_GLOBAL_CACHE']:
@runhourly
@no_aux_mode
def maintenance():
from . import AUX_MODE
if AUX_MODE: return
print_stats()
trim_cache()
def print_stats():
from . import AUX_MODE
if AUX_MODE: return
for name,c in (('Cache',cache),('Entity Cache',entitycache)):
hits, misses = c.get_stats()
log(f"{name}: Size: {len(c)} | Hits: {hits}/{hits+misses} | Estimated Memory: {human_readable_size(c)}")
@ -37,8 +34,6 @@ if malojaconfig['USE_GLOBAL_CACHE']:
def cached_wrapper(inner_func):
from . import AUX_MODE
if AUX_MODE: return inner_func
def outer_func(*args,**kwargs):
@ -63,8 +58,6 @@ if malojaconfig['USE_GLOBAL_CACHE']:
# we don't want a new cache entry for every single combination, but keep a common
# cache that's aware of what we're calling
def cached_wrapper_individual(inner_func):
from . import AUX_MODE
if AUX_MODE: return
def outer_func(set_arg,**kwargs):
if 'dbconn' in kwargs:
@ -88,9 +81,8 @@ if malojaconfig['USE_GLOBAL_CACHE']:
return outer_func
@no_aux_mode
def invalidate_caches(scrobbletime=None):
from . import AUX_MODE
if AUX_MODE: return
cleared, kept = 0, 0
for k in cache.keys():
@ -102,16 +94,11 @@ if malojaconfig['USE_GLOBAL_CACHE']:
kept += 1
log(f"Invalidated {cleared} of {cleared+kept} DB cache entries")
@no_aux_mode
def invalidate_entity_cache():
from . import AUX_MODE
if AUX_MODE: return
entitycache.clear()
def trim_cache():
from . import AUX_MODE
if AUX_MODE: return
ramprct = psutil.virtual_memory().percent
if ramprct > malojaconfig["DB_MAX_MEMORY"]:
log(f"{ramprct}% RAM usage, clearing cache!")

2
maloja/database/exceptions.py
View File

@ -16,7 +16,7 @@ class DatabaseNotBuilt(HTTPError):
def __init__(self):
super().__init__(
status=503,
body="The Maloja Database is being upgraded to Version 3. This could take quite a long time! (~ 2-5 minutes per 10 000 scrobbles)",
body="The Maloja Database is being upgraded to support new Maloja features. This could take a while.",
headers={"Retry-After":120}
)

83
maloja/database/sqldb.py
View File

@ -1,4 +1,5 @@
import sqlalchemy as sql
from sqlalchemy.dialects.sqlite import insert as sqliteinsert
import json
import unicodedata
import math
@ -8,6 +9,7 @@ from threading import Lock
from ..pkg_global.conf import data_dir
from .dbcache import cached_wrapper, cached_wrapper_individual, invalidate_caches, invalidate_entity_cache
from . import exceptions as exc
from . import no_aux_mode
from doreah.logging import log
from doreah.regular import runhourly, runmonthly
@ -19,6 +21,13 @@ from doreah.regular import runhourly, runmonthly
DBTABLES = {
# name - type - foreign key - kwargs
'_maloja':{
'columns':[
("key", sql.String, {'primary_key':True}),
("value", sql.String, {})
],
'extraargs':(),'extrakwargs':{}
},
'scrobbles':{
'columns':[
("timestamp", sql.Integer, {'primary_key':True}),
@ -150,6 +159,29 @@ def connection_provider(func):
wrapper.__innerfunc__ = func
return wrapper
@connection_provider
def get_maloja_info(keys,dbconn=None):
op = DB['_maloja'].select().where(
DB['_maloja'].c.key.in_(keys)
)
result = dbconn.execute(op).all()
info = {}
for row in result:
info[row.key] = row.value
return info
@connection_provider
def set_maloja_info(info,dbconn=None):
for k in info:
op = sqliteinsert(DB['_maloja']).values(
key=k, value=info[k]
).on_conflict_do_update(
index_elements=['key'],
set_={'value':info[k]}
)
dbconn.execute(op)
##### DB <-> Dict translations
## ATTENTION ALL ADVENTURERS
@ -451,14 +483,11 @@ def get_artist_id(artistname,create_new=True,dbconn=None):
@cached_wrapper
@connection_provider
def get_album_id(albumdict,create_new=True,dbconn=None):
def get_album_id(albumdict,create_new=True,ignore_albumartists=False,dbconn=None):
ntitle = normalize_name(albumdict['albumtitle'])
artist_ids = [get_artist_id(a,dbconn=dbconn) for a in albumdict.get('artists') or []]
artist_ids = list(set(artist_ids))
op = DB['albums'].select(
# DB['albums'].c.id
).where(
@ -466,20 +495,23 @@ def get_album_id(albumdict,create_new=True,dbconn=None):
)
result = dbconn.execute(op).all()
for row in result:
# check if the artists are the same
foundtrackartists = []
op = DB['albumartists'].select(
# DB['albumartists'].c.artist_id
).where(
DB['albumartists'].c.album_id==row.id
)
result = dbconn.execute(op).all()
match_artist_ids = [r.artist_id for r in result]
#print("required artists",artist_ids,"this match",match_artist_ids)
if set(artist_ids) == set(match_artist_ids):
#print("ID for",albumdict['title'],"was",row[0])
if ignore_albumartists:
return row.id
else:
# check if the artists are the same
foundtrackartists = []
op = DB['albumartists'].select(
# DB['albumartists'].c.artist_id
).where(
DB['albumartists'].c.album_id==row.id
)
result = dbconn.execute(op).all()
match_artist_ids = [r.artist_id for r in result]
#print("required artists",artist_ids,"this match",match_artist_ids)
if set(artist_ids) == set(match_artist_ids):
#print("ID for",albumdict['title'],"was",row[0])
return row.id
if not create_new: return None
@ -1361,11 +1393,9 @@ def search_album(searchterm,dbconn=None):
@runhourly
@connection_provider
@no_aux_mode
def clean_db(dbconn=None):
from . import AUX_MODE
if AUX_MODE: return
with SCROBBLE_LOCK:
log(f"Database Cleanup...")
@ -1410,11 +1440,9 @@ def clean_db(dbconn=None):
@runmonthly
@no_aux_mode
def renormalize_names():
from . import AUX_MODE
if AUX_MODE: return
with SCROBBLE_LOCK:
with engine.begin() as conn:
rows = conn.execute(DB['artists'].select()).all()
@ -1573,7 +1601,7 @@ def guess_albums(track_ids=None,replace=False,dbconn=None):
}}
if len(artists) == 0:
# for albums without artist, assume track artist
res[track_id]["guess_artists"] = True
res[track_id]["guess_artists"] = []
else:
res[track_id] = {"assigned":False,"reason":"Not enough data"}
@ -1582,7 +1610,7 @@ def guess_albums(track_ids=None,replace=False,dbconn=None):
missing_artists = [track_id for track_id in res if res[track_id].get("guess_artists")]
missing_artists = [track_id for track_id in res if "guess_artists" in res[track_id]]
#we're pointlessly getting the albumartist names here even though the IDs would be enough
#but it's better for function separation I guess
@ -1599,10 +1627,7 @@ def guess_albums(track_ids=None,replace=False,dbconn=None):
result = dbconn.execute(op).all()
for row in result:
res[row.track_id]["assigned"]["artists"].append(row.name)
for track_id in res:
if res[track_id].get("guess_artists"):
del res[track_id]["guess_artists"]
res[row.track_id]["guess_artists"].append(row.name)
return res

8
maloja/images.py
View File

@ -134,6 +134,14 @@ resolve_semaphore = BoundedSemaphore(8)
def resolve_track_image(track_id):
if malojaconfig["USE_ALBUM_ARTWORK_FOR_TRACKS"]:
track = database.sqldb.get_track(track_id)
if "album" in track:
album_id = database.sqldb.get_album_id(track["album"])
albumart = resolve_album_image(album_id)
if albumart:
return albumart
with resolve_semaphore:
# check cache
result = get_image_from_cache(track_id,'tracks')

15
maloja/pkg_global/conf.py
View File

@ -6,6 +6,8 @@ from doreah.configuration import types as tp
from ..__pkginfo__ import VERSION
# this mode specifies whether we run some auxiliary task instead of the main server
AUX_MODE = True
# if DATA_DIRECTORY is specified, this is the directory to use for EVERYTHING, no matter what
@ -194,6 +196,7 @@ malojaconfig = Configuration(
"album_showcase":(tp.Boolean(), "Display Album Showcase", True, "Display a graphical album showcase for artist overview pages instead of a chart list"),
"display_art_icons":(tp.Boolean(), "Display Album/Artist Icons", True),
"default_album_artist":(tp.String(), "Default Albumartist", "Various Artists"),
"use_album_artwork_for_tracks":(tp.Boolean(), "Use Album Artwork for tracks", True),
"discourage_cpu_heavy_stats":(tp.Boolean(), "Discourage CPU-heavy stats", False, "Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!"),
"use_local_images":(tp.Boolean(), "Use Local Images", True),
#"local_image_rotate":(tp.Integer(), "Local Image Rotate", 3600),
@ -300,15 +303,6 @@ data_dir = {
### write down the last ran version
with open(pthj(dir_settings['state'],".lastmalojaversion"),"w") as filed:
filed.write(VERSION)
filed.write("\n")
### DOREAH CONFIGURATION
from doreah import config
@ -334,7 +328,8 @@ config(
custom_css_files = [f for f in os.listdir(data_dir['css']()) if f.lower().endswith('.css')]
from ..database.sqldb import set_maloja_info
set_maloja_info({'last_run_version':VERSION})
# what the fuck did i just write
# this spaghetti file is proudly sponsored by the rice crackers i'm eating at the

3
maloja/proccontrol/tasks/import_scrobbles.py
View File

@ -21,9 +21,6 @@ outputs = {
def import_scrobbles(inputf):
from ...database import set_aux_mode
set_aux_mode()
from ...database.sqldb import add_scrobbles
result = {

111
maloja/proccontrol/tasks/parse_albums.py
View File

@ -1,23 +1,108 @@
from doreah.io import col
def parse_albums(strategy=None,prefer_existing=False):
if strategy not in ("track","none","all","majority","most"):
print("""
Please specify your album parsing strategy:
--strategy Specify what strategy to use when the scrobble contains
no information about album artists.
track Take the track artists. This can lead to
separate albums being created for compilation
albums or albums that have collaboration tracks.
none Merge all albums with the same name and assign
'Various Artists' as the album artist.
all Merge all albums with the same name and assign
every artist that appears on the album as an album
artist.
majority Merge all albums with the same name and assign
artists that appear in at least half the tracks
of the album as album artists. [RECOMMENDED]
most Merge all albums with the same name and assign
the artist that appears most on the album as album
artist.
--prefer_existing If an album with the same name already exists, use it
without further examination of track artists.
""")
return
def parse_albums(replace=False):
from ...database import set_aux_mode
set_aux_mode()
from ...database.sqldb import guess_albums, get_album_id, add_track_to_album
print("Parsing album information...")
result = guess_albums(replace=replace)
result = guess_albums()
result = {track_id:result[track_id] for track_id in result if result[track_id]["assigned"]}
print("Adding",len(result),"tracks to albums...")
print("Found",col['yellow'](len(result)),"Tracks to assign albums to")
result_authorative = {track_id:result[track_id] for track_id in result if result[track_id]["assigned"]["artists"]}
result_guesswork = {track_id:result[track_id] for track_id in result if not result[track_id]["assigned"]["artists"]}
i = 0
for track_id in result:
album_id = get_album_id(result[track_id]["assigned"])
add_track_to_album(track_id,album_id)
i += 1
def countup(i):
i+=1
if (i % 100) == 0:
print(i,"of",len(result))
print("Done!")
print(f"Added album information for {i} of {len(result)} tracks...")
return i
for track_id in result_authorative:
albuminfo = result[track_id]['assigned']
album_id = get_album_id(albuminfo)
add_track_to_album(track_id,album_id)
i=countup(i)
albums = {}
for track_id in result_guesswork:
albuminfo = result[track_id]['assigned']
# check if already exists
if prefer_existing:
album_id = get_album_id(albuminfo,ignore_albumartists=True,create_new=False)
if album_id:
add_track_to_album(track_id,album_id)
i=countup(i)
continue
if strategy == 'track':
albuminfo['artists'] = result[track_id]['guess_artists']
album_id = get_album_id(albuminfo)
add_track_to_album(track_id,album_id)
i=countup(i)
continue
if strategy == 'none':
albuminfo['artists'] = []
album_id = get_album_id(albuminfo)
add_track_to_album(track_id,album_id)
i=countup(i)
continue
if strategy in ['all','majority','most']:
cleantitle = albuminfo['albumtitle'].lower()
albums.setdefault(cleantitle,{'track_ids':[],'artists':{},'title':albuminfo['albumtitle']})
albums[cleantitle]['track_ids'].append(track_id)
for a in result[track_id]['guess_artists']:
albums[cleantitle]['artists'].setdefault(a,0)
albums[cleantitle]['artists'][a] += 1
for cleantitle in albums:
artistoptions = albums[cleantitle]['artists']
track_ids = albums[cleantitle]['track_ids']
realtitle = albums[cleantitle]['title']
if strategy == 'all':
artists = [a for a in artistoptions]
elif strategy == 'majority':
artists = [a for a in artistoptions if artistoptions[a] >= (len(track_ids) / 2)]
elif strategy == 'most':
artists = [max(artistoptions,key=artistoptions.get)]
for track_id in track_ids:
album_id = get_album_id({'albumtitle':realtitle,'artists':artists})
add_track_to_album(track_id,album_id)
i=countup(i)
print(col['lawngreen']("Done!"))

3
maloja/server.py
View File

@ -22,6 +22,7 @@ from .database.jinjaview import JinjaDBConnection
from .images import resolve_track_image, resolve_artist_image, resolve_album_image
from .malojauri import uri_to_internal, remove_identical
from .pkg_global.conf import malojaconfig, data_dir
from .pkg_global import conf
from .jinjaenv.context import jinja_environment
from .apis import init_apis, apikeystore
@ -285,6 +286,8 @@ logging.getLogger().addHandler(WaitressLogHandler())
def run_server():
conf.AUX_MODE = False
log("Starting up Maloja server...")
## start database

15
maloja/upgrade.py
View File

@ -11,6 +11,9 @@ from .pkg_global.conf import data_dir, dir_settings
from .apis import _apikeys
from .database.sqldb import get_maloja_info, set_maloja_info
# Dealing with old style tsv files - these should be phased out everywhere
def read_tsvs(path,types):
result = []
@ -40,7 +43,7 @@ def upgrade_apikeys():
except Exception:
pass
# v2 to v3 iupgrade
def upgrade_db(callback_add_scrobbles):
oldfolder = os.path.join(dir_settings['state'],"scrobbles")
@ -88,3 +91,13 @@ def upgrade_db(callback_add_scrobbles):
callback_add_scrobbles(scrobblelist)
os.rename(os.path.join(oldfolder,sf),os.path.join(newfolder,sf))
log("Done!",color='yellow')
# 3.2 album support
def parse_old_albums():
setting_name = "db_upgrade_albums"
if get_maloja_info([setting_name]).get(setting_name):
pass
else:
pass
#set_maloja_info({setting_name:True})

6
maloja/web/jinja/artist.jinja
View File

@ -90,7 +90,11 @@
</table>
{% if info["isalbumartist"] %}
{% set albums_info = dbc.get_albums_artist_appears_on(filterkeys,limitkeys) %}
{% set ownalbums = albums_info.own_albums %}
{% set otheralbums = albums_info.appears_on %}
{% if ownalbums or otheralbums %}
{% if settings['ALBUM_SHOWCASE'] %}
<h2><a href='{{ mlj_uri.create_uri("/charts_albums",filterkeys) }}'>Albums</a></h2>