Merge branch 'feature-asyncimages' into next_minor_version

Refactor container image to use linuxserverio alpine base

.dockerignore

@@ -1,7 +1,7 @@
 *
 !maloja
+!container
 !Containerfile
-!requirements_pre.txt
 !requirements.txt
 !pyproject.toml
 !README.md

Containerfile

@@ -1,40 +1,74 @@
-FROM alpine:3.15
-# Python image includes two Python versions, so use base Alpine
-
-# Based on the work of Jonathan Boeckel <jonathanboeckel1996@gmail.com>
+FROM lsiobase/alpine:3.17 as base
 
 WORKDIR /usr/src/app
 
-# Install run dependencies first
-RUN apk add --no-cache python3 py3-lxml tzdata
 
-# system pip could be removed after build, but apk then decides to also remove all its
-# python dependencies, even if they are explicitly installed as python packages
-# whut
+
+COPY --chown=abc:abc ./requirements.txt ./requirements.txt
+
+# based on https://github.com/linuxserver/docker-pyload-ng/blob/main/Dockerfile
+# everything but the app installation is run in one command so we can purge
+# all build dependencies and cache in the same layer
+# it may be possible to decrease image size slightly by using build stage and
+# copying all site-packages to runtime stage but the image is already pretty small
 RUN \
-	apk add py3-pip && \
-	pip install wheel
+  echo "**** install build packages ****" && \
+	apk add --no-cache --virtual=build-deps \
+		gcc \
+		g++ \
+		python3-dev \
+		libxml2-dev \
+		libxslt-dev \
+		libffi-dev \
+		libc-dev \
+		py3-pip \
+		linux-headers && \
+	echo "**** install runtime packages ****" && \
+	apk add --no-cache \
+		python3 \
+		py3-lxml \
+		tzdata && \
+	echo "**** install pip dependencies ****" && \
+	python3 -m ensurepip && \
+  pip3 install -U --no-cache-dir \
+	  pip \
+	  wheel && \
+  echo "**** install maloja requirements ****" && \
+  pip3 install --no-cache-dir -r requirements.txt && \
+	echo "**** cleanup ****" && \
+	apk del --purge \
+		build-deps && \
+	rm -rf \
+		/tmp/* \
+		${HOME}/.cache
 
+# actual installation in extra layer so we can cache the stuff above
 
-COPY ./requirements.txt ./requirements.txt
+COPY --chown=abc:abc . .
 
 RUN \
-	apk add --no-cache --virtual .build-deps gcc g++ python3-dev libxml2-dev libxslt-dev libffi-dev libc-dev py3-pip linux-headers && \
-	pip install --no-cache-dir -r requirements.txt && \
-	apk del .build-deps
+	echo "**** install maloja ****" && \
+	apk add --no-cache --virtual=install-deps \
+	  py3-pip && \
+	pip3 install /usr/src/app && \
+	apk del --purge \
+		install-deps && \
+	rm -rf \
+		/tmp/* \
+		${HOME}/.cache
 
 
-# no chance for caching below here
 
-COPY . .
+COPY container/root/ /
 
-RUN pip install /usr/src/app
-
-# Docker-specific configuration
-# defaulting to IPv4 is no longer necessary (default host is dual stack)
-ENV MALOJA_SKIP_SETUP=yes
-ENV PYTHONUNBUFFERED=1
+ENV	\
+	# Docker-specific configuration
+	MALOJA_SKIP_SETUP=yes \
+	PYTHONUNBUFFERED=1 \
+	# Prevents breaking change for previous container that ran maloja as root
+	# On linux hosts (non-podman rootless) these variables should be set to the
+	# host user that should own the host folder bound to MALOJA_DATA_DIRECTORY
+	PUID=0 \
+	PGID=0
 
 EXPOSE 42010
-# use exec form for better signal handling https://docs.docker.com/engine/reference/builder/#entrypoint
-ENTRYPOINT ["maloja", "run"]

README.md

@@ -9,7 +9,7 @@
 
 Simple self-hosted music scrobble database to create personal listening statistics. No recommendations, no social network, no nonsense.
 
-![screenshot](screenshot.png?raw=true)
+![screenshot](https://raw.githubusercontent.com/krateng/maloja/master/screenshot.png)
 
 You can check [my own Maloja page](https://maloja.krateng.ch) as an example instance.
 
@@ -20,17 +20,13 @@ You can check [my own Maloja page](https://maloja.krateng.ch) as an example inst
 	* [Requirements](#requirements)
 	* [PyPI](#pypi)
 	* [From Source](#from-source)
-	* [Docker / Podman](#docker-podman)
+	* [Docker / Podman](#docker--podman)
 	* [Extras](#extras)
 * [How to use](#how-to-use)
 	* [Basic control](#basic-control)
 	* [Data](#data)
 	* [Customization](#customization)
 * [How to scrobble](#how-to-scrobble)
-	* [Native support](#native-support)
-	* [Native API](#native-api)
-	* [Standard-compliant API](#standard-compliant-api)
-	* [Manual](#manual)
 * [How to extend](#how-to-extend)
 
 ## Features
@@ -100,6 +96,23 @@ An example of a minimum run configuration to access maloja via `localhost:42010`
 	docker run -p 42010:42010 -v $PWD/malojadata:/mljdata -e MALOJA_DATA_DIRECTORY=/mljdata krateng/maloja
 ```
 
+#### Linux Host
+
+**NOTE:** If you are using [rootless containers with Podman](https://developers.redhat.com/blog/2020/09/25/rootless-containers-with-podman-the-basics#why_podman_) this DOES NOT apply to you.
+
+If you are running Docker on a **Linux Host** you should specify `user:group` ids of the user who owns the folder on the host machine bound to `MALOJA_DATA_DIRECTORY` in order to avoid [docker file permission problems.](https://ikriv.com/blog/?p=4698) These can be specified using the [environmental variables **PUID** and **PGID**.](https://docs.linuxserver.io/general/understanding-puid-and-pgid)
+
+To get the UID and GID for the current user run these commands from a terminal:
+
+* `id -u` -- prints UID (EX `1000`)
+* `id -g` -- prints GID (EX `1001`)
+
+The modified run command with these variables would look like:
+
+```console
+	docker run -e PUID=1000 -e PGID=1001 -p 42010:42010 -v $PWD/malojadata:/mljdata -e MALOJA_DATA_DIRECTORY=/mljdata krateng/maloja
+```
+
 ### Extras
 
 * If you'd like to display images, you will need API keys for [Last.fm](https://www.last.fm/api/account/create) and [Spotify](https://developer.spotify.com/dashboard/applications). These are free of charge!
@@ -139,6 +152,7 @@ If you would like to import your previous scrobbles, use the command `maloja imp
 
 * a Last.fm export generated by [benfoxall's website](https://benjaminbenben.com/lastfm-to-csv/) ([GitHub page](https://github.com/benfoxall/lastfm-to-csv))
 * an official [Spotify data export file](https://www.spotify.com/us/account/privacy/)
+* an official [ListenBrainz export file](https://listenbrainz.org/profile/export/)
 * the export of another Maloja instance
 
 ⚠️ Never import your data while maloja is running. When you need to do import inside docker container start it in shell mode instead and perform import before starting the container as mentioned above.

auxiliary/chromium_scrobbler/maloja-scrobbler/background.js

@@ -11,7 +11,8 @@ const ALWAYS_SCROBBLE_SECONDS = 60*3;
 // Longer songs are always scrobbled when playing at least 2 minutes
 
 pages = {
-	"Plex Web":{
+	"plex":{
+		"name":"Plex",
 		"patterns":[
 			"https://app.plex.tv",
 			"http://app.plex.tv",
@@ -20,31 +21,36 @@ pages = {
 		],
 		"script":"plex.js"
 	},
-	"YouTube Music":{
+	"ytmusic":{
+		"name":"YouTube Music",
 		"patterns":[
 			"https://music.youtube.com"
 		],
 		"script":"ytmusic.js"
 	},
-	"Spotify Web":{
+	"spotify":{
+		"name":"Spotify",
 		"patterns":[
 			"https://open.spotify.com"
 		],
 		"script":"spotify.js"
 	},
-	"Bandcamp":{
+	"bandcamp":{
+		"name":"Bandcamp",
 		"patterns":[
 			"bandcamp.com"
 		],
 		"script":"bandcamp.js"
 	},
-	"Soundcloud":{
+	"soundcloud":{
+		"name":"Soundcloud",
 		"patterns":[
 			"https://soundcloud.com"
 		],
 		"script":"soundcloud.js"
 	},
-	"Navidrome":{
+	"navidrome":{
+		"name":"Navidrome",
 		"patterns":[
 			"https://navidrome.",
 			"http://navidrome."
@@ -77,6 +83,13 @@ function onTabUpdated(tabId, changeInfo, tab) {
 				//console.log("Still on same page!")
 				tabManagers[tabId].update();
 
+				// check if the setting for this page is still active
+				chrome.storage.local.get(["service_active_" + page],function(result){
+					if (!result["service_active_" + page]) {
+						delete tabManagers[tabId];
+					}
+				});
+
 				return
 			}
 		}
@@ -90,13 +103,21 @@ function onTabUpdated(tabId, changeInfo, tab) {
 			patterns = pages[key]["patterns"];
 			for (var i=0;i<patterns.length;i++) {
 				if (tab.url.includes(patterns[i])) {
-					console.log("New page on tab " + tabId + " will be handled by new " + key + " manager!");
-					tabManagers[tabId] = new Controller(tabId,key);
-					updateTabNum();
-					return
-					//chrome.tabs.executeScript(tab.id,{"file":"sitescripts/" + pages[key]["script"]})
 
+					// check if we even like that page
+					chrome.storage.local.get(["service_active_" + key],function(result){
+						if (result["service_active_" + key]) {
+							console.log("New page on tab " + tabId + " will be handled by new " + key + " manager!");
+							tabManagers[tabId] = new Controller(tabId,key);
+							updateTabNum();
+							//chrome.tabs.executeScript(tab.id,{"file":"sitescripts/" + pages[key]["script"]})
+						}
+						else {
+							console.log("New page on tab " + tabId + " is " + key + ", not enabled!");
+						}
+					});
 
+					return;
 				}
 			}
 		}
@@ -127,10 +148,10 @@ function onInternalMessage(request,sender) {
 		for (tabId in tabManagers) {
 			manager = tabManagers[tabId]
 			if (manager.currentlyPlaying) {
-				answer.push([manager.page,manager.currentArtist,manager.currentTitle]);
+				answer.push([pages[manager.page]['name'],manager.currentArtist,manager.currentTitle]);
 			}
 			else {
-				answer.push([manager.page,null]);
+				answer.push([pages[manager.page]['name'],null]);
 			}
 
 		}

auxiliary/chromium_scrobbler/maloja-scrobbler/manifest.json

@@ -1,6 +1,6 @@
 {
 	"name": "Maloja Scrobbler",
-	"version": "1.11",
+	"version": "1.13",
 	"description": "Scrobbles tracks from various sites to your Maloja server",
 	"manifest_version": 2,
 	"permissions": [

auxiliary/chromium_scrobbler/maloja-scrobbler/settings.html

@@ -14,7 +14,7 @@
 				color:beige;
 				font-family:'Ubuntu';
 			}
-			input {
+			input[type=text] {
 				width:270px;
 				font-family:'Ubuntu';
 				outline:none;
@@ -33,10 +33,14 @@
 			<br /><br />
 			<span id="checkmark_key"></span> <span>API key:</span><br />
 			<input type="text" id="apikey" />
-			<br/><br/>
+			<hr/>
 			<span>Tabs:</span>
 			<list id="playinglist">
 			</list>
+			<hr/>
+			<span>Services:</span>
+			<list id="sitelist">
+			</list>
 		</div>
 	</body>
 </html>

auxiliary/chromium_scrobbler/maloja-scrobbler/settings.js

@@ -1,26 +1,71 @@
+// duplicate this info for now, don't know if there is a better way than sending messages
+var pages = {
+	"plex":"Plex",
+	"ytmusic":"YouTube Music",
+	"spotify":"Spotify",
+	"bandcamp":"Bandcamp",
+	"soundcloud":"Soundcloud",
+	"navidrome":"Navidrome"
+}
+
 var config_defaults = {
 	serverurl:"http://localhost:42010",
 	apikey:"BlackPinkInYourArea"
 }
 
+for (var key in pages) {
+	config_defaults["service_active_" + key] = true;
+}
+
 
 document.addEventListener("DOMContentLoaded",function() {
 
+	var sitelist = document.getElementById("sitelist");
+
+
+	for (var identifier in pages) {
+		sitelist.append(document.createElement('br'));
+		var checkbox = document.createElement('input');
+		checkbox.type = "checkbox";
+		checkbox.id = "service_active_" + identifier;
+		var label = document.createElement('label');
+		label.for = checkbox.id;
+		label.textContent = pages[identifier];
+		sitelist.appendChild(checkbox);
+		sitelist.appendChild(label);
+
+		checkbox.addEventListener("change",toggleSite);
+
+	}
+
+
+
 	document.getElementById("serverurl").addEventListener("change",checkServer);
 	document.getElementById("apikey").addEventListener("change",checkServer);
 
 	document.getElementById("serverurl").addEventListener("focusout",checkServer);
 	document.getElementById("apikey").addEventListener("focusout",checkServer);
 
-	document.getElementById("serverurl").addEventListener("input",saveConfig);
-	document.getElementById("apikey").addEventListener("input",saveConfig);
+	document.getElementById("serverurl").addEventListener("input",saveServer);
+	document.getElementById("apikey").addEventListener("input",saveServer);
 
 
 	chrome.runtime.onMessage.addListener(onInternalMessage);
 
 	chrome.storage.local.get(config_defaults,function(result){
+		console.log(result);
 		for (var key in result) {
-			document.getElementById(key).value = result[key];
+
+			// booleans
+			if (result[key] == true || result[key] == false) {
+				document.getElementById(key).checked = result[key];
+			}
+
+			// text
+			else{
+				document.getElementById(key).value = result[key];
+			}
+
 		}
 		checkServer();
 	})
@@ -31,6 +76,11 @@ document.addEventListener("DOMContentLoaded",function() {
 
 });
 
+function toggleSite(evt) {
+	var element = evt.target;
+	chrome.storage.local.set({ [element.id]: element.checked });
+}
+
 
 function onInternalMessage(request,sender) {
 	if (request.type == "response") {
@@ -50,8 +100,8 @@ function onInternalMessage(request,sender) {
 
 
 
-function saveConfig() {
-	for (var key in config_defaults) {
+function saveServer() {
+	for (var key of ["serverurl","apikey"]) {
 		var value = document.getElementById(key).value;
 		chrome.storage.local.set({ [key]: value });
 	}

container/root/etc/s6-overlay/s6-rc.d/init-permission-check/run

@@ -0,0 +1,10 @@
+#!/usr/bin/with-contenv bash
+
+if [ "$(s6-setuidgid abc id -u)" = "0" ]; then
+   echo "-------------------------------------"
+   echo "WARN: Running as root! If you meant to do this than this message can be ignored."
+   echo "If you are running this container on a *linux* host and are not using podman rootless you SHOULD"
+   echo "change the ENVs PUID and PGID for this container to ensure correct permissions on your config folder."
+   echo -e "See: https://github.com/krateng/maloja#linux-host\n"
+   echo -e "-------------------------------------\n"
+fi

container/root/etc/s6-overlay/s6-rc.d/init-permission-check/type

@@ -0,0 +1 @@
+oneshot

container/root/etc/s6-overlay/s6-rc.d/init-permission-check/up

@@ -0,0 +1 @@
+/etc/s6-overlay/s6-rc.d/init-permission-check/run

container/root/etc/s6-overlay/s6-rc.d/svc-python/run

@@ -0,0 +1,7 @@
+#!/usr/bin/with-contenv bash
+
+# used https://github.com/linuxserver/docker-wikijs/blob/master/root/etc/s6-overlay/s6-rc.d/svc-wikijs/run as a template
+
+echo -e "\nMaloja is starting!"
+exec \
+	  s6-setuidgid abc python -m maloja run

container/root/etc/s6-overlay/s6-rc.d/svc-python/type

@@ -0,0 +1 @@
+longrun

dev/list_tags.sh

@@ -0,0 +1 @@
+git tag -l '*.0' -n1 --sort=v:refname

dev/releases/3.1.yml

@@ -10,3 +10,34 @@ minor_release_name: "Soyeon"
     - "[Performance] Adjusted cache sizes"
     - "[Logging] Added cache memory use information"
     - "[Technical] Bumped Python Version and various dependencies"
+3.1.1:
+    commit: "20aae955b2263be07c56bafe4794f622117116ef"
+    notes:
+    - "[Bugfix] Fixed inclusion of custom css files"
+    - "[Bugfix] Fixed list values in configuration"
+3.1.2:
+    commit: "a0739306013cd9661f028fb5b2620cfa2d298aa4"
+    notes:
+    - "[Feature] Added remix artist parsing"
+    - "[Feature] Added API debug mode"
+    - "[Bugfix] Fixed leftover whitespaces when parsing titles"
+    - "[Bugfix] Fixed handling of fallthrough values in config file"
+3.1.3:
+    commit: "f3a04c79b1c37597cdf3cafcd95e3c923cd6a53f"
+    notes:
+    - "[Bugfix] Fixed infinite recursion with capitalized featuring delimiters"
+    - "[Bugfix] Fixed favicon display"
+3.1.4:
+    commit: "ef06f2262205c903e7c3060e2d2d52397f8ffc9d"
+    notes:
+    - "[Feature] Expanded information saved from Listenbrainz API"
+    - "[Feature] Added import for Listenbrainz exports"
+    - "[Bugfix] Sanitized artists and tracks with html-like structure"
+3.1.5:
+    commit: "4330b0294bc0a01cdb841e2e3db370108da901db"
+    notes:
+    - "[Feature] Made image upload part of regular API"
+    - "[Bugfix] Additional entity name sanitization"
+    - "[Bugfix] Fixed image display on Safari"
+    - "[Bugfix] Fixed entity editing on Firefox"
+    - "[Bugfix] Made compatibile with SQLAlchemy 2.0"

dev/releases/3.2.yml

@@ -0,0 +1,9 @@
+minor_release_name: "Momo"
+3.2.0:
+    notes:
+    - "[Architecture] Switched to linuxserver.io container base image"
+    - "[Feature] Added basic support for albums"
+    - "[Performance] Improved image rendering"
+    - "[Bugfix] Fixed configuration of time format"
+    - "[Bugfix] Fixed search on manual scrobble page"
+    - "[Bugfix] Disabled DB maintenance while not running main server"

dev/testing/Maloja.postman_collection.json

@@ -189,7 +189,7 @@
 								],
 								"body": {
 									"mode": "raw",
-									"raw": "{\n    \"key\": \"{{api_key}}\",\n    \"artist\": \"{{data.artist1}}\",\n    \"title\": \"{{data.title1}}\"\n}"
+									"raw": "{\n    \"key\": \"{{api_key}}\",\n    \"artist\": \"{{data.artist1}}\",\n    \"title\": \"{{data.title1}}\",\n    \"album\": \"{{data.album}}\",\n    \"albumartists\":[\n        \"{{data.artist1}}\",\n        \"{{data.artist3}}\"\n    ]\n}"
 								},
 								"url": {
 									"raw": "{{url}}/apis/mlj_1/newscrobble",
@@ -219,7 +219,7 @@
 								],
 								"body": {
 									"mode": "raw",
-									"raw": "{\n    \"key\": \"{{api_key}}\",\n    \"artists\": [\"{{data.artist1}}\",\"{{data.artist2}}\"],\n    \"title\": \"{{data.title1}}\"\n}"
+									"raw": "{\n    \"key\": \"{{api_key}}\",\n    \"artists\": [\"{{data.artist1}}\",\"{{data.artist2}}\"],\n    \"title\": \"{{data.title1}}\",\n    \"album\": \"{{data.album}}\",\n    \"albumartists\":[\n        \"{{data.artist1}}\",\n        \"{{data.artist3}}\"\n    ]\n}"
 								},
 								"url": {
 									"raw": "{{url}}/apis/mlj_1/newscrobble",
@@ -867,6 +867,11 @@
 			"key": "data.title3",
 			"value": "One in a Million"
 		},
+		{
+			"key": "data.album",
+			"value": "The Epic Collection",
+			"type": "default"
+		},
 		{
 			"key": "data.timestamp1",
 			"value": ""

maloja/__main__.py

@@ -14,7 +14,7 @@ from . import __pkginfo__ as pkginfo
 from .pkg_global import conf
 from .proccontrol import tasks
 from .setup import setup
-from .dev import generate
+from .dev import generate, apidebug
 
 
 
@@ -137,12 +137,18 @@ def print_info():
 	print(col['lightblue']("Configuration Directory:"),conf.dir_settings['config'])
 	print(col['lightblue']("Data Directory:         "),conf.dir_settings['state'])
 	print(col['lightblue']("Log Directory:          "),conf.dir_settings['logs'])
-	print(col['lightblue']("Network:                "),f"IPv{ip_address(conf.malojaconfig['host']).version}, Port {conf.malojaconfig['port']}")
+	print(col['lightblue']("Network:                "),f"Dual Stack, Port {conf.malojaconfig['port']}" if conf.malojaconfig['host'] == "*" else f"IPv{ip_address(conf.malojaconfig['host']).version}, Port {conf.malojaconfig['port']}")
 	print(col['lightblue']("Timezone:               "),f"UTC{conf.malojaconfig['timezone']:+d}")
 	print()
+	try:
+		import pkg_resources
+		for pkg in ("sqlalchemy","waitress","bottle","doreah","jinja2"):
+			print(col['cyan']     (f"{pkg}:".ljust(13)),pkg_resources.get_distribution(pkg).version)
+	except ImportError:
+		print("Could not determine dependency versions.")
 	print()
 
-@mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images'],shield=True)
+@mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images','prefer_existing'],shield=True)
 def main(*args,**kwargs):
 
 	actions = {
@@ -159,6 +165,8 @@ def main(*args,**kwargs):
 		"backup":tasks.backup,					# maloja backup --targetfolder /x/y --include_images
 		"generate":generate.generate_scrobbles,	# maloja generate 400
 		"export":tasks.export,					# maloja export
+		"apidebug":apidebug.run,				# maloja apidebug
+		"parsealbums":tasks.parse_albums,		# maloja parsealbums --strategy majority
 		# aux
 		"info":print_info
 	}

maloja/__pkginfo__.py

@@ -4,7 +4,7 @@
 # you know what f*ck it
 # this is hardcoded for now because of that damn project / package name discrepancy
 # i'll fix it one day
-VERSION = "3.1.0"
+VERSION = "3.1.5"
 HOMEPAGE = "https://github.com/krateng/maloja"
 
 

maloja/apis/__init__.py

@@ -47,9 +47,12 @@ def init_apis(server):
 			server.get(altpath_empty_cl)(alias_api)
 			server.post(altpath_empty_cl)(alias_api)
 
-	def invalid_api(pth):
+	def invalid_api(pth=''):
 		response.status = 404
 		return {"error":"Invalid API"}
 
 	server.get("/apis/<pth:path>")(invalid_api)
 	server.post("/apis/<pth:path>")(invalid_api)
+
+	server.get("/apis")(invalid_api)
+	server.post("/apis")(invalid_api)

maloja/apis/audioscrobbler_legacy.py

@@ -73,6 +73,8 @@ class AudioscrobblerLegacy(APIHandler):
 		client = self.mobile_sessions.get(key)
 		for count in range(50):
 			artist_key = f"a[{count}]"
+			album_key = f"b[{count}]"
+			length_key = f"l[{count}]"
 			track_key = f"t[{count}]"
 			time_key = f"i[{count}]"
 			if artist_key not in keys or track_key not in keys:
@@ -82,12 +84,19 @@ class AudioscrobblerLegacy(APIHandler):
 				timestamp = int(keys[time_key])
 			except Exception:
 				timestamp = None
-			#database.createScrobble(artists,title,timestamp)
-			self.scrobble({
+
+			scrobble = {
 				'track_artists':[artiststr],
 				'track_title':titlestr,
-				'scrobble_time':timestamp
-			},client=client)
+				'scrobble_time':timestamp,
+			}
+			if album_key in keys:
+				scrobble['album_name'] = keys[album_key]
+			if length_key in keys:
+				scrobble['track_length'] = keys[length_key]
+
+			#database.createScrobble(artists,title,timestamp)
+			self.scrobble(scrobble, client=client)
 		return 200,"OK\n"
 
 

maloja/apis/listenbrainz.py

@@ -55,6 +55,8 @@ class Listenbrainz(APIHandler):
 				try:
 					metadata = listen["track_metadata"]
 					artiststr, titlestr = metadata["artist_name"], metadata["track_name"]
+					albumstr = metadata.get("release_name")
+					additional = metadata.get("additional_info",{})
 					try:
 						timestamp = int(listen["listened_at"])
 					except Exception:
@@ -62,10 +64,21 @@ class Listenbrainz(APIHandler):
 				except Exception:
 					raise MalformedJSONException()
 
+				extrafields = {
+					# fields that will not be consumed by regular scrobbling
+					# will go into 'extra'
+					k:additional[k]
+					for k in ['track_mbid', 'release_mbid', 'artist_mbids','recording_mbid','tags']
+					if k in additional
+				}
+
 				self.scrobble({
 					'track_artists':[artiststr],
 					'track_title':titlestr,
-					'scrobble_time':timestamp
+					'album_title':albumstr,
+					'scrobble_time':timestamp,
+					'track_length': additional.get("duration"),
+					**extrafields
 				},client=client)
 
 			return 200,{"status":"ok"}

maloja/apis/native_v1.py

@@ -72,6 +72,14 @@ errors = {
 			'desc':"The database is being upgraded. Please try again later."
 		}
 	}),
+	images.MalformedB64: lambda e: (400,{
+		"status":"failure",
+		"error":{
+			'type':'malformed_b64',
+			'value':None,
+			'desc':"The provided base 64 string is not valid."
+		}
+	}),
 	# for http errors, use their status code
 	Exception: lambda e: ((e.status_code if hasattr(e,'statuscode') else 500),{
 		"status":"failure",
@@ -462,7 +470,7 @@ def post_scrobble(
 	rawscrobble = {
 		'track_artists':(artist or []) + artists,
 		'track_title':title,
-		'album_name':album,
+		'album_title':album,
 		'album_artists':albumartists,
 		'scrobble_duration':duration,
 		'track_length':length,
@@ -486,24 +494,56 @@ def post_scrobble(
 			'artists':result['track']['artists'],
 			'title':result['track']['title']
 		},
-		'desc':f"Scrobbled {result['track']['title']} by {', '.join(result['track']['artists'])}"
+		'desc':f"Scrobbled {result['track']['title']} by {', '.join(result['track']['artists'])}",
+		'warnings':[]
 	}
 	if extra_kwargs:
-		responsedict['warnings'] = [
+		responsedict['warnings'] += [
 			{'type':'invalid_keyword_ignored','value':k,
 			'desc':"This key was not recognized by the server and has been discarded."}
 			for k in extra_kwargs
 		]
 	if artist and artists:
-		responsedict['warnings'] = [
+		responsedict['warnings'] += [
 			{'type':'mixed_schema','value':['artist','artists'],
 			'desc':"These two fields are meant as alternative methods to submit information. Use of both is discouraged, but works at the moment."}
 		]
+
+	if len(responsedict['warnings']) == 0: del responsedict['warnings']
+
 	return responsedict
 
 
 
 
+@api.post("addpicture")
+@authenticated_function(alternate=api_key_correct,api=True)
+@catch_exceptions
+def add_picture(b64,artist:Multi=[],title=None,albumtitle=None):
+	"""Uploads a new image for an artist or track.
+
+	param string b64: Base 64 representation of the image
+	param string artist: Artist name. Can be supplied multiple times for tracks with multiple artists.
+	param string title: Title of the track. Optional.
+
+	"""
+	keys = FormsDict()
+	for a in artist:
+		keys.append("artist",a)
+	if title is not None: keys.append("title",title)
+	elif albumtitle is not None: keys.append("albumtitle",albumtitle)
+	k_filter, _, _, _, _ = uri_to_internal(keys)
+	if "track" in k_filter: k_filter = k_filter["track"]
+	elif "album" in k_filter: k_filter = k_filter["album"]
+	url = images.set_image(b64,**k_filter)
+
+	return {
+		'status': 'success',
+		'url': url
+	}
+
+
+
 @api.post("importrules")
 @authenticated_function(api=True)
 @catch_exceptions
@@ -552,6 +592,7 @@ def search(**keys):
 
 	artists = database.db_search(query,type="ARTIST")
 	tracks = database.db_search(query,type="TRACK")
+	albums = database.db_search(query,type="ALBUM")
 
 
 
@@ -559,6 +600,7 @@ def search(**keys):
 	# also, shorter is better (because longer titles would be easier to further specify)
 	artists.sort(key=lambda x: ((0 if x.lower().startswith(query) else 1 if " " + query in x.lower() else 2),len(x)))
 	tracks.sort(key=lambda x: ((0 if x["title"].lower().startswith(query) else 1 if " " + query in x["title"].lower() else 2),len(x["title"])))
+	albums.sort(key=lambda x: ((0 if x["albumtitle"].lower().startswith(query) else 1 if " " + query in x["albumtitle"].lower() else 2),len(x["albumtitle"])))
 
 	# add links
 	artists_result = []
@@ -579,21 +621,17 @@ def search(**keys):
 		}
 		tracks_result.append(result)
 
-	return {"artists":artists_result[:max_],"tracks":tracks_result[:max_]}
+	albums_result = []
+	for al in albums:
+		result = {
+			'album': al,
+			'link': "/album?" + compose_querystring(internal_to_uri({"album":al})),
+			'image': images.get_album_image(al)
+		}
+		if not result['album']['artists']: result['album']['displayArtist'] = malojaconfig["DEFAULT_ALBUM_ARTIST"]
+		albums_result.append(result)
 
-
-@api.post("addpicture")
-@authenticated_function(api=True)
-@catch_exceptions
-def add_picture(b64,artist:Multi=[],title=None):
-	"""Internal Use Only"""
-	keys = FormsDict()
-	for a in artist:
-		keys.append("artist",a)
-	if title is not None: keys.append("title",title)
-	k_filter, _, _, _, _ = uri_to_internal(keys)
-	if "track" in k_filter: k_filter = k_filter["track"]
-	images.set_image(b64,**k_filter)
+	return {"artists":artists_result[:max_],"tracks":tracks_result[:max_],"albums":albums_result[:max_]}
 
 
 @api.post("newrule")
@@ -688,6 +726,16 @@ def edit_track(id,title):
 		"status":"success"
 	}
 
+@api.post("edit_album")
+@authenticated_function(api=True)
+@catch_exceptions
+def edit_album(id,albumtitle):
+	"""Internal Use Only"""
+	result = database.edit_album(id,{'albumtitle':albumtitle})
+	return {
+		"status":"success"
+	}
+
 
 @api.post("merge_tracks")
 @authenticated_function(api=True)
@@ -709,6 +757,16 @@ def merge_artists(target_id,source_ids):
 		"status":"success"
 	}
 
+@api.post("merge_albums")
+@authenticated_function(api=True)
+@catch_exceptions
+def merge_artists(target_id,source_ids):
+	"""Internal Use Only"""
+	result = database.merge_albums(target_id,source_ids)
+	return {
+		"status":"success"
+	}
+
 @api.post("reparse_scrobble")
 @authenticated_function(api=True)
 @catch_exceptions

maloja/cleanup.py

@@ -26,6 +26,7 @@ class CleanerAgent:
 		self.rules_belongtogether = [r[1] for r in rawrules if r[0]=="belongtogether"]
 		self.rules_notanartist = [r[1] for r in rawrules if r[0]=="notanartist"]
 		self.rules_replacetitle = {r[1].lower():r[2] for r in rawrules if r[0]=="replacetitle"}
+		self.rules_replacealbumtitle = {r[1].lower():r[2] for r in rawrules if r[0]=="replacealbumtitle"}
 		self.rules_replaceartist = {r[1].lower():r[2] for r in rawrules if r[0]=="replaceartist"}
 		self.rules_ignoreartist = [r[1].lower() for r in rawrules if r[0]=="ignoreartist"]
 		self.rules_addartists = {r[2].lower():(r[1].lower(),r[3]) for r in rawrules if r[0]=="addartists"}
@@ -55,7 +56,7 @@ class CleanerAgent:
 		artists = list(set(artists))
 		artists.sort()
 
-		return (artists,title)
+		return (artists,title.strip())
 
 	def removespecial(self,s):
 		if isinstance(s,list):
@@ -82,7 +83,7 @@ class CleanerAgent:
 
 	def parseArtists(self,a):
 
-		if isinstance(a,list):
+		if isinstance(a,list) or isinstance(a,tuple):
 			res = [self.parseArtists(art) for art in a]
 			return [a for group in res for a in group]
 
@@ -109,9 +110,9 @@ class CleanerAgent:
 
 
 		for d in self.delimiters_feat:
-			if re.match(r"(.*) [\(\[]" + d + " (.*)[\)\]]",a) is not None:
-				return self.parseArtists(re.sub(r"(.*) [\(\[]" + d + " (.*)[\)\]]",r"\1",a)) + \
-						self.parseArtists(re.sub(r"(.*) [\(\[]" + d + " (.*)[\)\]]",r"\2",a))
+			if re.match(r"(.*) [\(\[]" + d + " (.*)[\)\]]",a,flags=re.IGNORECASE) is not None:
+				return self.parseArtists(re.sub(r"(.*) [\(\[]" + d + " (.*)[\)\]]",r"\1",a,flags=re.IGNORECASE)) + \
+						self.parseArtists(re.sub(r"(.*) [\(\[]" + d + " (.*)[\)\]]",r"\2",a,flags=re.IGNORECASE))
 
 
 
@@ -156,29 +157,46 @@ class CleanerAgent:
 		#	t = p(t).strip()
 		return t
 
-	def parseTitleForArtists(self,t):
-		for d in self.delimiters_feat:
-			if re.match(r"(.*) [\(\[]" + d + " (.*?)[\)\]]",t) is not None:
-				(title,artists) = self.parseTitleForArtists(re.sub(r"(.*) [\(\[]" + d + " (.*?)[\)\]]",r"\1",t))
-				artists += self.parseArtists(re.sub(r"(.*) [\(\[]" + d + " (.*?)[\)\]].*",r"\2",t))
-				return (title,artists)
-			if re.match(r"(.*) - " + d + " (.*)",t) is not None:
-				(title,artists) = self.parseTitleForArtists(re.sub(r"(.*) - " + d + " (.*)",r"\1",t))
-				artists += self.parseArtists(re.sub(r"(.*) - " + d + " (.*).*",r"\2",t))
-				return (title,artists)
-			if re.match(r"(.*) " + d + " (.*)",t) is not None:
-				(title,artists) = self.parseTitleForArtists(re.sub(r"(.*) " + d + " (.*)",r"\1",t))
-				artists += self.parseArtists(re.sub(r"(.*) " + d + " (.*).*",r"\2",t))
-				return (title,artists)
-
+	def parseTitleForArtists(self,title):
 		artists = []
+		for delimiter in malojaconfig["DELIMITERS_FEAT"]:
+			for pattern in [
+				r" [\(\[]" + re.escape(delimiter) + " (.*?)[\)\]]",
+				r" - " + re.escape(delimiter) + " (.*)",
+				r" " + re.escape(delimiter) + " (.*)"
+			]:
+				matches = re.finditer(pattern,title,flags=re.IGNORECASE)
+				for match in matches:
+					title = match.re.sub('',match.string) # Remove matched part
+					artists += self.parseArtists(match.group(1)) # Parse matched artist string
+
+
+
+		if malojaconfig["PARSE_REMIX_ARTISTS"]:
+			for filter in malojaconfig["FILTERS_REMIX"]:
+				for pattern in [
+					r" [\(\[](.*)" + re.escape(filter) + "[\)\]]", # match remix in brackets
+					r" - (.*)" + re.escape(filter) # match remix split with "-"
+				]:
+					match = re.search(pattern,title,flags=re.IGNORECASE)
+					if match:
+						# title stays the same
+						artists += self.parseArtists(match.group(1))
+
+
+
 		for st in self.rules_artistintitle:
-			if st in t.lower(): artists += self.rules_artistintitle[st].split("␟")
-		return (t,artists)
-
+			if st in title.lower(): artists += self.rules_artistintitle[st].split("␟")
+		return (title,artists)
 
+	def parseAlbumtitle(self,t):
+		if t.strip().lower() in self.rules_replacealbumtitle:
+			return self.rules_replacealbumtitle[t.strip().lower()]
 
+		t = t.replace("[","(").replace("]",")")
 
+		t = t.strip()
+		return t
 
 
 def flatten(lis):

maloja/data_files/config/rules/predefined/krateng_artistsingroups.tsv

@@ -8,6 +8,7 @@ countas		Trouble Maker				HyunA
 countas		S Club 7				Tina Barrett
 countas		4Minute						HyunA
 countas		I.O.I						Chungha
+countas		TrySail					Sora Amamiya
 # Group more famous than single artist
 countas		RenoakRhythm				Approaching Nirvana
 countas		Shirley Manson				Garbage
@@ -18,3 +19,7 @@ countas		Airi Suzuki					℃-ute
 countas		CeeLo Green					Gnarls Barkley
 countas		Amelia Watson				Hololive EN
 countas		Gawr Gura					Hololive EN
+countas		Mori Calliope				Hololive EN
+countas		Ninomae Ina'nis				Hololive EN
+countas		Takanashi Kiara				Hololive EN
+countas		Ceres Fauna						Hololive EN

maloja/data_files/config/rules/predefined/krateng_jpop.tsv

@@ -0,0 +1,20 @@
+# NAME: JPop
+# DESC: Fixes and romanizes various Japanese tracks and artists
+
+
+belongtogether	Myth & Roid
+
+
+# Sora-chan
+replaceartist	Amamiya Sora						Sora Amamiya
+replacetitle	エデンの旅人						Eden no Tabibito
+replacetitle	月灯り							Tsukiakari
+replacetitle	火花							Hibana
+replacetitle	ロンリーナイト・ディスコティック				Lonely Night Discotheque
+replacetitle	羽根輪舞							Hane Rinbu
+replacetitle	メリーゴーランド						Merry-go-round
+replacetitle	フリイジア						Fressia
+replacetitle	誓い							Chikai
+
+# ReoNa
+replacetitle	ないない							nainai

maloja/data_files/config/rules/predefined/krateng_kpopgirlgroups.tsv

@@ -21,7 +21,7 @@ addartists		HyunA					Change		Jun Hyung
 # BLACKPINK
 countas		Jennie					BLACKPINK
 countas		Rosé					BLACKPINK
-countas		Lisa					BLACKPINK
+countas		Lalisa					BLACKPINK
 countas		Jisoo					BLACKPINK
 replacetitle	AS IF IT'S YOUR LAST			As If It's Your Last
 replacetitle	BOOMBAYAH				Boombayah
@@ -200,10 +200,13 @@ countas				ACE					IZ*ONE
 countas				Chaewon			IZ*ONE
 countas				Minju			IZ*ONE
 
-
+# ITZY
 countas				Yeji			ITZY
+
+# IVE
 countas				Wonyoung		IVE
 countas				Yujin			IVE
+countas				Gaeul			IVE
 
 # Popular Remixes
 artistintitle	Areia Remix				Areia

maloja/database/__init__.py

@@ -1,6 +1,17 @@
 # server
 from bottle import request, response, FormsDict
 
+
+# decorator that makes sure this function is only run in normal operation,
+# not when we run a task that needs to access the database
+def no_aux_mode(func):
+	def wrapper(*args,**kwargs):
+		from ..pkg_global import conf
+		if conf.AUX_MODE: return
+		return func(*args,**kwargs)
+	return wrapper
+
+
 # rest of the project
 from ..cleanup import CleanerAgent
 from .. import images
@@ -46,6 +57,9 @@ dbstatus = {
 
 
 
+
+
+
 def waitfordb(func):
 	def newfunc(*args,**kwargs):
 		if not dbstatus['healthy']: raise exceptions.DatabaseNotBuilt()
@@ -93,12 +107,15 @@ def incoming_scrobble(rawscrobble,fix=True,client=None,api=None,dbconn=None):
 	log(f"Incoming scrobble [Client: {client} | API: {api}]: {rawscrobble}")
 
 	scrobbledict = rawscrobble_to_scrobbledict(rawscrobble, fix, client)
+	albumupdate = (malojaconfig["ALBUM_INFORMATION_TRUST"] == 'last')
 
-	sqldb.add_scrobble(scrobbledict,dbconn=dbconn)
+
+	sqldb.add_scrobble(scrobbledict,update_album=albumupdate,dbconn=dbconn)
 	proxy_scrobble_all(scrobbledict['track']['artists'],scrobbledict['track']['title'],scrobbledict['time'])
 
 	dbcache.invalidate_caches(scrobbledict['time'])
 
+
 	#return {"status":"success","scrobble":scrobbledict}
 	return scrobbledict
 
@@ -130,8 +147,22 @@ def rawscrobble_to_scrobbledict(rawscrobble, fix=True, client=None):
 	scrobbleinfo = {**rawscrobble}
 	if fix:
 		scrobbleinfo['track_artists'],scrobbleinfo['track_title'] = cla.fullclean(scrobbleinfo['track_artists'],scrobbleinfo['track_title'])
+		if scrobbleinfo.get('album_artists'):
+			scrobbleinfo['album_artists'] = cla.parseArtists(scrobbleinfo['album_artists'])
+		if scrobbleinfo.get("album_title"):
+			scrobbleinfo['album_title'] = cla.parseAlbumtitle(scrobbleinfo['album_title'])
 	scrobbleinfo['scrobble_time'] = scrobbleinfo.get('scrobble_time') or int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp())
 
+	# if we send [] as albumartists, it means various
+	# if we send nothing, the scrobbler just doesnt support it and we assume track artists
+	if ('album_title' in scrobbleinfo) and ('album_artists' not in scrobbleinfo):
+		scrobbleinfo['album_artists'] = scrobbleinfo.get('track_artists')
+
+	# New plan, do this further down
+	# NONE always means there is simply no info, so make a guess or whatever the options say
+	# various artists always needs to be specified via []
+	# TODO
+
 	# processed info to internal scrobble dict
 	scrobbledict = {
 		"time":scrobbleinfo.get('scrobble_time'),
@@ -139,7 +170,7 @@ def rawscrobble_to_scrobbledict(rawscrobble, fix=True, client=None):
 			"artists":scrobbleinfo.get('track_artists'),
 			"title":scrobbleinfo.get('track_title'),
 			"album":{
-				"name":scrobbleinfo.get('album_name'),
+				"albumtitle":scrobbleinfo.get('album_title'),
 				"artists":scrobbleinfo.get('album_artists')
 			},
 			"length":scrobbleinfo.get('track_length')
@@ -148,11 +179,15 @@ def rawscrobble_to_scrobbledict(rawscrobble, fix=True, client=None):
 		"origin":f"client:{client}" if client else "generic",
 		"extra":{
 			k:scrobbleinfo[k] for k in scrobbleinfo if k not in
-			['scrobble_time','track_artists','track_title','track_length','scrobble_duration','album_name','album_artists']
+			['scrobble_time','track_artists','track_title','track_length','scrobble_duration']#,'album_title','album_artists']
+			# we still save album info in extra because the user might select majority album authority
 		},
 		"rawscrobble":rawscrobble
 	}
 
+	if scrobbledict["track"]["album"]["albumtitle"] is None:
+		del scrobbledict["track"]["album"]
+
 	return scrobbledict
 
 
@@ -184,6 +219,16 @@ def edit_track(id,trackinfo):
 
 	return result
 
+@waitfordb
+def edit_album(id,albuminfo):
+	album = sqldb.get_album(id)
+	log(f"Renaming {album['albumtitle']} to {albuminfo['albumtitle']}")
+	result = sqldb.edit_album(id,albuminfo)
+	dbcache.invalidate_entity_cache()
+	dbcache.invalidate_caches()
+
+	return result
+
 @waitfordb
 def merge_artists(target_id,source_ids):
 	sources = [sqldb.get_artist(id) for id in source_ids]
@@ -206,6 +251,17 @@ def merge_tracks(target_id,source_ids):
 
 	return result
 
+@waitfordb
+def merge_albums(target_id,source_ids):
+	sources = [sqldb.get_album(id) for id in source_ids]
+	target = sqldb.get_album(target_id)
+	log(f"Merging {sources} into {target}")
+	result = sqldb.merge_albums(target_id,source_ids)
+	dbcache.invalidate_entity_cache()
+	dbcache.invalidate_caches()
+
+	return result
+
 
 
 
@@ -216,6 +272,8 @@ def get_scrobbles(dbconn=None,**keys):
 		result = sqldb.get_scrobbles_of_artist(artist=keys['artist'],since=since,to=to,dbconn=dbconn)
 	elif 'track' in keys:
 		result = sqldb.get_scrobbles_of_track(track=keys['track'],since=since,to=to,dbconn=dbconn)
+	elif 'album' in keys:
+		result = sqldb.get_scrobbles_of_album(album=keys['album'],since=since,to=to,dbconn=dbconn)
 	else:
 		result = sqldb.get_scrobbles(since=since,to=to,dbconn=dbconn)
 	#return result[keys['page']*keys['perpage']:(keys['page']+1)*keys['perpage']]
@@ -229,6 +287,8 @@ def get_scrobbles_num(dbconn=None,**keys):
 		result = len(sqldb.get_scrobbles_of_artist(artist=keys['artist'],since=since,to=to,resolve_references=False,dbconn=dbconn))
 	elif 'track' in keys:
 		result = len(sqldb.get_scrobbles_of_track(track=keys['track'],since=since,to=to,resolve_references=False,dbconn=dbconn))
+	elif 'album' in keys:
+		result = len(sqldb.get_scrobbles_of_album(album=keys['album'],since=since,to=to,resolve_references=False,dbconn=dbconn))
 	else:
 		result = sqldb.get_scrobbles_num(since=since,to=to,dbconn=dbconn)
 	return result
@@ -248,19 +308,45 @@ def get_artists(dbconn=None):
 	return sqldb.get_artists(dbconn=dbconn)
 
 
+def get_albums_artist_appears_on(dbconn=None,**keys):
+
+	artist_id = sqldb.get_artist_id(keys['artist'],dbconn=dbconn)
+
+	albums = sqldb.get_albums_artists_appear_on([artist_id],dbconn=dbconn).get(artist_id) or []
+	ownalbums = sqldb.get_albums_of_artists([artist_id],dbconn=dbconn).get(artist_id) or []
+
+	result = {
+		"own_albums":ownalbums,
+		"appears_on":[a for a in albums if a not in ownalbums]
+	}
+
+	return result
+
+
 @waitfordb
-def get_charts_artists(dbconn=None,**keys):
+def get_charts_artists(dbconn=None,resolve_ids=True,**keys):
 	(since,to) = keys.get('timerange').timestamps()
-	result = sqldb.count_scrobbles_by_artist(since=since,to=to,dbconn=dbconn)
+	result = sqldb.count_scrobbles_by_artist(since=since,to=to,resolve_ids=resolve_ids,dbconn=dbconn)
 	return result
 
 @waitfordb
-def get_charts_tracks(dbconn=None,**keys):
+def get_charts_tracks(dbconn=None,resolve_ids=True,**keys):
 	(since,to) = keys.get('timerange').timestamps()
 	if 'artist' in keys:
-		result = sqldb.count_scrobbles_by_track_of_artist(since=since,to=to,artist=keys['artist'],dbconn=dbconn)
+		result = sqldb.count_scrobbles_by_track_of_artist(since=since,to=to,artist=keys['artist'],resolve_ids=resolve_ids,dbconn=dbconn)
+	elif 'album' in keys:
+		result = sqldb.count_scrobbles_by_track_of_album(since=since,to=to,album=keys['album'],resolve_ids=resolve_ids,dbconn=dbconn)
 	else:
-		result = sqldb.count_scrobbles_by_track(since=since,to=to,dbconn=dbconn)
+		result = sqldb.count_scrobbles_by_track(since=since,to=to,resolve_ids=resolve_ids,dbconn=dbconn)
+	return result
+
+@waitfordb
+def get_charts_albums(dbconn=None,resolve_ids=True,**keys):
+	(since,to) = keys.get('timerange').timestamps()
+	if 'artist' in keys:
+		result = sqldb.count_scrobbles_by_album_of_artist(since=since,to=to,artist=keys['artist'],resolve_ids=resolve_ids,dbconn=dbconn)
+	else:
+		result = sqldb.count_scrobbles_by_album(since=since,to=to,resolve_ids=resolve_ids,dbconn=dbconn)
 	return result
 
 @waitfordb
@@ -282,21 +368,32 @@ def get_performance(dbconn=None,**keys):
 
 	for rng in rngs:
 		if "track" in keys:
-			track = sqldb.get_track(sqldb.get_track_id(keys['track'],dbconn=dbconn),dbconn=dbconn)
-			charts = get_charts_tracks(timerange=rng,dbconn=dbconn)
+			track_id = sqldb.get_track_id(keys['track'],dbconn=dbconn)
+			#track = sqldb.get_track(track_id,dbconn=dbconn)
+			charts = get_charts_tracks(timerange=rng,resolve_ids=False,dbconn=dbconn)
 			rank = None
 			for c in charts:
-				if c["track"] == track:
+				if c["track_id"] == track_id:
 					rank = c["rank"]
 					break
 		elif "artist" in keys:
-			artist = sqldb.get_artist(sqldb.get_artist_id(keys['artist'],dbconn=dbconn),dbconn=dbconn)
+			artist_id = sqldb.get_artist_id(keys['artist'],dbconn=dbconn)
+			#artist = sqldb.get_artist(artist_id,dbconn=dbconn)
 			# ^this is the most useless line in programming history
 			# but I like consistency
-			charts = get_charts_artists(timerange=rng,dbconn=dbconn)
+			charts = get_charts_artists(timerange=rng,resolve_ids=False,dbconn=dbconn)
 			rank = None
 			for c in charts:
-				if c["artist"] == artist:
+				if c["artist_id"] == artist_id:
+					rank = c["rank"]
+					break
+		elif "album" in keys:
+			album_id = sqldb.get_album_id(keys['album'],dbconn=dbconn)
+			#album = sqldb.get_album(album_id,dbconn=dbconn)
+			charts = get_charts_albums(timerange=rng,resolve_ids=False,dbconn=dbconn)
+			rank = None
+			for c in charts:
+				if c["album_id"] == album_id:
 					rank = c["rank"]
 					break
 		else:
@@ -336,24 +433,53 @@ def get_top_tracks(dbconn=None,**keys):
 
 	return results
 
+@waitfordb
+def get_top_albums(dbconn=None,**keys):
+
+	rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]})
+	results = []
+
+	for rng in rngs:
+		try:
+			res = get_charts_albums(timerange=rng,dbconn=dbconn)[0]
+			results.append({"range":rng,"album":res["album"],"scrobbles":res["scrobbles"]})
+		except Exception:
+			results.append({"range":rng,"album":None,"scrobbles":0})
+
+	return results
+
 @waitfordb
 def artist_info(dbconn=None,**keys):
 
 	artist = keys.get('artist')
 	if artist is None: raise exceptions.MissingEntityParameter()
 
-	artist_id = sqldb.get_artist_id(artist,dbconn=dbconn)
+	artist_id = sqldb.get_artist_id(artist,create_new=False,dbconn=dbconn)
+	if not artist_id: raise exceptions.ArtistDoesNotExist(artist)
+
 	artist = sqldb.get_artist(artist_id,dbconn=dbconn)
 	alltimecharts = get_charts_artists(timerange=alltime(),dbconn=dbconn)
-	scrobbles = get_scrobbles_num(artist=artist,timerange=alltime(),dbconn=dbconn)
 	#we cant take the scrobble number from the charts because that includes all countas scrobbles
-	try:
-		c = [e for e in alltimecharts if e["artist"] == artist][0]
+	scrobbles = get_scrobbles_num(artist=artist,timerange=alltime(),dbconn=dbconn)
+	albums = sqldb.get_albums_of_artists(set([artist_id]),dbconn=dbconn)
+	isalbumartist = len(albums.get(artist_id,[]))>0
+
+
+	# base info for everyone
+	result = {
+		"artist":artist,
+		"scrobbles":scrobbles,
+		"id":artist_id,
+		"isalbumartist":isalbumartist
+	}
+
+	# check if credited to someone else
+	parent_artists = sqldb.get_credited_artists(artist)
+	if len(parent_artists) == 0:
+		c = [e for e in alltimecharts if e["artist"] == artist]
+		position = c[0]["rank"] if len(c) > 0 else None
 		others = sqldb.get_associated_artists(artist,dbconn=dbconn)
-		position = c["rank"]
-		return {
-			"artist":artist,
-			"scrobbles":scrobbles,
+		result.update({
 			"position":position,
 			"associated":others,
 			"medals":{
@@ -361,23 +487,19 @@ def artist_info(dbconn=None,**keys):
 				"silver": [year for year in cached.medals_artists if artist_id in cached.medals_artists[year]['silver']],
 				"bronze": [year for year in cached.medals_artists if artist_id in cached.medals_artists[year]['bronze']],
 			},
-			"topweeks":len([e for e in cached.weekly_topartists if e == artist_id]),
-			"id":artist_id
-		}
-	except Exception:
-		# if the artist isnt in the charts, they are not being credited and we
-		# need to show information about the credited one
-		replaceartist = sqldb.get_credited_artists(artist)[0]
+			"topweeks":len([e for e in cached.weekly_topartists if e == artist_id])
+		})
+
+	else:
+		replaceartist = parent_artists[0]
 		c = [e for e in alltimecharts if e["artist"] == replaceartist][0]
 		position = c["rank"]
-		return {
-			"artist":artist,
+		result.update({
 			"replace":replaceartist,
-			"scrobbles":scrobbles,
-			"position":position,
-			"id":artist_id
-		}
+			"position":position
+		})
 
+	return result
 
 
 
@@ -387,12 +509,14 @@ def track_info(dbconn=None,**keys):
 	track = keys.get('track')
 	if track is None: raise exceptions.MissingEntityParameter()
 
-	track_id = sqldb.get_track_id(track,dbconn=dbconn)
+	track_id = sqldb.get_track_id(track,create_new=False,dbconn=dbconn)
+	if not track_id: raise exceptions.TrackDoesNotExist(track['title'])
+
 	track = sqldb.get_track(track_id,dbconn=dbconn)
-	alltimecharts = get_charts_tracks(timerange=alltime(),dbconn=dbconn)
+	alltimecharts = get_charts_tracks(timerange=alltime(),resolve_ids=False,dbconn=dbconn)
 	#scrobbles = get_scrobbles_num(track=track,timerange=alltime())
 
-	c = [e for e in alltimecharts if e["track"] == track][0]
+	c = [e for e in alltimecharts if e["track_id"] == track_id][0]
 	scrobbles = c["scrobbles"]
 	position = c["rank"]
 	cert = None
@@ -417,6 +541,37 @@ def track_info(dbconn=None,**keys):
 	}
 
 
+@waitfordb
+def album_info(dbconn=None,**keys):
+
+	album = keys.get('album')
+	if album is None: raise exceptions.MissingEntityParameter()
+
+	album_id = sqldb.get_album_id(album,create_new=False,dbconn=dbconn)
+	if not album_id: raise exceptions.AlbumDoesNotExist(album['albumtitle'])
+
+	album = sqldb.get_album(album_id,dbconn=dbconn)
+	alltimecharts = get_charts_albums(timerange=alltime(),dbconn=dbconn)
+
+	#scrobbles = get_scrobbles_num(track=track,timerange=alltime())
+
+	c = [e for e in alltimecharts if e["album"] == album][0]
+	scrobbles = c["scrobbles"]
+	position = c["rank"]
+
+	return {
+		"album":album,
+		"scrobbles":scrobbles,
+		"position":position,
+		"medals":{
+			"gold": [year for year in cached.medals_albums if album_id in cached.medals_albums[year]['gold']],
+			"silver": [year for year in cached.medals_albums if album_id in cached.medals_albums[year]['silver']],
+			"bronze": [year for year in cached.medals_albums if album_id in cached.medals_albums[year]['bronze']],
+		},
+		"topweeks":len([e for e in cached.weekly_topalbums if e == album_id]),
+		"id":album_id
+	}
+
 
 def get_predefined_rulesets(dbconn=None):
 	validchars = "-_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
@@ -464,6 +619,7 @@ def start_db():
 	# Upgrade database
 	from .. import upgrade
 	upgrade.upgrade_db(sqldb.add_scrobbles)
+	upgrade.parse_old_albums()
 
 	# Load temporary tables
 	from . import associated
@@ -497,4 +653,7 @@ def db_search(query,type=None):
 		results = sqldb.search_artist(query)
 	if type=="TRACK":
 		results = sqldb.search_track(query)
+	if type=="ALBUM":
+		results = sqldb.search_album(query)
+
 	return results

maloja/database/cached.py

@@ -14,16 +14,21 @@ medals_artists = {
 medals_tracks = {
 	# year: {'gold':[],'silver':[],'bronze':[]}
 }
+medals_albums = {
+	# year: {'gold':[],'silver':[],'bronze':[]}
+}
 
 weekly_topartists = []
 weekly_toptracks = []
+weekly_topalbums = []
 
 @runyearly
 def update_medals():
 
-	global medals_artists, medals_tracks
+	global medals_artists, medals_tracks, medals_albums
 	medals_artists.clear()
 	medals_tracks.clear()
+	medals_albums.clear()
 
 	with sqldb.engine.begin() as conn:
 		for year in mjt.ranges(step="year"):
@@ -31,11 +36,14 @@ def update_medals():
 
 			charts_artists = sqldb.count_scrobbles_by_artist(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=conn)
 			charts_tracks = sqldb.count_scrobbles_by_track(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=conn)
+			charts_albums = sqldb.count_scrobbles_by_album(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=conn)
 
 			entry_artists = {'gold':[],'silver':[],'bronze':[]}
 			entry_tracks = {'gold':[],'silver':[],'bronze':[]}
+			entry_albums = {'gold':[],'silver':[],'bronze':[]}
 			medals_artists[year.desc()] = entry_artists
 			medals_tracks[year.desc()] = entry_tracks
+			medals_albums[year.desc()] = entry_albums
 
 			for entry in charts_artists:
 				if entry['rank'] == 1: entry_artists['gold'].append(entry['artist_id'])
@@ -47,6 +55,11 @@ def update_medals():
 				elif entry['rank'] == 2: entry_tracks['silver'].append(entry['track_id'])
 				elif entry['rank'] == 3: entry_tracks['bronze'].append(entry['track_id'])
 				else: break
+			for entry in charts_albums:
+				if entry['rank'] == 1: entry_albums['gold'].append(entry['album_id'])
+				elif entry['rank'] == 2: entry_albums['silver'].append(entry['album_id'])
+				elif entry['rank'] == 3: entry_albums['bronze'].append(entry['album_id'])
+				else: break
 
 
 
@@ -54,9 +67,10 @@ def update_medals():
 @rundaily
 def update_weekly():
 
-	global weekly_topartists, weekly_toptracks
+	global weekly_topartists, weekly_toptracks, weekly_topalbums
 	weekly_topartists.clear()
 	weekly_toptracks.clear()
+	weekly_topalbums.clear()
 
 	with sqldb.engine.begin() as conn:
 		for week in mjt.ranges(step="week"):
@@ -65,6 +79,7 @@ def update_weekly():
 
 			charts_artists = sqldb.count_scrobbles_by_artist(since=week.first_stamp(),to=week.last_stamp(),resolve_ids=False,dbconn=conn)
 			charts_tracks = sqldb.count_scrobbles_by_track(since=week.first_stamp(),to=week.last_stamp(),resolve_ids=False,dbconn=conn)
+			charts_albums = sqldb.count_scrobbles_by_album(since=week.first_stamp(),to=week.last_stamp(),resolve_ids=False,dbconn=conn)
 
 			for entry in charts_artists:
 				if entry['rank'] == 1: weekly_topartists.append(entry['artist_id'])
@@ -72,3 +87,6 @@ def update_weekly():
 			for entry in charts_tracks:
 				if entry['rank'] == 1: weekly_toptracks.append(entry['track_id'])
 				else: break
+			for entry in charts_albums:
+				if entry['rank'] == 1: weekly_topalbums.append(entry['album_id'])
+				else: break

maloja/database/dbcache.py

@@ -10,7 +10,7 @@ from doreah.regular import runhourly
 from doreah.logging import log
 
 from ..pkg_global.conf import malojaconfig
-
+from . import no_aux_mode
 
 
 if malojaconfig['USE_GLOBAL_CACHE']:
@@ -21,6 +21,7 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 
 
 	@runhourly
+	@no_aux_mode
 	def maintenance():
 		print_stats()
 		trim_cache()
@@ -80,7 +81,9 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 
 		return outer_func
 
+	@no_aux_mode
 	def invalidate_caches(scrobbletime=None):
+
 		cleared, kept = 0, 0
 		for k in cache.keys():
 			# VERY BIG TODO: differentiate between None as in 'unlimited timerange' and None as in 'time doesnt matter here'!
@@ -91,11 +94,10 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 				kept += 1
 		log(f"Invalidated {cleared} of {cleared+kept} DB cache entries")
 
-
+	@no_aux_mode
 	def invalidate_entity_cache():
 		entitycache.clear()
 
-
 	def trim_cache():
 		ramprct = psutil.virtual_memory().percent
 		if ramprct > malojaconfig["DB_MAX_MEMORY"]:
@@ -162,11 +164,15 @@ def get_size_of(obj,counted=None):
 	return size
 
 def human_readable_size(obj):
-	units = ['','K','M','G','T','P']
-	idx = 0
-	bytes = get_size_of(obj)
-	while bytes > 1024 and len(units) > idx+1:
-		bytes = bytes / 1024
-		idx += 1
+	units = ['','Ki','Mi','Gi','Ti','Pi']
+	magnitude = 0
 
-	return f"{bytes:.2f} {units[idx]}B"
+	bytes = get_size_of(obj)
+	while bytes > 1024 and len(units) > magnitude+1:
+		bytes = bytes / 1024
+		magnitude += 1
+
+	if magnitude > 2:
+		return f"{bytes:.2f} {units[magnitude]}B"
+	else:
+		return f"{bytes:.0f} {units[magnitude]}B"

maloja/database/exceptions.py

@@ -16,7 +16,7 @@ class DatabaseNotBuilt(HTTPError):
 	def __init__(self):
 		super().__init__(
 			status=503,
-			body="The Maloja Database is being upgraded to Version 3. This could take quite a long time! (~ 2-5 minutes per 10 000 scrobbles)",
+			body="The Maloja Database is being upgraded to support new Maloja features. This could take a while.",
 			headers={"Retry-After":120}
 		)
 
@@ -27,3 +27,19 @@ class MissingScrobbleParameters(Exception):
 
 class MissingEntityParameter(Exception):
 	pass
+
+class EntityDoesNotExist(HTTPError):
+	entitytype = 'Entity'
+	def __init__(self,name):
+		self.entityname = name
+		super().__init__(
+			status=404,
+			body=f"The {self.entitytype} '{self.entityname}' does not exist in the database."
+		)
+
+class ArtistDoesNotExist(EntityDoesNotExist):
+	entitytype = 'Artist'
+class AlbumDoesNotExist(EntityDoesNotExist):
+	entitytype = 'Album'
+class TrackDoesNotExist(EntityDoesNotExist):
+	entitytype = 'Track'

maloja/dev/apidebug.py

@@ -0,0 +1,28 @@
+import bottle, waitress
+
+from ..pkg_global.conf import malojaconfig
+
+from doreah.logging import log
+from nimrodel import EAPI as API
+
+
+PORT = malojaconfig["PORT"]
+HOST = malojaconfig["HOST"]
+
+the_listener = API(delay=True)
+
+@the_listener.get("{path}")
+@the_listener.post("{path}")
+def all_requests(path,**kwargs):
+	result = {
+		'path':path,
+		'payload': kwargs
+	}
+	log(result)
+	return result
+
+
+def run():
+	server = bottle.Bottle()
+	the_listener.mount(server,path="apis")
+	waitress.serve(server, listen=f"*:{PORT}")

maloja/images.py

@@ -12,195 +12,317 @@ import base64
 import requests
 import datauri
 import io
-from threading import Thread, Timer, BoundedSemaphore
+from threading import Lock
+from concurrent.futures import ThreadPoolExecutor
 import re
 import datetime
+import time
 
 import sqlalchemy as sql
 
 
 
+MAX_RESOLVE_THREADS = 5
+MAX_SECONDS_TO_RESOLVE_REQUEST = 5
+
+
+# remove old db file (columns missing)
+try:
+	os.remove(data_dir['cache']('images.sqlite'))
+except:
+	pass
 
 DB = {}
-engine = sql.create_engine(f"sqlite:///{data_dir['cache']('images.sqlite')}", echo = False)
+engine = sql.create_engine(f"sqlite:///{data_dir['cache']('imagecache.sqlite')}", echo = False)
 meta = sql.MetaData()
 
+dblock = Lock()
+
 DB['artists'] = sql.Table(
 	'artists', meta,
 	sql.Column('id',sql.Integer,primary_key=True),
 	sql.Column('url',sql.String),
 	sql.Column('expire',sql.Integer),
-	sql.Column('raw',sql.String)
+#	sql.Column('raw',sql.String)
+	sql.Column('local',sql.Boolean),
+	sql.Column('localproxyurl',sql.String)
 )
 DB['tracks'] = sql.Table(
 	'tracks', meta,
 	sql.Column('id',sql.Integer,primary_key=True),
 	sql.Column('url',sql.String),
 	sql.Column('expire',sql.Integer),
-	sql.Column('raw',sql.String)
+#	sql.Column('raw',sql.String)
+	sql.Column('local',sql.Boolean),
+	sql.Column('localproxyurl',sql.String)
+)
+DB['albums'] = sql.Table(
+	'albums', meta,
+	sql.Column('id',sql.Integer,primary_key=True),
+	sql.Column('url',sql.String),
+	sql.Column('expire',sql.Integer),
+#	sql.Column('raw',sql.String)
+	sql.Column('local',sql.Boolean),
+	sql.Column('localproxyurl',sql.String)
 )
 
 meta.create_all(engine)
 
-def get_image_from_cache(id,table):
+def get_id_and_table(track_id=None,artist_id=None,album_id=None):
+	if track_id:
+		return track_id,'tracks'
+	elif album_id:
+		return album_id,'albums'
+	elif artist_id:
+		return artist_id,'artists'
+
+def get_image_from_cache(track_id=None,artist_id=None,album_id=None):
 	now = int(datetime.datetime.now().timestamp())
+	entity_id, table = get_id_and_table(track_id=track_id,artist_id=artist_id,album_id=album_id)
+
 	with engine.begin() as conn:
 		op = DB[table].select().where(
-			DB[table].c.id==id,
+			DB[table].c.id==entity_id,
 			DB[table].c.expire>now
 		)
 		result = conn.execute(op).all()
 	for row in result:
-		if row.raw is not None:
-			return {'type':'raw','value':row.raw}
+		if row.local:
+			return {'type':'localurl','value':row.url}
+		elif row.localproxyurl:
+			return {'type':'localurl','value':row.localproxyurl}
 		else:
 			return {'type':'url','value':row.url} # returns None as value if nonexistence cached
 	return None # no cache entry
 
-def set_image_in_cache(id,table,url):
-	remove_image_from_cache(id,table)
-	now = int(datetime.datetime.now().timestamp())
-	if url is None:
-		expire = now + (malojaconfig["CACHE_EXPIRE_NEGATIVE"] * 24 * 3600)
-	else:
-		expire = now + (malojaconfig["CACHE_EXPIRE_POSITIVE"] * 24 * 3600)
+def set_image_in_cache(url,track_id=None,artist_id=None,album_id=None,local=False):
+	remove_image_from_cache(track_id=track_id,artist_id=artist_id,album_id=album_id)
+	entity_id, table = get_id_and_table(track_id=track_id,artist_id=artist_id,album_id=album_id)
 
-	raw = dl_image(url)
+	with dblock:
+		now = int(datetime.datetime.now().timestamp())
+		if url is None:
+			expire = now + (malojaconfig["CACHE_EXPIRE_NEGATIVE"] * 24 * 3600)
+		else:
+			expire = now + (malojaconfig["CACHE_EXPIRE_POSITIVE"] * 24 * 3600)
 
-	with engine.begin() as conn:
-		op = DB[table].insert().values(
-			id=id,
-			url=url,
-			expire=expire,
-			raw=raw
-		)
-		result = conn.execute(op)
+		if not local and malojaconfig["PROXY_IMAGES"] and url is not None:
+			localproxyurl = dl_image(url)
+		else:
+			localproxyurl = None
+
+		with engine.begin() as conn:
+			op = DB[table].insert().values(
+				id=entity_id,
+				url=url,
+				expire=expire,
+				local=local,
+				localproxyurl=localproxyurl
+			)
+			result = conn.execute(op)
+
+def remove_image_from_cache(track_id=None,artist_id=None,album_id=None):
+	entity_id, table = get_id_and_table(track_id=track_id,artist_id=artist_id,album_id=album_id)
+
+	with dblock:
+		with engine.begin() as conn:
+			op = DB[table].delete().where(
+				DB[table].c.id==entity_id,
+			).returning(
+				DB[table].c.id,
+				DB[table].c.localproxyurl
+			)
+			result = conn.execute(op).all()
+
+		for row in result:
+			targetpath = data_dir['cache']('images',row.localproxyurl.split('/')[-1])
+			try:
+				os.remove(targetpath)
+			except:
+				pass
 
-def remove_image_from_cache(id,table):
-	with engine.begin() as conn:
-		op = DB[table].delete().where(
-			DB[table].c.id==id,
-		)
-		result = conn.execute(op)
 
 def dl_image(url):
-	if not malojaconfig["PROXY_IMAGES"]: return None
-	if url is None: return None
-	if url.startswith("/"): return None #local image
 	try:
 		r = requests.get(url)
 		mime = r.headers.get('content-type') or 'image/jpg'
 		data = io.BytesIO(r.content).read()
-		uri = datauri.DataURI.make(mime,charset='ascii',base64=True,data=data)
-		log(f"Downloaded {url} for local caching")
-		return uri
+		#uri = datauri.DataURI.make(mime,charset='ascii',base64=True,data=data)
+		targetname = '%030x' % random.getrandbits(128)
+		targetpath = data_dir['cache']('images',targetname)
+		with open(targetpath,'wb') as fd:
+			fd.write(data)
+		return os.path.join("/cacheimages",targetname)
 	except Exception:
 		log(f"Image {url} could not be downloaded for local caching")
 		return None
 
 
 
+
+resolver = ThreadPoolExecutor(max_workers=MAX_RESOLVE_THREADS)
+
 ### getting images for any website embedding now ALWAYS returns just the generic link
 ### even if we have already cached it, we will handle that on request
 def get_track_image(track=None,track_id=None):
 	if track_id is None:
-		track_id = database.sqldb.get_track_id(track)
+		track_id = database.sqldb.get_track_id(track,create_new=False)
 
-	return f"/image?type=track&id={track_id}"
+	if malojaconfig["USE_ALBUM_ARTWORK_FOR_TRACKS"]:
+		if track is None:
+			track = database.sqldb.get_track(track_id)
+		if track.get("album"):
+			album_id = database.sqldb.get_album_id(track["album"])
+			return get_album_image(album_id=album_id)
 
+	resolver.submit(resolve_image,track_id=track_id)
+
+	return f"/image?track_id={track_id}"
 
 def get_artist_image(artist=None,artist_id=None):
 	if artist_id is None:
-		artist_id = database.sqldb.get_artist_id(artist)
+		artist_id = database.sqldb.get_artist_id(artist,create_new=False)
 
-	return f"/image?type=artist&id={artist_id}"
+	resolver.submit(resolve_image,artist_id=artist_id)
+
+	return f"/image?artist_id={artist_id}"
+
+def get_album_image(album=None,album_id=None):
+	if album_id is None:
+		album_id = database.sqldb.get_album_id(album,create_new=False)
+
+	resolver.submit(resolve_image,album_id=album_id)
+
+	return f"/image?album_id={album_id}"
 
 
+# this is to keep track of what is currently being resolved
+# so new requests know that they don't need to queue another resolve
+image_resolve_controller_lock = Lock()
+image_resolve_controller = {
+	'artists':set(),
+	'albums':set(),
+	'tracks':set()
+}
 
-resolve_semaphore = BoundedSemaphore(8)
+# this function doesn't need to return any info
+# it runs async to do all the work that takes time and only needs to write the result
+# to the cache so the synchronous functions (http requests) can access it
+def resolve_image(artist_id=None,track_id=None,album_id=None):
+	result = get_image_from_cache(artist_id=artist_id,track_id=track_id,album_id=album_id)
+	if result is not None:
+		# No need to do anything
+		return
 
+	if artist_id:
+		entitytype = 'artist'
+		table = 'artists'
+		getfunc, entity_id = database.sqldb.get_artist, artist_id
+	elif track_id:
+		entitytype = 'track'
+		table = 'tracks'
+		getfunc, entity_id = database.sqldb.get_track, track_id
+	elif album_id:
+		entitytype = 'album'
+		table = 'albums'
+		getfunc, entity_id = database.sqldb.get_album, album_id
 
-def resolve_track_image(track_id):
+	# is another thread already working on this?
+	with image_resolve_controller_lock:
+		if entity_id in image_resolve_controller[table]:
+			return
+		else:
+			image_resolve_controller[table].add(entity_id)
 
-	with resolve_semaphore:
-		# check cache
-		result = get_image_from_cache(track_id,'tracks')
-		if result is not None:
-			return result
-
-		track = database.sqldb.get_track(track_id)
+	try:
+		entity = getfunc(entity_id)
 
 		# local image
 		if malojaconfig["USE_LOCAL_IMAGES"]:
-			images = local_files(artists=track['artists'],title=track['title'])
+			images = local_files(**{entitytype: entity})
 			if len(images) != 0:
 				result = random.choice(images)
 				result = urllib.parse.quote(result)
-				result = {'type':'url','value':result}
-				set_image_in_cache(track_id,'tracks',result['value'])
+				result = {'type':'localurl','value':result}
+				set_image_in_cache(artist_id=artist_id,track_id=track_id,album_id=album_id,url=result['value'],local=True)
 				return result
 
 		# third party
-		result = thirdparty.get_image_track_all((track['artists'],track['title']))
+		if artist_id:
+			result = thirdparty.get_image_artist_all(entity)
+		elif track_id:
+			result = thirdparty.get_image_track_all((entity['artists'],entity['title']))
+		elif album_id:
+			result = thirdparty.get_image_album_all((entity['artists'],entity['albumtitle']))
+
 		result = {'type':'url','value':result}
-		set_image_in_cache(track_id,'tracks',result['value'])
-
-		return result
+		set_image_in_cache(artist_id=artist_id,track_id=track_id,album_id=album_id,url=result['value'])
+	finally:
+		with image_resolve_controller_lock:
+			image_resolve_controller[table].remove(entity_id)
 
 
-def resolve_artist_image(artist_id):
 
-	with resolve_semaphore:
+# the actual http request for the full image
+def image_request(artist_id=None,track_id=None,album_id=None):
+
+	# because we use lazyload, we can allow our http requests to take a little while at least
+	# not the full backend request, but a few seconds to give us time to fetch some images
+	# because 503 retry-after doesn't seem to be honored
+	attempt = 0
+	while attempt < MAX_SECONDS_TO_RESOLVE_REQUEST:
+		attempt += 1
 		# check cache
-		result = get_image_from_cache(artist_id,'artists')
+		result = get_image_from_cache(artist_id=artist_id,track_id=track_id,album_id=album_id)
 		if result is not None:
+			# we got an entry, even if it's that there is no image (value None)
+			if result['value'] is None:
+				# use placeholder
+				if malojaconfig["FANCY_PLACEHOLDER_ART"]:
+					placeholder_url = "https://generative-placeholders.glitch.me/image?width=300&height=300&style="
+					if artist_id:
+						result['value'] = placeholder_url + f"tiles&colors={artist_id % 100}"
+					if track_id:
+						result['value'] = placeholder_url + f"triangles&colors={track_id % 100}"
+					if album_id:
+						result['value'] = placeholder_url + f"joy-division&colors={album_id % 100}"
+				else:
+					if artist_id:
+						result['value'] = "/static/svg/placeholder_artist.svg"
+					if track_id:
+						result['value'] = "/static/svg/placeholder_track.svg"
+					if album_id:
+						result['value'] = "/static/svg/placeholder_album.svg"
 			return result
+		time.sleep(1)
 
-		artist = database.sqldb.get_artist(artist_id)
+	# no entry, which means we're still working on it
+	return {'type':'noimage','value':'wait'}
 
-		# local image
-		if malojaconfig["USE_LOCAL_IMAGES"]:
-			images = local_files(artist=artist)
-			if len(images) != 0:
-				result = random.choice(images)
-				result = urllib.parse.quote(result)
-				result = {'type':'url','value':result}
-				set_image_in_cache(artist_id,'artists',result['value'])
-				return result
-
-		# third party
-		result = thirdparty.get_image_artist_all(artist)
-		result = {'type':'url','value':result}
-		set_image_in_cache(artist_id,'artists',result['value'])
-
-		return result
 
 
 # removes emojis and weird shit from names
 def clean(name):
 	return "".join(c for c in name if c.isalnum() or c in []).strip()
 
-def get_all_possible_filenames(artist=None,artists=None,title=None):
-	# check if we're dealing with a track or artist, then clean up names
-	# (only remove non-alphanumeric, allow korean and stuff)
-
-	if title is not None and artists is not None:
-		track = True
-		title, artists = clean(title), [clean(a) for a in artists]
-	elif artist is not None:
-		track = False
+# new and improved
+def get_all_possible_filenames(artist=None,track=None,album=None):
+	if track:
+		title, artists = clean(track['title']), [clean(a) for a in track['artists']]
+		superfolder = "tracks/"
+	elif album:
+		title, artists = clean(album['albumtitle']), [clean(a) for a in album.get('artists') or []]
+		superfolder = "albums/"
+	elif artist:
 		artist = clean(artist)
-	else: return []
-
-
-	superfolder = "tracks/" if track else "artists/"
+		superfolder = "artists/"
+	else:
+		return []
 
 	filenames = []
 
-	if track:
-		#unsafeartists = [artist.translate(None,"-_./\\") for artist in artists]
+	if track or album:
 		safeartists = [re.sub("[^a-zA-Z0-9]","",artist) for artist in artists]
-		#unsafetitle = title.translate(None,"-_./\\")
 		safetitle = re.sub("[^a-zA-Z0-9]","",title)
 
 		if len(artists) < 4:
@@ -210,7 +332,6 @@ def get_all_possible_filenames(artist=None,artists=None,title=None):
 			unsafeperms = [sorted(artists)]
 			safeperms = [sorted(safeartists)]
 
-
 		for unsafeartistlist in unsafeperms:
 			filename = "-".join(unsafeartistlist) + "_" + title
 			if filename != "":
@@ -241,10 +362,11 @@ def get_all_possible_filenames(artist=None,artists=None,title=None):
 
 	return [superfolder + name for name in filenames]
 
-def local_files(artist=None,artists=None,title=None):
+
+def local_files(artist=None,album=None,track=None):
 
 
-	filenames = get_all_possible_filenames(artist,artists,title)
+	filenames = get_all_possible_filenames(artist=artist,album=album,track=track)
 
 	images = []
 
@@ -267,34 +389,50 @@ def local_files(artist=None,artists=None,title=None):
 
 
 
+class MalformedB64(Exception):
+	pass
+
 def set_image(b64,**keys):
-	track = "title" in keys
-	if track:
-		entity = {'artists':keys['artists'],'title':keys['title']}
-		id = database.sqldb.get_track_id(entity)
-	else:
-		entity = keys['artist']
-		id = database.sqldb.get_artist_id(entity)
+	if "title" in keys:
+		entity = {"track":keys}
+		id = database.sqldb.get_track_id(entity['track'])
+		idkeys = {'track_id':id}
+		dbtable = "tracks"
+	elif "albumtitle" in keys:
+		entity = {"album":keys}
+		id = database.sqldb.get_album_id(entity['album'])
+		idkeys = {'album_id':id}
+		dbtable = "albums"
+	elif "artist" in keys:
+		entity = keys
+		id = database.sqldb.get_artist_id(entity['artist'])
+		idkeys = {'artist_id':id}
+		dbtable = "artists"
 
 	log("Trying to set image, b64 string: " + str(b64[:30] + "..."),module="debug")
 
 	regex = r"data:image/(\w+);base64,(.+)"
-	type,b64 = re.fullmatch(regex,b64).groups()
+	match = re.fullmatch(regex,b64)
+	if not match: raise MalformedB64()
+
+	type,b64 = match.groups()
 	b64 = base64.b64decode(b64)
 	filename = "webupload" + str(int(datetime.datetime.now().timestamp())) + "." + type
-	for folder in get_all_possible_filenames(**keys):
+	for folder in get_all_possible_filenames(**entity):
 		if os.path.exists(data_dir['images'](folder)):
 			with open(data_dir['images'](folder,filename),"wb") as f:
 				f.write(b64)
 			break
 	else:
-		folder = get_all_possible_filenames(**keys)[0]
+		folder = get_all_possible_filenames(**entity)[0]
 		os.makedirs(data_dir['images'](folder))
 		with open(data_dir['images'](folder,filename),"wb") as f:
 			f.write(b64)
 
+
 	log("Saved image as " + data_dir['images'](folder,filename),module="debug")
 
 	# set as current picture in rotation
-	if track: set_image_in_cache(id,'tracks',os.path.join("/images",folder,filename))
-	else: set_image_in_cache(id,'artists',os.path.join("/images",folder,filename))
+	set_image_in_cache(**idkeys,url=os.path.join("/images",folder,filename),local=True)
+
+	return os.path.join("/images",folder,filename)

maloja/jinjaenv/context.py

@@ -1,5 +1,6 @@
 from . import filters
 from ..pkg_global.conf import malojaconfig
+from ..pkg_global import conf
 
 from .. import database, malojatime, images, malojauri, thirdparty, __pkginfo__
 from ..database import jinjaview
@@ -32,6 +33,7 @@ def update_jinja_environment():
 		"mlj_uri": malojauri,
 		"settings": malojaconfig,
 		"thirdparty": thirdparty,
+		"conf":conf,
 		"pkginfo": __pkginfo__,
 		# external
 		"urllib": urllib,

maloja/malojauri.py

@@ -4,7 +4,7 @@ import urllib
 import math
 
 # this also sets defaults!
-def uri_to_internal(keys,forceTrack=False,forceArtist=False,api=False):
+def uri_to_internal(keys,forceTrack=False,forceArtist=False,forceAlbum=False,api=False):
 
 	# output:
 	# 1	keys that define the filtered object like artist or track
@@ -12,12 +12,23 @@ def uri_to_internal(keys,forceTrack=False,forceArtist=False,api=False):
 	# 3	keys that define interal time ranges
 	# 4	keys that define amount limits
 
+	type = None
+	if forceTrack: type = "track"
+	if forceArtist: type = "artist"
+	if forceAlbum: type = "album"
+
+	if not type and "title" in keys: type = "track"
+	if not type and "albumtitle" in keys: type = "album"
+	if not type and "artist" in keys: type = "artist"
+
 	# 1
-	if "title" in keys and not forceArtist:
+	if type == "track":
 		filterkeys = {"track":{"artists":keys.getall("artist"),"title":keys.get("title")}}
-	elif "artist" in keys and not forceTrack:
+	elif type == "artist":
 		filterkeys = {"artist":keys.get("artist")}
 		if "associated" in keys: filterkeys["associated"] = True
+	elif type == "album":
+		filterkeys = {"album":{"artists":keys.getall("artist"),"albumtitle":keys.get("albumtitle") or keys.get("title")}}
 	else:
 		filterkeys = {}
 
@@ -84,6 +95,10 @@ def internal_to_uri(keys):
 		for a in keys["track"]["artists"]:
 			urikeys.append("artist",a)
 		urikeys.append("title",keys["track"]["title"])
+	elif "album" in keys:
+		for a in keys["album"].get("artists") or []:
+			urikeys.append("artist",a)
+		urikeys.append("albumtitle",keys["album"]["albumtitle"])
 
 	#time
 	if "timerange" in keys:

maloja/pkg_global/conf.py

@@ -6,6 +6,8 @@ from doreah.configuration import types as tp
 from ..__pkginfo__ import VERSION
 
 
+# this mode specifies whether we run some auxiliary task instead of the main server
+AUX_MODE = True
 
 
 # if DATA_DIRECTORY is specified, this is the directory to use for EVERYTHING, no matter what
@@ -177,17 +179,25 @@ malojaconfig = Configuration(
 
 		},
 		"Database":{
+			"album_information_trust":(tp.Choice({'first':"First",'last':"Last",'majority':"Majority"}),	"Album Information Authority","first",															"Whether to trust the first album information that is sent with a track or update every time a different album is sent"),
 			"invalid_artists":(tp.Set(tp.String()),								"Invalid Artists",				["[Unknown Artist]","Unknown Artist","Spotify"],											"Artists that should be discarded immediately"),
 			"remove_from_title":(tp.Set(tp.String()),							"Remove from Title",			["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)","(Bonus Track)"],	"Phrases that should be removed from song titles"),
-			"delimiters_feat":(tp.Set(tp.String()),								"Featuring Delimiters",			["ft.","ft","feat.","feat","featuring","Ft.","Ft","Feat.","Feat","Featuring"],				"Delimiters used for extra artists, even when in the title field"),
+			"delimiters_feat":(tp.Set(tp.String()),								"Featuring Delimiters",			["ft.","ft","feat.","feat","featuring"],													"Delimiters used for extra artists, even when in the title field"),
 			"delimiters_informal":(tp.Set(tp.String()),							"Informal Delimiters",			["vs.","vs","&"],																			"Delimiters in informal artist strings with spaces expected around them"),
-			"delimiters_formal":(tp.Set(tp.String()),							"Formal Delimiters",			[";","/","|","␝","␞","␟"],																					"Delimiters used to tag multiple artists when only one tag field is available")
+			"delimiters_formal":(tp.Set(tp.String()),							"Formal Delimiters",			[";","/","|","␝","␞","␟"],																	"Delimiters used to tag multiple artists when only one tag field is available"),
+			"filters_remix":(tp.Set(tp.String()),								"Remix Filters",				["Remix", "Remix Edit", "Short Mix", "Extended Mix", "Soundtrack Version"],					"Filters used to recognize the remix artists in the title"),
+			"parse_remix_artists":(tp.Boolean(),								"Parse Remix Artists",			False)
 		},
 		"Web Interface":{
 			"default_range_charts_artists":(tp.Choice({'alltime':'All Time','year':'Year','month':"Month",'week':'Week'}),	"Default Range Artist Charts",	"year"),
 			"default_range_charts_tracks":(tp.Choice({'alltime':'All Time','year':'Year','month':"Month",'week':'Week'}),	"Default Range Track Charts",	"year"),
 			"default_step_pulse":(tp.Choice({'year':'Year','month':"Month",'week':'Week','day':'Day'}),						"Default Pulse Step",			"month"),
 			"charts_display_tiles":(tp.Boolean(),								"Display Chart Tiles",			False),
+			"album_showcase":(tp.Boolean(),										"Display Album Showcase",		True,		"Display a graphical album showcase for artist overview pages instead of a chart list"),
+			"display_art_icons":(tp.Boolean(),									"Display Album/Artist Icons",	True),
+			"default_album_artist":(tp.String(),								"Default Albumartist",			"Various Artists"),
+			"use_album_artwork_for_tracks":(tp.Boolean(),						"Use Album Artwork for tracks",	True),
+			"fancy_placeholder_art":(tp.Boolean(),								"Use fancy placeholder artwork",True),
 			"discourage_cpu_heavy_stats":(tp.Boolean(),							"Discourage CPU-heavy stats",	False,					"Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!"),
 			"use_local_images":(tp.Boolean(),									"Use Local Images",				True),
 			#"local_image_rotate":(tp.Integer(),									"Local Image Rotate",			3600),
@@ -294,15 +304,6 @@ data_dir = {
 
 
 
-### write down the last ran version
-with open(pthj(dir_settings['state'],".lastmalojaversion"),"w") as filed:
-	filed.write(VERSION)
-	filed.write("\n")
-
-
-
-
-
 ### DOREAH CONFIGURATION
 
 from doreah import config
@@ -326,6 +327,10 @@ config(
 
 
 
+custom_css_files = [f for f in os.listdir(data_dir['css']()) if f.lower().endswith('.css')]
+
+from ..database.sqldb import set_maloja_info
+set_maloja_info({'last_run_version':VERSION})
 
 # what the fuck did i just write
 # this spaghetti file is proudly sponsored by the rice crackers i'm eating at the

maloja/proccontrol/tasks/__init__.py

@@ -1,3 +1,4 @@
 from .import_scrobbles import import_scrobbles
 from .backup import backup
 from .export import export # read that line out loud
+from .parse_albums import parse_albums

maloja/proccontrol/tasks/import_scrobbles.py

@@ -49,6 +49,11 @@ def import_scrobbles(inputf):
 		typeid,typedesc = "maloja","Maloja"
 		importfunc = parse_maloja
 
+	# username_lb-YYYY-MM-DD.json
+	elif re.match(".*_lb-[0-9-]+\.json",filename):
+		typeid,typedesc = "listenbrainz","ListenBrainz"
+		importfunc = parse_listenbrainz
+
 	else:
 		print("File",inputf,"could not be identified as a valid import source.")
 		return result
@@ -84,7 +89,7 @@ def import_scrobbles(inputf):
 				 	"track":{
 				 		"artists":scrobble['track_artists'],
 				 		"title":scrobble['track_title'],
-				 		"length":None
+				 		"length":scrobble['track_length'],
 				 	},
 				 	"duration":scrobble['scrobble_duration'],
 				 	"origin":"import:" + typeid,
@@ -154,6 +159,7 @@ def parse_spotify_lite(inputf):
 				yield ("CONFIDENT_IMPORT",{
 					'track_title':title,
 					'track_artists': artist,
+					'track_length': None,
 					'scrobble_time': timestamp,
 					'scrobble_duration':played,
 					'album_name': None
@@ -174,7 +180,7 @@ def parse_spotify_full(inputf):
 	if len(inputfiles) == 0:
 		print("No files found!")
 		return
-		
+
 	if inputfiles != [inputf]:
 		print("Spotify files should all be imported together to identify duplicates across the whole dataset.")
 		if not ask("Import " + ", ".join(col['yellow'](i) for i in inputfiles) + "?",default=True):
@@ -262,6 +268,7 @@ def parse_spotify_full(inputf):
 				yield (status,{
 					'track_title':title,
 					'track_artists': artist,
+					'track_length': None,
 					'album_name': album,
 					'scrobble_time': timestamp,
 					'scrobble_duration':played
@@ -294,10 +301,13 @@ def parse_lastfm(inputf):
 				yield ('CONFIDENT_IMPORT',{
 					'track_title': title,
 					'track_artists': artist,
+					'track_length': None,
 					'album_name': album,
 					'scrobble_time': int(datetime.datetime.strptime(
 						time + '+0000',
 						"%d %b %Y %H:%M%z"
+						# lastfm exports have time in UTC
+						# some old imports might have the wrong time here!
 					).timestamp()),
 					'scrobble_duration':None
 				},'')
@@ -305,6 +315,28 @@ def parse_lastfm(inputf):
 				yield ('FAIL',None,f"{row} (Line {line}) could not be parsed. Scrobble not imported. ({repr(e)})")
 				continue
 
+def parse_listenbrainz(inputf):
+
+	with open(inputf,'r') as inputfd:
+		data = json.load(inputfd)
+
+	for entry in data:
+
+		try:
+			track_metadata = entry['track_metadata']
+			additional_info = track_metadata.get('additional_info', {})
+
+			yield ("CONFIDENT_IMPORT",{
+				'track_title': track_metadata['track_name'],
+				'track_artists': additional_info.get('artist_names') or track_metadata['artist_name'],
+				'track_length': int(additional_info.get('duration_ms', 0) / 1000) or additional_info.get('duration'),
+				'album_name': track_metadata.get('release_name'),
+				'scrobble_time': entry['listened_at'],
+				'scrobble_duration': None,
+			},'')
+		except Exception as e:
+			yield ('FAIL',None,f"{entry} could not be parsed. Scrobble not imported. ({repr(e)})")
+			continue
 
 def parse_maloja(inputf):
 
@@ -318,6 +350,7 @@ def parse_maloja(inputf):
 			yield ('CONFIDENT_IMPORT',{
 				'track_title': s['track']['title'],
 				'track_artists': s['track']['artists'],
+				'track_length': s['track']['length'],
 				'album_name': s['track'].get('album',{}).get('name',''),
 				'scrobble_time': s['time'],
 				'scrobble_duration': s['duration']

maloja/proccontrol/tasks/parse_albums.py

@@ -0,0 +1,108 @@
+from doreah.io import col
+
+def parse_albums(strategy=None,prefer_existing=False):
+
+	if strategy not in ("track","none","all","majority","most"):
+		print("""
+Please specify your album parsing strategy:
+
+    --strategy           Specify what strategy to use when the scrobble contains
+                         no information about album artists.
+                         track      Take the track artists. This can lead to
+                                    separate albums being created for compilation
+                                    albums or albums that have collaboration tracks.
+                         none       Merge all albums with the same name and assign
+                                    'Various Artists' as the album artist.
+                         all        Merge all albums with the same name and assign
+                                    every artist that appears on the album as an album
+                                    artist.
+                         majority   Merge all albums with the same name and assign
+                                    artists that appear in at least half the tracks
+                                    of the album as album artists. [RECOMMENDED]
+                         most       Merge all albums with the same name and assign
+                                    the artist that appears most on the album as album
+                                    artist.
+    --prefer_existing    If an album with the same name already exists, use it
+                         without further examination of track artists.
+		""")
+		return
+
+
+
+	from ...database.sqldb import guess_albums, get_album_id, add_track_to_album
+
+	print("Parsing album information...")
+	result = guess_albums()
+
+	result = {track_id:result[track_id] for track_id in result if result[track_id]["assigned"]}
+	print("Found",col['yellow'](len(result)),"Tracks to assign albums to")
+
+	result_authorative = {track_id:result[track_id] for track_id in result if result[track_id]["assigned"]["artists"]}
+	result_guesswork = {track_id:result[track_id] for track_id in result if not result[track_id]["assigned"]["artists"]}
+
+	i = 0
+
+	def countup(i):
+		i+=1
+		if (i % 100) == 0:
+			print(f"Added album information for {i} of {len(result)} tracks...")
+		return i
+
+	for track_id in result_authorative:
+		albuminfo = result[track_id]['assigned']
+		album_id = get_album_id(albuminfo)
+		add_track_to_album(track_id,album_id)
+		i=countup(i)
+
+	albums = {}
+	for track_id in result_guesswork:
+		albuminfo = result[track_id]['assigned']
+
+		# check if already exists
+		if prefer_existing:
+			album_id = get_album_id(albuminfo,ignore_albumartists=True,create_new=False)
+			if album_id:
+				add_track_to_album(track_id,album_id)
+				i=countup(i)
+				continue
+
+		if strategy == 'track':
+			albuminfo['artists'] = result[track_id]['guess_artists']
+			album_id = get_album_id(albuminfo)
+			add_track_to_album(track_id,album_id)
+			i=countup(i)
+			continue
+
+		if strategy == 'none':
+			albuminfo['artists'] = []
+			album_id = get_album_id(albuminfo)
+			add_track_to_album(track_id,album_id)
+			i=countup(i)
+			continue
+
+		if strategy in ['all','majority','most']:
+			cleantitle = albuminfo['albumtitle'].lower()
+			albums.setdefault(cleantitle,{'track_ids':[],'artists':{},'title':albuminfo['albumtitle']})
+			albums[cleantitle]['track_ids'].append(track_id)
+			for a in result[track_id]['guess_artists']:
+				albums[cleantitle]['artists'].setdefault(a,0)
+				albums[cleantitle]['artists'][a] += 1
+
+
+	for cleantitle in albums:
+		artistoptions = albums[cleantitle]['artists']
+		track_ids = albums[cleantitle]['track_ids']
+		realtitle = albums[cleantitle]['title']
+		if strategy == 'all':
+			artists = [a for a in artistoptions]
+		elif strategy == 'majority':
+			artists = [a for a in artistoptions if artistoptions[a] >= (len(track_ids) / 2)]
+		elif strategy == 'most':
+			artists = [max(artistoptions,key=artistoptions.get)]
+
+		for track_id in track_ids:
+			album_id = get_album_id({'albumtitle':realtitle,'artists':artists})
+			add_track_to_album(track_id,album_id)
+			i=countup(i)
+
+	print(col['lawngreen']("Done!"))

maloja/server.py

@@ -10,6 +10,7 @@ import time
 # server stuff
 from bottle import Bottle, static_file, request, response, FormsDict, redirect, BaseRequest, abort
 import waitress
+from jinja2.exceptions import TemplateNotFound
 
 # doreah toolkit
 from doreah.logging import log
@@ -18,9 +19,10 @@ from doreah import auth
 # rest of the project
 from . import database
 from .database.jinjaview import JinjaDBConnection
-from .images import resolve_track_image, resolve_artist_image
+from .images import image_request
 from .malojauri import uri_to_internal, remove_identical
 from .pkg_global.conf import malojaconfig, data_dir
+from .pkg_global import conf
 from .jinjaenv.context import jinja_environment
 from .apis import init_apis, apikeystore
 
@@ -119,20 +121,14 @@ def deprecated_api(pth):
 @webserver.route("/image")
 def dynamic_image():
 	keys = FormsDict.decode(request.query)
-	if keys['type'] == 'track':
-		result = resolve_track_image(keys['id'])
-	elif keys['type'] == 'artist':
-		result = resolve_artist_image(keys['id'])
+	result = image_request(**{k:int(keys[k]) for k in keys})
 
-	if result is None or result['value'] in [None,'']:
-		return ""
-	if result['type'] == 'raw':
-		# data uris are directly served as image because a redirect to a data uri
-		# doesnt work
-		duri = datauri.DataURI(result['value'])
-		response.content_type = duri.mimetype
-		return duri.data
-	if result['type'] == 'url':
+	if result['type'] == 'noimage' and result['value'] == 'wait':
+		# still being worked on
+		response.status = 503
+		response.set_header('Retry-After',5)
+		return
+	if result['type'] in ('url','localurl'):
 		redirect(result['value'],307)
 
 @webserver.route("/images/<pth:re:.*\\.jpeg>")
@@ -159,6 +155,9 @@ def static_image(pth):
 	resp.set_header("Content-Type", "image/" + ext)
 	return resp
 
+@webserver.route("/cacheimages/<uuid>")
+def static_proxied_image(uuid):
+	return static_file(uuid,root=data_dir['cache']('images'))
 
 @webserver.route("/login")
 def login():
@@ -182,6 +181,15 @@ def static(path):
 	response.set_header("Cache-Control", "public, max-age=3600")
 	return response
 
+# static files not supplied by the package
+@webserver.get("/static_custom/<category>/<path:path>")
+def static_custom(category,path):
+	rootpath = {
+		'css':data_dir['css']()
+	}
+	response = static_file(path,root=rootpath[category])
+	response.set_header("Cache-Control", "public, max-age=3600")
+	return response
 
 
 ### DYNAMIC
@@ -203,12 +211,13 @@ def jinja_page(name):
 			"_urikeys":keys, #temporary!
 		}
 		loc_context["filterkeys"], loc_context["limitkeys"], loc_context["delimitkeys"], loc_context["amountkeys"], loc_context["specialkeys"] = uri_to_internal(keys)
-
-		template = jinja_environment.get_template(name + '.jinja')
 		try:
+			template = jinja_environment.get_template(name + '.jinja')
 			res = template.render(**loc_context)
-		except (ValueError, IndexError):
-			abort(404,"This Artist or Track does not exist")
+		except TemplateNotFound:
+			abort(404,f"Not found: '{name}'")
+		#except (ValueError, IndexError):
+		#	abort(404,"This Artist or Track does not exist")
 
 	if malojaconfig["DEV_MODE"]: jinja_environment.cache.clear()
 
@@ -272,6 +281,8 @@ logging.getLogger().addHandler(WaitressLogHandler())
 
 
 def run_server():
+	conf.AUX_MODE = False
+
 	log("Starting up Maloja server...")
 
 	## start database

maloja/thirdparty/__init__.py

@@ -63,7 +63,18 @@ def get_image_artist_all(artist):
 					log("Could not get artist image for " + str(artist) + " from " + service.name)
 			except Exception as e:
 				log("Error getting artist image from " + service.name + ": " + repr(e))
-
+def get_image_album_all(album):
+	with thirdpartylock:
+		for service in services["metadata"]:
+			try:
+				res = service.get_image_album(album)
+				if res is not None:
+					log("Got album image for " + str(album) + " from " + service.name)
+					return res
+				else:
+					log("Could not get album image for " + str(album) + " from " + service.name)
+			except Exception as e:
+				log("Error getting album image from " + service.name + ": " + repr(e))
 
 
 class GenericInterface:
@@ -217,6 +228,23 @@ class MetadataInterface(GenericInterface,abstract=True):
 		if imgurl is not None: imgurl = self.postprocess_url(imgurl)
 		return imgurl
 
+	def get_image_album(self,album):
+		artists, title = album
+		artiststring = urllib.parse.quote(", ".join(artists))
+		titlestring = urllib.parse.quote(title)
+		response = urllib.request.urlopen(
+			self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings)
+		)
+
+		responsedata = response.read()
+		if self.metadata["response_type"] == "json":
+			data = json.loads(responsedata)
+			imgurl = self.metadata_parse_response_album(data)
+		else:
+			imgurl = None
+		if imgurl is not None: imgurl = self.postprocess_url(imgurl)
+		return imgurl
+
 	# default function to parse response by descending down nodes
 	# override if more complicated
 	def metadata_parse_response_artist(self,data):
@@ -225,6 +253,9 @@ class MetadataInterface(GenericInterface,abstract=True):
 	def metadata_parse_response_track(self,data):
 		return self._parse_response("response_parse_tree_track", data)
 
+	def metadata_parse_response_album(self,data):
+		return self._parse_response("response_parse_tree_album", data)
+
 	def _parse_response(self, resp, data):
 		res = data
 		for node in self.metadata[resp]:

maloja/thirdparty/audiodb.py

@@ -9,13 +9,17 @@ class AudioDB(MetadataInterface):
 	}
 
 	metadata = {
-		#"trackurl": "https://theaudiodb.com/api/v1/json/{api_key}/searchtrack.php?s={artist}&t={title}",
+		#"trackurl": "https://theaudiodb.com/api/v1/json/{api_key}/searchtrack.php?s={artist}&t={title}", #patreon
 		"artisturl": "https://www.theaudiodb.com/api/v1/json/{api_key}/search.php?s={artist}",
+		#"albumurl": "https://www.theaudiodb.com/api/v1/json/{api_key}/searchalbum.php?s={artist}&a={title}", #patreon
 		"response_type":"json",
 		#"response_parse_tree_track": ["tracks",0,"astrArtistThumb"],
 		"response_parse_tree_artist": ["artists",0,"strArtistThumb"],
 		"required_settings": ["api_key"],
 	}
 
-	def get_image_track(self,artist):
+	def get_image_track(self,track):
+		return None
+
+	def get_image_album(self,album):
 		return None

maloja/thirdparty/deezer.py

@@ -8,10 +8,17 @@ class Deezer(MetadataInterface):
 	}
 
 	metadata = {
-		"trackurl": "https://api.deezer.com/search?q={artist}%20{title}",
+		#"trackurl": "https://api.deezer.com/search?q={artist}%20{title}",
 		"artisturl": "https://api.deezer.com/search?q={artist}",
+		"albumurl": "https://api.deezer.com/search?q={artist}%20{title}",
 		"response_type":"json",
-		"response_parse_tree_track": ["data",0,"album","cover_medium"],
+		#"response_parse_tree_track": ["data",0,"album","cover_medium"],
 		"response_parse_tree_artist": ["data",0,"artist","picture_medium"],
+		"response_parse_tree_album": ["data",0,"album","cover_medium"],
 		"required_settings": [],
 	}
+
+	def get_image_track(self,track):
+		return None
+		# we can use the album pic from the track search,
+		# but should do so via maloja logic

maloja/thirdparty/lastfm.py

@@ -22,15 +22,22 @@ class LastFM(MetadataInterface, ProxyScrobbleInterface):
 		"activated_setting": "SCROBBLE_LASTFM"
 	}
 	metadata = {
+		#"artisturl": "https://ws.audioscrobbler.com/2.0/?method=artist.getinfo&artist={artist}&api_key={apikey}&format=json"
 		"trackurl": "https://ws.audioscrobbler.com/2.0/?method=track.getinfo&track={title}&artist={artist}&api_key={apikey}&format=json",
+		"albumurl": "https://ws.audioscrobbler.com/2.0/?method=album.getinfo&api_key={apikey}&artist={artist}&album={title}&format=json",
 		"response_type":"json",
 		"response_parse_tree_track": ["track","album","image",-1,"#text"],
+		# technically just the album artwork, but we use it for now
+		#"response_parse_tree_artist": ["artist","image",-1,"#text"],
+		"response_parse_tree_album": ["album","image",-1,"#text"],
 		"required_settings": ["apikey"],
 	}
 
 	def get_image_artist(self,artist):
 		return None
-		# lastfm doesn't provide artist images
+		# lastfm still provides that endpoint with data,
+		# but doesn't provide actual images
+
 
 	def proxyscrobble_parse_response(self,data):
 		return data.attrib.get("status") == "ok" and data.find("scrobbles").attrib.get("ignored") == "0"

maloja/thirdparty/musicbrainz.py

@@ -18,7 +18,7 @@ class MusicBrainz(MetadataInterface):
 
 	metadata = {
 		"response_type":"json",
-		"response_parse_tree_track": ["images",0,"image"],
+		"response_parse_tree_track": ["images",0,"thumbnails","500"],
 		"required_settings": [],
 	}
 
@@ -26,6 +26,8 @@ class MusicBrainz(MetadataInterface):
 		return None
 		# not supported
 
+	def get_image_album(self,album):
+		return None
 
 	def get_image_track(self,track):
 		self.lock.acquire()

maloja/thirdparty/spotify.py

@@ -15,9 +15,11 @@ class Spotify(MetadataInterface):
 
 	metadata = {
 		"trackurl": "https://api.spotify.com/v1/search?q=artist:{artist}%20track:{title}&type=track&access_token={token}",
+		"albumurl": "https://api.spotify.com/v1/search?q=artist:{artist}%album:{title}&type=album&access_token={token}",
 		"artisturl": "https://api.spotify.com/v1/search?q=artist:{artist}&type=artist&access_token={token}",
 		"response_type":"json",
-		"response_parse_tree_track": ["tracks","items",0,"album","images",0,"url"],
+		"response_parse_tree_track": ["tracks","items",0,"album","images",0,"url"], # use album art
+		"response_parse_tree_album": ["albums","items",0,"images",0,"url"],
 		"response_parse_tree_artist": ["artists","items",0,"images",0,"url"],
 		"required_settings": ["apiid","secret"],
 	}

maloja/upgrade.py

@@ -11,6 +11,9 @@ from .pkg_global.conf import data_dir, dir_settings
 from .apis import _apikeys
 
 
+from .database.sqldb import get_maloja_info, set_maloja_info
+
+
 # Dealing with old style tsv files - these should be phased out everywhere
 def read_tsvs(path,types):
 	result = []
@@ -40,7 +43,7 @@ def upgrade_apikeys():
 		except Exception:
 			pass
 
-
+# v2 to v3 iupgrade
 def upgrade_db(callback_add_scrobbles):
 
 	oldfolder = os.path.join(dir_settings['state'],"scrobbles")
@@ -88,3 +91,13 @@ def upgrade_db(callback_add_scrobbles):
 				callback_add_scrobbles(scrobblelist)
 				os.rename(os.path.join(oldfolder,sf),os.path.join(newfolder,sf))
 			log("Done!",color='yellow')
+
+
+# 3.2 album support
+def parse_old_albums():
+	setting_name = "db_upgrade_albums"
+	if get_maloja_info([setting_name]).get(setting_name):
+		pass
+	else:
+		pass
+		#set_maloja_info({setting_name:True})

maloja/web/jinja/abstracts/base.jinja

@@ -8,12 +8,16 @@
 
 		<title>{% block title %}{% endblock %}</title>
 		<meta name="description" content='Maloja is a self-hosted music scrobble server.' />
+		<link rel="icon" type="image/x-icon" href="/favicon.ico" />
 
 		<meta name="color-scheme" content="dark" />
 		<meta name="darkreader" content="wat" />
 
 		<link rel="stylesheet" href="/maloja.css" />
 		<link rel="stylesheet" href="/static/css/themes/{{ settings.theme }}.css" />
+		{% for cssf in conf.custom_css_files %}
+			<link rel="stylesheet" href="/static_custom/css/{{ cssf }}" />
+		{% endfor %}
 
 		<script src="/search.js"></script>
 		<script src="/neopolitan.js"></script>
@@ -76,7 +80,11 @@
 					</table>
 					<br/><br/>
 					<span>Tracks</span>
-					<table class="searchresults_tracks" id="searchresults_tracks">
+					<table class="searchresults_tracks searchresults_extrainfo" id="searchresults_tracks">
+					</table>
+					<br/><br/>
+					<span>Albums</span>
+					<table class="searchresults_albums searchresults_extrainfo" id="searchresults_albums">
 					</table>
 				</div>
 			</div>
@@ -93,5 +101,10 @@
 		</div>
 
 
+    <!-- Load script as late as possible so DOM renders first -->
+    <script src="/lazyload17-8-2.min.js"></script>
+    <script>
+        var lazyLoadInstance = new LazyLoad({});
+    </script>
 	</body>
 </html>

maloja/web/jinja/admin_manual.jinja

@@ -16,7 +16,7 @@
 			<td style="padding-right:7px;">
 				Artists:
 			</td><td id="artists_td">
-				<input placeholder='Separate with Enter' class='simpleinput' id='artists' onKeydown='keyDetect(event)' />
+				<input placeholder='Separate with Enter' class='simpleinput' id='artists' onKeydown='keyDetect(event)' onblur='addEnteredArtist()' />
 			</td>
 		</tr>
 		<tr>

maloja/web/jinja/admin_overview.jinja

@@ -83,10 +83,10 @@
 
 	Backup your data.<br/><br/>
 
-	<a href="/apis/mlj_1/backup" download="maloja_backup.tar.gz">
+	<a class="hidelink" href="/apis/mlj_1/backup" download="maloja_backup.tar.gz">
 		<button type="button">Backup</button>
 	</a>
-	<a href="/apis/mlj_1/export" download="maloja_export.json">
+	<a class="hidelink" href="/apis/mlj_1/export" download="maloja_export.json">
 		<button type="button">Export</button>
 	</a>
 

maloja/web/jinja/admin_setup.jinja

@@ -90,7 +90,7 @@
 
 	<h2>Set up some rules</h2>
 
-	After you've scrobbled for a bit, you might want to check the <a class="textlink" href="/admin_issues">Issues page</a> to see if you need to set up some rules. You can also manually add rules in your server's "rules" directory - just add your own .tsv file and read the instructions on how to declare a rule.
+	You can add some rules in your server's "rules" directory - just add your own .tsv file and read the instructions on how to declare a rule.
 	<br/><br/>
 
 	You can also set up some predefined rulesets right away!

maloja/web/jinja/album.jinja

@@ -0,0 +1,163 @@
+{% extends "abstracts/base.jinja" %}
+{% block title %}Maloja - {{ info.album.albumtitle }}{% endblock %}
+
+{% import 'snippets/links.jinja' as links %}
+
+{% block scripts %}
+	<script src="/rangeselect.js"></script>
+	<script src="/edit.js"></script>
+{% endblock %}
+
+{% set album = filterkeys.album %}
+{% set info = dbc.album_info({'album':album}) %}
+
+{% set initialrange ='month' %}
+
+
+{% set encodedalbum = mlj_uri.uriencode({'album':album}) %}
+
+
+{% block icon_bar %}
+	{% if adminmode %}
+		{% include 'icons/edit.jinja' %}
+		{% include 'icons/merge.jinja' %}
+		{% include 'icons/merge_mark.jinja' %}
+		{% include 'icons/merge_cancel.jinja' %}
+		<script>showValidMergeIcons();</script>
+	{% endif %}
+{% endblock %}
+
+{% block content %}
+
+<script>
+	const entity_id = {{ info.id }};
+	const entity_type = 'album';
+	const entity_name = {{ album.albumtitle | tojson }};
+</script>
+
+
+{% import 'partials/awards_album.jinja' as awards %}
+
+
+<table class="top_info">
+	<tr>
+		<td class="image">
+			{% if adminmode %}
+			<div
+				class="changeable-image" data-uploader="b64=>upload('{{ encodedalbum }}',b64)"
+				style="background-image:url('{{ images.get_album_image(info.album) }}');"
+				title="Drag & Drop to upload new image"
+			></div>
+			{% else %}
+			<div style="background-image:url('{{ images.get_album_image(info.album) }}');">
+			</div>
+			{% endif %}
+		</td>
+		<td class="text">
+			<span>{{ links.links(album.artists) }}</span><br/>
+			<h1 id="main_entity_name" class="headerwithextra">{{ info.album.albumtitle | e }}</h1>
+				{# awards.certs(album) #}
+				<span class="rank"><a href="/charts_albums?max=100">#{{ info.position }}</a></span>
+				<br/>
+
+			<p class="stats">
+				<a href="{{ mlj_uri.create_uri("/scrobbles",filterkeys) }}">{{ info['scrobbles'] }} Scrobbles</a>
+			</p>
+
+
+
+
+
+			{{ awards.medals(info) }}
+			{{ awards.topweeks(info) }}
+
+
+		</td>
+	</tr>
+</table>
+
+<h2><a href='{{ mlj_uri.create_uri("/charts_tracks",filterkeys) }}'>Top Tracks</a></h2>
+
+
+{% with amountkeys={"perpage":15,"page":0} %}
+{% include 'partials/charts_tracks.jinja' %}
+{% endwith %}
+
+<br/>
+
+
+<table class="twopart">
+
+	<tr>
+		<td>
+			<h2 class="headerwithextra"><a href='{{ mlj_uri.create_uri("/pulse",filterkeys) }}'>Pulse</a></h2>
+			<br/>
+			{% for r in xranges %}
+			<span
+				onclick="showRangeManual('pulse','{{ r.identifier }}')"
+				class="stat_selector_pulse selector_pulse_{{ r.identifier }}"
+				style="{{ 'opacity:0.5;' if initialrange==r.identifier else '' }}">
+				{{ r.localisation }}
+			</span>
+			{% if not loop.last %}|{% endif %}
+			{% endfor %}
+
+      		<br/><br/>
+
+			{% for r in xranges %}
+
+			<span
+				class="stat_module_pulse pulse_{{ r.identifier }}"
+				style="{{ 'display:none;' if initialrange!=r.identifier else '' }}"
+				>
+
+			{% with limitkeys={"since":r.firstrange},delimitkeys={'step':r.identifier,'trail':1} %}
+			{% include 'partials/pulse.jinja' %}
+			{% endwith %}
+			</span>
+
+			{% endfor %}
+		</td>
+		<td>
+			<!-- We use the same classes / function calls here because we want it to switch together with pulse -->
+			<h2 class="headerwithextra"><a href='{{ mlj_uri.create_uri("/performance",filterkeys) }}'>Performance</a></h2>
+			<br/>
+			{% for r in xranges %}
+				<span
+					onclick="showRangeManual('pulse','{{ r.identifier }}')"
+					class="stat_selector_pulse selector_pulse_{{ r.identifier }}"
+					style="{{ 'opacity:0.5;' if initialrange==r.identifier else '' }}">
+					{{ r.localisation }}
+				</span>
+				{% if not loop.last %}|{% endif %}
+			{% endfor %}
+
+      		<br/><br/>
+
+			{% for r in xranges %}
+
+			<span
+				class="stat_module_pulse pulse_{{ r.identifier }}"
+				style="{{ 'display:none;' if initialrange!=r.identifier else '' }}"
+				>
+
+			{% with limitkeys={"since":r.firstrange},delimitkeys={'step':r.identifier,'trail':1} %}
+			{% include 'partials/performance.jinja' %}
+			{% endwith %}
+			</span>
+
+			{% endfor %}
+
+		</td>
+	</tr>
+</table>
+
+
+<h2><a href='{{ mlj_uri.create_uri("/scrobbles",filterkeys) }}'>Last Scrobbles</a></h2>
+
+{% with amountkeys = {"perpage":15,"page":0} %}
+{% include 'partials/scrobbles.jinja' %}
+{% endwith %}
+
+
+{% endblock %}

maloja/web/jinja/artist.jinja

@@ -53,16 +53,17 @@
 			{% if adminmode %}
 			<div
 				class="changeable-image" data-uploader="b64=>upload('{{ encodedartist }}',b64)"
-				style="background-image:url('{{ images.get_artist_image(artist) }}');"
+				style="background-image:url('{{ images.get_artist_image(info.artist) }}');"
+				title="Drag & Drop to upload new image"
 			></div>
 			{% else %}
-			<div style="background-image:url('{{ images.get_artist_image(artist) }}');">
+			<div style="background-image:url('{{ images.get_artist_image(info.artist) }}');">
 			</div>
 			{% endif %}
 		</td>
 		<td class="text">
-			<h1 id="main_entity_name" class="headerwithextra">{{ info.artist }}</h1>
-				{% if competes %}<span class="rank"><a href="/charts_artists?max=100">#{{ info.position }}</a></span>{% endif %}
+			<h1 id="main_entity_name" class="headerwithextra">{{ info.artist | e }}</h1>
+				{% if competes and info['scrobbles']>0 %}<span class="rank"><a href="/charts_artists?max=100">#{{ info.position }}</a></span>{% endif %}
 				<br/>
 				{% if competes and included %}
 					<span>associated: {{ links.links(included) }}</span>
@@ -89,13 +90,33 @@
 </table>
 
 
-<h2><a href='{{ mlj_uri.create_uri("/charts_tracks",filterkeys) }}'>Top Tracks</a></h2>
+{% set albums_info = dbc.get_albums_artist_appears_on(filterkeys,limitkeys) %}
+{% set ownalbums = albums_info.own_albums %}
+{% set otheralbums = albums_info.appears_on %}
 
+{% if ownalbums or otheralbums %}
+
+{% if settings['ALBUM_SHOWCASE'] %}
+	<h2><a href='{{ mlj_uri.create_uri("/charts_albums",filterkeys) }}'>Albums</a></h2>
+	{% include 'partials/album_showcase.jinja' %}
+{% else %}
+	<h2><a href='{{ mlj_uri.create_uri("/charts_albums",filterkeys) }}'>Top Albums</a></h2>
+
+	{% with amountkeys={"perpage":15,"page":0} %}
+	{% include 'partials/charts_albums.jinja' %}
+	{% endwith %}
+{% endif %}
+
+{% endif %}
+
+{% if info['scrobbles']>0 %}
+<h2><a href='{{ mlj_uri.create_uri("/charts_tracks",filterkeys) }}'>Top Tracks</a></h2>
 
 {% with amountkeys={"perpage":15,"page":0} %}
 {% include 'partials/charts_tracks.jinja' %}
 {% endwith %}
 
+
 <br/>
 
 <table class="twopart">
@@ -173,5 +194,6 @@
 {% with amountkeys = {"perpage":15,"page":0} %}
 {% include 'partials/scrobbles.jinja' %}
 {% endwith %}
+{% endif %}
 
 {% endblock %}

maloja/web/jinja/charts_albums.jinja

@@ -0,0 +1,52 @@
+{% extends "abstracts/base.jinja" %}
+{% block title %}Maloja - Album Charts{% endblock %}
+
+{% import 'snippets/links.jinja' as links %}
+{% import 'snippets/filterdescription.jinja' as filterdesc %}
+
+{% block scripts %}
+	<script src="/datechange.js" async></script>
+{% endblock %}
+
+{% set charts = dbc.get_charts_albums(filterkeys,limitkeys) %}
+{% set pages = math.ceil(charts.__len__() / amountkeys.perpage) %}
+{% if charts[0] is defined %}
+	{% set topalbum = charts[0].album %}
+	{% set img = images.get_album_image(topalbum) %}
+{% else %}
+	{% set img = "/favicon.png" %}
+{% endif %}
+
+
+{% block content %}
+
+<table class="top_info">
+	<tr>
+		<td class="image">
+			<div style="background-image:url('{{ img }}')"></div>
+		</td>
+		<td class="text">
+			<h1>Album Charts</h1><a href="/top_albums"><span>View #1 Albums</span></a><br/>
+			{{ filterdesc.desc(filterkeys,limitkeys) }}
+			<br/><br/>
+			{% with delimitkeys = {} %}
+			{% include 'snippets/timeselection.jinja' %}
+			{% endwith %}
+
+		</td>
+	</tr>
+</table>
+
+{% if settings['CHARTS_DISPLAY_TILES'] %}
+	{% include 'partials/charts_albums_tiles.jinja' %}
+	<br/><br/>
+{% endif %}
+
+{% with compare=true %}
+{% include 'partials/charts_albums.jinja' %}
+{% endwith %}
+
+{% import 'snippets/pagination.jinja' as pagination %}
+{{ pagination.pagination(filterkeys,limitkeys,delimitkeys,amountkeys,pages) }}
+
+{% endblock %}

maloja/web/jinja/charts_artists.jinja

@@ -1,6 +1,8 @@
 {% extends "abstracts/base.jinja" %}
 {% block title %}Maloja - Artist Charts{% endblock %}
 
+{% import 'snippets/filterdescription.jinja' as filterdesc %}
+
 {% block scripts %}
 	<script src="/datechange.js" async></script>
 {% endblock %}
@@ -25,7 +27,7 @@
 		</td>
 		<td class="text">
 			<h1>Artist Charts</h1><a href="/top_artists"><span>View #1 Artists</span></a><br/>
-			<span>{{ limitkeys.timerange.desc(prefix=True) }}</span>
+			{{ filterdesc.desc(filterkeys,limitkeys) }}
 			<br/><br/>
 			{% with delimitkeys = {} %}
 			{% include 'snippets/timeselection.jinja' %}

maloja/web/jinja/charts_tracks.jinja

@@ -2,6 +2,7 @@
 {% block title %}Maloja - Track Charts{% endblock %}
 
 {% import 'snippets/links.jinja' as links %}
+{% import 'snippets/filterdescription.jinja' as filterdesc %}
 
 {% block scripts %}
 	<script src="/datechange.js" async></script>
@@ -26,8 +27,7 @@
 		</td>
 		<td class="text">
 			<h1>Track Charts</h1><a href="/top_tracks"><span>View #1 Tracks</span></a><br/>
-			{% if filterkeys.get('artist') is not none %}by {{ links.link(filterkeys.get('artist')) }}{% endif %}
-			<span>{{ limitkeys.timerange.desc(prefix=True) }}</span>
+			{{ filterdesc.desc(filterkeys,limitkeys) }}
 			<br/><br/>
 			{% with delimitkeys = {} %}
 			{% include 'snippets/timeselection.jinja' %}

maloja/web/jinja/partials/album_showcase.jinja

@@ -0,0 +1,78 @@
+{% import 'snippets/links.jinja' as links %}
+
+{% set info = dbc.get_albums_artist_appears_on(filterkeys,limitkeys) %}
+{% set ownalbums = info.own_albums %}
+{% set otheralbums = info.appears_on %}
+
+<div id="showcase_container">
+
+{% for album in ownalbums %}
+<table class="album">
+	<tr><td>&nbsp</td></tr>
+	<tr><td>
+		<a href="{{ links.url(album) }}">
+			<div class="lazy" data-bg="{{ images.get_album_image(album) }}"'></div>
+		</a>
+	</td></tr>
+	<tr><td>
+		<span class="album_artists">{{ links.links(album.artists) }}</span><br/>
+		<span class="album_title">{{ links.link(album) }}</span>
+	</td></tr>
+
+</table>
+{% endfor %}
+
+{% for album in otheralbums %}
+<table class="album">
+	<tr><td>Appears on</td></tr>
+	<tr><td>
+		<a href="{{ links.url(album) }}">
+			<div class="lazy" data-bg="{{ images.get_album_image(album) }}"'></div>
+		</a>
+	</td></tr>
+	<tr><td>
+		<span class="album_artists">{{ links.links(album.artists) }}</span><br/>
+		<span class="album_title">{{ links.link(album) }}</span>
+	</td></tr>
+
+</table>
+{% endfor %}
+
+<!--
+<table class="album_showcase">
+<tr>
+{% for album in ownalbums %}<td></td>{% endfor %}
+{% if ownalbums and otheralbums%}<td class="album_separator_column"></td>{% endif %}
+{% for album in otheralbums %}<td>Appears on</td>{% endfor %}
+</tr>
+
+<tr>
+{% for album in ownalbums %}
+	<td>
+		<a href="{{ links.url(album) }}">
+			<div class="lazy" data-bg="{{ images.get_album_image(album) }}"'></div>
+		</a>
+	</td>
+{% endfor %}
+{% if ownalbums and otheralbums%}<td class="album_separator_column"></td>{% endif %}
+{% for album in otheralbums %}
+	<td class="album_appearon">
+		<a href="{{ links.url(album) }}">
+			<div class="lazy" data-bg="{{ images.get_album_image(album) }}"'></div>
+		</a>
+	</td>
+{% endfor %}
+</tr>
+
+<tr>
+{% for album in ownalbums %}
+	<td>{{ album.albumtitle }}</td>
+{% endfor %}
+{% if ownalbums and otheralbums%}<td class="album_separator_column"></td>{% endif %}
+{% for album in otheralbums %}
+	<td>{{ album.albumtitle }}</td>
+{% endfor %}
+</tr>
+</table>
+-->
+</div>

maloja/web/jinja/partials/awards_album.jinja

@@ -0,0 +1,42 @@
+{% macro medals(info) %}
+
+<!-- MEDALS -->
+{% for year in info.medals.gold -%}
+<a title="Best Album in {{ year }}" class="hidelink medal shiny gold" href='/charts_albums?max=50&amp;in={{ year }}'>
+	<span>{{ year }}</span>
+</a>
+{%- endfor %}
+{% for year in info.medals.silver -%}
+<a title="Second best Album in {{ year }}" class="hidelink medal shiny silver" href='/charts_albums?max=50&amp;in={{ year }}'>
+	<span>{{ year }}</span>
+</a>
+{%- endfor %}
+{% for year in info.medals.bronze -%}
+<a title="Third best Album in {{ year }}" class="hidelink medal shiny bronze" href='/charts_albums?max=50&amp;in={{ year }}'>
+	<span>{{ year }}</span>
+</a>
+{%- endfor %}
+
+{%- endmacro %}
+
+
+
+
+
+
+
+
+{% macro topweeks(info) %}
+
+{% set encodedalbum = mlj_uri.uriencode({'album':info.album}) %}
+
+<!-- TOPWEEKS -->
+<span>
+	{% if info.topweeks > 0 %}
+	<a title="{{ info.topweeks }} weeks on #1" href="/performance?{{ encodedalbum }}&step=week">
+		<img class="star" src="/media/star.png" />{{ info.topweeks }}
+	</a>
+	{% endif %}
+</span>
+
+{%- endmacro %}

maloja/web/jinja/partials/charts_albums.jinja

@@ -0,0 +1,56 @@
+{% import 'snippets/links.jinja' as links %}
+{% import 'snippets/entityrow.jinja' as entityrow %}
+
+{% if charts is undefined %}
+	{% set charts = dbc.get_charts_albums(filterkeys,limitkeys) %}
+{% endif %}
+{% if compare %}
+	{% if compare is true %}
+		{% set compare = limitkeys.timerange.next(step=-1) %}
+		{% if compare is none %}{% set compare = False %}{% endif %}
+	{% endif %}
+	{% if compare %}
+		{% set prevalbums = dbc.get_charts_albums(filterkeys,{'timerange':compare}) %}
+
+		{% set lastranks = {} %}
+		{% for t in prevalbums %}
+			{% if lastranks.update({"|".join(t.album.artists or [])+"||"+t.album.albumtitle:t.rank}) %}{% endif %}
+		{% endfor %}
+
+		{% for t in charts %}
+			{% if "|".join(t.album.artists or [])+"||"+t.album.albumtitle in lastranks %}
+				{% if t.update({'last_rank':lastranks["|".join(t.album.artists or [])+"||"+t.album.albumtitle]}) %}{% endif %}
+			{% endif %}
+		{% endfor %}
+	{% endif %}
+{% endif %}
+
+{% set firstindex = amountkeys.page * amountkeys.perpage %}
+{% set lastindex = firstindex + amountkeys.perpage %}
+
+{% set maxbar = charts[0]['scrobbles'] if charts != [] else 0 %}
+<table class='list'>
+	{% for e in charts %}
+		{% if loop.index0 >= firstindex and loop.index0 < lastindex %}
+		<tr>
+			<!-- Rank -->
+			<td class="rank">{%if loop.changed(e.scrobbles) %}#{{ e.rank }}{% endif %}</td>
+			<!-- Rank change -->
+			{% if compare %}
+				{% if e.last_rank is undefined %}<td class='rankup' title='New'>🆕</td>
+				{% elif e.last_rank < e.rank %}<td class='rankdown' title='Down from #{{ e.last_rank }}'>↘</td>
+				{% elif e.last_rank > e.rank %}<td class='rankup' title='Up from #{{ e.last_rank }}'>↗</td>
+				{% elif e.last_rank == e.rank %}<td class='ranksame' title='Unchanged'>➡</td>
+				{% endif %}
+			{% endif %}
+
+			<!-- artist -->
+			{{ entityrow.row(e['album']) }}
+
+			<!-- scrobbles -->
+			<td class="amount">{{ links.link_scrobbles([{'album':e.album,'timerange':limitkeys.timerange}],amount=e['scrobbles']) }}</td>
+			<td class="bar">{{ links.link_scrobbles([{'album':e.album,'timerange':limitkeys.timerange}],percent=e['scrobbles']*100/maxbar) }}</td>
+		</tr>
+		{% endif %}
+	{% endfor %}
+</table>

maloja/web/jinja/partials/charts_albums_tiles.jinja

@@ -0,0 +1,45 @@
+{% import 'snippets/links.jinja' as links %}
+
+
+{% if charts is undefined %}
+	{% set charts = dbc.get_charts_albums(limitkeys) %}
+{% endif %}
+
+{% set charts_14 = charts | fixlength(14) %}
+{% set charts_cycler = cycler(*charts_14) %}
+
+
+
+<table class="tiles_top"><tr>
+{% for segment in range(3) %}
+	{% if charts_14[0] is none and loop.first %}
+	{% include 'icons/nodata.jinja' %}
+	{% else %}
+	<td>
+	{% set segmentsize = segment+1 %}
+	<table class="tiles_{{ segmentsize }}x{{ segmentsize }} tiles_sub">
+		{% for row in range(segmentsize) -%}
+			<tr>
+				{% for col in range(segmentsize) %}
+					{% set entry = charts_cycler.next() %}
+					{% if entry is not none %}
+						{% set album = entry.album %}
+						{% set rank = entry.rank %}
+						<td>
+							<a href="{{ links.url(album) }}">
+							<div class="lazy" data-bg="{{ images.get_album_image(album) }}"'>
+								<span class='stats'>#{{ rank }}</span> <span>{{ album.albumtitle }}</span>
+							</div>
+							</a>
+						</td>
+					{% else -%}
+						<td></td>
+					{%- endif -%}
+				{%- endfor -%}
+			</tr>
+		{%- endfor -%}
+	</table>
+	</td>
+	{% endif %}
+{% endfor %}
+</tr></table>

maloja/web/jinja/partials/charts_artists_tiles.jinja

@@ -27,7 +27,7 @@
 						{% set rank = entry.rank %}
 						<td>
 							<a href="{{ links.url(artist) }}">
-							<div style='background-image:url("{{ images.get_artist_image(artist) }}")'>
+							<div class="lazy" data-bg="{{ images.get_artist_image(artist) }}"'>
 								<span class='stats'>#{{ rank }}</span> <span>{{ artist }}</span>
 							</div>
 							</a>

maloja/web/jinja/partials/charts_tracks_tiles.jinja

@@ -26,7 +26,7 @@
 						{% set rank = entry.rank %}
 						<td>
 							<a href="{{ links.url(track) }}">
-							<div style='background-image:url("{{ images.get_track_image(track) }}")'>
+							<div class="lazy" data-bg="{{ images.get_track_image(track) }}")'>
 								<span class='stats'>#{{ rank }}</span> <span>{{ track.title }}</span>
 							</div>
 							</a>

maloja/web/jinja/partials/top_albums.jinja

@@ -0,0 +1,30 @@
+{% import 'snippets/links.jinja' as links %}
+{% import 'snippets/entityrow.jinja' as entityrow %}
+
+{% set ranges = dbc.get_top_albums(filterkeys,limitkeys,delimitkeys) %}
+
+{% set maxbar = ranges|map(attribute="scrobbles")|max|default(1) %}
+{% if maxbar < 1 %}{% set maxbar = 1 %}{% endif %}
+
+<table class="list">
+	{% for e in ranges %}
+
+	{% set thisrange = e.range %}
+	{% set album = e.album %}
+	<tr>
+		<td>{{ thisrange.desc() }}</td>
+
+		{% if album is none %}
+			<td><div></div></td>
+			<td class='stats'>n/a</td>
+			<td class='amount'>0</td>
+			<td class='bar'></td>
+		{% else %}
+		{{ entityrow.row(album) }}
+			<td class='amount'>{{ links.link_scrobbles([{'album':album,'timerange':thisrange}],amount=e.scrobbles) }}</td>
+			<td class='bar'> {{ links.link_scrobbles([{'album':album,'timerange':thisrange}],percent=e.scrobbles*100/maxbar) }}</td>
+		{% endif %}
+
+	</tr>
+	{% endfor %}
+</table>

maloja/web/jinja/snippets/entityrow.jinja

@@ -2,13 +2,19 @@
 
 {% import 'snippets/links.jinja' as links %}
 
-{% if entity is mapping and 'artists' in entity %}
+{% if entity is mapping and 'title' in entity %}
 	{% set img = images.get_track_image(entity) %}
+{% elif entity is mapping and 'albumtitle' in entity %}
+	{% set img = images.get_album_image(entity) %}
 {% else %}
 	{% set img = images.get_artist_image(entity) %}
 {% endif %}
 
-<td class='icon'><div style="background-image:url('{{ img }}')"></div></td>
+<td class='icon'>
+{% if settings['DISPLAY_ART_ICONS'] %}
+    <div class="lazy" data-bg="{{ img }}"></div>
+{% endif %}
+</td>
 {% if entity is mapping and 'artists' in entity %}
 	{% if settings['TRACK_SEARCH_PROVIDER'] %}
 		<td class='searchProvider'>{{ links.link_search(entity) }}</td>
@@ -16,6 +22,10 @@
 	<td class='track'>
 		<span class='artist_in_trackcolumn'>{{ links.links(entity.artists) }}</span> – {{ links.link(entity) }}
 	</td>
+{% elif entity is mapping and 'albumtitle' in entity %}
+	<td class='album'>
+		<span class='artist_in_trackcolumn'>{{ links.links(entity.artists) }}</span> – {{ links.link(entity) }}
+	</td>
 {% else %}
 	<td class='artist'>{{ links.link(entity) }}
 	{% if counting != [] %}

maloja/web/jinja/snippets/filterdescription.jinja

@@ -7,6 +7,9 @@
 	{% elif filterkeys.get('track') is not none %}
 		of {{ links.link(filterkeys.get('track')) }}
 		by {{ links.links(filterkeys["track"]["artists"]) }}
+	{% elif filterkeys.get('album') is not none %}
+		of {{ links.link(filterkeys.get('album')) }}
+		by {{ links.links(filterkeys["album"]["artists"]) }}
 	{% endif %}
 	{{ limitkeys.timerange.desc(prefix=True) }}
 

maloja/web/jinja/snippets/links.jinja

@@ -1,23 +1,29 @@
 {% macro link(entity) -%}
-	{% if entity is mapping and 'artists' in entity %}
-		{% set name = entity.title %}
+	{% if entity is mapping and 'title' in entity or 'albumtitle' in entity %}
+		{% set name = entity.title or entity.albumtitle %}
 	{% else %}
 		{% set name = entity %}
 	{% endif %}
 
-	<a href="{{ url(entity) }}">{{ name }}</a>
+	<a href="{{ url(entity) }}">{{ name | e }}</a>
 {%- endmacro %}
 
 {% macro links(entities) -%}
-	{% for entity in entities -%}
-		{{ link(entity) }}{{ ", " if not loop.last }}
-	{%- endfor %}
+	{% if entities is none or entities == [] %}
+		{{ settings["DEFAULT_ALBUM_ARTIST"] }}
+	{% else %}
+		{% for entity in entities -%}
+			{{ link(entity) }}{{ ", " if not loop.last }}
+		{%- endfor %}
+	{% endif %}
 {%- endmacro %}
 
 
 
 {% macro url(entity) %}
-	{% if entity is mapping and 'artists' in entity -%}
+  {% if entity is mapping and 'albumtitle' in entity -%}
+	  {{ mlj_uri.create_uri("/album",{'album':entity}) }}
+	{% elif entity is mapping and 'artists' in entity -%}
 		{{ mlj_uri.create_uri("/track",{'track':entity}) }}
 	{%- else -%}
 		{{ mlj_uri.create_uri("/artist",{'artist':entity}) }}
@@ -43,6 +49,8 @@
 
 	{% if 'track' in filterkeys %}
 		{% set url = mlj_uri.create_uri("/charts_tracks",{'timerange':timerange}) %}
+	{% elif 'album' in filterkeys %}
+		{% set url = mlj_uri.create_uri("/charts_albums",{'timerange':timerange}) %}
 	{% elif 'artist' in filterkeys %}
 		{% set url = mlj_uri.create_uri("/charts_artists",{'timerange':timerange}) %}
 	{% endif %}

maloja/web/jinja/snippets/pagination.jinja

@@ -10,8 +10,7 @@
 	{% if pages > 1 %}
 		{% if page > 1 %}
 			<a href='{{ mlj_uri.create_uri("",filterkeys,limitkeys,delimitkeys,amountkeys,{'page':0}) }}'>
-				<span class='stat_selector'>1</span>
-			</a> |
+				<span class='stat_selector'>1</span></a> |
 		{% endif %}
 
 		{% if page > 2 %}
@@ -20,8 +19,7 @@
 
 		{% if page > 0 %}
 			<a href='{{ mlj_uri.create_uri("",filterkeys,limitkeys,delimitkeys,amountkeys,{'page':page-1}) }}'>
-				<span class='stat_selector'>{{ page }}</span>
-			</a> «
+				<span class='stat_selector'>{{ page }}</span></a> «
 		{% endif %}
 
 		<span style='opacity:0.5;' class='stat_selector'>
@@ -30,8 +28,7 @@
 
 		{% if page < pages-1 %}
 			 » <a href='{{ mlj_uri.create_uri("",filterkeys,limitkeys,delimitkeys,amountkeys,{'page':page+1}) }}'>
-				 <span class='stat_selector'>{{ page+2 }}</span>
-			 </a>
+				 <span class='stat_selector'>{{ page+2 }}</span></a>
 		{% endif %}
 
 		{% if page < pages-3 %}
@@ -40,8 +37,7 @@
 
 		{% if page < pages-2 %}
 			 | <a href='{{ mlj_uri.create_uri("",filterkeys,limitkeys,delimitkeys,amountkeys,{'page':pages-1}) }}'>
-				 <span class='stat_selector'>{{ pages }}</span>
-			 </a>
+				 <span class='stat_selector'>{{ pages }}</span></a>
 		{% endif %}
 	{% endif %}
 

maloja/web/jinja/top_albums.jinja

@@ -0,0 +1,31 @@
+{% extends "abstracts/base.jinja" %}
+{% block title %}Maloja - #1 Albums{% endblock %}
+
+{% import 'snippets/filterdescription.jinja' as filterdesc %}
+
+<!-- find representative -->
+
+{% set entries = dbc.get_top_albums(filterkeys,limitkeys,delimitkeys) %}
+{% set repr = entries | find_representative('album','scrobbles') %}
+{% set img = "/favicon.png" if repr is none else images.get_album_image(repr.album) %}
+
+
+{% block content %}
+		<table class="top_info">
+			<tr>
+				<td class="image">
+					<div style="background-image:url('{{ img }}')"></div>
+				</td>
+				<td class="text">
+					<h1>#1 Albums</h1><br/>
+					{{ filterdesc.desc(filterkeys,limitkeys) }}
+
+					<br/><br/>
+					{% include 'snippets/timeselection.jinja' %}
+				</td>
+			</tr>
+		</table>
+
+		{% include 'partials/top_albums.jinja' %}
+
+{% endblock %}

maloja/web/jinja/top_artists.jinja

@@ -1,6 +1,7 @@
 {% extends "abstracts/base.jinja" %}
 {% block title %}Maloja - #1 Artists{% endblock %}
 
+{% import 'snippets/filterdescription.jinja' as filterdesc %}
 
 <!-- find representative -->
 
@@ -17,7 +18,7 @@
 				</td>
 				<td class="text">
 					<h1>#1 Artists</h1><br/>
-					<span>{{ limitkeys.timerange.desc(prefix=True) }}</span>
+					{{ filterdesc.desc(filterkeys,limitkeys) }}
 
 					<br/><br/>
 					{% include 'snippets/timeselection.jinja' %}

maloja/web/jinja/top_tracks.jinja

@@ -1,6 +1,7 @@
 {% extends "abstracts/base.jinja" %}
 {% block title %}Maloja - #1 Tracks{% endblock %}
 
+{% import 'snippets/filterdescription.jinja' as filterdesc %}
 
 <!-- find representative -->
 
@@ -17,7 +18,7 @@
 				</td>
 				<td class="text">
 					<h1>#1 Tracks</h1><br/>
-					<span>{{ limitkeys.timerange.desc(prefix=True) }}</span>
+					{{ filterdesc.desc(filterkeys,limitkeys) }}
 
 					<br/><br/>
 					{% include 'snippets/timeselection.jinja' %}

maloja/web/jinja/track.jinja

@@ -50,19 +50,23 @@
 			{% if adminmode %}
 			<div
 				class="changeable-image" data-uploader="b64=>upload('{{ encodedtrack }}',b64)"
-				style="background-image:url('{{ images.get_track_image(track) }}');"
+				style="background-image:url('{{ images.get_track_image(info.track) }}');"
+				title="Drag & Drop to upload new image"
 			></div>
 			{% else %}
-			<div style="background-image:url('{{ images.get_track_image(track) }}');">
+			<div style="background-image:url('{{ images.get_track_image(info.track) }}');">
 			</div>
 			{% endif %}
 		</td>
 		<td class="text">
 			<span>{{ links.links(track.artists) }}</span><br/>
-			<h1 id="main_entity_name" class="headerwithextra">{{ info.track.title }}</h1>
+			<h1 id="main_entity_name" class="headerwithextra">{{ info.track.title | e }}</h1>
 				{{ awards.certs(track) }}
 				<span class="rank"><a href="/charts_tracks?max=100">#{{ info.position }}</a></span>
 				<br/>
+				{% if info.track.album %}
+				from {{ links.link(info.track.album) }}<br/>
+				{% endif %}
 
 			<p class="stats">
 				{% if adminmode %}<button type="button" onclick="scrobble('{{ encodedtrack }}')">Scrobble now</button>{% endif %}

maloja/web/static/css/maloja.css

@@ -189,7 +189,7 @@ div.searchresults tr td:nth-child(2) {
 	padding-left:10px;
 }
 
-div.searchresults table.searchresults_tracks td span:nth-child(1) {
+div.searchresults table.searchresults_extrainfo td span:nth-child(1) {
 	font-size:12px;
 	color:grey;
 
@@ -716,7 +716,7 @@ table.list td.amount {
 	text-align:right;
 }
 table.list td.bar {
-	width:500px;
+	width:400px;
 	/* background-color: var(--base-color); */
 	/* Remove 5er separators for bars */
 	/*border-color:rgba(0,0,0,0)!important;*/
@@ -734,7 +734,7 @@ table.list tr:hover td.bar div {
 }
 
 table.list td.chart {
-	width:500px;
+	width:400px;
 	/* background-color: var(--base-color); */
 	/* Remove 5er separators for bars */
 	/*border-color:rgba(0,0,0,0)!important;*/
@@ -848,8 +848,11 @@ table.tiles_top td div {
 
 table.tiles_top td span {
 	background-color:rgba(0,0,0,0.7);
-	display: table-cell;
+	display: inline-block;
+	margin-top:2%;
 	padding: 3px;
+	max-width: 67%;
+	vertical-align: text-top;
 }
 table.tiles_top td a:hover {
 	text-decoration: none;
@@ -863,12 +866,12 @@ table.tiles_1x1 td {
 table.tiles_2x2 td {
 	height:50%;
 	width:50%;
-	font-size:90%
+	font-size:80%
 }
 table.tiles_3x3 td {
 	height:33.333%;
 	width:33.333%;
-	font-size:70%
+	font-size:60%
 }
 table.tiles_4x4 td {
 	font-size:50%
@@ -877,6 +880,74 @@ table.tiles_5x5 td {
 	font-size:40%
 }
 
+/* Safari fix */
+table.tiles_sub.tiles_3x3 td div {
+        min-height: 100px;
+        min-width: 100px;
+}
+table.tiles_sub.tiles_2x2 td div {
+        min-height: 150px;
+        min-width: 150px;
+}
+table.tiles_sub.tiles_1x1 td div {
+        min-height: 300px;
+        min-width: 300px;
+}
+table.tiles_sub a span {
+        overflow-wrap: anywhere;
+}
+
+
+div#showcase_container {
+		display: flex;
+		margin-top: -15px;
+		padding-bottom: 20px;
+		align-items: flex-start;
+		flex-wrap: wrap;
+}
+
+div#showcase_container table.album {
+	width: 180px;
+}
+
+div#showcase_container table.album tr td {
+	padding-left: 15px;
+	padding-right: 15px;
+}
+div#showcase_container table.album tr:nth-child(1) td {
+	height:8px;
+	opacity: 0.3;
+	text-align: center;
+}
+div#showcase_container table.album tr:nth-child(2) td {
+	height:150px;
+	padding-top:2px;
+	padding-bottom:2px;
+}
+div#showcase_container table.album tr:nth-child(3) td {
+	height:15px;
+}
+div#showcase_container div {
+	height: 150px;
+	width: 150px;
+
+	background-size: cover;
+	background-position: top;
+
+	box-shadow: 0px 0px 10px 10px rgba(0,0,0,0.5);
+}
+
+div#showcase_container table:hover div {
+	box-shadow: 0px 0px 10px 10px var(--ctrl-element-color-main);
+}
+
+div#showcase_container span.album_artists {
+	font-size: 80%;
+}
+div#showcase_container span.album_title {
+	font-weight: bold;
+}
+
 
 .summary_rank {
 	background-size:cover;

maloja/web/static/js/edit.js

@@ -104,7 +104,13 @@ function createTrackCell(trackinfo) {
 function editEntity() {
 
 	var namefield = document.getElementById('main_entity_name');
-	namefield.contentEditable = "plaintext-only";
+	try {
+		namefield.contentEditable = "plaintext-only"; // not supported by Firefox
+	}
+	catch (e) {
+		namefield.contentEditable = true;
+	}
+
 
 	namefield.addEventListener('keydown',function(e){
 		// dont allow new lines, done on enter
@@ -155,6 +161,11 @@ function doneEditing() {
 		    searchParams.set("title", newname);
 			var payload = {'id':entity_id,'title':newname}
 		}
+		else if (entity_type == 'album') {
+			var endpoint = "/apis/mlj_1/edit_album";
+		    searchParams.set("albumtitle", newname);
+			var payload = {'id':entity_id,'albumtitle':newname}
+		}
 
 		callback_func = function(req){
 			if (req.status == 200) {

maloja/web/static/js/manualscrobble.js

@@ -126,14 +126,14 @@ function searchresult_manualscrobbling() {
 		console.log(tracks);
 		for (let t of tracks) {
 			track = document.createElement("span");
-			trackstr = t["artists"].join(", ") + " - " + t["title"];
+			trackstr = t.track["artists"].join(", ") + " - " + t.track["title"];
 			tracklink = t["link"];
 			track.innerHTML = "<a href='" + tracklink + "'>" +  trackstr + "</a>";
 			row = document.createElement("tr")
 			col1 = document.createElement("td")
 			button = document.createElement("button")
 			button.innerHTML = "Scrobble!"
-			button.onclick = function(){ scrobble(t["artists"],t["title"])};
+			button.onclick = function(){ scrobble(t.track["artists"],t.track["title"])};
 			col2 = document.createElement("td")
 			row.appendChild(col1)
 			col1.appendChild(button)

maloja/web/static/js/search.js

@@ -23,11 +23,13 @@ function html_to_fragment(html) {
 
 var results_artists;
 var results_tracks;
+var results_albums;
 var searchresultwrap;
 
 window.addEventListener("DOMContentLoaded",function(){
 	results_artists = document.getElementById("searchresults_artists");
 	results_tracks = document.getElementById("searchresults_tracks");
+	results_albums = document.getElementById("searchresults_albums");
 	searchresultwrap = document.getElementById("resultwrap");
 });
 
@@ -50,8 +52,9 @@ function searchresult() {
 		// any older searches are now rendered irrelevant
 		while (searches[0] != this) { searches.splice(0,1) }
 		var result = JSON.parse(this.responseText);
-		var artists = result["artists"].slice(0,5)
-		var tracks = result["tracks"].slice(0,5)
+		var artists = result["artists"].slice(0,4)
+		var tracks = result["tracks"].slice(0,4)
+		var albums = result["albums"].slice(0,4)
 
 		while (results_artists.firstChild) {
 			results_artists.removeChild(results_artists.firstChild);
@@ -59,6 +62,9 @@ function searchresult() {
 		while (results_tracks.firstChild) {
 			results_tracks.removeChild(results_tracks.firstChild);
 		}
+		while (results_albums.firstChild) {
+			results_albums.removeChild(results_albums.firstChild);
+		}
 
 		for (var i=0;i<artists.length;i++) {
 			name = artists[i]["artist"];
@@ -68,7 +74,7 @@ function searchresult() {
 			var node = oneresult.cloneNode(true);
 			node.setAttribute("onclick","goto('" + link + "')");
 			node.children[0].style.backgroundImage = "url('" + image + "')";
-			node.children[1].children[0].innerHTML = name;
+			node.children[1].children[0].textContent = name;
 
 			results_artists.appendChild(node);
 		}
@@ -82,11 +88,26 @@ function searchresult() {
 			var node = oneresult.cloneNode(true);
 			node.setAttribute("onclick","goto('" + link + "')");
 			node.children[0].style.backgroundImage = "url('" + image + "')";
-			node.children[1].children[0].innerHTML = artists;
-			node.children[1].children[2].innerHTML = title;
+			node.children[1].children[0].textContent = artists;
+			node.children[1].children[2].textContent = title;
 
 			results_tracks.appendChild(node);
 		}
+		for (var i=0;i<albums.length;i++) {
+
+			artists = albums[i]["album"].hasOwnProperty("displayArtist") ? albums[i]["album"]["displayArtist"] : albums[i]["album"]["artists"].join(", ");
+			albumtitle = albums[i]["album"]["albumtitle"];
+			link = albums[i]["link"];
+			image = albums[i]["image"];
+
+			var node = oneresult.cloneNode(true);
+			node.setAttribute("onclick","goto('" + link + "')");
+			node.children[0].style.backgroundImage = "url('" + image + "')";
+			node.children[1].children[0].textContent = artists;
+			node.children[1].children[2].textContent = albumtitle;
+
+			results_albums.appendChild(node);
+		}
 		searchresultwrap.classList.remove("hide")
 
 	}

maloja/web/static/svg/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015-present Ionic (http://ionic.io/)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

maloja/web/static/svg/placeholder_album.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="ionicon" viewBox="0 0 512 512"><circle cx="256" cy="256" r="208" fill="none" stroke="#f5f5dc" stroke-miterlimit="10" stroke-width="32"/><circle cx="256" cy="256" r="96" fill="none" stroke="#f5f5dc" stroke-miterlimit="10" stroke-width="32"/><circle cx="256" cy="256" r="32"/></svg>

maloja/web/static/svg/placeholder_artist.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="ionicon" viewBox="0 0 512 512"><path fill="none" stroke="#f5f5dc" stroke-linecap="round" stroke-linejoin="round" stroke-width="32" d="M192 448h128M384 208v32c0 70.4-57.6 128-128 128h0c-70.4 0-128-57.6-128-128v-32M256 368v80"/><path d="M256 64a63.68 63.68 0 00-64 64v111c0 35.2 29 65 64 65s64-29 64-65V128c0-36-28-64-64-64z" fill="none" stroke="#f5f5dc" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/></svg>

maloja/web/static/svg/placeholder_track.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="ionicon" viewBox="0 0 512 512"><path d="M192 218v-6c0-14.84 10-27 24.24-30.59l174.59-46.68A20 20 0 01416 154v22" fill="none" stroke="#f5f5dc" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/><path d="M416 295.94v80c0 13.91-8.93 25.59-22 30l-22 8c-25.9 8.72-52-10.42-52-38h0a33.37 33.37 0 0123-32l51-18.15c13.07-4.4 22-15.94 22-29.85V58a10 10 0 00-12.6-9.61L204 102a16.48 16.48 0 00-12 16v226c0 13.91-8.93 25.6-22 30l-52 18c-13.88 4.68-22 17.22-22 32h0c0 27.58 26.52 46.55 52 38l22-8c13.07-4.4 22-16.08 22-30v-80" fill="none" stroke="#f5f5dc" stroke-linecap="round" stroke-linejoin="round" stroke-width="32"/></svg>

maloja/web/static/txt/robots.txt

@@ -1,2 +1,2 @@
 User-agent: *
-Disallow: *
+Disallow: /

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "malojaserver"
-version = "3.1.0"
+version = "3.1.5"
 description = "Self-hosted music scrobble database"
 readme = "./README.md"
 requires-python = ">=3.7"
@@ -21,7 +21,7 @@ classifiers = [
 dependencies = [
 	"bottle>=0.12.16",
 	"waitress>=2.1.0",
-	"doreah>=1.9.1, <2",
+	"doreah>=1.9.4, <2",
 	"nimrodel>=0.8.0",
 	"setproctitle>=1.1.10",
 	#"pyvips>=2.1.16",

requirements.txt

@@ -1,6 +1,6 @@
 bottle>=0.12.16
 waitress>=2.1.0
-doreah>=1.9.1, <2
+doreah>=1.9.4, <2
 nimrodel>=0.8.0
 setproctitle>=1.1.10
 jinja2>=3.0.0
@@ -9,4 +9,3 @@ psutil>=5.8.0
 sqlalchemy>=1.4
 python-datauri>=1.1.0
 requests>=2.27.1
-

settings.md

@@ -1,4 +1,14 @@
-Technically, each setting can be set via environment variable or the settings file - simply add the prefix `MALOJA_` for environment variables. The columns are filled according to what is reasonable, it is recommended to use the settings file where possible and not configure each aspect of your server via environment variables!
+If you wish to adjust settings in the settings.ini file, do so while the server
+	is not running in order to avoid data being overwritten.
+
+	Technically, each setting can be set via environment variable or the settings
+	file - simply add the prefix `MALOJA_` for environment variables. It is recommended
+	to use the settings file where possible and not configure each aspect of your
+	server via environment variables!
+
+	You also can specify additional settings in the files`/run/secrets/maloja.yml` or
+	`/run/secrets/maloja.ini`, as well as their values directly in files of the respective
+	name in `/run/secrets/` (e.g. `/run/secrets/lastfm_api_key`).
 
 Settings File			| Environment Variable			| Type			| Description
 ------					| ---------						| ---------		| ---------
@@ -15,16 +25,14 @@ Settings File			| Environment Variable			| Type			| Description
 `logging`	| `MALOJA_LOGGING`  | Boolean | Enable Logging
 `dev_mode`	| `MALOJA_DEV_MODE`  | Boolean | Enable developer mode
 **Network**
-`host`	| `MALOJA_HOST`  | String | Host for your server - most likely :: for IPv6 or 0.0.0.0 for IPv4
+`host`	| `MALOJA_HOST`  | String | Host for your server, e.g. '*' for dual stack, '::' for IPv6 or '0.0.0.0' for IPv4
 `port`	| `MALOJA_PORT`  | Integer | Port
 **Technical**
 `cache_expire_positive`	| `MALOJA_CACHE_EXPIRE_POSITIVE`  | Integer | Days until images are refetched
 `cache_expire_negative`	| `MALOJA_CACHE_EXPIRE_NEGATIVE`  | Integer | Days until failed image fetches are reattempted
-`use_db_cache`	| `MALOJA_USE_DB_CACHE`  | Boolean | Use DB Cache
-`cache_database_short`	| `MALOJA_CACHE_DATABASE_SHORT`  | Boolean | Use volatile Database Cache
-`cache_database_perm`	| `MALOJA_CACHE_DATABASE_PERM`  | Boolean | Use permanent Database Cache
-`db_cache_entries`	| `MALOJA_DB_CACHE_ENTRIES`  | Integer | Maximal Cache entries
-`db_max_memory`	| `MALOJA_DB_MAX_MEMORY`  | Integer | Maximal percentage of RAM that should be used by whole system before Maloja discards cache entries. Use a higher number if your Maloja runs on a dedicated instance (e.g. a container)
+`db_max_memory`	| `MALOJA_DB_MAX_MEMORY`  | Integer | RAM Usage in percent at which Maloja should no longer increase its database cache.
+`use_request_cache`	| `MALOJA_USE_REQUEST_CACHE`  | Boolean | Use request-local DB Cache
+`use_global_cache`	| `MALOJA_USE_GLOBAL_CACHE`  | Boolean | Use global DB Cache
 **Fluff**
 `scrobbles_gold`	| `MALOJA_SCROBBLES_GOLD`  | Integer | How many scrobbles a track needs to be considered 'Gold' status
 `scrobbles_platinum`	| `MALOJA_SCROBBLES_PLATINUM`  | Integer | How many scrobbles a track needs to be considered 'Platinum' status
@@ -35,24 +43,33 @@ Settings File			| Environment Variable			| Type			| Description
 `scrobble_lastfm`	| `MALOJA_SCROBBLE_LASTFM`  | Boolean | Proxy-Scrobble to Last.fm
 `lastfm_api_key`	| `MALOJA_LASTFM_API_KEY`  | String | Last.fm API Key
 `lastfm_api_secret`	| `MALOJA_LASTFM_API_SECRET`  | String | Last.fm API Secret
+`lastfm_api_sk`	| `MALOJA_LASTFM_API_SK`  | String | Last.fm API Session Key
+`lastfm_username`	| `MALOJA_LASTFM_USERNAME`  | String | Last.fm Username
+`lastfm_password`	| `MALOJA_LASTFM_PASSWORD`  | String | Last.fm Password
 `spotify_api_id`	| `MALOJA_SPOTIFY_API_ID`  | String | Spotify API ID
 `spotify_api_secret`	| `MALOJA_SPOTIFY_API_SECRET`  | String | Spotify API Secret
 `audiodb_api_key`	| `MALOJA_AUDIODB_API_KEY`  | String | TheAudioDB API Key
+`other_maloja_url`	| `MALOJA_OTHER_MALOJA_URL`  | String | Other Maloja Instance URL
+`other_maloja_api_key`	| `MALOJA_OTHER_MALOJA_API_KEY`  | String | Other Maloja Instance API Key
 `track_search_provider`	| `MALOJA_TRACK_SEARCH_PROVIDER`  | String | Track Search Provider
 `send_stats`	| `MALOJA_SEND_STATS`  | Boolean | Send Statistics
+`proxy_images`	| `MALOJA_PROXY_IMAGES`  | Boolean | Whether third party images should be downloaded and served directly by Maloja (instead of just linking their URL)
 **Database**
 `invalid_artists`	| `MALOJA_INVALID_ARTISTS`  | Set | Artists that should be discarded immediately
 `remove_from_title`	| `MALOJA_REMOVE_FROM_TITLE`  | Set | Phrases that should be removed from song titles
 `delimiters_feat`	| `MALOJA_DELIMITERS_FEAT`  | Set | Delimiters used for extra artists, even when in the title field
 `delimiters_informal`	| `MALOJA_DELIMITERS_INFORMAL`  | Set | Delimiters in informal artist strings with spaces expected around them
 `delimiters_formal`	| `MALOJA_DELIMITERS_FORMAL`  | Set | Delimiters used to tag multiple artists when only one tag field is available
+`filters_remix`	| `MALOJA_FILTERS_REMIX`  | Set | Filters used to recognize the remix artists in the title
+`parse_remix_artists`	| `MALOJA_PARSE_REMIX_ARTISTS`  | Boolean | Parse Remix Artists
 **Web Interface**
 `default_range_charts_artists`	| `MALOJA_DEFAULT_RANGE_CHARTS_ARTISTS`  | Choice | Default Range Artist Charts
 `default_range_charts_tracks`	| `MALOJA_DEFAULT_RANGE_CHARTS_TRACKS`  | Choice | Default Range Track Charts
 `default_step_pulse`	| `MALOJA_DEFAULT_STEP_PULSE`  | Choice | Default Pulse Step
 `charts_display_tiles`	| `MALOJA_CHARTS_DISPLAY_TILES`  | Boolean | Display Chart Tiles
+`display_art_icons`	| `MALOJA_DISPLAY_ART_ICONS`  | Boolean | Display Album/Artist Icons
 `discourage_cpu_heavy_stats`	| `MALOJA_DISCOURAGE_CPU_HEAVY_STATS`  | Boolean | Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!
 `use_local_images`	| `MALOJA_USE_LOCAL_IMAGES`  | Boolean | Use Local Images
-`local_image_rotate`	| `MALOJA_LOCAL_IMAGE_ROTATE`  | Integer | Local Image Rotate
 `timezone`	| `MALOJA_TIMEZONE`  | Integer | UTC Offset
 `time_format`	| `MALOJA_TIME_FORMAT`  | String | Time Format
+`theme`	| `MALOJA_THEME`  | String | Theme