Reworked image proxying / caching

maloja/globalconf.py

@@ -174,7 +174,7 @@ malojaconfig = Configuration(
 			"other_maloja_api_key":(tp.String(),								"Other Maloja Instance API Key",None),
 			"track_search_provider":(tp.String(),								"Track Search Provider",		None),
 			"send_stats":(tp.Boolean(),											"Send Statistics",				None),
-			"proxy_images":(tp.Boolean(),										"Image Proxy",					False,	"Whether third party images should be downloaded and served directly by Maloja (instead of just linking their URL)")
+			"proxy_images":(tp.Boolean(),										"Image Proxy",					True,	"Whether third party images should be downloaded and served directly by Maloja (instead of just linking their URL)")
 
 		},
 		"Database":{

maloja/server.py

@@ -19,7 +19,7 @@ from doreah import auth
 # rest of the project
 from . import database
 from .database.jinjaview import JinjaDBConnection
-from .utilities import get_track_image, get_artist_image
+from .utilities import resolve_track_image, resolve_artist_image
 from .malojauri import uri_to_internal, remove_identical
 from .globalconf import malojaconfig, data_dir
 from .jinjaenv.context import jinja_environment
@@ -162,21 +162,20 @@ def deprecated_api(pth):
 @webserver.route("/image")
 def dynamic_image():
 	keys = FormsDict.decode(request.query)
-	relevant, _, _, _, _ = uri_to_internal(keys)
-	if 'track' in relevant:
-		result = get_track_image(relevant['track'])
-	elif 'artist' in relevant:
-		result = get_artist_image(relevant['artist'])
+	if keys['type'] == 'track':
+		result = resolve_track_image(keys['id'])
+	elif keys['type'] == 'artist':
+		result = resolve_artist_image(keys['id'])
 
 	if result is None: return ""
-	if result.startswith("data:"):
+	if result['type'] == 'raw':
 		# data uris are directly served as image because a redirect to a data uri
 		# doesnt work
-		duri = datauri.DataURI(result)
+		duri = datauri.DataURI(result['value'])
 		response.content_type = duri.mimetype
 		return duri.data
 	else:
-		redirect(result,307)
+		redirect(result['value'],307)
 
 @webserver.route("/images/<pth:re:.*\\.jpeg>")
 @webserver.route("/images/<pth:re:.*\\.jpg>")

maloja/utilities/images.py

@@ -30,13 +30,15 @@ DB['artists'] = sql.Table(
 	'artists', meta,
 	sql.Column('id',sql.Integer,primary_key=True),
 	sql.Column('url',sql.String),
-	sql.Column('expire',sql.Integer)
+	sql.Column('expire',sql.Integer),
+	sql.Column('raw',sql.String)
 )
 DB['tracks'] = sql.Table(
 	'tracks', meta,
 	sql.Column('id',sql.Integer,primary_key=True),
 	sql.Column('url',sql.String),
-	sql.Column('expire',sql.Integer)
+	sql.Column('expire',sql.Integer),
+	sql.Column('raw',sql.String)
 )
 
 meta.create_all(engine)
@@ -50,8 +52,11 @@ def get_image_from_cache(id,table):
 		)
 		result = conn.execute(op).all()
 	for row in result:
-		return row.url # returns None if nonexistence cached
-	return False # no cache entry
+		if row.raw is not None:
+			return {'type':'raw','value':row.raw}
+		else:
+			return {'type':'url','value':row.url} # returns None as value if nonexistence cached
+	return None # no cache entry
 
 def set_image_in_cache(id,table,url):
 	remove_image_from_cache(id,table)
@@ -60,11 +65,15 @@ def set_image_in_cache(id,table,url):
 		expire = now + (malojaconfig["CACHE_EXPIRE_NEGATIVE"] * 24 * 3600)
 	else:
 		expire = now + (malojaconfig["CACHE_EXPIRE_POSITIVE"] * 24 * 3600)
+
+	raw = dl_image(url)
+
 	with engine.begin() as conn:
 		op = DB[table].insert().values(
 			id=id,
 			url=url,
-			expire=expire
+			expire=expire,
+			raw=raw
 		)
 		result = conn.execute(op)
 
@@ -76,6 +85,7 @@ def remove_image_from_cache(id,table):
 		result = conn.execute(op)
 
 def dl_image(url):
+	if not malojaconfig["PROXY_IMAGES"]: return None
 	if url is None: return None
 	try:
 		r = requests.get(url)
@@ -86,95 +96,84 @@ def dl_image(url):
 		return uri
 	except:
 		log(f"Image {url} could not be downloaded for local caching")
-		return url
+		return None
 
-def get_track_image(track=None,track_id=None,fast=False):
 
+
+### getting images for any website embedding now ALWAYS returns just the generic link
+### even if we have already cached it, we will handle that on request
+def get_track_image(track=None,track_id=None):
 	if track_id is None:
 		track_id = database.sqldb.get_track_id(track)
-	title = track['title']
-	artists = track['artists']
 
-	# check cache
-	result = get_image_from_cache(track_id,'tracks')
-	if result is None:
-		# nonexistence cached, redirect to artist
-		for a in artists:
-			return get_artist_image(artist=a,fast=True)
-	elif result is False:
-		# no cache entry
-		pass
-	else:
-		return result
-
-	# local image
-	if malojaconfig["USE_LOCAL_IMAGES"]:
-		images = local_files(artists=artists,title=title)
-		if len(images) != 0:
-			result = random.choice(images)
-			result = urllib.parse.quote(result)
-			set_image_in_cache(track_id,'tracks',result)
-			return result
-
-	# forward
-	if fast:
-		titlequery = "title=" + urllib.parse.quote(title)
-		artistquery = "&".join("artist=" + urllib.parse.quote(a) for a in artists)
-		return (f"/image?{titlequery}&{artistquery}")
-
-	# third party
-	result = thirdparty.get_image_track_all((artists,title))
-
-	# dl image and proxy
-	result = dl_image(result)
-
-	set_image_in_cache(track_id,'tracks',result)
-	if result is not None: return result
-	for a in artists:
-		res = get_artist_image(artist=a,fast=False)
-		if res != "": return res
-	return ""
+	return f"/image?type=track&id={track_id}"
 
 
-def get_artist_image(artist=None,artist_id=None,fast=False):
-
+def get_artist_image(artist=None,artist_id=None):
 	if artist_id is None:
 		artist_id = database.sqldb.get_artist_id(artist)
 
+	return f"/image?type=artist&id={artist_id}"
+
+
+
+
+
+
+def resolve_track_image(track_id):
+
+	# check cache
+	result = get_image_from_cache(track_id,'tracks')
+	if result is not None:
+		return result
+
+	track = database.sqldb.get_track(track_id)
+	print(track)
+
+	# local image
+	if malojaconfig["USE_LOCAL_IMAGES"]:
+		images = local_files(artists=track['artists'],title=track['title'])
+		if len(images) != 0:
+			result = random.choice(images)
+			result = urllib.parse.quote(result)
+			result = {'type':'url','value':result}
+			set_image_in_cache(track_id,'tracks',result['value'])
+			return result
+
+	# third party
+	result = thirdparty.get_image_track_all((track['artists'],track['title']))
+	result = {'type':'url','value':result}
+	set_image_in_cache(track_id,'tracks',result['value'])
+
+	return result
+
+
+def resolve_artist_image(artist_id):
+
 	# check cache
 	result = get_image_from_cache(artist_id,'artists')
-	if result is None:
-		# nonexistence cached, whatevs
-		return ""
-	elif result is False:
-		# no cache entry
-		pass
-	else:
+	if result is not None:
 		return result
 
+	artist = database.sqldb.get_artist(artist_id)
+	print(artist)
+
 	# local image
 	if malojaconfig["USE_LOCAL_IMAGES"]:
 		images = local_files(artist=artist)
 		if len(images) != 0:
 			result = random.choice(images)
 			result = urllib.parse.quote(result)
-			set_image_in_cache(artist_id,'artists',result)
+			result = {'type':'url','value':result}
+			set_image_in_cache(artist_id,'artists',result['value'])
 			return result
 
-	# forward
-	if fast:
-		artistquery = "artist=" + urllib.parse.quote(artist)
-		return (f"/image?{artistquery}")
-
 	# third party
 	result = thirdparty.get_image_artist_all(artist)
+	result = {'type':'url','value':result}
+	set_image_in_cache(artist_id,'artists',result['value'])
 
-	# dl image and proxy
-	result = dl_image(result)
-
-	set_image_in_cache(artist_id,'artists',result)
-	if result is not None: return result
-	return ""
+	return result
 
 
 # removes emojis and weird shit from names

maloja/web/jinja/artist.jinja

@@ -39,10 +39,10 @@
 			{% if adminmode %}
 			<div
 				class="changeable-image" data-uploader="b64=>upload('{{ encodedartist }}',b64)"
-				style="background-image:url('{{ utilities.get_artist_image(artist,fast=True) }}');"
+				style="background-image:url('{{ utilities.get_artist_image(artist) }}');"
 			></div>
 			{% else %}
-			<div style="background-image:url('{{ utilities.get_artist_image(artist,fast=True) }}');">
+			<div style="background-image:url('{{ utilities.get_artist_image(artist) }}');">
 			</div>
 			{% endif %}
 		</td>

maloja/web/jinja/charts_artists.jinja

@@ -9,7 +9,7 @@
 {% set pages = math.ceil(charts.__len__() / amountkeys.perpage) %}
 {% if charts[0] is defined %}
 	{% set topartist = charts[0].artist %}
-	{% set img = utilities.get_artist_image(topartist,fast=True) %}
+	{% set img = utilities.get_artist_image(topartist) %}
 {% else %}
 	{% set img = "/favicon.png" %}
 {% endif %}

maloja/web/jinja/charts_tracks.jinja

@@ -11,7 +11,7 @@
 {% set pages = math.ceil(charts.__len__() / amountkeys.perpage) %}
 {% if charts[0] is defined %}
 	{% set toptrack = charts[0].track %}
-	{% set img = utilities.get_track_image(toptrack,fast=True) %}
+	{% set img = utilities.get_track_image(toptrack) %}
 {% else %}
 	{% set img = "/favicon.png" %}
 {% endif %}

maloja/web/jinja/partials/charts_artists_tiles.jinja

@@ -23,7 +23,7 @@
 						{% set rank = entry.rank %}
 						<td>
 							<a href="{{ links.url(artist) }}">
-							<div style='background-image:url("{{ utilities.get_artist_image(artist,fast=True) }}")'>
+							<div style='background-image:url("{{ utilities.get_artist_image(artist) }}")'>
 								<span class='stats'>#{{ rank }}</span> <span>{{ artist }}</span>
 							</div>
 							</a>

maloja/web/jinja/partials/charts_tracks_tiles.jinja

@@ -23,7 +23,7 @@
 						{% set rank = entry.rank %}
 						<td>
 							<a href="{{ links.url(track) }}">
-							<div style='background-image:url("{{ utilities.get_track_image(track,fast=True) }}")'>
+							<div style='background-image:url("{{ utilities.get_track_image(track) }}")'>
 								<span class='stats'>#{{ rank }}</span> <span>{{ track.title }}</span>
 							</div>
 							</a>

maloja/web/jinja/performance.jinja

@@ -4,9 +4,9 @@
 {% import 'snippets/filterdescription.jinja' as filterdesc %}
 
 {% if filterkeys.get('track') is not none %}
-	{% set img = utilities.get_track_image(filterkeys.track,fast=True) %}
+	{% set img = utilities.get_track_image(filterkeys.track) %}
 {% elif filterkeys.get('artist') is not none %}
-	{% set img = utilities.get_artist_image(filterkeys.artist,fast=True) %}
+	{% set img = utilities.get_artist_image(filterkeys.artist) %}
 {% else %}
 	{% set img = "/favicon.png" %}
 {% endif %}

maloja/web/jinja/pulse.jinja

@@ -4,9 +4,9 @@
 {% import 'snippets/filterdescription.jinja' as filterdesc %}
 
 {% if filterkeys.get('track') is not none %}
-	{% set img = utilities.get_track_image(filterkeys.track,fast=True) %}
+	{% set img = utilities.get_track_image(filterkeys.track) %}
 {% elif filterkeys.get('artist') is not none %}
-	{% set img = utilities.get_artist_image(filterkeys.artist,fast=True) %}
+	{% set img = utilities.get_artist_image(filterkeys.artist) %}
 {% else %}
 	{% set img = "/favicon.png" %}
 {% endif %}

maloja/web/jinja/scrobbles.jinja

@@ -8,11 +8,11 @@
 {% set pages = math.ceil(scrobbles.__len__() / amountkeys.perpage) %}
 
 {% if filterkeys.get('track') is not none %}
-	{% set img = utilities.get_track_image(filterkeys.track,fast=True) %}
+	{% set img = utilities.get_track_image(filterkeys.track) %}
 {% elif filterkeys.get('artist') is not none %}
-	{% set img = utilities.get_artist_image(filterkeys.artist,fast=True) %}
+	{% set img = utilities.get_artist_image(filterkeys.artist) %}
 {% elif scrobbles.__len__() > 0 %}
-	{% set img = utilities.get_track_image(scrobbles[0].track,fast=True) %}
+	{% set img = utilities.get_track_image(scrobbles[0].track) %}
 {% else %}
 	{% set img = "/favicon.png" %}
 {% endif %}

maloja/web/jinja/snippets/entityrow.jinja

@@ -3,9 +3,9 @@
 {% import 'snippets/links.jinja' as links %}
 
 {% if 'artists' in entity %}
-	{% set img = utilities.get_track_image(entity,fast=True) %}
+	{% set img = utilities.get_track_image(entity) %}
 {% else %}
-	{% set img = utilities.get_artist_image(entity,fast=True) %}
+	{% set img = utilities.get_artist_image(entity) %}
 {% endif %}
 
 <td class='icon'><div style="background-image:url('{{ img }}')"></div></td>

maloja/web/jinja/track.jinja

@@ -33,10 +33,10 @@
 			{% if adminmode %}
 			<div
 				class="changeable-image" data-uploader="b64=>upload('{{ encodedtrack }}',b64)"
-				style="background-image:url('{{ utilities.get_track_image(track,fast=True) }}');"
+				style="background-image:url('{{ utilities.get_track_image(track) }}');"
 			></div>
 			{% else %}
-			<div style="background-image:url('{{ utilities.get_track_image(track,fast=True) }}');">
+			<div style="background-image:url('{{ utilities.get_track_image(track) }}');">
 			</div>
 			{% endif %}
 		</td>