Changed way of serving third-party images to improve page loading time

database.py

@@ -312,7 +312,7 @@ def get_pulse_external():
 	results = get_pulse(**ckeys)
 	return {"list":results}
 
-def get_pulse(step="month",stepn=1,trail=3,**keys):
+def get_pulse(step="month",stepn=1,trail=1,**keys):
 
 	(ts_start,ts_end) = getTimestamps(**{k:keys[k] for k in keys if k in ["since","to","within"]})
 	d_start = getStartOf(ts_start,step)

htmlgenerators.py

@@ -127,7 +127,7 @@ def getRangeDesc(timeA,timeB,inclusiveB=True):
 # it does fecking everything
 # it's the best
 # fantastic
-def KeySplit(keys):
+def KeySplit(keys,forceTrack=False,forceArtist=False):
 
 	# output:
 	# 1	keys that define the filtered object like artist or track
@@ -136,9 +136,9 @@ def KeySplit(keys):
 	# 4	keys that define amount limits
 
 	# 1
-	if "title" in keys:
+	if "title" in keys and not forceArtist:
 		resultkeys1 = {"track":{"artists":keys.getall("artist"),"title":keys.get("title")}}
-	elif "artist" in keys:
+	elif "artist" in keys and not forceTrack:
 		resultkeys1 = {"artist":keys.get("artist")}
 		if "associated" in keys: resultkeys1["associated"] = True
 	else:

htmlmodules.py

@@ -1,6 +1,7 @@
 from htmlgenerators import *
 import database
 from utilities import getArtistsInfo, getTracksInfo
+import urllib
 
 
 def getpictures(ls,result,tracks=False):
@@ -22,7 +23,8 @@ def module_scrobblelist(max_=None,pictures=False,shortTimeDesc=False,**kwargs):
 	scrobbles = database.get_scrobbles(**kwargs_time,**kwargs_filter) #we're getting all scrobbles for the number and only filtering them on site
 	if pictures:
 		scrobbleswithpictures = scrobbles if max_ is None else scrobbles[:max_]
-		scrobbleimages = [e.get("image") for e in getTracksInfo(scrobbleswithpictures)] #will still work with scrobble objects as they are a technically a subset of track objects
+		#scrobbleimages = [e.get("image") for e in getTracksInfo(scrobbleswithpictures)] #will still work with scrobble objects as they are a technically a subset of track objects
+		scrobbleimages = ["/image?title=" + urllib.parse.quote(t["title"]) + "&" + "&".join(["artist=" + urllib.parse.quote(a) for a in t["artists"]])  for t in scrobbleswithpictures]
 	
 	representative = scrobbles[0] if scrobbles is not [] else None
 	

server.py

@@ -1,18 +1,23 @@
 #!/usr/bin/env python
 
+# server stuff
 from bottle import Bottle, route, get, post, error, run, template, static_file, request, response, FormsDict, redirect, template
-from importlib.machinery import SourceFileLoader
+import waitress
+# rest of the project
 from htmlgenerators import removeIdentical
 from utilities import *
+from htmlgenerators import KeySplit
+# technical
+from importlib.machinery import SourceFileLoader
 import _thread
-import waitress
-import urllib.request
-import urllib.parse
-from urllib.error import *
 import sys
 import signal
 import os
 import setproctitle
+# url handling
+import urllib.request
+import urllib.parse
+from urllib.error import *
 
 
 
@@ -73,6 +78,13 @@ def graceful_exit(sig=None,frame=None):
 	os._exit(42)
 
 
+@webserver.route("/image")
+def dynamic_image():
+	keys = FormsDict.decode(request.query)
+	relevant, _, _, _ = KeySplit(keys)
+	result = resolveImage(**relevant)
+	redirect(result)
+
 @webserver.route("/images/<pth:re:.*\\.jpeg>")
 @webserver.route("/images/<pth:re:.*\\.jpg>")
 @webserver.route("/images/<pth:re:.*\\.png>")

utilities.py

@@ -349,3 +349,14 @@ def getArtistsInfo(artistlist):
 	return [getArtistInfo(a) for a in artistlist]
 
 
+
+# new way of serving images
+# instead always generate a link locally, but redirect that on the fly
+# this way the page can load faster and images will trickle in without having to resort to XHTTP requests
+
+def resolveImage(artist=None,track=None):
+	if track is not None:
+		return getTrackInfo(track["artists"],track["title"])["image"]
+	elif artist is not None:
+		return getArtistInfo(artist)["image"]
+		

website/artist.html

@@ -23,7 +23,7 @@
 			</tr>
 		</table>
 		
-		<h2><a href='/toptracks?artist=KEY_ENC_ARTISTNAME'>Tracks</a></h2>
+		<h2><a href='/toptracks?artist=KEY_ENC_ARTISTNAME'>Top Tracks</a></h2>
 		KEY_TRACKLIST
 		
 		

website/artist.py

@@ -1,40 +1,42 @@
 import urllib
 import json
+import database
 
 		
 def instructions(keys,dbport):
 	from utilities import getArtistInfo
-	from htmlgenerators import clean, artistLink, artistLinks, trackLink, scrobblesTrackLink, getRangeDesc, scrobblesLink
+	from htmlgenerators import clean, artistLink, artistLinks, KeySplit
 	from htmlmodules import module_pulse, module_trackcharts
 
-	allowedkeys = {"artist":keys.get("artist")}
+	filterkeys, _, _, _ = KeySplit(keys,forceArtist=True)
-#	clean(keys)
+	info = getArtistInfo(filterkeys["artist"])
-	info = getArtistInfo(keys["artist"])
 	imgurl = info.get("image")
-	#desc = info.get("info")
 	pushresources = [{"file":imgurl,"type":"image"}] if imgurl.startswith("/") else []
 	
-	response = urllib.request.urlopen("http://[::1]:" + str(dbport) + "/artistinfo?artist=" + urllib.parse.quote(keys["artist"]))
+	data = database.artistInfo(filterkeys["artist"])
-	db_data = json.loads(response.read())
+	scrobbles = str(data["scrobbles"])
-	scrobbles = str(db_data["scrobbles"])
+	pos = "#" + str(data["position"])
-	pos = "#" + str(db_data["position"])
 	
-	credited = db_data.get("replace")
+	credited = data.get("replace")
 	includestr = " "
 	if credited is not None:
 		includestr = "Competing under " + artistLink(credited) + " (" + pos + ")"
 		pos = ""
-	included = db_data.get("associated")
+	included = data.get("associated")
 	if included is not None and included != []:
 		includestr = "associated: "
 		includestr += artistLinks(included)
 	
 
-	html_tracks, _ = module_trackcharts(**allowedkeys)	
+	html_tracks, _ = module_trackcharts(**filterkeys,max_=15)	
 	
 	
-	html_pulse = module_pulse(**allowedkeys,step="year",stepn=1,trail=1)
+	html_pulse = module_pulse(**filterkeys,step="year",stepn=1,trail=1)
 
-	replace = {"KEY_ARTISTNAME":keys["artist"],"KEY_ENC_ARTISTNAME":urllib.parse.quote(keys["artist"]),"KEY_IMAGEURL":imgurl, "KEY_DESCRIPTION":"","KEY_TRACKLIST":html_tracks,"KEY_SCROBBLES":scrobbles,"KEY_POSITION":pos,"KEY_ASSOCIATED":includestr,"KEY_PULSE":html_pulse}
+	replace = {"KEY_ARTISTNAME":keys["artist"],"KEY_ENC_ARTISTNAME":urllib.parse.quote(keys["artist"]),
+	"KEY_IMAGEURL":imgurl, "KEY_DESCRIPTION":"",
+	"KEY_TRACKLIST":html_tracks,"KEY_PULSE":html_pulse,
+	"KEY_SCROBBLES":scrobbles,"KEY_POSITION":pos,
+	"KEY_ASSOCIATED":includestr}
 	
 	return (replace,pushresources)

website/scrobbles.py

@@ -1,10 +1,11 @@
 import urllib
 import json
+import database
 
 		
 def instructions(keys,dbport):
 	from utilities import getArtistInfo, getTrackInfo
-	from htmlgenerators import getTimeDesc, artistLink, artistLinks, trackLink, keysToUrl, KeySplit
+	from htmlgenerators import artistLink, artistLinks, trackLink, KeySplit
 	from htmlmodules import module_scrobblelist
 	
 	
@@ -12,16 +13,15 @@ def instructions(keys,dbport):
 	
 	# describe the scope
 	limitstring = ""
-	if keys.get("title") is not None:
+	if filterkeys.get("track") is not None:
-		limitstring += "of " + trackLink({"title":keys.get("title"),"artists":keys.getall("artist")}) + " "
+		limitstring += "of " + trackLink(filterkeys["track"]) + " "
-		limitstring += "by " + artistLinks(keys.getall("artist"))
+		limitstring += "by " + artistLinks(filterkeys["track"]["artists"])
 	
-	elif keys.get("artist") is not None:
+	elif filterkeys.get("artist") is not None:
-		limitstring += "by " + artistLink(keys.get("artist"))
+		limitstring += "by " + artistLink(filterkeys.get("artist"))
-		if keys.get("associated") is not None:
+		if filterkeys.get("associated"):
-			response = urllib.request.urlopen("http://[::1]:" + str(dbport) + "/artistinfo?artist=" + urllib.parse.quote(keys["artist"]))
+			data = database.artistInfo(filterkeys["artist"])
-			db_data = json.loads(response.read())
+			moreartists = data.get("associated")
-			moreartists = db_data["associated"]
 			if moreartists != []:
 				limitstring += " <span class='extra'>including " + artistLinks(moreartists) + "</span>"
 		

website/start.py

@@ -38,9 +38,10 @@ def instructions(keys,dbport):
 	topartist = charts[0]["artist"]
 	
 	artisttitles = [c["artist"] for c in charts]
-	artistimages = []
+	#artistimages = []
-	t1 = Thread(target=getpictures,args=(artisttitles,artistimages,))
+	#t1 = Thread(target=getpictures,args=(artisttitles,artistimages,))
-	t1.start()
+	#t1.start()
+	artistimages = ["/image?artist=" + urllib.parse.quote(a) for a in artisttitles]
 	#artistimages = [info.get("image") for info in getArtistsInfo(artisttitles)]
 	artistlinks = [artistLink(a) for a in artisttitles]
 	
@@ -54,9 +55,10 @@ def instructions(keys,dbport):
 	trackobjects = [t["track"] for t in charts]
 	tracktitles = [t["title"] for t in trackobjects]
 	trackartists = [", ".join(t["artists"]) for t in trackobjects]
-	trackimages = []
+	#trackimages = []
-	t2 = Thread(target=getpictures,args=(trackobjects,trackimages,),kwargs={"tracks":True})
+	#t2 = Thread(target=getpictures,args=(trackobjects,trackimages,),kwargs={"tracks":True})
-	t2.start()
+	#t2.start()
+	trackimages = ["/image?title=" + urllib.parse.quote(t["title"]) + "&" + "&".join(["artist=" + urllib.parse.quote(a) for a in t["artists"]])  for t in trackobjects]
 	#trackimages = [info.get("image") for info in getTracksInfo(trackobjects)]
 	tracklinks = [trackLink(t) for t in trackobjects]
 	
@@ -120,8 +122,8 @@ def instructions(keys,dbport):
 	
 	
 	
-	t1.join()
+	#t1.join()
-	t2.join()
+	#t2.join()
 	#t3.join()
 	
 	#pushresources = [{"file":img,"type":"image"} for img in artistimages + trackimages + scrobbleimages if img.startswith("/")]

website/track.py

@@ -9,7 +9,7 @@ def instructions(keys,dbport):
 	from htmlmodules import module_scrobblelist, module_pulse
 
 	
-	filterkeys, _, _, _ = KeySplit(keys)	
+	filterkeys, _, _, _ = KeySplit(keys,forceTrack=True)	
 	
 	track = filterkeys.get("track")
 	imgurl = getTrackInfo(track["artists"],track["title"]).get("image")
@@ -61,7 +61,8 @@ def instructions(keys,dbport):
 	html_pulse = module_pulse(track=track,step="year",stepn=1,trail=1)
 
 
-	replace = {"KEY_TRACKTITLE":track.get("title"),"KEY_ARTISTS":artistLinks(track.get("artists")),"KEY_SCROBBLES":scrobblesnum,"KEY_IMAGEURL":imgurl,
+	replace = {"KEY_TRACKTITLE":track.get("title"),"KEY_ARTISTS":artistLinks(track.get("artists")),"KEY_SCROBBLES":scrobblesnum,"KEY_POSITION":pos,"KEY_IMAGEURL":imgurl,
-		"KEY_SCROBBLELINK":keysToUrl(keys),"KEY_SCROBBLELIST":html_scrobbles,"KEY_POSITION":pos,"KEY_PULSE":html_pulse}
+		"KEY_SCROBBLELINK":keysToUrl(keys),
+		"KEY_SCROBBLELIST":html_scrobbles,"KEY_PULSE":html_pulse}
 	
 	return (replace,pushresources)