From f1007d6da5638f7ff807af1e0c4bfa96e4989139 Mon Sep 17 00:00:00 2001 From: Krateng Date: Wed, 20 Feb 2019 21:10:58 +0100 Subject: [PATCH] Changed way of serving third-party images to improve page loading time --- database.py | 2 +- htmlgenerators.py | 6 +++--- htmlmodules.py | 4 +++- server.py | 22 +++++++++++++++++----- utilities.py | 11 +++++++++++ website/artist.html | 2 +- website/artist.py | 30 ++++++++++++++++-------------- website/scrobbles.py | 20 ++++++++++---------- website/start.py | 18 ++++++++++-------- website/track.py | 7 ++++--- 10 files changed, 76 insertions(+), 46 deletions(-) diff --git a/database.py b/database.py index 8dd86ce..57e5c50 100644 --- a/database.py +++ b/database.py @@ -312,7 +312,7 @@ def get_pulse_external(): results = get_pulse(**ckeys) return {"list":results} -def get_pulse(step="month",stepn=1,trail=3,**keys): +def get_pulse(step="month",stepn=1,trail=1,**keys): (ts_start,ts_end) = getTimestamps(**{k:keys[k] for k in keys if k in ["since","to","within"]}) d_start = getStartOf(ts_start,step) diff --git a/htmlgenerators.py b/htmlgenerators.py index dcf3d1a..314957e 100644 --- a/htmlgenerators.py +++ b/htmlgenerators.py @@ -127,7 +127,7 @@ def getRangeDesc(timeA,timeB,inclusiveB=True): # it does fecking everything # it's the best # fantastic -def KeySplit(keys): +def KeySplit(keys,forceTrack=False,forceArtist=False): # output: # 1 keys that define the filtered object like artist or track @@ -136,9 +136,9 @@ def KeySplit(keys): # 4 keys that define amount limits # 1 - if "title" in keys: + if "title" in keys and not forceArtist: resultkeys1 = {"track":{"artists":keys.getall("artist"),"title":keys.get("title")}} - elif "artist" in keys: + elif "artist" in keys and not forceTrack: resultkeys1 = {"artist":keys.get("artist")} if "associated" in keys: resultkeys1["associated"] = True else: diff --git a/htmlmodules.py b/htmlmodules.py index eb513ba..7c07bc7 100644 --- a/htmlmodules.py +++ b/htmlmodules.py @@ -1,6 +1,7 @@ from htmlgenerators import * import database from utilities import getArtistsInfo, getTracksInfo +import urllib def getpictures(ls,result,tracks=False): @@ -22,7 +23,8 @@ def module_scrobblelist(max_=None,pictures=False,shortTimeDesc=False,**kwargs): scrobbles = database.get_scrobbles(**kwargs_time,**kwargs_filter) #we're getting all scrobbles for the number and only filtering them on site if pictures: scrobbleswithpictures = scrobbles if max_ is None else scrobbles[:max_] - scrobbleimages = [e.get("image") for e in getTracksInfo(scrobbleswithpictures)] #will still work with scrobble objects as they are a technically a subset of track objects + #scrobbleimages = [e.get("image") for e in getTracksInfo(scrobbleswithpictures)] #will still work with scrobble objects as they are a technically a subset of track objects + scrobbleimages = ["/image?title=" + urllib.parse.quote(t["title"]) + "&" + "&".join(["artist=" + urllib.parse.quote(a) for a in t["artists"]]) for t in scrobbleswithpictures] representative = scrobbles[0] if scrobbles is not [] else None diff --git a/server.py b/server.py index 701e069..50b097f 100755 --- a/server.py +++ b/server.py @@ -1,18 +1,23 @@ #!/usr/bin/env python +# server stuff from bottle import Bottle, route, get, post, error, run, template, static_file, request, response, FormsDict, redirect, template -from importlib.machinery import SourceFileLoader +import waitress +# rest of the project from htmlgenerators import removeIdentical from utilities import * +from htmlgenerators import KeySplit +# technical +from importlib.machinery import SourceFileLoader import _thread -import waitress -import urllib.request -import urllib.parse -from urllib.error import * import sys import signal import os import setproctitle +# url handling +import urllib.request +import urllib.parse +from urllib.error import * @@ -73,6 +78,13 @@ def graceful_exit(sig=None,frame=None): os._exit(42) +@webserver.route("/image") +def dynamic_image(): + keys = FormsDict.decode(request.query) + relevant, _, _, _ = KeySplit(keys) + result = resolveImage(**relevant) + redirect(result) + @webserver.route("/images/") @webserver.route("/images/") @webserver.route("/images/") diff --git a/utilities.py b/utilities.py index 1a630e3..d1c84e3 100644 --- a/utilities.py +++ b/utilities.py @@ -348,4 +348,15 @@ def getArtistsInfo(artistlist): # async calls only cached results, now we need to get them return [getArtistInfo(a) for a in artistlist] + + +# new way of serving images +# instead always generate a link locally, but redirect that on the fly +# this way the page can load faster and images will trickle in without having to resort to XHTTP requests + +def resolveImage(artist=None,track=None): + if track is not None: + return getTrackInfo(track["artists"],track["title"])["image"] + elif artist is not None: + return getArtistInfo(artist)["image"] diff --git a/website/artist.html b/website/artist.html index 5ea4ff1..f14fa13 100644 --- a/website/artist.html +++ b/website/artist.html @@ -23,7 +23,7 @@ -

Tracks

+

Top Tracks

KEY_TRACKLIST diff --git a/website/artist.py b/website/artist.py index 7a6b15c..5443c71 100644 --- a/website/artist.py +++ b/website/artist.py @@ -1,40 +1,42 @@ import urllib import json +import database def instructions(keys,dbport): from utilities import getArtistInfo - from htmlgenerators import clean, artistLink, artistLinks, trackLink, scrobblesTrackLink, getRangeDesc, scrobblesLink + from htmlgenerators import clean, artistLink, artistLinks, KeySplit from htmlmodules import module_pulse, module_trackcharts - allowedkeys = {"artist":keys.get("artist")} -# clean(keys) - info = getArtistInfo(keys["artist"]) + filterkeys, _, _, _ = KeySplit(keys,forceArtist=True) + info = getArtistInfo(filterkeys["artist"]) imgurl = info.get("image") - #desc = info.get("info") pushresources = [{"file":imgurl,"type":"image"}] if imgurl.startswith("/") else [] - response = urllib.request.urlopen("http://[::1]:" + str(dbport) + "/artistinfo?artist=" + urllib.parse.quote(keys["artist"])) - db_data = json.loads(response.read()) - scrobbles = str(db_data["scrobbles"]) - pos = "#" + str(db_data["position"]) + data = database.artistInfo(filterkeys["artist"]) + scrobbles = str(data["scrobbles"]) + pos = "#" + str(data["position"]) - credited = db_data.get("replace") + credited = data.get("replace") includestr = " " if credited is not None: includestr = "Competing under " + artistLink(credited) + " (" + pos + ")" pos = "" - included = db_data.get("associated") + included = data.get("associated") if included is not None and included != []: includestr = "associated: " includestr += artistLinks(included) - html_tracks, _ = module_trackcharts(**allowedkeys) + html_tracks, _ = module_trackcharts(**filterkeys,max_=15) - html_pulse = module_pulse(**allowedkeys,step="year",stepn=1,trail=1) + html_pulse = module_pulse(**filterkeys,step="year",stepn=1,trail=1) - replace = {"KEY_ARTISTNAME":keys["artist"],"KEY_ENC_ARTISTNAME":urllib.parse.quote(keys["artist"]),"KEY_IMAGEURL":imgurl, "KEY_DESCRIPTION":"","KEY_TRACKLIST":html_tracks,"KEY_SCROBBLES":scrobbles,"KEY_POSITION":pos,"KEY_ASSOCIATED":includestr,"KEY_PULSE":html_pulse} + replace = {"KEY_ARTISTNAME":keys["artist"],"KEY_ENC_ARTISTNAME":urllib.parse.quote(keys["artist"]), + "KEY_IMAGEURL":imgurl, "KEY_DESCRIPTION":"", + "KEY_TRACKLIST":html_tracks,"KEY_PULSE":html_pulse, + "KEY_SCROBBLES":scrobbles,"KEY_POSITION":pos, + "KEY_ASSOCIATED":includestr} return (replace,pushresources) diff --git a/website/scrobbles.py b/website/scrobbles.py index 4d396d8..32ddc24 100644 --- a/website/scrobbles.py +++ b/website/scrobbles.py @@ -1,10 +1,11 @@ import urllib import json +import database def instructions(keys,dbport): from utilities import getArtistInfo, getTrackInfo - from htmlgenerators import getTimeDesc, artistLink, artistLinks, trackLink, keysToUrl, KeySplit + from htmlgenerators import artistLink, artistLinks, trackLink, KeySplit from htmlmodules import module_scrobblelist @@ -12,16 +13,15 @@ def instructions(keys,dbport): # describe the scope limitstring = "" - if keys.get("title") is not None: - limitstring += "of " + trackLink({"title":keys.get("title"),"artists":keys.getall("artist")}) + " " - limitstring += "by " + artistLinks(keys.getall("artist")) + if filterkeys.get("track") is not None: + limitstring += "of " + trackLink(filterkeys["track"]) + " " + limitstring += "by " + artistLinks(filterkeys["track"]["artists"]) - elif keys.get("artist") is not None: - limitstring += "by " + artistLink(keys.get("artist")) - if keys.get("associated") is not None: - response = urllib.request.urlopen("http://[::1]:" + str(dbport) + "/artistinfo?artist=" + urllib.parse.quote(keys["artist"])) - db_data = json.loads(response.read()) - moreartists = db_data["associated"] + elif filterkeys.get("artist") is not None: + limitstring += "by " + artistLink(filterkeys.get("artist")) + if filterkeys.get("associated"): + data = database.artistInfo(filterkeys["artist"]) + moreartists = data.get("associated") if moreartists != []: limitstring += " including " + artistLinks(moreartists) + "" diff --git a/website/start.py b/website/start.py index 0965ccc..c0232b5 100644 --- a/website/start.py +++ b/website/start.py @@ -38,9 +38,10 @@ def instructions(keys,dbport): topartist = charts[0]["artist"] artisttitles = [c["artist"] for c in charts] - artistimages = [] - t1 = Thread(target=getpictures,args=(artisttitles,artistimages,)) - t1.start() + #artistimages = [] + #t1 = Thread(target=getpictures,args=(artisttitles,artistimages,)) + #t1.start() + artistimages = ["/image?artist=" + urllib.parse.quote(a) for a in artisttitles] #artistimages = [info.get("image") for info in getArtistsInfo(artisttitles)] artistlinks = [artistLink(a) for a in artisttitles] @@ -54,9 +55,10 @@ def instructions(keys,dbport): trackobjects = [t["track"] for t in charts] tracktitles = [t["title"] for t in trackobjects] trackartists = [", ".join(t["artists"]) for t in trackobjects] - trackimages = [] - t2 = Thread(target=getpictures,args=(trackobjects,trackimages,),kwargs={"tracks":True}) - t2.start() + #trackimages = [] + #t2 = Thread(target=getpictures,args=(trackobjects,trackimages,),kwargs={"tracks":True}) + #t2.start() + trackimages = ["/image?title=" + urllib.parse.quote(t["title"]) + "&" + "&".join(["artist=" + urllib.parse.quote(a) for a in t["artists"]]) for t in trackobjects] #trackimages = [info.get("image") for info in getTracksInfo(trackobjects)] tracklinks = [trackLink(t) for t in trackobjects] @@ -120,8 +122,8 @@ def instructions(keys,dbport): - t1.join() - t2.join() + #t1.join() + #t2.join() #t3.join() #pushresources = [{"file":img,"type":"image"} for img in artistimages + trackimages + scrobbleimages if img.startswith("/")] diff --git a/website/track.py b/website/track.py index 9731892..5949b50 100644 --- a/website/track.py +++ b/website/track.py @@ -9,7 +9,7 @@ def instructions(keys,dbport): from htmlmodules import module_scrobblelist, module_pulse - filterkeys, _, _, _ = KeySplit(keys) + filterkeys, _, _, _ = KeySplit(keys,forceTrack=True) track = filterkeys.get("track") imgurl = getTrackInfo(track["artists"],track["title"]).get("image") @@ -61,7 +61,8 @@ def instructions(keys,dbport): html_pulse = module_pulse(track=track,step="year",stepn=1,trail=1) - replace = {"KEY_TRACKTITLE":track.get("title"),"KEY_ARTISTS":artistLinks(track.get("artists")),"KEY_SCROBBLES":scrobblesnum,"KEY_IMAGEURL":imgurl, - "KEY_SCROBBLELINK":keysToUrl(keys),"KEY_SCROBBLELIST":html_scrobbles,"KEY_POSITION":pos,"KEY_PULSE":html_pulse} + replace = {"KEY_TRACKTITLE":track.get("title"),"KEY_ARTISTS":artistLinks(track.get("artists")),"KEY_SCROBBLES":scrobblesnum,"KEY_POSITION":pos,"KEY_IMAGEURL":imgurl, + "KEY_SCROBBLELINK":keysToUrl(keys), + "KEY_SCROBBLELIST":html_scrobbles,"KEY_PULSE":html_pulse} return (replace,pushresources)