1
0
mirror of https://github.com/krateng/maloja.git synced 2023-08-10 21:12:55 +03:00

Improved image requesting and caching

This commit is contained in:
Krateng 2019-03-12 11:39:36 +01:00
parent d46ba0a366
commit f310cf4b20
9 changed files with 84 additions and 64 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
__pycache__
*.sh
apikey
*.txt

View File

@ -1,6 +1,6 @@
from htmlgenerators import *
import database
from utilities import getArtistsInfo, getTracksInfo, getArtistInfo, getTrackInfo
from utilities import getArtistImage, getTrackImage
from malojatime import *
import urllib
@ -26,7 +26,7 @@ def module_scrobblelist(max_=None,pictures=False,shortTimeDesc=False,**kwargs):
scrobbleswithpictures = scrobbles if max_ is None else scrobbles[:max_]
#scrobbleimages = [e.get("image") for e in getTracksInfo(scrobbleswithpictures)] #will still work with scrobble objects as they are a technically a subset of track objects
#scrobbleimages = ["/image?title=" + urllib.parse.quote(t["title"]) + "&" + "&".join(["artist=" + urllib.parse.quote(a) for a in t["artists"]]) for t in scrobbleswithpictures]
scrobbleimages = [getTrackInfo(t["artists"],t["title"],fast=True).get("image") for t in scrobbleswithpictures]
scrobbleimages = [getTrackImage(t["artists"],t["title"],fast=True) for t in scrobbleswithpictures]
representative = scrobbles[0] if len(scrobbles) is not 0 else None
@ -186,7 +186,7 @@ def module_artistcharts_tiles(**kwargs):
rnk = (rank,e["scrobbles"])
rank = "#" + str(rank)
#image = "/image?artist=" + urllib.parse.quote(e["artist"])
image = getArtistInfo(e["artist"],fast=True).get("image")
image = getArtistImage(e["artist"],fast=True)
link = artistLink(e["artist"])
else:
rank = ""
@ -242,7 +242,7 @@ def module_trackcharts_tiles(**kwargs):
rnk = (rank,e["scrobbles"])
rank = "#" + str(rank)
#image = "/image?title=" + urllib.parse.quote(e["track"]["title"]) + "&" + "&".join(["artist=" + urllib.parse.quote(a) for a in e["track"]["artists"]])
image = getTrackInfo(e["track"]["artists"],e["track"]["title"],fast=True).get("image")
image = getTrackImage(e["track"]["artists"],e["track"]["title"],fast=True)
link = trackLink(e["track"])
else:
rank = ""

View File

@ -196,9 +196,9 @@ def apirequest(artists=None,artist=None,title=None):
with open("apikey","r") as keyfile:
apikey = keyfile.read().replace("\n","")
if apikey == "NONE": return {"image":None}
if apikey == "NONE": return None
except:
return {"image":None}
return None
sites = [
@ -221,36 +221,38 @@ def apirequest(artists=None,artist=None,title=None):
artiststr = urllib.parse.quote(", ".join(artists))
titlestr = urllib.parse.quote(title)
response = urllib.request.urlopen(s["trackurl"].format(artist=artiststr,title=titlestr))
log("API: " + s["name"] + "; Image request: " + "/".join(artists) + " - " + title,module="external")
data = json.loads(response.read())
if s["result_track_imgurl"](data) != "":
return {"image":s["result_track_imgurl"](data)}
return s["result_track_imgurl"](data)
except:
pass
if len(artists) == 1:
#return {"image":apirequest(artist=artists[0])["image"]}
return {"image":None}
return None
# try the same track with every single artist
for a in artists:
rec = apirequest(artists=[a],title=title)
if rec["image"] is not None:
if rec is not None:
return rec
return {"image":None}
return None
# ARTISTS
else:
for s in sites:
try:
response = urllib.request.urlopen(s["artisturl"].format(artist=urllib.parse.quote(artist)))
log("API: " + s["name"] + "; Image request: " + artist,module="external")
data = json.loads(response.read())
if s["result_artist_imgurl"](data) != "":
return {"image":s["result_artist_imgurl"](data)}
return s["result_artist_imgurl"](data)
except:
pass
return {"image":None}
return None
# I think I've only just understood modules
cachedTracks = {}
@ -275,7 +277,7 @@ def loadCache():
finally:
fl.close()
def getTrackInfo(artists,title,fast=False):
def getTrackImage(artists,title,fast=False):
obj = (frozenset(artists),title)
filename = "-".join([re.sub("[^a-zA-Z0-9]","",artist) for artist in artists]) + "_" + re.sub("[^a-zA-Z0-9]","",title)
@ -285,33 +287,49 @@ def getTrackInfo(artists,title,fast=False):
# check if custom image exists
if os.path.exists(filepath + ".png"):
imgurl = "/" + filepath + ".png"
return {"image":imgurl}
return imgurl
elif os.path.exists(filepath + ".jpg"):
imgurl = "/" + filepath + ".jpg"
return {"image":imgurl}
return imgurl
elif os.path.exists(filepath + ".jpeg"):
imgurl = "/" + filepath + ".jpeg"
return {"image":imgurl}
return imgurl
try:
return {"image":cachedTracks[(frozenset(artists),title)]}
# check our cache
# if we have cached the nonexistence of that image, we immediately return the redirect to the artist and let the resolver handle it
# (even if we're not in a fast lookup right now)
result = cachedTracks[(frozenset(artists),title)]
if result is not None: return result
else:
for a in artists:
res = getArtistImage(artist=a,fast=True)
if res != "": return res
return ""
except:
pass
# fast request only retuns cached and local results, generates redirect link for rest
if fast:
return {"image":"/image?title=" + urllib.parse.quote(title) + "&" + "&".join(["artist=" + urllib.parse.quote(a) for a in artists])}
if fast: return "/image?title=" + urllib.parse.quote(title) + "&" + "&".join(["artist=" + urllib.parse.quote(a) for a in artists])
# non-fast lookup (esentially only the resolver lookup)
result = apirequest(artists=artists,title=title)
if result.get("image") is not None:
cachedTracks[(frozenset(artists),title)] = result["image"]
return result
else:
result = getArtistInfo(artist=artists[0])
#cachedTracks[(frozenset(artists),title)] = result["image"]
return result
def getArtistInfo(artist,fast=False):
# cache results (even negative ones)
cachedTracks[(frozenset(artists),title)] = result
# return either result or redirect to artist
if result is not None: return result
else:
for a in artists:
res = getArtistImage(artist=a,fast=False)
if res != "": return res
return ""
def getArtistImage(artist,fast=False):
obj = artist
filename = re.sub("[^a-zA-Z0-9]","",artist)
@ -322,40 +340,42 @@ def getArtistInfo(artist,fast=False):
# check if custom image exists
if os.path.exists(filepath + ".png"):
imgurl = "/" + filepath + ".png"
return {"image":imgurl}
return imgurl
elif os.path.exists(filepath + ".jpg"):
imgurl = "/" + filepath + ".jpg"
return {"image":imgurl}
return imgurl
elif os.path.exists(filepath + ".jpeg"):
imgurl = "/" + filepath + ".jpeg"
return {"image":imgurl}
return imgurl
try:
return {"image":cachedArtists[artist]}
result = cachedArtists[artist]
if result is not None: return result
else: return ""
except:
pass
# fast request only retuns cached and local results, generates redirect link for rest
if fast:
return {"image":"/image?artist=" + urllib.parse.quote(artist)}
if fast: return "/image?artist=" + urllib.parse.quote(artist)
# non-fast lookup (esentially only the resolver lookup)
result = apirequest(artist=artist)
if result.get("image") is not None:
cachedArtists[artist] = result["image"]
return result
else:
return {"image":""}
def getTracksInfo(trackobjectlist,fast=False):
# cache results (even negative ones)
cachedArtists[artist] = result
if result is not None: return result
else: return ""
def getTrackImages(trackobjectlist,fast=False):
threads = []
for track in trackobjectlist:
t = Thread(target=getTrackInfo,args=(track["artists"],track["title"],),kwargs={"fast":fast})
t = Thread(target=getTrackImage,args=(track["artists"],track["title"],),kwargs={"fast":fast})
t.start()
threads.append(t)
@ -363,14 +383,14 @@ def getTracksInfo(trackobjectlist,fast=False):
t.join()
return [getTrackInfo(t["artists"],t["title"]) for t in trackobjectlist]
return [getTrackImage(t["artists"],t["title"]) for t in trackobjectlist]
def getArtistsInfo(artistlist,fast=False):
def getArtistImages(artistlist,fast=False):
threads = []
for artist in artistlist:
t = Thread(target=getArtistInfo,args=(artist,),kwargs={"fast":fast})
t = Thread(target=getArtistImage,args=(artist,),kwargs={"fast":fast})
t.start()
threads.append(t)
@ -378,7 +398,7 @@ def getArtistsInfo(artistlist,fast=False):
t.join()
# async calls only cached results, now we need to get them
return [getArtistInfo(a) for a in artistlist]
return [getArtistImage(a) for a in artistlist]
@ -388,7 +408,7 @@ def getArtistsInfo(artistlist,fast=False):
def resolveImage(artist=None,track=None):
if track is not None:
return getTrackInfo(track["artists"],track["title"])["image"]
return getTrackImage(track["artists"],track["title"])
elif artist is not None:
return getArtistInfo(artist)["image"]
return getArtistImage(artist)

View File

@ -3,13 +3,12 @@ import database
def instructions(keys):
from utilities import getArtistInfo
from utilities import getArtistImage
from htmlgenerators import artistLink, artistLinks, KeySplit
from htmlmodules import module_pulse, module_trackcharts
filterkeys, _, _, _ = KeySplit(keys,forceArtist=True)
info = getArtistInfo(filterkeys["artist"],fast=True)
imgurl = info.get("image")
imgurl = getArtistImage(filterkeys["artist"],fast=True)
pushresources = [{"file":imgurl,"type":"image"}] if imgurl.startswith("/") else []
data = database.artistInfo(filterkeys["artist"])

View File

@ -3,7 +3,7 @@ import database
def instructions(keys):
from utilities import getArtistInfo, getTrackInfo
from utilities import getArtistImage, getTrackImage
from htmlgenerators import artistLink, artistLinks, trackLink, scrobblesLink, keysToUrl, KeySplit
from htmlmodules import module_pulse
from malojatime import range_desc, delimit_desc
@ -35,9 +35,9 @@ def instructions(keys):
# get image
if filterkeys.get("track") is not None:
imgurl = getTrackInfo(filterkeys.get("track")["artists"],filterkeys.get("track")["title"]).get("image")
imgurl = getTrackImage(filterkeys.get("track")["artists"],filterkeys.get("track")["title"])
elif filterkeys.get("artist") is not None:
imgurl = getArtistInfo(keys.get("artist")).get("image")
imgurl = getArtistImage(keys.get("artist"))
else:
imgurl = ""

View File

@ -3,7 +3,7 @@ import database
def instructions(keys):
from utilities import getArtistInfo, getTrackInfo
from utilities import getArtistImage, getTrackImage
from htmlgenerators import artistLink, artistLinks, trackLink, KeySplit
from htmlmodules import module_scrobblelist
from malojatime import range_desc
@ -32,11 +32,11 @@ def instructions(keys):
# get image
if filterkeys.get("track") is not None:
imgurl = getTrackInfo(filterkeys.get("track")["artists"],filterkeys.get("track")["title"],fast=True).get("image")
imgurl = getTrackImage(filterkeys.get("track")["artists"],filterkeys.get("track")["title"],fast=True)
elif filterkeys.get("artist") is not None:
imgurl = getArtistInfo(keys.get("artist"),fast=True).get("image")
imgurl = getArtistImage(keys.get("artist"),fast=True)
elif rep is not None:
imgurl = getTrackInfo(rep["artists"],rep["title"],fast=True).get("image")
imgurl = getTrackImage(rep["artists"],rep["title"],fast=True)
else:
imgurl = ""

View File

@ -2,7 +2,7 @@ import urllib
def instructions(keys):
from utilities import getArtistInfo
from utilities import getArtistImage
from htmlgenerators import KeySplit
from htmlmodules import module_artistcharts
from malojatime import range_desc
@ -17,7 +17,7 @@ def instructions(keys):
html_charts, rep = module_artistcharts(**amountkeys,**timekeys)
if rep is not None:
imgurl = getArtistInfo(rep).get("image")
imgurl = getArtistImage(rep)
else:
imgurl = ""

View File

@ -2,7 +2,7 @@ import urllib
def instructions(keys):
from utilities import getArtistInfo, getTrackInfo
from utilities import getArtistImage, getTrackImage
from htmlgenerators import artistLink, KeySplit
from htmlmodules import module_trackcharts
from malojatime import range_desc
@ -17,10 +17,10 @@ def instructions(keys):
if filterkeys.get("artist") is not None:
imgurl = getArtistInfo(filterkeys.get("artist")).get("image")
imgurl = getArtistImage(filterkeys.get("artist"))
limitstring = "by " + artistLink(filterkeys.get("artist"))
elif rep is not None:
imgurl = getTrackInfo(rep["artists"],rep["title"]).get("image")
imgurl = getTrackImage(rep["artists"],rep["title"])
else:
imgurl = ""

View File

@ -3,7 +3,7 @@ import database
def instructions(keys):
from utilities import getArtistInfo, getTrackInfo
from utilities import getArtistImage, getTrackImage
from htmlgenerators import artistLinks, keysToUrl, KeySplit
from htmlmodules import module_scrobblelist, module_pulse
@ -11,7 +11,7 @@ def instructions(keys):
filterkeys, _, _, _ = KeySplit(keys,forceTrack=True)
track = filterkeys.get("track")
imgurl = getTrackInfo(track["artists"],track["title"],fast=True).get("image")
imgurl = getTrackImage(track["artists"],track["title"],fast=True)
pushresources = [{"file":imgurl,"type":"image"}] if imgurl.startswith("/") else []
data = database.trackInfo(track["artists"],track["title"])