mirror/maloja
1
0
Fork 0
mirror of https://github.com/krateng/maloja.git synced 2023-08-10 21:12:55 +03:00
Code Issues Projects Releases Wiki Activity

Proper handling of shared ranks everywhere

Browse Source
This commit is contained in:
Krateng 2019-03-14 11:07:20 +01:00
parent 8e37a902e5
commit 958337cadb
3 changed files with 276 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

348
database.py
View File

@ -52,10 +52,10 @@ def checkAPIkey(k):
def getScrobbleObject(o): def getScrobbleObject(o):
track = getTrackObject(TRACKS[o[0]]) track = getTrackObject(TRACKS[o[0]])
return {"artists":track["artists"],"title":track["title"],"time":o[1]} return {"artists":track["artists"],"title":track["title"],"time":o[1]}
def getArtistObject(o): def getArtistObject(o):
return o return o
def getTrackObject(o): def getTrackObject(o):
artists = [getArtistObject(ARTISTS[a]) for a in o[0]] artists = [getArtistObject(ARTISTS[a]) for a in o[0]]
return {"artists":artists,"title":o[1]} return {"artists":artists,"title":o[1]}
@ -66,7 +66,7 @@ def getTrackObject(o):
#### ####
def createScrobble(artists,title,time,volatile=False): def createScrobble(artists,title,time,volatile=False):
while (time in SCROBBLESDICT): while (time in SCROBBLESDICT):
time += 1 time += 1
@ -89,14 +89,14 @@ def readScrobble(artists,title,time):
SCROBBLES.append(obj) SCROBBLES.append(obj)
SCROBBLESDICT[time] = obj SCROBBLESDICT[time] = obj
#STAMPS.append(time) #STAMPS.append(time)
def getArtistID(name): def getArtistID(name):
obj = name obj = name
objlower = name.lower() objlower = name.lower()
try: try:
return ARTISTS.index(obj) return ARTISTS.index(obj)
except: except:
@ -107,14 +107,14 @@ def getArtistID(name):
i = len(ARTISTS) i = len(ARTISTS)
ARTISTS.append(obj) ARTISTS.append(obj)
return i return i
def getTrackID(artists,title): def getTrackID(artists,title):
artistset = set() artistset = set()
for a in artists: for a in artists:
artistset.add(getArtistID(name=a)) artistset.add(getArtistID(name=a))
obj = (frozenset(artistset),title) obj = (frozenset(artistset),title)
objlower = (frozenset(artistset),title.lower()) objlower = (frozenset(artistset),title.lower())
try: try:
return TRACKS.index(obj) return TRACKS.index(obj)
except: except:
@ -150,14 +150,14 @@ def test_server():
if apikey is not None and not (checkAPIkey(apikey)): if apikey is not None and not (checkAPIkey(apikey)):
response.status = 403 response.status = 403
return "Wrong API key" return "Wrong API key"
elif db_rulestate: elif db_rulestate:
response.status = 204 response.status = 204
return return
else: else:
response.status = 205 response.status = 205
return return
# 204 Database server is up and operational # 204 Database server is up and operational
# 205 Database server is up, but DB is not fully built or is inconsistent # 205 Database server is up, but DB is not fully built or is inconsistent
# 403 Database server is up, but provided API key is not valid # 403 Database server is up, but provided API key is not valid
@ -173,7 +173,7 @@ def get_scrobbles_external():
ckeys["since"], ckeys["to"], ckeys["within"] = keys.get("since"), keys.get("to"), keys.get("in") ckeys["since"], ckeys["to"], ckeys["within"] = keys.get("since"), keys.get("to"), keys.get("in")
ckeys["associated"] = (keys.get("associated")!=None) ckeys["associated"] = (keys.get("associated")!=None)
ckeys["max_"] = keys.get("max") ckeys["max_"] = keys.get("max")
result = get_scrobbles(**ckeys) result = get_scrobbles(**ckeys)
return {"list":result} return {"list":result}
@ -196,7 +196,7 @@ def get_scrobbles(**keys):
# #
#def get_amounts(): #def get_amounts():
# return {"scrobbles":len(SCROBBLES),"tracks":len(TRACKS),"artists":len(ARTISTS)} # return {"scrobbles":len(SCROBBLES),"tracks":len(TRACKS),"artists":len(ARTISTS)}
@dbserver.route("/numscrobbles") @dbserver.route("/numscrobbles")
def get_scrobbles_num_external(): def get_scrobbles_num_external():
@ -205,7 +205,7 @@ def get_scrobbles_num_external():
ckeys["artists"], ckeys["title"] = keys.getall("artist"), keys.get("title") ckeys["artists"], ckeys["title"] = keys.getall("artist"), keys.get("title")
ckeys["since"], ckeys["to"], ckeys["within"] = keys.get("since"), keys.get("to"), keys.get("in") ckeys["since"], ckeys["to"], ckeys["within"] = keys.get("since"), keys.get("to"), keys.get("in")
ckeys["associated"] = (keys.get("associated")!=None) ckeys["associated"] = (keys.get("associated")!=None)
result = get_scrobbles_num(**ckeys) result = get_scrobbles_num(**ckeys)
return {"amount":result} return {"amount":result}
@ -219,17 +219,17 @@ def get_scrobbles_num(**keys):
# REEVALUATE # REEVALUATE
#def get_scrobbles_num_multiple(sinces=[],to=None,**keys): #def get_scrobbles_num_multiple(sinces=[],to=None,**keys):
# #
# sinces_stamps = [time_stamps(since,to,None)[0] for since in sinces] # sinces_stamps = [time_stamps(since,to,None)[0] for since in sinces]
# #print(sinces) # #print(sinces)
# #print(sinces_stamps) # #print(sinces_stamps)
# minsince = sinces[-1] # minsince = sinces[-1]
# r = db_query(**{k:keys[k] for k in keys if k in ["artist","track","artists","title","associated","to"]},since=minsince) # r = db_query(**{k:keys[k] for k in keys if k in ["artist","track","artists","title","associated","to"]},since=minsince)
# #
# #print(r) # #print(r)
# #
# validtracks = [0 for s in sinces] # validtracks = [0 for s in sinces]
# #
# i = 0 # i = 0
# si = 0 # si = 0
# while True: # while True:
@ -241,10 +241,10 @@ def get_scrobbles_num(**keys):
# si += 1 # si += 1
# continue # continue
# i += 1 # i += 1
# #
# #
# return validtracks # return validtracks
# UNUSED # UNUSED
#@dbserver.route("/charts") #@dbserver.route("/charts")
@ -252,8 +252,8 @@ def get_scrobbles_num(**keys):
# keys = FormsDict.decode(request.query) # keys = FormsDict.decode(request.query)
# ckeys = {} # ckeys = {}
# ckeys["since"], ckeys["to"], ckeys["within"] = keys.get("since"), keys.get("to"), keys.get("in") # ckeys["since"], ckeys["to"], ckeys["within"] = keys.get("since"), keys.get("to"), keys.get("in")
# #
# result = get_scrobbles_num(**ckeys) # result = get_scrobbles_num(**ckeys)
# return {"number":result} # return {"number":result}
#def get_charts(**keys): #def get_charts(**keys):
@ -275,7 +275,7 @@ def get_tracks_external():
return {"list":result} return {"list":result}
def get_tracks(artist=None): def get_tracks(artist=None):
if artist is not None: if artist is not None:
artistid = ARTISTS.index(artist) artistid = ARTISTS.index(artist)
else: else:
@ -283,7 +283,7 @@ def get_tracks(artist=None):
# Option 1 # Option 1
return [getTrackObject(t) for t in TRACKS if (artistid in t[0]) or (artistid==None)] return [getTrackObject(t) for t in TRACKS if (artistid in t[0]) or (artistid==None)]
# Option 2 is a bit more elegant but much slower # Option 2 is a bit more elegant but much slower
#tracklist = [getTrackObject(t) for t in TRACKS] #tracklist = [getTrackObject(t) for t in TRACKS]
#ls = [t for t in tracklist if (artist in t["artists"]) or (artist==None)] #ls = [t for t in tracklist if (artist in t["artists"]) or (artist==None)]
@ -296,19 +296,19 @@ def get_artists_external():
def get_artists(): def get_artists():
return ARTISTS #well return ARTISTS #well
@dbserver.route("/charts/artists") @dbserver.route("/charts/artists")
def get_charts_artists_external(): def get_charts_artists_external():
keys = FormsDict.decode(request.query) keys = FormsDict.decode(request.query)
ckeys = {} ckeys = {}
ckeys["since"], ckeys["to"], ckeys["within"] = keys.get("since"), keys.get("to"), keys.get("in") ckeys["since"], ckeys["to"], ckeys["within"] = keys.get("since"), keys.get("to"), keys.get("in")
result = get_charts_artists(**ckeys) result = get_charts_artists(**ckeys)
return {"list":result} return {"list":result}
@ -326,21 +326,21 @@ def get_charts_tracks_external():
ckeys = {} ckeys = {}
ckeys["since"], ckeys["to"], ckeys["within"] = keys.get("since"), keys.get("to"), keys.get("in") ckeys["since"], ckeys["to"], ckeys["within"] = keys.get("since"), keys.get("to"), keys.get("in")
ckeys["artist"] = keys.get("artist") ckeys["artist"] = keys.get("artist")
result = get_charts_tracks(**ckeys) result = get_charts_tracks(**ckeys)
return {"list":result} return {"list":result}
def get_charts_tracks(**keys): def get_charts_tracks(**keys):
return db_aggregate(by="TRACK",**{k:keys[k] for k in keys if k in ["since","to","within","artist"]}) return db_aggregate(by="TRACK",**{k:keys[k] for k in keys if k in ["since","to","within","artist"]})
@dbserver.route("/pulse") @dbserver.route("/pulse")
def get_pulse_external(): def get_pulse_external():
keys = FormsDict.decode(request.query) keys = FormsDict.decode(request.query)
@ -351,20 +351,20 @@ def get_pulse_external():
ckeys["associated"] = (keys.get("associated")!=None) ckeys["associated"] = (keys.get("associated")!=None)
if ckeys["step"] is not None: [ckeys["step"],ckeys["stepn"]] = (ckeys["step"].split("-") + [1])[:2] # makes the multiplier 1 if not assigned if ckeys["step"] is not None: [ckeys["step"],ckeys["stepn"]] = (ckeys["step"].split("-") + [1])[:2] # makes the multiplier 1 if not assigned
if "stepn" in ckeys: ckeys["stepn"] = int(ckeys["stepn"]) if "stepn" in ckeys: ckeys["stepn"] = int(ckeys["stepn"])
cleandict(ckeys) cleandict(ckeys)
results = get_pulse(**ckeys) results = get_pulse(**ckeys)
return {"list":results} return {"list":results}
def get_pulse(**keys): def get_pulse(**keys):
rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","step","stepn","trail"]}) rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","step","stepn","trail"]})
results = [] results = []
for (a,b) in rngs: for (a,b) in rngs:
res = len(db_query(since=a,to=b,**{k:keys[k] for k in keys if k in ["artists","artist","track","title","associated"]})) res = len(db_query(since=a,to=b,**{k:keys[k] for k in keys if k in ["artists","artist","track","title","associated"]}))
results.append({"from":a,"to":b,"scrobbles":res}) results.append({"from":a,"to":b,"scrobbles":res})
return results return results
@ -372,8 +372,8 @@ def get_pulse(**keys):
@dbserver.route("/top/artists") @dbserver.route("/top/artists")
def get_top_artists_external(): def get_top_artists_external():
@ -383,23 +383,23 @@ def get_top_artists_external():
ckeys["step"], ckeys["trail"] = keys.get("step"), int_or_none(keys.get("trail")) ckeys["step"], ckeys["trail"] = keys.get("step"), int_or_none(keys.get("trail"))
if ckeys["step"] is not None: [ckeys["step"],ckeys["stepn"]] = (ckeys["step"].split("-") + [1])[:2] # makes the multiplier 1 if not assigned if ckeys["step"] is not None: [ckeys["step"],ckeys["stepn"]] = (ckeys["step"].split("-") + [1])[:2] # makes the multiplier 1 if not assigned
if "stepn" in ckeys: ckeys["stepn"] = int(ckeys["stepn"]) if "stepn" in ckeys: ckeys["stepn"] = int(ckeys["stepn"])
cleandict(ckeys) cleandict(ckeys)
results = get_top_artists(**ckeys) results = get_top_artists(**ckeys)
return {"list":results} return {"list":results}
def get_top_artists(**keys): def get_top_artists(**keys):
rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","step","stepn","trail"]}) rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","step","stepn","trail"]})
results = [] results = []
for (a,b) in rngs: for (a,b) in rngs:
try: try:
res = db_aggregate(since=a,to=b,by="ARTIST")[0] res = db_aggregate(since=a,to=b,by="ARTIST")[0]
results.append({"from":a,"to":b,"artist":res["artist"],"scrobbles":res["scrobbles"]}) results.append({"from":a,"to":b,"artist":res["artist"],"scrobbles":res["scrobbles"]})
except: except:
results.append({"from":a,"to":b,"artist":None,"scrobbles":0}) results.append({"from":a,"to":b,"artist":None,"scrobbles":0})
return results return results
@ -419,7 +419,7 @@ def get_top_tracks_external():
ckeys["step"], ckeys["trail"] = keys.get("step"), int_or_none(keys.get("trail")) ckeys["step"], ckeys["trail"] = keys.get("step"), int_or_none(keys.get("trail"))
if ckeys["step"] is not None: [ckeys["step"],ckeys["stepn"]] = (ckeys["step"].split("-") + [1])[:2] # makes the multiplier 1 if not assigned if ckeys["step"] is not None: [ckeys["step"],ckeys["stepn"]] = (ckeys["step"].split("-") + [1])[:2] # makes the multiplier 1 if not assigned
if "stepn" in ckeys: ckeys["stepn"] = int(ckeys["stepn"]) if "stepn" in ckeys: ckeys["stepn"] = int(ckeys["stepn"])
cleandict(ckeys) cleandict(ckeys)
results = get_top_tracks(**ckeys) results = get_top_tracks(**ckeys)
return {"list":results} return {"list":results}
@ -428,14 +428,14 @@ def get_top_tracks(**keys):
rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","step","stepn","trail"]}) rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","step","stepn","trail"]})
results = [] results = []
for (a,b) in rngs: for (a,b) in rngs:
try: try:
res = db_aggregate(since=a,to=b,by="TRACK")[0] res = db_aggregate(since=a,to=b,by="TRACK")[0]
results.append({"from":a,"to":b,"track":res["track"],"scrobbles":res["scrobbles"]}) results.append({"from":a,"to":b,"track":res["track"],"scrobbles":res["scrobbles"]})
except: except:
results.append({"from":a,"to":b,"track":None,"scrobbles":0}) results.append({"from":a,"to":b,"track":None,"scrobbles":0})
return results return results
@ -444,52 +444,59 @@ def get_top_tracks(**keys):
@dbserver.route("/artistinfo") @dbserver.route("/artistinfo")
def artistInfo_external(): def artistInfo_external():
keys = FormsDict.decode(request.query) keys = FormsDict.decode(request.query)
ckeys = {} ckeys = {}
ckeys["artist"] = keys.get("artist") ckeys["artist"] = keys.get("artist")
results = artistInfo(**ckeys) results = artistInfo(**ckeys)
return results return results
def artistInfo(artist): def artistInfo(artist):
charts = db_aggregate(by="ARTIST") charts = db_aggregate(by="ARTIST")
scrobbles = len(db_query(artists=[artist])) #we cant take the scrobble number from the charts because that includes all countas scrobbles scrobbles = len(db_query(artists=[artist])) #we cant take the scrobble number from the charts because that includes all countas scrobbles
try: try:
c = [e for e in charts if e["artist"] == artist][0] c = [e for e in charts if e["artist"] == artist][0]
others = coa.getAllAssociated(artist) others = coa.getAllAssociated(artist)
return {"scrobbles":scrobbles,"position":charts.index(c) + 1,"associated":others} position = charts.index(c)
while position != 0 and c["scrobbles"] == charts[position-1]["scrobbles"]: position -= 1
return {"scrobbles":scrobbles,"position":position + 1,"associated":others}
except: except:
# if the artist isnt in the charts, they are not being credited and we need to show information about the credited one # if the artist isnt in the charts, they are not being credited and we need to show information about the credited one
artist = coa.getCredited(artist) artist = coa.getCredited(artist)
c = [e for e in charts if e["artist"] == artist][0] c = [e for e in charts if e["artist"] == artist][0]
return {"replace":artist,"scrobbles":scrobbles,"position":charts.index(c) + 1} position = charts.index(c)
while position != 0 and c["scrobbles"] == charts[position-1]["scrobbles"]: position -= 1
return {"replace":artist,"scrobbles":scrobbles,"position":position + 1}
@dbserver.route("/trackinfo")
@dbserver.route("/trackinfo")
def trackInfo_external(): def trackInfo_external():
keys = FormsDict.decode(request.query) keys = FormsDict.decode(request.query)
ckeys = {} ckeys = {}
ckeys["artists"],ckeys["title"] = keys.getall("artist"), keys.get("title") ckeys["artists"],ckeys["title"] = keys.getall("artist"), keys.get("title")
results = trackInfo(**ckeys) results = trackInfo(**ckeys)
return results return results
def trackInfo(artists,title): def trackInfo(artists,title):
charts = db_aggregate(by="TRACK") charts = db_aggregate(by="TRACK")
scrobbles = len(db_query(artists=artists,title=title)) #we cant take the scrobble number from the charts because that includes all countas scrobbles #scrobbles = len(db_query(artists=artists,title=title)) #chart entry of track always has right scrobble number, no countas rules here
c = [e for e in charts if set(e["track"]["artists"]) == set(artists) and e["track"]["title"] == title][0] c = [e for e in charts if set(e["track"]["artists"]) == set(artists) and e["track"]["title"] == title][0]
return {"scrobbles":scrobbles,"position":charts.index(c) + 1} scrobbles = c["scrobbles"]
position = charts.index(c)
while position != 0 and c["scrobbles"] == charts[position-1]["scrobbles"]: position -= 1
return {"scrobbles":scrobbles,"position":position + 1}
@ -514,14 +521,14 @@ def pseudo_post_scrobble():
## this is necessary for localhost testing ## this is necessary for localhost testing
response.set_header("Access-Control-Allow-Origin","*") response.set_header("Access-Control-Allow-Origin","*")
createScrobble(artists,title,time) createScrobble(artists,title,time)
if (time - lastsync) > 3600: if (time - lastsync) > 3600:
sync() sync()
return "" return ""
@dbserver.post("/newscrobble") @dbserver.post("/newscrobble")
def post_scrobble(): def post_scrobble():
keys = FormsDict.decode(request.forms) # The Dal★Shabet handler keys = FormsDict.decode(request.forms) # The Dal★Shabet handler
@ -531,7 +538,7 @@ def post_scrobble():
if not (checkAPIkey(apikey)): if not (checkAPIkey(apikey)):
response.status = 403 response.status = 403
return "" return ""
try: try:
time = int(keys.get("time")) time = int(keys.get("time"))
except: except:
@ -540,15 +547,15 @@ def post_scrobble():
## this is necessary for localhost testing ## this is necessary for localhost testing
response.set_header("Access-Control-Allow-Origin","*") response.set_header("Access-Control-Allow-Origin","*")
createScrobble(artists,title,time) createScrobble(artists,title,time)
#if (time - lastsync) > 3600: #if (time - lastsync) > 3600:
# sync() # sync()
sync() #let's just always sync, not like one filesystem access every three minutes is a problem and it avoids lost tracks when we lose power sync() #let's just always sync, not like one filesystem access every three minutes is a problem and it avoids lost tracks when we lose power
return "" return ""
@dbserver.route("/sync") @dbserver.route("/sync")
def abouttoshutdown(): def abouttoshutdown():
sync() sync()
@ -562,31 +569,31 @@ def newrule():
addEntry("rules/webmade.tsv",[k for k in keys]) addEntry("rules/webmade.tsv",[k for k in keys])
global db_rulestate global db_rulestate
db_rulestate = False db_rulestate = False
@dbserver.route("/issues") @dbserver.route("/issues")
def issues_external(): #probably not even needed def issues_external(): #probably not even needed
return issues() return issues()
def issues(): def issues():
combined = [] combined = []
duplicates = [] duplicates = []
newartists = [] newartists = []
inconsistent = not db_rulestate inconsistent = not db_rulestate
# if the user manually edits files while the server is running this won't show, but too lazy to check the rulestate here # if the user manually edits files while the server is running this won't show, but too lazy to check the rulestate here
import itertools import itertools
import difflib import difflib
sortedartists = ARTISTS.copy() sortedartists = ARTISTS.copy()
sortedartists.sort(key=len,reverse=True) sortedartists.sort(key=len,reverse=True)
reversesortedartists = sortedartists.copy() reversesortedartists = sortedartists.copy()
reversesortedartists.reverse() reversesortedartists.reverse()
for a in reversesortedartists: for a in reversesortedartists:
nochange = cla.confirmedReal(a) nochange = cla.confirmedReal(a)
st = a st = a
lis = [] lis = []
reachedmyself = False reachedmyself = False
@ -596,11 +603,11 @@ def issues():
elif not reachedmyself: elif not reachedmyself:
reachedmyself = True reachedmyself = True
continue continue
if (ar.lower() == a.lower()) or ("the " + ar.lower() == a.lower()) or ("a " + ar.lower() == a.lower()): if (ar.lower() == a.lower()) or ("the " + ar.lower() == a.lower()) or ("a " + ar.lower() == a.lower()):
duplicates.append((ar,a)) duplicates.append((ar,a))
break break
if (ar + " " in st) or (" " + ar in st): if (ar + " " in st) or (" " + ar in st):
lis.append(ar) lis.append(ar)
st = st.replace(ar,"").strip() st = st.replace(ar,"").strip()
@ -610,10 +617,10 @@ def issues():
if not nochange: if not nochange:
combined.append((a,lis)) combined.append((a,lis))
break break
elif (ar in st) and len(ar)*2 > len(st): elif (ar in st) and len(ar)*2 > len(st):
duplicates.append((a,ar)) duplicates.append((a,ar))
st = st.replace("&","").replace("and","").replace("with","").strip() st = st.replace("&","").replace("and","").replace("with","").strip()
if st != "" and st != a: if st != "" and st != a:
if len(st) < 5 and len(lis) == 1: if len(st) < 5 and len(lis) == 1:
@ -626,7 +633,7 @@ def issues():
#check if we havent just randomly found the string in another word #check if we havent just randomly found the string in another word
if (" " + st + " ") in a or (a.endswith(" " + st)) or (a.startswith(st + " ")): if (" " + st + " ") in a or (a.endswith(" " + st)) or (a.startswith(st + " ")):
newartists.append((st,a,lis)) newartists.append((st,a,lis))
#for c in itertools.combinations(ARTISTS,3): #for c in itertools.combinations(ARTISTS,3):
# l = list(c) # l = list(c)
# print(l) # print(l)
@ -641,8 +648,8 @@ def issues():
# #
# if (c[0].lower == c[1].lower): # if (c[0].lower == c[1].lower):
# duplicates.append((c[0],c[1])) # duplicates.append((c[0],c[1]))
# elif (c[0] + " " in c[1]) or (" " + c[0] in c[1]) or (c[1] + " " in c[0]) or (" " + c[1] in c[0]): # elif (c[0] + " " in c[1]) or (" " + c[0] in c[1]) or (c[1] + " " in c[0]) or (" " + c[1] in c[0]):
# if (c[0] in c[1]): # if (c[0] in c[1]):
# full, part = c[1],c[0] # full, part = c[1],c[0]
@ -652,16 +659,16 @@ def issues():
# rest = c[0].replace(c[1],"").strip() # rest = c[0].replace(c[1],"").strip()
# if rest in ARTISTS and full not in [c[0] for c in combined]: # if rest in ARTISTS and full not in [c[0] for c in combined]:
# combined.append((full,part,rest)) # combined.append((full,part,rest))
# elif (c[0] in c[1]) or (c[1] in c[0]): # elif (c[0] in c[1]) or (c[1] in c[0]):
# duplicates.append((c[0],c[1])) # duplicates.append((c[0],c[1]))
return {"duplicates":duplicates,"combined":combined,"newartists":newartists,"inconsistent":inconsistent} return {"duplicates":duplicates,"combined":combined,"newartists":newartists,"inconsistent":inconsistent}
@dbserver.post("/rebuild") @dbserver.post("/rebuild")
def rebuild(): def rebuild():
keys = FormsDict.decode(request.forms) keys = FormsDict.decode(request.forms)
apikey = keys.pop("key",None) apikey = keys.pop("key",None)
if (checkAPIkey(apikey)): if (checkAPIkey(apikey)):
@ -669,6 +676,7 @@ def rebuild():
global db_rulestate global db_rulestate
db_rulestate = False db_rulestate = False
sync() sync()
invalidate_caches()
os.system("python3 fixexisting.py") os.system("python3 fixexisting.py")
global cla, coa global cla, coa
cla = CleanerAgent() cla = CleanerAgent()
@ -685,7 +693,7 @@ def search():
max_ = keys.get("max") max_ = keys.get("max")
if max_ is not None: max_ = int(max_) if max_ is not None: max_ = int(max_)
query = query.lower() query = query.lower()
artists = db_search(query,type="ARTIST") artists = db_search(query,type="ARTIST")
tracks = db_search(query,type="TRACK") tracks = db_search(query,type="TRACK")
# if the string begins with the query it's a better match, if a word in it begins with it, still good # if the string begins with the query it's a better match, if a word in it begins with it, still good
@ -693,7 +701,7 @@ def search():
artists.sort(key=lambda x: ((0 if x.lower().startswith(query) else 1 if " " + query in x.lower() else 2),len(x))) artists.sort(key=lambda x: ((0 if x.lower().startswith(query) else 1 if " " + query in x.lower() else 2),len(x)))
tracks.sort(key=lambda x: ((0 if x["title"].lower().startswith(query) else 1 if " " + query in x["title"].lower() else 2),len(x["title"]))) tracks.sort(key=lambda x: ((0 if x["title"].lower().startswith(query) else 1 if " " + query in x["title"].lower() else 2),len(x["title"])))
return {"artists":artists[:max_],"tracks":tracks[:max_]} return {"artists":artists[:max_],"tracks":tracks[:max_]}
#### ####
## Server operation ## Server operation
#### ####
@ -706,7 +714,7 @@ def runserver(PORT):
global lastsync global lastsync
lastsync = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()) lastsync = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp())
build_db() build_db()
loadAPIkeys() loadAPIkeys()
@ -714,91 +722,91 @@ def runserver(PORT):
log("Database server reachable!") log("Database server reachable!")
def build_db(): def build_db():
log("Building database...") log("Building database...")
global SCROBBLES, ARTISTS, TRACKS global SCROBBLES, ARTISTS, TRACKS
global SCROBBLESDICT, STAMPS global SCROBBLESDICT, STAMPS
SCROBBLES = [] SCROBBLES = []
ARTISTS = [] ARTISTS = []
TRACKS = [] TRACKS = []
# parse files # parse files
db = parseAllTSV("scrobbles","int","string","string",escape=False) db = parseAllTSV("scrobbles","int","string","string",escape=False)
for sc in db: for sc in db:
artists = sc[1].split("␟") artists = sc[1].split("␟")
title = sc[2] title = sc[2]
time = sc[0] time = sc[0]
readScrobble(artists,title,time) readScrobble(artists,title,time)
# optimize database # optimize database
SCROBBLES.sort(key = lambda tup: tup[1]) SCROBBLES.sort(key = lambda tup: tup[1])
#SCROBBLESDICT = {obj[1]:obj for obj in SCROBBLES} #SCROBBLESDICT = {obj[1]:obj for obj in SCROBBLES}
STAMPS = [t for t in SCROBBLESDICT] STAMPS = [t for t in SCROBBLESDICT]
STAMPS.sort() STAMPS.sort()
# inform malojatime module about earliest scrobble # inform malojatime module about earliest scrobble
register_scrobbletime(STAMPS[0]) register_scrobbletime(STAMPS[0])
# get extra artists with zero scrobbles from countas rules # get extra artists with zero scrobbles from countas rules
for artist in coa.getAllArtists(): for artist in coa.getAllArtists():
if artist not in ARTISTS: if artist not in ARTISTS:
ARTISTS.append(artist) ARTISTS.append(artist)
coa.updateIDs(ARTISTS) coa.updateIDs(ARTISTS)
global db_rulestate global db_rulestate
db_rulestate = consistentRulestate("scrobbles",cla.checksums) db_rulestate = consistentRulestate("scrobbles",cla.checksums)
# load cached images # load cached images
loadCache() loadCache()
log("Database fully built!") log("Database fully built!")
# Saves all cached entries to disk
# Saves all cached entries to disk
def sync(): def sync():
# all entries by file collected # all entries by file collected
# so we don't open the same file for every entry # so we don't open the same file for every entry
entries = {} entries = {}
for idx in range(len(SCROBBLES)): for idx in range(len(SCROBBLES)):
if not SCROBBLES[idx][2]: if not SCROBBLES[idx][2]:
t = getScrobbleObject(SCROBBLES[idx]) t = getScrobbleObject(SCROBBLES[idx])
artistlist = list(t["artists"]) artistlist = list(t["artists"])
artistlist.sort() #we want the order of artists to be deterministic so when we update files with new rules a diff can see what has actually been changed artistlist.sort() #we want the order of artists to be deterministic so when we update files with new rules a diff can see what has actually been changed
artistss = "␟".join(artistlist) artistss = "␟".join(artistlist)
timestamp = datetime.date.fromtimestamp(t["time"]) timestamp = datetime.date.fromtimestamp(t["time"])
entry = [str(t["time"]),artistss,t["title"]] entry = [str(t["time"]),artistss,t["title"]]
monthcode = str(timestamp.year) + "_" + str(timestamp.month) monthcode = str(timestamp.year) + "_" + str(timestamp.month)
entries.setdefault(monthcode,[]).append(entry) #i feckin love the setdefault function entries.setdefault(monthcode,[]).append(entry) #i feckin love the setdefault function
SCROBBLES[idx] = (SCROBBLES[idx][0],SCROBBLES[idx][1],True) SCROBBLES[idx] = (SCROBBLES[idx][0],SCROBBLES[idx][1],True)
for e in entries: for e in entries:
addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False) addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False)
combineChecksums("scrobbles/" + e + ".tsv",cla.checksums) combineChecksums("scrobbles/" + e + ".tsv",cla.checksums)
global lastsync global lastsync
lastsync = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()) lastsync = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp())
log("Database saved to disk.") log("Database saved to disk.")
# save cached images # save cached images
saveCache() saveCache()
### ###
@ -816,7 +824,7 @@ def db_query(**kwargs):
global cache_query global cache_query
key = json.dumps(kwargs) key = json.dumps(kwargs)
if key in cache_query: return copy.copy(cache_query[key]) if key in cache_query: return copy.copy(cache_query[key])
result = db_query_full(**kwargs) result = db_query_full(**kwargs)
cache_query[key] = copy.copy(result) cache_query[key] = copy.copy(result)
return result return result
@ -827,22 +835,24 @@ def db_aggregate(**kwargs):
global cache_aggregate global cache_aggregate
key = json.dumps(kwargs) key = json.dumps(kwargs)
if key in cache_aggregate: return copy.copy(cache_aggregate[key]) if key in cache_aggregate: return copy.copy(cache_aggregate[key])
result = db_aggregate_full(**kwargs) result = db_aggregate_full(**kwargs)
cache_aggregate[key] = copy.copy(result) cache_aggregate[key] = copy.copy(result)
return result return result
def invalidate_caches(): def invalidate_caches():
global cache_query, cache_aggregate global cache_query, cache_aggregate
cache_query = {} cache_query = {}
cache_aggregate = {} cache_aggregate = {}
now = datetime.datetime.now() now = datetime.datetime.now()
global cacheday global cacheday
cacheday = (now.year,now.month,now.day) cacheday = (now.year,now.month,now.day)
log("Database caches invalidated.")
def check_cache_age(): def check_cache_age():
now = datetime.datetime.now() now = datetime.datetime.utcnow()
global cacheday global cacheday
if cacheday != (now.year,now.month,now.day): invalidate_caches() if cacheday != (now.year,now.month,now.day): invalidate_caches()
@ -853,70 +863,70 @@ def check_cache_age():
# Queries the database # Queries the database
def db_query_full(artist=None,artists=None,title=None,track=None,since=None,to=None,within=None,associated=False,max_=None): def db_query_full(artist=None,artists=None,title=None,track=None,since=None,to=None,within=None,associated=False,max_=None):
(since, to) = time_stamps(since,to,within) (since, to) = time_stamps(since,to,within)
# this is not meant as a search function. we *can* query the db with a string, but it only works if it matches exactly # this is not meant as a search function. we *can* query the db with a string, but it only works if it matches exactly
# if a title is specified, we assume that a specific track (with the exact artist combination) is requested # if a title is specified, we assume that a specific track (with the exact artist combination) is requested
# if not, duplicate artist arguments are ignored # if not, duplicate artist arguments are ignored
#artist = None #artist = None
if artist is not None and isinstance(artist,str): if artist is not None and isinstance(artist,str):
artist = ARTISTS.index(artist) artist = ARTISTS.index(artist)
# artists to numbers # artists to numbers
if artists is not None: if artists is not None:
artists = set([(ARTISTS.index(a) if isinstance(a,str) else a) for a in artists]) artists = set([(ARTISTS.index(a) if isinstance(a,str) else a) for a in artists])
# track to number # track to number
if track is not None and isinstance(track,dict): if track is not None and isinstance(track,dict):
trackartists = set([(ARTISTS.index(a) if isinstance(a,str) else a) for a in track["artists"]]) trackartists = set([(ARTISTS.index(a) if isinstance(a,str) else a) for a in track["artists"]])
track = TRACKS.index((frozenset(trackartists),track["title"])) track = TRACKS.index((frozenset(trackartists),track["title"]))
artists = None artists = None
#check if track is requested via title #check if track is requested via title
if title!=None and track==None: if title!=None and track==None:
track = TRACKS.index((frozenset(artists),title)) track = TRACKS.index((frozenset(artists),title))
artists = None artists = None
# if we're not looking for a track (either directly or per title artist arguments, which is converted to track above) # if we're not looking for a track (either directly or per title artist arguments, which is converted to track above)
# we only need one artist # we only need one artist
elif artist is None and track is None and artists is not None and len(artists) != 0: elif artist is None and track is None and artists is not None and len(artists) != 0:
artist = artists.pop() artist = artists.pop()
# db query always reverse by default # db query always reverse by default
result = [] result = []
i = 0 i = 0
for s in scrobbles_in_range(since,to,reverse=True): for s in scrobbles_in_range(since,to,reverse=True):
if i == max_: break if i == max_: break
if (track is None or s[0] == track) and (artist is None or artist in TRACKS[s[0]][0] or associated and artist in coa.getCreditedList(TRACKS[s[0]][0])): if (track is None or s[0] == track) and (artist is None or artist in TRACKS[s[0]][0] or associated and artist in coa.getCreditedList(TRACKS[s[0]][0])):
result.append(getScrobbleObject(s)) result.append(getScrobbleObject(s))
i += 1 i += 1
return result return result
# pointless to check for artist when track is checked because every track has a fixed set of artists, but it's more elegant this way # pointless to check for artist when track is checked because every track has a fixed set of artists, but it's more elegant this way
# Queries that... well... aggregate # Queries that... well... aggregate
def db_aggregate_full(by=None,since=None,to=None,within=None,artist=None): def db_aggregate_full(by=None,since=None,to=None,within=None,artist=None):
(since, to) = time_stamps(since,to,within) (since, to) = time_stamps(since,to,within)
if isinstance(artist, str): if isinstance(artist, str):
artist = ARTISTS.index(artist) artist = ARTISTS.index(artist)
if (by=="ARTIST"): if (by=="ARTIST"):
#this is probably a really bad idea #this is probably a really bad idea
#for a in ARTISTS: #for a in ARTISTS:
# num = len(db_query(artist=a,since=since,to=to)) # num = len(db_query(artist=a,since=since,to=to))
# #
# alright let's try for real # alright let's try for real
charts = {} charts = {}
#for s in [scr for scr in SCROBBLES if since < scr[1] < to]: #for s in [scr for scr in SCROBBLES if since < scr[1] < to]:
@ -925,10 +935,10 @@ def db_aggregate_full(by=None,since=None,to=None,within=None,artist=None):
for a in coa.getCreditedList(artists): for a in coa.getCreditedList(artists):
# this either creates the new entry or increments the existing one # this either creates the new entry or increments the existing one
charts[a] = charts.setdefault(a,0) + 1 charts[a] = charts.setdefault(a,0) + 1
ls = [{"artist":getArtistObject(ARTISTS[a]),"scrobbles":charts[a],"counting":coa.getAllAssociated(ARTISTS[a])} for a in charts] ls = [{"artist":getArtistObject(ARTISTS[a]),"scrobbles":charts[a],"counting":coa.getAllAssociated(ARTISTS[a])} for a in charts]
return sorted(ls,key=lambda k:k["scrobbles"], reverse=True) return sorted(ls,key=lambda k:k["scrobbles"], reverse=True)
elif (by=="TRACK"): elif (by=="TRACK"):
charts = {} charts = {}
#for s in [scr for scr in SCROBBLES if since < scr[1] < to and (artist==None or (artist in TRACKS[scr[0]][0]))]: #for s in [scr for scr in SCROBBLES if since < scr[1] < to and (artist==None or (artist in TRACKS[scr[0]][0]))]:
@ -936,10 +946,10 @@ def db_aggregate_full(by=None,since=None,to=None,within=None,artist=None):
track = s[0] track = s[0]
# this either creates the new entry or increments the existing one # this either creates the new entry or increments the existing one
charts[track] = charts.setdefault(track,0) + 1 charts[track] = charts.setdefault(track,0) + 1
ls = [{"track":getTrackObject(TRACKS[t]),"scrobbles":charts[t]} for t in charts] ls = [{"track":getTrackObject(TRACKS[t]),"scrobbles":charts[t]} for t in charts]
return sorted(ls,key=lambda k:k["scrobbles"], reverse=True) return sorted(ls,key=lambda k:k["scrobbles"], reverse=True)
else: else:
#return len([scr for scr in SCROBBLES if since < scr[1] < to]) #return len([scr for scr in SCROBBLES if since < scr[1] < to])
return len(list(scrobbles_in_range(since,to))) return len(list(scrobbles_in_range(since,to)))
@ -953,14 +963,14 @@ def db_search(query,type=None):
#if query.lower() in a.lower(): #if query.lower() in a.lower():
if simplestr(query) in simplestr(a): if simplestr(query) in simplestr(a):
results.append(a) results.append(a)
if type=="TRACK": if type=="TRACK":
results = [] results = []
for t in TRACKS: for t in TRACKS:
#if query.lower() in t[1].lower(): #if query.lower() in t[1].lower():
if simplestr(query) in simplestr(t[1]): if simplestr(query) in simplestr(t[1]):
results.append(getTrackObject(t)) results.append(getTrackObject(t))
return results return results
@ -978,7 +988,7 @@ def simplestr(input,ignorecapitalization=True):
return clear return clear
def getArtistId(nameorid): def getArtistId(nameorid):
if isinstance(nameorid,int): if isinstance(nameorid,int):
return nameorid return nameorid
@ -987,8 +997,8 @@ def getArtistId(nameorid):
return ARTISTS.index(nameorid) return ARTISTS.index(nameorid)
except: except:
return -1 return -1
def insert(list_,item,key=lambda x:x): def insert(list_,item,key=lambda x:x):
i = 0 i = 0
while len(list_) > i: while len(list_) > i:
@ -996,10 +1006,10 @@ def insert(list_,item,key=lambda x:x):
list_.insert(i,item) list_.insert(i,item)
return i return i
i += 1 i += 1
list_.append(item) list_.append(item)
return i return i
def scrobbles_in_range(start,end,reverse=False): def scrobbles_in_range(start,end,reverse=False):
if reverse: if reverse:
@ -1014,7 +1024,7 @@ def scrobbles_in_range(start,end,reverse=False):
if stamp < start: continue if stamp < start: continue
if stamp > end: return if stamp > end: return
yield SCROBBLESDICT[stamp] yield SCROBBLESDICT[stamp]
# for performance testing # for performance testing
def generateStuff(num=0,pertrack=0,mult=0): def generateStuff(num=0,pertrack=0,mult=0):
@ -1024,14 +1034,14 @@ def generateStuff(num=0,pertrack=0,mult=0):
t = getTrackObject(track) t = getTrackObject(track)
time = random.randint(STAMPS[0],STAMPS[-1]) time = random.randint(STAMPS[0],STAMPS[-1])
createScrobble(t["artists"],t["title"],time,volatile=True) createScrobble(t["artists"],t["title"],time,volatile=True)
for track in TRACKS: for track in TRACKS:
t = getTrackObject(track) t = getTrackObject(track)
for i in range(pertrack): for i in range(pertrack):
time = random.randint(STAMPS[0],STAMPS[-1]) time = random.randint(STAMPS[0],STAMPS[-1])
createScrobble(t["artists"],t["title"],time,volatile=True) createScrobble(t["artists"],t["title"],time,volatile=True)
for scrobble in SCROBBLES: for scrobble in SCROBBLES:
s = getScrobbleObject(scrobble) s = getScrobbleObject(scrobble)
for i in range(mult): for i in range(mult):
createScrobble(s["artists"],s["title"],s["time"] - i*500,volatile=True) createScrobble(s["artists"],s["title"],s["time"] - i*500,volatile=True)

155
htmlmodules.py
View File

@ -17,11 +17,11 @@ import urllib
# artist=None,track=None,since=None,to=None,within=None,associated=False,max_=None,pictures=False # artist=None,track=None,since=None,to=None,within=None,associated=False,max_=None,pictures=False
def module_scrobblelist(max_=None,pictures=False,shortTimeDesc=False,earlystop=False,**kwargs): def module_scrobblelist(max_=None,pictures=False,shortTimeDesc=False,earlystop=False,**kwargs):
kwargs_filter = pickKeys(kwargs,"artist","track","associated") kwargs_filter = pickKeys(kwargs,"artist","track","associated")
kwargs_time = pickKeys(kwargs,"since","to","within") kwargs_time = pickKeys(kwargs,"since","to","within")
# if earlystop, we don't care about the actual amount and only request as many from the db # if earlystop, we don't care about the actual amount and only request as many from the db
# without, we request everything and filter on site # without, we request everything and filter on site
maxkey = {"max_":max_} if earlystop else {} maxkey = {"max_":max_} if earlystop else {}
@ -31,14 +31,14 @@ def module_scrobblelist(max_=None,pictures=False,shortTimeDesc=False,earlystop=F
#scrobbleimages = [e.get("image") for e in getTracksInfo(scrobbleswithpictures)] #will still work with scrobble objects as they are a technically a subset of track objects #scrobbleimages = [e.get("image") for e in getTracksInfo(scrobbleswithpictures)] #will still work with scrobble objects as they are a technically a subset of track objects
#scrobbleimages = ["/image?title=" + urllib.parse.quote(t["title"]) + "&" + "&".join(["artist=" + urllib.parse.quote(a) for a in t["artists"]]) for t in scrobbleswithpictures] #scrobbleimages = ["/image?title=" + urllib.parse.quote(t["title"]) + "&" + "&".join(["artist=" + urllib.parse.quote(a) for a in t["artists"]]) for t in scrobbleswithpictures]
scrobbleimages = [getTrackImage(t["artists"],t["title"],fast=True) for t in scrobbleswithpictures] scrobbleimages = [getTrackImage(t["artists"],t["title"],fast=True) for t in scrobbleswithpictures]
representative = scrobbles[0] if len(scrobbles) is not 0 else None representative = scrobbles[0] if len(scrobbles) is not 0 else None
# build list # build list
i = 0 i = 0
html = "<table class='list'>" html = "<table class='list'>"
for s in scrobbles: for s in scrobbles:
html += "<tr>" html += "<tr>"
html += "<td class='time'>" + time_desc(s["time"],short=shortTimeDesc) + "</td>" html += "<td class='time'>" + time_desc(s["time"],short=shortTimeDesc) + "</td>"
if pictures: if pictures:
@ -48,34 +48,34 @@ def module_scrobblelist(max_=None,pictures=False,shortTimeDesc=False,earlystop=F
# Alternative way: Do it in one cell # Alternative way: Do it in one cell
#html += "<td class='title'><span>" + artistLinks(s["artists"]) + "</span> — " + trackLink({"artists":s["artists"],"title":s["title"]}) + "</td>" #html += "<td class='title'><span>" + artistLinks(s["artists"]) + "</span> — " + trackLink({"artists":s["artists"],"title":s["title"]}) + "</td>"
html += "</tr>" html += "</tr>"
i += 1 i += 1
if max_ is not None and i>=max_: if max_ is not None and i>=max_:
break break
html += "</table>" html += "</table>"
return (html,len(scrobbles),representative) return (html,len(scrobbles),representative)
def module_pulse(max_=None,**kwargs): def module_pulse(max_=None,**kwargs):
kwargs_filter = pickKeys(kwargs,"artist","track","associated") kwargs_filter = pickKeys(kwargs,"artist","track","associated")
kwargs_time = pickKeys(kwargs,"since","to","within","step","stepn","trail") kwargs_time = pickKeys(kwargs,"since","to","within","step","stepn","trail")
ranges = database.get_pulse(**kwargs_time,**kwargs_filter) ranges = database.get_pulse(**kwargs_time,**kwargs_filter)
if max_ is not None: ranges = ranges[:max_] if max_ is not None: ranges = ranges[:max_]
# if time range not explicitly specified, only show from first appearance # if time range not explicitly specified, only show from first appearance
# if "since" not in kwargs: # if "since" not in kwargs:
# while ranges[0]["scrobbles"] == 0: # while ranges[0]["scrobbles"] == 0:
# del ranges[0] # del ranges[0]
maxbar = max([t["scrobbles"] for t in ranges]) maxbar = max([t["scrobbles"] for t in ranges])
maxbar = max(maxbar,1) maxbar = max(maxbar,1)
#build list #build list
html = "<table class='list'>" html = "<table class='list'>"
for t in ranges: for t in ranges:
@ -87,24 +87,24 @@ def module_pulse(max_=None,**kwargs):
html += "<td class='bar'>" + scrobblesLink({"since":fromstr,"to":tostr},percent=t["scrobbles"]*100/maxbar,**kwargs_filter) + "</td>" html += "<td class='bar'>" + scrobblesLink({"since":fromstr,"to":tostr},percent=t["scrobbles"]*100/maxbar,**kwargs_filter) + "</td>"
html += "</tr>" html += "</tr>"
html += "</table>" html += "</table>"
return html return html
def module_trackcharts(max_=None,**kwargs): def module_trackcharts(max_=None,**kwargs):
kwargs_filter = pickKeys(kwargs,"artist","associated") kwargs_filter = pickKeys(kwargs,"artist","associated")
kwargs_time = pickKeys(kwargs,"since","to","within") kwargs_time = pickKeys(kwargs,"since","to","within")
tracks = database.get_charts_tracks(**kwargs_filter,**kwargs_time) tracks = database.get_charts_tracks(**kwargs_filter,**kwargs_time)
if tracks != []: if tracks != []:
maxbar = tracks[0]["scrobbles"] maxbar = tracks[0]["scrobbles"]
representative = tracks[0]["track"] representative = tracks[0]["track"]
else: else:
representative = None representative = None
i = 0 i = 0
html = "<table class='list'>" html = "<table class='list'>"
for e in tracks: for e in tracks:
@ -112,31 +112,35 @@ def module_trackcharts(max_=None,**kwargs):
if max_ is not None and i>max_: if max_ is not None and i>max_:
break break
html += "<tr>" html += "<tr>"
html += "<td class='rank'>#" + str(i) + "</td>" if i == 1 or e["scrobbles"] < prev["scrobbles"]:
html += "<td class='rank'>#" + str(i) + "</td>"
else:
html += "<td class='rank'></td>"
html += "<td class='artists'>" + artistLinks(e["track"]["artists"]) + "</td>" html += "<td class='artists'>" + artistLinks(e["track"]["artists"]) + "</td>"
html += "<td class='title'>" + trackLink(e["track"]) + "</td>" html += "<td class='title'>" + trackLink(e["track"]) + "</td>"
html += "<td class='amount'>" + scrobblesTrackLink(e["track"],kwargs_time,amount=e["scrobbles"]) + "</td>" html += "<td class='amount'>" + scrobblesTrackLink(e["track"],kwargs_time,amount=e["scrobbles"]) + "</td>"
html += "<td class='bar'>" + scrobblesTrackLink(e["track"],kwargs_time,percent=e["scrobbles"]*100/maxbar) + "</td>" html += "<td class='bar'>" + scrobblesTrackLink(e["track"],kwargs_time,percent=e["scrobbles"]*100/maxbar) + "</td>"
html += "</tr>" html += "</tr>"
prev = e
html += "</table>" html += "</table>"
return (html,representative) return (html,representative)
def module_artistcharts(max_=None,**kwargs): def module_artistcharts(max_=None,**kwargs):
kwargs_filter = pickKeys(kwargs,"associated") #not used right now kwargs_filter = pickKeys(kwargs,"associated") #not used right now
kwargs_time = pickKeys(kwargs,"since","to","within") kwargs_time = pickKeys(kwargs,"since","to","within")
artists = database.get_charts_artists(**kwargs_filter,**kwargs_time) artists = database.get_charts_artists(**kwargs_filter,**kwargs_time)
if artists != []: if artists != []:
maxbar = artists[0]["scrobbles"] maxbar = artists[0]["scrobbles"]
representative = artists[0]["artist"] representative = artists[0]["artist"]
else: else:
representative = None representative = None
i = 0 i = 0
html = "<table class='list'>" html = "<table class='list'>"
for e in artists: for e in artists:
@ -144,7 +148,10 @@ def module_artistcharts(max_=None,**kwargs):
if max_ is not None and i>max_: if max_ is not None and i>max_:
break break
html += "<tr>" html += "<tr>"
html += "<td class='rank'>#" + str(i) + "</td>" if i == 1 or e["scrobbles"] < prev["scrobbles"]:
html += "<td class='rank'>#" + str(i) + "</td>"
else:
html += "<td class='rank'></td>"
html += "<td class='artist'>" + artistLink(e["artist"]) html += "<td class='artist'>" + artistLink(e["artist"])
if (e["counting"] != []): if (e["counting"] != []):
html += " <span class='extra'>incl. " + ", ".join([artistLink(a) for a in e["counting"]]) + "</span>" html += " <span class='extra'>incl. " + ", ".join([artistLink(a) for a in e["counting"]]) + "</span>"
@ -152,39 +159,40 @@ def module_artistcharts(max_=None,**kwargs):
html += "<td class='amount'>" + scrobblesArtistLink(e["artist"],kwargs_time,amount=e["scrobbles"],associated=True) + "</td>" html += "<td class='amount'>" + scrobblesArtistLink(e["artist"],kwargs_time,amount=e["scrobbles"],associated=True) + "</td>"
html += "<td class='bar'>" + scrobblesArtistLink(e["artist"],kwargs_time,percent=e["scrobbles"]*100/maxbar,associated=True) + "</td>" html += "<td class='bar'>" + scrobblesArtistLink(e["artist"],kwargs_time,percent=e["scrobbles"]*100/maxbar,associated=True) + "</td>"
html += "</tr>" html += "</tr>"
prev = e
html += "</table>" html += "</table>"
return (html, representative) return (html, representative)
def module_artistcharts_tiles(**kwargs): def module_artistcharts_tiles(**kwargs):
kwargs_filter = pickKeys(kwargs,"associated") #not used right now kwargs_filter = pickKeys(kwargs,"associated") #not used right now
kwargs_time = pickKeys(kwargs,"since","to","within") kwargs_time = pickKeys(kwargs,"since","to","within")
artists = database.get_charts_artists(**kwargs_filter,**kwargs_time)[:14] artists = database.get_charts_artists(**kwargs_filter,**kwargs_time)[:14]
while len(artists)<14: artists.append(None) while len(artists)<14: artists.append(None)
i = 1 i = 1
bigpart = [0,1,2,6,15] bigpart = [0,1,2,6,15]
smallpart = [0,1,2,4,6,9,12,15] smallpart = [0,1,2,4,6,9,12,15]
rnk = (0,0) #temporary store so entries with the same scrobble amount get the same rank rnk = (0,0) #temporary store so entries with the same scrobble amount get the same rank
html = """<table class="tiles_top"><tr>""" html = """<table class="tiles_top"><tr>"""
for e in artists: for e in artists:
if i in bigpart: if i in bigpart:
n = bigpart.index(i) n = bigpart.index(i)
html += """<td><table class="tiles_""" + str(n) + """x""" + str(n) + """ tiles_sub">""" html += """<td><table class="tiles_""" + str(n) + """x""" + str(n) + """ tiles_sub">"""
if i in smallpart: if i in smallpart:
html += "<tr>" html += "<tr>"
if e is not None: if e is not None:
rank = i if e["scrobbles"] != rnk[1] else rnk[0] rank = i if e["scrobbles"] != rnk[1] else rnk[0]
rnk = (rank,e["scrobbles"]) rnk = (rank,e["scrobbles"])
@ -196,51 +204,51 @@ def module_artistcharts_tiles(**kwargs):
rank = "" rank = ""
image = "" image = ""
link = "" link = ""
html += """<td style="background-image:url('""" + image + """')"><span class="stats">""" + rank + "</span> <span>" + link + "</span></td>" html += """<td style="background-image:url('""" + image + """')"><span class="stats">""" + rank + "</span> <span>" + link + "</span></td>"
i += 1 i += 1
if i in smallpart: if i in smallpart:
html += "</tr>" html += "</tr>"
if i in bigpart: if i in bigpart:
html += "</table></td>" html += "</table></td>"
html += """</tr></table>""" html += """</tr></table>"""
return html return html
def module_trackcharts_tiles(**kwargs): def module_trackcharts_tiles(**kwargs):
kwargs_filter = pickKeys(kwargs,"artist","associated") kwargs_filter = pickKeys(kwargs,"artist","associated")
kwargs_time = pickKeys(kwargs,"since","to","within") kwargs_time = pickKeys(kwargs,"since","to","within")
tracks = database.get_charts_tracks(**kwargs_filter,**kwargs_time)[:14] tracks = database.get_charts_tracks(**kwargs_filter,**kwargs_time)[:14]
while len(tracks)<14: tracks.append(None) #{"track":{"title":"","artists":[]}} while len(tracks)<14: tracks.append(None) #{"track":{"title":"","artists":[]}}
i = 1 i = 1
bigpart = [0,1,2,6,15] bigpart = [0,1,2,6,15]
smallpart = [0,1,2,4,6,9,12,15] smallpart = [0,1,2,4,6,9,12,15]
rnk = (0,0) #temporary store so entries with the same scrobble amount get the same rank rnk = (0,0) #temporary store so entries with the same scrobble amount get the same rank
html = """<table class="tiles_top"><tr>""" html = """<table class="tiles_top"><tr>"""
for e in tracks: for e in tracks:
if i in bigpart: if i in bigpart:
n = bigpart.index(i) n = bigpart.index(i)
html += """<td><table class="tiles_""" + str(n) + """x""" + str(n) + """ tiles_sub">""" html += """<td><table class="tiles_""" + str(n) + """x""" + str(n) + """ tiles_sub">"""
if i in smallpart: if i in smallpart:
html += "<tr>" html += "<tr>"
if e is not None: if e is not None:
rank = i if e["scrobbles"] != rnk[1] else rnk[0] rank = i if e["scrobbles"] != rnk[1] else rnk[0]
rnk = (rank,e["scrobbles"]) rnk = (rank,e["scrobbles"])
@ -252,18 +260,17 @@ def module_trackcharts_tiles(**kwargs):
rank = "" rank = ""
image = "" image = ""
link = "" link = ""
html += """<td style="background-image:url('""" + image + """')"><span class="stats">""" + rank + "</span> <span>" + link + "</span></td>" html += """<td style="background-image:url('""" + image + """')"><span class="stats">""" + rank + "</span> <span>" + link + "</span></td>"
i += 1 i += 1
if i in smallpart: if i in smallpart:
html += "</tr>" html += "</tr>"
if i in bigpart: if i in bigpart:
html += "</table></td>" html += "</table></td>"
html += """</tr></table>"""
return html
html += """</tr></table>"""
return html

33
server.py
View File

@ -53,7 +53,7 @@ def database_get(pth):
except HTTPError as e: except HTTPError as e:
response.status = e.code response.status = e.code
return return
@webserver.post("/db/<pth:path>") @webserver.post("/db/<pth:path>")
def database_post(pth): def database_post(pth):
response.set_header("Access-Control-Allow-Origin","*") response.set_header("Access-Control-Allow-Origin","*")
@ -66,12 +66,10 @@ def database_post(pth):
except HTTPError as e: except HTTPError as e:
response.status = e.code response.status = e.code
return return
return return
def graceful_exit(sig=None,frame=None): def graceful_exit(sig=None,frame=None):
urllib.request.urlopen("http://[::1]:" + str(DATABASE_PORT) + "/sync") urllib.request.urlopen("http://[::1]:" + str(DATABASE_PORT) + "/sync")
log("Server shutting down...") log("Server shutting down...")
@ -110,7 +108,7 @@ def static_image(pth):
response = static_file("images/" + pth,root="") response = static_file("images/" + pth,root="")
except: except:
response = static_file("images/" + pth,root="") response = static_file("images/" + pth,root="")
#response = static_file("images/" + pth,root="") #response = static_file("images/" + pth,root="")
response.set_header("Cache-Control", "public, max-age=604800") response.set_header("Cache-Control", "public, max-age=604800")
return response return response
@ -121,19 +119,19 @@ def static_image(pth):
@webserver.route("/<name:re:.*\\.png>") @webserver.route("/<name:re:.*\\.png>")
@webserver.route("/<name:re:.*\\.jpeg>") @webserver.route("/<name:re:.*\\.jpeg>")
@webserver.route("/<name:re:.*\\.ico>") @webserver.route("/<name:re:.*\\.ico>")
def static(name): def static(name):
response = static_file("website/" + name,root="") response = static_file("website/" + name,root="")
response.set_header("Cache-Control", "public, max-age=604800") response.set_header("Cache-Control", "public, max-age=604800")
return response return response
@webserver.route("/<name>") @webserver.route("/<name>")
def static_html(name): def static_html(name):
linkheaders = ["</maloja.css>; rel=preload; as=style"] linkheaders = ["</maloja.css>; rel=preload; as=style"]
keys = removeIdentical(FormsDict.decode(request.query)) keys = removeIdentical(FormsDict.decode(request.query))
with open("website/" + name + ".html") as htmlfile: with open("website/" + name + ".html") as htmlfile:
html = htmlfile.read() html = htmlfile.read()
# apply global substitutions # apply global substitutions
with open("website/common/footer.html") as footerfile: with open("website/common/footer.html") as footerfile:
footerhtml = footerfile.read() footerhtml = footerfile.read()
@ -141,16 +139,16 @@ def static_html(name):
headerhtml = headerfile.read() headerhtml = headerfile.read()
html = html.replace("</body>",footerhtml + "</body>").replace("</head>",headerhtml + "</head>") html = html.replace("</body>",footerhtml + "</body>").replace("</head>",headerhtml + "</head>")
# If a python file exists, it provides the replacement dict for the html file # If a python file exists, it provides the replacement dict for the html file
if os.path.exists("website/" + name + ".py"): if os.path.exists("website/" + name + ".py"):
#txt_keys = SourceFileLoader(name,"website/" + name + ".py").load_module().replacedict(keys,DATABASE_PORT) #txt_keys = SourceFileLoader(name,"website/" + name + ".py").load_module().replacedict(keys,DATABASE_PORT)
txt_keys,resources = SourceFileLoader(name,"website/" + name + ".py").load_module().instructions(keys) txt_keys,resources = SourceFileLoader(name,"website/" + name + ".py").load_module().instructions(keys)
# add headers for server push # add headers for server push
for resource in resources: for resource in resources:
linkheaders.append("<" + resource["file"] + ">; rel=preload; as=" + resource["type"]) linkheaders.append("<" + resource["file"] + ">; rel=preload; as=" + resource["type"])
# apply key substitutions # apply key substitutions
for k in txt_keys: for k in txt_keys:
if isinstance(txt_keys[k],list): if isinstance(txt_keys[k],list):
@ -160,9 +158,9 @@ def static_html(name):
else: else:
html = html.replace(k,txt_keys[k]) html = html.replace(k,txt_keys[k])
response.set_header("Link",",".join(linkheaders)) response.set_header("Link",",".join(linkheaders))
return html return html
#return static_file("website/" + name + ".html",root="") #return static_file("website/" + name + ".html",root="")
@ -172,8 +170,9 @@ signal.signal(signal.SIGTERM, graceful_exit)
#rename process, this is now required for the daemon manager to work #rename process, this is now required for the daemon manager to work
setproctitle.setproctitle("Maloja") setproctitle.setproctitle("Maloja")
## start database server ## start database server
_thread.start_new_thread(SourceFileLoader("database","database.py").load_module().runserver,(DATABASE_PORT,)) _thread.start_new_thread(SourceFileLoader("database","database.py").load_module().runserver,(DATABASE_PORT,))
log("Starting up Maloja server...")
run(webserver, host='::', port=MAIN_PORT, server='waitress') run(webserver, host='::', port=MAIN_PORT, server='waitress')