mirror of
https://github.com/krateng/maloja.git
synced 2023-08-10 21:12:55 +03:00
Reimplemented caching of yearly and weekly stats
This commit is contained in:
krateng
parent
df07dd7b00
commit
eb9cd4aba4
@ -10,6 +10,7 @@ from ..thirdparty import proxy_scrobble_all
|
||||
from ..globalconf import data_dir, malojaconfig, apikeystore
|
||||
#db
|
||||
from . import sqldb
|
||||
from . import cached
|
||||
|
||||
# doreah toolkit
|
||||
from doreah.logging import log
|
||||
@ -42,9 +43,9 @@ import urllib
|
||||
|
||||
|
||||
dbstatus = {
|
||||
"healthy":False,
|
||||
"healthy":False, # we can access the db
|
||||
"rebuildinprogress":False,
|
||||
"complete":False
|
||||
"complete":False # information is complete
|
||||
}
|
||||
class DatabaseNotBuilt(HTTPError):
|
||||
def __init__(self):
|
||||
@ -235,19 +236,17 @@ def artist_info(artist):
|
||||
c = [e for e in alltimecharts if e["artist"] == artist][0]
|
||||
others = sqldb.get_associated_artists(artist)
|
||||
position = c["rank"]
|
||||
performance_weekly = get_performance(artist=artist,step="week")[:-1] #current week doesn't count
|
||||
performance_yearly = get_performance(artist=artist,step="year")[:-1] #current year doesn't count
|
||||
return {
|
||||
"artist":artist,
|
||||
"scrobbles":scrobbles,
|
||||
"position":position,
|
||||
"associated":others,
|
||||
"medals":{
|
||||
"gold":[e['range'] for e in performance_yearly if e['rank'] == 1],
|
||||
"silver":[e['range'] for e in performance_yearly if e['rank'] == 2],
|
||||
"bronze":[e['range'] for e in performance_yearly if e['rank'] == 3]
|
||||
"gold": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['gold']],
|
||||
"silver": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['silver']],
|
||||
"bronze": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['bronze']],
|
||||
},
|
||||
"topweeks":len([e for e in performance_weekly if e['rank'] == 1])
|
||||
"topweeks":len([e for e in cached.weekly_topartists if e == artist])
|
||||
}
|
||||
except:
|
||||
# if the artist isnt in the charts, they are not being credited and we
|
||||
@ -276,21 +275,18 @@ def track_info(track):
|
||||
elif scrobbles >= threshold_platinum: cert = "platinum"
|
||||
elif scrobbles >= threshold_gold: cert = "gold"
|
||||
|
||||
performance_weekly = get_performance(track=track,step="week")[:-1] #current week doesn't count
|
||||
performance_yearly = get_performance(track=track,step="year")[:-1] #current year doesn't count
|
||||
|
||||
|
||||
return {
|
||||
"track":track,
|
||||
"scrobbles":scrobbles,
|
||||
"position":position,
|
||||
"medals":{
|
||||
"gold":[e['range'] for e in performance_yearly if e['rank'] == 1],
|
||||
"silver":[e['range'] for e in performance_yearly if e['rank'] == 2],
|
||||
"bronze":[e['range'] for e in performance_yearly if e['rank'] == 3]
|
||||
"gold": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['gold']],
|
||||
"silver": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['silver']],
|
||||
"bronze": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['bronze']],
|
||||
},
|
||||
"certification":cert,
|
||||
"topweeks":len([e for e in performance_weekly if e['rank'] == 1])
|
||||
"topweeks":len([e for e in cached.weekly_toptracks if e == track])
|
||||
}
|
||||
|
||||
|
||||
@ -452,18 +448,28 @@ def get_predefined_rulesets():
|
||||
|
||||
|
||||
def start_db():
|
||||
# Upgrade database
|
||||
from .. import upgrade
|
||||
upgrade.upgrade_db(sqldb.add_scrobbles)
|
||||
|
||||
# Load temporary tables
|
||||
from . import associated
|
||||
associated.load_associated_rules()
|
||||
|
||||
dbstatus['healthy'] = True
|
||||
dbstatus['complete'] = True
|
||||
|
||||
# inform time module about begin of scrobbling
|
||||
firstscrobble = sqldb.get_scrobbles()[0]
|
||||
register_scrobbletime(firstscrobble['time'])
|
||||
|
||||
# create cached information
|
||||
cached.update_medals()
|
||||
cached.update_weekly()
|
||||
|
||||
dbstatus['complete'] = True
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
69
maloja/database/cached.py
Normal file
69
maloja/database/cached.py
Normal file
@ -0,0 +1,69 @@
|
||||
# for information that is not authorative, but should be saved anyway because it
|
||||
# changes infrequently and DB access is expensive
|
||||
|
||||
from doreah.regular import yearly, daily
|
||||
from .. import database
|
||||
from .. import malojatime as mjt
|
||||
|
||||
|
||||
|
||||
medals_artists = {
|
||||
# year: {'gold':[],'silver':[],'bronze':[]}
|
||||
}
|
||||
medals_tracks = {
|
||||
# year: {'gold':[],'silver':[],'bronze':[]}
|
||||
}
|
||||
|
||||
weekly_topartists = []
|
||||
weekly_toptracks = []
|
||||
|
||||
@yearly
|
||||
def update_medals():
|
||||
|
||||
global medals_artists, medals_tracks
|
||||
medals_artists.clear()
|
||||
medals_tracks.clear()
|
||||
|
||||
for year in mjt.ranges(step="year"):
|
||||
if year == mjt.thisyear(): break
|
||||
|
||||
charts_artists = database.get_charts_artists(timerange=year)
|
||||
charts_tracks = database.get_charts_tracks(timerange=year)
|
||||
|
||||
entry_artists = {'gold':[],'silver':[],'bronze':[]}
|
||||
entry_tracks = {'gold':[],'silver':[],'bronze':[]}
|
||||
medals_artists[year.desc()] = entry_artists
|
||||
medals_tracks[year.desc()] = entry_tracks
|
||||
|
||||
for entry in charts_artists:
|
||||
if entry['rank'] == 1: entry_artists['gold'].append(entry['artist'])
|
||||
elif entry['rank'] == 2: entry_artists['silver'].append(entry['artist'])
|
||||
elif entry['rank'] == 3: entry_artists['bronze'].append(entry['artist'])
|
||||
else: break
|
||||
for entry in charts_tracks:
|
||||
if entry['rank'] == 1: entry_tracks['gold'].append(entry['track'])
|
||||
elif entry['rank'] == 2: entry_tracks['silver'].append(entry['track'])
|
||||
elif entry['rank'] == 3: entry_tracks['bronze'].append(entry['track'])
|
||||
else: break
|
||||
|
||||
|
||||
|
||||
@daily
|
||||
def update_weekly():
|
||||
|
||||
global weekly_topartists, weekly_toptracks
|
||||
weekly_topartists.clear()
|
||||
weekly_toptracks.clear()
|
||||
|
||||
for week in mjt.ranges(step="week"):
|
||||
if week == mjt.thisweek(): break
|
||||
|
||||
charts_artists = database.get_charts_artists(timerange=week)
|
||||
charts_tracks = database.get_charts_tracks(timerange=week)
|
||||
|
||||
for entry in charts_artists:
|
||||
if entry['rank'] == 1: weekly_topartists.append(entry['artist'])
|
||||
else: break
|
||||
for entry in charts_tracks:
|
||||
if entry['rank'] == 1: weekly_toptracks.append(entry['track'])
|
||||
else: break
|
||||
@ -1,3 +1,2 @@
|
||||
from .images import *
|
||||
from .maintenance import *
|
||||
from .utils import *
|
||||
|
||||
@ -1,114 +0,0 @@
|
||||
from ..__pkginfo__ import VERSION
|
||||
from ..malojatime import ranges, thisweek, thisyear
|
||||
from ..globalconf import malojaconfig
|
||||
|
||||
from doreah.regular import yearly, daily
|
||||
from doreah.logging import log
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import urllib
|
||||
import itertools
|
||||
|
||||
|
||||
|
||||
get_track = lambda x:(frozenset(x["track"]["artists"]),x["track"]["title"])
|
||||
get_artist = lambda x:x["artist"]
|
||||
|
||||
def group_by_attribute(sequence,attribute):
|
||||
grouped = itertools.groupby(sequence,key=lambda x:x[attribute])
|
||||
for attrvalue,members in grouped:
|
||||
yield attrvalue,list(members)
|
||||
|
||||
def collect_rankings(chart,identify,collection,iteration=None,count=True):
|
||||
grouped = group_by_attribute(chart,"rank")
|
||||
for rank, members in grouped:
|
||||
if not count and rank not in rankmedals: break
|
||||
if count and rank != 1: break
|
||||
|
||||
for m in members:
|
||||
# get the actual object that we're counting
|
||||
entity = identify(m)
|
||||
|
||||
# count no1 spots
|
||||
if count:
|
||||
collection[entity] = collection.setdefault(entity,0) + 1
|
||||
|
||||
# collect instances of top3 spots
|
||||
else:
|
||||
medal = rankmedals[rank]
|
||||
collection.setdefault(entity,{}).setdefault(medal,[]).append(iteration)
|
||||
|
||||
|
||||
rankmedals = {
|
||||
1:'gold',
|
||||
2:'silver',
|
||||
3:'bronze'
|
||||
}
|
||||
|
||||
@yearly
|
||||
def update_medals():
|
||||
|
||||
|
||||
from ..database import MEDALS_ARTISTS, MEDALS_TRACKS, STAMPS, get_charts_artists, get_charts_tracks
|
||||
|
||||
|
||||
MEDALS_ARTISTS.clear()
|
||||
MEDALS_TRACKS.clear()
|
||||
|
||||
for year in ranges(step="year"):
|
||||
if year == thisyear(): break
|
||||
|
||||
charts_artists = get_charts_artists(timerange=year)
|
||||
charts_tracks = get_charts_tracks(timerange=year)
|
||||
|
||||
collect_rankings(charts_artists,get_artist,MEDALS_ARTISTS,iteration=year,count=False)
|
||||
collect_rankings(charts_tracks,get_track,MEDALS_TRACKS,iteration=year,count=False)
|
||||
|
||||
|
||||
@daily
|
||||
def update_weekly():
|
||||
|
||||
from ..database import WEEKLY_TOPTRACKS, WEEKLY_TOPARTISTS, get_charts_artists, get_charts_tracks
|
||||
|
||||
|
||||
WEEKLY_TOPARTISTS.clear()
|
||||
WEEKLY_TOPTRACKS.clear()
|
||||
|
||||
for week in ranges(step="week"):
|
||||
if week == thisweek(): break
|
||||
|
||||
charts_artists = get_charts_artists(timerange=week)
|
||||
charts_tracks = get_charts_tracks(timerange=week)
|
||||
|
||||
collect_rankings(charts_artists,get_artist,WEEKLY_TOPARTISTS)
|
||||
collect_rankings(charts_tracks,get_track,WEEKLY_TOPTRACKS)
|
||||
|
||||
|
||||
@daily
|
||||
def send_stats():
|
||||
if malojaconfig["SEND_STATS"]:
|
||||
|
||||
log("Sending daily stats report...")
|
||||
|
||||
from ..database import ARTISTS, TRACKS, SCROBBLES
|
||||
|
||||
keys = {
|
||||
"url":"https://myrcella.krateng.ch/malojastats",
|
||||
"method":"POST",
|
||||
"headers":{"Content-Type": "application/json"},
|
||||
"data":json.dumps({
|
||||
"name":malojaconfig["NAME"],
|
||||
"url":malojaconfig["PUBLIC_URL"],
|
||||
"version":VERSION,
|
||||
"artists":len(ARTISTS),
|
||||
"tracks":len(TRACKS),
|
||||
"scrobbles":len(SCROBBLES)
|
||||
}).encode("utf-8")
|
||||
}
|
||||
try:
|
||||
req = urllib.request.Request(**keys)
|
||||
response = urllib.request.urlopen(req)
|
||||
log("Sent daily report!")
|
||||
except:
|
||||
log("Could not send daily report!")
|
||||
Loading…
Reference in New Issue
Block a user