Reimplemented caching of yearly and weekly stats

maloja/database/__init__.py

@@ -10,6 +10,7 @@ from ..thirdparty import proxy_scrobble_all
 from ..globalconf import data_dir, malojaconfig, apikeystore
 #db
 from . import sqldb
+from . import cached
 
 # doreah toolkit
 from doreah.logging import log
@@ -42,9 +43,9 @@ import urllib
 
 
 dbstatus = {
-	"healthy":False,
+	"healthy":False,			# we can access the db
 	"rebuildinprogress":False,
-	"complete":False
+	"complete":False			# information is complete
 }
 class DatabaseNotBuilt(HTTPError):
 	def __init__(self):
@@ -235,19 +236,17 @@ def artist_info(artist):
 		c = [e for e in alltimecharts if e["artist"] == artist][0]
 		others = sqldb.get_associated_artists(artist)
 		position = c["rank"]
-		performance_weekly = get_performance(artist=artist,step="week")[:-1] #current week doesn't count
-		performance_yearly = get_performance(artist=artist,step="year")[:-1] #current year doesn't count
 		return {
 			"artist":artist,
 			"scrobbles":scrobbles,
 			"position":position,
 			"associated":others,
 			"medals":{
-				"gold":[e['range'] for e in performance_yearly if e['rank'] == 1],
+				"gold": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['gold']],
-				"silver":[e['range'] for e in performance_yearly if e['rank'] == 2],
+				"silver": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['silver']],
-				"bronze":[e['range'] for e in performance_yearly if e['rank'] == 3]
+				"bronze": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['bronze']],
 			},
-			"topweeks":len([e for e in performance_weekly if e['rank'] == 1])
+			"topweeks":len([e for e in cached.weekly_topartists if e == artist])
 		}
 	except:
 		# if the artist isnt in the charts, they are not being credited and we
@@ -276,21 +275,18 @@ def track_info(track):
 	elif scrobbles >= threshold_platinum: cert = "platinum"
 	elif scrobbles >= threshold_gold: cert = "gold"
 
-	performance_weekly = get_performance(track=track,step="week")[:-1] #current week doesn't count
-	performance_yearly = get_performance(track=track,step="year")[:-1] #current year doesn't count
-
 
 	return {
 		"track":track,
 		"scrobbles":scrobbles,
 		"position":position,
 		"medals":{
-			"gold":[e['range'] for e in performance_yearly if e['rank'] == 1],
+			"gold": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['gold']],
-			"silver":[e['range'] for e in performance_yearly if e['rank'] == 2],
+			"silver": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['silver']],
-			"bronze":[e['range'] for e in performance_yearly if e['rank'] == 3]
+			"bronze": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['bronze']],
 		},
 		"certification":cert,
-		"topweeks":len([e for e in performance_weekly if e['rank'] == 1])
+		"topweeks":len([e for e in cached.weekly_toptracks if e == track])
 	}
 
 
@@ -452,18 +448,28 @@ def get_predefined_rulesets():
 
 
 def start_db():
+	# Upgrade database
 	from .. import upgrade
 	upgrade.upgrade_db(sqldb.add_scrobbles)
 
+	# Load temporary tables
 	from . import associated
 	associated.load_associated_rules()
 
 	dbstatus['healthy'] = True
-	dbstatus['complete'] = True
 
+	# inform time module about begin of scrobbling
 	firstscrobble = sqldb.get_scrobbles()[0]
 	register_scrobbletime(firstscrobble['time'])
 
+	# create cached information
+	cached.update_medals()
+	cached.update_weekly()
+
+	dbstatus['complete'] = True
+
+
+
 
 
 

maloja/database/cached.py

@@ -0,0 +1,69 @@
+# for information that is not authorative, but should be saved anyway because it
+# changes infrequently and DB access is expensive
+
+from doreah.regular import yearly, daily
+from .. import database
+from .. import malojatime as mjt
+
+
+
+medals_artists = {
+	# year: {'gold':[],'silver':[],'bronze':[]}
+}
+medals_tracks = {
+	# year: {'gold':[],'silver':[],'bronze':[]}
+}
+
+weekly_topartists = []
+weekly_toptracks = []
+
+@yearly
+def update_medals():
+
+	global medals_artists, medals_tracks
+	medals_artists.clear()
+	medals_tracks.clear()
+
+	for year in mjt.ranges(step="year"):
+		if year == mjt.thisyear(): break
+
+		charts_artists = database.get_charts_artists(timerange=year)
+		charts_tracks = database.get_charts_tracks(timerange=year)
+
+		entry_artists = {'gold':[],'silver':[],'bronze':[]}
+		entry_tracks = {'gold':[],'silver':[],'bronze':[]}
+		medals_artists[year.desc()] = entry_artists
+		medals_tracks[year.desc()] = entry_tracks
+
+		for entry in charts_artists:
+			if entry['rank'] == 1: entry_artists['gold'].append(entry['artist'])
+			elif entry['rank'] == 2: entry_artists['silver'].append(entry['artist'])
+			elif entry['rank'] == 3: entry_artists['bronze'].append(entry['artist'])
+			else: break
+		for entry in charts_tracks:
+			if entry['rank'] == 1: entry_tracks['gold'].append(entry['track'])
+			elif entry['rank'] == 2: entry_tracks['silver'].append(entry['track'])
+			elif entry['rank'] == 3: entry_tracks['bronze'].append(entry['track'])
+			else: break
+
+
+
+@daily
+def update_weekly():
+
+	global weekly_topartists, weekly_toptracks
+	weekly_topartists.clear()
+	weekly_toptracks.clear()
+
+	for week in mjt.ranges(step="week"):
+		if week == mjt.thisweek(): break
+
+		charts_artists = database.get_charts_artists(timerange=week)
+		charts_tracks = database.get_charts_tracks(timerange=week)
+
+		for entry in charts_artists:
+			if entry['rank'] == 1: weekly_topartists.append(entry['artist'])
+			else: break
+		for entry in charts_tracks:
+			if entry['rank'] == 1: weekly_toptracks.append(entry['track'])
+			else: break

maloja/utilities/__init__.py

@@ -1,3 +1,2 @@
 from .images import *
-from .maintenance import *
 from .utils import *

maloja/utilities/maintenance.py

@@ -1,114 +0,0 @@
-from ..__pkginfo__ import VERSION
-from ..malojatime import ranges, thisweek, thisyear
-from ..globalconf import malojaconfig
-
-from doreah.regular import yearly, daily
-from doreah.logging import log
-
-import datetime
-import json
-import urllib
-import itertools
-
-
-
-get_track = lambda x:(frozenset(x["track"]["artists"]),x["track"]["title"])
-get_artist = lambda x:x["artist"]
-
-def group_by_attribute(sequence,attribute):
-	grouped = itertools.groupby(sequence,key=lambda x:x[attribute])
-	for attrvalue,members in grouped:
-		yield attrvalue,list(members)
-
-def collect_rankings(chart,identify,collection,iteration=None,count=True):
-	grouped = group_by_attribute(chart,"rank")
-	for rank, members in grouped:
-		if not count and rank not in rankmedals: break
-		if count and rank != 1: break
-
-		for m in members:
-			# get the actual object that we're counting
-			entity = identify(m)
-
-			# count no1 spots
-			if count:
-				collection[entity] = collection.setdefault(entity,0) + 1
-
-			# collect instances of top3 spots
-			else:
-				medal = rankmedals[rank]
-				collection.setdefault(entity,{}).setdefault(medal,[]).append(iteration)
-
-
-rankmedals = {
-	1:'gold',
-	2:'silver',
-	3:'bronze'
-}
-
-@yearly
-def update_medals():
-
-
-	from ..database import MEDALS_ARTISTS, MEDALS_TRACKS, STAMPS, get_charts_artists, get_charts_tracks
-
-
-	MEDALS_ARTISTS.clear()
-	MEDALS_TRACKS.clear()
-
-	for year in ranges(step="year"):
-		if year == thisyear(): break
-
-		charts_artists = get_charts_artists(timerange=year)
-		charts_tracks = get_charts_tracks(timerange=year)
-
-		collect_rankings(charts_artists,get_artist,MEDALS_ARTISTS,iteration=year,count=False)
-		collect_rankings(charts_tracks,get_track,MEDALS_TRACKS,iteration=year,count=False)
-
-
-@daily
-def update_weekly():
-
-	from ..database import WEEKLY_TOPTRACKS, WEEKLY_TOPARTISTS, get_charts_artists, get_charts_tracks
-
-
-	WEEKLY_TOPARTISTS.clear()
-	WEEKLY_TOPTRACKS.clear()
-
-	for week in ranges(step="week"):
-		if week == thisweek(): break
-
-		charts_artists = get_charts_artists(timerange=week)
-		charts_tracks = get_charts_tracks(timerange=week)
-
-		collect_rankings(charts_artists,get_artist,WEEKLY_TOPARTISTS)
-		collect_rankings(charts_tracks,get_track,WEEKLY_TOPTRACKS)
-
-
-@daily
-def send_stats():
-	if malojaconfig["SEND_STATS"]:
-
-		log("Sending daily stats report...")
-
-		from ..database import ARTISTS, TRACKS, SCROBBLES
-
-		keys = {
-			"url":"https://myrcella.krateng.ch/malojastats",
-			"method":"POST",
-			"headers":{"Content-Type": "application/json"},
-			"data":json.dumps({
-				"name":malojaconfig["NAME"],
-				"url":malojaconfig["PUBLIC_URL"],
-				"version":VERSION,
-				"artists":len(ARTISTS),
-				"tracks":len(TRACKS),
-				"scrobbles":len(SCROBBLES)
-			}).encode("utf-8")
-		}
-		try:
-			req = urllib.request.Request(**keys)
-			response = urllib.request.urlopen(req)
-			log("Sent daily report!")
-		except:
-			log("Could not send daily report!")