More experimenting with database architecture

maloja/database/__init__.py

@@ -56,6 +56,13 @@ class DatabaseNotBuilt(HTTPError):
 		)
 
 
+def waitfordb(func):
+	def newfunc(*args,**kwargs):
+		if not dbstatus['healthy']: raise DatabaseNotBuilt()
+		return func(*args,**kwargs)
+	return newfunc
+
+
 
 MEDALS_ARTISTS = {}	#literally only changes once per year, no need to calculate that on the fly
 MEDALS_TRACKS = {}
@@ -103,53 +110,33 @@ def createScrobble(artists,title,time,album=None,duration=None,volatile=False):
 
 
 
-########
+@waitfordb
-########
-## HTTP requests and their associated functions
-########
-########
-
-
-
-
-
-
-
-
 def get_scrobbles(**keys):
-	r = db_query(**{k:keys[k] for k in keys if k in ["artist","artists","title","since","to","within","timerange","associated","track"]})
+	(since,to) = keys.get('timerange').timestamps()
-	return r
+	if 'artist' in keys:
-
+		result = sqldb.get_scrobbles_of_artist(artist=keys['artist'],since=since,to=to)
-
+	elif 'track' in keys:
-def info():
+		result = sqldb.get_scrobbles_of_track(track=keys['track'],since=since,to=to)
-	totalscrobbles = get_scrobbles_num()
+	else:
-	artists = {}
+		result = sqldb.get_scrobbles(since=since,to=to)
-
+	#return result[keys['page']*keys['perpage']:(keys['page']+1)*keys['perpage']]
-	return {
+	return result
-		"name":malojaconfig["NAME"],
-		"artists":{
-			chartentry["artist"]:round(chartentry["scrobbles"] * 100 / totalscrobbles,3)
-			for chartentry in get_charts_artists() if chartentry["scrobbles"]/totalscrobbles >= 0
-		},
-		"known_servers":list(KNOWN_SERVERS)
-	}
-
-
 
+@waitfordb
 def get_scrobbles_num(**keys):
-	r = db_query(**{k:keys[k] for k in keys if k in ["artist","track","artists","title","since","to","within","timerange","associated"]})
+	return len(get_scrobbles(**keys))
-	return len(r)
 
+@waitfordb
 def get_tracks(artist=None):
+	if artist is None:
+		result = sqldb.get_tracks()
+	else:
+		result = sqldb.get_tracks_of_artist(artist)
+	return result
 
-	artistid = ARTISTS.index(artist) if artist is not None else None
+@waitfordb
-	return [get_track_dict(t) for t in TRACKS if (artistid in t.artists) or (artistid==None)]
-
-
-
 def get_artists():
-	if not dbstatus['healthy']: raise DatabaseNotBuilt()
+	return sqldb.get_artists()
-	return ARTISTS #well
 
 
 
@@ -504,6 +491,9 @@ def start_db():
 	dbstatus['healthy'] = True
 	dbstatus['complete'] = True
 
+	firstscrobble = sqldb.get_scrobbles(max=1)[0]
+	register_scrobbletime(firstscrobble['time'])
+
 
 
 
@@ -520,7 +510,7 @@ def start_db():
 
 # Queries the database
 def db_query_full(artist=None,artists=None,title=None,track=None,timerange=None,associated=False,max_=None):
-	print((artist,artists,title,track,timerange))
+
 	if not dbstatus['healthy']: raise DatabaseNotBuilt()
 	(since, to) = time_stamps(range=timerange)
 
@@ -529,12 +519,12 @@ def db_query_full(artist=None,artists=None,title=None,track=None,timerange=None,
 		track = {'artists':artists,'title':title}
 
 	if track is not None:
-		return sqldb.get_scrobbles_of_track(track=track,since=since,to=to)
+		return list(reversed(sqldb.get_scrobbles_of_track(track=track,since=since,to=to)))
 
 	if artist is not None:
-		return sqldb.get_scrobbles_of_artist(artist=artist,since=since,to=to)
+		return list(reversed(sqldb.get_scrobbles_of_artist(artist=artist,since=since,to=to)))
 
-	return sqldb.get_scrobbles(since=since,to=to)
+	return list(reversed(sqldb.get_scrobbles(since=since,to=to)))
 
 
 
@@ -592,8 +582,7 @@ def db_aggregate_full(by=None,timerange=None,artist=None):
 		return ls
 
 	else:
-		#return len([scr for scr in SCROBBLES if since < scr[1] < to])
+		return len(sqldb.get_scrobbles(since=since,to=to,resolve_references=False))
-		return len(list(scrobbles_in_range(since,to)))
 
 
 # Search for strings

maloja/database/sqldb.py

@@ -1,6 +1,8 @@
 import sqlalchemy as sql
 import json
 import unicodedata
+import math
+from datetime import datetime
 
 from ..globalconf import data_dir
 
@@ -74,27 +76,69 @@ meta.create_all(engine)
 
 
 ##### Conversions between DB and dicts
-## These should only take the row info from their respective table and fill in
-## other information by calling the respective id lookup functions
 
-def scrobble_db_to_dict(row,resolve_references=True):
+# These should work on whole lists and collect all the references,
-	return {
+# then look them up once and fill them in
-		"time":row.timestamp,
-		"track":get_track(row.track_id) if resolve_references else row.track_id,
-		"duration":row.duration,
-		"origin":row.origin
-	}
 
-def track_db_to_dict(row):
+
-	return {
+### DB -> DICT
-		"artists":get_artists_of_track(row.id),
+
-		"title":row.title,
+#def scrobble_db_to_dict(row,resolve_references=True):
-		"length":row.length
+#	return {
-	}
+#		"time":row.timestamp,
+#		"track":get_track(row.track_id) if resolve_references else row.track_id,
+#		"duration":row.duration,
+#		"origin":row.origin
+#	}
+
+
+def scrobbles_db_to_dict(rows):
+	#track_ids = set(row.track_id for row in rows)
+	#tracks = {
+	#	track_id:get_track(track_id) for track_id in track_ids
+	#}
+	tracks = get_tracks_map(set(row.track_id for row in rows))
+	return [
+		{
+			"time":row.timestamp,
+			"track":tracks[row.track_id],
+			"duration":row.duration,
+			"origin":row.origin
+		}
+		for row in rows
+	]
+
+#def track_db_to_dict(row):
+#	return {
+#		"artists":get_artists_of_track(row.id),
+#		"title":row.title,
+#		"length":row.length
+#	}
+
+def tracks_db_to_dict(rows):
+	artists = get_artists_of_tracks(set(row.id for row in rows))
+	return [
+		{
+			"artists":artists[row.id],
+			"title":row.title,
+			"length":row.length
+		}
+		for row in rows
+	]
 
 def artist_db_to_dict(row):
 	return row.name
 
+def artists_db_to_dict(rows):
+	return [
+		row.name
+		for row in rows
+	]
+
+
+### DICT -> DB
+
+
 def scrobble_dict_to_db(info):
 	return {
 		"rawscrobble":json.dumps(info),
@@ -122,6 +166,8 @@ def artist_dict_to_db(info):
 
 
 
+##### Actual Database interactions
+
 
 def add_scrobble(scrobbledict):
 	add_scrobbles([scrobbledict])
@@ -130,11 +176,7 @@ def add_scrobbles(scrobbleslist):
 
 	ops = [
 		DB['scrobbles'].insert().values(
-			rawscrobble=json.dumps(s),
+			**scrobble_dict_to_db(s)
-			timestamp=s['time'],
-			origin=s['origin'],
-			duration=s['duration'],
-			track_id=get_track_id(s['track'])
 		) for s in scrobbleslist
 	]
 
@@ -146,8 +188,7 @@ def add_scrobbles(scrobbleslist):
 				pass
 
 
-### DB interface functions - these will 'get' the ID of an entity,
+### these will 'get' the ID of an entity, creating it if necessary
-### creating it if necessary
 
 
 def get_track_id(trackdict):
@@ -222,7 +263,16 @@ def get_artist_id(artistname):
 		return result.inserted_primary_key[0]
 
 
-def get_scrobbles_of_artist(artist,since,to,resolve_references=True):
+
+
+
+### Functions that get rows according to parameters
+
+
+def get_scrobbles_of_artist(artist,since=None,to=None,resolve_references=True):
+
+	if since is None: since=0
+	if to is None: to=now()
 
 	artist_id = get_artist_id(artist)
 
@@ -235,11 +285,15 @@ def get_scrobbles_of_artist(artist,since,to,resolve_references=True):
 		)
 		result = conn.execute(op).all()
 
-	result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for row in result]
+	result = scrobbles_db_to_dict(result)
+	#result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for row in result]
 	return result
 
 
-def get_scrobbles_of_track(track,since,to):
+def get_scrobbles_of_track(track,since=None,to=None):
+
+	if since is None: since=0
+	if to is None: to=now()
 
 	track_id = get_track_id(track)
 
@@ -251,11 +305,15 @@ def get_scrobbles_of_track(track,since,to):
 		)
 		result = conn.execute(op).all()
 
-	result = [scrobble_db_to_dict(row) for row in result]
+	result = scrobbles_db_to_dict(result)
+	#result = [scrobble_db_to_dict(row) for row in result]
 	return result
 
 
-def get_scrobbles(since,to,resolve_references=True):
+def get_scrobbles(since=None,to=None,resolve_references=True,max=math.inf):
+
+	if since is None: since=0
+	if to is None: to=now()
 
 	with engine.begin() as conn:
 		op = DB['scrobbles'].select().where(
@@ -264,9 +322,77 @@ def get_scrobbles(since,to,resolve_references=True):
 		)
 		result = conn.execute(op).all()
 
-	result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for row in result]
+	result = scrobbles_db_to_dict(result)
+	#result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for i,row in enumerate(result) if i<max]
 	return result
 
+
+def get_artists_of_track(track_id,resolve_references=True):
+	with engine.begin() as conn:
+		op = DB['trackartists'].select().where(
+			DB['trackartists'].c.track_id==track_id
+		)
+		result = conn.execute(op).all()
+
+	artists = [get_artist(row.artist_id) if resolve_references else row.artist_id for row in result]
+	return artists
+
+def get_artists_of_tracks(track_ids):
+	with engine.begin() as conn:
+		op = sql.join(DB['trackartists'],DB['artists']).select().where(
+			DB['trackartists'].c.track_id.in_(track_ids)
+		)
+		result = conn.execute(op).all()
+
+	artists = {}
+	for row in result:
+		artists.setdefault(row.track_id,[]).append(artist_db_to_dict(row))
+	return artists
+
+def get_tracks_map(track_ids):
+	with engine.begin() as conn:
+		op = DB['tracks'].select().where(
+			DB['tracks'].c.id.in_(track_ids)
+		)
+		result = conn.execute(op).all()
+
+	tracks = {}
+	trackids = [row.id for row in result]
+	trackdicts = tracks_db_to_dict(result)
+	for i in range(len(trackids)):
+		tracks[trackids[i]] = trackdicts[i]
+	return tracks
+
+def get_tracks():
+	with engine.begin() as conn:
+		op = DB['tracks'].select()
+		result = conn.execute(op).all()
+
+	return tracks_db_to_dict(result)
+
+def get_tracks_of_artist(artist):
+
+	artist_id = get_artist_id(artist)
+
+	with engine.begin() as conn:
+		op = sql.join(DB['tracks'],DB['trackartists']).select().where(
+			DB['trackartists'].c.artist_id==artist_id
+		)
+		result = conn.execute(op).all()
+
+	return tracks_db_to_dict(result)
+
+def get_artists():
+	with engine.begin() as conn:
+		op = DB['artists'].select()
+		result = conn.execute(op).all()
+
+	return artists_db_to_dict(result)
+
+
+### get a specific entity by id
+
+
 def get_track(id):
 	with engine.begin() as conn:
 		op = DB['tracks'].select().where(
@@ -291,16 +417,6 @@ def get_artist(id):
 	artistinfo = result[0]
 	return artist_db_to_dict(artistinfo)
 
-def get_artists_of_track(track_id,resolve_references=True):
-	with engine.begin() as conn:
-		op = DB['trackartists'].select().where(
-			DB['trackartists'].c.track_id==track_id
-		)
-		result = conn.execute(op).all()
-
-	artists = [get_artist(row.artist_id) if resolve_references else row.artist_id for row in result]
-	return artists
-
 
 
 
@@ -313,3 +429,7 @@ def normalize_name(name):
 	name = "".join(char for char in unicodedata.normalize('NFD',name.lower())
 		if char not in remove_symbols and unicodedata.category(char) != 'Mn')
 	return name
+
+
+def now():
+	return int(datetime.now().timestamp())

maloja/malojatime.py

@@ -58,6 +58,9 @@ class MTRangeGeneric:
 	def unlimited(self):
 		return False
 
+	def timestamps(self):
+		return (self.first_stamp(),self.last_stamp())
+
 	# whether we currently live or will ever again live in this range
 	def active(self):
 		return (self.last_stamp() > datetime.utcnow().timestamp())