Updated tasks to new database architecture

maloja/proccontrol/tasks/generate.py

@@ -1,6 +1,8 @@
 import random
 import datetime
 
+from ...database.sqldb import add_scrobbles
+
 
 artists = [
 	"Chou Tzuyu","Jennie Kim","Kim Seolhyun","Nancy McDonie","Park Junghwa","Hirai Momo","Rosé Park","Laura Brehm","HyunA",
@@ -66,14 +68,22 @@ def generate_track():
 
 
 def generate():
-	# TODO
-	pass
 
+	scrobbles = []
 	for _ in range(200):
 		track = generate_track()
+		print("Generated",track)
 		for _ in range(random.randint(1, 50)):
 			timestamp = random.randint(1, int(datetime.datetime.now().timestamp()))
 
-			entry = "\t".join([str(timestamp),"␟".join(track['artists']),track['title'],"-"])
+			scrobbles.append({
-			fd.write(entry)
+				"time":timestamp,
-			fd.write("\n")
+				 	"track":{
+				 		"artists":track['artists'],
+				 		"title":track['title']
+				 	},
+					"duration":None,
+				 	"origin":"generated"
+			})
+
+	add_scrobbles(scrobbles)

maloja/proccontrol/tasks/import_scrobbles.py

@@ -5,6 +5,8 @@ import json, csv
 from ...cleanup import *
 from doreah.io import col, ask
 from ...globalconf import data_dir
+
+from ...database.sqldb import add_scrobbles
 #from ...utilities import *
 
 
@@ -23,60 +25,58 @@ def import_scrobbles(fromfile):
 	ext = fromfile.split('.')[-1].lower()
 
 	if ext == 'csv':
-		type = "Last.fm"
+		import_type = "Last.fm"
-		outputf = data_dir['scrobbles']("lastfmimport.tsv")
 		importfunc = parse_lastfm
 
 
 	elif ext == 'json':
-		type = "Spotify"
+		import_type = "Spotify"
-		outputf = data_dir['scrobbles']("spotifyimport.tsv")
 		importfunc = parse_spotify
 
 
-	print(f"Parsing {col['yellow'](fromfile)} as {col['cyan'](type)} export")
+	print(f"Parsing {col['yellow'](fromfile)} as {col['cyan'](import_type)} export")
 
-	if os.path.exists(outputf):
+	success = 0
-		overwrite = ask("Already imported data. Overwrite?",default=False)
+	failed = 0
-		if not overwrite: return
+	timestamps = set()
+	scrobblebuffer = []
 
-	with open(outputf,"w") as outputfd:
-		success = 0
-		failed = 0
-		timestamps = set()
 
-		for scrobble in importfunc(fromfile):
+	for scrobble in importfunc(fromfile):
-			if scrobble is None:
+		if scrobble is None:
-				failed += 1
+			failed += 1
-			else:
+		else:
-				success += 1
+			success += 1
 
-				## We prevent double timestamps in the database creation, so we
+			# prevent duplicate timestamps within one import file
-				## technically don't need them in the files
+			while scrobble['timestamp'] in timestamps:
-				## however since the conversion to maloja is a one-time thing,
+				scrobble['timestamp'] += 1
-				## we should take any effort to make the file as good as possible
+			timestamps.add(scrobble['timestamp'])
-				while scrobble['timestamp'] in timestamps:
+			# clean up
-					scrobble['timestamp'] += 1
+			(scrobble['artists'],scrobble['title']) = c.fullclean(scrobble['artists'],scrobble['title'])
-				timestamps.add(scrobble['timestamp'])
 
-				# Format fields for tsv
+			scrobblebuffer.append({
-				scrobble['timestamp'] = str(scrobble['timestamp'])
+				"time":scrobble['timestamp'],
-				scrobble['duration'] = str(scrobble['duration']) if scrobble['duration'] is not None else '-'
+				 	"track":{
-				(artists,scrobble['title']) = c.fullclean(scrobble['artiststr'],scrobble['title'])
+				 		"artists":scrobble['artists'],
-				scrobble['artiststr'] = "␟".join(artists)
+				 		"title":scrobble['title'],
+				 		"album":{
+				 			"name":scrobble['album'],
+				 			"artists":scrobble['artists']
+				 		},
+				 		"length":None
+				 	},
+				 	"duration":scrobble['duration'],
+				 	"origin":"import:" + import_type,
+					"extra":{}
+			})
 
-				outputline = "\t".join([
+			if success % 1000 == 0:
-					scrobble['timestamp'],
+				print(f"Imported {success} scrobbles...")
-					scrobble['artiststr'],
+				add_scrobbles(scrobblebuffer)
-					scrobble['title'],
+				scrobblebuffer = []
-					scrobble['album'],
-					scrobble['duration']
-				])
-				outputfd.write(outputline + '\n')
-
-				if success % 100 == 0:
-					print(f"Imported {success} scrobbles...")
 
+	add_scrobbles(scrobblebuffer)
 	print("Successfully imported",success,"scrobbles!")
 	if failed > 0:
 		print(col['red'](str(failed) + " Errors!"))
@@ -95,7 +95,7 @@ def parse_spotify(inputf):
 			try:
 				yield {
 					'title':entry['master_metadata_track_name'],
-					'artiststr': entry['master_metadata_album_artist_name'],
+					'artists': entry['master_metadata_album_artist_name'],
 					'album': entry['master_metadata_album_album_name'],
 					'timestamp': int(datetime.datetime.strptime(
 						entry['ts'].replace('Z','+0000',),
@@ -124,7 +124,7 @@ def parse_lastfm(inputf):
 			try:
 				yield {
 					'title': row[2],
-					'artiststr': row[0],
+					'artists': row[0],
 					'album': row[1],
 					'timestamp': int(datetime.datetime.strptime(
 						row[3] + '+0000',

maloja/utilities/images.py

@@ -128,7 +128,6 @@ def resolve_track_image(track_id):
 		return result
 
 	track = database.sqldb.get_track(track_id)
-	print(track)
 
 	# local image
 	if malojaconfig["USE_LOCAL_IMAGES"]:
@@ -156,7 +155,6 @@ def resolve_artist_image(artist_id):
 		return result
 
 	artist = database.sqldb.get_artist(artist_id)
-	print(artist)
 
 	# local image
 	if malojaconfig["USE_LOCAL_IMAGES"]:
@@ -275,7 +273,6 @@ def set_image(b64,**keys):
 	else:
 		entity = keys['artist']
 		id = database.sqldb.get_artist_id(entity)
-	print('id is',id)
 
 	log("Trying to set image, b64 string: " + str(b64[:30] + "..."),module="debug")