Database can now analyze consistency of scrobble data

README.md

@@ -46,4 +46,4 @@ I wouldn't recommend it yet. But if you want to test Maloja, it's fairly easy:
 			proxy_pass http://yoururl:42011;
 		}
 		
-If you would like to import all your previous last.fm scrobbles, use [benfoxall's website](https://benjaminbenben.com/lastfm-to-csv/) ([GitHub page](https://github.com/benfoxall/lastfm-to-csv)). Use the python script lastfmconverter.py with two arguments - the downloaded csv file and your new tsv file - to convert your data. Place the tsv file in logs/ and the server will recognize it on startup.
+If you would like to import all your previous last.fm scrobbles, use [benfoxall's website](https://benjaminbenben.com/lastfm-to-csv/) ([GitHub page](https://github.com/benfoxall/lastfm-to-csv)). Use the python script lastfmconverter.py with two arguments - the downloaded csv file and your new tsv file - to convert your data. Place the tsv file in scrobbles/ and the server will recognize it on startup.

database.py

@@ -23,7 +23,8 @@ clients = []
 
 lastsync = 0
 
-rulescheck = ""
+# rulestate that the entire current database was built with, or False if the database was built from inconsistent scrobble files
+db_rulestate = False
 
 
 
@@ -109,13 +110,16 @@ def getTrackID(artists,title):
 def test_server():
 	apikey = request.query.get("key")
 	response.set_header("Access-Control-Allow-Origin","*")
-	if not (checkAPIkey(apikey)):
+	if apikey is not None and not (checkAPIkey(apikey)):
 		response.status = 403
-		return "Wrong or Missing API key"
+		return "Wrong API key"
 	
-	else:
+	elif db_rulestate:
 		response.status = 204
 		return
+	else:
+		response.status = 205
+		return
 
 @dbserver.route("/scrobbles")
 def get_scrobbles():
@@ -384,12 +388,15 @@ def abouttoshutdown():
 def newrule():
 	keys = FormsDict.decode(request.forms)
 	addEntry("rules/webmade.tsv",[k for k in keys])
+	global db_rulestate
+	db_rulestate = False
 	
 @dbserver.route("/issues")
 def issues():
 	combined = []
 	duplicates = []
 	newartists = []
+	inconsistent = not db_rulestate
 	
 	
 	import itertools
@@ -473,7 +480,16 @@ def issues():
 	#		duplicates.append((c[0],c[1]))
 			
 	
-	return {"duplicates":duplicates,"combined":combined,"newartists":newartists}
+	return {"duplicates":duplicates,"combined":combined,"newartists":newartists,"inconsistent":inconsistent}
+
+@dbserver.post("/rebuild")
+def rebuild():
+	sync()
+	os.system("python3 fixexisting.py")
+	global cla, coa
+	cla = CleanerAgent()
+	coa = CollectorAgent()
+	build_db()
 
 
 	
@@ -497,17 +513,14 @@ def runserver(PORT):
 
 def build_db():
 	
-	global rulescheck
 	
 	
-	global SCROBBLES
 	
-	SCROBBLESNEW = []
-	for t in SCROBBLES:
-		if not t[2]:
-			SCROBBLESNEW.append(t)
-
-	SCROBBLES = SCROBBLESNEW
+	global SCROBBLES, ARTISTS, TRACKS
+	
+	SCROBBLES = []
+	ARTISTS = []
+	TRACKS = []
 	
 	db = parseAllTSV("scrobbles","int","string","string")
 	for sc in db:
@@ -538,7 +551,8 @@ def build_db():
 			
 	SCROBBLES.sort(key = lambda tup: tup[1])
 			
-	
+	global db_rulestate
+	db_rulestate = consistentRulestate("scrobbles",cla.checksums)
 	
 		
 
@@ -691,6 +705,14 @@ def db_search(query,type=None):
 # Takes user inputs like YYYY/MM and returns the timestamps. Returns timestamp if timestamp was already given.	
 def getTimestamps(f,t):
 	#(f,t) = inp
+	if isinstance(f, str) and f.lower() == "today":
+		tod = datetime.date.today()
+		f = [tod.year,tod.month,tod.day]
+	if isinstance(t, str) and t.lower() == "today":
+		tod = datetime.date.today()
+		t = [tod.year,tod.month,tod.day]
+	
+	
 	if isinstance(f, str):
 		f = [int(x) for x in f.split("/")]
 		

scrobbler-vivaldi-plex/settings.js

@@ -55,7 +55,7 @@ function checkServer() {
 
 function createCheckmarks() {
 	if (this.readyState == 4) {
-		if (this.status == 204) {
+		if ((this.status == 204) || (this.status == 205)) {
 			//document.getElementById("checkmark_url").innerHTML = "✔️"
 			//document.getElementById("checkmark_key").innerHTML = "✔️"
 			document.getElementById("serverurl").style.backgroundColor = "lawngreen"

utilities.py

@@ -77,7 +77,27 @@ def combineChecksums(filename,checksums):
 		f.close()
 		return True
 	
+# checks ALL files for their rule state. if they are all the same as the current loaded one, the entire database can be assumed to be consistent with the current ruleset
+# in any other case, get out
+def consistentRulestate(folder,checksums):
+	import os
 	
+	result = []
+	for scrobblefile in os.listdir(folder + "/"):
+		
+		if (scrobblefile.endswith(".tsv")):
+		
+			try:
+				f = open(folder + "/" + scrobblefile + ".rulestate","r")
+				if f.read() != checksums:
+					return False
+			
+			except:
+				return False
+			finally:
+				f.close()
+	
+	return True
 	
 	
 def parseAllTSV(path,*args):

website/issues.html

@@ -40,6 +40,13 @@
 			line.parentNode.removeChild(line)
 	
 		}
+		
+		function fullrebuild() {
+			var xhttp = new XMLHttpRequest();
+			xhttp.open("POST","/db/rebuild", true);
+			xhttp.send()
+			window.location = "/wait";
+		}
 	
 	</script>
 </html>

website/issues.py

@@ -9,6 +9,12 @@ def replacedict(keys,dbport):
 	i = 0
 	
 	html = "<table>"
+	if db_data["inconsistent"]:
+		html += "<tr>"
+		html += "<td>The current database wasn't built with all current rules in effect. Any problem below might be a false alarm and fixing it could create redundant rules.</td>"
+		html += """<td class='button important' onclick="fullrebuild()">Rebuild the database</td>"""
+		html += "</tr>"
+		i += 1
 	for d in db_data["duplicates"]:
 		html += "<tr>"
 		html += "<td>'" + artistLink(d[0]) + "'"

website/maloja.css

@@ -60,7 +60,12 @@ table td.button {
 	width:200px;
 	background-color:yellow;
 	color:#333337;
-	padding:1px;
+	padding:4px;
 	border-radius:4px;
 	cursor:pointer;
 }
+
+td.button.important {
+	background-color:red;
+	color:white;
+}

website/topartists.py

@@ -6,7 +6,7 @@ def replacedict(keys,dbport):
 	from utilities import getArtistInfo
 	
 	#hand down the since and from arguments
-	extrakeys = urllib.parse.urlencode(keys)
+	extrakeys = urllib.parse.urlencode(keys,quote_via=urllib.parse.quote,safe="/")
 	
 	response = urllib.request.urlopen("http://localhost:" + str(dbport) + "/charts/artists?" + extrakeys)
 	db_data = json.loads(response.read())

website/wait.html

@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+
+<html>
+	<head>
+		<meta charset="UTF-8" />
+		<title>Maloja - Please wait</title>
+		<link rel="stylesheet" href="maloja.css" />
+	</head>
+	
+	<body>
+		<table class="top_info">
+			<tr>
+				
+				<td class="text">
+					<h1>Maloja</h1><br/>
+					<span>Redbuilding the database</span>
+					
+					<p>Please wait...</p>
+				</td>
+			</tr>
+		</table>
+		
+		
+	</body>
+	
+	<script>
+		var pending = false;
+		setInterval(probeServer,1500);
+	
+		function probeServer() {
+			if (!pending) {
+				console.log("Probing...");
+				pending = true;
+				var xhttp = new XMLHttpRequest();
+				xhttp.open("GET","/db/test", true);
+				xhttp.onreadystatechange = goback;
+				xhttp.send();
+				
+			}
+			
+		}
+		
+		function goback() {
+			if ((this.readyState == 4) && (this.status == 205)) {
+				console.log("Not ready yet!")
+				pending = false;
+			}
+			if ((this.readyState == 4) && (this.status == 204)) {
+				console.log("K");
+				pending = false;
+				window.location = "/issues";
+			}
+		}
+	
+	</script>
+</html>