Added web interface to audit library data

cleanup.py

@@ -27,6 +27,10 @@ class CleanerAgent:
 	def removespecial(self,s):
 		return s.replace("\t","").replace("␟","").replace("\n","")
 
+	# if an artist appears in any created rule, we can assume that artist is meant to exist and be spelled like that
+	def confirmedReal(self,a):
+		confirmed = self.rules_belongtogether + [self.rules_replaceartist[r] for r in self.rules_replaceartist]
+		return (a in confirmed)
 
 	delimiters_feat = ["ft.","ft","feat.","feat","featuring","Ft.","Ft","Feat.","Feat","Featuring"]			#Delimiters used for extra artists, even when in the title field
 	delimiters = ["vs.","vs","&"]											#Delimiters in informal artist strings, spaces expected around them

database.py

@@ -23,6 +23,9 @@ clients = []
 
 lastsync = 0
 
+rulescheck = ""
+
+
 
 ### symmetric keys are fine for now since we hopefully use HTTPS
 def loadAPIkeys():
@@ -376,7 +379,103 @@ def post_scrobble():
 def abouttoshutdown():
 	sync()
 	#sys.exit()
+
+@dbserver.post("/newrule")
+def newrule():
+	keys = FormsDict.decode(request.forms)
+	addEntry("rules/webmade.tsv",[k for k in keys])
 	
+@dbserver.route("/issues")
+def issues():
+	combined = []
+	duplicates = []
+	newartists = []
+	
+	
+	import itertools
+	import difflib
+	
+	sortedartists = ARTISTS.copy()
+	sortedartists.sort(key=len,reverse=True)
+	reversesortedartists = sortedartists.copy()
+	reversesortedartists.reverse()
+	for a in reversesortedartists:
+		
+		nochange = cla.confirmedReal(a)
+		
+		st = a
+		lis = []
+		reachedmyself = False
+		for ar in sortedartists:
+			if (ar != a) and not reachedmyself:
+				continue
+			elif not reachedmyself:
+				reachedmyself = True
+				continue
+			
+			if (ar.lower() == a.lower()) or ("the " + ar.lower() == a.lower()) or ("a " + ar.lower() == a.lower()):
+				duplicates.append((ar,a))
+				break
+		
+			if (ar + " " in st) or (" " + ar in st):
+				lis.append(ar)
+				st = st.replace(ar,"").strip()
+			elif (ar == st):
+				lis.append(ar)
+				st = ""
+				if not nochange:
+					combined.append((a,lis))
+				break
+				
+			elif (ar in st) and len(ar)*2 > len(st):
+				duplicates.append((a,ar))
+		
+		st = st.replace("&","").replace("and","").replace("with","").strip()
+		if st != "" and st != a:
+			if len(st) < 5 and len(lis) == 1:
+				#check if we havent just randomly found the string in another word
+				#if (" " + st + " ") in lis[0] or (lis[0].endswith(" " + st)) or (lis[0].startswith(st + " ")):
+				duplicates.append((a,lis[0]))
+			elif len(st) < 5 and len(lis) > 1 and not nochange:
+				combined.append((a,lis))
+			elif len(st) >= 5 and not nochange:
+				#check if we havent just randomly found the string in another word
+				if (" " + st + " ") in a or (a.endswith(" " + st)) or (a.startswith(st + " ")):
+					newartists.append((st,a,lis))
+	
+	#for c in itertools.combinations(ARTISTS,3):
+	#	l = list(c)
+	#	print(l)
+	#	l.sort(key=len,reverse=True)
+	#	[full,a1,a2] = l
+	#	if (a1 + " " + a2 in full) or (a2 + " " + a1 in full):
+	#		combined.append((full,a1,a2))
+
+
+	#for c in itertools.combinations(ARTISTS,2):
+	#	if
+	#
+	#	if (c[0].lower == c[1].lower):
+	#		duplicates.append((c[0],c[1]))
+			
+			
+	#	elif (c[0] + " " in c[1]) or (" " + c[0] in c[1]) or (c[1] + " " in c[0]) or (" " + c[1] in c[0]):
+	#		if (c[0] in c[1]):
+	#			full, part = c[1],c[0]
+	#			rest = c[1].replace(c[0],"").strip()
+	#		else:
+	#			full, part = c[0],c[1]
+	#			rest = c[0].replace(c[1],"").strip()
+	#		if rest in ARTISTS and full not in [c[0] for c in combined]:
+	#			combined.append((full,part,rest))
+	
+	#	elif (c[0] in c[1]) or (c[1] in c[0]):
+	#		duplicates.append((c[0],c[1]))
+			
+	
+	return {"duplicates":duplicates,"combined":combined,"newartists":newartists}
+
+
 	
 ####
 ## Server operation
@@ -388,8 +487,6 @@ def abouttoshutdown():
 def runserver(PORT):
 	global lastsync
 	lastsync = time = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp())
-	#reload()
-	#buildh()
 	build_db()
 	coa.updateIDs(ARTISTS)
 
@@ -400,6 +497,9 @@ def runserver(PORT):
 
 def build_db():
 	
+	global rulescheck
+	
+	
 	global SCROBBLES
 	
 	SCROBBLESNEW = []

utilities.py

@@ -54,6 +54,15 @@ def createTSV(filename):
 	
 	if not os.path.exists(filename):
 		open(filename,"w").close()
+
+def addEntry(filename,args):
+
+	createTSV(filename)
+	
+	line = "\t".join(args)
+	with open(filename,"a") as f:
+		f.write(line + "\n")
+
 		
 ### Logging
 		
@@ -148,6 +157,9 @@ def cacheImage(url,path,filename):
 	target = path + "/" + filename + "." + response.info().get_content_subtype()	
 	urllib.request.urlretrieve(url,target)
 	
+def artistLink(name):
+	import urllib
+	return "<a href='/artist?artist=" + urllib.parse.quote(name) + "'>" + name + "</a>"
 	
 	
 def getTimeDesc(timestamp):

website/issues.html

@@ -0,0 +1,45 @@
+<!DOCTYPE html>
+
+<html>
+	<head>
+		<meta charset="UTF-8" />
+		<title>Maloja - Issues</title>
+		<link rel="stylesheet" href="maloja.css" />
+	</head>
+	
+	<body>
+		<table class="top_info">
+			<tr>
+				
+				<td class="text">
+					<h1>Possible Issues</h1><br/>
+					<span>with your library</span>
+					<p class="stats"><a href="/scrobbles?artist=KEY_ENC_ARTISTNAME">KEY_ISSUES Issues</a></p>
+					
+					<p>Maloja can identify possible problems with consistency or redundancy in your library.</p>
+				</td>
+			</tr>
+		</table>
+
+		KEY_ISSUESLIST
+		
+	</body>
+	
+	<script>
+		function newrule() {
+			keys = ""
+			for (var i = 1; i < arguments.length; i++) {
+				keys += encodeURIComponent(arguments[i]) + "&"
+			}
+			console.log(keys)
+			var xhttp = new XMLHttpRequest();
+			xhttp.open("POST","/db/newrule?", true);
+			xhttp.send(keys);
+			e = arguments[0]
+			line = e.parentNode
+			line.parentNode.removeChild(line)
+	
+		}
+	
+	</script>
+</html>

website/issues.py

@@ -0,0 +1,39 @@
+import urllib
+import json
+from utilities import artistLink
+
+def replacedict(keys,dbport):
+	
+	response = urllib.request.urlopen("http://localhost:" + str(dbport) + "/issues")
+	db_data = json.loads(response.read())
+	i = 0
+	
+	html = "<table>"
+	for d in db_data["duplicates"]:
+		html += "<tr>"
+		html += "<td>'" + artistLink(d[0]) + "'"
+		html += " is a possible duplicate of "
+		html += "'" + artistLink(d[1]) + "'</td>"
+		html += """<td class='button' onclick="newrule(this,'replaceartist','""" + d[0] + """','""" + d[1] + """')">""" + d[1] + """ is correct</td>"""
+		html += """<td class='button' onclick="newrule(this,'replaceartist','""" + d[1] + """','""" + d[0] + """')">""" + d[0] + """ is correct</td>"""
+		html += "</tr>"
+		i += 1
+	for c in db_data["combined"]:
+		html += "<tr>"
+		html += "<td>'" + artistLink(c[0]) + "' sounds like the combination of " + str(len(c[1])) + " artists: "
+		for a in c[1]:
+			html += "'" + artistLink(a) + "' "
+		html += "</td>"
+		html += """<td class='button' onclick="newrule(this,'replaceartist','""" + c[0] + """','""" + "␟".join(c[1]) + """')">Fix it</td>"""
+		html += "</tr>"
+		i += 1
+	for n in db_data["newartists"]:
+		html += "<tr>"
+		html += "<td>Is '" + n[0] + "' in '" + artistLink(n[1]) + "' an artist?</td>"
+		html += """<td class='button' onclick="newrule(this,'replaceartist','""" + n[1] + """','""" + "␟".join(n[2] + [n[0]]) + """')">Yes</td>"""
+		html += "</tr>"
+		i += 1
+	
+	html += "</table>"
+
+	return {"KEY_ISSUESLIST":html,"KEY_ISSUES":str(i)}

website/maloja.css

@@ -55,3 +55,12 @@ table td.time {
 table td.artists,td.artist,td.title,td.amount {
 	width:300px;
 }
+
+table td.button {
+	width:150px;
+	background-color:yellow;
+	color:#333337;
+	padding:1px;
+	border-radius:4px;
+	cursor:pointer;
+}