Implemented custom rules

cleanup.py

@@ -1,65 +1,103 @@
 import re
+import utilities
+
+# need to do this as a class so it can retain loaded settings from file
+class CleanerAgent:
+	
+	def __init__(self):
+		self.updateRules()
+	
+	def updateRules(self):
+		raw = utilities.parseAllTSV("rules","string","string","string")
+		self.rules_belongtogether = [b for [a,b,c] in raw if a=="belongtogether"]
+		self.rules_notanartist = [b for [a,b,c] in raw if a=="notanartist"]
+		self.rules_replacetitle = {b:c for [a,b,c] in raw if a=="replacetitle"}
+		self.rules_replaceartist = {b:c for [a,b,c] in raw if a=="replaceartist"}
+			
+	
+	
+	def fullclean(self,artist,title):
+		artists = self.parseArtists(self.removespecial(artist))
+		title = self.parseTitle(self.removespecial(title))
+		(title,moreartists) = self.parseTitleForArtists(title)
+		artists += moreartists
+		
+		return (list(set(artists)),title)
+
+	def removespecial(self,s):
+		return s.replace("\t","").replace("␟","").replace("\n","")
+
+
+	delimiters_feat = ["ft.","ft","feat.","feat","featuring"]			#Delimiters used for extra artists, even when in the title field
+	delimiters = ["vs.","vs","&"]							#Delimiters in informal titles, spaces expected around them
+	delimiters_formal = ["; ",";"]							#Delimiters used specifically to tag multiple artists when only one tag field is available, no spaces used
+
+
+	def parseArtists(self,a):
+
+		if a.strip() == "":
+			return []
+			
+		if a.strip() in self.rules_belongtogether:
+			return [a.strip()]
+		if a.strip() in self.rules_replaceartist:
+			return [self.rules_replaceartist[a.strip()]]
+			
+		
+		
+		for d in self.delimiters_feat:
+			if re.match(r"(.*) \(" + d + " (.*)\)",a) is not None:
+				return self.parseArtists(re.sub(r"(.*) \(" + d + " (.*)\)",r"\1",a)) + self.parseArtists(re.sub(r"(.*) \(" + d + " (.*)\)",r"\2",a))
+		
+		for d in (self.delimiters_feat + self.delimiters):
+			if ((" " + d + " ") in a):
+				ls = []
+				for i in a.split(" " + d + " "):
+					ls += self.parseArtists(i)
+				return ls
+				
+		for d in self.delimiters_formal:
+			if (d in a):
+				ls = []
+				for i in a.split(d):
+					ls += self.parseArtists(i)
+				return ls
+			
+		
+			
+		return [a.strip()]
+
+	def parseTitle(self,t):
+		
+		if t.strip() in self.rules_replacetitle:
+			return self.rules_replacetitle[t.strip()]
+	
+		t = t.replace("[","(").replace("]",")")
+		
+		t = re.sub(r" \(as made famous by .*?\)","",t)
+		t = re.sub(r" \(originally by .*?\)","",t)
+		
+		return t.strip()
+
+	def parseTitleForArtists(self,t):
+		for d in self.delimiters_feat:
+			if re.match(r"(.*) \(" + d + " (.*?)\)",t) is not None:
+				(title,artists) = self.parseTitleForArtists(re.sub(r"(.*) \(" + d + " (.*?)\)",r"\1",t))
+				artists += self.parseArtists(re.sub(r"(.*) \(" + d + " (.*?)\).*",r"\2",t))
+				return (title,artists)
+		
+		return (t,[])
 		
-def fullclean(artist,title):
-	artists = parseArtists(removespecial(artist))
-	title = parseTitle(removespecial(title))
-	(title,moreartists) = parseTitleForArtists(title)
-	artists += moreartists
 		
-	return (list(set(artists)),title)
 		
-def removespecial(s):
-	return s.replace("\t","").replace("␟","").replace("\n","")
-
-
-delimiters_feat = ["ft.","ft","feat.","feat","featuring"]			#Delimiters used for extra artists, even when in the title field
-delimiters = ["vs.","vs","&"]							#Delimiters in informal titles, spaces expected around them
-delimiters_formal = ["; ",";"]							#Delimiters used specifically to tag multiple artists when only one tag field is available, no spaces used
-
-
-def parseArtists(a):
-
-	if a.strip() == "":
-		return []
-	
-	for d in delimiters_feat:
-		if re.match(r"(.*) \(" + d + " (.*)\)",a) is not None:
-			return parseArtists(re.sub(r"(.*) \(" + d + " (.*)\)",r"\1",a)) + parseArtists(re.sub(r"(.*) \(" + d + " (.*)\)",r"\2",a))
-	
-	for d in (delimiters + delimiters_feat):
-		if ((" " + d + " ") in a):
-			ls = []
-			for i in a.split(" " + d + " "):
-				ls += parseArtists(i)
-			return ls
-			
-	for d in delimiters_formal:
-		if (d in a):
-			ls = []
-			for i in a.split(d):
-				ls += parseArtists(i)
-			return ls
 		
 		
 		
-	return [a.strip()]
 		
-def parseTitle(t):
-	t = t.replace("[","(").replace("]",")")
 		
-	t = re.sub(r" \(as made famous by .*?\)","",t)
-	t = re.sub(r" \(originally by .*?\)","",t)
 		
-	return t
 		
-def parseTitleForArtists(t):
-	for d in delimiters_feat:
-		if re.match(r"(.*) \(" + d + " (.*?)\)",t) is not None:
-			(title,artists) = parseTitleForArtists(re.sub(r"(.*) \(" + d + " (.*?)\)",r"\1",t))
-			artists += parseArtists(re.sub(r"(.*) \(" + d + " (.*?)\).*",r"\2",t))
-			return (title,artists)
 		
-	return (t,[])
 		
 def flatten(lis):
 

database.py

@@ -4,7 +4,7 @@ import urllib
 import waitress
 import os
 import datetime
-import cleanup
+from cleanup import *
 import sys
 
 
@@ -12,6 +12,8 @@ SCROBBLES = []	# Format: tuple(track_ref,timestamp,saved)
 ARTISTS = []	# Format: artist
 TRACKS = []	# Format: tuple(frozenset(artist_ref,...),title)
 
+c = CleanerAgent()
+
 lastsync = 0
 
 
@@ -118,10 +120,11 @@ def post_scrobble():
 	#title = urllib.parse.unquote(keys.get("title"))
 	artists = keys.get("artist")
 	title = keys.get("title")
-	time = int(keys.get("time"))
-	(artists,title) = cleanup.fullclean(artists,title)
-	if time is None:
+	try:
+		time = int(keys.get("time"))
+	except:
 		time = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp())
+	(artists,title) = c.fullclean(artists,title)
 
 	## this is necessary for localhost testing
 	response.set_header("Access-Control-Allow-Origin","*")

lastfmconverter.py

@@ -1,9 +1,12 @@
 import sys, os, datetime, re, cleanup
+from cleanup import *
 
 log = open(sys.argv[1])
 
 outputlog = open(sys.argv[2],"a")
 
+c = CleanerAgent()
+
 for l in log:
 	l = l.replace("\n","")
 	data = l.split(",")
@@ -14,7 +17,7 @@ for l in log:
 	time = data[3]
 	
 	
-	(artists,title) = cleanup.fullclean(artist,title)
+	(artists,title) = c.fullclean(artist,title)
 	
 	artistsstr = "␟".join(artists)
 	

rules/examplerules.tsv

@@ -7,8 +7,8 @@
 ###	countas: defines an artist that should be counted together with another artist for chart statistics etc. This will not change the separation in the database and all effects of this rule will disappear as soon as it is no longer active. Second column is the artist, third column the replacement artist
 ###
 ### THE RULES IN THIS EXAMPLE FILE ARE IGNORED
-notanartist	In Dreams
-belongtogether	Darth & Vader
-replacetitle	첫 사랑니 (Rum Pum Pum Pum)	Rum Pum Pum Pum
-replaceartist	Dal Shabet			Dal★Shabet
-countas		Trouble Maker			HyunA
+#notanartist	In Dreams
+#belongtogether	Darth & Vader
+#replacetitle	첫 사랑니 (Rum Pum Pum Pum)	Rum Pum Pum Pum
+#replaceartist	Dal Shabet			Dal★Shabet
+#countas		Trouble Maker			HyunA

utilities.py

@@ -0,0 +1,52 @@
+
+
+
+def parseTSV(filename,*args):
+	f = open(filename)
+	
+	result = []
+	for l in [l for l in f if (not l.startswith("#")) and (not l.strip()=="")]:
+		
+		l = l.replace("\n","").split("#")[0]
+		data = list(filter(None,l.split("\t"))) # Multiple tabs are okay, we don't accept empty fields unless trailing
+		entry = [] * len(args)
+		for i in range(len(args)):
+			if args[i]=="list":
+				try:
+					entry.append(data[i].split("␟"))
+				except:
+					entry.append([])
+			elif args[i]=="string":
+				try:
+					entry.append(data[i])
+				except:
+					entry.append("")
+			elif args[i]=="int":
+				try:
+					entry.append(int(data[i]))
+				except:
+					entry.append(0)
+			elif args[i]=="bool":
+				try:
+					entry.append((data[i].lower() in ["true","yes","1","y"]))
+				except:
+					entry.append(False)
+				
+		result.append(entry)
+		
+	f.close()
+	return result
+	
+def parseAllTSV(path,*args):
+	
+	import os
+	
+	result = []
+	for f in os.listdir(path + "/"):
+		
+		if (".tsv" in f):
+			
+			result += parseTSV(path + "/" + f,*args)
+			
+	return result
+