1
0
mirror of https://github.com/krateng/maloja.git synced 2023-08-10 21:12:55 +03:00

Implemented custom rules

This commit is contained in:
Krateng 2018-11-28 17:45:52 +01:00
parent 144198f933
commit 54bffc5642
5 changed files with 163 additions and 67 deletions

View File

@ -1,65 +1,103 @@
import re
import utilities
# need to do this as a class so it can retain loaded settings from file
class CleanerAgent:
def __init__(self):
self.updateRules()
def updateRules(self):
raw = utilities.parseAllTSV("rules","string","string","string")
self.rules_belongtogether = [b for [a,b,c] in raw if a=="belongtogether"]
self.rules_notanartist = [b for [a,b,c] in raw if a=="notanartist"]
self.rules_replacetitle = {b:c for [a,b,c] in raw if a=="replacetitle"}
self.rules_replaceartist = {b:c for [a,b,c] in raw if a=="replaceartist"}
def fullclean(self,artist,title):
artists = self.parseArtists(self.removespecial(artist))
title = self.parseTitle(self.removespecial(title))
(title,moreartists) = self.parseTitleForArtists(title)
artists += moreartists
return (list(set(artists)),title)
def removespecial(self,s):
return s.replace("\t","").replace("","").replace("\n","")
delimiters_feat = ["ft.","ft","feat.","feat","featuring"] #Delimiters used for extra artists, even when in the title field
delimiters = ["vs.","vs","&"] #Delimiters in informal titles, spaces expected around them
delimiters_formal = ["; ",";"] #Delimiters used specifically to tag multiple artists when only one tag field is available, no spaces used
def parseArtists(self,a):
if a.strip() == "":
return []
if a.strip() in self.rules_belongtogether:
return [a.strip()]
if a.strip() in self.rules_replaceartist:
return [self.rules_replaceartist[a.strip()]]
for d in self.delimiters_feat:
if re.match(r"(.*) \(" + d + " (.*)\)",a) is not None:
return self.parseArtists(re.sub(r"(.*) \(" + d + " (.*)\)",r"\1",a)) + self.parseArtists(re.sub(r"(.*) \(" + d + " (.*)\)",r"\2",a))
for d in (self.delimiters_feat + self.delimiters):
if ((" " + d + " ") in a):
ls = []
for i in a.split(" " + d + " "):
ls += self.parseArtists(i)
return ls
for d in self.delimiters_formal:
if (d in a):
ls = []
for i in a.split(d):
ls += self.parseArtists(i)
return ls
return [a.strip()]
def parseTitle(self,t):
if t.strip() in self.rules_replacetitle:
return self.rules_replacetitle[t.strip()]
t = t.replace("[","(").replace("]",")")
t = re.sub(r" \(as made famous by .*?\)","",t)
t = re.sub(r" \(originally by .*?\)","",t)
return t.strip()
def parseTitleForArtists(self,t):
for d in self.delimiters_feat:
if re.match(r"(.*) \(" + d + " (.*?)\)",t) is not None:
(title,artists) = self.parseTitleForArtists(re.sub(r"(.*) \(" + d + " (.*?)\)",r"\1",t))
artists += self.parseArtists(re.sub(r"(.*) \(" + d + " (.*?)\).*",r"\2",t))
return (title,artists)
return (t,[])
def fullclean(artist,title):
artists = parseArtists(removespecial(artist))
title = parseTitle(removespecial(title))
(title,moreartists) = parseTitleForArtists(title)
artists += moreartists
return (list(set(artists)),title)
def removespecial(s):
return s.replace("\t","").replace("","").replace("\n","")
delimiters_feat = ["ft.","ft","feat.","feat","featuring"] #Delimiters used for extra artists, even when in the title field
delimiters = ["vs.","vs","&"] #Delimiters in informal titles, spaces expected around them
delimiters_formal = ["; ",";"] #Delimiters used specifically to tag multiple artists when only one tag field is available, no spaces used
def parseArtists(a):
if a.strip() == "":
return []
for d in delimiters_feat:
if re.match(r"(.*) \(" + d + " (.*)\)",a) is not None:
return parseArtists(re.sub(r"(.*) \(" + d + " (.*)\)",r"\1",a)) + parseArtists(re.sub(r"(.*) \(" + d + " (.*)\)",r"\2",a))
for d in (delimiters + delimiters_feat):
if ((" " + d + " ") in a):
ls = []
for i in a.split(" " + d + " "):
ls += parseArtists(i)
return ls
for d in delimiters_formal:
if (d in a):
ls = []
for i in a.split(d):
ls += parseArtists(i)
return ls
return [a.strip()]
def parseTitle(t):
t = t.replace("[","(").replace("]",")")
t = re.sub(r" \(as made famous by .*?\)","",t)
t = re.sub(r" \(originally by .*?\)","",t)
return t
def parseTitleForArtists(t):
for d in delimiters_feat:
if re.match(r"(.*) \(" + d + " (.*?)\)",t) is not None:
(title,artists) = parseTitleForArtists(re.sub(r"(.*) \(" + d + " (.*?)\)",r"\1",t))
artists += parseArtists(re.sub(r"(.*) \(" + d + " (.*?)\).*",r"\2",t))
return (title,artists)
return (t,[])
def flatten(lis):

View File

@ -4,7 +4,7 @@ import urllib
import waitress
import os
import datetime
import cleanup
from cleanup import *
import sys
@ -12,6 +12,8 @@ SCROBBLES = [] # Format: tuple(track_ref,timestamp,saved)
ARTISTS = [] # Format: artist
TRACKS = [] # Format: tuple(frozenset(artist_ref,...),title)
c = CleanerAgent()
lastsync = 0
@ -118,10 +120,11 @@ def post_scrobble():
#title = urllib.parse.unquote(keys.get("title"))
artists = keys.get("artist")
title = keys.get("title")
time = int(keys.get("time"))
(artists,title) = cleanup.fullclean(artists,title)
if time is None:
try:
time = int(keys.get("time"))
except:
time = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp())
(artists,title) = c.fullclean(artists,title)
## this is necessary for localhost testing
response.set_header("Access-Control-Allow-Origin","*")

View File

@ -1,9 +1,12 @@
import sys, os, datetime, re, cleanup
from cleanup import *
log = open(sys.argv[1])
outputlog = open(sys.argv[2],"a")
c = CleanerAgent()
for l in log:
l = l.replace("\n","")
data = l.split(",")
@ -14,7 +17,7 @@ for l in log:
time = data[3]
(artists,title) = cleanup.fullclean(artist,title)
(artists,title) = c.fullclean(artist,title)
artistsstr = "".join(artists)

View File

@ -7,8 +7,8 @@
### countas: defines an artist that should be counted together with another artist for chart statistics etc. This will not change the separation in the database and all effects of this rule will disappear as soon as it is no longer active. Second column is the artist, third column the replacement artist
###
### THE RULES IN THIS EXAMPLE FILE ARE IGNORED
notanartist In Dreams
belongtogether Darth & Vader
replacetitle 첫 사랑니 (Rum Pum Pum Pum) Rum Pum Pum Pum
replaceartist Dal Shabet Dal★Shabet
countas Trouble Maker HyunA
#notanartist In Dreams
#belongtogether Darth & Vader
#replacetitle 첫 사랑니 (Rum Pum Pum Pum) Rum Pum Pum Pum
#replaceartist Dal Shabet Dal★Shabet
#countas Trouble Maker HyunA

Can't render this file because it contains an unexpected character in line 3 and column 58.

52
utilities.py Normal file
View File

@ -0,0 +1,52 @@
def parseTSV(filename,*args):
f = open(filename)
result = []
for l in [l for l in f if (not l.startswith("#")) and (not l.strip()=="")]:
l = l.replace("\n","").split("#")[0]
data = list(filter(None,l.split("\t"))) # Multiple tabs are okay, we don't accept empty fields unless trailing
entry = [] * len(args)
for i in range(len(args)):
if args[i]=="list":
try:
entry.append(data[i].split(""))
except:
entry.append([])
elif args[i]=="string":
try:
entry.append(data[i])
except:
entry.append("")
elif args[i]=="int":
try:
entry.append(int(data[i]))
except:
entry.append(0)
elif args[i]=="bool":
try:
entry.append((data[i].lower() in ["true","yes","1","y"]))
except:
entry.append(False)
result.append(entry)
f.close()
return result
def parseAllTSV(path,*args):
import os
result = []
for f in os.listdir(path + "/"):
if (".tsv" in f):
result += parseTSV(path + "/" + f,*args)
return result