Updated database fix

maloja/controller.py

@@ -198,6 +198,10 @@ def backup(level="full"):
 def update():
 	os.system("pip3 install malojaserver --upgrade --no-cache-dir")
 
+def fixdb():
+	from .fixexisting import fix
+	fix()
+
 @mainfunction({"l":"level"},shield=True)
 def main(action,*args,**kwargs):
 	actions = {
@@ -207,7 +211,8 @@ def main(action,*args,**kwargs):
 		"import":loadlastfm,
 		"debug":direct,
 		"backup":backup,
-		"update":update
+		"update":update,
+		"fix":fixdb
 	}
 	if action in actions: actions[action](*args,**kwargs)
 	else: print("Valid commands: " + " ".join(a for a in actions))

maloja/fixexisting.py

@@ -1,46 +1,64 @@
 import os
+from .__init__ import DATA_DIR
+os.chdir(DATA_DIR)
+
 import re
 from .cleanup import CleanerAgent
 from doreah.logging import log
 import difflib
+import datetime
 
 wendigo = CleanerAgent()
 
 exp = r"([0-9]*)(\t+)([^\t]+?)(\t+)([^\t]+)(\t*)([^\t]*)\n"
 
-for fn in os.listdir("scrobbles/"):
-	if fn.endswith(".tsv"):
-		f = open("scrobbles/" + fn)
-		fnew = open("scrobbles/" + fn + "_new","w")
-		for l in f:
+pthj = os.path.join
 
-			a,t = re.sub(exp,r"\3",l), re.sub(exp,r"\5",l)
-			r1,r2,r3 = re.sub(exp,r"\1\2",l),re.sub(exp,r"\4",l),re.sub(exp,r"\6\7",l)
 
-			a = a.replace("␟",";")
+def fix():
 
-			(al,t) = wendigo.fullclean(a,t)
-			a = "␟".join(al)
-			fnew.write(r1 + a + r2 + t + r3 + "\n")
+	now = datetime.datetime.utcnow()
+	nowstr = now.strftime("%Y_%m_%d_%H_%M_%S")
+	datestr = now.strftime("%Y/%m/%d")
+	timestr = now.strftime("%H:%M:%S")
 
-			#print("Artists: " + a)
-			#print("Title: " + t)
-			#print("1: " + r1)
-			#print("2: " + r2)
-			#print("3: " + r3)
+	with open(pthj(DATA_DIR,"logs","dbfix",nowstr + ".log"),"a") as logfile:
 
-		f.close()
-		fnew.close()
+		logfile.write("Database fix initiated on " + datestr + " " + timestr + " UTC")
+		logfile.write("\n\n")
 
-		#os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn)
-		with open("scrobbles/" + fn + "_new","r") as newfile:
-			with open("scrobbles/" + fn,"r") as oldfile:
-				diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="")
-				diff = list(diff)[2:]
-				log("Diff for scrobbles/" + fn + "".join("\n\t" + d for d in diff),module="fixer")
+		for filename in os.listdir(pthj(DATA_DIR,"scrobbles")):
+			if filename.endswith(".tsv"):
+				filename_new = filename + "_new"
 
-		os.rename("scrobbles/" + fn + "_new","scrobbles/" + fn)
+				with open(pthj(DATA_DIR,"scrobbles",filename_new),"w") as newfile:
+					with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile:
 
-		checkfile = open("scrobbles/" + fn + ".rulestate","w")
-		checkfile.write(wendigo.checksums)
-		checkfile.close()
+						for l in oldfile:
+
+							a,t = re.sub(exp,r"\3",l), re.sub(exp,r"\5",l)
+							r1,r2,r3 = re.sub(exp,r"\1\2",l),re.sub(exp,r"\4",l),re.sub(exp,r"\6\7",l)
+
+							a = a.replace("␟",";")
+
+							(al,t) = wendigo.fullclean(a,t)
+							a = "␟".join(al)
+							newfile.write(r1 + a + r2 + t + r3 + "\n")
+
+
+				#os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn)
+				with open(pthj(DATA_DIR,"scrobbles",filename_new),"r") as newfile:
+					with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile:
+
+						diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="")
+						diff = list(diff)[2:]
+						#log("Diff for scrobbles/" + filename + "".join("\n\t" + d for d in diff),module="fixer")
+						output = "Diff for scrobbles/" + filename + "".join("\n\t" + d for d in diff)
+						print(output)
+						logfile.write(output)
+						logfile.write("\n")
+
+				os.rename(pthj(DATA_DIR,"scrobbles",filename_new),pthj(DATA_DIR,"scrobbles",filename))
+
+				with open(pthj(DATA_DIR,"scrobbles",filename + ".rulestate"),"w") as checkfile:
+					checkfile.write(wendigo.checksums)