Reorganized tasks

maloja/proccontrol/control.py

@@ -6,6 +6,7 @@ import os
 import signal
 
 from .setup import setup
+from . import tasks
 
 def getInstance():
 	try:
@@ -82,11 +83,10 @@ def main(action,*args,**kwargs):
 		"run":direct,
 		"debug":direct,
 
-	#	"import":loadlastfm,
-
-	#	"backup":backuphere,
+		"import":tasks.loadlastfm,
+		"backup":tasks.backuphere,
 	#	"update":update,
-	#	"fix":fixdb
+		"fix":tasks.fixdb
 	}
 	if action in actions: actions[action](*args,**kwargs)
 	else: print("Valid commands: " + " ".join(a for a in actions))

maloja/proccontrol/tasks/__init__.py

@@ -1,9 +1,10 @@
 import os
-from ..lastfmconverter import convert
-from ..backup import backup
-from ..fixexisting import fix
-from ..globalconf import datadir
-from .control import restart
+from .lastfmconverter import convert
+from .backup import backup
+from .fixexisting import fix
+
+from ...globalconf import datadir
+from ..control import restart
 from doreah.io import ask
 
 def loadlastfm(filename):

maloja/proccontrol/tasks/backup.py

@@ -0,0 +1,35 @@
+import tarfile
+from datetime import datetime
+import glob
+import os
+from ...globalconf import datadir
+
+
+user_files = {
+	"minimal":[
+		"rules/*.tsv",
+		"scrobbles"
+	],
+	"full":[
+		"clients/authenticated_machines.tsv",
+		"images/artists",
+		"images/tracks",
+		"settings/settings.ini"
+	]
+}
+
+def backup(folder,level="full"):
+
+	selected_files = user_files["minimal"] if level == "minimal" else user_files["minimal"] + user_files["full"]
+	real_files = []
+	for g in selected_files:
+		real_files += glob.glob(datadir(g))
+
+	now = datetime.utcnow()
+	timestr = now.strftime("%Y_%m_%d_%H_%M_%S")
+	filename = "maloja_backup_" + timestr + ".tar.gz"
+	archivefile = os.path.join(folder,filename)
+	assert not os.path.exists(archivefile)
+	with tarfile.open(name=archivefile,mode="x:gz") as archive:
+		for f in real_files:
+			archive.add(f)

maloja/proccontrol/tasks/fixexisting.py

@@ -0,0 +1,70 @@
+import os
+from ...globalconf import datadir
+import re
+from ...cleanup import CleanerAgent
+from doreah.logging import log
+import difflib
+import datetime
+from .backup import backup
+
+wendigo = CleanerAgent()
+
+exp = r"([0-9]*)(\t+)([^\t]+?)(\t+)([^\t]+)([^\n]*)\n"
+#        1       2    3        4    5       6
+# groups:
+# 1 - timestamp
+# 2 - sep
+# 3 - artists
+# 4 - sep
+# 5 - title
+# 6 - rest
+
+
+
+def fix():
+
+	backup(level="minimal",folder=datadir("backups"))
+
+	now = datetime.datetime.utcnow()
+	nowstr = now.strftime("%Y_%m_%d_%H_%M_%S")
+	datestr = now.strftime("%Y/%m/%d")
+	timestr = now.strftime("%H:%M:%S")
+
+	patchfolder = datadir("logs","dbfix",nowstr)
+	os.makedirs(patchfolder)
+
+	#with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile:
+
+
+	for filename in os.listdir(datadir("scrobbles")):
+		if filename.endswith(".tsv"):
+			filename_new = filename + "_new"
+
+			with open(datadir("scrobbles",filename_new),"w") as newfile:
+				with open(datadir("scrobbles",filename),"r") as oldfile:
+
+					for l in oldfile:
+
+						a,t = re.sub(exp,r"\3",l), re.sub(exp,r"\5",l)
+						r1,r2,r3 = re.sub(exp,r"\1\2",l),re.sub(exp,r"\4",l),re.sub(exp,r"\6",l)
+
+						a = a.replace("␟",";")
+
+						(al,t) = wendigo.fullclean(a,t)
+						a = "␟".join(al)
+						newfile.write(r1 + a + r2 + t + r3 + "\n")
+
+
+			#os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn)
+			with open(datadir("scrobbles",filename_new),"r") as newfile, open(datadir("scrobbles",filename),"r") as oldfile:
+
+				diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="")
+				diff = list(diff)
+
+				with open(os.path.join(patchfolder,filename + ".diff"),"w") as patchfile:
+					patchfile.write("\n".join(diff))
+
+			os.rename(datadir("scrobbles",filename_new),datadir("scrobbles",filename))
+
+			with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile:
+				checkfile.write(wendigo.checksums)

maloja/proccontrol/tasks/lastfmconverter.py

@@ -0,0 +1,67 @@
+import os, datetime, re
+from ...cleanup import *
+from ...utilities import *
+
+
+
+
+c = CleanerAgent()
+
+
+
+def convert(input,output):
+
+	with open(input,"r",encoding="utf-8") as log:
+		with open(output,"w") as outputlog:
+
+
+			stamps = [99999999999999]
+
+			for l in log:
+				l = l.replace("\n","")
+				data = l.split(",")
+
+				artist = data[0]
+				album = data[1]
+				title = data[2]
+				time = data[3]
+
+
+				(artists,title) = c.fullclean(artist,title)
+
+				artistsstr = "␟".join(artists)
+
+
+				timeparts = time.split(" ")
+				(h,m) = timeparts[3].split(":")
+
+				months = {"Jan":1,"Feb":2,"Mar":3,"Apr":4,"May":5,"Jun":6,"Jul":7,"Aug":8,"Sep":9,"Oct":10,"Nov":11,"Dec":12}
+
+				timestamp = int(datetime.datetime(int(timeparts[2]),months[timeparts[1]],int(timeparts[0]),int(h),int(m)).timestamp())
+
+
+				## We prevent double timestamps in the database creation, so we technically don't need them in the files
+				## however since the conversion from lastfm to maloja is a one-time thing, we should take any effort to make the file as good as possible
+				if (timestamp < stamps[-1]):
+					pass
+				elif (timestamp == stamps[-1]):
+					timestamp -= 1
+				else:
+					while(timestamp in stamps):
+						timestamp -= 1
+
+				if (timestamp < stamps[-1]):
+					stamps.append(timestamp)
+				else:
+					stamps.insert(0,timestamp)
+
+
+				entry = "\t".join([str(timestamp),artistsstr,title,album])
+				entry = entry.replace("#",r"\num")
+
+				outputlog.write(entry)
+				outputlog.write("\n")
+
+	with open(output + ".rulestate","w") as checksumfile:
+		#this file stores an identifier for all rules that were in place when the corresponding file was created
+		checksumfile.write(c.checksums)