Moved tsv handling to doreah

cleanup.py

@@ -1,5 +1,6 @@
 import re
 import utilities
+from doreah import tsv
 
 # need to do this as a class so it can retain loaded settings from file
 # apparently this is not true
@@ -10,7 +11,7 @@ class CleanerAgent:
 		self.updateRules()
 
 	def updateRules(self):
-		raw = utilities.parseAllTSV("rules","string","string","string")
+		raw = tsv.parse_all("rules","string","string","string")
 		self.rules_belongtogether = [b for [a,b,c] in raw if a=="belongtogether"]
 		self.rules_notanartist = [b for [a,b,c] in raw if a=="notanartist"]
 		self.rules_replacetitle = {b:c for [a,b,c] in raw if a=="replacetitle"}
@@ -127,7 +128,7 @@ class CollectorAgent:
 		self.updateRules()
 
 	def updateRules(self):
-		raw = utilities.parseAllTSV("rules","string","string","string")
+		raw = tsv.parse_all("rules","string","string","string")
 		self.rules_countas = {b:c for [a,b,c] in raw if a=="countas"}
 		self.rules_include = {} #Twice the memory, double the performance! (Yes, we're saving redundant information here, but it's not unelegant if it's within a closed object!)
 		for a in self.rules_countas:

database.py

@@ -7,6 +7,7 @@ import datetime
 from cleanup import *
 from utilities import *
 from doreah.logging import log
+from doreah import tsv
 from malojatime import *
 import sys
 import unicodedata
@@ -39,8 +40,10 @@ db_rulestate = False
 ### symmetric keys are fine for now since we hopefully use HTTPS
 def loadAPIkeys():
 	global clients
-	createTSV("clients/authenticated_machines.tsv")
+	tsv.create("clients/authenticated_machines.tsv")
-	clients = parseTSV("clients/authenticated_machines.tsv","string","string")
+	#createTSV("clients/authenticated_machines.tsv")
+	clients = tsv.parse("clients/authenticated_machines.tsv","string","string")
+	#clients = parseTSV("clients/authenticated_machines.tsv","string","string")
 	log("Authenticated Machines: " + ", ".join([m[1] for m in clients]))
 
 def checkAPIkey(k):
@@ -550,7 +553,8 @@ def newrule():
 	keys = FormsDict.decode(request.forms)
 	apikey = keys.pop("key",None)
 	if (checkAPIkey(apikey)):
-		addEntry("rules/webmade.tsv",[k for k in keys])
+		tsv.add_entry("rules/webmade.tsv",[k for k in keys])
+		#addEntry("rules/webmade.tsv",[k for k in keys])
 		global db_rulestate
 		db_rulestate = False
 
@@ -742,7 +746,8 @@ def build_db():
 
 
 	# parse files
-	db = parseAllTSV("scrobbles","int","string","string",escape=False)
+	db = tsv.parse_all("scrobbles","int","string","string",comments=False)
+	#db = parseAllTSV("scrobbles","int","string","string",escape=False)
 	for sc in db:
 		artists = sc[1].split("␟")
 		title = sc[2]
@@ -803,7 +808,8 @@ def sync():
 			SCROBBLES[idx] = (SCROBBLES[idx][0],SCROBBLES[idx][1],True)
 
 	for e in entries:
-		addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False)
+		tsv.add_entries("scrobbles/" + e + ".tsv",entries[e],comments=False)
+		#addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False)
 		combineChecksums("scrobbles/" + e + ".tsv",cla.checksums)
 
 

utilities.py

@@ -6,48 +6,49 @@ import pickle
 import urllib
 import datetime
 from doreah import settings
+from doreah.logging import log
 
 
 ### TSV files
 
-def parseTSV(filename,*args,escape=True):
+#def parseTSV(filename,*args,escape=True):
-	f = open(filename)
+#	f = open(filename)
+#
+#	result = []
+#	for l in [l for l in f if (not l.startswith("#")) and (not l.strip()=="")]:
+#
+#		l = l.replace("\n","")
+#		if escape:
+#			l = l.split("#")[0]
+#		l = l.replace(r"\num","#") # translate escape sequences even if we don't support comments in the file and they are not actually necessary (they might still be used for some reason)
+#		data = list(filter(None,l.split("\t"))) # Multiple tabs are okay, we don't accept empty fields unless trailing
+#		entry = [] * len(args)
+#		for i in range(len(args)):
+#			if args[i]=="list":
+#				try:
+#					entry.append(data[i].split("␟"))
+#				except:
+#					entry.append([])
+#			elif args[i]=="string":
+#				try:
+#					entry.append(data[i])
+#				except:
+#					entry.append("")
+#			elif args[i]=="int":
+#				try:
+#					entry.append(int(data[i]))
+#				except:
+#					entry.append(0)
+#			elif args[i]=="bool":
+#				try:
+#					entry.append((data[i].lower() in ["true","yes","1","y"]))
+#				except:
+#					entry.append(False)
+#
+#		result.append(entry)
 
-	result = []
+#	f.close()
-	for l in [l for l in f if (not l.startswith("#")) and (not l.strip()=="")]:
+#	return result
-
-		l = l.replace("\n","")
-		if escape:
-			l = l.split("#")[0]
-		l = l.replace(r"\num","#") # translate escape sequences even if we don't support comments in the file and they are not actually necessary (they might still be used for some reason)
-		data = list(filter(None,l.split("\t"))) # Multiple tabs are okay, we don't accept empty fields unless trailing
-		entry = [] * len(args)
-		for i in range(len(args)):
-			if args[i]=="list":
-				try:
-					entry.append(data[i].split("␟"))
-				except:
-					entry.append([])
-			elif args[i]=="string":
-				try:
-					entry.append(data[i])
-				except:
-					entry.append("")
-			elif args[i]=="int":
-				try:
-					entry.append(int(data[i]))
-				except:
-					entry.append(0)
-			elif args[i]=="bool":
-				try:
-					entry.append((data[i].lower() in ["true","yes","1","y"]))
-				except:
-					entry.append(False)
-
-		result.append(entry)
-
-	f.close()
-	return result
 
 def checksumTSV(folder):
 
@@ -110,40 +111,40 @@ def consistentRulestate(folder,checksums):
 	return True
 
 
-def parseAllTSV(path,*args,escape=True):
+#def parseAllTSV(path,*args,escape=True):
+#
+#
+#	result = []
+#	for f in os.listdir(path + "/"):
+#
+#		if (f.endswith(".tsv")):
+#
+#			result += parseTSV(path + "/" + f,*args,escape=escape)
+#
+#	return result
 
+#def createTSV(filename):
+#
+#	if not os.path.exists(filename):
+#		open(filename,"w").close()
 
-	result = []
+#def addEntry(filename,a,escape=True):
-	for f in os.listdir(path + "/"):
+#
-
+#	createTSV(filename)
-		if (f.endswith(".tsv")):
+#
-
+#	line = "\t".join(a)
-			result += parseTSV(path + "/" + f,*args,escape=escape)
+#	if escape: line = line.replace("#",r"\num")
-
+#	with open(filename,"a") as f:
-	return result
+#		f.write(line + "\n")
-
-def createTSV(filename):
-
-	if not os.path.exists(filename):
-		open(filename,"w").close()
-
-def addEntry(filename,a,escape=True):
-
-	createTSV(filename)
-
-	line = "\t".join(a)
-	if escape: line = line.replace("#",r"\num")
-	with open(filename,"a") as f:
-		f.write(line + "\n")
-
-def addEntries(filename,al,escape=True):
-
-	with open(filename,"a") as f:
-		for a in al:
-			line = "\t".join(a)
-			if escape: line = line.replace("#",r"\num")
-			f.write(line + "\n")
 
+#def addEntries(filename,al,escape=True):
+#
+#	with open(filename,"a") as f:
+#		for a in al:
+#			line = "\t".join(a)
+#			if escape: line = line.replace("#",r"\num")
+#			f.write(line + "\n")
+#
 
 
 ### Useful functions
@@ -273,6 +274,7 @@ def cache_track(artists,title,result):
 	day = datetime.date.today().toordinal()
 	cachedTracksDays[(frozenset(artists),title)] = day
 def cache_artist(artist,result):
+	if result is None: log("Caching None for " + artist,module="debug")
 	cachedArtists[artist] = result
 	day = datetime.date.today().toordinal()
 	cachedArtistsDays[artist] = day