Moved logging and timing to doreah

database.py

@@ -6,6 +6,7 @@ import os
 import datetime
 from cleanup import *
 from utilities import *
+from doreah.logging import log
 from malojatime import *
 import sys
 import unicodedata

doreah/logging.py

@@ -0,0 +1,102 @@
+import datetime
+import inspect
+import os
+
+from ._internal import defaultarguments, gopen
+
+_config = {}
+
+_queue = []
+_locked = False
+
+# set configuration
+# logfolder			folder to store logfiles in
+# timeformat		strftime format for log files
+# defaultmodule		name for the main running script
+# verbosity			higher means more (less important) messages are shown on console
+def config(logfolder="logs",timeformat="%Y/%m/%d %H:%M:%S",defaultmodule="main",verbosity=0):
+	global _config
+	_config["logfolder"] = logfolder
+	_config["timeformat"] = timeformat
+	_config["defaultmodule"] = defaultmodule
+	_config["verbosity"] = verbosity
+
+
+# initial config on import, set everything to default
+config()
+
+
+
+
+
+# Log entry
+# module		allows discrimination between modules of a program. Will be prepended in console output and will determine the separate file for disk storage
+# 				defaults to actual name of the calling module or "main" for the main script
+# header		determines the hierarchical position of the entry.
+# indent		adds indent to the log entry
+# importance	low means important. if higher than the configured verbosity, entry will not be shown on console
+def log(*msgs,module=None,header=None,indent=0,importance=0):
+
+	now = datetime.datetime.utcnow().strftime(_config["timeformat"])
+
+	# log() can be used to add empty line
+	if len(msgs) == 0: msgs = ("",)
+
+	# make it easier to log data structures and such
+	msgs = tuple([str(msg) for msg in msgs])
+
+	# header formating
+	if header == 2:
+		msgs = ("","","####") + msgs + ("####","")
+	elif header == 1:
+		msgs = ("","","","# # # # #","") + msgs + ("","# # # # #","","")
+
+	# indent
+	prefix = "\t" * indent
+
+	# module name
+	if module is None:
+		try:
+			module = inspect.getmodule(inspect.stack()[1][0]).__name__
+			if module == "__main__": module = _config["defaultmodule"]
+		except:
+			module = "interpreter"
+
+	global _locked, _queue
+	if _locked:
+		for msg in msgs:
+			_queue.append({"time":now,"prefix":prefix,"msg":msg,"module":module,"console":(importance <= _config["verbosity"])})
+	else:
+		# console output
+		if (importance <= _config["verbosity"]):
+			for msg in msgs:
+				print("[" + module + "] " + prefix + msg)
+
+		# file output
+		logfilename = _config["logfolder"] + "/" + module + ".log"
+		#os.makedirs(os.path.dirname(logfilename), exist_ok=True)
+		with gopen(logfilename,"a") as logfile:
+			for msg in msgs:
+				logfile.write(now + "  " + prefix + msg + "\n")
+
+
+def flush():
+	global _queue
+	for entry in _queue:
+		# console output
+		if entry["console"]:
+			print("[" + entry["module"] + "] " + entry["prefix"] + entry["msg"])
+
+		# file output
+		logfilename = _config["logfolder"] + "/" + entry["module"] + ".log"
+		#os.makedirs(os.path.dirname(logfilename), exist_ok=True)
+		with gopen(logfilename,"a") as logfile:
+			logfile.write(entry["time"] + "  " + entry["prefix"] + entry["msg"] + "\n")
+
+	_queue = []
+
+# Quicker way to add header
+def logh1(*args,**kwargs):
+	return log(*args,**kwargs,header=1)
+def logh2(*args,**kwargs):
+	return log(*args,**kwargs,header=2)

doreah/persistence.py

@@ -0,0 +1,40 @@
+import pickle
+import os
+
+from ._internal import defaultarguments, gopen
+
+_config = {}
+
+# set configuration
+# folder	folder to store log files
+def config(folder="storage"):
+	global _config
+	_config["folder"] = folder
+
+
+# initial config on import, set everything to default
+config()
+
+@defaultarguments(_config,folder="folder")
+def save(data,name,folder):
+
+	filename = os.path.join(folder,name + ".gilly")
+
+	fl = gopen(filename,"wb")
+	stream = pickle.dumps(data)
+	fl.write(stream)
+	fl.close()
+
+@defaultarguments(_config,folder="folder")
+def load(name,folder):
+
+	filename = os.path.join(folder,name + ".gilly")
+
+	try:
+		fl = gopen(filename,"rb")
+		ob = pickle.loads(fl.read())
+	except: ob = None
+	finally:
+		fl.close()
+
+	return ob

doreah/timing.py

@@ -0,0 +1,44 @@
+import time
+
+from ._internal import defaultarguments
+
+_config = {}
+
+
+# set configuration
+# si		0 means seconds, 1 ms, 2 μs, 3 ns etc
+def config(si=0):
+	global _config
+	_config["si"] = si
+
+
+# initial config on import, set everything to default
+config()
+
+
+# Take clock. Returns time passed since last call of this function. if called with an identifier, will only
+# consider calls with that identifier. No identifier means any call is valid.
+# identifiers	arbitrary strings to remember different timers. guaranteed to set all timers to exactly the same time for
+#				all identifiers in one call. will return tuple of all identifiers, singular value if only one identifier
+def clock(*identifiers,lastcalls={None:None}):
+
+	if len(identifiers) == 0: identifiers = (None,)
+
+	now = time.time()
+	# get last calls
+	stamps = (lastcalls.get(i) for i in identifiers)
+	results = tuple(None if lc is None else (now - lc) * (1000**_config["si"]) for lc in stamps)
+	if len(results) == 1: results = results[0]
+
+	# set new stamps
+	for i in identifiers:
+		lastcalls[i] = now
+	lastcalls[None] = now			# always save last overall call so we can directly access it
+
+	return results
+
+
+
+def clockp(name,*identifiers):
+	time = clock(*identifiers)
+	print(name + ": " + str(time))

doreah/tsv.py

@@ -0,0 +1,115 @@
+import os
+
+from ._internal import defaultarguments
+
+_config = {}
+
+# set configuration
+# defaultextension	files with this extension will be regarded as valid files. can be overwritten per request.
+# comments			whether files may include commenting (indicated by #)
+# multitab			whether fields can be separated by multiple tabs (this makes empty fields impossible except when trailing)
+def config(defaultextension=".tsv",comments=True,multitab=True):
+	global _config
+	_config["defaultextension"] = defaultextension
+	_config["comments"] = comments
+	_config["multitab"] = multitab
+
+
+# initial config on import, set everything to default
+config()
+
+
+@defaultarguments(_config,comments="comments",multitab="multitab")
+def parse(filename,*args,comments,multitab):
+
+	if not os.path.exists(filename):
+		filename = filename + _config["defaultextension"]
+
+	f = open(filename)
+
+	result = []
+	for l in [l for l in f if (not l.startswith("#")) and (not l.strip()=="")]:
+		l = l.replace("\n","")
+
+		# if the file allows comments, we need to replace the escape sequence and properly stop parsing for inline comments
+		if comments:
+			l = l.split("#")[0]
+			l = l.replace(r"\num","#")
+			l = l.replace(r"\hashtag","#")
+
+		# we either allow multiple tabs, or we don't (in which case empty fields are possible)
+		if multitab:
+			data = list(filter(None,l.split("\t")))
+		else:
+			data = list(l.split("\t"))
+
+		entry = [] * len(args)
+		for i in range(len(args)):
+			if args[i] in ["list","ls","array"]:
+				try:
+					entry.append(data[i].split("␟"))
+				except:
+					entry.append([])
+			elif args[i] in ["string","str","text"]:
+				try:
+					entry.append(data[i])
+				except:
+					entry.append("")
+			elif args[i] in ["int","integer","num","number"]:
+				try:
+					entry.append(int(data[i]))
+				except:
+					entry.append(0)
+			elif args[i] in ["bool","boolean"]:
+				try:
+					entry.append((data[i].lower() in ["true","yes","1","y"]))
+				except:
+					entry.append(False)
+			else:
+				raise TypeError()
+
+		result.append(entry)
+
+	f.close()
+	return result
+
+@defaultarguments(_config,extension="defaultextension")
+def parse_all(path,*args,extension,**kwargs):
+
+	result = []
+	for f in os.listdir(path + "/"):
+		if (f.endswith(extension)): # use "" if all files are valid
+			result += parse(path + "/" + f,*args,**kwargs)
+
+	return result
+
+
+
+def create(filename):
+
+	if not os.path.exists(filename):
+		open(filename,"w").close()
+
+@defaultarguments(_config,comments="comments")
+def add_entry(filename,a,comments):
+
+	create(filename)
+	# remove all tabs and create tab-separated string
+	line = "\t".join([str(e).replace("\t"," ") for e in a])
+
+	# replace comment symbol
+	if comments: line = line.replace("#",r"\num")
+
+	with open(filename,"a") as f:
+		f.write(line + "\n")
+
+@defaultarguments(_config,comments="comments")
+def add_entries(filename,al,comments):
+
+	create(filename)
+
+	with open(filename,"a") as f:
+		for a in al:
+			line = "\t".join([str(e).replace("\t"," ") for e in a])
+			if comments: line = line.replace("#",r"\num")
+			f.write(line + "\n")

fixexisting.py

@@ -1,7 +1,7 @@
 import os
 import re
 from cleanup import CleanerAgent
-from utilities import log
+from doreah.logging import log
 import difflib
 
 wendigo = CleanerAgent()

lastfmconverter.py

@@ -61,6 +61,3 @@ checksumfile.write(c.checksums)
 log.close()
 outputlog.close()
 checksumfile.close()
-	
-
-

server.py

@@ -10,6 +10,7 @@ from utilities import *
 from htmlgenerators import KeySplit
 # doreah toolkit
 from doreah import settings
+from doreah.logging import log
 # technical
 from importlib.machinery import SourceFileLoader
 import _thread

utilities.py

@@ -164,27 +164,29 @@ def addEntries(filename,al,escape=True):
 
 
 ### Logging
+# now handled by doreah
 
-def log(msg,module=None):
+#def log(msg,module=None):
-	now = datetime.datetime.utcnow().strftime("%Y/%m/%d %H:%M:%S")
+#	now = datetime.datetime.utcnow().strftime("%Y/%m/%d %H:%M:%S")
-	if module is None:
+#	if module is None:
-		import inspect
+#		import inspect
-		module = inspect.getmodule(inspect.stack()[1][0]).__name__
+#		module = inspect.getmodule(inspect.stack()[1][0]).__name__
-		if module == "__main__": module = "mainserver"
+#		if module == "__main__": module = "mainserver"
-	print("[" + module + "] " + msg)
+#	print("[" + module + "] " + msg)
-	with open("logs/" + module + ".log","a") as logfile:
+#	with open("logs/" + module + ".log","a") as logfile:
-		logfile.write(now + "  " + msg + "\n")
+#		logfile.write(now + "  " + msg + "\n")
 
 
 ### not meant to be precise, just for a rough idea
-measurement = 0
+# now handled by doreah
-def clock(*args):
+#measurement = 0
-	import time
+#def clock(*args):
-	global measurement
+#	import time
-	now = time.time()
+#	global measurement
-	if len(args) > 0:
+#	now = time.time()
-		print(args[0] + ": " + str(now - measurement))
+#	if len(args) > 0:
-	measurement = now
+#		print(args[0] + ": " + str(now - measurement))
+#	measurement = now
 
 
 

website/pulse.py

@@ -50,4 +50,3 @@ def instructions(keys):
 	replace = {"KEY_PULSE_TABLE":html_pulse,"KEY_IMAGEURL":imgurl,"KEY_LIMITS":limitstring,"KEY_PULSEDETAILS":delimitstring}
 
 	return (replace,pushresources)
-		

website/scrobbles.py

@@ -47,4 +47,3 @@ def instructions(keys):
 	replace = {"KEY_SCROBBLELIST":html,"KEY_SCROBBLES":str(amount),"KEY_IMAGEURL":imgurl,"KEY_LIMITS":limitstring}
 
 	return (replace,pushresources)
-		

website/start.py

@@ -1,7 +1,7 @@
 import urllib
 from datetime import datetime, timedelta
 import database
-from utilities import clock
+from doreah.timing import clock, clockp
 
 from htmlmodules import module_scrobblelist, module_pulse, module_artistcharts_tiles, module_trackcharts_tiles
 
@@ -24,7 +24,7 @@ def instructions(keys):
 	topartists_month = module_artistcharts_tiles(since="month")
 	topartists_week = module_artistcharts_tiles(since=weekstart)
 
-	clock("Artists")
+	clockp("Artists")
 
 	# tracks
 
@@ -34,13 +34,13 @@ def instructions(keys):
 	toptracks_week = module_trackcharts_tiles(since=weekstart)
 
 
-	clock("Tracks")
+	clockp("Tracks")
 
 
 	# scrobbles
 	html_scrobbles, _, _ = module_scrobblelist(max_=15,shortTimeDesc=True,pictures=True,earlystop=True)
 
-	clock("Scrobbles")
+	clockp("Scrobbles")
 
 	# stats
 
@@ -61,7 +61,7 @@ def instructions(keys):
 	amount_total = database.get_scrobbles_num()
 	scrobbles_total = "<a href='/scrobbles'>" + str(amount_total) + "</a>"
 
-	clock("Amounts")
+	clockp("Amounts")
 
 	# pulse
 	dt = datetime.utcnow()
@@ -83,7 +83,7 @@ def instructions(keys):
 	#html_pulse_month = module_pulse(max_=30,since=[dt.year,dt.month],step="day",trail=1)
 	#html_pulse_year = module_pulse(max_=12,since=[dt.year],step="month",trail=1)
 
-	clock("Pulse")
+	clockp("Pulse")
 
 	#pushresources = [{"file":img,"type":"image"} for img in artistimages + trackimages] #can't push scrobble images as we don't get them from the module function, need to think about that
 	pushresources = []

website/toptracks.py

@@ -33,4 +33,3 @@ def instructions(keys):
 	replace = {"KEY_TOPARTIST_IMAGEURL":imgurl,"KEY_TRACKLIST":html_charts,"KEY_LIMITS":limitstring}
 
 	return (replace,pushresources)
-