Moved logging and timing to doreah

doreah/logging.py

@@ -0,0 +1,102 @@
+import datetime
+import inspect
+import os
+
+from ._internal import defaultarguments, gopen
+
+_config = {}
+
+_queue = []
+_locked = False
+
+# set configuration
+# logfolder			folder to store logfiles in
+# timeformat		strftime format for log files
+# defaultmodule		name for the main running script
+# verbosity			higher means more (less important) messages are shown on console
+def config(logfolder="logs",timeformat="%Y/%m/%d %H:%M:%S",defaultmodule="main",verbosity=0):
+	global _config
+	_config["logfolder"] = logfolder
+	_config["timeformat"] = timeformat
+	_config["defaultmodule"] = defaultmodule
+	_config["verbosity"] = verbosity
+
+
+# initial config on import, set everything to default
+config()
+
+
+
+
+
+# Log entry
+# module		allows discrimination between modules of a program. Will be prepended in console output and will determine the separate file for disk storage
+# 				defaults to actual name of the calling module or "main" for the main script
+# header		determines the hierarchical position of the entry.
+# indent		adds indent to the log entry
+# importance	low means important. if higher than the configured verbosity, entry will not be shown on console
+def log(*msgs,module=None,header=None,indent=0,importance=0):
+
+	now = datetime.datetime.utcnow().strftime(_config["timeformat"])
+
+	# log() can be used to add empty line
+	if len(msgs) == 0: msgs = ("",)
+
+	# make it easier to log data structures and such
+	msgs = tuple([str(msg) for msg in msgs])
+
+	# header formating
+	if header == 2:
+		msgs = ("","","####") + msgs + ("####","")
+	elif header == 1:
+		msgs = ("","","","# # # # #","") + msgs + ("","# # # # #","","")
+
+	# indent
+	prefix = "\t" * indent
+
+	# module name
+	if module is None:
+		try:
+			module = inspect.getmodule(inspect.stack()[1][0]).__name__
+			if module == "__main__": module = _config["defaultmodule"]
+		except:
+			module = "interpreter"
+
+	global _locked, _queue
+	if _locked:
+		for msg in msgs:
+			_queue.append({"time":now,"prefix":prefix,"msg":msg,"module":module,"console":(importance <= _config["verbosity"])})
+	else:
+		# console output
+		if (importance <= _config["verbosity"]):
+			for msg in msgs:
+				print("[" + module + "] " + prefix + msg)
+
+		# file output
+		logfilename = _config["logfolder"] + "/" + module + ".log"
+		#os.makedirs(os.path.dirname(logfilename), exist_ok=True)
+		with gopen(logfilename,"a") as logfile:
+			for msg in msgs:
+				logfile.write(now + "  " + prefix + msg + "\n")
+
+
+def flush():
+	global _queue
+	for entry in _queue:
+		# console output
+		if entry["console"]:
+			print("[" + entry["module"] + "] " + entry["prefix"] + entry["msg"])
+
+		# file output
+		logfilename = _config["logfolder"] + "/" + entry["module"] + ".log"
+		#os.makedirs(os.path.dirname(logfilename), exist_ok=True)
+		with gopen(logfilename,"a") as logfile:
+			logfile.write(entry["time"] + "  " + entry["prefix"] + entry["msg"] + "\n")
+
+	_queue = []
+
+# Quicker way to add header
+def logh1(*args,**kwargs):
+	return log(*args,**kwargs,header=1)
+def logh2(*args,**kwargs):
+	return log(*args,**kwargs,header=2)

doreah/persistence.py

@@ -0,0 +1,40 @@
+import pickle
+import os
+
+from ._internal import defaultarguments, gopen
+
+_config = {}
+
+# set configuration
+# folder	folder to store log files
+def config(folder="storage"):
+	global _config
+	_config["folder"] = folder
+
+
+# initial config on import, set everything to default
+config()
+
+@defaultarguments(_config,folder="folder")
+def save(data,name,folder):
+
+	filename = os.path.join(folder,name + ".gilly")
+
+	fl = gopen(filename,"wb")
+	stream = pickle.dumps(data)
+	fl.write(stream)
+	fl.close()
+
+@defaultarguments(_config,folder="folder")
+def load(name,folder):
+
+	filename = os.path.join(folder,name + ".gilly")
+
+	try:
+		fl = gopen(filename,"rb")
+		ob = pickle.loads(fl.read())
+	except: ob = None
+	finally:
+		fl.close()
+
+	return ob

doreah/timing.py

@@ -0,0 +1,44 @@
+import time
+
+from ._internal import defaultarguments
+
+_config = {}
+
+
+# set configuration
+# si		0 means seconds, 1 ms, 2 μs, 3 ns etc
+def config(si=0):
+	global _config
+	_config["si"] = si
+
+
+# initial config on import, set everything to default
+config()
+
+
+# Take clock. Returns time passed since last call of this function. if called with an identifier, will only
+# consider calls with that identifier. No identifier means any call is valid.
+# identifiers	arbitrary strings to remember different timers. guaranteed to set all timers to exactly the same time for
+#				all identifiers in one call. will return tuple of all identifiers, singular value if only one identifier
+def clock(*identifiers,lastcalls={None:None}):
+
+	if len(identifiers) == 0: identifiers = (None,)
+
+	now = time.time()
+	# get last calls
+	stamps = (lastcalls.get(i) for i in identifiers)
+	results = tuple(None if lc is None else (now - lc) * (1000**_config["si"]) for lc in stamps)
+	if len(results) == 1: results = results[0]
+
+	# set new stamps
+	for i in identifiers:
+		lastcalls[i] = now
+	lastcalls[None] = now			# always save last overall call so we can directly access it
+
+	return results
+
+
+
+def clockp(name,*identifiers):
+	time = clock(*identifiers)
+	print(name + ": " + str(time))

doreah/tsv.py

@@ -0,0 +1,115 @@
+import os
+
+from ._internal import defaultarguments
+
+_config = {}
+
+# set configuration
+# defaultextension	files with this extension will be regarded as valid files. can be overwritten per request.
+# comments			whether files may include commenting (indicated by #)
+# multitab			whether fields can be separated by multiple tabs (this makes empty fields impossible except when trailing)
+def config(defaultextension=".tsv",comments=True,multitab=True):
+	global _config
+	_config["defaultextension"] = defaultextension
+	_config["comments"] = comments
+	_config["multitab"] = multitab
+
+
+# initial config on import, set everything to default
+config()
+
+
+@defaultarguments(_config,comments="comments",multitab="multitab")
+def parse(filename,*args,comments,multitab):
+
+	if not os.path.exists(filename):
+		filename = filename + _config["defaultextension"]
+
+	f = open(filename)
+
+	result = []
+	for l in [l for l in f if (not l.startswith("#")) and (not l.strip()=="")]:
+		l = l.replace("\n","")
+
+		# if the file allows comments, we need to replace the escape sequence and properly stop parsing for inline comments
+		if comments:
+			l = l.split("#")[0]
+			l = l.replace(r"\num","#")
+			l = l.replace(r"\hashtag","#")
+
+		# we either allow multiple tabs, or we don't (in which case empty fields are possible)
+		if multitab:
+			data = list(filter(None,l.split("\t")))
+		else:
+			data = list(l.split("\t"))
+
+		entry = [] * len(args)
+		for i in range(len(args)):
+			if args[i] in ["list","ls","array"]:
+				try:
+					entry.append(data[i].split("␟"))
+				except:
+					entry.append([])
+			elif args[i] in ["string","str","text"]:
+				try:
+					entry.append(data[i])
+				except:
+					entry.append("")
+			elif args[i] in ["int","integer","num","number"]:
+				try:
+					entry.append(int(data[i]))
+				except:
+					entry.append(0)
+			elif args[i] in ["bool","boolean"]:
+				try:
+					entry.append((data[i].lower() in ["true","yes","1","y"]))
+				except:
+					entry.append(False)
+			else:
+				raise TypeError()
+
+		result.append(entry)
+
+	f.close()
+	return result
+
+@defaultarguments(_config,extension="defaultextension")
+def parse_all(path,*args,extension,**kwargs):
+
+	result = []
+	for f in os.listdir(path + "/"):
+		if (f.endswith(extension)): # use "" if all files are valid
+			result += parse(path + "/" + f,*args,**kwargs)
+
+	return result
+
+
+
+def create(filename):
+
+	if not os.path.exists(filename):
+		open(filename,"w").close()
+
+@defaultarguments(_config,comments="comments")
+def add_entry(filename,a,comments):
+
+	create(filename)
+	# remove all tabs and create tab-separated string
+	line = "\t".join([str(e).replace("\t"," ") for e in a])
+
+	# replace comment symbol
+	if comments: line = line.replace("#",r"\num")
+
+	with open(filename,"a") as f:
+		f.write(line + "\n")
+
+@defaultarguments(_config,comments="comments")
+def add_entries(filename,al,comments):
+
+	create(filename)
+
+	with open(filename,"a") as f:
+		for a in al:
+			line = "\t".join([str(e).replace("\t"," ") for e in a])
+			if comments: line = line.replace("#",r"\num")
+			f.write(line + "\n")