mirror of
https://github.com/krateng/maloja.git
synced 2023-08-10 21:12:55 +03:00
Moved tsv handling to doreah
This commit is contained in:
Krateng
parent
5765687f9d
commit
2f30157b04
@ -1,5 +1,6 @@
|
||||
import re
|
||||
import utilities
|
||||
from doreah import tsv
|
||||
|
||||
# need to do this as a class so it can retain loaded settings from file
|
||||
# apparently this is not true
|
||||
@ -10,7 +11,7 @@ class CleanerAgent:
|
||||
self.updateRules()
|
||||
|
||||
def updateRules(self):
|
||||
raw = utilities.parseAllTSV("rules","string","string","string")
|
||||
raw = tsv.parse_all("rules","string","string","string")
|
||||
self.rules_belongtogether = [b for [a,b,c] in raw if a=="belongtogether"]
|
||||
self.rules_notanartist = [b for [a,b,c] in raw if a=="notanartist"]
|
||||
self.rules_replacetitle = {b:c for [a,b,c] in raw if a=="replacetitle"}
|
||||
@ -127,7 +128,7 @@ class CollectorAgent:
|
||||
self.updateRules()
|
||||
|
||||
def updateRules(self):
|
||||
raw = utilities.parseAllTSV("rules","string","string","string")
|
||||
raw = tsv.parse_all("rules","string","string","string")
|
||||
self.rules_countas = {b:c for [a,b,c] in raw if a=="countas"}
|
||||
self.rules_include = {} #Twice the memory, double the performance! (Yes, we're saving redundant information here, but it's not unelegant if it's within a closed object!)
|
||||
for a in self.rules_countas:
|
||||
|
||||
16
database.py
16
database.py
@ -7,6 +7,7 @@ import datetime
|
||||
from cleanup import *
|
||||
from utilities import *
|
||||
from doreah.logging import log
|
||||
from doreah import tsv
|
||||
from malojatime import *
|
||||
import sys
|
||||
import unicodedata
|
||||
@ -39,8 +40,10 @@ db_rulestate = False
|
||||
### symmetric keys are fine for now since we hopefully use HTTPS
|
||||
def loadAPIkeys():
|
||||
global clients
|
||||
createTSV("clients/authenticated_machines.tsv")
|
||||
clients = parseTSV("clients/authenticated_machines.tsv","string","string")
|
||||
tsv.create("clients/authenticated_machines.tsv")
|
||||
#createTSV("clients/authenticated_machines.tsv")
|
||||
clients = tsv.parse("clients/authenticated_machines.tsv","string","string")
|
||||
#clients = parseTSV("clients/authenticated_machines.tsv","string","string")
|
||||
log("Authenticated Machines: " + ", ".join([m[1] for m in clients]))
|
||||
|
||||
def checkAPIkey(k):
|
||||
@ -550,7 +553,8 @@ def newrule():
|
||||
keys = FormsDict.decode(request.forms)
|
||||
apikey = keys.pop("key",None)
|
||||
if (checkAPIkey(apikey)):
|
||||
addEntry("rules/webmade.tsv",[k for k in keys])
|
||||
tsv.add_entry("rules/webmade.tsv",[k for k in keys])
|
||||
#addEntry("rules/webmade.tsv",[k for k in keys])
|
||||
global db_rulestate
|
||||
db_rulestate = False
|
||||
|
||||
@ -742,7 +746,8 @@ def build_db():
|
||||
|
||||
|
||||
# parse files
|
||||
db = parseAllTSV("scrobbles","int","string","string",escape=False)
|
||||
db = tsv.parse_all("scrobbles","int","string","string",comments=False)
|
||||
#db = parseAllTSV("scrobbles","int","string","string",escape=False)
|
||||
for sc in db:
|
||||
artists = sc[1].split("␟")
|
||||
title = sc[2]
|
||||
@ -803,7 +808,8 @@ def sync():
|
||||
SCROBBLES[idx] = (SCROBBLES[idx][0],SCROBBLES[idx][1],True)
|
||||
|
||||
for e in entries:
|
||||
addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False)
|
||||
tsv.add_entries("scrobbles/" + e + ".tsv",entries[e],comments=False)
|
||||
#addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False)
|
||||
combineChecksums("scrobbles/" + e + ".tsv",cla.checksums)
|
||||
|
||||
|
||||
|
||||
138
utilities.py
138
utilities.py
@ -6,48 +6,49 @@ import pickle
|
||||
import urllib
|
||||
import datetime
|
||||
from doreah import settings
|
||||
from doreah.logging import log
|
||||
|
||||
|
||||
### TSV files
|
||||
|
||||
def parseTSV(filename,*args,escape=True):
|
||||
f = open(filename)
|
||||
#def parseTSV(filename,*args,escape=True):
|
||||
# f = open(filename)
|
||||
#
|
||||
# result = []
|
||||
# for l in [l for l in f if (not l.startswith("#")) and (not l.strip()=="")]:
|
||||
#
|
||||
# l = l.replace("\n","")
|
||||
# if escape:
|
||||
# l = l.split("#")[0]
|
||||
# l = l.replace(r"\num","#") # translate escape sequences even if we don't support comments in the file and they are not actually necessary (they might still be used for some reason)
|
||||
# data = list(filter(None,l.split("\t"))) # Multiple tabs are okay, we don't accept empty fields unless trailing
|
||||
# entry = [] * len(args)
|
||||
# for i in range(len(args)):
|
||||
# if args[i]=="list":
|
||||
# try:
|
||||
# entry.append(data[i].split("␟"))
|
||||
# except:
|
||||
# entry.append([])
|
||||
# elif args[i]=="string":
|
||||
# try:
|
||||
# entry.append(data[i])
|
||||
# except:
|
||||
# entry.append("")
|
||||
# elif args[i]=="int":
|
||||
# try:
|
||||
# entry.append(int(data[i]))
|
||||
# except:
|
||||
# entry.append(0)
|
||||
# elif args[i]=="bool":
|
||||
# try:
|
||||
# entry.append((data[i].lower() in ["true","yes","1","y"]))
|
||||
# except:
|
||||
# entry.append(False)
|
||||
#
|
||||
# result.append(entry)
|
||||
|
||||
result = []
|
||||
for l in [l for l in f if (not l.startswith("#")) and (not l.strip()=="")]:
|
||||
|
||||
l = l.replace("\n","")
|
||||
if escape:
|
||||
l = l.split("#")[0]
|
||||
l = l.replace(r"\num","#") # translate escape sequences even if we don't support comments in the file and they are not actually necessary (they might still be used for some reason)
|
||||
data = list(filter(None,l.split("\t"))) # Multiple tabs are okay, we don't accept empty fields unless trailing
|
||||
entry = [] * len(args)
|
||||
for i in range(len(args)):
|
||||
if args[i]=="list":
|
||||
try:
|
||||
entry.append(data[i].split("␟"))
|
||||
except:
|
||||
entry.append([])
|
||||
elif args[i]=="string":
|
||||
try:
|
||||
entry.append(data[i])
|
||||
except:
|
||||
entry.append("")
|
||||
elif args[i]=="int":
|
||||
try:
|
||||
entry.append(int(data[i]))
|
||||
except:
|
||||
entry.append(0)
|
||||
elif args[i]=="bool":
|
||||
try:
|
||||
entry.append((data[i].lower() in ["true","yes","1","y"]))
|
||||
except:
|
||||
entry.append(False)
|
||||
|
||||
result.append(entry)
|
||||
|
||||
f.close()
|
||||
return result
|
||||
# f.close()
|
||||
# return result
|
||||
|
||||
def checksumTSV(folder):
|
||||
|
||||
@ -110,40 +111,40 @@ def consistentRulestate(folder,checksums):
|
||||
return True
|
||||
|
||||
|
||||
def parseAllTSV(path,*args,escape=True):
|
||||
#def parseAllTSV(path,*args,escape=True):
|
||||
#
|
||||
#
|
||||
# result = []
|
||||
# for f in os.listdir(path + "/"):
|
||||
#
|
||||
# if (f.endswith(".tsv")):
|
||||
#
|
||||
# result += parseTSV(path + "/" + f,*args,escape=escape)
|
||||
#
|
||||
# return result
|
||||
|
||||
#def createTSV(filename):
|
||||
#
|
||||
# if not os.path.exists(filename):
|
||||
# open(filename,"w").close()
|
||||
|
||||
result = []
|
||||
for f in os.listdir(path + "/"):
|
||||
|
||||
if (f.endswith(".tsv")):
|
||||
|
||||
result += parseTSV(path + "/" + f,*args,escape=escape)
|
||||
|
||||
return result
|
||||
|
||||
def createTSV(filename):
|
||||
|
||||
if not os.path.exists(filename):
|
||||
open(filename,"w").close()
|
||||
|
||||
def addEntry(filename,a,escape=True):
|
||||
|
||||
createTSV(filename)
|
||||
|
||||
line = "\t".join(a)
|
||||
if escape: line = line.replace("#",r"\num")
|
||||
with open(filename,"a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def addEntries(filename,al,escape=True):
|
||||
|
||||
with open(filename,"a") as f:
|
||||
for a in al:
|
||||
line = "\t".join(a)
|
||||
if escape: line = line.replace("#",r"\num")
|
||||
f.write(line + "\n")
|
||||
#def addEntry(filename,a,escape=True):
|
||||
#
|
||||
# createTSV(filename)
|
||||
#
|
||||
# line = "\t".join(a)
|
||||
# if escape: line = line.replace("#",r"\num")
|
||||
# with open(filename,"a") as f:
|
||||
# f.write(line + "\n")
|
||||
|
||||
#def addEntries(filename,al,escape=True):
|
||||
#
|
||||
# with open(filename,"a") as f:
|
||||
# for a in al:
|
||||
# line = "\t".join(a)
|
||||
# if escape: line = line.replace("#",r"\num")
|
||||
# f.write(line + "\n")
|
||||
#
|
||||
|
||||
|
||||
### Useful functions
|
||||
@ -273,6 +274,7 @@ def cache_track(artists,title,result):
|
||||
day = datetime.date.today().toordinal()
|
||||
cachedTracksDays[(frozenset(artists),title)] = day
|
||||
def cache_artist(artist,result):
|
||||
if result is None: log("Caching None for " + artist,module="debug")
|
||||
cachedArtists[artist] = result
|
||||
day = datetime.date.today().toordinal()
|
||||
cachedArtistsDays[artist] = day
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user