Replaced directory changing with fully generated file paths

This commit is contained in:
Krateng 2019-12-15 15:18:33 +01:00
parent 2029e5d522
commit dbc23ca73c
11 changed files with 102 additions and 89 deletions

View File

@ -33,23 +33,4 @@ commands = {
"maloja":"controller:main"
}
### DOREAH CONFIGURATION
from doreah import config
config(
logging={
"logfolder": "logs"
},
settings={
"files":[
"settings/default.ini",
"settings/settings.ini"
]
},
caching={
"folder": "cache/"
},
regular={
"autostart": False
}
)
from . import globalconf

View File

@ -2,6 +2,7 @@ import tarfile
from datetime import datetime
import glob
import os
from globalconf import datadir
user_files = {
@ -22,7 +23,7 @@ def backup(folder,level="full"):
selected_files = user_files["minimal"] if level == "minimal" else user_files["minimal"] + user_files["full"]
real_files = []
for g in selected_files:
real_files += glob.glob(g)
real_files += glob.glob(datadir(g))
now = datetime.utcnow()
timestr = now.strftime("%Y_%m_%d_%H_%M_%S")

View File

@ -1,6 +1,8 @@
import re
from . import utilities
from doreah import tsv, settings
from .globalconf import datadir
import pkg_resources
# need to do this as a class so it can retain loaded settings from file
# apparently this is not true
@ -11,7 +13,7 @@ class CleanerAgent:
self.updateRules()
def updateRules(self):
raw = tsv.parse_all("rules","string","string","string","string")
raw = tsv.parse_all(datadir("rules"),"string","string","string","string")
self.rules_belongtogether = [b for [a,b,c,d] in raw if a=="belongtogether"]
self.rules_notanartist = [b for [a,b,c,d] in raw if a=="notanartist"]
self.rules_replacetitle = {b.lower():c for [a,b,c,d] in raw if a=="replacetitle"}
@ -160,7 +162,7 @@ class CollectorAgent:
# rules_include dict: credited artist -> all real artists
def updateRules(self):
raw = tsv.parse_all("rules","string","string","string")
raw = tsv.parse_all(datadir("rules"),"string","string","string")
self.rules_countas = {b:c for [a,b,c] in raw if a=="countas"}
self.rules_countas_id = {}
self.rules_include = {} #Twice the memory, double the performance!

View File

@ -12,18 +12,17 @@ import pkg_resources
from doreah.control import mainfunction
from doreah.io import col
from .globalconf import DATA_DIR
from .globalconf import datadir
from .backup import backup
origpath = os.getcwd()
os.chdir(DATA_DIR)
def copy_initial_local_files():
folder = pkg_resources.resource_filename(__name__,"data_files")
#shutil.copy(folder,DATA_DIR)
dir_util.copy_tree(folder,DATA_DIR,update=False)
dir_util.copy_tree(folder,datadir(),update=False)
def setup():
@ -49,13 +48,13 @@ def setup():
print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.")
key = input()
if key == "": key = None
settings.update_settings("settings/settings.ini",{k:key},create_new=True)
settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True)
else:
print("\t" + apikeys[k] + " found.")
# OWN API KEY
if os.path.exists("./clients/authenticated_machines.tsv"):
if os.path.exists(datadir("clients/authenticated_machines.tsv")):
pass
else:
print("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database. [Y/n]")
@ -66,7 +65,7 @@ def setup():
for i in range(64):
key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")))
print("Your API Key: " + col["yellow"](key))
with open("./clients/authenticated_machines.tsv","w") as keyfile:
with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile:
keyfile.write(key + "\t" + "Default Generated Key")
elif answer.lower() in ["n","no","nay","0","negative","false"]:
pass
@ -76,16 +75,15 @@ def setup():
print("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.")
name = input()
if name == "": name = "Generic Maloja User"
settings.update_settings("settings/settings.ini",{"NAME":name},create_new=True)
settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True)
if settings.get_settings("SEND_STATS") is None:
print("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)? [Y/n]")
answer = input()
if answer.lower() in ["y","yes","yea","1","positive","true",""]:
settings.update_settings("settings/settings.ini",{"SEND_STATS":True},create_new=True)
settings.update_settings("settings/settings.ini",{"PUBLIC_URL":None},create_new=True)
settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True)
else:
settings.update_settings("settings/settings.ini",{"SEND_STATS":False},create_new=True)
settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True)
def getInstance():
@ -107,7 +105,7 @@ def getInstanceSupervisor():
def start():
setup()
try:
with open("logs/stderr.log","w") as logf:
with open(datadir("logs/stderr.log"),"w") as logf:
p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=logf,cwd=DATA_DIR)
sp = subprocess.Popen(["python3","-m","maloja.supervisor"],stdout=subprocess.DEVNULL,stderr=logf,cwd=DATA_DIR)
print(col["green"]("Maloja started!") + " PID: " + str(p.pid))
@ -152,14 +150,14 @@ def loadlastfm(filename):
print("Please specify a file!")
return
if os.path.exists("./scrobbles/lastfmimport.tsv"):
if os.path.exists(datadir("scrobbles/lastfmimport.tsv")):
print("Already imported Last.FM data. Overwrite? [y/N]")
if input().lower() in ["y","yes","yea","1","positive","true"]:
pass
else:
return
print("Please wait...")
os.system("python3 -m maloja.lastfmconverter " + filename + " ./scrobbles/lastfmimport.tsv")
os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv"))
print("Successfully imported your Last.FM scrobbles!")
def direct():

View File

@ -8,6 +8,7 @@ from .urihandler import uri_to_internal, internal_to_uri, compose_querystring
from . import compliant_api
from .external import proxy_scrobble
from .__init__ import version
from .globalconf import datadir
# doreah toolkit
from doreah.logging import log
from doreah import tsv
@ -68,7 +69,7 @@ lastsync = 0
db_rulestate = False
try:
with open("known_servers.yml","r") as f:
with open(datadir("known_servers.yml"),"r") as f:
KNOWN_SERVERS = set(yaml.safe_load(f))
except:
KNOWN_SERVERS = set()
@ -76,7 +77,7 @@ except:
def add_known_server(url):
KNOWN_SERVERS.add(url)
with open("known_servers.yml","w") as f:
with open(datadir("known_servers.yml"),"w") as f:
f.write(yaml.dump(list(KNOWN_SERVERS)))
@ -84,9 +85,9 @@ def add_known_server(url):
### symmetric keys are fine for now since we hopefully use HTTPS
def loadAPIkeys():
global clients
tsv.create("clients/authenticated_machines.tsv")
tsv.create(datadir("clients/authenticated_machines.tsv"))
#createTSV("clients/authenticated_machines.tsv")
clients = tsv.parse("clients/authenticated_machines.tsv","string","string")
clients = tsv.parse(datadir("clients/authenticated_machines.tsv"),"string","string")
#clients = parseTSV("clients/authenticated_machines.tsv","string","string")
log("Authenticated Machines: " + ", ".join([m[1] for m in clients]))
@ -847,10 +848,10 @@ def import_rulemodule(**keys):
if remove:
log("Deactivating predefined rulefile " + filename)
os.remove("rules/" + filename + ".tsv")
os.remove(datadir("rules/" + filename + ".tsv"))
else:
log("Importing predefined rulefile " + filename)
os.symlink("predefined/" + filename + ".tsv","rules/" + filename + ".tsv")
os.symlink(datadir("predefined/" + filename + ".tsv"),datadir("rules/" + filename + ".tsv"))
@ -951,7 +952,7 @@ def build_db():
# parse files
db = tsv.parse_all("scrobbles","int","string","string",comments=False)
db = tsv.parse_all(datadir("scrobbles"),"int","string","string",comments=False)
#db = parseAllTSV("scrobbles","int","string","string",escape=False)
for sc in db:
artists = sc[1].split("")
@ -985,7 +986,7 @@ def build_db():
utilities.send_stats()
global db_rulestate
db_rulestate = utilities.consistentRulestate("scrobbles",cla.checksums)
db_rulestate = utilities.consistentRulestate(datadir("scrobbles"),cla.checksums)
log("Database fully built!")
@ -1019,9 +1020,9 @@ def sync():
#log("Sorted into months",module="debug")
for e in entries:
tsv.add_entries("scrobbles/" + e + ".tsv",entries[e],comments=False)
tsv.add_entries(datadir("scrobbles/" + e + ".tsv"),entries[e],comments=False)
#addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False)
utilities.combineChecksums("scrobbles/" + e + ".tsv",cla.checksums)
utilities.combineChecksums(datadir("scrobbles/" + e + ".tsv"),cla.checksums)
#log("Written files",module="debug")
@ -1046,7 +1047,7 @@ import copy
cache_query = {}
if doreah.version >= (0,7,1) and settings.get_settings("EXPERIMENTAL_FEATURES"):
cache_query_permanent = DiskDict(name="dbquery",folder="cache",maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE"))
cache_query_permanent = DiskDict(name="dbquery",folder=datadir("cache"),maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE"))
else:
cache_query_permanent = Cache(maxmemory=1024*1024*500)
cacheday = (0,0,0)

View File

@ -1,7 +1,5 @@
import os
from .globalconf import DATA_DIR
os.chdir(DATA_DIR)
from .globalconf import datadir
import re
from .cleanup import CleanerAgent
from doreah.logging import log
@ -13,29 +11,28 @@ wendigo = CleanerAgent()
exp = r"([0-9]*)(\t+)([^\t]+?)(\t+)([^\t]+)(\t*)([^\t]*)\n"
pthj = os.path.join
def fix():
backup(level="minimal",folder=pthj(DATA_DIR,"backups"))
backup(level="minimal",folder=datadir("backups"))
now = datetime.datetime.utcnow()
nowstr = now.strftime("%Y_%m_%d_%H_%M_%S")
datestr = now.strftime("%Y/%m/%d")
timestr = now.strftime("%H:%M:%S")
with open(pthj(DATA_DIR,"logs","dbfix",nowstr + ".log"),"a") as logfile:
with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile:
logfile.write("Database fix initiated on " + datestr + " " + timestr + " UTC")
logfile.write("\n\n")
for filename in os.listdir(pthj(DATA_DIR,"scrobbles")):
for filename in os.listdir(datadir("scrobbles")):
if filename.endswith(".tsv"):
filename_new = filename + "_new"
with open(pthj(DATA_DIR,"scrobbles",filename_new),"w") as newfile:
with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile:
with open(datadir("scrobbles",filename_new),"w") as newfile:
with open(datadir("scrobbles",filename),"r") as oldfile:
for l in oldfile:
@ -50,8 +47,8 @@ def fix():
#os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn)
with open(pthj(DATA_DIR,"scrobbles",filename_new),"r") as newfile:
with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile:
with open(datadir("scrobbles",filename_new),"r") as newfile:
with open(datadir("scrobbles",filename),"r") as oldfile:
diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="")
diff = list(diff)[2:]
@ -61,7 +58,7 @@ def fix():
logfile.write(output)
logfile.write("\n")
os.rename(pthj(DATA_DIR,"scrobbles",filename_new),pthj(DATA_DIR,"scrobbles",filename))
os.rename(datadir("scrobbles",filename_new),datadir("scrobbles",filename))
with open(pthj(DATA_DIR,"scrobbles",filename + ".rulestate"),"w") as checkfile:
with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile:
checkfile.write(wendigo.checksums)

View File

@ -1,6 +1,51 @@
from doreah.settings import get_settings
import os
# data folder
# must be determined first because getting settings relies on it
try:
DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0]
assert os.path.exists(DATA_DIR)
except:
DATA_DIR = os.path.join(os.environ["HOME"],".local/share/")
DATA_DIR = os.path.join(DATA_DIR,"maloja")
os.makedirs(DATA_DIR,exist_ok=True)
def datadir(*args):
return os.path.join(DATA_DIR,*args)
### DOREAH CONFIGURATION
from doreah import config
config(
logging={
"logfolder": datadir("logs")
},
settings={
"files":[
datadir("settings/default.ini"),
datadir("settings/settings.ini")
]
},
caching={
"folder": datadir("cache")
},
regular={
"autostart": False
}
)
from doreah.settings import get_settings
# thumbor
THUMBOR_SERVER, THUMBOR_SECRET = get_settings("THUMBOR_SERVER","THUMBOR_SECRET")
try:
USE_THUMBOR = THUMBOR_SERVER is not None and THUMBOR_SECRET is not None
@ -12,15 +57,3 @@ try:
except:
USE_THUMBOR = False
log("Thumbor could not be initialized. Is libthumbor installed?")
try:
DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0]
assert os.path.exists(DATA_DIR)
except:
DATA_DIR = os.path.join(os.environ["HOME"],".local/share/")
DATA_DIR = os.path.join(DATA_DIR,"maloja")
os.makedirs(DATA_DIR,exist_ok=True)

View File

@ -1,7 +1,6 @@
#!/usr/bin/env python
import os
from .globalconf import DATA_DIR
os.chdir(DATA_DIR)
from .globalconf import datadir, DATA_DIR
# server stuff
@ -50,6 +49,7 @@ BaseRequest.MEMFILE_MAX = 15 * 1024 * 1024
WEBFOLDER = pkg_resources.resource_filename(__name__,"web")
STATICFOLDER = pkg_resources.resource_filename(__name__,"static")
DATAFOLDER = DATADIR
webserver = Bottle()
@ -122,27 +122,27 @@ def dynamic_image():
@webserver.route("/images/<pth:re:.*\\.gif>")
def static_image(pth):
if globalconf.USE_THUMBOR:
return static_file("images/" + pth,root="")
return static_file(pthjoin("images",pth),root=DATAFOLDER)
type = pth.split(".")[-1]
small_pth = pth + "-small"
if os.path.exists("images/" + small_pth):
response = static_file("images/" + small_pth,root="")
if os.path.exists(datadir("images",small_pth)):
response = static_file(pthjoin("images",small_pth),root=DATAFOLDER)
else:
try:
from wand.image import Image
img = Image(filename="images/" + pth)
img = Image(filename=datadir("images",pth))
x,y = img.size[0], img.size[1]
smaller = min(x,y)
if smaller > 300:
ratio = 300/smaller
img.resize(int(ratio*x),int(ratio*y))
img.save(filename="images/" + small_pth)
response = static_file("images/" + small_pth,root="")
img.save(filename=datadir("images",small_pth))
response = static_file(pthjoin("images",small_pth),root=DATAFOLDER)
else:
response = static_file("images/" + pth,root="")
response = static_file(pthjoin("images",pth),root=DATAFOLDER)
except:
response = static_file("images/" + pth,root="")
response = static_file(pthjoin("images",pth),root=DATAFOLDER)
#response = static_file("images/" + pth,root="")
response.set_header("Cache-Control", "public, max-age=86400")

View File

@ -1,7 +1,5 @@
#!/usr/bin/env python3
import os
from .globalconf import DATA_DIR
os.chdir(DATA_DIR)
import subprocess
import time

View File

@ -17,6 +17,7 @@ from doreah.regular import yearly, daily
from .external import api_request_track, api_request_artist
from .__init__ import version
from . import globalconf
from .globalconf import datadir
@ -170,7 +171,7 @@ def get_all_possible_filenames(artist=None,artists=None,title=None):
else: return []
superfolder = "images/tracks/" if track else "images/artists/"
superfolder = datadir("images/tracks/") if track else datadir("images/artists/")
filenames = []

View File

@ -1,4 +1,5 @@
import os
from ..globalconf import datadir
def instructions(keys):
@ -8,7 +9,7 @@ def instructions(keys):
validchars = "-_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
for f in os.listdir("rules/predefined"):
for f in os.listdir(datadir("rules/predefined")):
if f.endswith(".tsv"):
rawf = f.replace(".tsv","")
@ -22,7 +23,7 @@ def instructions(keys):
if not "_" in rawf: continue
try:
with open("rules/predefined/" + f) as tsvfile:
with open(datadir("rules/predefined",f)) as tsvfile:
line1 = tsvfile.readline()
line2 = tsvfile.readline()
@ -39,7 +40,7 @@ def instructions(keys):
html += "<tr>"
if os.path.exists("rules/" + f):
if os.path.exists(datadir("rules",f)):
html += "<td class='interaction' onclick=deactivateRuleModule(this,'" + rawf + "')><a class='textlink'>Remove:</a></td>"
else:
html += "<td class='interaction' onclick=activateRuleModule(this,'" + rawf + "')><a class='textlink'>Add:</a></td>"