mirror of https://github.com/krateng/maloja.git
Replaced directory changing with fully generated file paths
This commit is contained in:
parent
2029e5d522
commit
dbc23ca73c
|
@ -33,23 +33,4 @@ commands = {
|
|||
"maloja":"controller:main"
|
||||
}
|
||||
|
||||
### DOREAH CONFIGURATION
|
||||
|
||||
from doreah import config
|
||||
config(
|
||||
logging={
|
||||
"logfolder": "logs"
|
||||
},
|
||||
settings={
|
||||
"files":[
|
||||
"settings/default.ini",
|
||||
"settings/settings.ini"
|
||||
]
|
||||
},
|
||||
caching={
|
||||
"folder": "cache/"
|
||||
},
|
||||
regular={
|
||||
"autostart": False
|
||||
}
|
||||
)
|
||||
from . import globalconf
|
||||
|
|
|
@ -2,6 +2,7 @@ import tarfile
|
|||
from datetime import datetime
|
||||
import glob
|
||||
import os
|
||||
from globalconf import datadir
|
||||
|
||||
|
||||
user_files = {
|
||||
|
@ -22,7 +23,7 @@ def backup(folder,level="full"):
|
|||
selected_files = user_files["minimal"] if level == "minimal" else user_files["minimal"] + user_files["full"]
|
||||
real_files = []
|
||||
for g in selected_files:
|
||||
real_files += glob.glob(g)
|
||||
real_files += glob.glob(datadir(g))
|
||||
|
||||
now = datetime.utcnow()
|
||||
timestr = now.strftime("%Y_%m_%d_%H_%M_%S")
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
import re
|
||||
from . import utilities
|
||||
from doreah import tsv, settings
|
||||
from .globalconf import datadir
|
||||
import pkg_resources
|
||||
|
||||
# need to do this as a class so it can retain loaded settings from file
|
||||
# apparently this is not true
|
||||
|
@ -11,7 +13,7 @@ class CleanerAgent:
|
|||
self.updateRules()
|
||||
|
||||
def updateRules(self):
|
||||
raw = tsv.parse_all("rules","string","string","string","string")
|
||||
raw = tsv.parse_all(datadir("rules"),"string","string","string","string")
|
||||
self.rules_belongtogether = [b for [a,b,c,d] in raw if a=="belongtogether"]
|
||||
self.rules_notanartist = [b for [a,b,c,d] in raw if a=="notanartist"]
|
||||
self.rules_replacetitle = {b.lower():c for [a,b,c,d] in raw if a=="replacetitle"}
|
||||
|
@ -160,7 +162,7 @@ class CollectorAgent:
|
|||
# rules_include dict: credited artist -> all real artists
|
||||
|
||||
def updateRules(self):
|
||||
raw = tsv.parse_all("rules","string","string","string")
|
||||
raw = tsv.parse_all(datadir("rules"),"string","string","string")
|
||||
self.rules_countas = {b:c for [a,b,c] in raw if a=="countas"}
|
||||
self.rules_countas_id = {}
|
||||
self.rules_include = {} #Twice the memory, double the performance!
|
||||
|
|
|
@ -12,18 +12,17 @@ import pkg_resources
|
|||
from doreah.control import mainfunction
|
||||
from doreah.io import col
|
||||
|
||||
from .globalconf import DATA_DIR
|
||||
from .globalconf import datadir
|
||||
from .backup import backup
|
||||
|
||||
|
||||
|
||||
origpath = os.getcwd()
|
||||
os.chdir(DATA_DIR)
|
||||
|
||||
def copy_initial_local_files():
|
||||
folder = pkg_resources.resource_filename(__name__,"data_files")
|
||||
#shutil.copy(folder,DATA_DIR)
|
||||
dir_util.copy_tree(folder,DATA_DIR,update=False)
|
||||
dir_util.copy_tree(folder,datadir(),update=False)
|
||||
|
||||
|
||||
def setup():
|
||||
|
@ -49,13 +48,13 @@ def setup():
|
|||
print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.")
|
||||
key = input()
|
||||
if key == "": key = None
|
||||
settings.update_settings("settings/settings.ini",{k:key},create_new=True)
|
||||
settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True)
|
||||
else:
|
||||
print("\t" + apikeys[k] + " found.")
|
||||
|
||||
|
||||
# OWN API KEY
|
||||
if os.path.exists("./clients/authenticated_machines.tsv"):
|
||||
if os.path.exists(datadir("clients/authenticated_machines.tsv")):
|
||||
pass
|
||||
else:
|
||||
print("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database. [Y/n]")
|
||||
|
@ -66,7 +65,7 @@ def setup():
|
|||
for i in range(64):
|
||||
key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")))
|
||||
print("Your API Key: " + col["yellow"](key))
|
||||
with open("./clients/authenticated_machines.tsv","w") as keyfile:
|
||||
with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile:
|
||||
keyfile.write(key + "\t" + "Default Generated Key")
|
||||
elif answer.lower() in ["n","no","nay","0","negative","false"]:
|
||||
pass
|
||||
|
@ -76,16 +75,15 @@ def setup():
|
|||
print("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.")
|
||||
name = input()
|
||||
if name == "": name = "Generic Maloja User"
|
||||
settings.update_settings("settings/settings.ini",{"NAME":name},create_new=True)
|
||||
settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True)
|
||||
|
||||
if settings.get_settings("SEND_STATS") is None:
|
||||
print("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)? [Y/n]")
|
||||
answer = input()
|
||||
if answer.lower() in ["y","yes","yea","1","positive","true",""]:
|
||||
settings.update_settings("settings/settings.ini",{"SEND_STATS":True},create_new=True)
|
||||
settings.update_settings("settings/settings.ini",{"PUBLIC_URL":None},create_new=True)
|
||||
settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True)
|
||||
else:
|
||||
settings.update_settings("settings/settings.ini",{"SEND_STATS":False},create_new=True)
|
||||
settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True)
|
||||
|
||||
|
||||
def getInstance():
|
||||
|
@ -107,7 +105,7 @@ def getInstanceSupervisor():
|
|||
def start():
|
||||
setup()
|
||||
try:
|
||||
with open("logs/stderr.log","w") as logf:
|
||||
with open(datadir("logs/stderr.log"),"w") as logf:
|
||||
p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=logf,cwd=DATA_DIR)
|
||||
sp = subprocess.Popen(["python3","-m","maloja.supervisor"],stdout=subprocess.DEVNULL,stderr=logf,cwd=DATA_DIR)
|
||||
print(col["green"]("Maloja started!") + " PID: " + str(p.pid))
|
||||
|
@ -152,14 +150,14 @@ def loadlastfm(filename):
|
|||
print("Please specify a file!")
|
||||
return
|
||||
|
||||
if os.path.exists("./scrobbles/lastfmimport.tsv"):
|
||||
if os.path.exists(datadir("scrobbles/lastfmimport.tsv")):
|
||||
print("Already imported Last.FM data. Overwrite? [y/N]")
|
||||
if input().lower() in ["y","yes","yea","1","positive","true"]:
|
||||
pass
|
||||
else:
|
||||
return
|
||||
print("Please wait...")
|
||||
os.system("python3 -m maloja.lastfmconverter " + filename + " ./scrobbles/lastfmimport.tsv")
|
||||
os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv"))
|
||||
print("Successfully imported your Last.FM scrobbles!")
|
||||
|
||||
def direct():
|
||||
|
|
|
@ -8,6 +8,7 @@ from .urihandler import uri_to_internal, internal_to_uri, compose_querystring
|
|||
from . import compliant_api
|
||||
from .external import proxy_scrobble
|
||||
from .__init__ import version
|
||||
from .globalconf import datadir
|
||||
# doreah toolkit
|
||||
from doreah.logging import log
|
||||
from doreah import tsv
|
||||
|
@ -68,7 +69,7 @@ lastsync = 0
|
|||
db_rulestate = False
|
||||
|
||||
try:
|
||||
with open("known_servers.yml","r") as f:
|
||||
with open(datadir("known_servers.yml"),"r") as f:
|
||||
KNOWN_SERVERS = set(yaml.safe_load(f))
|
||||
except:
|
||||
KNOWN_SERVERS = set()
|
||||
|
@ -76,7 +77,7 @@ except:
|
|||
|
||||
def add_known_server(url):
|
||||
KNOWN_SERVERS.add(url)
|
||||
with open("known_servers.yml","w") as f:
|
||||
with open(datadir("known_servers.yml"),"w") as f:
|
||||
f.write(yaml.dump(list(KNOWN_SERVERS)))
|
||||
|
||||
|
||||
|
@ -84,9 +85,9 @@ def add_known_server(url):
|
|||
### symmetric keys are fine for now since we hopefully use HTTPS
|
||||
def loadAPIkeys():
|
||||
global clients
|
||||
tsv.create("clients/authenticated_machines.tsv")
|
||||
tsv.create(datadir("clients/authenticated_machines.tsv"))
|
||||
#createTSV("clients/authenticated_machines.tsv")
|
||||
clients = tsv.parse("clients/authenticated_machines.tsv","string","string")
|
||||
clients = tsv.parse(datadir("clients/authenticated_machines.tsv"),"string","string")
|
||||
#clients = parseTSV("clients/authenticated_machines.tsv","string","string")
|
||||
log("Authenticated Machines: " + ", ".join([m[1] for m in clients]))
|
||||
|
||||
|
@ -847,10 +848,10 @@ def import_rulemodule(**keys):
|
|||
|
||||
if remove:
|
||||
log("Deactivating predefined rulefile " + filename)
|
||||
os.remove("rules/" + filename + ".tsv")
|
||||
os.remove(datadir("rules/" + filename + ".tsv"))
|
||||
else:
|
||||
log("Importing predefined rulefile " + filename)
|
||||
os.symlink("predefined/" + filename + ".tsv","rules/" + filename + ".tsv")
|
||||
os.symlink(datadir("predefined/" + filename + ".tsv"),datadir("rules/" + filename + ".tsv"))
|
||||
|
||||
|
||||
|
||||
|
@ -951,7 +952,7 @@ def build_db():
|
|||
|
||||
|
||||
# parse files
|
||||
db = tsv.parse_all("scrobbles","int","string","string",comments=False)
|
||||
db = tsv.parse_all(datadir("scrobbles"),"int","string","string",comments=False)
|
||||
#db = parseAllTSV("scrobbles","int","string","string",escape=False)
|
||||
for sc in db:
|
||||
artists = sc[1].split("␟")
|
||||
|
@ -985,7 +986,7 @@ def build_db():
|
|||
utilities.send_stats()
|
||||
|
||||
global db_rulestate
|
||||
db_rulestate = utilities.consistentRulestate("scrobbles",cla.checksums)
|
||||
db_rulestate = utilities.consistentRulestate(datadir("scrobbles"),cla.checksums)
|
||||
|
||||
log("Database fully built!")
|
||||
|
||||
|
@ -1019,9 +1020,9 @@ def sync():
|
|||
#log("Sorted into months",module="debug")
|
||||
|
||||
for e in entries:
|
||||
tsv.add_entries("scrobbles/" + e + ".tsv",entries[e],comments=False)
|
||||
tsv.add_entries(datadir("scrobbles/" + e + ".tsv"),entries[e],comments=False)
|
||||
#addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False)
|
||||
utilities.combineChecksums("scrobbles/" + e + ".tsv",cla.checksums)
|
||||
utilities.combineChecksums(datadir("scrobbles/" + e + ".tsv"),cla.checksums)
|
||||
|
||||
#log("Written files",module="debug")
|
||||
|
||||
|
@ -1046,7 +1047,7 @@ import copy
|
|||
|
||||
cache_query = {}
|
||||
if doreah.version >= (0,7,1) and settings.get_settings("EXPERIMENTAL_FEATURES"):
|
||||
cache_query_permanent = DiskDict(name="dbquery",folder="cache",maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE"))
|
||||
cache_query_permanent = DiskDict(name="dbquery",folder=datadir("cache"),maxmemory=1024*1024*500,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE"))
|
||||
else:
|
||||
cache_query_permanent = Cache(maxmemory=1024*1024*500)
|
||||
cacheday = (0,0,0)
|
||||
|
|
|
@ -1,7 +1,5 @@
|
|||
import os
|
||||
from .globalconf import DATA_DIR
|
||||
os.chdir(DATA_DIR)
|
||||
|
||||
from .globalconf import datadir
|
||||
import re
|
||||
from .cleanup import CleanerAgent
|
||||
from doreah.logging import log
|
||||
|
@ -13,29 +11,28 @@ wendigo = CleanerAgent()
|
|||
|
||||
exp = r"([0-9]*)(\t+)([^\t]+?)(\t+)([^\t]+)(\t*)([^\t]*)\n"
|
||||
|
||||
pthj = os.path.join
|
||||
|
||||
|
||||
def fix():
|
||||
|
||||
backup(level="minimal",folder=pthj(DATA_DIR,"backups"))
|
||||
backup(level="minimal",folder=datadir("backups"))
|
||||
|
||||
now = datetime.datetime.utcnow()
|
||||
nowstr = now.strftime("%Y_%m_%d_%H_%M_%S")
|
||||
datestr = now.strftime("%Y/%m/%d")
|
||||
timestr = now.strftime("%H:%M:%S")
|
||||
|
||||
with open(pthj(DATA_DIR,"logs","dbfix",nowstr + ".log"),"a") as logfile:
|
||||
with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile:
|
||||
|
||||
logfile.write("Database fix initiated on " + datestr + " " + timestr + " UTC")
|
||||
logfile.write("\n\n")
|
||||
|
||||
for filename in os.listdir(pthj(DATA_DIR,"scrobbles")):
|
||||
for filename in os.listdir(datadir("scrobbles")):
|
||||
if filename.endswith(".tsv"):
|
||||
filename_new = filename + "_new"
|
||||
|
||||
with open(pthj(DATA_DIR,"scrobbles",filename_new),"w") as newfile:
|
||||
with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile:
|
||||
with open(datadir("scrobbles",filename_new),"w") as newfile:
|
||||
with open(datadir("scrobbles",filename),"r") as oldfile:
|
||||
|
||||
for l in oldfile:
|
||||
|
||||
|
@ -50,8 +47,8 @@ def fix():
|
|||
|
||||
|
||||
#os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn)
|
||||
with open(pthj(DATA_DIR,"scrobbles",filename_new),"r") as newfile:
|
||||
with open(pthj(DATA_DIR,"scrobbles",filename),"r") as oldfile:
|
||||
with open(datadir("scrobbles",filename_new),"r") as newfile:
|
||||
with open(datadir("scrobbles",filename),"r") as oldfile:
|
||||
|
||||
diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="")
|
||||
diff = list(diff)[2:]
|
||||
|
@ -61,7 +58,7 @@ def fix():
|
|||
logfile.write(output)
|
||||
logfile.write("\n")
|
||||
|
||||
os.rename(pthj(DATA_DIR,"scrobbles",filename_new),pthj(DATA_DIR,"scrobbles",filename))
|
||||
os.rename(datadir("scrobbles",filename_new),datadir("scrobbles",filename))
|
||||
|
||||
with open(pthj(DATA_DIR,"scrobbles",filename + ".rulestate"),"w") as checkfile:
|
||||
with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile:
|
||||
checkfile.write(wendigo.checksums)
|
||||
|
|
|
@ -1,6 +1,51 @@
|
|||
from doreah.settings import get_settings
|
||||
import os
|
||||
|
||||
|
||||
# data folder
|
||||
# must be determined first because getting settings relies on it
|
||||
|
||||
try:
|
||||
DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0]
|
||||
assert os.path.exists(DATA_DIR)
|
||||
except:
|
||||
DATA_DIR = os.path.join(os.environ["HOME"],".local/share/")
|
||||
|
||||
DATA_DIR = os.path.join(DATA_DIR,"maloja")
|
||||
os.makedirs(DATA_DIR,exist_ok=True)
|
||||
|
||||
def datadir(*args):
|
||||
return os.path.join(DATA_DIR,*args)
|
||||
|
||||
|
||||
|
||||
### DOREAH CONFIGURATION
|
||||
|
||||
from doreah import config
|
||||
|
||||
config(
|
||||
logging={
|
||||
"logfolder": datadir("logs")
|
||||
},
|
||||
settings={
|
||||
"files":[
|
||||
datadir("settings/default.ini"),
|
||||
datadir("settings/settings.ini")
|
||||
]
|
||||
},
|
||||
caching={
|
||||
"folder": datadir("cache")
|
||||
},
|
||||
regular={
|
||||
"autostart": False
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
|
||||
from doreah.settings import get_settings
|
||||
|
||||
# thumbor
|
||||
|
||||
THUMBOR_SERVER, THUMBOR_SECRET = get_settings("THUMBOR_SERVER","THUMBOR_SECRET")
|
||||
try:
|
||||
USE_THUMBOR = THUMBOR_SERVER is not None and THUMBOR_SECRET is not None
|
||||
|
@ -12,15 +57,3 @@ try:
|
|||
except:
|
||||
USE_THUMBOR = False
|
||||
log("Thumbor could not be initialized. Is libthumbor installed?")
|
||||
|
||||
|
||||
|
||||
|
||||
try:
|
||||
DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0]
|
||||
assert os.path.exists(DATA_DIR)
|
||||
except:
|
||||
DATA_DIR = os.path.join(os.environ["HOME"],".local/share/")
|
||||
|
||||
DATA_DIR = os.path.join(DATA_DIR,"maloja")
|
||||
os.makedirs(DATA_DIR,exist_ok=True)
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
#!/usr/bin/env python
|
||||
import os
|
||||
from .globalconf import DATA_DIR
|
||||
os.chdir(DATA_DIR)
|
||||
from .globalconf import datadir, DATA_DIR
|
||||
|
||||
|
||||
# server stuff
|
||||
|
@ -50,6 +49,7 @@ BaseRequest.MEMFILE_MAX = 15 * 1024 * 1024
|
|||
|
||||
WEBFOLDER = pkg_resources.resource_filename(__name__,"web")
|
||||
STATICFOLDER = pkg_resources.resource_filename(__name__,"static")
|
||||
DATAFOLDER = DATADIR
|
||||
|
||||
webserver = Bottle()
|
||||
|
||||
|
@ -122,27 +122,27 @@ def dynamic_image():
|
|||
@webserver.route("/images/<pth:re:.*\\.gif>")
|
||||
def static_image(pth):
|
||||
if globalconf.USE_THUMBOR:
|
||||
return static_file("images/" + pth,root="")
|
||||
return static_file(pthjoin("images",pth),root=DATAFOLDER)
|
||||
|
||||
type = pth.split(".")[-1]
|
||||
small_pth = pth + "-small"
|
||||
if os.path.exists("images/" + small_pth):
|
||||
response = static_file("images/" + small_pth,root="")
|
||||
if os.path.exists(datadir("images",small_pth)):
|
||||
response = static_file(pthjoin("images",small_pth),root=DATAFOLDER)
|
||||
else:
|
||||
try:
|
||||
from wand.image import Image
|
||||
img = Image(filename="images/" + pth)
|
||||
img = Image(filename=datadir("images",pth))
|
||||
x,y = img.size[0], img.size[1]
|
||||
smaller = min(x,y)
|
||||
if smaller > 300:
|
||||
ratio = 300/smaller
|
||||
img.resize(int(ratio*x),int(ratio*y))
|
||||
img.save(filename="images/" + small_pth)
|
||||
response = static_file("images/" + small_pth,root="")
|
||||
img.save(filename=datadir("images",small_pth))
|
||||
response = static_file(pthjoin("images",small_pth),root=DATAFOLDER)
|
||||
else:
|
||||
response = static_file("images/" + pth,root="")
|
||||
response = static_file(pthjoin("images",pth),root=DATAFOLDER)
|
||||
except:
|
||||
response = static_file("images/" + pth,root="")
|
||||
response = static_file(pthjoin("images",pth),root=DATAFOLDER)
|
||||
|
||||
#response = static_file("images/" + pth,root="")
|
||||
response.set_header("Cache-Control", "public, max-age=86400")
|
||||
|
|
|
@ -1,7 +1,5 @@
|
|||
#!/usr/bin/env python3
|
||||
import os
|
||||
from .globalconf import DATA_DIR
|
||||
os.chdir(DATA_DIR)
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
|
|
|
@ -17,6 +17,7 @@ from doreah.regular import yearly, daily
|
|||
from .external import api_request_track, api_request_artist
|
||||
from .__init__ import version
|
||||
from . import globalconf
|
||||
from .globalconf import datadir
|
||||
|
||||
|
||||
|
||||
|
@ -170,7 +171,7 @@ def get_all_possible_filenames(artist=None,artists=None,title=None):
|
|||
else: return []
|
||||
|
||||
|
||||
superfolder = "images/tracks/" if track else "images/artists/"
|
||||
superfolder = datadir("images/tracks/") if track else datadir("images/artists/")
|
||||
|
||||
filenames = []
|
||||
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import os
|
||||
from ..globalconf import datadir
|
||||
|
||||
def instructions(keys):
|
||||
|
||||
|
@ -8,7 +9,7 @@ def instructions(keys):
|
|||
|
||||
|
||||
validchars = "-_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
for f in os.listdir("rules/predefined"):
|
||||
for f in os.listdir(datadir("rules/predefined")):
|
||||
if f.endswith(".tsv"):
|
||||
|
||||
rawf = f.replace(".tsv","")
|
||||
|
@ -22,7 +23,7 @@ def instructions(keys):
|
|||
if not "_" in rawf: continue
|
||||
|
||||
try:
|
||||
with open("rules/predefined/" + f) as tsvfile:
|
||||
with open(datadir("rules/predefined",f)) as tsvfile:
|
||||
line1 = tsvfile.readline()
|
||||
line2 = tsvfile.readline()
|
||||
|
||||
|
@ -39,7 +40,7 @@ def instructions(keys):
|
|||
|
||||
html += "<tr>"
|
||||
|
||||
if os.path.exists("rules/" + f):
|
||||
if os.path.exists(datadir("rules",f)):
|
||||
html += "<td class='interaction' onclick=deactivateRuleModule(this,'" + rawf + "')><a class='textlink'>Remove:</a></td>"
|
||||
else:
|
||||
html += "<td class='interaction' onclick=activateRuleModule(this,'" + rawf + "')><a class='textlink'>Add:</a></td>"
|
||||
|
|
Loading…
Reference in New Issue