mirror of https://github.com/krateng/maloja.git
Reorganized tasks
This commit is contained in:
krateng
parent
3f098b6993
commit
e31c0dce57
|
|
@ -105,21 +105,22 @@ def print_info():
|
|||
print("#####")
|
||||
print()
|
||||
|
||||
@mainfunction({"l":"level","v":"version","V":"version"},flags=['version'],shield=True)
|
||||
@mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images'],shield=True)
|
||||
def main(*args,**kwargs):
|
||||
|
||||
actions = {
|
||||
# server
|
||||
"start":start,
|
||||
"restart":restart,
|
||||
"stop":stop,
|
||||
"run":direct,
|
||||
"debug":debug,
|
||||
"setup":onlysetup,
|
||||
"import":tasks.loadexternal,
|
||||
"backup":tasks.backuphere,
|
||||
# "update":update,
|
||||
"fix":tasks.fixdb,
|
||||
"generate":tasks.generate_scrobbles,
|
||||
# admin scripts
|
||||
"import":tasks.import_scrobbles,
|
||||
"backup":tasks.backup, # maloja backup --folder x/y --include_images
|
||||
"generate":tasks.generate,
|
||||
# aux
|
||||
"info":print_info
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -35,8 +35,6 @@ def start():
|
|||
|
||||
while True:
|
||||
log("Maloja is not running, starting...",module="supervisor")
|
||||
if malojaconfig["UPDATE_AFTER_CRASH"]:
|
||||
update()
|
||||
process = start()
|
||||
|
||||
process.wait()
|
||||
|
|
|
|||
|
|
@ -1,39 +1,3 @@
|
|||
import os
|
||||
from doreah.io import ask,col
|
||||
|
||||
from ...globalconf import data_dir
|
||||
|
||||
|
||||
## TODO: import to db
|
||||
def loadexternal(filename):
|
||||
|
||||
if not os.path.exists(filename):
|
||||
print("File could not be found.")
|
||||
return
|
||||
|
||||
print("Please wait...")
|
||||
|
||||
from .importer import import_scrobbles
|
||||
imported,failed = import_scrobbles(filename)
|
||||
print("Successfully imported",imported,"scrobbles!")
|
||||
if failed > 0:
|
||||
print(col['red'](str(failed) + " Errors!"))
|
||||
|
||||
def backuphere():
|
||||
from .backup import backup
|
||||
backup(folder=os.getcwd())
|
||||
|
||||
def update():
|
||||
os.system("pip3 install malojaserver --upgrade --no-cache-dir")
|
||||
from ..control import restart
|
||||
restart()
|
||||
|
||||
def fixdb():
|
||||
from .fixexisting import fix
|
||||
fix()
|
||||
|
||||
def generate_scrobbles():
|
||||
targetfile = data_dir['scrobbles']("randomgenerated.tsv")
|
||||
|
||||
from .generate import generate
|
||||
generate(targetfile)
|
||||
from .import_scrobbles import import_scrobbles
|
||||
from .backup import backup
|
||||
from .generate import generate
|
||||
|
|
|
|||
|
|
@ -2,46 +2,50 @@ import tarfile
|
|||
from datetime import datetime
|
||||
import glob
|
||||
import os
|
||||
from ...globalconf import data_dir
|
||||
from ...globalconf import dir_settings
|
||||
from pathlib import PurePath
|
||||
|
||||
from doreah.logging import log
|
||||
from doreah.io import col
|
||||
|
||||
|
||||
user_files = {
|
||||
"minimal":{
|
||||
"rules":["*.tsv"],
|
||||
"scrobbles":["malojadb.sqlite"]
|
||||
},
|
||||
"full":{
|
||||
"clients":["apikeys.yml"],
|
||||
"images":["artists","tracks"],
|
||||
"settings":["settings.ini"]
|
||||
}
|
||||
}
|
||||
basic_files = [
|
||||
('config', ['rules/*.tsv','settings.ini','apikeys.yml','custom_css/*.css']),
|
||||
('state', ['auth/auth.ddb','malojadb.sqlite'])
|
||||
]
|
||||
expanded_files = [
|
||||
('state', ['images'])
|
||||
]
|
||||
|
||||
def backup(folder,level="full"):
|
||||
def backup(targetfolder=None,include_images=False):
|
||||
|
||||
selected_files = user_files["minimal"] if level == "minimal" else {**user_files["minimal"], **user_files["full"]}
|
||||
real_files = {cat:[] for cat in selected_files}
|
||||
for cat in selected_files:
|
||||
catfolder = data_dir[cat]
|
||||
for g in selected_files[cat]:
|
||||
real_files[cat] += glob.glob(catfolder(g))
|
||||
if targetfolder is None:
|
||||
targetfolder = os.getcwd()
|
||||
|
||||
if include_images:
|
||||
file_patterns = basic_files + expanded_files
|
||||
else:
|
||||
file_patterns = basic_files
|
||||
|
||||
real_files = {}
|
||||
for category,patterns in file_patterns:
|
||||
real_files.setdefault(category,[])
|
||||
for pattern in patterns:
|
||||
real_files[category] += glob.glob(os.path.join(dir_settings[category],pattern))
|
||||
|
||||
log("Creating backup...")
|
||||
|
||||
|
||||
now = datetime.utcnow()
|
||||
timestr = now.strftime("%Y_%m_%d_%H_%M_%S")
|
||||
filename = "maloja_backup_" + timestr + ".tar.gz"
|
||||
archivefile = os.path.join(folder,filename)
|
||||
filename = f"maloja_backup_{timestr}.tar.gz"
|
||||
archivefile = os.path.join(targetfolder,filename)
|
||||
assert not os.path.exists(archivefile)
|
||||
with tarfile.open(name=archivefile,mode="x:gz") as archive:
|
||||
for cat, value in real_files.items():
|
||||
for f in value:
|
||||
for category, filelist in real_files.items():
|
||||
for f in filelist:
|
||||
p = PurePath(f)
|
||||
r = p.relative_to(data_dir[cat]())
|
||||
archive.add(f,arcname=os.path.join(cat,r))
|
||||
log("Backup created!")
|
||||
r = p.relative_to(dir_settings[category])
|
||||
archive.add(f,arcname=os.path.join(category,r))
|
||||
log("Backup created: " + col['yellow'](archivefile))
|
||||
return archivefile
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
import os
|
||||
from ...globalconf import data_dir
|
||||
import re
|
||||
from ...cleanup import CleanerAgent
|
||||
from doreah.logging import log
|
||||
import difflib
|
||||
import datetime
|
||||
from .backup import backup
|
||||
|
||||
wendigo = CleanerAgent()
|
||||
|
||||
exp = r"([0-9]*)(\t+)([^\t]+?)(\t+)([^\t]+)([^\n]*)\n"
|
||||
# 1 2 3 4 5 6
|
||||
# groups:
|
||||
# 1 - timestamp
|
||||
# 2 - sep
|
||||
# 3 - artists
|
||||
# 4 - sep
|
||||
# 5 - title
|
||||
# 6 - rest
|
||||
|
||||
|
||||
|
||||
def fix():
|
||||
|
||||
backup(level="minimal",folder=data_dir['backups']())
|
||||
|
||||
now = datetime.datetime.utcnow()
|
||||
nowstr = now.strftime("%Y_%m_%d_%H_%M_%S")
|
||||
datestr = now.strftime("%Y/%m/%d")
|
||||
timestr = now.strftime("%H:%M:%S")
|
||||
|
||||
patchfolder = data_dir['logs']("dbfix",nowstr)
|
||||
os.makedirs(patchfolder)
|
||||
|
||||
|
||||
log("Fixing database...")
|
||||
for filename in os.listdir(data_dir['scrobbles']()):
|
||||
if filename.endswith(".tsv"):
|
||||
log("Fix file " + filename)
|
||||
filename_new = filename + "_new"
|
||||
|
||||
with open(data_dir['scrobbles'](filename_new),"w") as newfile:
|
||||
with open(data_dir['scrobbles'](filename),"r") as oldfile:
|
||||
|
||||
for l in oldfile:
|
||||
|
||||
a,t = re.sub(exp,r"\3",l), re.sub(exp,r"\5",l)
|
||||
r1,r2,r3 = re.sub(exp,r"\1\2",l),re.sub(exp,r"\4",l),re.sub(exp,r"\6",l)
|
||||
|
||||
a = a.split("␟")
|
||||
|
||||
(al,t) = wendigo.fullclean(a,t)
|
||||
a = "␟".join(al)
|
||||
newfile.write(r1 + a + r2 + t + r3 + "\n")
|
||||
|
||||
|
||||
#os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn)
|
||||
with open(data_dir['scrobbles'](filename_new),"r") as newfile, open(data_dir['scrobbles'](filename),"r") as oldfile:
|
||||
|
||||
diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="",n=0)
|
||||
diff = list(diff)
|
||||
|
||||
with open(os.path.join(patchfolder,filename + ".diff"),"w") as patchfile:
|
||||
patchfile.write("\n".join(diff))
|
||||
|
||||
os.replace(data_dir['scrobbles'](filename_new),data_dir['scrobbles'](filename))
|
||||
|
||||
|
||||
log("Database fixed!")
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import random
|
||||
import datetime
|
||||
|
||||
|
||||
artists = [
|
||||
"Chou Tzuyu","Jennie Kim","Kim Seolhyun","Nancy McDonie","Park Junghwa","Hirai Momo","Rosé Park","Laura Brehm","HyunA",
|
||||
"Jeremy Soule","Jerry Goldsmith","Howard Shore","Tilman Sillescu","James Newton Howard","Bear McCreary","David Newman",
|
||||
|
|
@ -64,13 +65,15 @@ def generate_track():
|
|||
|
||||
|
||||
|
||||
def generate(targetfile):
|
||||
with open(targetfile,"a") as fd:
|
||||
for _ in range(200):
|
||||
track = generate_track()
|
||||
for _ in range(random.randint(1, 50)):
|
||||
timestamp = random.randint(1, int(datetime.datetime.now().timestamp()))
|
||||
def generate():
|
||||
# TODO
|
||||
pass
|
||||
|
||||
entry = "\t".join([str(timestamp),"␟".join(track['artists']),track['title'],"-"])
|
||||
fd.write(entry)
|
||||
fd.write("\n")
|
||||
for _ in range(200):
|
||||
track = generate_track()
|
||||
for _ in range(random.randint(1, 50)):
|
||||
timestamp = random.randint(1, int(datetime.datetime.now().timestamp()))
|
||||
|
||||
entry = "\t".join([str(timestamp),"␟".join(track['artists']),track['title'],"-"])
|
||||
fd.write(entry)
|
||||
fd.write("\n")
|
||||
|
|
|
|||
|
|
@ -13,10 +13,14 @@ from ...globalconf import data_dir
|
|||
c = CleanerAgent()
|
||||
|
||||
|
||||
# TODO db import
|
||||
def import_scrobbles(fromfile):
|
||||
|
||||
def import_scrobbles(inputf):
|
||||
if not os.path.exists(fromfile):
|
||||
print("File could not be found.")
|
||||
return
|
||||
|
||||
ext = inputf.split('.')[-1].lower()
|
||||
ext = fromfile.split('.')[-1].lower()
|
||||
|
||||
if ext == 'csv':
|
||||
type = "Last.fm"
|
||||
|
|
@ -30,7 +34,7 @@ def import_scrobbles(inputf):
|
|||
importfunc = parse_spotify
|
||||
|
||||
|
||||
print(f"Parsing {col['yellow'](inputf)} as {col['cyan'](type)} export")
|
||||
print(f"Parsing {col['yellow'](fromfile)} as {col['cyan'](type)} export")
|
||||
|
||||
if os.path.exists(outputf):
|
||||
overwrite = ask("Already imported data. Overwrite?",default=False)
|
||||
|
|
@ -41,7 +45,7 @@ def import_scrobbles(inputf):
|
|||
failed = 0
|
||||
timestamps = set()
|
||||
|
||||
for scrobble in importfunc(inputf):
|
||||
for scrobble in importfunc(fromfile):
|
||||
if scrobble is None:
|
||||
failed += 1
|
||||
else:
|
||||
|
|
@ -73,6 +77,9 @@ def import_scrobbles(inputf):
|
|||
if success % 100 == 0:
|
||||
print(f"Imported {success} scrobbles...")
|
||||
|
||||
print("Successfully imported",success,"scrobbles!")
|
||||
if failed > 0:
|
||||
print(col['red'](str(failed) + " Errors!"))
|
||||
return success,failed
|
||||
|
||||
|
||||
Loading…
Reference in New Issue