From e50cce28fad2913264d6223240a0f6cae897a891 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sun, 15 Dec 2019 15:27:16 +0100 Subject: [PATCH] Finally refactored lastfm import --- maloja/backup.py | 2 +- maloja/cleanup.py | 18 ++++++-- maloja/controller.py | 10 ++--- maloja/lastfmconverter.py | 88 +++++++++++++++++++++------------------ 4 files changed, 68 insertions(+), 50 deletions(-) diff --git a/maloja/backup.py b/maloja/backup.py index b08bfe0..3a7e9da 100644 --- a/maloja/backup.py +++ b/maloja/backup.py @@ -2,7 +2,7 @@ import tarfile from datetime import datetime import glob import os -from globalconf import datadir +from .globalconf import datadir user_files = { diff --git a/maloja/cleanup.py b/maloja/cleanup.py index d9a427f..7d435fd 100644 --- a/maloja/cleanup.py +++ b/maloja/cleanup.py @@ -25,8 +25,17 @@ class CleanerAgent: #self.rules_regextitle = [[b,c] for [a,b,c,d] in raw if a=="regextitle"] # TODO + #self.plugin_artistparsers = [] + #self.plugin_titleparsers = [] + #if settings.get_settings("USE_PARSE_PLUGINS"): + # for ep in pkg_resources.iter_entry_points(group='maloja.artistparsers'): + # self.plugin_artistparsers.append(ep.load()) + # for ep in pkg_resources.iter_entry_points(group='maloja.titleparsers'): + # self.plugin_titleparsers.append(ep.load()) + + # we always need to be able to tell if our current database is made with the current rules - self.checksums = utilities.checksumTSV("rules") + self.checksums = utilities.checksumTSV(datadir("rules")) @@ -125,9 +134,12 @@ class CleanerAgent: for s in settings.get_settings("REMOVE_FROM_TITLE"): if s in t: - t = t.replace(s,"").strip() + t = t.replace(s,"") - return t.strip() + t = t.strip() + #for p in self.plugin_titleparsers: + # t = p(t).strip() + return t def parseTitleForArtists(self,t): for d in self.delimiters_feat: diff --git a/maloja/controller.py b/maloja/controller.py index da88d6b..199f7b8 100755 --- a/maloja/controller.py +++ b/maloja/controller.py @@ -144,10 +144,8 @@ def stop(): def loadlastfm(filename): - try: - filename = os.path.join(origpath,filename) - except: - print("Please specify a file!") + if not os.path.exists(filename): + print("File could not be found.") return if os.path.exists(datadir("scrobbles/lastfmimport.tsv")): @@ -157,7 +155,9 @@ def loadlastfm(filename): else: return print("Please wait...") - os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv")) + from .lastfmconverter import convert + convert(filename,datadir("scrobbles/lastfmimport.tsv")) + #os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv")) print("Successfully imported your Last.FM scrobbles!") def direct(): diff --git a/maloja/lastfmconverter.py b/maloja/lastfmconverter.py index 8f2959a..9f58990 100644 --- a/maloja/lastfmconverter.py +++ b/maloja/lastfmconverter.py @@ -1,63 +1,69 @@ -import sys, os, datetime, re, cleanup +import os, datetime, re from .cleanup import * from .utilities import * -log = open(sys.argv[1]) -outputlog = open(sys.argv[2],"w") -checksumfile = open(sys.argv[2] + ".rulestate","w") #this file stores an identifier for all rules that were in place when the corresponding file was created c = CleanerAgent() -stamps = [99999999999999] - -for l in log: - l = l.replace("\n","") - data = l.split(",") - - artist = data[0] - album = data[1] - title = data[2] - time = data[3] - (artists,title) = c.fullclean(artist,title) - artistsstr = "␟".join(artists) +def convert(input,output): + + log = open(input,"r") + outputlog = open(output,"w") + checksumfile = open(output + ".rulestate","w") #this file stores an identifier for all rules that were in place when the corresponding file was created + + stamps = [99999999999999] + + for l in log: + l = l.replace("\n","") + data = l.split(",") + + artist = data[0] + album = data[1] + title = data[2] + time = data[3] - timeparts = time.split(" ") - (h,m) = timeparts[3].split(":") + (artists,title) = c.fullclean(artist,title) - months = {"Jan":1,"Feb":2,"Mar":3,"Apr":4,"May":5,"Jun":6,"Jul":7,"Aug":8,"Sep":9,"Oct":10,"Nov":11,"Dec":12} - - timestamp = int(datetime.datetime(int(timeparts[2]),months[timeparts[1]],int(timeparts[0]),int(h),int(m)).timestamp()) + artistsstr = "␟".join(artists) - ## We prevent double timestamps in the database creation, so we technically don't need them in the files - ## however since the conversion from lastfm to maloja is a one-time thing, we should take any effort to make the file as good as possible - if (timestamp < stamps[-1]): - pass - elif (timestamp == stamps[-1]): - timestamp -= 1 - else: - while(timestamp in stamps): + timeparts = time.split(" ") + (h,m) = timeparts[3].split(":") + + months = {"Jan":1,"Feb":2,"Mar":3,"Apr":4,"May":5,"Jun":6,"Jul":7,"Aug":8,"Sep":9,"Oct":10,"Nov":11,"Dec":12} + + timestamp = int(datetime.datetime(int(timeparts[2]),months[timeparts[1]],int(timeparts[0]),int(h),int(m)).timestamp()) + + + ## We prevent double timestamps in the database creation, so we technically don't need them in the files + ## however since the conversion from lastfm to maloja is a one-time thing, we should take any effort to make the file as good as possible + if (timestamp < stamps[-1]): + pass + elif (timestamp == stamps[-1]): timestamp -= 1 + else: + while(timestamp in stamps): + timestamp -= 1 - if (timestamp < stamps[-1]): - stamps.append(timestamp) - else: - stamps.insert(0,timestamp) + if (timestamp < stamps[-1]): + stamps.append(timestamp) + else: + stamps.insert(0,timestamp) - entry = "\t".join([str(timestamp),artistsstr,title,album]) - entry = entry.replace("#",r"\num") + entry = "\t".join([str(timestamp),artistsstr,title,album]) + entry = entry.replace("#",r"\num") - outputlog.write(entry) - outputlog.write("\n") + outputlog.write(entry) + outputlog.write("\n") -checksumfile.write(c.checksums) + checksumfile.write(c.checksums) -log.close() -outputlog.close() -checksumfile.close() + log.close() + outputlog.close() + checksumfile.close()