From e0ec4b31375e25577f83d8e35dc532d0d349850a Mon Sep 17 00:00:00 2001 From: Krateng Date: Fri, 21 Dec 2018 16:32:21 +0100 Subject: [PATCH] Database can now analyze consistency of scrobble data --- README.md | 2 +- database.py | 50 ++++++++++++++++++-------- scrobbler-vivaldi-plex/settings.js | 2 +- utilities.py | 20 +++++++++++ website/issues.html | 7 ++++ website/issues.py | 6 ++++ website/maloja.css | 7 +++- website/topartists.py | 2 +- website/wait.html | 56 ++++++++++++++++++++++++++++++ 9 files changed, 134 insertions(+), 18 deletions(-) create mode 100644 website/wait.html diff --git a/README.md b/README.md index cf744e5..76c0890 100644 --- a/README.md +++ b/README.md @@ -46,4 +46,4 @@ I wouldn't recommend it yet. But if you want to test Maloja, it's fairly easy: proxy_pass http://yoururl:42011; } -If you would like to import all your previous last.fm scrobbles, use [benfoxall's website](https://benjaminbenben.com/lastfm-to-csv/) ([GitHub page](https://github.com/benfoxall/lastfm-to-csv)). Use the python script lastfmconverter.py with two arguments - the downloaded csv file and your new tsv file - to convert your data. Place the tsv file in logs/ and the server will recognize it on startup. +If you would like to import all your previous last.fm scrobbles, use [benfoxall's website](https://benjaminbenben.com/lastfm-to-csv/) ([GitHub page](https://github.com/benfoxall/lastfm-to-csv)). Use the python script lastfmconverter.py with two arguments - the downloaded csv file and your new tsv file - to convert your data. Place the tsv file in scrobbles/ and the server will recognize it on startup. diff --git a/database.py b/database.py index fa665b8..d15f28a 100644 --- a/database.py +++ b/database.py @@ -23,7 +23,8 @@ clients = [] lastsync = 0 -rulescheck = "" +# rulestate that the entire current database was built with, or False if the database was built from inconsistent scrobble files +db_rulestate = False @@ -109,13 +110,16 @@ def getTrackID(artists,title): def test_server(): apikey = request.query.get("key") response.set_header("Access-Control-Allow-Origin","*") - if not (checkAPIkey(apikey)): + if apikey is not None and not (checkAPIkey(apikey)): response.status = 403 - return "Wrong or Missing API key" + return "Wrong API key" - else: + elif db_rulestate: response.status = 204 return + else: + response.status = 205 + return @dbserver.route("/scrobbles") def get_scrobbles(): @@ -384,12 +388,15 @@ def abouttoshutdown(): def newrule(): keys = FormsDict.decode(request.forms) addEntry("rules/webmade.tsv",[k for k in keys]) + global db_rulestate + db_rulestate = False @dbserver.route("/issues") def issues(): combined = [] duplicates = [] newartists = [] + inconsistent = not db_rulestate import itertools @@ -473,7 +480,16 @@ def issues(): # duplicates.append((c[0],c[1])) - return {"duplicates":duplicates,"combined":combined,"newartists":newartists} + return {"duplicates":duplicates,"combined":combined,"newartists":newartists,"inconsistent":inconsistent} + +@dbserver.post("/rebuild") +def rebuild(): + sync() + os.system("python3 fixexisting.py") + global cla, coa + cla = CleanerAgent() + coa = CollectorAgent() + build_db() @@ -497,17 +513,14 @@ def runserver(PORT): def build_db(): - global rulescheck - global SCROBBLES - SCROBBLESNEW = [] - for t in SCROBBLES: - if not t[2]: - SCROBBLESNEW.append(t) - - SCROBBLES = SCROBBLESNEW + global SCROBBLES, ARTISTS, TRACKS + + SCROBBLES = [] + ARTISTS = [] + TRACKS = [] db = parseAllTSV("scrobbles","int","string","string") for sc in db: @@ -538,7 +551,8 @@ def build_db(): SCROBBLES.sort(key = lambda tup: tup[1]) - + global db_rulestate + db_rulestate = consistentRulestate("scrobbles",cla.checksums) @@ -691,6 +705,14 @@ def db_search(query,type=None): # Takes user inputs like YYYY/MM and returns the timestamps. Returns timestamp if timestamp was already given. def getTimestamps(f,t): #(f,t) = inp + if isinstance(f, str) and f.lower() == "today": + tod = datetime.date.today() + f = [tod.year,tod.month,tod.day] + if isinstance(t, str) and t.lower() == "today": + tod = datetime.date.today() + t = [tod.year,tod.month,tod.day] + + if isinstance(f, str): f = [int(x) for x in f.split("/")] diff --git a/scrobbler-vivaldi-plex/settings.js b/scrobbler-vivaldi-plex/settings.js index 8fbc921..ac403e2 100644 --- a/scrobbler-vivaldi-plex/settings.js +++ b/scrobbler-vivaldi-plex/settings.js @@ -55,7 +55,7 @@ function checkServer() { function createCheckmarks() { if (this.readyState == 4) { - if (this.status == 204) { + if ((this.status == 204) || (this.status == 205)) { //document.getElementById("checkmark_url").innerHTML = "✔️" //document.getElementById("checkmark_key").innerHTML = "✔️" document.getElementById("serverurl").style.backgroundColor = "lawngreen" diff --git a/utilities.py b/utilities.py index 76a602d..9950c05 100644 --- a/utilities.py +++ b/utilities.py @@ -77,7 +77,27 @@ def combineChecksums(filename,checksums): f.close() return True +# checks ALL files for their rule state. if they are all the same as the current loaded one, the entire database can be assumed to be consistent with the current ruleset +# in any other case, get out +def consistentRulestate(folder,checksums): + import os + result = [] + for scrobblefile in os.listdir(folder + "/"): + + if (scrobblefile.endswith(".tsv")): + + try: + f = open(folder + "/" + scrobblefile + ".rulestate","r") + if f.read() != checksums: + return False + + except: + return False + finally: + f.close() + + return True def parseAllTSV(path,*args): diff --git a/website/issues.html b/website/issues.html index 268e8d3..8aa1b6c 100644 --- a/website/issues.html +++ b/website/issues.html @@ -40,6 +40,13 @@ line.parentNode.removeChild(line) } + + function fullrebuild() { + var xhttp = new XMLHttpRequest(); + xhttp.open("POST","/db/rebuild", true); + xhttp.send() + window.location = "/wait"; + } diff --git a/website/issues.py b/website/issues.py index 655b2dd..363c056 100644 --- a/website/issues.py +++ b/website/issues.py @@ -9,6 +9,12 @@ def replacedict(keys,dbport): i = 0 html = "" + if db_data["inconsistent"]: + html += "" + html += "" + html += """""" + html += "" + i += 1 for d in db_data["duplicates"]: html += "" html += "
The current database wasn't built with all current rules in effect. Any problem below might be a false alarm and fixing it could create redundant rules.Rebuild the database
'" + artistLink(d[0]) + "'" diff --git a/website/maloja.css b/website/maloja.css index 945e4b3..bcfa6fd 100644 --- a/website/maloja.css +++ b/website/maloja.css @@ -60,7 +60,12 @@ table td.button { width:200px; background-color:yellow; color:#333337; - padding:1px; + padding:4px; border-radius:4px; cursor:pointer; } + +td.button.important { + background-color:red; + color:white; +} diff --git a/website/topartists.py b/website/topartists.py index cc1f734..a961cf8 100644 --- a/website/topartists.py +++ b/website/topartists.py @@ -6,7 +6,7 @@ def replacedict(keys,dbport): from utilities import getArtistInfo #hand down the since and from arguments - extrakeys = urllib.parse.urlencode(keys) + extrakeys = urllib.parse.urlencode(keys,quote_via=urllib.parse.quote,safe="/") response = urllib.request.urlopen("http://localhost:" + str(dbport) + "/charts/artists?" + extrakeys) db_data = json.loads(response.read()) diff --git a/website/wait.html b/website/wait.html new file mode 100644 index 0000000..dc6e8f5 --- /dev/null +++ b/website/wait.html @@ -0,0 +1,56 @@ + + + + + + Maloja - Please wait + + + + + + + + + +
+

Maloja


+ Redbuilding the database + +

Please wait...

+
+ + + + + +