From a4812a66dabb772de548b2792953a164718a3e98 Mon Sep 17 00:00:00 2001 From: Krateng Date: Thu, 3 Sep 2020 02:57:51 +0200 Subject: [PATCH] First working version for new api achitecture --- maloja/__pkginfo__.py | 2 +- maloja/apis/__init__.py | 10 +- maloja/apis/_base.py | 95 +++++++++++++++ maloja/apis/_exceptions.py | 6 + maloja/apis/audioscrobbler.py | 68 +++++++++++ maloja/apis/listenbrainz.py | 62 ++++++++++ maloja/compliant_api.py | 221 ---------------------------------- maloja/database.py | 1 - maloja/server.py | 5 +- 9 files changed, 244 insertions(+), 226 deletions(-) create mode 100644 maloja/apis/_base.py create mode 100644 maloja/apis/_exceptions.py create mode 100644 maloja/apis/audioscrobbler.py create mode 100644 maloja/apis/listenbrainz.py delete mode 100644 maloja/compliant_api.py diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index f443d02..6c37b4f 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -16,7 +16,7 @@ requires = [ "bottle>=0.12.16", "waitress>=1.3", "doreah>=1.6.10", - "nimrodel>=0.6.3", + "nimrodel>=0.6.4", "setproctitle>=1.1.10", "wand>=0.5.4", "lesscpy>=0.13", diff --git a/maloja/apis/__init__.py b/maloja/apis/__init__.py index d2743e7..10088ec 100644 --- a/maloja/apis/__init__.py +++ b/maloja/apis/__init__.py @@ -1,11 +1,19 @@ from . import native_v1 +from .audioscrobbler import Audioscrobbler +from .listenbrainz import Listenbrainz + import copy apis = { - "mlj_1":native_v1.api + "mlj_1":native_v1.api, + "listenbrainz/1":Listenbrainz().nimrodelapi, + "audioscrobbler/2.0":Audioscrobbler().nimrodelapi } +aliases = { + "native_1":"mlj_1" +} def init_apis(server): diff --git a/maloja/apis/_base.py b/maloja/apis/_base.py new file mode 100644 index 0000000..3ec2d3b --- /dev/null +++ b/maloja/apis/_base.py @@ -0,0 +1,95 @@ +from nimrodel import EAPI as API +from nimrodel import Multi + +from ._exceptions import * + +from copy import deepcopy +from types import FunctionType +import sys + +from doreah.logging import log + +from bottle import response + +from ..cleanup import CleanerAgent +from .. import database + +__logmodulename__ = "apis" + + +#def singleton(cls): +# return cls() + + + +cla = CleanerAgent() + +class APIHandler: + # make these classes singletons + _instance = None + def __new__(cls, *args, **kwargs): + if not isinstance(cls._instance, cls): + cls._instance = object.__new__(cls, *args, **kwargs) + return cls._instance + + def __init_subclass__(cls): + # Copy the handle function so we can have a unique docstring + sf = cls.__base__.wrapper + cls.wrapper = FunctionType(sf.__code__,sf.__globals__,sf.__name__,sf.__defaults__,sf.__closure__) + cls.wrapper.__annotations__ = sf.__annotations__ + # need to copy annotations so nimrodel properly passes path argument + + # create docstring + doc = "Accepts requests according to the {name} Standard" + cls.wrapper.__doc__ = doc.format(name=cls.__apiname__,link=cls.__doclink__) + + def __init__(self): + self.init() + + # creates a rump api object that exposes one generic endpoint + # we don't want the api explorer to show the different structures of + # third party apis, just mention they exist + self.nimrodelapi = API(delay=True) + self.nimrodelapi.get("{path}",pass_headers=True)(self.wrapper) + self.nimrodelapi.post("{path}",pass_headers=True)(self.wrapper) + self.nimrodelapi.get("",pass_headers=True)(self.wrapper) + self.nimrodelapi.post("",pass_headers=True)(self.wrapper) + + + def wrapper(self,path:Multi=[],**keys): + log("API request: " + str(path))# + " | Keys: " + str({k:keys.get(k) for k in keys})) + + try: + response.status,result = self.handle(path,keys) + except: + exceptiontype = sys.exc_info()[0] + if exceptiontype in self.errors: + response.status,result = self.errors[exceptiontype] + else: + response.status,result = 500,{"status":"Unknown error","code":500} + + return result + #else: + # result = {"error":"Invalid scrobble protocol"} + # response.status = 500 + + + def handle(self,path,keys): + + try: + methodname = self.get_method(path,keys) + method = self.methods[methodname] + except: + raise InvalidMethodException() + return method(path,keys) + + + def scrobble(self,artiststr,titlestr,time=None,duration=None,album=None): + logmsg = "Incoming scrobble (API: {api}): ARTISTS: {artiststr}, TRACK: {titlestr}" + log(logmsg.format(api=self.__apiname__,artiststr=artiststr,titlestr=titlestr)) + try: + (artists,title) = cla.fullclean(artiststr,titlestr) + database.createScrobble(artists,title,time) + database.sync() + except: + raise ScrobblingException() diff --git a/maloja/apis/_exceptions.py b/maloja/apis/_exceptions.py new file mode 100644 index 0000000..62139c9 --- /dev/null +++ b/maloja/apis/_exceptions.py @@ -0,0 +1,6 @@ +class BadAuthException(Exception): pass +class InvalidAuthException(Exception): pass +class InvalidMethodException(Exception): pass +class InvalidSessionKey(Exception): pass +class MalformedJSONException(Exception): pass +class ScrobblingException(Exception): pass diff --git a/maloja/apis/audioscrobbler.py b/maloja/apis/audioscrobbler.py new file mode 100644 index 0000000..df25651 --- /dev/null +++ b/maloja/apis/audioscrobbler.py @@ -0,0 +1,68 @@ +from ._base import APIHandler +from ._exceptions import * +from .. import database + +class Audioscrobbler(APIHandler): + __apiname__ = "Audioscrobbler" + __doclink__ = "https://www.last.fm/api/scrobbling" + + def init(self): + + # no need to save these on disk, clients can always request a new session + self.mobile_sessions = [] + self.methods = { + "auth.getMobileSession":self.authmobile, + "track.scrobble":self.submit_scrobble + } + self.errors = { + BadAuthException:(400,{"error":6,"message":"Requires authentication"}), + InvalidAuthException:(401,{"error":4,"message":"Invalid credentials"}), + InvalidMethodException:(200,{"error":3,"message":"Invalid method"}), + InvalidSessionKey:(403,{"error":9,"message":"Invalid session key"}), + ScrobblingException:(500,{"error":8,"message":"Operation failed"}) + } + + def get_method(self,pathnodes,keys): + return keys.get("method") + + def authmobile(self,pathnodes,keys): + token = keys.get("authToken") + user = keys.get("username") + password = keys.get("password") + # either username and password + if user is not None and password is not None: + if password in database.allAPIkeys(): + sessionkey = generate_key(self.mobile_sessions) + return 200,{"session":{"key":sessionkey}} + else: + raise InvalidAuthException() + # or username and token (deprecated by lastfm) + elif user is not None and token is not None: + for key in database.allAPIkeys(): + if md5(user + md5(key)) == token: + sessionkey = generate_key(self.mobile_sessions) + return 200,{"session":{"key":sessionkey}} + raise InvalidAuthException() + else: + raise BadAuthException() + + def submit_scrobble(self,pathnodes,keys): + if keys.get("sk") is None or keys.get("sk") not in self.mobile_sessions: + raise InvalidSessionKey() + else: + if "track" in keys and "artist" in keys: + artiststr,titlestr = keys["artist"], keys["track"] + #(artists,title) = cla.fullclean(artiststr,titlestr) + timestamp = int(keys["timestamp"]) + #database.createScrobble(artists,title,timestamp) + self.scrobble(artiststr,titlestr,time=timestamp) + return 200,{"scrobbles":{"@attr":{"ignored":0}}} + else: + for num in range(50): + if "track[" + str(num) + "]" in keys: + artiststr,titlestr = keys["artist[" + str(num) + "]"], keys["track[" + str(num) + "]"] + #(artists,title) = cla.fullclean(artiststr,titlestr) + timestamp = int(keys["timestamp[" + str(num) + "]"]) + #database.createScrobble(artists,title,timestamp) + self.scrobble(artiststr,titlestr,time=timestamp) + return 200,{"scrobbles":{"@attr":{"ignored":0}}} diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py new file mode 100644 index 0000000..16ee0e2 --- /dev/null +++ b/maloja/apis/listenbrainz.py @@ -0,0 +1,62 @@ +from ._base import APIHandler +from ._exceptions import * +from .. import database +import datetime + + +class Listenbrainz(APIHandler): + __apiname__ = "Listenbrainz" + __doclink__ = "https://listenbrainz.readthedocs.io/en/production/" + + def init(self): + self.methods = { + "submit-listens":self.submit, + "validate-token":self.validate_token + } + self.errors = { + BadAuthException:(401,{"code":401,"error":"You need to provide an Authorization header."}), + InvalidAuthException:(401,{"code":401,"error":"Incorrect Authorization"}), + InvalidMethodException:(200,{"code":200,"error":"Invalid Method"}), + MalformedJSONException:(400,{"code":400,"error":"Invalid JSON document submitted."}), + ScrobblingException:(500,{"code":500,"error":"Unspecified server error."}) + } + + def get_method(self,pathnodes,keys): + return pathnodes.pop(0) + + def submit(self,pathnodes,keys): + try: + token = keys.get("Authorization").replace("token ","").replace("Token ","").strip() + except: + raise BadAuthException() + + if token not in database.allAPIkeys(): + raise InvalidAuthException() + + try: + listentype = keys["listen_type"] + payload = keys["payload"] + if listentype in ["single","import"]: + for listen in payload: + metadata = listen["track_metadata"] + artiststr, titlestr = metadata["artist_name"], metadata["track_name"] + #(artists,title) = cla.fullclean(artiststr,titlestr) + try: + timestamp = int(listen["listened_at"]) + except: + timestamp = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()) + except: + raise MalformedJSONException() + + self.scrobble(artiststr,titlestr,timestamp) + return 200,{"status":"ok"} + + def validate_token(self,pathnodes,keys): + try: + token = keys.get("token").strip() + except: + raise BadAuthException() + if token not in database.allAPIkeys(): + raise InvalidAuthException() + else: + return 200,{"code":200,"message":"Token valid.","valid":True,"user_name":"n/a"} diff --git a/maloja/compliant_api.py b/maloja/compliant_api.py deleted file mode 100644 index 7c13399..0000000 --- a/maloja/compliant_api.py +++ /dev/null @@ -1,221 +0,0 @@ -from doreah.logging import log -import hashlib -import random -from . import database -import datetime -import itertools -import sys -from .cleanup import CleanerAgent -from bottle import response - -## GNU-FM-compliant scrobbling - - -cla = CleanerAgent() - -def md5(input): - m = hashlib.md5() - m.update(bytes(input,encoding="utf-8")) - return m.hexdigest() - -def generate_key(ls): - key = "" - for i in range(64): - key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) - ls.append(key) - return key - -#def check_sig(keys): -# try: -# sig = keys.pop("api_sig") -# text = "".join([key + keys[key] for key in sorted(keys.keys())]) + # secret -# assert sig == md5(text) -# return True -# except: -# return False - - -handlers = {} - -def handler(apiname,version): - def deco(cls): - handlers[(apiname,version)] = cls() - return cls - return deco - -def handle(path,keys): - log("API request: " + str(path))# + " | Keys: " + str({k:keys.get(k) for k in keys})) - - if len(path)>1 and (path[0],path[1]) in handlers: - handler = handlers[(path[0],path[1])] - path = path[2:] - try: - response.status,result = handler.handle(path,keys) - except: - type = sys.exc_info()[0] - response.status,result = handler.errors[type] - else: - result = {"error":"Invalid scrobble protocol"} - response.status = 500 - - - log("Response: " + str(result)) - return result - -def scrobbletrack(artiststr,titlestr,timestamp): - try: - log("Incoming scrobble (compliant API): ARTISTS: " + artiststr + ", TRACK: " + titlestr,module="debug") - (artists,title) = cla.fullclean(artiststr,titlestr) - database.createScrobble(artists,title,timestamp) - database.sync() - except: - raise ScrobblingException() - - -class BadAuthException(Exception): pass -class InvalidAuthException(Exception): pass -class InvalidMethodException(Exception): pass -class InvalidSessionKey(Exception): pass -class MalformedJSONException(Exception): pass -class ScrobblingException(Exception): pass - -class APIHandler: - # make these classes singletons - _instance = None - def __new__(cls, *args, **kwargs): - if not isinstance(cls._instance, cls): - cls._instance = object.__new__(cls, *args, **kwargs) - return cls._instance - - - def handle(self,pathnodes,keys): - try: - methodname = self.get_method(pathnodes,keys) - method = self.methods[methodname] - except: - raise InvalidMethodException() - return method(pathnodes,keys) - -@handler("audioscrobbler","2.0") -@handler("gnufm","2.0") -@handler("gnukebox","2.0") -class GNUFM2(APIHandler): - def __init__(self): - # no need to save these on disk, clients can always request a new session - self.mobile_sessions = [] - self.methods = { - "auth.getMobileSession":self.authmobile, - "track.scrobble":self.scrobble - } - self.errors = { - BadAuthException:(400,{"error":6,"message":"Requires authentication"}), - InvalidAuthException:(401,{"error":4,"message":"Invalid credentials"}), - InvalidMethodException:(200,{"error":3,"message":"Invalid method"}), - InvalidSessionKey:(403,{"error":9,"message":"Invalid session key"}), - ScrobblingException:(500,{"error":8,"message":"Operation failed"}) - } - - def get_method(self,pathnodes,keys): - return keys.get("method") - - def authmobile(self,pathnodes,keys): - token = keys.get("authToken") - user = keys.get("username") - password = keys.get("password") - # either username and password - if user is not None and password is not None: - if password in database.allAPIkeys(): - sessionkey = generate_key(self.mobile_sessions) - return 200,{"session":{"key":sessionkey}} - else: - raise InvalidAuthException() - # or username and token (deprecated by lastfm) - elif user is not None and token is not None: - for key in database.allAPIkeys(): - if md5(user + md5(key)) == token: - sessionkey = generate_key(self.mobile_sessions) - return 200,{"session":{"key":sessionkey}} - raise InvalidAuthException() - else: - raise BadAuthException() - - def scrobble(self,pathnodes,keys): - if keys.get("sk") is None or keys.get("sk") not in self.mobile_sessions: - raise InvalidSessionKey() - else: - if "track" in keys and "artist" in keys: - artiststr,titlestr = keys["artist"], keys["track"] - #(artists,title) = cla.fullclean(artiststr,titlestr) - timestamp = int(keys["timestamp"]) - #database.createScrobble(artists,title,timestamp) - scrobbletrack(artiststr,titlestr,timestamp) - return 200,{"scrobbles":{"@attr":{"ignored":0}}} - else: - for num in range(50): - if "track[" + str(num) + "]" in keys: - artiststr,titlestr = keys["artist[" + str(num) + "]"], keys["track[" + str(num) + "]"] - #(artists,title) = cla.fullclean(artiststr,titlestr) - timestamp = int(keys["timestamp[" + str(num) + "]"]) - #database.createScrobble(artists,title,timestamp) - scrobbletrack(artiststr,titlestr,timestamp) - return 200,{"scrobbles":{"@attr":{"ignored":0}}} - - - -@handler("listenbrainz","1") -@handler("lbrnz","1") -class LBrnz1(APIHandler): - def __init__(self): - self.methods = { - "submit-listens":self.submit, - "validate-token":self.validate_token - } - self.errors = { - BadAuthException:(401,{"code":401,"error":"You need to provide an Authorization header."}), - InvalidAuthException:(401,{"code":401,"error":"Incorrect Authorization"}), - InvalidMethodException:(200,{"code":200,"error":"Invalid Method"}), - MalformedJSONException:(400,{"code":400,"error":"Invalid JSON document submitted."}), - ScrobblingException:(500,{"code":500,"error":"Unspecified server error."}) - } - - def get_method(self,pathnodes,keys): - return pathnodes.pop(0) - - def submit(self,pathnodes,keys): - try: - token = keys.get("Authorization").replace("token ","").replace("Token ","").strip() - except: - raise BadAuthException() - - if token not in database.allAPIkeys(): - raise InvalidAuthException() - - try: - #log("scrobbling to listenbrainz, keys "+str(keys),module="debug") - if keys["listen_type"] in ["single","import"]: - payload = keys["payload"] - for listen in payload: - metadata = listen["track_metadata"] - artiststr, titlestr = metadata["artist_name"], metadata["track_name"] - #(artists,title) = cla.fullclean(artiststr,titlestr) - try: - timestamp = int(listen["listened_at"]) - except: - timestamp = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()) - #database.createScrobble(artists,title,timestamp) - scrobbletrack(artiststr,titlestr,timestamp) - return 200,{"status":"ok"} - else: - return 200,{"status":"ok"} - except: - raise MalformedJSONException() - - def validate_token(self,pathnodes,keys): - try: - token = keys.get("token").strip() - except: - raise BadAuthException() - if token not in database.allAPIkeys(): - raise InvalidAuthException() - else: - return 200,{"code":200,"message":"Token valid.","valid":True,"user_name":"n/a"} diff --git a/maloja/database.py b/maloja/database.py index 0ddd751..c0c82e8 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -6,7 +6,6 @@ from .cleanup import CleanerAgent, CollectorAgent from . import utilities from .malojatime import register_scrobbletime, time_stamps, ranges from .malojauri import uri_to_internal, internal_to_uri, compose_querystring -from . import compliant_api from .thirdparty import proxy_scrobble_all diff --git a/maloja/server.py b/maloja/server.py index 92c834d..05d72f8 100755 --- a/maloja/server.py +++ b/maloja/server.py @@ -18,6 +18,7 @@ from .utilities import resolveImage from .malojauri import uri_to_internal, remove_identical, compose_querystring from . import globalconf from .jinjaenv.context import jinja_environment +from jinja2.exceptions import TemplateNotFound # doreah toolkit from doreah import settings from doreah.logging import log @@ -112,8 +113,8 @@ def mainpage(): def customerror(error): errorcode = error.status_code errordesc = error.status - traceback = error.traceback.strip() - + traceback = error.traceback + traceback = traceback.strip() if traceback is not None else "No Traceback" adminmode = request.cookies.get("adminmode") == "true" and auth.check(request) template = jinja_environment.get_template('error.jinja')