From d81f8374c9eb68e5da3d6753d280d2d058a343e8 Mon Sep 17 00:00:00 2001 From: krateng Date: Sat, 23 Apr 2022 20:13:50 +0200 Subject: [PATCH 01/13] Simplified container build --- Containerfile | 16 +++------------- dev/templates/Containerfile.jinja | 9 --------- install/install_alpine.sh | 1 + install/install_dependencies_alpine.sh | 1 + pyproject.toml | 1 + requirements_pre.txt | 3 --- 6 files changed, 6 insertions(+), 25 deletions(-) delete mode 100644 requirements_pre.txt diff --git a/Containerfile b/Containerfile index 8e11da8..b1d1981 100644 --- a/Containerfile +++ b/Containerfile @@ -6,7 +6,7 @@ FROM alpine:3.15 WORKDIR /usr/src/app # Install run dependencies first -RUN apk add --no-cache python3 tzdata +RUN apk add --no-cache python3 py3-lxml tzdata # system pip could be removed after build, but apk then decides to also remove all its # python dependencies, even if they are explicitly installed as python packages @@ -15,16 +15,7 @@ RUN \ apk add py3-pip && \ pip install wheel -# these are more static than the real requirements, which means caching -COPY ./requirements_pre.txt ./requirements_pre.txt -RUN \ - apk add --no-cache --virtual .build-deps gcc g++ python3-dev libxml2-dev libxslt-dev libffi-dev libc-dev py3-pip linux-headers && \ - pip install --no-cache-dir -r requirements_pre.txt && \ - apk del .build-deps - - -# less likely to be cached COPY ./requirements.txt ./requirements.txt RUN \ @@ -39,10 +30,9 @@ COPY . . RUN pip install /usr/src/app -# Docker-specific configuration -# defaulting to IPv4 is no longer necessary (default host is dual stack) +# Docker-specific configuration and default to IPv4 ENV MALOJA_SKIP_SETUP=yes -ENV PYTHONUNBUFFERED=1 +ENV MALOJA_HOST=0.0.0.0 EXPOSE 42010 # use exec form for better signal handling https://docs.docker.com/engine/reference/builder/#entrypoint diff --git a/dev/templates/Containerfile.jinja b/dev/templates/Containerfile.jinja index e180abd..0eb8063 100644 --- a/dev/templates/Containerfile.jinja +++ b/dev/templates/Containerfile.jinja @@ -15,16 +15,7 @@ RUN \ apk add py3-pip && \ pip install wheel -# these are more static than the real requirements, which means caching -COPY ./requirements_pre.txt ./requirements_pre.txt -RUN \ - apk add --no-cache --virtual .build-deps {{ tool.osreqs.alpine.build | join(' ') }} && \ - pip install --no-cache-dir -r requirements_pre.txt && \ - apk del .build-deps - - -# less likely to be cached COPY ./requirements.txt ./requirements.txt RUN \ diff --git a/install/install_alpine.sh b/install/install_alpine.sh index 88b23c2..5de1f41 100644 --- a/install/install_alpine.sh +++ b/install/install_alpine.sh @@ -11,6 +11,7 @@ apk add \ py3-pip \ linux-headers \ python3 \ + py3-lxml \ tzdata \ vips diff --git a/install/install_dependencies_alpine.sh b/install/install_dependencies_alpine.sh index 5a14d85..20bca2b 100644 --- a/install/install_dependencies_alpine.sh +++ b/install/install_dependencies_alpine.sh @@ -11,5 +11,6 @@ apk add \ py3-pip \ linux-headers \ python3 \ + py3-lxml \ tzdata \ vips diff --git a/pyproject.toml b/pyproject.toml index ea61747..dba3ae1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ build =[ ] run = [ "python3", + "py3-lxml", "tzdata" ] opt = [ diff --git a/requirements_pre.txt b/requirements_pre.txt deleted file mode 100644 index c4c0dbe..0000000 --- a/requirements_pre.txt +++ /dev/null @@ -1,3 +0,0 @@ -# this is a more static file that enables container images to be cached -# it should contain packages that take long to build and don't change frequently -lxml From 3cb72f46bc658b255987776b9e49fb33c1bac054 Mon Sep 17 00:00:00 2001 From: krateng Date: Sat, 23 Apr 2022 20:59:18 +0200 Subject: [PATCH 02/13] Didn't actually mean to reset this --- Containerfile | 5 +++-- dev/templates/Containerfile.jinja | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Containerfile b/Containerfile index b1d1981..ed8a983 100644 --- a/Containerfile +++ b/Containerfile @@ -30,9 +30,10 @@ COPY . . RUN pip install /usr/src/app -# Docker-specific configuration and default to IPv4 +# Docker-specific configuration +# defaulting to IPv4 is no longer necessary (default host is dual stack) ENV MALOJA_SKIP_SETUP=yes -ENV MALOJA_HOST=0.0.0.0 +ENV PYTHONUNBUFFERED=1 EXPOSE 42010 # use exec form for better signal handling https://docs.docker.com/engine/reference/builder/#entrypoint diff --git a/dev/templates/Containerfile.jinja b/dev/templates/Containerfile.jinja index 0eb8063..7a99f03 100644 --- a/dev/templates/Containerfile.jinja +++ b/dev/templates/Containerfile.jinja @@ -30,9 +30,10 @@ COPY . . RUN pip install /usr/src/app -# Docker-specific configuration and default to IPv4 +# Docker-specific configuration +# defaulting to IPv4 is no longer necessary (default host is dual stack) ENV MALOJA_SKIP_SETUP=yes -ENV MALOJA_HOST=0.0.0.0 +ENV PYTHONUNBUFFERED=1 EXPOSE 42010 # use exec form for better signal handling https://docs.docker.com/engine/reference/builder/#entrypoint From 082d11309bc75cebe64e6056a1981a4f6359471b Mon Sep 17 00:00:00 2001 From: Jiri Travnicek Date: Sat, 23 Apr 2022 17:12:01 +0200 Subject: [PATCH 03/13] build arm64 --- .github/workflows/docker.yml | 85 +++++++++++++++++++++++++++++++++ .github/workflows/dockerhub.yml | 51 -------------------- 2 files changed, 85 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/dockerhub.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..408be1e --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,85 @@ +name: Build and release docker image + +on: + push: + tags: + - 'v*' + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Log in to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + + - name: Login to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v3 + with: + images: | + krateng/maloja + ghcr.io/krateng/maloja + # generate Docker tags based on the following events/attributes + tags: | + type=semver,pattern={{version}} + flavor: | + latest=true + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + file: Containerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Update Readme and short description + uses: peter-evans/dockerhub-description@1cf9afbac3c5d2fdc66416a464c2c38260cb6f8d + continue-on-error: true + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + repository: krateng/maloja + short-description: ${{ github.event.repository.description }} diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml deleted file mode 100644 index f20b655..0000000 --- a/.github/workflows/dockerhub.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Publish to Dockerhub - -on: - push: - tags: - - 'v*' - -jobs: - push_to_registry: - name: Push Docker image to Docker Hub - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - - - name: Log in to Docker Hub - if: github.event_name != 'pull_request' - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 - with: - images: krateng/maloja - # generate Docker tags based on the following events/attributes - tags: | - type=semver,pattern={{version}} - flavor: | - latest=true - - - name: Build and push Docker image - uses: docker/build-push-action@7f9d37fa544684fb73bfe4835ed7214c255ce02b - with: - context: . - file: Containerfile - #platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - - name: Update Readme and short description - uses: peter-evans/dockerhub-description@1cf9afbac3c5d2fdc66416a464c2c38260cb6f8d - continue-on-error: true - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} - repository: krateng/maloja - short-description: ${{ github.event.repository.description }} From d54f2f8d35222844ec0bd8c382b92c6fad97a6bd Mon Sep 17 00:00:00 2001 From: Jiri Travnicek Date: Sat, 23 Apr 2022 20:38:18 +0200 Subject: [PATCH 04/13] ci: use specific commit tag for github actions --- .github/workflows/docker.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 408be1e..fc775b7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,18 +11,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b - name: Log in to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@v1 + uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} - name: Login to GHCR if: github.event_name != 'pull_request' - uses: docker/login-action@v1 + uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -30,7 +30,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 with: images: | krateng/maloja @@ -42,13 +42,13 @@ jobs: latest=true - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Cache Docker layers - uses: actions/cache@v2 + uses: actions/cache@48af2dc4a9e8278b89d7fa154b955c30c6aaab09 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} @@ -56,7 +56,7 @@ jobs: ${{ runner.os }}-buildx- - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a with: context: . file: Containerfile @@ -76,7 +76,7 @@ jobs: mv /tmp/.buildx-cache-new /tmp/.buildx-cache - name: Update Readme and short description - uses: peter-evans/dockerhub-description@1cf9afbac3c5d2fdc66416a464c2c38260cb6f8d + uses: peter-evans/dockerhub-description@836d7e6aa8f6f32dce26f5a1dd46d3dc24997eae continue-on-error: true with: username: ${{ secrets.DOCKERHUB_USERNAME }} From 9589a6a5c9f609a1a975778cf5da4dfe649ff8c8 Mon Sep 17 00:00:00 2001 From: Jiri Travnicek Date: Sat, 23 Apr 2022 20:39:42 +0200 Subject: [PATCH 05/13] use repository owner variable instead of hardcoding it --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fc775b7..f33c65e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,8 +33,8 @@ jobs: uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 with: images: | - krateng/maloja - ghcr.io/krateng/maloja + ${{ github.repository_owner }}/maloja + ghcr.io/${{ github.repository_owner }}/maloja # generate Docker tags based on the following events/attributes tags: | type=semver,pattern={{version}} From 0c94dc845b6c1da3b3e2c9991c10e4250eb57188 Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 24 Apr 2022 03:06:44 +0200 Subject: [PATCH 06/13] Updated release information --- dev/releases/3.0.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/releases/3.0.yml b/dev/releases/3.0.yml index 6ac393d..6ccf8b2 100644 --- a/dev/releases/3.0.yml +++ b/dev/releases/3.0.yml @@ -35,7 +35,9 @@ minor_release_name: "Yeonhee" - "[Bugfix] Fixed crash when encountering error in Lastfm import" 3.0.6: notes: + - "[Performance] Implemented search in database" - "[Bugfix] Better parsing of featuring artists" - "[Bugfix] Fixed buffered output in Docker" - "[Bugfix] Fixed importing a Spotify file without path" - "[Bugfix] No longer releasing database lock during scrobble creation" + - "[Distribution] Experimental arm64 image" From b3d4cb7a153845d1f5a5eef67a6508754e338f2f Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 24 Apr 2022 03:08:22 +0200 Subject: [PATCH 07/13] Version bump --- maloja/__pkginfo__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index ecf899c..44acbe8 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -4,7 +4,7 @@ # you know what f*ck it # this is hardcoded for now because of that damn project / package name discrepancy # i'll fix it one day -VERSION = "3.0.5" +VERSION = "3.0.6" HOMEPAGE = "https://github.com/krateng/maloja" diff --git a/pyproject.toml b/pyproject.toml index dba3ae1..aa20899 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "malojaserver" -version = "3.0.5" +version = "3.0.6" description = "Self-hosted music scrobble database" readme = "./README.md" requires-python = ">=3.6" From e1074ba259ae46d0b3f5a1fef3b3dbb231461057 Mon Sep 17 00:00:00 2001 From: Jiri Travnicek Date: Sun, 24 Apr 2022 16:11:58 +0200 Subject: [PATCH 08/13] actions: drop ghcr support --- .github/workflows/docker.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f33c65e..e95305c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -20,21 +20,12 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} - - name: Login to GHCR - if: github.event_name != 'pull_request' - uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67 with: images: | ${{ github.repository_owner }}/maloja - ghcr.io/${{ github.repository_owner }}/maloja # generate Docker tags based on the following events/attributes tags: | type=semver,pattern={{version}} From 00b3e6fc573e8fc76daef6149dd50651a5546737 Mon Sep 17 00:00:00 2001 From: Jiri Travnicek Date: Sun, 24 Apr 2022 16:12:11 +0200 Subject: [PATCH 09/13] actions: build image for linux/arm/v7 (raspberry pi) --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e95305c..1aad71c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -54,7 +54,7 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64,linux/arm/v7 cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max From bfc83fdbb08ff02acb134d529a3a85c614f68dc3 Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 24 Apr 2022 20:47:17 +0200 Subject: [PATCH 10/13] Ported signal handling fix from 3.1 --- dev/releases/3.0.yml | 1 + dev/write_tags.py | 2 +- maloja/apis/_base.py | 6 +++--- maloja/apis/audioscrobbler.py | 2 +- maloja/apis/audioscrobbler_legacy.py | 2 +- maloja/apis/listenbrainz.py | 10 +++++----- maloja/database/__init__.py | 8 ++++---- maloja/database/dbcache.py | 4 ++-- maloja/globalconf.py | 2 +- maloja/images.py | 4 ++-- maloja/jinjaenv/filters.py | 2 +- maloja/malojatime.py | 4 ++-- maloja/malojauri.py | 2 +- maloja/monkey.py | 8 ++++---- maloja/proccontrol/control.py | 6 +++--- maloja/proccontrol/profiler.py | 2 +- maloja/thirdparty/__init__.py | 2 +- maloja/thirdparty/musicbrainz.py | 2 +- maloja/upgrade.py | 2 +- 19 files changed, 36 insertions(+), 35 deletions(-) diff --git a/dev/releases/3.0.yml b/dev/releases/3.0.yml index 6ccf8b2..6a52c9d 100644 --- a/dev/releases/3.0.yml +++ b/dev/releases/3.0.yml @@ -34,6 +34,7 @@ minor_release_name: "Yeonhee" - "[Feature] Added notification system for web interface" - "[Bugfix] Fixed crash when encountering error in Lastfm import" 3.0.6: + commit: "b3d4cb7a153845d1f5a5eef67a6508754e338f2f" notes: - "[Performance] Implemented search in database" - "[Bugfix] Better parsing of featuring artists" diff --git a/dev/write_tags.py b/dev/write_tags.py index b3976e7..4a3cb42 100644 --- a/dev/write_tags.py +++ b/dev/write_tags.py @@ -43,7 +43,7 @@ for version in releases: try: prev_tag = sp.check_output(["git","show",f'v{maj}.{min}.{hot}']).decode() prev_tag_commit = prev_tag.split('\n')[6].split(" ")[1] - except: + except Exception: pass else: assert prev_tag_commit == info['commit'] diff --git a/maloja/apis/_base.py b/maloja/apis/_base.py index 938c701..d81232e 100644 --- a/maloja/apis/_base.py +++ b/maloja/apis/_base.py @@ -62,7 +62,7 @@ class APIHandler: try: response.status,result = self.handle(path,keys) - except: + except Exception: exceptiontype = sys.exc_info()[0] if exceptiontype in self.errors: response.status,result = self.errors[exceptiontype] @@ -82,7 +82,7 @@ class APIHandler: try: methodname = self.get_method(path,keys) method = self.methods[methodname] - except: + except Exception: log("Could not find a handler for method " + str(methodname) + " in API " + self.__apiname__,module="debug") log("Keys: " + str(keys),module="debug") raise InvalidMethodException() @@ -94,5 +94,5 @@ class APIHandler: # fixing etc is handled by the main scrobble function try: return database.incoming_scrobble(rawscrobble,api=self.__apiname__,client=client) - except: + except Exception: raise ScrobblingException() diff --git a/maloja/apis/audioscrobbler.py b/maloja/apis/audioscrobbler.py index 586f2c5..6699618 100644 --- a/maloja/apis/audioscrobbler.py +++ b/maloja/apis/audioscrobbler.py @@ -76,7 +76,7 @@ class Audioscrobbler(APIHandler): #(artists,title) = cla.fullclean(artiststr,titlestr) try: timestamp = int(keys["timestamp"]) - except: + except Exception: timestamp = None #database.createScrobble(artists,title,timestamp) self.scrobble({'track_artists':[artiststr],'track_title':titlestr,'scrobble_time':timestamp},client=client) diff --git a/maloja/apis/audioscrobbler_legacy.py b/maloja/apis/audioscrobbler_legacy.py index 322fe78..2a3ac33 100644 --- a/maloja/apis/audioscrobbler_legacy.py +++ b/maloja/apis/audioscrobbler_legacy.py @@ -80,7 +80,7 @@ class AudioscrobblerLegacy(APIHandler): artiststr,titlestr = keys[artist_key], keys[track_key] try: timestamp = int(keys[time_key]) - except: + except Exception: timestamp = None #database.createScrobble(artists,title,timestamp) self.scrobble({ diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py index 457573f..507de60 100644 --- a/maloja/apis/listenbrainz.py +++ b/maloja/apis/listenbrainz.py @@ -34,7 +34,7 @@ class Listenbrainz(APIHandler): def submit(self,pathnodes,keys): try: token = self.get_token_from_request_keys(keys) - except: + except Exception: raise BadAuthException() client = apikeystore.check_and_identify_key(token) @@ -45,7 +45,7 @@ class Listenbrainz(APIHandler): try: listentype = keys["listen_type"] payload = keys["payload"] - except: + except Exception: raise MalformedJSONException() if listentype == "playing_now": @@ -57,9 +57,9 @@ class Listenbrainz(APIHandler): artiststr, titlestr = metadata["artist_name"], metadata["track_name"] try: timestamp = int(listen["listened_at"]) - except: + except Exception: timestamp = None - except: + except Exception: raise MalformedJSONException() self.scrobble({ @@ -74,7 +74,7 @@ class Listenbrainz(APIHandler): def validate_token(self,pathnodes,keys): try: token = self.get_token_from_request_keys(keys) - except: + except Exception: raise BadAuthException() if not apikeystore.check_key(token): raise InvalidAuthException() diff --git a/maloja/database/__init__.py b/maloja/database/__init__.py index 6595e88..dbd51c0 100644 --- a/maloja/database/__init__.py +++ b/maloja/database/__init__.py @@ -256,7 +256,7 @@ def get_top_artists(dbconn=None,**keys): try: res = get_charts_artists(timerange=rng,dbconn=dbconn)[0] results.append({"range":rng,"artist":res["artist"],"scrobbles":res["scrobbles"]}) - except: + except Exception: results.append({"range":rng,"artist":None,"scrobbles":0}) return results @@ -272,7 +272,7 @@ def get_top_tracks(dbconn=None,**keys): try: res = get_charts_tracks(timerange=rng,dbconn=dbconn)[0] results.append({"range":rng,"track":res["track"],"scrobbles":res["scrobbles"]}) - except: + except Exception: results.append({"range":rng,"track":None,"scrobbles":0}) return results @@ -302,7 +302,7 @@ def artist_info(dbconn=None,**keys): }, "topweeks":len([e for e in cached.weekly_topartists if e == artist]) } - except: + except Exception: # if the artist isnt in the charts, they are not being credited and we # need to show information about the credited one replaceartist = sqldb.get_credited_artists(artist)[0] @@ -370,7 +370,7 @@ def get_predefined_rulesets(dbconn=None): else: name = rawf.split("_")[1] desc = line2.replace("# DESC: ","") if "# DESC: " in line2 else "" author = rawf.split("_")[0] - except: + except Exception: continue ruleset = {"file":rawf} diff --git a/maloja/database/dbcache.py b/maloja/database/dbcache.py index b3d4ce3..78174b7 100644 --- a/maloja/database/dbcache.py +++ b/maloja/database/dbcache.py @@ -132,10 +132,10 @@ def trim_cache(): def serialize(obj): try: return serialize(obj.hashable()) - except: + except Exception: try: return json.dumps(obj) - except: + except Exception: if isinstance(obj, (list, tuple, set)): return "[" + ",".join(serialize(o) for o in obj) + "]" elif isinstance(obj,dict): diff --git a/maloja/globalconf.py b/maloja/globalconf.py index 16206a8..b70f075 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -28,7 +28,7 @@ def is_dir_usable(pth): os.mknod(pthj(pth,".test")) os.remove(pthj(pth,".test")) return True - except: + except Exception: return False def get_env_vars(key,pathsuffix=[]): diff --git a/maloja/images.py b/maloja/images.py index ba399f7..607885f 100644 --- a/maloja/images.py +++ b/maloja/images.py @@ -94,7 +94,7 @@ def dl_image(url): uri = datauri.DataURI.make(mime,charset='ascii',base64=True,data=data) log(f"Downloaded {url} for local caching") return uri - except: + except Exception: log(f"Image {url} could not be downloaded for local caching") return None @@ -260,7 +260,7 @@ def local_files(artist=None,artists=None,title=None): for f in os.listdir(data_dir['images'](purename)): if f.split(".")[-1] in ["png","jpg","jpeg","gif"]: images.append("/images/" + purename + "/" + f) - except: + except Exception: pass return images diff --git a/maloja/jinjaenv/filters.py b/maloja/jinjaenv/filters.py index 0e94c25..8119178 100644 --- a/maloja/jinjaenv/filters.py +++ b/maloja/jinjaenv/filters.py @@ -13,7 +13,7 @@ def find_representative(sequence,attribute_id,attribute_count): newsequence = [e for e in newsequence if e[attribute_count] == max(el[attribute_count] for el in newsequence)] return newsequence[0] - except: + except Exception: return None finally: for e in newsequence: diff --git a/maloja/malojatime.py b/maloja/malojatime.py index 5be3c72..73b1731 100644 --- a/maloja/malojatime.py +++ b/maloja/malojatime.py @@ -430,7 +430,7 @@ def time_fix(t): try: t = [int(p) for p in t] return MTRangeGregorian(t[:3]) - except: + except Exception: pass if isinstance(t[1],str) and t[1].startswith("w"): @@ -438,7 +438,7 @@ def time_fix(t): year = int(t[0]) weeknum = int(t[1][1:]) return MTRangeWeek(year=year,week=weeknum) - except: + except Exception: raise diff --git a/maloja/malojauri.py b/maloja/malojauri.py index d93caeb..6b1af6d 100644 --- a/maloja/malojauri.py +++ b/maloja/malojauri.py @@ -146,7 +146,7 @@ def remove_identical(*dicts): try: #multidicts for v in d.getall(k): keys.append(k,v) - except: #normaldicts + except Exception: #normaldicts v = d.get(k) keys.append(k,v) diff --git a/maloja/monkey.py b/maloja/monkey.py index f4780b4..78a68b4 100644 --- a/maloja/monkey.py +++ b/maloja/monkey.py @@ -11,21 +11,21 @@ try: from simplejson import JSONEncoder JSONEncoder._olddefault = JSONEncoder.default JSONEncoder.default = newdefault -except: +except Exception: pass try: from json import JSONEncoder JSONEncoder._olddefault = JSONEncoder.default JSONEncoder.default = newdefault -except: +except Exception: pass try: from ujson import JSONEncoder JSONEncoder._olddefault = JSONEncoder.default JSONEncoder.default = newdefault -except: +except Exception: pass @@ -51,7 +51,7 @@ class expandeddate(date): def fromchrcalendar(cls,y,w,d): try: return datetime.date.fromisocalendar(y,w,d) - timedelta(days=1) #sunday instead of monday - except: + except Exception: # pre python3.8 compatibility firstdayofyear = datetime.date(y,1,1) diff --git a/maloja/proccontrol/control.py b/maloja/proccontrol/control.py index 691e3c0..2442c9e 100644 --- a/maloja/proccontrol/control.py +++ b/maloja/proccontrol/control.py @@ -27,14 +27,14 @@ def getInstance(): try: output = subprocess.check_output(["pidof","Maloja"]) return int(output) - except: + except Exception: return None def getInstanceSupervisor(): try: output = subprocess.check_output(["pidof","maloja_supervisor"]) return int(output) - except: + except Exception: return None def restart(): @@ -59,7 +59,7 @@ def start(): print("\t" + col["blue"]("http://localhost:" + str(port))) print("\t" + col["blue"]("http://localhost:" + str(port) + "/admin_setup")) return True - except: + except Exception: print("Error while starting Maloja.") return False diff --git a/maloja/proccontrol/profiler.py b/maloja/proccontrol/profiler.py index 068bf61..e8342a8 100644 --- a/maloja/proccontrol/profiler.py +++ b/maloja/proccontrol/profiler.py @@ -33,7 +33,7 @@ def profile(func): if FULL_PROFILE: try: pstats.Stats(profiler).dump_stats(os.path.join(benchmarkfolder,f"{func.__name__}.stats")) - except: + except Exception: pass return result diff --git a/maloja/thirdparty/__init__.py b/maloja/thirdparty/__init__.py index e38518e..8d23109 100644 --- a/maloja/thirdparty/__init__.py +++ b/maloja/thirdparty/__init__.py @@ -230,7 +230,7 @@ class MetadataInterface(GenericInterface,abstract=True): for node in self.metadata[resp]: try: res = res[node] - except: + except Exception: return None return res diff --git a/maloja/thirdparty/musicbrainz.py b/maloja/thirdparty/musicbrainz.py index c100d3f..3dde429 100644 --- a/maloja/thirdparty/musicbrainz.py +++ b/maloja/thirdparty/musicbrainz.py @@ -57,7 +57,7 @@ class MusicBrainz(MetadataInterface): if imgurl is not None: imgurl = self.postprocess_url(imgurl) return imgurl - except: + except Exception: return None finally: time.sleep(2) diff --git a/maloja/upgrade.py b/maloja/upgrade.py index ca84001..03b6a26 100644 --- a/maloja/upgrade.py +++ b/maloja/upgrade.py @@ -37,7 +37,7 @@ def upgrade_apikeys(): for key,identifier in entries: _apikeys.apikeystore[identifier] = key os.remove(oldfile) - except: + except Exception: pass From 342b8867d97375aa21b490e11894551030e97403 Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 24 Apr 2022 20:55:07 +0200 Subject: [PATCH 11/13] Ported cache cleanup from 3.1 --- maloja/database/dbcache.py | 193 ++++++++++++++++++++----------------- maloja/globalconf.py | 4 +- 2 files changed, 106 insertions(+), 91 deletions(-) diff --git a/maloja/database/dbcache.py b/maloja/database/dbcache.py index 78174b7..57117d3 100644 --- a/maloja/database/dbcache.py +++ b/maloja/database/dbcache.py @@ -10,96 +10,97 @@ from doreah.logging import log from ..globalconf import malojaconfig -HIGH_NUMBER = 1000000 -CACHE_SIZE = 10000 -ENTITY_CACHE_SIZE = 1000000 -CACHE_ADJUST_STEP = 100 - -cache = lru.LRU(CACHE_SIZE) -entitycache = lru.LRU(ENTITY_CACHE_SIZE) - -hits, misses = 0, 0 -@runhourly -def maintenance(): - if malojaconfig['USE_GLOBAL_CACHE']: + +if malojaconfig['USE_GLOBAL_CACHE']: + CACHE_SIZE = 1000 + ENTITY_CACHE_SIZE = 100000 + + cache = lru.LRU(CACHE_SIZE) + entitycache = lru.LRU(ENTITY_CACHE_SIZE) + + hits, misses = 0, 0 + + + + @runhourly + def maintenance(): print_stats() trim_cache() -def print_stats(): - log(f"Cache Size: {len(cache)} [{len(entitycache)} E], System RAM Utilization: {psutil.virtual_memory().percent}%, Cache Hits: {hits}/{hits+misses}") - #print("Full rundown:") - #import sys - #for k in cache.keys(): - # print(f"\t{k}\t{sys.getsizeof(cache[k])}") + def print_stats(): + log(f"Cache Size: {len(cache)} [{len(entitycache)} E], System RAM Utilization: {psutil.virtual_memory().percent}%, Cache Hits: {hits}/{hits+misses}") + #print("Full rundown:") + #import sys + #for k in cache.keys(): + # print(f"\t{k}\t{sys.getsizeof(cache[k])}") -def cached_wrapper(inner_func): + def cached_wrapper(inner_func): - if not malojaconfig['USE_GLOBAL_CACHE']: return inner_func - def outer_func(*args,**kwargs): - if 'dbconn' in kwargs: - conn = kwargs.pop('dbconn') - else: - conn = None - global hits, misses - key = (serialize(args),serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to")) + def outer_func(*args,**kwargs): - if key in cache: - hits += 1 - return cache.get(key) + if 'dbconn' in kwargs: + conn = kwargs.pop('dbconn') + else: + conn = None + global hits, misses + key = (serialize(args),serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to")) + + if key in cache: + hits += 1 + return cache.get(key) + + else: + misses += 1 + result = inner_func(*args,**kwargs,dbconn=conn) + cache[key] = result + return result + + return outer_func + + + # cache for functions that call with a whole list of entity ids + # we don't want a new cache entry for every single combination, but keep a common + # cache that's aware of what we're calling + def cached_wrapper_individual(inner_func): + + + def outer_func(set_arg,**kwargs): + + + if 'dbconn' in kwargs: + conn = kwargs.pop('dbconn') + else: + conn = None + + #global hits, misses + result = {} + for id in set_arg: + if (inner_func,id) in entitycache: + result[id] = entitycache[(inner_func,id)] + #hits += 1 + else: + pass + #misses += 1 + + + remaining = inner_func(set(e for e in set_arg if e not in result),dbconn=conn) + for id in remaining: + entitycache[(inner_func,id)] = remaining[id] + result[id] = remaining[id] - else: - misses += 1 - result = inner_func(*args,**kwargs,dbconn=conn) - cache[key] = result return result - return outer_func + return outer_func - -# cache for functions that call with a whole list of entity ids -# we don't want a new cache entry for every single combination, but keep a common -# cache that's aware of what we're calling -def cached_wrapper_individual(inner_func): - - if not malojaconfig['USE_GLOBAL_CACHE']: return inner_func - def outer_func(set_arg,**kwargs): - - - if 'dbconn' in kwargs: - conn = kwargs.pop('dbconn') - else: - conn = None - - #global hits, misses - result = {} - for id in set_arg: - if (inner_func,id) in entitycache: - result[id] = entitycache[(inner_func,id)] - #hits += 1 - else: - pass - #misses += 1 - - - remaining = inner_func(set(e for e in set_arg if e not in result),dbconn=conn) - for id in remaining: - entitycache[(inner_func,id)] = remaining[id] - result[id] = remaining[id] - - return result - - return outer_func - -def invalidate_caches(scrobbletime): - if malojaconfig['USE_GLOBAL_CACHE']: + def invalidate_caches(scrobbletime=None): cleared, kept = 0, 0 for k in cache.keys(): # VERY BIG TODO: differentiate between None as in 'unlimited timerange' and None as in 'time doesnt matter here'! - if (k[3] is None or scrobbletime >= k[3]) and (k[4] is None or scrobbletime <= k[4]): + if scrobbletime is None or (k[3] is None or scrobbletime >= k[3]) and (k[4] is None or scrobbletime <= k[4]): cleared += 1 del cache[k] else: @@ -107,28 +108,42 @@ def invalidate_caches(scrobbletime): log(f"Invalidated {cleared} of {cleared+kept} DB cache entries") -def invalidate_entity_cache(): - entitycache.clear() + def invalidate_entity_cache(): + entitycache.clear() -def trim_cache(): - ramprct = psutil.virtual_memory().percent - if ramprct > malojaconfig["DB_MAX_MEMORY"]: - log(f"{ramprct}% RAM usage, clearing cache and adjusting size!") - #ratio = 0.6 - #targetsize = max(int(len(cache) * ratio),50) - #log(f"Reducing to {targetsize} entries") - #cache.set_size(targetsize) - #cache.set_size(HIGH_NUMBER) - cache.clear() - if cache.get_size() > CACHE_ADJUST_STEP: - cache.set_size(cache.get_size() - CACHE_ADJUST_STEP) + def trim_cache(): + ramprct = psutil.virtual_memory().percent + if ramprct > malojaconfig["DB_MAX_MEMORY"]: + log(f"{ramprct}% RAM usage, clearing cache and adjusting size!") + #ratio = 0.6 + #targetsize = max(int(len(cache) * ratio),50) + #log(f"Reducing to {targetsize} entries") + #cache.set_size(targetsize) + #cache.set_size(HIGH_NUMBER) + cache.clear() + #if cache.get_size() > CACHE_ADJUST_STEP: + # cache.set_size(cache.get_size() - CACHE_ADJUST_STEP) - #log(f"New RAM usage: {psutil.virtual_memory().percent}%") - print_stats() + #log(f"New RAM usage: {psutil.virtual_memory().percent}%") + print_stats() + + + +else: + def cached_wrapper(func): + return func + def cached_wrapper_individual(func): + return func + def invalidate_caches(scrobbletime=None): + return None + def invalidate_entity_cache(): + return None + + def serialize(obj): try: return serialize(obj.hashable()) diff --git a/maloja/globalconf.py b/maloja/globalconf.py index b70f075..2ab8f09 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -149,8 +149,8 @@ malojaconfig = Configuration( "cache_expire_positive":(tp.Integer(), "Image Cache Expiration", 60, "Days until images are refetched"), "cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 5, "Days until failed image fetches are reattempted"), "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 80, "RAM Usage in percent at which Maloja should no longer increase its database cache."), - "use_request_cache":(tp.Boolean(), "Use request-local DB Cache", True), - "use_global_cache":(tp.Boolean(), "Use global DB Cache", True) + "use_request_cache":(tp.Boolean(), "Use request-local DB Cache", False), + "use_global_cache":(tp.Boolean(), "Use global DB Cache", False) }, "Fluff":{ "scrobbles_gold":(tp.Integer(), "Scrobbles for Gold", 250, "How many scrobbles a track needs to be considered 'Gold' status"), From e470e2e43fc855997a6e17125cca76a1b49f0e9d Mon Sep 17 00:00:00 2001 From: krateng Date: Mon, 25 Apr 2022 02:42:07 +0200 Subject: [PATCH 12/13] Potentially fixed nonsensical caching, GH-132 --- maloja/malojatime.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maloja/malojatime.py b/maloja/malojatime.py index 73b1731..28ac383 100644 --- a/maloja/malojatime.py +++ b/maloja/malojatime.py @@ -320,7 +320,8 @@ class MTRangeComposite(MTRangeGeneric): if self.since is None: return FIRST_SCROBBLE else: return self.since.first_stamp() def last_stamp(self): - if self.to is None: return int(datetime.utcnow().replace(tzinfo=timezone.utc).timestamp()) + #if self.to is None: return int(datetime.utcnow().replace(tzinfo=timezone.utc).timestamp()) + if self.to is None: return today().last_stamp() else: return self.to.last_stamp() def next(self,step=1): From 95f98370cfdd45d35582d07a39d10eea455eab77 Mon Sep 17 00:00:00 2001 From: krateng Date: Mon, 25 Apr 2022 02:47:03 +0200 Subject: [PATCH 13/13] Updated branch notes --- dev/releases/branch.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 dev/releases/branch.yml diff --git a/dev/releases/branch.yml b/dev/releases/branch.yml new file mode 100644 index 0000000..61415ea --- /dev/null +++ b/dev/releases/branch.yml @@ -0,0 +1,3 @@ +- "[Bugix] Improved signal handling" +- "[Bugix] Fixed constant re-caching of all-time stats" +- "[Performance] Disabled caches per default"