Merge branch 'master' into feature-webedit

Build image for raspberry pi 2 (arm/v7)

.github/workflows/docker.yml

@@ -0,0 +1,76 @@
+name: Build and release docker image
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+
+      - name: Log in to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@f2a13332ac1ce8c0a71aeac48a150dbb1838ab67
+        with:
+          images: |
+            ${{ github.repository_owner }}/maloja
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=semver,pattern={{version}}
+          flavor: |
+            latest=true
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
+
+      - name: Cache Docker layers
+        uses: actions/cache@48af2dc4a9e8278b89d7fa154b955c30c6aaab09
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        with:
+          context: .
+          file: Containerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+      - name: Update Readme and short description
+        uses: peter-evans/dockerhub-description@836d7e6aa8f6f32dce26f5a1dd46d3dc24997eae
+        continue-on-error: true
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
+          repository: krateng/maloja
+          short-description: ${{ github.event.repository.description }}

.github/workflows/dockerhub.yml

@@ -1,51 +0,0 @@
-name: Publish to Dockerhub
-
-on:
-  push:
-    tags:
-      - 'v*'
-
-jobs:
-  push_to_registry:
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
-
-      - name: Log in to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681
-        with:
-          images: krateng/maloja
-          # generate Docker tags based on the following events/attributes
-          tags: |
-            type=semver,pattern={{version}}
-          flavor: |
-            latest=true
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@7f9d37fa544684fb73bfe4835ed7214c255ce02b
-        with:
-          context: .
-          file: Containerfile
-          #platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-
-      - name: Update Readme and short description
-        uses: peter-evans/dockerhub-description@1cf9afbac3c5d2fdc66416a464c2c38260cb6f8d
-        continue-on-error: true
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
-          repository: krateng/maloja
-          short-description: ${{ github.event.repository.description }}

Containerfile

@@ -6,7 +6,7 @@ FROM alpine:3.15
 WORKDIR /usr/src/app
 
 # Install run dependencies first
-RUN apk add --no-cache python3 tzdata
+RUN apk add --no-cache python3 py3-lxml tzdata
 
 # system pip could be removed after build, but apk then decides to also remove all its
 # python dependencies, even if they are explicitly installed as python packages
@@ -15,16 +15,7 @@ RUN \
 	apk add py3-pip && \
 	pip install wheel
 
-# these are more static than the real requirements, which means caching
-COPY ./requirements_pre.txt ./requirements_pre.txt
 
-RUN \
-	apk add --no-cache --virtual .build-deps gcc g++ python3-dev libxml2-dev libxslt-dev libffi-dev libc-dev py3-pip linux-headers && \
-	pip install --no-cache-dir -r requirements_pre.txt && \
-	apk del .build-deps
-
-
-# less likely to be cached
 COPY ./requirements.txt ./requirements.txt
 
 RUN \

dev/releases/3.0.yml

@@ -33,3 +33,12 @@ minor_release_name: "Yeonhee"
     notes:
     - "[Feature] Added notification system for web interface"
     - "[Bugfix] Fixed crash when encountering error in Lastfm import"
+3.0.6:
+    commit: "b3d4cb7a153845d1f5a5eef67a6508754e338f2f"
+    notes:
+    - "[Performance] Implemented search in database"
+    - "[Bugfix] Better parsing of featuring artists"
+    - "[Bugfix] Fixed buffered output in Docker"
+    - "[Bugfix] Fixed importing a Spotify file without path"
+    - "[Bugfix] No longer releasing database lock during scrobble creation"
+    - "[Distribution] Experimental arm64 image"

dev/templates/Containerfile.jinja

@@ -15,16 +15,7 @@ RUN \
 	apk add py3-pip && \
 	pip install wheel
 
-# these are more static than the real requirements, which means caching
-COPY ./requirements_pre.txt ./requirements_pre.txt
 
-RUN \
-	apk add --no-cache --virtual .build-deps {{ tool.osreqs.alpine.build | join(' ') }} && \
-	pip install --no-cache-dir -r requirements_pre.txt && \
-	apk del .build-deps
-
-
-# less likely to be cached
 COPY ./requirements.txt ./requirements.txt
 
 RUN \

dev/testing/stresstest.py

@@ -0,0 +1,43 @@
+import threading
+import subprocess
+import time
+import requests
+import os
+
+ACTIVE = True
+
+build_cmd = ["docker","build","-t","maloja",".","-f","Containerfile"]
+subprocess.run(build_cmd)
+
+common_prc = (
+	["docker","run","--rm","-v",f"{os.path.abspath('./testdata')}:/mlj","-e","MALOJA_DATA_DIRECTORY=/mlj"],
+	["maloja"]
+)
+
+servers = [
+	{'port': 42010},
+	{'port': 42011,	'extraargs':["--memory=1g"]},
+	{'port': 42012,	'extraargs':["--memory=500m"]}
+]
+for s in servers:
+	cmd = common_prc[0] + ["-p",f"{s['port']}:42010"] + s.get('extraargs',[]) + common_prc[1]
+	print(cmd)
+	t = threading.Thread(target=subprocess.run,args=(cmd,))
+	s['thread'] = t
+	t.daemon = True
+	t.start()
+	time.sleep(5)
+
+time.sleep(5)
+while ACTIVE:
+	time.sleep(1)
+	try:
+		for s in servers:
+			requests.get(f"http://localhost:{s['port']}")
+	except KeyboardInterrupt:
+		ACTIVE = False
+	except Exception:
+		pass
+
+for s in servers:
+	s['thread'].join()

dev/write_tags.py

@@ -43,7 +43,7 @@ for version in releases:
 	try:
 		prev_tag = sp.check_output(["git","show",f'v{maj}.{min}.{hot}']).decode()
 		prev_tag_commit = prev_tag.split('\n')[6].split(" ")[1]
-	except:
+	except Exception:
 		pass
 	else:
 		assert prev_tag_commit == info['commit']

install/install_alpine.sh

@@ -11,6 +11,7 @@ apk add \
 	py3-pip \
 	linux-headers \
 	python3 \
+	py3-lxml \
 	tzdata \
 	vips
 

install/install_dependencies_alpine.sh

@@ -11,5 +11,6 @@ apk add \
 	py3-pip \
 	linux-headers \
 	python3 \
+	py3-lxml \
 	tzdata \
 	vips

maloja/__pkginfo__.py

@@ -4,7 +4,7 @@
 # you know what f*ck it
 # this is hardcoded for now because of that damn project / package name discrepancy
 # i'll fix it one day
-VERSION = "3.0.5"
+VERSION = "3.0.6"
 HOMEPAGE = "https://github.com/krateng/maloja"
 
 

maloja/malojatime.py

@@ -320,7 +320,8 @@ class MTRangeComposite(MTRangeGeneric):
 		if self.since is None: return FIRST_SCROBBLE
 		else: return self.since.first_stamp()
 	def last_stamp(self):
-		if self.to is None: return int(datetime.utcnow().replace(tzinfo=timezone.utc).timestamp())
+		#if self.to is None: return int(datetime.utcnow().replace(tzinfo=timezone.utc).timestamp())
+		if self.to is None: return today().last_stamp()
 		else: return self.to.last_stamp()
 
 	def next(self,step=1):

maloja/pkg_global/conf.py

@@ -149,8 +149,8 @@ malojaconfig = Configuration(
 			"cache_expire_positive":(tp.Integer(),								"Image Cache Expiration", 				60,		"Days until images are refetched"),
 			"cache_expire_negative":(tp.Integer(),								"Image Cache Negative Expiration",		5,		"Days until failed image fetches are reattempted"),
 			"db_max_memory":(tp.Integer(min=0,max=100),							"RAM Percentage soft limit",			80,		"RAM Usage in percent at which Maloja should no longer increase its database cache."),
-			"use_request_cache":(tp.Boolean(),									"Use request-local DB Cache",			True),
-			"use_global_cache":(tp.Boolean(),									"Use global DB Cache",					True)
+			"use_request_cache":(tp.Boolean(),									"Use request-local DB Cache",			False),
+			"use_global_cache":(tp.Boolean(),									"Use global DB Cache",					False)
 		},
 		"Fluff":{
 			"scrobbles_gold":(tp.Integer(),										"Scrobbles for Gold",			250,				"How many scrobbles a track needs to be considered 'Gold' status"),

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "malojaserver"
-version = "3.0.5"
+version = "3.0.6"
 description = "Self-hosted music scrobble database"
 readme = "./README.md"
 requires-python = ">=3.7"
@@ -62,6 +62,7 @@ build =[
 ]
 run = [
 	"python3",
+	"py3-lxml",
 	"tzdata"
 ]
 opt = [

requirements_pre.txt

@@ -1,3 +0,0 @@
-# this is a more static file that enables container images to be cached
-# it should contain packages that take long to build and don't change frequently
-lxml