diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5faabd8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +* +!maloja +!Containerfile +!requirements_pre.txt +!requirements.txt +!pyproject.toml +!README.md +!LICENSE diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..393f47d --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: ["https://flattr.com/@Krateng", "https://paypal.me/krateng", "bitcoin:1krat8JMniJBTiHftMfR1LtF3Y1w5DAxx"] diff --git a/.github/workflows/aux.library.pypi.yml b/.github/workflows/aux.library.pypi.yml index 123b843..97841d8 100644 --- a/.github/workflows/aux.library.pypi.yml +++ b/.github/workflows/aux.library.pypi.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@7f80679172b057fc5e90d70d197929d454754a5a with: python-version: '3.x' @@ -30,7 +30,7 @@ jobs: run: python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@717ba43cfbb0387f6ce311b169a825772f54d295 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/aux.scrobbler.upload.yml b/.github/workflows/aux.scrobbler.upload.yml index 528ee9b..d65cf6d 100644 --- a/.github/workflows/aux.scrobbler.upload.yml +++ b/.github/workflows/aux.scrobbler.upload.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Push Extension to Web Store - uses: Klemensas/chrome-extension-upload-action@$VERSION + uses: Klemensas/chrome-extension-upload-action@1e8ede84548583abf1a2a495f4242c4c51539337 with: refresh-token: '${{ secrets.GOOGLE_REFRESHTOKEN }}' client-id: '${{ secrets.GOOGLE_CLIENTID }}' diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml index 534be43..f20b655 100644 --- a/.github/workflows/dockerhub.yml +++ b/.github/workflows/dockerhub.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - name: Log in to Docker Hub if: github.event_name != 'pull_request' @@ -22,7 +22,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 with: images: krateng/maloja # generate Docker tags based on the following events/attributes @@ -32,19 +32,20 @@ jobs: latest=true - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@7f9d37fa544684fb73bfe4835ed7214c255ce02b with: context: . file: Containerfile + #platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} -# skipping this until https://github.com/peter-evans/dockerhub-description/issues/10 is fixed -# - name: Update Readme and short description -# uses: peter-evans/dockerhub-description@v2 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} -# repository: krateng/maloja -# short-description: ${{ github.event.repository.description }} + - name: Update Readme and short description + uses: peter-evans/dockerhub-description@1cf9afbac3c5d2fdc66416a464c2c38260cb6f8d + continue-on-error: true + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + repository: krateng/maloja + short-description: ${{ github.event.repository.description }} diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 67fd36a..f067ea1 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@7f80679172b057fc5e90d70d197929d454754a5a with: python-version: '3.x' @@ -25,7 +25,7 @@ jobs: run: python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@717ba43cfbb0387f6ce311b169a825772f54d295 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index e0bc5d9..7e6010c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,21 +1,14 @@ -# generic temporary / dev files +# temporary / generated files *.pyc -*.note -*.xcf -/nohup.out -/*-old -/*.yml -/pylintrc -.venv/* -/testdata* -# build +# environments / builds +.venv/* +testdata* /dist /build /*.egg-info -# currently not using -/screenshot*.png - -# only for development, normally external -/doreah +# dev files +*.xcf +*.note +*-old diff --git a/APKBUILD b/APKBUILD new file mode 100644 index 0000000..4f27581 --- /dev/null +++ b/APKBUILD @@ -0,0 +1,36 @@ +# Contributor: Johannes Krattenmacher +# Maintainer: Johannes Krattenmacher +pkgname=maloja +pkgver=3.0.0-dev +pkgrel=0 +pkgdesc="Self-hosted music scrobble database" +url="https://github.com/krateng/maloja" +arch="noarch" +license="GPL-3.0" +depends="python3 tzdata" +pkgusers=$pkgname +pkggroups=$pkgname +depends_dev="gcc g++ python3-dev libxml2-dev libxslt-dev libffi-dev libc-dev py3-pip linux-headers" +makedepends="$depends_dev" +source=" + $pkgname-$pkgver.tar.gz::https://github.com/krateng/maloja/archive/refs/tags/v$pkgver.tar.gz +" +builddir="$srcdir"/$pkgname-$pkgver + + + +build() { + cd $builddir + python3 -m build . + pip3 install dist/*.tar.gz +} + +package() { + mkdir -p /etc/$pkgname || return 1 + mkdir -p /var/lib/$pkgname || return 1 + mkdir -p /var/cache/$pkgname || return 1 + mkdir -p /var/logs/$pkgname || return 1 +} + +# TODO +sha512sums="a674eaaaa248fc2b315514d79f9a7a0bac6aa1582fe29554d9176e8b551e8aa3aa75abeebdd7713e9e98cc987e7bd57dc7a5e9a2fb85af98b9c18cb54de47bf7 $pkgname-${pkgver}.tar.gz" diff --git a/Containerfile b/Containerfile index 2c8a278..22b0b72 100644 --- a/Containerfile +++ b/Containerfile @@ -1,29 +1,47 @@ -FROM python:3-alpine +FROM alpine:3.15 +# Python image includes two Python versions, so use base Alpine # Based on the work of Jonathan Boeckel -# https://gitlab.com/Joniator/docker-maloja -# https://github.com/Joniator WORKDIR /usr/src/app +# Install run dependencies first +RUN apk add --no-cache python3 tzdata -# Copy project into dir -COPY . . +# system pip could be removed after build, but apk then decides to also remove all its +# python dependencies, even if they are explicitly installed as python packages +# whut +RUN \ + apk add py3-pip && \ + pip install wheel + +# these are more static than the real requirements, which means caching +COPY ./requirements_pre.txt ./requirements_pre.txt RUN \ - # Build dependencies (This will pipe all packages from the file) - sed 's/#.*//' ./install/dependencies_build.txt | xargs apk add --no-cache --virtual .build-deps && \ - # Runtime dependencies (Same) - sed 's/#.*//' ./install/dependencies_run.txt | xargs apk add --no-cache && \ - # Python dependencies - pip3 install --no-cache-dir -r requirements.txt && \ - # Local project install - pip3 install /usr/src/app && \ - # Remove build dependencies - apk del .build-deps + apk add --no-cache --virtual .build-deps gcc g++ python3-dev libxml2-dev libxslt-dev libffi-dev libc-dev py3-pip linux-headers && \ + pip install --no-cache-dir -r requirements_pre.txt && \ + apk del .build-deps -# expected behavior for a default setup is for maloja to "just work" + +# less likely to be cached +COPY ./requirements.txt ./requirements.txt + +RUN \ + apk add --no-cache --virtual .build-deps gcc g++ python3-dev libxml2-dev libxslt-dev libffi-dev libc-dev py3-pip linux-headers && \ + pip install --no-cache-dir -r requirements.txt && \ + apk del .build-deps + + +# no chance for caching below here + +COPY . . + +RUN pip install /usr/src/app + +# Docker-specific configuration and default to IPv4 ENV MALOJA_SKIP_SETUP=yes +ENV MALOJA_HOST=0.0.0.0 EXPOSE 42010 # use exec form for better signal handling https://docs.docker.com/engine/reference/builder/#entrypoint diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 748cc39..b38de80 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -11,10 +11,10 @@ Clone the repository and enter it. To avoid cluttering your system, consider using a [virtual environment](https://docs.python.org/3/tutorial/venv.html). -Your system needs several packages installed. On Alpine, this can be done with +Your system needs several packages installed. For supported distributions, this can be done with e.g. ```console - sh ./install/install_dependencies.sh + sh ./install/install_dependencies_alpine.sh ``` For other distros, try to find the equivalents of the packages listed or simply check your error output. @@ -41,6 +41,18 @@ You can also build the package with ```console pip install . ``` + + +## Docker + +You can also always build and run the server with + +```console + sh ./dev/run_docker.sh +``` + +This will use the directory `testdata`. + ## Further help Feel free to [ask](https://github.com/krateng/maloja/discussions) if you need some help! diff --git a/Dockerfile-pypi b/Dockerfile-pypi deleted file mode 100644 index b68e430..0000000 --- a/Dockerfile-pypi +++ /dev/null @@ -1,31 +0,0 @@ -FROM python:3-alpine - -# Based on the work of Jonathan Boeckel -# https://gitlab.com/Joniator/docker-maloja -# https://github.com/Joniator - -ARG MALOJA_RELEASE -WORKDIR /usr/src/app - -# Copy needed information -COPY ./install ./install - -RUN \ - # Build dependencies (This will pipe all packages from the file) - sed 's/#.*//' ./install/dependencies_build.txt | xargs apk add --no-cache --virtual .build-deps && \ - # Runtime dependencies (Same) - sed 's/#.*//' ./install/dependencies_run.txt | xargs apk add --no-cache && \ - - - # PyPI install - pip3 install malojaserver==$MALOJA_RELEASE && \ - # Remove build dependencies - apk del .build-deps - -# expected behavior for a default setup is for maloja to "just work" -ENV MALOJA_SKIP_SETUP=yes -ENV MAGICK_HOME=/usr - -EXPOSE 42010 -# use exec form for better signal handling https://docs.docker.com/engine/reference/builder/#entrypoint -ENTRYPOINT ["maloja", "run"] diff --git a/README.md b/README.md index 97750e0..ae4c17c 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,26 @@ # Maloja -[![](https://img.shields.io/github/v/tag/krateng/maloja?label=GitHub&style=for-the-badge)](https://github.com/krateng/maloja) -[![](https://img.shields.io/pypi/v/malojaserver?label=PyPI&style=for-the-badge)](https://pypi.org/project/malojaserver/) -[![](https://img.shields.io/docker/v/krateng/maloja?label=Docker&style=for-the-badge)](https://hub.docker.com/r/krateng/maloja) +[![](https://img.shields.io/github/v/tag/krateng/maloja?label=GitHub&style=for-the-badge&logo=github&logoColor=white)](https://github.com/krateng/maloja) +[![](https://img.shields.io/pypi/v/malojaserver?label=PyPI&style=for-the-badge&logo=pypi&logoColor=white)](https://pypi.org/project/malojaserver/) +[![](https://img.shields.io/docker/v/krateng/maloja?label=Dockerhub&style=for-the-badge&logo=docker&logoColor=white)](https://hub.docker.com/r/krateng/maloja) [![](https://img.shields.io/pypi/l/malojaserver?style=for-the-badge)](https://github.com/krateng/maloja/blob/master/LICENSE) [![](https://img.shields.io/codeclimate/maintainability/krateng/maloja?style=for-the-badge)](https://codeclimate.com/github/krateng/maloja) Simple self-hosted music scrobble database to create personal listening statistics. No recommendations, no social network, no nonsense. -You can check [my own Maloja page](https://maloja.krateng.ch) to see what it looks like (it's down fairly often because I use it as staging environment, that doesn't reflect the stability of the Maloja software!). +![screenshot](screenshot.png?raw=true) -> If you're a new Maloja user, consider installing [Version 3 Beta](https://github.com/krateng/maloja/tree/v3) right away. This way, you avoid having to upgrade later and you might help catch some bugs. -> -> Simply clone this repository and change branches with `git checkout v3`, then follow the [Readme](https://github.com/krateng/maloja/blob/v3/README.md) of that branch to install from source or run the included Containerfile. -> -> Thank you for your help testing the new release! +You can check [my own Maloja page](https://maloja.krateng.ch) as an example instance. ## Table of Contents * [Features](#features) * [How to install](#how-to-install) - * [LXC / VM / Bare Metal](#lxc--vm--bare-metal) - * [Docker](#docker) + * [Requirements](#requirements) + * [PyPI](#pypi) + * [From Source](#from-source) + * [Docker / Podman](#docker-podman) * [Extras](#extras) * [How to use](#how-to-use) * [Basic control](#basic-control) @@ -49,27 +47,25 @@ You can check [my own Maloja page](https://maloja.krateng.ch) to see what it loo ## How to install -### LXC / VM / Bare Metal +### Requirements + +Maloja should run on any x86 or ARM machine that runs Python. I can support you with issues best if you use **Alpine Linux**. -#### From PyPI +Your CPU should have a single core passmark score of at the very least 1500. 500 MB RAM should give you a decent experience, but performance will benefit greatly from up to 2 GB. -You can download the included script `install_alpine.sh` and run it with +### PyPI -```console - sh install_alpine.sh -``` - -You can also simply call the install command +You can install Maloja with ```console pip install malojaserver ``` -directly (e.g. if you're not on Alpine) - make sure you have all the system packages installed. +To make sure all dependencies are installed, you can also use one of the included scripts in the `install` folder. -#### From Source +### From Source Clone this repository and enter the directory with @@ -78,15 +74,15 @@ Clone this repository and enter the directory with cd maloja ``` -Then install all the requirements and build the package: +Then install all the requirements and build the package, e.g.: ```console - sh ./install/install_dependencies.sh + sh ./install/install_dependencies_alpine.sh pip install -r requirements.txt pip install . ``` -### Docker +### Docker / Podman Pull the [latest image](https://hub.docker.com/r/krateng/maloja) or check out the repository and use the included Containerfile. @@ -95,21 +91,20 @@ Of note are these settings which should be passed as environmental variables to * `MALOJA_DATA_DIRECTORY` -- Set the directory in the container where configuration folders/files should be located * Mount a [volume](https://docs.docker.com/engine/reference/builder/#volume) to the specified directory to access these files outside the container (and to make them persistent) * `MALOJA_FORCE_PASSWORD` -- Set an admin password for maloja -* `MALOJA_HOST` (Optional) -- Maloja uses IPv6 by default for the host. Set this to `0.0.0.0` if you cannot initially access the webserver -You must also publish a port on your host machine to bind to the container's web port (default 42010) +You must publish a port on your host machine to bind to the container's web port (default 42010). The container uses IPv4 per default. -An example of a minimum run configuration when accessing maloja from an IPv4 address IE `localhost:42010`: +An example of a minimum run configuration to access maloja via `localhost:42010`: ```console - docker run -p 42010:42010 -e MALOJA_HOST=0.0.0.0 maloja + docker run -p 42010:42010 -v $PWD/malojadata:/mljdata -e MALOJA_DATA_DIRECTORY=/mljdata maloja ``` ### Extras * If you'd like to display images, you will need API keys for [Last.fm](https://www.last.fm/api/account/create) and [Spotify](https://developer.spotify.com/dashboard/applications). These are free of charge! -* Put your server behind a reverse proxy for SSL encryption. Make sure that you're proxying to the IPv6 address unless you changed your settings to use IPv4. +* Put your server behind a reverse proxy for SSL encryption. Make sure that you're proxying to the IPv6 or IPv4 address according to your settings. * You can set up a cronjob to start your server on system boot, and potentially restart it on a regular basis: @@ -140,11 +135,13 @@ If you need to run the server in the foreground, use ### Data -* If you would like to import all your previous last.fm scrobbles, use [benfoxall's website](https://benjaminbenben.com/lastfm-to-csv/) ([GitHub page](https://github.com/benfoxall/lastfm-to-csv)). Use the command `maloja import *filename*` to import the downloaded file into Maloja. +If you would like to import your previous scrobbles, use the command `maloja import *filename*`. This works on: -* To backup your data, run `maloja backup` or, to only backup essential data (no artwork etc), `maloja backup -l minimal`. +* a Last.fm export generated by [benfoxall's website](https://benjaminbenben.com/lastfm-to-csv/) ([GitHub page](https://github.com/benfoxall/lastfm-to-csv)) +* an official [Spotify data export file](https://www.spotify.com/us/account/privacy/) +* the export of another Maloja instance -* To fix your database (e.g. after you've added new rules), use `maloja fix`. +To backup your data, run `maloja backup`, optional with `--include_images`. ### Customization @@ -157,7 +154,7 @@ If you need to run the server in the foreground, use ## How to scrobble -You can set up any amount of API keys in the file `authenticated_machines.tsv` in the `/etc/maloja/clients` folder. It is recommended to define a different API key for every scrobbler you use. +You can set up any amount of API keys in the file `apikeys.yml` in your configuration folder (or via the web interface). It is recommended to define a different API key for every scrobbler you use. ### Native support @@ -167,7 +164,6 @@ These solutions allow you to directly setup scrobbling to your Maloja server: * [Multi Scrobbler](https://github.com/FoxxMD/multi-scrobbler) Desktop Application * [Cmus-maloja-scrobbler](https://git.sr.ht/~xyank/cmus-maloja-scrobbler) Script * [OngakuKiroku](https://github.com/Atelier-Shiori/OngakuKiroku) Desktop Application (Mac) -* [Albula](https://github.com/krateng/albula) Music Server * [Maloja Scrobbler](https://chrome.google.com/webstore/detail/maloja-scrobbler/cfnbifdmgbnaalphodcbandoopgbfeeh) Chromium Extension (also included in the repository) for Plex Web, Spotify, Bandcamp, Soundcloud or Youtube Music ### Native API diff --git a/auxiliary/chromium_scrobbler/maloja-scrobbler/background.js b/auxiliary/chromium_scrobbler/maloja-scrobbler/background.js index a6353b9..d12ca99 100644 --- a/auxiliary/chromium_scrobbler/maloja-scrobbler/background.js +++ b/auxiliary/chromium_scrobbler/maloja-scrobbler/background.js @@ -6,6 +6,10 @@ chrome.runtime.onMessage.addListener(onInternalMessage); tabManagers = {} + +const ALWAYS_SCROBBLE_SECONDS = 60*3; +// Longer songs are always scrobbled when playing at least 2 minutes + pages = { "Plex Web":{ "patterns":[ @@ -282,7 +286,7 @@ class Controller { //ONLY CASE 2: Playback ended if (artist != this.currentArtist || title != this.currentTitle) { - if (this.alreadyPlayed > this.currentLength / 2) { + if ((this.alreadyPlayed > this.currentLength / 2) || (this.alreadyPlayed > ALWAYS_SCROBBLE_SECONDS)) { scrobble(this.currentArtist,this.currentTitle,this.alreadyPlayed) this.alreadyPlayed = 0 } diff --git a/auxiliary/chromium_scrobbler/maloja-scrobbler/sitescript.js b/auxiliary/chromium_scrobbler/maloja-scrobbler/sitescript.js index f24969d..b8a2af9 100644 --- a/auxiliary/chromium_scrobbler/maloja-scrobbler/sitescript.js +++ b/auxiliary/chromium_scrobbler/maloja-scrobbler/sitescript.js @@ -32,7 +32,7 @@ Node.prototype.xpath = getxpath; bar = document.xpath(maloja_scrobbler_selector_playbar, XPathResult.FIRST_ORDERED_NODE_TYPE); if (bar == null) { - console.log("Nothing playing right now!"); + console.log("[Maloja Scrobbler] Nothing playing right now!"); chrome.runtime.sendMessage({type:"stopPlayback",time:Math.floor(Date.now()),artist:"",title:""}); } else { @@ -78,12 +78,12 @@ else { label_paused = "Play" } if (control == label_paused) { - console.log("Not playing right now"); + console.log("[Maloja Scrobbler] Not playing right now"); chrome.runtime.sendMessage({type:"stopPlayback",time:Math.floor(Date.now()),artist:artist,title:title}); //stopPlayback() } else if (control == label_playing) { - console.log("Playing " + artist + " - " + title + " (" + durationSeconds + " sec)"); + console.log("[Maloja Scrobbler] Playing " + artist + " - " + title + " (" + durationSeconds + " sec)"); chrome.runtime.sendMessage({type:"startPlayback",time:Math.floor(Date.now()),artist:artist,title:title,duration:durationSeconds}); //startPlayback(artist,title,durationSeconds) } diff --git a/auxiliary/malojalib/pyproject.toml b/auxiliary/malojalib/pyproject.toml index 0b2eb1a..de4ca35 100644 --- a/auxiliary/malojalib/pyproject.toml +++ b/auxiliary/malojalib/pyproject.toml @@ -21,7 +21,6 @@ dependencies = [ "requests" ] -[project.scripts] [build-system] requires = ["flit_core >=3.2,<4"] diff --git a/dev-scripts/package.py b/dev/package.py similarity index 100% rename from dev-scripts/package.py rename to dev/package.py diff --git a/dev/run_docker.sh b/dev/run_docker.sh new file mode 100644 index 0000000..10c58b6 --- /dev/null +++ b/dev/run_docker.sh @@ -0,0 +1,2 @@ +docker build -t maloja . -f Containerfile +docker run --rm -p 42010:42010 -v $PWD/testdata:/mlj -e MALOJA_DATA_DIRECTORY=/mlj maloja diff --git a/dev/run_podman.sh b/dev/run_podman.sh new file mode 100644 index 0000000..aebbfee --- /dev/null +++ b/dev/run_podman.sh @@ -0,0 +1,2 @@ +podman build -t maloja . -f Containerfile +podman run --rm -p 42010:42010 -v $PWD/testdata:/mlj -e MALOJA_DATA_DIRECTORY=/mlj maloja diff --git a/dev/templates/APKBUILD.jinja b/dev/templates/APKBUILD.jinja new file mode 100644 index 0000000..42e70f0 --- /dev/null +++ b/dev/templates/APKBUILD.jinja @@ -0,0 +1,36 @@ +# Contributor: Johannes Krattenmacher +# Maintainer: Johannes Krattenmacher +pkgname={{ tool.flit.module.name }} +pkgver={{ project.version }} +pkgrel=0 +pkgdesc="{{ project.description }}" +url="{{ project.urls.homepage }}" +arch="noarch" +license="GPL-3.0" +depends="{{ tool.osreqs.alpine.run | join(' ') }}" +pkgusers=$pkgname +pkggroups=$pkgname +depends_dev="{{ tool.osreqs.alpine.build | join(' ') }}" +makedepends="$depends_dev" +source=" + $pkgname-$pkgver.tar.gz::{{ project.urls.repository }}/archive/refs/tags/v$pkgver.tar.gz +" +builddir="$srcdir"/$pkgname-$pkgver + + + +build() { + cd $builddir + python3 -m build . + pip3 install dist/*.tar.gz +} + +package() { + mkdir -p /etc/$pkgname || return 1 + mkdir -p /var/lib/$pkgname || return 1 + mkdir -p /var/cache/$pkgname || return 1 + mkdir -p /var/logs/$pkgname || return 1 +} + +# TODO +sha512sums="a674eaaaa248fc2b315514d79f9a7a0bac6aa1582fe29554d9176e8b551e8aa3aa75abeebdd7713e9e98cc987e7bd57dc7a5e9a2fb85af98b9c18cb54de47bf7 $pkgname-${pkgver}.tar.gz" diff --git a/dev/templates/Containerfile.jinja b/dev/templates/Containerfile.jinja new file mode 100644 index 0000000..e180abd --- /dev/null +++ b/dev/templates/Containerfile.jinja @@ -0,0 +1,48 @@ +FROM alpine:3.15 +# Python image includes two Python versions, so use base Alpine + +# Based on the work of Jonathan Boeckel + +WORKDIR /usr/src/app + +# Install run dependencies first +RUN apk add --no-cache {{ tool.osreqs.alpine.run | join(' ') }} + +# system pip could be removed after build, but apk then decides to also remove all its +# python dependencies, even if they are explicitly installed as python packages +# whut +RUN \ + apk add py3-pip && \ + pip install wheel + +# these are more static than the real requirements, which means caching +COPY ./requirements_pre.txt ./requirements_pre.txt + +RUN \ + apk add --no-cache --virtual .build-deps {{ tool.osreqs.alpine.build | join(' ') }} && \ + pip install --no-cache-dir -r requirements_pre.txt && \ + apk del .build-deps + + +# less likely to be cached +COPY ./requirements.txt ./requirements.txt + +RUN \ + apk add --no-cache --virtual .build-deps {{ tool.osreqs.alpine.build | join(' ') }} && \ + pip install --no-cache-dir -r requirements.txt && \ + apk del .build-deps + + +# no chance for caching below here + +COPY . . + +RUN pip install /usr/src/app + +# Docker-specific configuration and default to IPv4 +ENV MALOJA_SKIP_SETUP=yes +ENV MALOJA_HOST=0.0.0.0 + +EXPOSE 42010 +# use exec form for better signal handling https://docs.docker.com/engine/reference/builder/#entrypoint +ENTRYPOINT ["maloja", "run"] diff --git a/dev/templates/install/install_alpine.sh.jinja b/dev/templates/install/install_alpine.sh.jinja new file mode 100644 index 0000000..021ebc0 --- /dev/null +++ b/dev/templates/install/install_alpine.sh.jinja @@ -0,0 +1,4 @@ +{% include 'install/install_dependencies_alpine.sh.jinja' %} +apk add py3-pip +pip install wheel +pip install malojaserver diff --git a/dev/templates/install/install_debian.sh.jinja b/dev/templates/install/install_debian.sh.jinja new file mode 100644 index 0000000..bd3f903 --- /dev/null +++ b/dev/templates/install/install_debian.sh.jinja @@ -0,0 +1,4 @@ +{% include 'install/install_dependencies_debian.sh.jinja' %} +apt install python3-pip +pip install wheel +pip install malojaserver diff --git a/dev/templates/install/install_dependencies_alpine.sh.jinja b/dev/templates/install/install_dependencies_alpine.sh.jinja new file mode 100644 index 0000000..862d63c --- /dev/null +++ b/dev/templates/install/install_dependencies_alpine.sh.jinja @@ -0,0 +1,4 @@ +#!/usr/bin/env sh +apk update +apk add \ + {{ (tool.osreqs.alpine.build + tool.osreqs.alpine.run + tool.osreqs.alpine.opt) | join(' \\\n\t') }} diff --git a/dev/templates/install/install_dependencies_debian.sh.jinja b/dev/templates/install/install_dependencies_debian.sh.jinja new file mode 100644 index 0000000..b840935 --- /dev/null +++ b/dev/templates/install/install_dependencies_debian.sh.jinja @@ -0,0 +1,4 @@ +#!/usr/bin/env sh +apt update +apt install \ + {{ (tool.osreqs.debian.build + tool.osreqs.debian.run + tool.osreqs.debian.opt) | join(' \\\n\t') }} diff --git a/dev/templates/requirements.txt.jinja b/dev/templates/requirements.txt.jinja new file mode 100644 index 0000000..4ef0bbe --- /dev/null +++ b/dev/templates/requirements.txt.jinja @@ -0,0 +1,3 @@ +{% for dep in project.dependencies -%} +{{ dep }} +{% endfor %} diff --git a/dev/templates/requirements_extra.txt.jinja b/dev/templates/requirements_extra.txt.jinja new file mode 100644 index 0000000..33cf997 --- /dev/null +++ b/dev/templates/requirements_extra.txt.jinja @@ -0,0 +1,3 @@ +{% for dep in project['optional-dependencies'].full -%} +{{ dep }} +{% endfor %} diff --git a/testing/Maloja.postman_collection.json b/dev/testing/Maloja.postman_collection.json similarity index 100% rename from testing/Maloja.postman_collection.json rename to dev/testing/Maloja.postman_collection.json diff --git a/dev/update_dist_files.py b/dev/update_dist_files.py new file mode 100644 index 0000000..0bd6c8e --- /dev/null +++ b/dev/update_dist_files.py @@ -0,0 +1,33 @@ +import toml +import os +import jinja2 + +env = jinja2.Environment( + loader=jinja2.FileSystemLoader('dev/templates'), + autoescape=jinja2.select_autoescape(['html', 'xml']), + keep_trailing_newline=True +) + +with open("pyproject.toml") as filed: + data = toml.load(filed) + +templatedir = "./dev/templates" + +for root,dirs,files in os.walk(templatedir): + + reldirpath = os.path.relpath(root,start=templatedir) + for f in files: + + relfilepath = os.path.join(reldirpath,f) + + if not f.endswith('.jinja'): continue + + srcfile = os.path.join(root,f) + trgfile = os.path.join(reldirpath,f.replace(".jinja","")) + + + template = env.get_template(relfilepath) + result = template.render(**data) + + with open(trgfile,"w") as filed: + filed.write(result) diff --git a/dev-scripts/update_scrobbler.sh b/dev/update_scrobbler.sh similarity index 100% rename from dev-scripts/update_scrobbler.sh rename to dev/update_scrobbler.sh diff --git a/example-compose.yml b/example-compose.yml new file mode 100644 index 0000000..a8a23eb --- /dev/null +++ b/example-compose.yml @@ -0,0 +1,20 @@ +services: + maloja: + # from dockerhub + image: "krateng/maloja:latest" + # or built locally + #build: + # context: . + # dockerfile: ./Containerfile + ports: + - "42010:42010" + # different directories for configuration, state and logs + volumes: + - "$PWD/config:/etc/maloja" + - "$PWD/data:/var/lib/maloja" + - "$PWD/logs:/var/log/maloja" + #you can also have everything together instead: + #volumes: + #- "$PWD/data:/data" + #environment: + #- "MALOJA_DATA_DIRECTORY=/data" diff --git a/install/dependencies_basic.txt b/install/dependencies_basic.txt deleted file mode 100644 index 273cd54..0000000 --- a/install/dependencies_basic.txt +++ /dev/null @@ -1 +0,0 @@ -python3 diff --git a/install/dependencies_build.txt b/install/dependencies_build.txt deleted file mode 100644 index 7980da1..0000000 --- a/install/dependencies_build.txt +++ /dev/null @@ -1,8 +0,0 @@ -gcc -python3-dev -libxml2-dev -libxslt-dev -libffi-dev -libc-dev -py3-pip -linux-headers diff --git a/install/dependencies_run.txt b/install/dependencies_run.txt deleted file mode 100644 index 0883ff0..0000000 --- a/install/dependencies_run.txt +++ /dev/null @@ -1 +0,0 @@ -tzdata diff --git a/install/dependencies_run_opt.txt b/install/dependencies_run_opt.txt deleted file mode 100644 index c2850f6..0000000 --- a/install/dependencies_run_opt.txt +++ /dev/null @@ -1 +0,0 @@ -vips diff --git a/install/install_alpine.sh b/install/install_alpine.sh new file mode 100644 index 0000000..88b23c2 --- /dev/null +++ b/install/install_alpine.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +apk update +apk add \ + gcc \ + g++ \ + python3-dev \ + libxml2-dev \ + libxslt-dev \ + libffi-dev \ + libc-dev \ + py3-pip \ + linux-headers \ + python3 \ + tzdata \ + vips + +apk add py3-pip +pip install wheel +pip install malojaserver diff --git a/install/install_debian.sh b/install/install_debian.sh new file mode 100644 index 0000000..3886e39 --- /dev/null +++ b/install/install_debian.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh +apt update +apt install \ + python3-pip \ + python3 + +apt install python3-pip +pip install wheel +pip install malojaserver diff --git a/install/install_dependencies.sh b/install/install_dependencies.sh deleted file mode 100644 index bbdc570..0000000 --- a/install/install_dependencies.sh +++ /dev/null @@ -1,4 +0,0 @@ -sed 's/#.*//' ./install/dependencies_basic.txt | xargs apk add -sed 's/#.*//' ./install/dependencies_build.txt | xargs apk add -sed 's/#.*//' ./install/dependencies_run.txt | xargs apk add -sed 's/#.*//' ./install/dependencies_run_opt.txt | xargs apk add diff --git a/install_alpine.sh b/install/install_dependencies_alpine.sh similarity index 68% rename from install_alpine.sh rename to install/install_dependencies_alpine.sh index 890e033..5a14d85 100644 --- a/install_alpine.sh +++ b/install/install_dependencies_alpine.sh @@ -1,15 +1,15 @@ -#!/usr/bin/env bash +#!/usr/bin/env sh +apk update apk add \ - python3 \ - python3-dev \ gcc \ + g++ \ + python3-dev \ libxml2-dev \ libxslt-dev \ libffi-dev \ libc-dev \ py3-pip \ linux-headers \ + python3 \ tzdata \ vips -pip3 install wheel -pip3 install malojaserver diff --git a/install/install_dependencies_debian.sh b/install/install_dependencies_debian.sh new file mode 100644 index 0000000..776bb6c --- /dev/null +++ b/install/install_dependencies_debian.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh +apt update +apt install \ + python3-pip \ + python3 diff --git a/install_ubuntu.sh b/install_ubuntu.sh deleted file mode 100644 index ff0bd66..0000000 --- a/install_ubuntu.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -apt update -apt install python3 python3-pip -pip3 install malojaserver diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 27a8f75..d7ba9e6 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -4,7 +4,7 @@ # you know what f*ck it # this is hardcoded for now because of that damn project / package name discrepancy # i'll fix it one day -VERSION = "2.14.10" +VERSION = "3.0.0-beta.6" HOMEPAGE = "https://github.com/krateng/maloja" diff --git a/maloja/apis/__init__.py b/maloja/apis/__init__.py index 87f17a3..3dfd490 100644 --- a/maloja/apis/__init__.py +++ b/maloja/apis/__init__.py @@ -1,22 +1,28 @@ -from . import native_v1 -from .audioscrobbler import Audioscrobbler -from .audioscrobbler_legacy import AudioscrobblerLegacy -from .listenbrainz import Listenbrainz +from ._apikeys import apikeystore + import copy from bottle import redirect, request, response from urllib.parse import urlencode -native_apis = [ - native_v1.api -] -standardized_apis = [ - Listenbrainz(), - Audioscrobbler(), - AudioscrobblerLegacy() -] + def init_apis(server): + + from . import native_v1 + from .audioscrobbler import Audioscrobbler + from .audioscrobbler_legacy import AudioscrobblerLegacy + from .listenbrainz import Listenbrainz + + native_apis = [ + native_v1.api + ] + standardized_apis = [ + Listenbrainz(), + Audioscrobbler(), + AudioscrobblerLegacy() + ] + for api in native_apis: api.mount(server=server,path="apis/"+api.__apipath__) diff --git a/maloja/apis/_apikeys.py b/maloja/apis/_apikeys.py new file mode 100644 index 0000000..cc49c92 --- /dev/null +++ b/maloja/apis/_apikeys.py @@ -0,0 +1,29 @@ +### API KEYS +### symmetric keys are fine since we hopefully use HTTPS + +from doreah.keystore import KeyStore +from doreah.logging import log + +from ..globalconf import data_dir + +apikeystore = KeyStore(file=data_dir['clients']("apikeys.yml"),save_endpoint="/apis/mlj_1/apikeys") + + +from .. import upgrade +upgrade.upgrade_apikeys() + + +# skip regular authentication if api key is present in request +# an api key now ONLY permits scrobbling tracks, no other admin tasks +def api_key_correct(request,args,kwargs): + if "key" in kwargs: + apikey = kwargs.pop("key") + elif "apikey" in kwargs: + apikey = kwargs.pop("apikey") + else: return False + + client = apikeystore.check_and_identify_key(apikey) + if client: + return {'client':client} + else: + return False diff --git a/maloja/apis/_base.py b/maloja/apis/_base.py index 9a365d9..938c701 100644 --- a/maloja/apis/_base.py +++ b/maloja/apis/_base.py @@ -58,7 +58,7 @@ class APIHandler: def wrapper(self,path:Multi=[],**keys): - log("API request: " + str(path))# + " | Keys: " + str({k:keys.get(k) for k in keys})) + log(f"{self.__apiname__} API request: {path}")# + " | Keys: " + str({k:keys.get(k) for k in keys})) try: response.status,result = self.handle(path,keys) @@ -89,13 +89,10 @@ class APIHandler: return method(path,keys) - def scrobble(self,artiststr,titlestr,time=None,duration=None,album=None): - logmsg = "Incoming scrobble (API: {api}): ARTISTS: {artiststr}, TRACK: {titlestr}" - log(logmsg.format(api=self.__apiname__,artiststr=artiststr,titlestr=titlestr)) - if time is None: time = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()) + def scrobble(self,rawscrobble,client=None): + + # fixing etc is handled by the main scrobble function try: - (artists,title) = cla.fullclean(artiststr,titlestr) - database.createScrobble(artists,title,time) - database.sync() + return database.incoming_scrobble(rawscrobble,api=self.__apiname__,client=client) except: raise ScrobblingException() diff --git a/maloja/apis/audioscrobbler.py b/maloja/apis/audioscrobbler.py index 560c966..586f2c5 100644 --- a/maloja/apis/audioscrobbler.py +++ b/maloja/apis/audioscrobbler.py @@ -1,6 +1,7 @@ from ._base import APIHandler from ._exceptions import * from .. import database +from ._apikeys import apikeystore class Audioscrobbler(APIHandler): __apiname__ = "Audioscrobbler" @@ -14,7 +15,7 @@ class Audioscrobbler(APIHandler): def init(self): # no need to save these on disk, clients can always request a new session - self.mobile_sessions = [] + self.mobile_sessions = {} self.methods = { "auth.getMobileSession":self.authmobile, "track.scrobble":self.submit_scrobble @@ -30,29 +31,45 @@ class Audioscrobbler(APIHandler): def get_method(self,pathnodes,keys): return keys.get("method") + def generate_key(self,client): + key = "".join( + str( + random.choice( + list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) for _ in range(64)) + + self.mobile_sessions[key] = client + return key + def authmobile(self,pathnodes,keys): token = keys.get("authToken") user = keys.get("username") password = keys.get("password") # either username and password if user is not None and password is not None: - if password in database.allAPIkeys(): - sessionkey = generate_key(self.mobile_sessions) + client = apikeystore.check_and_identify_key(password) + if client: + sessionkey = self.generate_key(client) return 200,{"session":{"key":sessionkey}} else: raise InvalidAuthException() # or username and token (deprecated by lastfm) elif user is not None and token is not None: - for key in database.allAPIkeys(): + for client in apikeystore: + key = apikeystore[client] if md5(user + md5(key)) == token: - sessionkey = generate_key(self.mobile_sessions) + sessionkey = self.generate_key(client) return 200,{"session":{"key":sessionkey}} raise InvalidAuthException() else: raise BadAuthException() def submit_scrobble(self,pathnodes,keys): - if keys.get("sk") is None or keys.get("sk") not in self.mobile_sessions: + key = keys.get("sk") + if key is None: + raise InvalidSessionKey() + client = self.mobile_sessions.get(key) + if not client: raise InvalidSessionKey() if "track" in keys and "artist" in keys: artiststr,titlestr = keys["artist"], keys["track"] @@ -62,7 +79,7 @@ class Audioscrobbler(APIHandler): except: timestamp = None #database.createScrobble(artists,title,timestamp) - self.scrobble(artiststr,titlestr,time=timestamp) + self.scrobble({'track_artists':[artiststr],'track_title':titlestr,'scrobble_time':timestamp},client=client) else: for num in range(50): if "track[" + str(num) + "]" in keys: @@ -82,13 +99,3 @@ def md5(input): m = hashlib.md5() m.update(bytes(input,encoding="utf-8")) return m.hexdigest() - -def generate_key(ls): - key = "".join( - str( - random.choice( - list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + - list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) for _ in range(64)) - - ls.append(key) - return key diff --git a/maloja/apis/audioscrobbler_legacy.py b/maloja/apis/audioscrobbler_legacy.py index 09cd7bb..322fe78 100644 --- a/maloja/apis/audioscrobbler_legacy.py +++ b/maloja/apis/audioscrobbler_legacy.py @@ -1,6 +1,7 @@ from ._base import APIHandler from ._exceptions import * from .. import database +from ._apikeys import apikeystore from bottle import request @@ -15,7 +16,7 @@ class AudioscrobblerLegacy(APIHandler): def init(self): # no need to save these on disk, clients can always request a new session - self.mobile_sessions = [] + self.mobile_sessions = {} self.methods = { "handshake":self.handshake, "nowplaying":self.now_playing, @@ -41,9 +42,10 @@ class AudioscrobblerLegacy(APIHandler): protocol = 'http' if (keys.get("u") == 'nossl') else request.urlparts.scheme if auth is not None: - for key in database.allAPIkeys(): - if check_token(auth, key, timestamp): - sessionkey = generate_key(self.mobile_sessions) + for client in apikeystore: + key = apikeystore[client] + if self.check_token(auth,key,timestamp): + sessionkey = self.generate_key(client) return 200, ( "OK\n" f"{sessionkey}\n" @@ -65,8 +67,10 @@ class AudioscrobblerLegacy(APIHandler): return 200,"OK\n" def submit_scrobble(self,pathnodes,keys): - if keys.get("s") is None or keys.get("s") not in self.mobile_sessions: + key = keys.get("s") + if key is None or key not in self.mobile_sessions: raise InvalidSessionKey() + client = self.mobile_sessions.get(key) for count in range(50): artist_key = f"a[{count}]" track_key = f"t[{count}]" @@ -79,10 +83,29 @@ class AudioscrobblerLegacy(APIHandler): except: timestamp = None #database.createScrobble(artists,title,timestamp) - self.scrobble(artiststr,titlestr,time=timestamp) + self.scrobble({ + 'track_artists':[artiststr], + 'track_title':titlestr, + 'scrobble_time':timestamp + },client=client) return 200,"OK\n" + def check_token(self, received_token, expected_key, ts): + expected_token = md5(md5(expected_key) + ts) + return received_token == expected_token + + def generate_key(self,client): + key = "".join( + str( + random.choice( + list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) for _ in range(64)) + + self.mobile_sessions[key] = client + return key + + import hashlib import random @@ -90,20 +113,3 @@ def md5(input): m = hashlib.md5() m.update(bytes(input,encoding="utf-8")) return m.hexdigest() - -def generate_key(ls): - key = "".join( - str( - random.choice( - list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + - list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) for _ in range(64)) - - ls.append(key) - return key - -def lastfm_token(password, ts): - return md5(md5(password) + ts) - -def check_token(received_token, expected_key, ts): - expected_token = lastfm_token(expected_key, ts) - return received_token == expected_token diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py index a254ffa..457573f 100644 --- a/maloja/apis/listenbrainz.py +++ b/maloja/apis/listenbrainz.py @@ -2,6 +2,7 @@ from ._base import APIHandler from ._exceptions import * from .. import database import datetime +from ._apikeys import apikeystore from ..globalconf import malojaconfig @@ -36,7 +37,9 @@ class Listenbrainz(APIHandler): except: raise BadAuthException() - if token not in database.allAPIkeys(): + client = apikeystore.check_and_identify_key(token) + + if not client: raise InvalidAuthException() try: @@ -59,7 +62,11 @@ class Listenbrainz(APIHandler): except: raise MalformedJSONException() - self.scrobble(artiststr,titlestr,timestamp) + self.scrobble({ + 'track_artists':[artiststr], + 'track_title':titlestr, + 'scrobble_time':timestamp + },client=client) return 200,{"status":"ok"} @@ -69,7 +76,7 @@ class Listenbrainz(APIHandler): token = self.get_token_from_request_keys(keys) except: raise BadAuthException() - if token not in database.allAPIkeys(): + if not apikeystore.check_key(token): raise InvalidAuthException() else: return 200,{"code":200,"message":"Token valid.","valid":True,"user_name":malojaconfig["NAME"]} diff --git a/maloja/apis/native_v1.py b/maloja/apis/native_v1.py index 672d4f3..6e5ccac 100644 --- a/maloja/apis/native_v1.py +++ b/maloja/apis/native_v1.py @@ -1,20 +1,43 @@ -from ..database import * -from ..globalconf import malojaconfig, apikeystore -from ..__pkginfo__ import VERSION -from ..malojauri import uri_to_internal -from .. import utilities +import os -from bottle import response, static_file +from bottle import response, static_file, request, FormsDict + +from doreah.logging import log +from doreah.auth import authenticated_api, authenticated_api_with_alternate, authenticated_function # nimrodel API from nimrodel import EAPI as API from nimrodel import Multi +from .. import database +from ..globalconf import malojaconfig, data_dir + + + +from ..__pkginfo__ import VERSION +from ..malojauri import uri_to_internal, compose_querystring, internal_to_uri +from .. import images +from ._apikeys import apikeystore, api_key_correct + + + + + + + + + + + + api = API(delay=True) api.__apipath__ = "mlj_1" + + + @api.get("test") def test_server(key=None): """Pings the server. If an API key is supplied, the server will respond with 200 @@ -24,7 +47,7 @@ def test_server(key=None): :param string key: An API key to be tested. Optional. """ response.set_header("Access-Control-Allow-Origin","*") - if key is not None and not (checkAPIkey(key)): + if key is not None and not apikeystore.check_key(key): response.status = 403 return {"error":"Wrong API key"} @@ -44,7 +67,7 @@ def server_info(): "name":malojaconfig["NAME"], "version":VERSION.split("."), "versionstring":VERSION, - "db_status":dbstatus + "db_status":database.dbstatus } @@ -56,7 +79,7 @@ def get_scrobbles_external(**keys): k_filter, k_time, _, k_amount, _ = uri_to_internal(keys,api=True) ckeys = {**k_filter, **k_time, **k_amount} - result = get_scrobbles(**ckeys) + result = database.get_scrobbles(**ckeys) offset = (k_amount.get('page') * k_amount.get('perpage')) if k_amount.get('perpage') is not math.inf else 0 result = result[offset:] @@ -81,7 +104,7 @@ def get_scrobbles_num_external(**keys): k_filter, k_time, _, k_amount, _ = uri_to_internal(keys) ckeys = {**k_filter, **k_time, **k_amount} - result = get_scrobbles_num(**ckeys) + result = database.get_scrobbles_num(**ckeys) return {"amount":result} @@ -91,14 +114,14 @@ def get_tracks_external(**keys): k_filter, _, _, _, _ = uri_to_internal(keys,forceArtist=True) ckeys = {**k_filter} - result = get_tracks(**ckeys) + result = database.get_tracks(**ckeys) return {"list":result} @api.get("artists") def get_artists_external(): - result = get_artists() + result = database.get_artists() return {"list":result} @@ -110,7 +133,7 @@ def get_charts_artists_external(**keys): _, k_time, _, _, _ = uri_to_internal(keys) ckeys = {**k_time} - result = get_charts_artists(**ckeys) + result = database.get_charts_artists(**ckeys) return {"list":result} @@ -120,7 +143,7 @@ def get_charts_tracks_external(**keys): k_filter, k_time, _, _, _ = uri_to_internal(keys,forceArtist=True) ckeys = {**k_filter, **k_time} - result = get_charts_tracks(**ckeys) + result = database.get_charts_tracks(**ckeys) return {"list":result} @@ -131,7 +154,7 @@ def get_pulse_external(**keys): k_filter, k_time, k_internal, k_amount, _ = uri_to_internal(keys) ckeys = {**k_filter, **k_time, **k_internal, **k_amount} - results = get_pulse(**ckeys) + results = database.get_pulse(**ckeys) return {"list":results} @@ -142,7 +165,7 @@ def get_performance_external(**keys): k_filter, k_time, k_internal, k_amount, _ = uri_to_internal(keys) ckeys = {**k_filter, **k_time, **k_internal, **k_amount} - results = get_performance(**ckeys) + results = database.get_performance(**ckeys) return {"list":results} @@ -153,7 +176,7 @@ def get_top_artists_external(**keys): _, k_time, k_internal, _, _ = uri_to_internal(keys) ckeys = {**k_time, **k_internal} - results = get_top_artists(**ckeys) + results = database.get_top_artists(**ckeys) return {"list":results} @@ -166,23 +189,23 @@ def get_top_tracks_external(**keys): # IMPLEMENT THIS FOR TOP TRACKS OF ARTIST AS WELL? - results = get_top_tracks(**ckeys) + results = database.get_top_tracks(**ckeys) return {"list":results} @api.get("artistinfo") -def artistInfo_external(**keys): +def artist_info_external(**keys): k_filter, _, _, _, _ = uri_to_internal(keys,forceArtist=True) ckeys = {**k_filter} - return artistInfo(**ckeys) + return database.artist_info(**ckeys) @api.get("trackinfo") -def trackInfo_external(artist:Multi[str],**keys): +def track_info_external(artist:Multi[str],**keys): # transform into a multidict so we can use our nomral uri_to_internal function keys = FormsDict(keys) for a in artist: @@ -190,50 +213,60 @@ def trackInfo_external(artist:Multi[str],**keys): k_filter, _, _, _, _ = uri_to_internal(keys,forceTrack=True) ckeys = {**k_filter} - return trackInfo(**ckeys) + return database.track_info(**ckeys) @api.get("compare") def compare_external(**keys): - return compare(keys["remote"]) + return database.compare(keys["remote"]) - -@api.get("newscrobble") -@authenticated_api_with_alternate(api_key_correct) -def get_post_scrobble(artist:Multi,**keys): - """DEPRECATED. Use the equivalent POST method instead.""" - artists = artist - title = keys.get("title") - album = keys.get("album") - duration = keys.get("seconds") - time = keys.get("time") - if time is not None: time = int(time) - - return incoming_scrobble(artists,title,album=album,duration=duration,time=time) - @api.post("newscrobble") -@authenticated_api_with_alternate(api_key_correct) -def post_scrobble(artist:Multi=None,**keys): +@authenticated_function(alternate=api_key_correct,api=True,pass_auth_result_as='auth_result') +def post_scrobble(artist:Multi=None,auth_result=None,**keys): """Submit a new scrobble. :param string artist: Artist. Can be submitted multiple times as query argument for multiple artists. :param string artists: List of artists. Overwritten by artist parameter. :param string title: Title of the track. :param string album: Name of the album. Optional. + :param string albumartists: Album artists. Optional. :param int duration: Actual listened duration of the scrobble in seconds. Optional. + :param int length: Total length of the track in seconds. Optional. :param int time: UNIX timestamp of the scrobble. Optional, not needed if scrobble is at time of request. + :param boolean nofix: Skip server-side metadata parsing. Optional. """ - #artists = "/".join(artist) - artists = artist if artist is not None else keys.get("artists") - title = keys.get("title") - album = keys.get("album") - duration = keys.get("seconds") - time = keys.get("time") - nofix = keys.get("nofix") is not None - if time is not None: time = int(time) - return incoming_scrobble(artists,title,album=album,duration=duration,time=time,fix=not nofix) + rawscrobble = { + 'track_artists':artist if artist is not None else keys.get("artists"), + 'track_title':keys.get('title'), + 'album_name':keys.get('album'), + 'album_artists':keys.get('albumartists'), + 'scrobble_duration':keys.get('duration'), + 'track_length':keys.get('length'), + 'scrobble_time':int(keys.get('time')) if (keys.get('time') is not None) else None + } + + # for logging purposes, don't pass values that we didn't actually supply + rawscrobble = {k:rawscrobble[k] for k in rawscrobble if rawscrobble[k]} + + result = database.incoming_scrobble( + rawscrobble, + client='browser' if auth_result.get('doreah_native_auth_check') else auth_result.get('client'), + api='native/v1', + fix=(keys.get("nofix") is None) + ) + + if result: + return { + 'status': 'success', + 'track': { + 'artists':result['track']['artists'], + 'title':result['track']['title'] + } + } + else: + return {"status":"failure"} @@ -259,15 +292,14 @@ def import_rulemodule(**keys): @authenticated_api def rebuild(**keys): log("Database rebuild initiated!") - sync() + database.sync() dbstatus['rebuildinprogress'] = True from ..proccontrol.tasks.fixexisting import fix fix() - global cla, coa + global cla cla = CleanerAgent() - coa = CollectorAgent() - build_db() - invalidate_caches() + database.build_db() + database.invalidate_caches() @@ -279,8 +311,8 @@ def search(**keys): if max_ is not None: max_ = int(max_) query = query.lower() - artists = db_search(query,type="ARTIST") - tracks = db_search(query,type="TRACK") + artists = database.db_search(query,type="ARTIST") + tracks = database.db_search(query,type="TRACK") @@ -296,14 +328,14 @@ def search(**keys): 'name': a, 'link': "/artist?" + compose_querystring(internal_to_uri({"artist": a})), } - result["image"] = "/image?" + compose_querystring(internal_to_uri({"artist":a})) + result["image"] = images.get_artist_image(a) artists_result.append(result) tracks_result = [] for t in tracks: result = t result["link"] = "/track?" + compose_querystring(internal_to_uri({"track":t})) - result["image"] = "/image?" + compose_querystring(internal_to_uri({"track":t})) + result["image"] = images.get_track_image(t) tracks_result.append(result) return {"artists":artists_result[:max_],"tracks":tracks_result[:max_]} @@ -318,13 +350,15 @@ def add_picture(b64,artist:Multi=[],title=None): if title is not None: keys.append("title",title) k_filter, _, _, _, _ = uri_to_internal(keys) if "track" in k_filter: k_filter = k_filter["track"] - utilities.set_image(b64,**k_filter) + images.set_image(b64,**k_filter) @api.post("newrule") @authenticated_api def newrule(**keys): - tsv.add_entry(data_dir['rules']("webmade.tsv"),[k for k in keys]) + pass + # TODO after implementing new rule system + #tsv.add_entry(data_dir['rules']("webmade.tsv"),[k for k in keys]) #addEntry("rules/webmade.tsv",[k for k in keys]) @@ -354,3 +388,20 @@ def get_backup(**keys): archivefile = backup(tmpfolder) return static_file(os.path.basename(archivefile),root=tmpfolder) + +@api.get("export") +@authenticated_api +def get_export(**keys): + from ..proccontrol.tasks.export import export + import tempfile + + tmpfolder = tempfile.gettempdir() + resultfile = export(tmpfolder) + + return static_file(os.path.basename(resultfile),root=tmpfolder) + + +@api.post("delete_scrobble") +@authenticated_api +def delete_scrobble(timestamp): + database.remove_scrobble(timestamp) diff --git a/maloja/cleanup.py b/maloja/cleanup.py index 0f428e7..e94fdae 100644 --- a/maloja/cleanup.py +++ b/maloja/cleanup.py @@ -1,8 +1,8 @@ import re -#from . import utilities -from doreah import tsv +import os +import csv + from .globalconf import data_dir, malojaconfig -import pkg_resources # need to do this as a class so it can retain loaded settings from file # apparently this is not true @@ -13,19 +13,29 @@ class CleanerAgent: self.updateRules() def updateRules(self): - raw = tsv.parse_all(data_dir["rules"](),"string","string","string","string") - self.rules_belongtogether = [b for [a,b,c,d] in raw if a=="belongtogether"] - self.rules_notanartist = [b for [a,b,c,d] in raw if a=="notanartist"] - self.rules_replacetitle = {b.lower():c for [a,b,c,d] in raw if a=="replacetitle"} - self.rules_replaceartist = {b.lower():c for [a,b,c,d] in raw if a=="replaceartist"} - self.rules_ignoreartist = [b.lower() for [a,b,c,d] in raw if a=="ignoreartist"] - self.rules_addartists = {c.lower():(b.lower(),d) for [a,b,c,d] in raw if a=="addartists"} - self.rules_fixartists = {c.lower():b for [a,b,c,d] in raw if a=="fixartists"} - self.rules_artistintitle = {b.lower():c for [a,b,c,d] in raw if a=="artistintitle"} + + rawrules = [] + for f in os.listdir(data_dir["rules"]()): + if f.split('.')[-1].lower() != 'tsv': continue + filepath = data_dir["rules"](f) + with open(filepath,'r') as filed: + reader = csv.reader(filed,delimiter="\t") + rawrules += [[col for col in entry if col] for entry in reader if len(entry)>0 and not entry[0].startswith('#')] + + + self.rules_belongtogether = [r[1] for r in rawrules if r[0]=="belongtogether"] + self.rules_notanartist = [r[1] for r in rawrules if r[0]=="notanartist"] + self.rules_replacetitle = {r[1].lower():r[2] for r in rawrules if r[0]=="replacetitle"} + self.rules_replaceartist = {r[1].lower():r[2] for r in rawrules if r[0]=="replaceartist"} + self.rules_ignoreartist = [r[1].lower() for r in rawrules if r[0]=="ignoreartist"] + self.rules_addartists = {r[2].lower():(r[1].lower(),r[3]) for r in rawrules if r[0]=="addartists"} + self.rules_fixartists = {r[2].lower():r[1] for r in rawrules if r[0]=="fixartists"} + self.rules_artistintitle = {r[1].lower():r[2] for r in rawrules if r[0]=="artistintitle"} #self.rules_regexartist = [[b,c] for [a,b,c,d] in raw if a=="regexartist"] #self.rules_regextitle = [[b,c] for [a,b,c,d] in raw if a=="regextitle"] + def fullclean(self,artist,title): artists = self.parseArtists(self.removespecial(artist)) title = self.parseTitle(self.removespecial(title)) @@ -166,65 +176,6 @@ class CleanerAgent: -#this is for all the runtime changes (counting Trouble Maker as HyunA for charts etc) -class CollectorAgent: - - def __init__(self): - self.updateRules() - - # rules_countas dict: real artist -> credited artist - # rules_countas_id dict: real artist ID -> credited artist ID - # rules_include dict: credited artist -> all real artists - - def updateRules(self): - raw = tsv.parse_all(data_dir["rules"](),"string","string","string") - self.rules_countas = {b:c for [a,b,c] in raw if a=="countas"} - self.rules_countas_id = {} - self.rules_include = {} #Twice the memory, double the performance! - # (Yes, we're saving redundant information here, but it's not unelegant if it's within a closed object!) - for a in self.rules_countas: - self.rules_include[self.rules_countas[a]] = self.rules_include.setdefault(self.rules_countas[a],[]) + [a] - - # this agent needs to be aware of the current id assignment in the main program - # unelegant, but the best way i can think of - def updateIDs(self,artistlist): - self.rules_countas_id = {artistlist.index(a):artistlist.index(self.rules_countas[a]) for a in self.rules_countas if a in artistlist} - #self.rules_include_id = {artistlist.index(a):artistlist.index(self.rules_include[a]) for a in self.rules_include} - #this needs to take lists into account - - - # get who is credited for this artist - def getCredited(self,artist): - if artist in self.rules_countas: - return self.rules_countas[artist] - if artist in self.rules_countas_id: - return self.rules_countas_id[artist] - - else: - return artist - - # get all credited artists for the artists given - def getCreditedList(self,artists): - updatedArtists = [self.getCredited(artist) for artist in artists] - return list(set(updatedArtists)) - - # get artists who the given artist is given credit for - def getAllAssociated(self,artist): - return self.rules_include.get(artist,[]) - - # this function is there to check for artists that we should include in the - # database even though they never have any scrobble. - def getAllArtists(self): - return list({self.rules_countas[a] for a in self.rules_countas}) - # artists that count can be nonexisting (counting HyunA as 4Minute even - # though 4Minute has never been listened to) - # but artists that are counted as someone else are only relevant if they - # exist (so we can preemptively declare lots of rules just in case) - #return list(set([a for a in self.rules_countas] + [self.rules_countas[a] for a in self.rules_countas])) - - - - diff --git a/maloja/data_files/config/rules/predefined/krateng_artistsingroups.tsv b/maloja/data_files/config/rules/predefined/krateng_artistsingroups.tsv index 39639fa..2ad2169 100644 --- a/maloja/data_files/config/rules/predefined/krateng_artistsingroups.tsv +++ b/maloja/data_files/config/rules/predefined/krateng_artistsingroups.tsv @@ -16,3 +16,5 @@ countas Sips The Yogscast countas Sjin The Yogscast countas Airi Suzuki ℃-ute countas CeeLo Green Gnarls Barkley +countas Amelia Watson Hololive EN +countas Gawr Gura Hololive EN diff --git a/maloja/database.py b/maloja/database.py deleted file mode 100644 index 484d187..0000000 --- a/maloja/database.py +++ /dev/null @@ -1,1175 +0,0 @@ -# server -from bottle import request, response, FormsDict, HTTPError - -# rest of the project -from .cleanup import CleanerAgent, CollectorAgent -from . import utilities -from .malojatime import register_scrobbletime, time_stamps, ranges -from .malojauri import uri_to_internal, internal_to_uri, compose_querystring - -from .thirdparty import proxy_scrobble_all - -from .globalconf import data_dir, malojaconfig, apikeystore - -# doreah toolkit -from doreah.logging import log -from doreah import tsv -from doreah.caching import Cache, DeepCache -from doreah.auth import authenticated_api, authenticated_api_with_alternate -from doreah.io import ProgressBar -try: - from doreah.persistence import DiskDict -except: pass -import doreah - - - -# technical -import os -import datetime -import sys -import unicodedata -from collections import namedtuple -from threading import Lock -import yaml -import lru -import math - -# url handling -from importlib.machinery import SourceFileLoader -import urllib - - - -dblock = Lock() #global database lock -dbstatus = { - "healthy":False, - "rebuildinprogress":False, - "complete":False -} -class DatabaseNotBuilt(HTTPError): - def __init__(self): - super().__init__( - status=503, - body="The Maloja Database is still being built. Try again in a few seconds.", - headers={"Retry-After":10} - ) - -SCROBBLES = [] # Format: tuple(track_ref,timestamp,saved) -ARTISTS = [] # Format: artist -TRACKS = [] # Format: namedtuple(artists=frozenset(artist_ref,...),title=title) - - -Track = namedtuple("Track",["artists","title"]) -Scrobble = namedtuple("Scrobble",["track","timestamp","album","duration","saved"]) -# album is saved in the scrobble because it's not actually authorative information about the track, just info -# what was sent with this scrobble - -### OPTIMIZATION -SCROBBLESDICT = {} # timestamps to scrobble mapping -STAMPS = [] # sorted -#STAMPS_SET = set() # as set for easier check if exists # we use the scrobbles dict for that now -TRACKS_NORMALIZED = [] -ARTISTS_NORMALIZED = [] -ARTISTS_NORMALIZED_SET = set() -TRACKS_NORMALIZED_SET = set() - -MEDALS_ARTISTS = {} #literally only changes once per year, no need to calculate that on the fly -MEDALS_TRACKS = {} -WEEKLY_TOPTRACKS = {} -WEEKLY_TOPARTISTS = {} - -ISSUES = {} - -cla = CleanerAgent() -coa = CollectorAgent() -clients = [] - -lastsync = 0 - - -try: - with open(data_dir['state']("known_servers.yml"),"r") as f: - KNOWN_SERVERS = set(yaml.safe_load(f)) -except: - KNOWN_SERVERS = set() - - -def add_known_server(url): - KNOWN_SERVERS.add(url) - with open(data_dir['state']("known_servers.yml"),"w") as f: - f.write(yaml.dump(list(KNOWN_SERVERS))) - - - - -log("Authenticated Machines: " + ", ".join([k for k in apikeystore])) - -def checkAPIkey(key): - return apikeystore.check_key(key) - -def allAPIkeys(): - return [apikeystore[k] for k in apikeystore] - - -#### -## Getting dict representations of database objects -#### - -def get_scrobble_dict(o): - track = get_track_dict(TRACKS[o.track]) - return {"artists":track["artists"],"title":track["title"],"time":o.timestamp,"album":o.album,"duration":o.duration} - -def get_artist_dict(o): - return o - #technically not a dict, but... you know - -def get_track_dict(o): - artists = [get_artist_dict(ARTISTS[a]) for a in o.artists] - return {"artists":artists,"title":o.title} - - -#### -## Creating or finding existing database entries -#### - - - -def createScrobble(artists,title,time,album=None,duration=None,volatile=False): - - if len(artists) == 0 or title == "": - return {} - - dblock.acquire() - - i = getTrackID(artists,title) - - # idempotence - if time in SCROBBLESDICT and i == SCROBBLESDICT[time].track: - dblock.release() - return get_track_dict(TRACKS[i]) - # timestamp as unique identifier - while (time in SCROBBLESDICT): - time += 1 - - obj = Scrobble(i,time,album,duration,volatile) # if volatile generated, we simply pretend we have already saved it to disk - #SCROBBLES.append(obj) - # immediately insert scrobble correctly so we can guarantee sorted list - index = insert(SCROBBLES,obj,key=lambda x:x[1]) - SCROBBLESDICT[time] = obj - STAMPS.insert(index,time) #should be same index as scrobblelist - register_scrobbletime(time) - invalidate_caches() - dblock.release() - - proxy_scrobble_all(artists,title,time) - - return get_track_dict(TRACKS[obj.track]) - - -# this will never be called from different threads, so no lock -def readScrobble(artists,title,time): - while (time in SCROBBLESDICT): - time += 1 - i = getTrackID(artists,title) - obj = Scrobble(i,time,None,None,True) - SCROBBLES.append(obj) - SCROBBLESDICT[time] = obj - #STAMPS.append(time) - - - -def getArtistID(name): - - obj = name - obj_normalized = normalize_name(name) - - if obj_normalized in ARTISTS_NORMALIZED_SET: - return ARTISTS_NORMALIZED.index(obj_normalized) - - i = len(ARTISTS) - ARTISTS.append(obj) - ARTISTS_NORMALIZED_SET.add(obj_normalized) - ARTISTS_NORMALIZED.append(obj_normalized) - - # with a new artist added, we might also get new artists that they are credited as - cr = coa.getCredited(name) - getArtistID(cr) - - coa.updateIDs(ARTISTS) - - return i - -def getTrackID(artists,title): - artistset = {getArtistID(name=a) for a in artists} - obj = Track(artists=frozenset(artistset),title=title) - obj_normalized = Track(artists=frozenset(artistset),title=normalize_name(title)) - - if obj_normalized in TRACKS_NORMALIZED_SET: - return TRACKS_NORMALIZED.index(obj_normalized) - i = len(TRACKS) - TRACKS.append(obj) - TRACKS_NORMALIZED_SET.add(obj_normalized) - TRACKS_NORMALIZED.append(obj_normalized) - return i - -import unicodedata - -# function to turn the name into a representation that can be easily compared, ignoring minor differences -remove_symbols = ["'","`","’"] -replace_with_space = [" - ",": "] -def normalize_name(name): - for r in replace_with_space: - name = name.replace(r," ") - name = "".join(char for char in unicodedata.normalize('NFD',name.lower()) - if char not in remove_symbols and unicodedata.category(char) != 'Mn') - return name - - - - - -######## -######## -## HTTP requests and their associated functions -######## -######## - -# skip regular authentication if api key is present in request -# an api key now ONLY permits scrobbling tracks, no other admin tasks -def api_key_correct(request): - args = request.params - try: - args.update(request.json) - except: - pass - if "key" in args: - apikey = args["key"] - del args["key"] - elif "apikey" in args: - apikey = args["apikey"] - del args["apikey"] - else: return False - - return checkAPIkey(apikey) - - - - - - -def get_scrobbles(**keys): - r = db_query(**{k:keys[k] for k in keys if k in ["artist","artists","title","since","to","within","timerange","associated","track"]}) - #offset = (keys.get('page') * keys.get('perpage')) if keys.get('perpage') is not math.inf else 0 - #r = r[offset:] - #if keys.get('perpage') is not math.inf: r = r[:keys.get('perpage')] - return r - - -def info(): - totalscrobbles = get_scrobbles_num() - artists = {} - - return { - "name":malojaconfig["NAME"], - "artists":{ - chartentry["artist"]:round(chartentry["scrobbles"] * 100 / totalscrobbles,3) - for chartentry in get_charts_artists() if chartentry["scrobbles"]/totalscrobbles >= 0 - }, - "known_servers":list(KNOWN_SERVERS) - } - - - -def get_scrobbles_num(**keys): - r = db_query(**{k:keys[k] for k in keys if k in ["artist","track","artists","title","since","to","within","timerange","associated"]}) - return len(r) - - -#for multiple since values (must be ordered) -# DOESN'T SEEM TO ACTUALLY BE FASTER -# REEVALUATE - -#def get_scrobbles_num_multiple(sinces=[],to=None,**keys): -# -# sinces_stamps = [time_stamps(since,to,None)[0] for since in sinces] -# #print(sinces) -# #print(sinces_stamps) -# minsince = sinces[-1] -# r = db_query(**{k:keys[k] for k in keys if k in ["artist","track","artists","title","associated","to"]},since=minsince) -# -# #print(r) -# -# validtracks = [0 for s in sinces] -# -# i = 0 -# si = 0 -# while True: -# if si == len(sinces): break -# if i == len(r): break -# if r[i]["time"] >= sinces_stamps[si]: -# validtracks[si] += 1 -# else: -# si += 1 -# continue -# i += 1 -# -# -# return validtracks - - - -def get_tracks(artist=None): - - artistid = ARTISTS.index(artist) if artist is not None else None - # Option 1 - return [get_track_dict(t) for t in TRACKS if (artistid in t.artists) or (artistid==None)] - - # Option 2 is a bit more elegant but much slower - #tracklist = [get_track_dict(t) for t in TRACKS] - #ls = [t for t in tracklist if (artist in t["artists"]) or (artist==None)] - - -def get_artists(): - if not dbstatus['healthy']: raise DatabaseNotBuilt() - return ARTISTS #well - - - -def get_charts_artists(**keys): - return db_aggregate(by="ARTIST",**{k:keys[k] for k in keys if k in ["since","to","within","timerange"]}) - - -def get_charts_tracks(**keys): - return db_aggregate(by="TRACK",**{k:keys[k] for k in keys if k in ["since","to","within","timerange","artist"]}) - -def get_pulse(**keys): - - rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) - results = [] - for rng in rngs: - res = len(db_query(timerange=rng,**{k:keys[k] for k in keys if k in ["artists","artist","track","title","associated"]})) - results.append({"range":rng,"scrobbles":res}) - - return results - - -def get_performance(**keys): - - rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) - results = [] - - for rng in rngs: - if "track" in keys: - charts = get_charts_tracks(timerange=rng) - rank = None - for c in charts: - if c["track"] == keys["track"]: - rank = c["rank"] - break - elif "artist" in keys: - charts = get_charts_artists(timerange=rng) - rank = None - for c in charts: - if c["artist"] == keys["artist"]: - rank = c["rank"] - break - results.append({"range":rng,"rank":rank}) - - return results - - -def get_top_artists(**keys): - - rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) - results = [] - - for rng in rngs: - try: - res = db_aggregate(timerange=rng,by="ARTIST")[0] - results.append({"range":rng,"artist":res["artist"],"counting":res["counting"],"scrobbles":res["scrobbles"]}) - except: - results.append({"range":rng,"artist":None,"scrobbles":0}) - - return results - - - - - - - - - -def get_top_tracks(**keys): - - rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) - results = [] - - for rng in rngs: - try: - res = db_aggregate(timerange=rng,by="TRACK")[0] - results.append({"range":rng,"track":res["track"],"scrobbles":res["scrobbles"]}) - except: - results.append({"range":rng,"track":None,"scrobbles":0}) - - return results - - - - - - - - - - - -def artistInfo(artist): - - charts = db_aggregate(by="ARTIST") - scrobbles = len(db_query(artists=[artist])) - #we cant take the scrobble number from the charts because that includes all countas scrobbles - try: - c = [e for e in charts if e["artist"] == artist][0] - others = [a for a in coa.getAllAssociated(artist) if a in ARTISTS] - position = c["rank"] - performance = get_performance(artist=artist,step="week") - return { - "artist":artist, - "scrobbles":scrobbles, - "position":position, - "associated":others, - "medals":{"gold":[],"silver":[],"bronze":[],**MEDALS_ARTISTS.get(artist,{})}, - "topweeks":WEEKLY_TOPARTISTS.get(artist,0) - } - except: - # if the artist isnt in the charts, they are not being credited and we - # need to show information about the credited one - artist = coa.getCredited(artist) - c = [e for e in charts if e["artist"] == artist][0] - position = c["rank"] - return {"replace":artist,"scrobbles":scrobbles,"position":position} - - - - - - -def trackInfo(track): - charts = db_aggregate(by="TRACK") - #scrobbles = len(db_query(artists=artists,title=title)) #chart entry of track always has right scrobble number, no countas rules here - #c = [e for e in charts if set(e["track"]["artists"]) == set(artists) and e["track"]["title"] == title][0] - c = [e for e in charts if e["track"] == track][0] - scrobbles = c["scrobbles"] - position = c["rank"] - cert = None - threshold_gold, threshold_platinum, threshold_diamond = malojaconfig["SCROBBLES_GOLD","SCROBBLES_PLATINUM","SCROBBLES_DIAMOND"] - if scrobbles >= threshold_diamond: cert = "diamond" - elif scrobbles >= threshold_platinum: cert = "platinum" - elif scrobbles >= threshold_gold: cert = "gold" - - - return { - "track":track, - "scrobbles":scrobbles, - "position":position, - "medals":{"gold":[],"silver":[],"bronze":[],**MEDALS_TRACKS.get((frozenset(track["artists"]),track["title"]),{})}, - "certification":cert, - "topweeks":WEEKLY_TOPTRACKS.get(((frozenset(track["artists"]),track["title"])),0) - } - - - - -def compare(remoteurl): - import json - compareurl = remoteurl + "/api/info" - - response = urllib.request.urlopen(compareurl) - strangerinfo = json.loads(response.read()) - owninfo = info() - - #add_known_server(compareto) - - artists = {} - - for a in owninfo["artists"]: - artists[a.lower()] = {"name":a,"self":int(owninfo["artists"][a]*1000),"other":0} - - for a in strangerinfo["artists"]: - artists[a.lower()] = artists.setdefault(a.lower(),{"name":a,"self":0}) - artists[a.lower()]["other"] = int(strangerinfo["artists"][a]*1000) - - for a in artists: - common = min(artists[a]["self"],artists[a]["other"]) - artists[a]["self"] -= common - artists[a]["other"] -= common - artists[a]["common"] = common - - best = sorted((artists[a]["name"] for a in artists),key=lambda x: artists[x.lower()]["common"],reverse=True) - - result = { - "unique_self":sum(artists[a]["self"] for a in artists if artists[a]["common"] == 0), - "more_self":sum(artists[a]["self"] for a in artists if artists[a]["common"] != 0), - "common":sum(artists[a]["common"] for a in artists), - "more_other":sum(artists[a]["other"] for a in artists if artists[a]["common"] != 0), - "unique_other":sum(artists[a]["other"] for a in artists if artists[a]["common"] == 0) - } - - total = sum(result[c] for c in result) - - for r in result: - result[r] = (result[r],result[r]/total) - - - - return { - "result":result, - "info":{ - "ownname":owninfo["name"], - "remotename":strangerinfo["name"] - }, - "commonartist":best[0] - } - - -def incoming_scrobble(artists,title,album=None,duration=None,time=None,fix=True): - if time is None: - time = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()) - - log("Incoming scrobble (): ARTISTS: " + str(artists) + ", TRACK: " + title,module="debug") - if fix: - (artists,title) = cla.fullclean(artists,title) - trackdict = createScrobble(artists,title,time,album,duration) - - sync() - - return {"status":"success","track":trackdict} - - - - - - - - - -def issues(): - return ISSUES - -def check_issues(): - combined = [] - duplicates = [] - newartists = [] - - import itertools - import difflib - - sortedartists = ARTISTS.copy() - sortedartists.sort(key=len,reverse=True) - reversesortedartists = sortedartists.copy() - reversesortedartists.reverse() - for a in reversesortedartists: - - nochange = cla.confirmedReal(a) - - st = a - lis = [] - reachedmyself = False - for ar in sortedartists: - if (ar != a) and not reachedmyself: - continue - elif not reachedmyself: - reachedmyself = True - continue - - if (ar.lower() == a.lower()) or ("the " + ar.lower() == a.lower()) or ("a " + ar.lower() == a.lower()): - duplicates.append((ar,a)) - break - - if (ar + " " in st) or (" " + ar in st): - lis.append(ar) - st = st.replace(ar,"").strip() - elif (ar == st): - lis.append(ar) - st = "" - if not nochange: - combined.append((a,lis)) - break - - elif (ar in st) and len(ar)*2 > len(st): - duplicates.append((a,ar)) - - st = st.replace("&","").replace("and","").replace("with","").strip() - if st not in ["", a]: - if len(st) < 5 and len(lis) == 1: - #check if we havent just randomly found the string in another word - #if (" " + st + " ") in lis[0] or (lis[0].endswith(" " + st)) or (lis[0].startswith(st + " ")): - duplicates.append((a,lis[0])) - elif len(st) < 5 and len(lis) > 1 and not nochange: - combined.append((a,lis)) - elif len(st) >= 5 and not nochange: - #check if we havent just randomly found the string in another word - if (" " + st + " ") in a or (a.endswith(" " + st)) or (a.startswith(st + " ")): - newartists.append((st,a,lis)) - - #for c in itertools.combinations(ARTISTS,3): - # l = list(c) - # print(l) - # l.sort(key=len,reverse=True) - # [full,a1,a2] = l - # if (a1 + " " + a2 in full) or (a2 + " " + a1 in full): - # combined.append((full,a1,a2)) - - - #for c in itertools.combinations(ARTISTS,2): - # if - # - # if (c[0].lower == c[1].lower): - # duplicates.append((c[0],c[1])) - - - # elif (c[0] + " " in c[1]) or (" " + c[0] in c[1]) or (c[1] + " " in c[0]) or (" " + c[1] in c[0]): - # if (c[0] in c[1]): - # full, part = c[1],c[0] - # rest = c[1].replace(c[0],"").strip() - # else: - # full, part = c[0],c[1] - # rest = c[0].replace(c[1],"").strip() - # if rest in ARTISTS and full not in [c[0] for c in combined]: - # combined.append((full,part,rest)) - - # elif (c[0] in c[1]) or (c[1] in c[0]): - # duplicates.append((c[0],c[1])) - - - return {"duplicates":duplicates,"combined":combined,"newartists":newartists} - - - -def get_predefined_rulesets(): - validchars = "-_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" - - rulesets = [] - - for f in os.listdir(data_dir['rules']("predefined")): - if f.endswith(".tsv"): - - rawf = f.replace(".tsv","") - valid = all(char in validchars for char in rawf) - if not valid: continue - if "_" not in rawf: continue - - try: - with open(data_dir['rules']("predefined",f)) as tsvfile: - line1 = tsvfile.readline() - line2 = tsvfile.readline() - - if "# NAME: " in line1: - name = line1.replace("# NAME: ","") - else: name = rawf.split("_")[1] - desc = line2.replace("# DESC: ","") if "# DESC: " in line2 else "" - author = rawf.split("_")[0] - except: - continue - - ruleset = {"file":rawf} - rulesets.append(ruleset) - ruleset["active"] = bool(os.path.exists(data_dir['rules'](f))) - ruleset["name"] = name - ruleset["author"] = author - ruleset["desc"] = desc - - return rulesets - - -#### -## Server operation -#### - - - -# Starts the server -def start_db(): - log("Starting database...") - global lastsync - lastsync = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()) - build_db() - #run(dbserver, host='::', port=PORT, server='waitress') - log("Database reachable!") - -def build_db(): - - global dbstatus - dbstatus['healthy'] = False - dbstatus['complete'] = False - dbstatus['rebuildinprogress'] = True - - log("Building database...") - - global SCROBBLES, ARTISTS, TRACKS - global TRACKS_NORMALIZED_SET, TRACKS_NORMALIZED, ARTISTS_NORMALIZED_SET, ARTISTS_NORMALIZED - global SCROBBLESDICT, STAMPS - - SCROBBLES = [] - ARTISTS = [] - TRACKS = [] - STAMPS = [] - SCROBBLESDICT = {} - - TRACKS_NORMALIZED = [] - ARTISTS_NORMALIZED = [] - ARTISTS_NORMALIZED_SET = set() - TRACKS_NORMALIZED_SET = set() - - - # parse files - db = tsv.parse_all(data_dir['scrobbles'](),"int","string","string",comments=False) - scrobblenum = len(db) - log(f"Found {scrobblenum} scrobbles...") - - usebar = not malojaconfig["CLEAN_OUTPUT"] - if usebar: pbar = ProgressBar(max=scrobblenum,prefix="Loading scrobbles") - else: - n = 0 - m = max(int(scrobblenum / 25),20) - #db = parseAllTSV("scrobbles","int","string","string",escape=False) - for sc in db: - artists = sc[1].split("␟") - title = sc[2] - time = sc[0] - - readScrobble(artists,title,time) - if usebar: pbar.progress() - else: - n += 1 - if n % m == 0: log(f"Loaded {n}/{scrobblenum}...") - - if usebar: pbar.done() - - - log("Database loaded, optimizing...") - - # optimize database - SCROBBLES.sort(key = lambda tup: tup[1]) - #SCROBBLESDICT = {obj[1]:obj for obj in SCROBBLES} - STAMPS = [t for t in SCROBBLESDICT] - STAMPS.sort() - - # inform malojatime module about earliest scrobble - if STAMPS: register_scrobbletime(STAMPS[0]) - - # NOT NEEDED BECAUSE WE DO THAT ON ADDING EVERY ARTIST ANYWAY - # get extra artists with no real scrobbles from countas rules - #for artist in coa.getAllArtists(): - #for artist in coa.getCreditedList(ARTISTS): - # if artist not in ARTISTS: - # log(artist + " is added to database because of countas rules",module="debug") - # ARTISTS.append(artist) - # coa.updateIDs(ARTISTS) - - dbstatus['healthy'] = True - - - #start regular tasks - utilities.update_medals() - utilities.update_weekly() - utilities.send_stats() - - - global ISSUES - ISSUES = check_issues() - - - dbstatus['complete'] = True - dbstatus['rebuildinprogress'] = False - - log("Database fully built!") - - - -# Saves all cached entries to disk -def sync(): - - # all entries by file collected - # so we don't open the same file for every entry - #log("Syncing",module="debug") - entries = {} - - for idx in range(len(SCROBBLES)): - if not SCROBBLES[idx].saved: - - t = get_scrobble_dict(SCROBBLES[idx]) - - artistlist = list(t["artists"]) - artistlist.sort() #we want the order of artists to be deterministic so when we update files with new rules a diff can see what has actually been changed - artistss = "␟".join(artistlist) - timestamp = datetime.date.fromtimestamp(t["time"]) - - album = t["album"] or "-" - duration = t["duration"] or "-" - - entry = [str(t["time"]),artistss,t["title"],album,duration] - - monthcode = str(timestamp.year) + "_" + str(timestamp.month) - entries.setdefault(monthcode,[]).append(entry) #i feckin love the setdefault function - - SCROBBLES[idx] = Scrobble(*SCROBBLES[idx][:-1],True) - # save copy with last tuple entry set to true - - #log("Sorted into months",module="debug") - - for e in entries: - tsv.add_entries(data_dir['scrobbles'](e + ".tsv"),entries[e],comments=False) - #addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False) - - #log("Written files",module="debug") - - - global lastsync - lastsync = int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()) - #log("Database saved to disk.") - - # save cached images - #saveCache() - - - -### -## Caches in front of DB -## the volatile caches are intended mainly for excessive site navigation during one session -## the permanent caches are there to save data that is hard to calculate and never changes (old charts) -### - - - - -import copy - -if malojaconfig["USE_DB_CACHE"]: - def db_query(**kwargs): - return db_query_cached(**kwargs) - def db_aggregate(**kwargs): - return db_aggregate_cached(**kwargs) -else: - def db_query(**kwargs): - return db_query_full(**kwargs) - def db_aggregate(**kwargs): - return db_aggregate_full(**kwargs) - - -csz = malojaconfig["DB_CACHE_ENTRIES"] -cmp = malojaconfig["DB_MAX_MEMORY"] -try: - import psutil - use_psutil = True -except: - use_psutil = False - -cache_query = lru.LRU(csz) -cache_query_perm = lru.LRU(csz) -cache_aggregate = lru.LRU(csz) -cache_aggregate_perm = lru.LRU(csz) - -perm_caching = malojaconfig["CACHE_DATABASE_PERM"] -temp_caching = malojaconfig["CACHE_DATABASE_SHORT"] - -cachestats = { - "cache_query":{ - "hits_perm":0, - "hits_tmp":0, - "misses":0, - "objperm":cache_query_perm, - "objtmp":cache_query, - "name":"Query Cache" - }, - "cache_aggregate":{ - "hits_perm":0, - "hits_tmp":0, - "misses":0, - "objperm":cache_aggregate_perm, - "objtmp":cache_aggregate, - "name":"Aggregate Cache" - } -} - -from doreah.regular import runhourly - -@runhourly -def log_stats(): - logstr = "{name}: {hitsperm} Perm Hits, {hitstmp} Tmp Hits, {misses} Misses; Current Size: {sizeperm}/{sizetmp}" - for s in (cachestats["cache_query"],cachestats["cache_aggregate"]): - log(logstr.format(name=s["name"],hitsperm=s["hits_perm"],hitstmp=s["hits_tmp"],misses=s["misses"], - sizeperm=len(s["objperm"]),sizetmp=len(s["objtmp"])),module="debug") - -def db_query_cached(**kwargs): - global cache_query, cache_query_perm - key = utilities.serialize(kwargs) - - eligible_permanent_caching = ( - "timerange" in kwargs and - not kwargs["timerange"].active() and - perm_caching - ) - eligible_temporary_caching = ( - not eligible_permanent_caching and - temp_caching - ) - - # hit permanent cache for past timeranges - if eligible_permanent_caching and key in cache_query_perm: - cachestats["cache_query"]["hits_perm"] += 1 - return copy.copy(cache_query_perm.get(key)) - - # hit short term cache - elif eligible_temporary_caching and key in cache_query: - cachestats["cache_query"]["hits_tmp"] += 1 - return copy.copy(cache_query.get(key)) - - else: - cachestats["cache_query"]["misses"] += 1 - result = db_query_full(**kwargs) - if eligible_permanent_caching: cache_query_perm[key] = result - elif eligible_temporary_caching: cache_query[key] = result - - if use_psutil: - reduce_caches_if_low_ram() - - return result - - -def db_aggregate_cached(**kwargs): - global cache_aggregate, cache_aggregate_perm - key = utilities.serialize(kwargs) - - eligible_permanent_caching = ( - "timerange" in kwargs and - not kwargs["timerange"].active() and - perm_caching - ) - eligible_temporary_caching = ( - not eligible_permanent_caching and - temp_caching - ) - - # hit permanent cache for past timeranges - if eligible_permanent_caching and key in cache_aggregate_perm: - cachestats["cache_aggregate"]["hits_perm"] += 1 - return copy.copy(cache_aggregate_perm.get(key)) - - # hit short term cache - elif eligible_temporary_caching and key in cache_aggregate: - cachestats["cache_aggregate"]["hits_tmp"] += 1 - return copy.copy(cache_aggregate.get(key)) - - else: - cachestats["cache_aggregate"]["misses"] += 1 - result = db_aggregate_full(**kwargs) - if eligible_permanent_caching: cache_aggregate_perm[key] = result - elif eligible_temporary_caching: cache_aggregate[key] = result - - if use_psutil: - reduce_caches_if_low_ram() - - return result - -def invalidate_caches(): - global cache_query, cache_aggregate - cache_query.clear() - cache_aggregate.clear() - log("Database caches invalidated.") - -def reduce_caches(to=0.75): - global cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm - for c in cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm: - currentsize = len(c) - if currentsize > 100: - targetsize = max(int(currentsize * to),10) - c.set_size(targetsize) - c.set_size(csz) - -def reduce_caches_if_low_ram(): - ramprct = psutil.virtual_memory().percent - if ramprct > cmp: - log("{prct}% RAM usage, reducing caches!".format(prct=ramprct),module="debug") - ratio = (cmp / ramprct) ** 3 - reduce_caches(to=ratio) - -#### -## Database queries -#### - - - -# Queries the database -def db_query_full(artist=None,artists=None,title=None,track=None,since=None,to=None,within=None,timerange=None,associated=False,max_=None): - - if not dbstatus['healthy']: raise DatabaseNotBuilt() - (since, to) = time_stamps(since=since,to=to,within=within,range=timerange) - - # this is not meant as a search function. we *can* query the db with a string, but it only works if it matches exactly - # if a title is specified, we assume that a specific track (with the exact artist combination) is requested - # if not, duplicate artist arguments are ignored - - #artist = None - - if artist is not None and isinstance(artist,str): - artist = ARTISTS.index(artist) - - # artists to numbers - if artists is not None: - artists = set([(ARTISTS.index(a) if isinstance(a,str) else a) for a in artists]) - - # track to number - if track is not None and isinstance(track,dict): - trackartists = set([(ARTISTS.index(a) if isinstance(a,str) else a) for a in track["artists"]]) - track = TRACKS.index((frozenset(trackartists),track["title"])) - artists = None - - #check if track is requested via title - if title!=None and track==None: - track = TRACKS.index((frozenset(artists),title)) - artists = None - - # if we're not looking for a track (either directly or per title artist arguments, which is converted to track above) - # we only need one artist - elif artist is None and track is None and artists is not None and len(artists) != 0: - artist = artists.pop() - - - # db query always reverse by default - - result = [] - - i = 0 - for s in scrobbles_in_range(since,to,reverse=True): - if i == max_: break - if (track is None or s[0] == track) and (artist is None or artist in TRACKS[s[0]][0] or associated and artist in coa.getCreditedList(TRACKS[s[0]][0])): - result.append(get_scrobble_dict(s)) - i += 1 - - return result - - # pointless to check for artist when track is checked because every track has a fixed set of artists, but it's more elegant this way - - -# Queries that... well... aggregate -def db_aggregate_full(by=None,since=None,to=None,within=None,timerange=None,artist=None): - - if not dbstatus['healthy']: raise DatabaseNotBuilt() - (since, to) = time_stamps(since=since,to=to,within=within,range=timerange) - - if isinstance(artist, str): - artist = ARTISTS.index(artist) - - if (by=="ARTIST"): - #this is probably a really bad idea - #for a in ARTISTS: - # num = len(db_query(artist=a,since=since,to=to)) - # - - # alright let's try for real - charts = {} - #for s in [scr for scr in SCROBBLES if since < scr[1] < to]: - for s in scrobbles_in_range(since,to): - artists = TRACKS[s[0]][0] - for a in coa.getCreditedList(artists): - # this either creates the new entry or increments the existing one - charts[a] = charts.setdefault(a,0) + 1 - - ls = [{"artist":get_artist_dict(ARTISTS[a]),"scrobbles":charts[a],"counting":[arti for arti in coa.getAllAssociated(ARTISTS[a]) if arti in ARTISTS]} for a in charts] - ls.sort(key=lambda k:k["scrobbles"],reverse=True) - # add ranks - for rnk in range(len(ls)): - if rnk == 0 or ls[rnk]["scrobbles"] < ls[rnk-1]["scrobbles"]: - ls[rnk]["rank"] = rnk + 1 - else: - ls[rnk]["rank"] = ls[rnk-1]["rank"] - return ls - - elif (by=="TRACK"): - charts = {} - #for s in [scr for scr in SCROBBLES if since < scr[1] < to and (artist==None or (artist in TRACKS[scr[0]][0]))]: - for s in [scr for scr in scrobbles_in_range(since,to) if (artist is None or (artist in TRACKS[scr[0]][0]))]: - track = s[0] - # this either creates the new entry or increments the existing one - charts[track] = charts.setdefault(track,0) + 1 - - ls = [{"track":get_track_dict(TRACKS[t]),"scrobbles":charts[t]} for t in charts] - ls.sort(key=lambda k:k["scrobbles"],reverse=True) - # add ranks - for rnk in range(len(ls)): - if rnk == 0 or ls[rnk]["scrobbles"] < ls[rnk-1]["scrobbles"]: - ls[rnk]["rank"] = rnk + 1 - else: - ls[rnk]["rank"] = ls[rnk-1]["rank"] - return ls - - else: - #return len([scr for scr in SCROBBLES if since < scr[1] < to]) - return len(list(scrobbles_in_range(since,to))) - - -# Search for strings -def db_search(query,type=None): - results = [] - if type=="ARTIST": - results = [a for a in ARTISTS if simplestr(query) in simplestr(a)] - if type=="TRACK": - results = [ - get_track_dict(t) for t in TRACKS if simplestr(query) in simplestr(t[1]) - ] - return results - - -#### -## Useful functions -#### - -# makes a string usable for searching (special characters are blanks, accents and stuff replaced with their real part) -def simplestr(input,ignorecapitalization=True): - norm = unicodedata.normalize("NFKD",input) - norm = [c for c in norm if not unicodedata.combining(c)] - norm = [c if len(c.encode())==1 else " " for c in norm] - clear = ''.join(c for c in norm) - if ignorecapitalization: clear = clear.lower() - return clear - - - -#def getArtistId(nameorid): -# if isinstance(nameorid,int): -# return nameorid -# else: -# try: -# return ARTISTS.index(nameorid) -# except: -# return -1 - - -def insert(list_,item,key=lambda x:x): - i = 0 - while len(list_) > i: - if key(list_[i]) > key(item): - list_.insert(i,item) - return i - i += 1 - - list_.append(item) - return i - - -def scrobbles_in_range(start,end,reverse=False): - if reverse: - for stamp in reversed(STAMPS): - #print("Checking " + str(stamp)) - if stamp < start: return - if stamp > end: continue - yield SCROBBLESDICT[stamp] - else: - for stamp in STAMPS: - #print("Checking " + str(stamp)) - if stamp < start: continue - if stamp > end: return - yield SCROBBLESDICT[stamp] diff --git a/maloja/database/__init__.py b/maloja/database/__init__.py new file mode 100644 index 0000000..ac3ec86 --- /dev/null +++ b/maloja/database/__init__.py @@ -0,0 +1,420 @@ +# server +from bottle import request, response, FormsDict, HTTPError + +# rest of the project +from ..cleanup import CleanerAgent +from .. import images +from ..malojatime import register_scrobbletime, time_stamps, ranges, alltime +from ..malojauri import uri_to_internal, internal_to_uri, compose_querystring +from ..thirdparty import proxy_scrobble_all +from ..globalconf import data_dir, malojaconfig +from ..apis import apikeystore +#db +from . import sqldb +from . import cached +from . import dbcache + +# doreah toolkit +from doreah.logging import log +from doreah.auth import authenticated_api, authenticated_api_with_alternate +import doreah + + + + +# technical +import os +import datetime +import sys +import unicodedata +from collections import namedtuple +from threading import Lock +import yaml, json +import math + +# url handling +import urllib + + + +dbstatus = { + "healthy":False, # we can access the db + "rebuildinprogress":False, + "complete":False # information is complete +} +class DatabaseNotBuilt(HTTPError): + def __init__(self): + super().__init__( + status=503, + body="The Maloja Database is being upgraded to Version 3. This could take several minutes.", + headers={"Retry-After":120} + ) + + +def waitfordb(func): + def newfunc(*args,**kwargs): + if not dbstatus['healthy']: raise DatabaseNotBuilt() + return func(*args,**kwargs) + return newfunc + + + +ISSUES = {} + +cla = CleanerAgent() + + + + + +## this function accepts a flat dict - all info of the scrobble should be top level key +## but can contain a list as value +## the following keys are valid: +## scrobble_duration int +## scrobble_time int +## track_title str, mandatory +## track_artists list, mandatory +## track_length int +## album_name str +## album_artists list +## +## +## +## +## +## + +def incoming_scrobble(rawscrobble,fix=True,client=None,api=None,dbconn=None): + + if (not "track_artists" in rawscrobble) or (len(rawscrobble['track_artists']) == 0) or (not "track_title" in rawscrobble): + log(f"Invalid Scrobble [Client: {client} | API: {api}]: {rawscrobble} ",color='red') + #return {"status":"failure"} + return False + + log(f"Incoming scrobble [Client: {client} | API: {api}]: {rawscrobble}") + + # raw scrobble to processed info + scrobbleinfo = {**rawscrobble} + if fix: + scrobbleinfo['track_artists'],scrobbleinfo['track_title'] = cla.fullclean(scrobbleinfo['track_artists'],scrobbleinfo['track_title']) + scrobbleinfo['scrobble_time'] = scrobbleinfo.get('scrobble_time') or int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()) + + # processed info to internal scrobble dict + scrobbledict = { + "time":scrobbleinfo.get('scrobble_time'), + "track":{ + "artists":scrobbleinfo.get('track_artists'), + "title":scrobbleinfo.get('track_title'), + "album":{ + "name":scrobbleinfo.get('album_name'), + "artists":scrobbleinfo.get('album_artists') + }, + "length":scrobbleinfo.get('track_length') + }, + "duration":scrobbleinfo.get('scrobble_duration'), + "origin":f"client:{client}" if client else "generic", + "extra":{ + k:scrobbleinfo[k] for k in scrobbleinfo if k not in + ['scrobble_time','track_artists','track_title','track_length','scrobble_duration','album_name','album_artists'] + }, + "rawscrobble":rawscrobble + } + + + sqldb.add_scrobble(scrobbledict,dbconn=dbconn) + proxy_scrobble_all(scrobbledict['track']['artists'],scrobbledict['track']['title'],scrobbledict['time']) + + dbcache.invalidate_caches(scrobbledict['time']) + + #return {"status":"success","scrobble":scrobbledict} + return scrobbledict + + + +@waitfordb +def remove_scrobble(timestamp): + log(f"Deleting Scrobble {timestamp}") + result = sqldb.delete_scrobble(timestamp) + dbcache.invalidate_caches(timestamp) + + + + + + + + +@waitfordb +def get_scrobbles(dbconn=None,**keys): + (since,to) = keys.get('timerange').timestamps() + if 'artist' in keys: + result = sqldb.get_scrobbles_of_artist(artist=keys['artist'],since=since,to=to,dbconn=dbconn) + elif 'track' in keys: + result = sqldb.get_scrobbles_of_track(track=keys['track'],since=since,to=to,dbconn=dbconn) + else: + result = sqldb.get_scrobbles(since=since,to=to,dbconn=dbconn) + #return result[keys['page']*keys['perpage']:(keys['page']+1)*keys['perpage']] + return list(reversed(result)) + +@waitfordb +def get_scrobbles_num(dbconn=None,**keys): + (since,to) = keys.get('timerange').timestamps() + if 'artist' in keys: + result = len(sqldb.get_scrobbles_of_artist(artist=keys['artist'],since=since,to=to,resolve_references=False,dbconn=dbconn)) + elif 'track' in keys: + result = len(sqldb.get_scrobbles_of_track(track=keys['track'],since=since,to=to,resolve_references=False,dbconn=dbconn)) + else: + result = sqldb.get_scrobbles_num(since=since,to=to,dbconn=dbconn) + return result + + + +@waitfordb +def get_tracks(dbconn=None,**keys): + if keys.get('artist') is None: + result = sqldb.get_tracks(dbconn=dbconn) + else: + result = sqldb.get_tracks_of_artist(keys.get('artist'),dbconn=dbconn) + return result + +@waitfordb +def get_artists(dbconn=None): + return sqldb.get_artists(dbconn=dbconn) + + +@waitfordb +def get_charts_artists(dbconn=None,**keys): + (since,to) = keys.get('timerange').timestamps() + result = sqldb.count_scrobbles_by_artist(since=since,to=to,dbconn=dbconn) + return result + +@waitfordb +def get_charts_tracks(dbconn=None,**keys): + (since,to) = keys.get('timerange').timestamps() + if 'artist' in keys: + result = sqldb.count_scrobbles_by_track_of_artist(since=since,to=to,artist=keys['artist'],dbconn=dbconn) + else: + result = sqldb.count_scrobbles_by_track(since=since,to=to,dbconn=dbconn) + return result + +@waitfordb +def get_pulse(dbconn=None,**keys): + + rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) + results = [] + for rng in rngs: + res = get_scrobbles_num(timerange=rng,**{k:keys[k] for k in keys if k != 'timerange'},dbconn=dbconn) + results.append({"range":rng,"scrobbles":res}) + + return results + +@waitfordb +def get_performance(dbconn=None,**keys): + + rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) + results = [] + + for rng in rngs: + if "track" in keys: + track = sqldb.get_track(sqldb.get_track_id(keys['track'],dbconn=dbconn),dbconn=dbconn) + charts = get_charts_tracks(timerange=rng,dbconn=dbconn) + rank = None + for c in charts: + if c["track"] == track: + rank = c["rank"] + break + elif "artist" in keys: + artist = sqldb.get_artist(sqldb.get_artist_id(keys['artist'],dbconn=dbconn),dbconn=dbconn) + # ^this is the most useless line in programming history + # but I like consistency + charts = get_charts_artists(timerange=rng,dbconn=dbconn) + rank = None + for c in charts: + if c["artist"] == artist: + rank = c["rank"] + break + results.append({"range":rng,"rank":rank}) + + return results + +@waitfordb +def get_top_artists(dbconn=None,**keys): + + rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) + results = [] + + for rng in rngs: + try: + res = get_charts_artists(timerange=rng,dbconn=dbconn)[0] + results.append({"range":rng,"artist":res["artist"],"scrobbles":res["scrobbles"]}) + except: + results.append({"range":rng,"artist":None,"scrobbles":0}) + + return results + + +@waitfordb +def get_top_tracks(dbconn=None,**keys): + + rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) + results = [] + + for rng in rngs: + try: + res = get_charts_tracks(timerange=rng,dbconn=dbconn)[0] + results.append({"range":rng,"track":res["track"],"scrobbles":res["scrobbles"]}) + except: + results.append({"range":rng,"track":None,"scrobbles":0}) + + return results + +@waitfordb +def artist_info(dbconn=None,**keys): + + artist = keys.get('artist') + + artist = sqldb.get_artist(sqldb.get_artist_id(artist,dbconn=dbconn),dbconn=dbconn) + alltimecharts = get_charts_artists(timerange=alltime(),dbconn=dbconn) + scrobbles = get_scrobbles_num(artist=artist,timerange=alltime(),dbconn=dbconn) + #we cant take the scrobble number from the charts because that includes all countas scrobbles + try: + c = [e for e in alltimecharts if e["artist"] == artist][0] + others = sqldb.get_associated_artists(artist,dbconn=dbconn) + position = c["rank"] + return { + "artist":artist, + "scrobbles":scrobbles, + "position":position, + "associated":others, + "medals":{ + "gold": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['gold']], + "silver": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['silver']], + "bronze": [year for year in cached.medals_artists if artist in cached.medals_artists[year]['bronze']], + }, + "topweeks":len([e for e in cached.weekly_topartists if e == artist]) + } + except: + # if the artist isnt in the charts, they are not being credited and we + # need to show information about the credited one + replaceartist = sqldb.get_credited_artists(artist)[0] + c = [e for e in alltimecharts if e["artist"] == replaceartist][0] + position = c["rank"] + return {"artist":artist,"replace":replaceartist,"scrobbles":scrobbles,"position":position} + + + + +@waitfordb +def track_info(dbconn=None,**keys): + + track = keys.get('track') + + track = sqldb.get_track(sqldb.get_track_id(track,dbconn=dbconn),dbconn=dbconn) + alltimecharts = get_charts_tracks(timerange=alltime(),dbconn=dbconn) + #scrobbles = get_scrobbles_num(track=track,timerange=alltime()) + + c = [e for e in alltimecharts if e["track"] == track][0] + scrobbles = c["scrobbles"] + position = c["rank"] + cert = None + threshold_gold, threshold_platinum, threshold_diamond = malojaconfig["SCROBBLES_GOLD","SCROBBLES_PLATINUM","SCROBBLES_DIAMOND"] + if scrobbles >= threshold_diamond: cert = "diamond" + elif scrobbles >= threshold_platinum: cert = "platinum" + elif scrobbles >= threshold_gold: cert = "gold" + + + return { + "track":track, + "scrobbles":scrobbles, + "position":position, + "medals":{ + "gold": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['gold']], + "silver": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['silver']], + "bronze": [year for year in cached.medals_tracks if track in cached.medals_tracks[year]['bronze']], + }, + "certification":cert, + "topweeks":len([e for e in cached.weekly_toptracks if e == track]) + } + + + +def get_predefined_rulesets(dbconn=None): + validchars = "-_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + + rulesets = [] + + for f in os.listdir(data_dir['rules']("predefined")): + if f.endswith(".tsv"): + + rawf = f.replace(".tsv","") + valid = all(char in validchars for char in rawf) + if not valid: continue + if "_" not in rawf: continue + + try: + with open(data_dir['rules']("predefined",f)) as tsvfile: + line1 = tsvfile.readline() + line2 = tsvfile.readline() + + if "# NAME: " in line1: + name = line1.replace("# NAME: ","") + else: name = rawf.split("_")[1] + desc = line2.replace("# DESC: ","") if "# DESC: " in line2 else "" + author = rawf.split("_")[0] + except: + continue + + ruleset = {"file":rawf} + rulesets.append(ruleset) + ruleset["active"] = bool(os.path.exists(data_dir['rules'](f))) + ruleset["name"] = name + ruleset["author"] = author + ruleset["desc"] = desc + + return rulesets + + +#### +## Server operation +#### + + + +def start_db(): + # Upgrade database + from .. import upgrade + upgrade.upgrade_db(sqldb.add_scrobbles) + + # Load temporary tables + from . import associated + associated.load_associated_rules() + + dbstatus['healthy'] = True + + # inform time module about begin of scrobbling + try: + firstscrobble = sqldb.get_scrobbles()[0] + register_scrobbletime(firstscrobble['time']) + except IndexError: + register_scrobbletime(int(datetime.datetime.now().timestamp())) + + + # create cached information + cached.update_medals() + cached.update_weekly() + + dbstatus['complete'] = True + + + + + + +# Search for strings +def db_search(query,type=None): + results = [] + if type=="ARTIST": + results = [a for a in sqldb.get_artists() if sqldb.normalize_name(query) in sqldb.normalize_name(a)] + if type=="TRACK": + results = [t for t in sqldb.get_tracks() if sqldb.normalize_name(query) in sqldb.normalize_name(t['title'])] + return results diff --git a/maloja/database/associated.py b/maloja/database/associated.py new file mode 100644 index 0000000..00f6a5f --- /dev/null +++ b/maloja/database/associated.py @@ -0,0 +1,49 @@ +## dealing with loading the associated artists rules into a database +## right now this is kind of absurd because we're storing it in a db while not +## actually using its permanence, but this makes it possible to use the information +## directly in sql + + +import csv +import os + +from . import sqldb +from ..globalconf import data_dir + + +def load_associated_rules(): + # delete old + with sqldb.engine.begin() as conn: + op = sqldb.DB['associated_artists'].delete().where() + conn.execute(op) + + # load from file + rawrules = [] + for f in os.listdir(data_dir["rules"]()): + if f.split('.')[-1].lower() != 'tsv': continue + filepath = data_dir["rules"](f) + with open(filepath,'r') as filed: + reader = csv.reader(filed,delimiter="\t") + rawrules += [[col for col in entry if col] for entry in reader if len(entry)>0 and not entry[0].startswith('#')] + rules = [{'source_artist':r[1],'target_artist':r[2]} for r in rawrules if r[0]=="countas"] + + #for rule in rules: + # print(f"Rule to replace {rule['source_artist']} with {rule['target_artist']}:") + # test = {k:sqldb.get_artist_id(rule[k],create_new=False) for k in rule} + # if test['source_artist'] is None: print("axed") + + #allartists = set([*[r['source_artist'] for r in rules],*[r['target_artist'] for r in rules]]) + + # find ids + rules = [{k:sqldb.get_artist_id(rule[k],create_new=False) for k in rule} for rule in rules] + rules = [r for r in rules if r['source_artist'] is not None] + + # write to db + ops = [ + sqldb.DB['associated_artists'].insert().values(**r).prefix_with('OR IGNORE') + for r in rules + ] + + with sqldb.engine.begin() as conn: + for op in ops: + conn.execute(op) diff --git a/maloja/database/cached.py b/maloja/database/cached.py new file mode 100644 index 0000000..7967665 --- /dev/null +++ b/maloja/database/cached.py @@ -0,0 +1,69 @@ +# for information that is not authorative, but should be saved anyway because it +# changes infrequently and DB access is expensive + +from doreah.regular import runyearly, rundaily +from .. import database +from .. import malojatime as mjt + + + +medals_artists = { + # year: {'gold':[],'silver':[],'bronze':[]} +} +medals_tracks = { + # year: {'gold':[],'silver':[],'bronze':[]} +} + +weekly_topartists = [] +weekly_toptracks = [] + +@runyearly +def update_medals(): + + global medals_artists, medals_tracks + medals_artists.clear() + medals_tracks.clear() + + for year in mjt.ranges(step="year"): + if year == mjt.thisyear(): break + + charts_artists = database.get_charts_artists(timerange=year) + charts_tracks = database.get_charts_tracks(timerange=year) + + entry_artists = {'gold':[],'silver':[],'bronze':[]} + entry_tracks = {'gold':[],'silver':[],'bronze':[]} + medals_artists[year.desc()] = entry_artists + medals_tracks[year.desc()] = entry_tracks + + for entry in charts_artists: + if entry['rank'] == 1: entry_artists['gold'].append(entry['artist']) + elif entry['rank'] == 2: entry_artists['silver'].append(entry['artist']) + elif entry['rank'] == 3: entry_artists['bronze'].append(entry['artist']) + else: break + for entry in charts_tracks: + if entry['rank'] == 1: entry_tracks['gold'].append(entry['track']) + elif entry['rank'] == 2: entry_tracks['silver'].append(entry['track']) + elif entry['rank'] == 3: entry_tracks['bronze'].append(entry['track']) + else: break + + + +@rundaily +def update_weekly(): + + global weekly_topartists, weekly_toptracks + weekly_topartists.clear() + weekly_toptracks.clear() + + for week in mjt.ranges(step="week"): + if week == mjt.thisweek(): break + + charts_artists = database.get_charts_artists(timerange=week) + charts_tracks = database.get_charts_tracks(timerange=week) + + for entry in charts_artists: + if entry['rank'] == 1: weekly_topartists.append(entry['artist']) + else: break + for entry in charts_tracks: + if entry['rank'] == 1: weekly_toptracks.append(entry['track']) + else: break diff --git a/maloja/database/dbcache.py b/maloja/database/dbcache.py new file mode 100644 index 0000000..b3d4ce3 --- /dev/null +++ b/maloja/database/dbcache.py @@ -0,0 +1,143 @@ +# the more generalized caching for DB queries +# mostly to avoid long loading times for pages that show lots of information +# that changes very infrequently or not at all + +import lru +import psutil +import json +from doreah.regular import runhourly +from doreah.logging import log + +from ..globalconf import malojaconfig + +HIGH_NUMBER = 1000000 +CACHE_SIZE = 10000 +ENTITY_CACHE_SIZE = 1000000 +CACHE_ADJUST_STEP = 100 + +cache = lru.LRU(CACHE_SIZE) +entitycache = lru.LRU(ENTITY_CACHE_SIZE) + +hits, misses = 0, 0 + + + +@runhourly +def maintenance(): + if malojaconfig['USE_GLOBAL_CACHE']: + print_stats() + trim_cache() + +def print_stats(): + log(f"Cache Size: {len(cache)} [{len(entitycache)} E], System RAM Utilization: {psutil.virtual_memory().percent}%, Cache Hits: {hits}/{hits+misses}") + #print("Full rundown:") + #import sys + #for k in cache.keys(): + # print(f"\t{k}\t{sys.getsizeof(cache[k])}") + + +def cached_wrapper(inner_func): + + if not malojaconfig['USE_GLOBAL_CACHE']: return inner_func + def outer_func(*args,**kwargs): + if 'dbconn' in kwargs: + conn = kwargs.pop('dbconn') + else: + conn = None + global hits, misses + key = (serialize(args),serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to")) + + if key in cache: + hits += 1 + return cache.get(key) + + else: + misses += 1 + result = inner_func(*args,**kwargs,dbconn=conn) + cache[key] = result + return result + + return outer_func + + +# cache for functions that call with a whole list of entity ids +# we don't want a new cache entry for every single combination, but keep a common +# cache that's aware of what we're calling +def cached_wrapper_individual(inner_func): + + if not malojaconfig['USE_GLOBAL_CACHE']: return inner_func + def outer_func(set_arg,**kwargs): + + + if 'dbconn' in kwargs: + conn = kwargs.pop('dbconn') + else: + conn = None + + #global hits, misses + result = {} + for id in set_arg: + if (inner_func,id) in entitycache: + result[id] = entitycache[(inner_func,id)] + #hits += 1 + else: + pass + #misses += 1 + + + remaining = inner_func(set(e for e in set_arg if e not in result),dbconn=conn) + for id in remaining: + entitycache[(inner_func,id)] = remaining[id] + result[id] = remaining[id] + + return result + + return outer_func + +def invalidate_caches(scrobbletime): + if malojaconfig['USE_GLOBAL_CACHE']: + cleared, kept = 0, 0 + for k in cache.keys(): + # VERY BIG TODO: differentiate between None as in 'unlimited timerange' and None as in 'time doesnt matter here'! + if (k[3] is None or scrobbletime >= k[3]) and (k[4] is None or scrobbletime <= k[4]): + cleared += 1 + del cache[k] + else: + kept += 1 + log(f"Invalidated {cleared} of {cleared+kept} DB cache entries") + + +def invalidate_entity_cache(): + entitycache.clear() + + +def trim_cache(): + ramprct = psutil.virtual_memory().percent + if ramprct > malojaconfig["DB_MAX_MEMORY"]: + log(f"{ramprct}% RAM usage, clearing cache and adjusting size!") + #ratio = 0.6 + #targetsize = max(int(len(cache) * ratio),50) + #log(f"Reducing to {targetsize} entries") + #cache.set_size(targetsize) + #cache.set_size(HIGH_NUMBER) + cache.clear() + if cache.get_size() > CACHE_ADJUST_STEP: + cache.set_size(cache.get_size() - CACHE_ADJUST_STEP) + + #log(f"New RAM usage: {psutil.virtual_memory().percent}%") + print_stats() + + + +def serialize(obj): + try: + return serialize(obj.hashable()) + except: + try: + return json.dumps(obj) + except: + if isinstance(obj, (list, tuple, set)): + return "[" + ",".join(serialize(o) for o in obj) + "]" + elif isinstance(obj,dict): + return "{" + ",".join(serialize(o) + ":" + serialize(obj[o]) for o in obj) + "}" + return json.dumps(obj.hashable()) diff --git a/maloja/database/jinjaview.py b/maloja/database/jinjaview.py new file mode 100644 index 0000000..ab19e73 --- /dev/null +++ b/maloja/database/jinjaview.py @@ -0,0 +1,50 @@ +from .. import database +from . sqldb import engine + +from .dbcache import serialize + +from ..globalconf import malojaconfig + +from doreah.logging import log + + +# this is a wrapper object that provides a DB connection so that one jinja page +# (with all its included partials) can use it for all functions +# it also translates the non-unpacked calls to unpacked calls that the DB wants +# it also maintains a request-local cache since many webpages use the same stats +# several times +class JinjaDBConnection: + def __init__(self): + self.cache = {} + self.hits = 0 + self.misses = 0 + def __enter__(self): + self.conn = engine.connect() + return self + def __exit__(self, exc_type, exc_value, exc_traceback): + self.conn.close() + log(f"Generated page with {self.hits}/{self.hits+self.misses} local Cache hits",module="debug_performance") + del self.cache + def __getattr__(self,name): + originalmethod = getattr(database,name) + + def packedmethod(*keys): + kwargs = {} + for k in keys: + kwargs.update(k) + if malojaconfig['USE_REQUEST_CACHE']: + cachekey = serialize((id(originalmethod),kwargs)) + if cachekey in self.cache: + self.hits += 1 + return self.cache[cachekey] + else: + self.misses += 1 + result = originalmethod(**kwargs,dbconn=self.conn) + self.cache[cachekey] = result + return result + else: + result = originalmethod(**kwargs,dbconn=self.conn) + return result + + + return packedmethod diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py new file mode 100644 index 0000000..629dc7c --- /dev/null +++ b/maloja/database/sqldb.py @@ -0,0 +1,780 @@ +import sqlalchemy as sql +import json +import unicodedata +import math +from datetime import datetime +from threading import Lock + +from ..globalconf import data_dir +from .dbcache import cached_wrapper, cached_wrapper_individual + +from doreah.logging import log +from doreah.regular import runhourly, runmonthly + + + +##### DB Technical + + +DBTABLES = { + # name - type - foreign key - kwargs + 'scrobbles':{ + 'columns':[ + ("timestamp", sql.Integer, {'primary_key':True}), + ("rawscrobble", sql.String, {}), + ("origin", sql.String, {}), + ("duration", sql.Integer, {}), + ("track_id", sql.Integer, sql.ForeignKey('tracks.id'), {}), + ("extra", sql.String, {}) + ], + 'extraargs':(),'extrakwargs':{} + }, + 'tracks':{ + 'columns':[ + ("id", sql.Integer, {'primary_key':True}), + ("title", sql.String, {}), + ("title_normalized",sql.String, {}), + ("length", sql.Integer, {}) + ], + 'extraargs':(),'extrakwargs':{'sqlite_autoincrement':True} + }, + 'artists':{ + 'columns':[ + ("id", sql.Integer, {'primary_key':True}), + ("name", sql.String, {}), + ("name_normalized", sql.String, {}) + ], + 'extraargs':(),'extrakwargs':{'sqlite_autoincrement':True} + }, + 'trackartists':{ + 'columns':[ + ("id", sql.Integer, {'primary_key':True}), + ("artist_id", sql.Integer, sql.ForeignKey('artists.id'), {}), + ("track_id", sql.Integer, sql.ForeignKey('tracks.id'), {}) + ], + 'extraargs':(sql.UniqueConstraint('artist_id', 'track_id'),),'extrakwargs':{} + }, + 'associated_artists':{ + 'columns':[ + ("source_artist", sql.Integer, sql.ForeignKey('artists.id'), {}), + ("target_artist", sql.Integer, sql.ForeignKey('artists.id'), {}) + ], + 'extraargs':(sql.UniqueConstraint('source_artist', 'target_artist'),),'extrakwargs':{} + } +} + + + + +DB = {} + +engine = sql.create_engine(f"sqlite:///{data_dir['scrobbles']('malojadb.sqlite')}", echo = False) +meta = sql.MetaData() + +# create table definitions +for tablename in DBTABLES: + + DB[tablename] = sql.Table( + tablename, meta, + *[sql.Column(colname,*args,**kwargs) for colname,*args,kwargs in DBTABLES[tablename]['columns']], + *DBTABLES[tablename]['extraargs'], + **DBTABLES[tablename]['extrakwargs'] + ) + +# actually create tables for new databases +meta.create_all(engine) + +# upgrade old database with new columns +with engine.begin() as conn: + for tablename in DBTABLES: + info = DBTABLES[tablename] + table = DB[tablename] + + for colname,datatype,*args,kwargs in info['columns']: + try: + statement = f"ALTER TABLE {tablename} ADD {colname} {datatype().compile()}" + conn.execute(sql.text(statement)) + log(f"Column {colname} was added to table {tablename}!") + # TODO figure out how to compile foreign key references! + except sql.exc.OperationalError as e: + pass + + +# adding a scrobble could consist of multiple write operations that sqlite doesn't +# see as belonging together +SCROBBLE_LOCK = Lock() + + +# decorator that passes either the provided dbconn, or creates a separate one +# just for this function call +def connection_provider(func): + + def wrapper(*args,**kwargs): + if kwargs.get("dbconn") is not None: + return func(*args,**kwargs) + else: + with engine.connect() as connection: + kwargs['dbconn'] = connection + return func(*args,**kwargs) + return wrapper + +##### DB <-> Dict translations + +## ATTENTION ALL ADVENTURERS +## this is what a scrobble dict will look like from now on +## this is the single canonical source of truth +## stop making different little dicts in every single function +## this is the schema that will definitely 100% stay like this and not +## randomly get changed two versions later +## here we go +# +# { +# "time":int, +# "track":{ +# "artists":list, +# "title":string, +# "album":{ +# "name":string, +# "artists":list +# }, +# "length":None +# }, +# "duration":int, +# "origin":string, +# "extra":{string-keyed mapping for all flags with the scrobble}, +# "rawscrobble":{string-keyed mapping of the original scrobble received} +# } +# +# The last two fields are not returned under normal circumstances + + + + +##### Conversions between DB and dicts + +# These should work on whole lists and collect all the references, +# then look them up once and fill them in + + +### DB -> DICT +def scrobbles_db_to_dict(rows,include_internal=False): + tracks = get_tracks_map(set(row.track_id for row in rows)) + return [ + { + **{ + "time":row.timestamp, + "track":tracks[row.track_id], + "duration":row.duration, + "origin":row.origin, + }, + **({ + "extra":json.loads(row.extra or '{}'), + "rawscrobble":json.loads(row.rawscrobble or '{}') + } if include_internal else {}) + } + + for row in rows + ] + +def scrobble_db_to_dict(row): + return scrobbles_db_to_dict([row])[0] + +def tracks_db_to_dict(rows): + artists = get_artists_of_tracks(set(row.id for row in rows)) + return [ + { + "artists":artists[row.id], + "title":row.title, + #"album": + "length":row.length + } + for row in rows + ] + +def track_db_to_dict(row): + return tracks_db_to_dict([row])[0] + +def artists_db_to_dict(rows): + return [ + row.name + for row in rows + ] + +def artist_db_to_dict(row): + return artists_db_to_dict([row])[0] + + + + +### DICT -> DB + +def scrobble_dict_to_db(info): + return { + "timestamp":info['time'], + "origin":info['origin'], + "duration":info['duration'], + "track_id":get_track_id(info['track']), + "extra":json.dumps(info.get('extra',{})), + "rawscrobble":json.dumps(info.get('rawscrobble',{})) + } + +def track_dict_to_db(info): + return { + "title":info['title'], + "title_normalized":normalize_name(info['title']), + "length":info.get('length') + } + +def artist_dict_to_db(info): + return { + "name": info, + "name_normalized":normalize_name(info) + } + + + + + +##### Actual Database interactions + + +@connection_provider +def add_scrobble(scrobbledict,dbconn=None): + add_scrobbles([scrobbledict],dbconn=dbconn) + +@connection_provider +def add_scrobbles(scrobbleslist,dbconn=None): + + with SCROBBLE_LOCK: + + ops = [ + DB['scrobbles'].insert().values( + **scrobble_dict_to_db(s) + ) for s in scrobbleslist + ] + + success,errors = 0,0 + for op in ops: + try: + dbconn.execute(op) + success += 1 + except sql.exc.IntegrityError as e: + errors += 1 + + # TODO check if actual duplicate + + if errors > 0: log(f"{errors} Scrobbles have not been written to database!",color='red') + return success,errors + +@connection_provider +def delete_scrobble(scrobble_id,dbconn=None): + + with SCROBBLE_LOCK: + + op = DB['scrobbles'].delete().where( + DB['scrobbles'].c.timestamp == scrobble_id + ) + + dbconn.execute(op) + +### these will 'get' the ID of an entity, creating it if necessary + +@cached_wrapper +@connection_provider +def get_track_id(trackdict,dbconn=None): + ntitle = normalize_name(trackdict['title']) + artist_ids = [get_artist_id(a) for a in trackdict['artists']] + + + + + op = DB['tracks'].select( + DB['tracks'].c.id + ).where( + DB['tracks'].c.title_normalized==ntitle + ) + result = dbconn.execute(op).all() + for row in result: + # check if the artists are the same + foundtrackartists = [] + + op = DB['trackartists'].select( + DB['trackartists'].c.artist_id + ).where( + DB['trackartists'].c.track_id==row[0] + ) + result = dbconn.execute(op).all() + match_artist_ids = [r.artist_id for r in result] + #print("required artists",artist_ids,"this match",match_artist_ids) + if set(artist_ids) == set(match_artist_ids): + #print("ID for",trackdict['title'],"was",row[0]) + return row.id + + + op = DB['tracks'].insert().values( + **track_dict_to_db(trackdict) + ) + result = dbconn.execute(op) + track_id = result.inserted_primary_key[0] + + for artist_id in artist_ids: + op = DB['trackartists'].insert().values( + track_id=track_id, + artist_id=artist_id + ) + result = dbconn.execute(op) + #print("Created",trackdict['title'],track_id) + return track_id + +@cached_wrapper +@connection_provider +def get_artist_id(artistname,create_new=True,dbconn=None): + nname = normalize_name(artistname) + #print("looking for",nname) + + op = DB['artists'].select( + DB['artists'].c.id + ).where( + DB['artists'].c.name_normalized==nname + ) + result = dbconn.execute(op).all() + for row in result: + #print("ID for",artistname,"was",row[0]) + return row.id + + if not create_new: return None + + op = DB['artists'].insert().values( + name=artistname, + name_normalized=nname + ) + result = dbconn.execute(op) + #print("Created",artistname,result.inserted_primary_key) + return result.inserted_primary_key[0] + + + + + +### Functions that get rows according to parameters + +@cached_wrapper +@connection_provider +def get_scrobbles_of_artist(artist,since=None,to=None,resolve_references=True,dbconn=None): + + if since is None: since=0 + if to is None: to=now() + + artist_id = get_artist_id(artist) + + jointable = sql.join(DB['scrobbles'],DB['trackartists'],DB['scrobbles'].c.track_id == DB['trackartists'].c.track_id) + + op = jointable.select().where( + DB['scrobbles'].c.timestamp<=to, + DB['scrobbles'].c.timestamp>=since, + DB['trackartists'].c.artist_id==artist_id + ).order_by(sql.asc('timestamp')) + result = dbconn.execute(op).all() + + if resolve_references: + result = scrobbles_db_to_dict(result) + #result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for row in result] + return result + +@cached_wrapper +@connection_provider +def get_scrobbles_of_track(track,since=None,to=None,resolve_references=True,dbconn=None): + + if since is None: since=0 + if to is None: to=now() + + track_id = get_track_id(track) + + op = DB['scrobbles'].select().where( + DB['scrobbles'].c.timestamp<=to, + DB['scrobbles'].c.timestamp>=since, + DB['scrobbles'].c.track_id==track_id + ).order_by(sql.asc('timestamp')) + result = dbconn.execute(op).all() + + if resolve_references: + result = scrobbles_db_to_dict(result) + #result = [scrobble_db_to_dict(row) for row in result] + return result + +@cached_wrapper +@connection_provider +def get_scrobbles(since=None,to=None,resolve_references=True,dbconn=None): + + if since is None: since=0 + if to is None: to=now() + + op = DB['scrobbles'].select().where( + DB['scrobbles'].c.timestamp<=to, + DB['scrobbles'].c.timestamp>=since, + ).order_by(sql.asc('timestamp')) + result = dbconn.execute(op).all() + + if resolve_references: + result = scrobbles_db_to_dict(result) + #result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for i,row in enumerate(result) if i=since, + ) + result = dbconn.execute(op).all() + + return result[0][0] + +@cached_wrapper +@connection_provider +def get_artists_of_track(track_id,resolve_references=True,dbconn=None): + + op = DB['trackartists'].select().where( + DB['trackartists'].c.track_id==track_id + ) + result = dbconn.execute(op).all() + + artists = [get_artist(row.artist_id) if resolve_references else row.artist_id for row in result] + return artists + + +@cached_wrapper +@connection_provider +def get_tracks_of_artist(artist,dbconn=None): + + artist_id = get_artist_id(artist) + + op = sql.join(DB['tracks'],DB['trackartists']).select().where( + DB['trackartists'].c.artist_id==artist_id + ) + result = dbconn.execute(op).all() + + return tracks_db_to_dict(result) + +@cached_wrapper +@connection_provider +def get_artists(dbconn=None): + + op = DB['artists'].select() + result = dbconn.execute(op).all() + + return artists_db_to_dict(result) + +@cached_wrapper +@connection_provider +def get_tracks(dbconn=None): + + op = DB['tracks'].select() + result = dbconn.execute(op).all() + + return tracks_db_to_dict(result) + +### functions that count rows for parameters + +@cached_wrapper +@connection_provider +def count_scrobbles_by_artist(since,to,dbconn=None): + jointable = sql.join( + DB['scrobbles'], + DB['trackartists'], + DB['scrobbles'].c.track_id == DB['trackartists'].c.track_id + ) + + jointable2 = sql.join( + jointable, + DB['associated_artists'], + DB['trackartists'].c.artist_id == DB['associated_artists'].c.source_artist, + isouter=True + ) + op = sql.select( + sql.func.count(sql.func.distinct(DB['scrobbles'].c.timestamp)).label('count'), + # only count distinct scrobbles - because of artist replacement, we could end up + # with two artists of the same scrobble counting it twice for the same artist + # e.g. Irene and Seulgi adding two scrobbles to Red Velvet for one real scrobble + sql.func.coalesce(DB['associated_artists'].c.target_artist,DB['trackartists'].c.artist_id).label('artist_id') + # use the replaced artist as artist to count if it exists, otherwise original one + ).select_from(jointable2).where( + DB['scrobbles'].c.timestamp<=to, + DB['scrobbles'].c.timestamp>=since + ).group_by( + sql.func.coalesce(DB['associated_artists'].c.target_artist,DB['trackartists'].c.artist_id) + ).order_by(sql.desc('count')) + result = dbconn.execute(op).all() + + + counts = [row.count for row in result] + artists = get_artists_map([row.artist_id for row in result]) + result = [{'scrobbles':row.count,'artist':artists[row.artist_id]} for row in result] + result = rank(result,key='scrobbles') + return result + +@cached_wrapper +@connection_provider +def count_scrobbles_by_track(since,to,dbconn=None): + + + op = sql.select( + sql.func.count(sql.func.distinct(DB['scrobbles'].c.timestamp)).label('count'), + DB['scrobbles'].c.track_id + ).select_from(DB['scrobbles']).where( + DB['scrobbles'].c.timestamp<=to, + DB['scrobbles'].c.timestamp>=since + ).group_by(DB['scrobbles'].c.track_id).order_by(sql.desc('count')) + result = dbconn.execute(op).all() + + + counts = [row.count for row in result] + tracks = get_tracks_map([row.track_id for row in result]) + result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result] + result = rank(result,key='scrobbles') + return result + +@cached_wrapper +@connection_provider +def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None): + + artist_id = get_artist_id(artist) + + jointable = sql.join( + DB['scrobbles'], + DB['trackartists'], + DB['scrobbles'].c.track_id == DB['trackartists'].c.track_id + ) + + op = sql.select( + sql.func.count(sql.func.distinct(DB['scrobbles'].c.timestamp)).label('count'), + DB['scrobbles'].c.track_id + ).select_from(jointable).filter( + DB['scrobbles'].c.timestamp<=to, + DB['scrobbles'].c.timestamp>=since, + DB['trackartists'].c.artist_id==artist_id + ).group_by(DB['scrobbles'].c.track_id).order_by(sql.desc('count')) + result = dbconn.execute(op).all() + + + counts = [row.count for row in result] + tracks = get_tracks_map([row.track_id for row in result]) + result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result] + result = rank(result,key='scrobbles') + return result + + + + +### functions that get mappings for several entities -> rows + +@cached_wrapper_individual +@connection_provider +def get_artists_of_tracks(track_ids,dbconn=None): + op = sql.join(DB['trackartists'],DB['artists']).select().where( + DB['trackartists'].c.track_id.in_(track_ids) + ) + result = dbconn.execute(op).all() + + artists = {} + for row in result: + artists.setdefault(row.track_id,[]).append(artist_db_to_dict(row)) + return artists + + +@cached_wrapper_individual +@connection_provider +def get_tracks_map(track_ids,dbconn=None): + op = DB['tracks'].select().where( + DB['tracks'].c.id.in_(track_ids) + ) + result = dbconn.execute(op).all() + + tracks = {} + result = list(result) + # this will get a list of artistdicts in the correct order of our rows + trackdicts = tracks_db_to_dict(result) + for row,trackdict in zip(result,trackdicts): + tracks[row.id] = trackdict + return tracks + +@cached_wrapper_individual +@connection_provider +def get_artists_map(artist_ids,dbconn=None): + + op = DB['artists'].select().where( + DB['artists'].c.id.in_(artist_ids) + ) + result = dbconn.execute(op).all() + + artists = {} + result = list(result) + # this will get a list of artistdicts in the correct order of our rows + artistdicts = artists_db_to_dict(result) + for row,artistdict in zip(result,artistdicts): + artists[row.id] = artistdict + return artists + + +### associations + +@cached_wrapper +@connection_provider +def get_associated_artists(*artists,dbconn=None): + artist_ids = [get_artist_id(a) for a in artists] + + jointable = sql.join( + DB['associated_artists'], + DB['artists'], + DB['associated_artists'].c.source_artist == DB['artists'].c.id + ) + + op = jointable.select().where( + DB['associated_artists'].c.target_artist.in_(artist_ids) + ) + result = dbconn.execute(op).all() + + artists = artists_db_to_dict(result) + return artists + +@cached_wrapper +@connection_provider +def get_credited_artists(*artists,dbconn=None): + artist_ids = [get_artist_id(a) for a in artists] + + jointable = sql.join( + DB['associated_artists'], + DB['artists'], + DB['associated_artists'].c.target_artist == DB['artists'].c.id + ) + + + op = jointable.select().where( + DB['associated_artists'].c.source_artist.in_(artist_ids) + ) + result = dbconn.execute(op).all() + + artists = artists_db_to_dict(result) + return artists + + +### get a specific entity by id + +@cached_wrapper +@connection_provider +def get_track(id,dbconn=None): + op = DB['tracks'].select().where( + DB['tracks'].c.id==id + ) + result = dbconn.execute(op).all() + + trackinfo = result[0] + return track_db_to_dict(trackinfo) + +@cached_wrapper +@connection_provider +def get_artist(id,dbconn=None): + op = DB['artists'].select().where( + DB['artists'].c.id==id + ) + result = dbconn.execute(op).all() + + artistinfo = result[0] + return artist_db_to_dict(artistinfo) + + + + + +##### MAINTENANCE + +@runhourly +def clean_db(): + + with SCROBBLE_LOCK: + with engine.begin() as conn: + log(f"Database Cleanup...") + + to_delete = [ + # tracks with no scrobbles (trackartist entries first) + "from trackartists where track_id in (select id from tracks where id not in (select track_id from scrobbles))", + "from tracks where id not in (select track_id from scrobbles)", + # artists with no tracks + "from artists where id not in (select artist_id from trackartists) and id not in (select target_artist from associated_artists)", + # tracks with no artists (scrobbles first) + "from scrobbles where track_id in (select id from tracks where id not in (select track_id from trackartists))", + "from tracks where id not in (select track_id from trackartists)" + ] + + for d in to_delete: + selection = conn.execute(sql.text(f"select * {d}")) + for row in selection.all(): + log(f"Deleting {row}") + deletion = conn.execute(sql.text(f"delete {d}")) + + log("Database Cleanup complete!") + + + + #if a2+a1>0: log(f"Deleted {a2} tracks without scrobbles ({a1} track artist entries)") + + #if a3>0: log(f"Deleted {a3} artists without tracks") + + #if a5+a4>0: log(f"Deleted {a5} tracks without artists ({a4} scrobbles)") + + + +@runmonthly +def renormalize_names(): + with SCROBBLE_LOCK: + with engine.begin() as conn: + rows = conn.execute(DB['artists'].select()).all() + + for row in rows: + id = row.id + name = row.name + norm_actual = row.name_normalized + norm_target = normalize_name(name) + if norm_actual != norm_target: + log(f"{name} should be normalized to {norm_target}, but is instead {norm_actual}, fixing...") + + with engine.begin() as conn: + rows = conn.execute(DB['artists'].update().where(DB['artists'].c.id == id).values(name_normalized=norm_target)) + + + + + + +##### AUX FUNCS + + + +# function to turn the name into a representation that can be easily compared, ignoring minor differences +remove_symbols = ["'","`","’"] +replace_with_space = [" - ",": "] +def normalize_name(name): + for r in replace_with_space: + name = name.replace(r," ") + name = "".join(char for char in unicodedata.normalize('NFD',name.lower()) + if char not in remove_symbols and unicodedata.category(char) != 'Mn') + return name + + +def now(): + return int(datetime.now().timestamp()) + +def rank(ls,key): + for rnk in range(len(ls)): + if rnk == 0 or ls[rnk][key] < ls[rnk-1][key]: + ls[rnk]["rank"] = rnk + 1 + else: + ls[rnk]["rank"] = ls[rnk-1]["rank"] + return ls diff --git a/maloja/database_packed.py b/maloja/database_packed.py deleted file mode 100644 index a41cade..0000000 --- a/maloja/database_packed.py +++ /dev/null @@ -1,28 +0,0 @@ -from . import database - -# this is simply an object to expose all database functions with their arguments packed into dicts -# because jinja doesn't accept **kwargs -class DB: - def __getattr__(self,name): - originalmethod = getattr(database,name) - - def packedmethod(*keys): - kwargs = {} - for k in keys: - kwargs.update(k) - return originalmethod(**kwargs) - - return packedmethod - - -# class that is initialized with all the uri keys of the currently requested page and exposes curried db functions -class View: - def __init__(self,filterkeys,limitkeys,delimitkeys,amountkeys): - self.filterkeys = filterkeys - self.limitkeys = limitkeys - self.delimitkeys = delimitkeys - self.amountkeys = amountkeys - - - def get_pulse(self): - return database.get_pulse(**self.limitkeys,**self.delimitkeys,**self.filterkeys) diff --git a/maloja/globalconf.py b/maloja/globalconf.py index 0859ed9..938e542 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -1,12 +1,13 @@ import os from doreah.configuration import Configuration from doreah.configuration import types as tp -from doreah.keystore import KeyStore + from .__pkginfo__ import VERSION + # if DATA_DIRECTORY is specified, this is the directory to use for EVERYTHING, no matter what # but with asynnetrical structure, cache and logs in subfolders # otherwise, each directory is treated seperately @@ -14,10 +15,9 @@ from .__pkginfo__ import VERSION # DIRECRORY_CONFIG, DIRECRORY_STATE, DIRECTORY_LOGS and DIRECTORY_CACHE # config can only be determined by environment variable, the others can be loaded # from the config files -# explicit settings will always be respected. if there are none: -# first check if there is any indication of one of the possibilities being populated already -# if not, use the first we have permissions for -# after we decide which to use, fix it in settings to avoid future heuristics +# explicit settings will always be respected, fallback to default + +# if default isn't usable, and config writable, find alternative and fix it in settings # USEFUL FUNCS pthj = os.path.join @@ -146,13 +146,11 @@ malojaconfig = Configuration( "port":(tp.Integer(), "Port", 42010), }, "Technical":{ - "cache_expire_positive":(tp.Integer(), "Image Cache Expiration", 300, "Days until images are refetched"), - "cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 30, "Days until failed image fetches are reattempted"), - "use_db_cache":(tp.Boolean(), "Use DB Cache", True), - "cache_database_short":(tp.Boolean(), "Use volatile Database Cache", True), - "cache_database_perm":(tp.Boolean(), "Use permanent Database Cache", True), - "db_cache_entries":(tp.Integer(), "Maximal Cache entries", 10000), - "db_max_memory":(tp.Integer(max=100,min=20), "RAM Percentage Theshold", 75, "Maximal percentage of RAM that should be used by whole system before Maloja discards cache entries. Use a higher number if your Maloja runs on a dedicated instance (e.g. a container)") + "cache_expire_positive":(tp.Integer(), "Image Cache Expiration", 60, "Days until images are refetched"), + "cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 5, "Days until failed image fetches are reattempted"), + "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 80, "RAM Usage in percent at which Maloja should no longer increase its database cache."), + "use_request_cache":(tp.Boolean(), "Use request-local DB Cache", True), + "use_global_cache":(tp.Boolean(), "Use global DB Cache", True) }, "Fluff":{ "scrobbles_gold":(tp.Integer(), "Scrobbles for Gold", 250, "How many scrobbles a track needs to be considered 'Gold' status"), @@ -175,6 +173,7 @@ malojaconfig = Configuration( "other_maloja_api_key":(tp.String(), "Other Maloja Instance API Key",None), "track_search_provider":(tp.String(), "Track Search Provider", None), "send_stats":(tp.Boolean(), "Send Statistics", None), + "proxy_images":(tp.Boolean(), "Image Proxy", True, "Whether third party images should be downloaded and served directly by Maloja (instead of just linking their URL)") }, "Database":{ @@ -182,7 +181,7 @@ malojaconfig = Configuration( "remove_from_title":(tp.Set(tp.String()), "Remove from Title", ["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)","(Bonus Track)"], "Phrases that should be removed from song titles"), "delimiters_feat":(tp.Set(tp.String()), "Featuring Delimiters", ["ft.","ft","feat.","feat","featuring","Ft.","Ft","Feat.","Feat","Featuring"], "Delimiters used for extra artists, even when in the title field"), "delimiters_informal":(tp.Set(tp.String()), "Informal Delimiters", ["vs.","vs","&"], "Delimiters in informal artist strings with spaces expected around them"), - "delimiters_formal":(tp.Set(tp.String()), "Formal Delimiters", [";","/"], "Delimiters used to tag multiple artists when only one tag field is available") + "delimiters_formal":(tp.Set(tp.String()), "Formal Delimiters", [";","/","|","␝","␞","␟"], "Delimiters used to tag multiple artists when only one tag field is available") }, "Web Interface":{ "default_range_charts_artists":(tp.Choice({'alltime':'All Time','year':'Year','month':"Month",'week':'Week'}), "Default Range Artist Charts", "year"), @@ -191,43 +190,55 @@ malojaconfig = Configuration( "charts_display_tiles":(tp.Boolean(), "Display Chart Tiles", False), "discourage_cpu_heavy_stats":(tp.Boolean(), "Discourage CPU-heavy stats", False, "Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!"), "use_local_images":(tp.Boolean(), "Use Local Images", True), - "local_image_rotate":(tp.Integer(), "Local Image Rotate", 3600), + #"local_image_rotate":(tp.Integer(), "Local Image Rotate", 3600), "timezone":(tp.Integer(), "UTC Offset", 0), - "time_format":(tp.String(), "Time Format", "%d. %b %Y %I:%M %p") + "time_format":(tp.String(), "Time Format", "%d. %b %Y %I:%M %p"), + "theme":(tp.String(), "Theme", "maloja") } }, configfile=newsettingsfile, save_endpoint="/apis/mlj_1/settings", - env_prefix="MALOJA_" + env_prefix="MALOJA_", + extra_files=["/run/secrets/maloja.yml","/run/secrets/maloja.ini"] ) if found_new_config_dir: - malojaconfig["DIRECTORY_CONFIG"] = maloja_dir_config + try: + malojaconfig["DIRECTORY_CONFIG"] = maloja_dir_config + except PermissionError as e: + pass # this really doesn't matter because when are we gonna load info about where # the settings file is stored from the settings file # but oh well -malojaconfig.render_help(pthj(maloja_dir_config,"settings.md"), - top_text='''If you wish to adjust settings in the settings.ini file, do so while the server -is not running in order to avoid data being overwritten. +try: + malojaconfig.render_help(pthj(maloja_dir_config,"settings.md"), + top_text='''If you wish to adjust settings in the settings.ini file, do so while the server + is not running in order to avoid data being overwritten. -Technically, each setting can be set via environment variable or the settings -file - simply add the prefix `MALOJA_` for environment variables. It is recommended -to use the settings file where possible and not configure each aspect of your -server via environment variables!''') + Technically, each setting can be set via environment variable or the settings + file - simply add the prefix `MALOJA_` for environment variables. It is recommended + to use the settings file where possible and not configure each aspect of your + server via environment variables! + + You also can specify additional settings in the files`/run/secrets/maloja.yml` or + `/run/secrets/maloja.ini`, as well as their values directly in files of the respective + name in `/run/secrets/` (e.g. `/run/secrets/lastfm_api_key`).''') +except PermissionError as e: + pass ### STEP 3 - check all possible folders for files (old installation) - -for datatype in ("state","cache","logs"): - # obviously default values shouldn't trigger this - # if user has nothing specified, we need to use this - if malojaconfig.get_specified(directory_info[datatype]['setting']) is None and malojaconfig.get_specified('DATA_DIRECTORY') is None: - find_good_folder(datatype,malojaconfig) +if not malojaconfig.readonly: + for datatype in ("state","cache","logs"): + # obviously default values shouldn't trigger this + # if user has nothing specified, we need to use this + if malojaconfig.get_specified(directory_info[datatype]['setting']) is None and malojaconfig.get_specified('DATA_DIRECTORY') is None: + find_good_folder(datatype,malojaconfig) @@ -261,15 +272,21 @@ data_directories = { "auth":pthj(dir_settings['state'],"auth"), "backups":pthj(dir_settings['state'],"backups"), "images":pthj(dir_settings['state'],"images"), - "scrobbles":pthj(dir_settings['state'],"scrobbles"), + "scrobbles":pthj(dir_settings['state']), "rules":pthj(dir_settings['config'],"rules"), "clients":pthj(dir_settings['config']), "settings":pthj(dir_settings['config']), "css":pthj(dir_settings['config'],"custom_css"), - "logs":pthj(dir_settings['logs']), - "cache":pthj(dir_settings['cache']), + + "config":dir_settings['config'], + "state":dir_settings['state'], + "logs":dir_settings['logs'], + "cache":dir_settings['cache'], } +for identifier,path in data_directories.items(): + os.makedirs(path,exist_ok=True) + data_dir = { k:lambda *x,k=k: pthj(data_directories[k],*x) for k in data_directories @@ -291,9 +308,6 @@ with open(pthj(dir_settings['state'],".lastmalojaversion"),"w") as filed: from doreah import config config( - caching={ - "folder": data_dir['cache']() - }, auth={ "multiuser":False, "cookieprefix":"maloja", @@ -304,7 +318,6 @@ config( "logfolder": data_dir['logs']() if malojaconfig["LOGGING"] else None }, regular={ - "autostart": False, "offset": malojaconfig["TIMEZONE"] } ) @@ -312,23 +325,6 @@ config( -### API KEYS - - - -### symmetric keys are fine for now since we hopefully use HTTPS -apikeystore = KeyStore(file=data_dir['clients']("apikeys.yml"),save_endpoint="/apis/mlj_1/apikeys") - -oldfile = pthj(dir_settings['config'],"clients","authenticated_machines.tsv") -if os.path.exists(oldfile): - try: - from doreah import tsv - clients = tsv.parse(oldfile,"string","string") - for key,identifier in clients: - apikeystore[identifier] = key - os.remove(oldfile) - except: - pass # what the fuck did i just write diff --git a/maloja/images.py b/maloja/images.py new file mode 100644 index 0000000..ba399f7 --- /dev/null +++ b/maloja/images.py @@ -0,0 +1,300 @@ +from .globalconf import data_dir, malojaconfig +from . import thirdparty +from . import database + +from doreah.logging import log + +import itertools +import os +import urllib +import random +import base64 +import requests +import datauri +import io +from threading import Thread, Timer, BoundedSemaphore +import re +import datetime + +import sqlalchemy as sql + + + + +DB = {} +engine = sql.create_engine(f"sqlite:///{data_dir['cache']('images.sqlite')}", echo = False) +meta = sql.MetaData() + +DB['artists'] = sql.Table( + 'artists', meta, + sql.Column('id',sql.Integer,primary_key=True), + sql.Column('url',sql.String), + sql.Column('expire',sql.Integer), + sql.Column('raw',sql.String) +) +DB['tracks'] = sql.Table( + 'tracks', meta, + sql.Column('id',sql.Integer,primary_key=True), + sql.Column('url',sql.String), + sql.Column('expire',sql.Integer), + sql.Column('raw',sql.String) +) + +meta.create_all(engine) + +def get_image_from_cache(id,table): + now = int(datetime.datetime.now().timestamp()) + with engine.begin() as conn: + op = DB[table].select().where( + DB[table].c.id==id, + DB[table].c.expire>now + ) + result = conn.execute(op).all() + for row in result: + if row.raw is not None: + return {'type':'raw','value':row.raw} + else: + return {'type':'url','value':row.url} # returns None as value if nonexistence cached + return None # no cache entry + +def set_image_in_cache(id,table,url): + remove_image_from_cache(id,table) + now = int(datetime.datetime.now().timestamp()) + if url is None: + expire = now + (malojaconfig["CACHE_EXPIRE_NEGATIVE"] * 24 * 3600) + else: + expire = now + (malojaconfig["CACHE_EXPIRE_POSITIVE"] * 24 * 3600) + + raw = dl_image(url) + + with engine.begin() as conn: + op = DB[table].insert().values( + id=id, + url=url, + expire=expire, + raw=raw + ) + result = conn.execute(op) + +def remove_image_from_cache(id,table): + with engine.begin() as conn: + op = DB[table].delete().where( + DB[table].c.id==id, + ) + result = conn.execute(op) + +def dl_image(url): + if not malojaconfig["PROXY_IMAGES"]: return None + if url is None: return None + if url.startswith("/"): return None #local image + try: + r = requests.get(url) + mime = r.headers.get('content-type') or 'image/jpg' + data = io.BytesIO(r.content).read() + uri = datauri.DataURI.make(mime,charset='ascii',base64=True,data=data) + log(f"Downloaded {url} for local caching") + return uri + except: + log(f"Image {url} could not be downloaded for local caching") + return None + + + +### getting images for any website embedding now ALWAYS returns just the generic link +### even if we have already cached it, we will handle that on request +def get_track_image(track=None,track_id=None): + if track_id is None: + track_id = database.sqldb.get_track_id(track) + + return f"/image?type=track&id={track_id}" + + +def get_artist_image(artist=None,artist_id=None): + if artist_id is None: + artist_id = database.sqldb.get_artist_id(artist) + + return f"/image?type=artist&id={artist_id}" + + + +resolve_semaphore = BoundedSemaphore(8) + + +def resolve_track_image(track_id): + + with resolve_semaphore: + # check cache + result = get_image_from_cache(track_id,'tracks') + if result is not None: + return result + + track = database.sqldb.get_track(track_id) + + # local image + if malojaconfig["USE_LOCAL_IMAGES"]: + images = local_files(artists=track['artists'],title=track['title']) + if len(images) != 0: + result = random.choice(images) + result = urllib.parse.quote(result) + result = {'type':'url','value':result} + set_image_in_cache(track_id,'tracks',result['value']) + return result + + # third party + result = thirdparty.get_image_track_all((track['artists'],track['title'])) + result = {'type':'url','value':result} + set_image_in_cache(track_id,'tracks',result['value']) + + return result + + +def resolve_artist_image(artist_id): + + with resolve_semaphore: + # check cache + result = get_image_from_cache(artist_id,'artists') + if result is not None: + return result + + artist = database.sqldb.get_artist(artist_id) + + # local image + if malojaconfig["USE_LOCAL_IMAGES"]: + images = local_files(artist=artist) + if len(images) != 0: + result = random.choice(images) + result = urllib.parse.quote(result) + result = {'type':'url','value':result} + set_image_in_cache(artist_id,'artists',result['value']) + return result + + # third party + result = thirdparty.get_image_artist_all(artist) + result = {'type':'url','value':result} + set_image_in_cache(artist_id,'artists',result['value']) + + return result + + +# removes emojis and weird shit from names +def clean(name): + return "".join(c for c in name if c.isalnum() or c in []).strip() + +def get_all_possible_filenames(artist=None,artists=None,title=None): + # check if we're dealing with a track or artist, then clean up names + # (only remove non-alphanumeric, allow korean and stuff) + + if title is not None and artists is not None: + track = True + title, artists = clean(title), [clean(a) for a in artists] + elif artist is not None: + track = False + artist = clean(artist) + else: return [] + + + superfolder = "tracks/" if track else "artists/" + + filenames = [] + + if track: + #unsafeartists = [artist.translate(None,"-_./\\") for artist in artists] + safeartists = [re.sub("[^a-zA-Z0-9]","",artist) for artist in artists] + #unsafetitle = title.translate(None,"-_./\\") + safetitle = re.sub("[^a-zA-Z0-9]","",title) + + if len(artists) < 4: + unsafeperms = itertools.permutations(artists) + safeperms = itertools.permutations(safeartists) + else: + unsafeperms = [sorted(artists)] + safeperms = [sorted(safeartists)] + + + for unsafeartistlist in unsafeperms: + filename = "-".join(unsafeartistlist) + "_" + title + if filename != "": + filenames.append(filename) + filenames.append(filename.lower()) + for safeartistlist in safeperms: + filename = "-".join(safeartistlist) + "_" + safetitle + if filename != "": + filenames.append(filename) + filenames.append(filename.lower()) + filenames = list(set(filenames)) + if len(filenames) == 0: filenames.append(str(hash((frozenset(artists),title)))) + else: + #unsafeartist = artist.translate(None,"-_./\\") + safeartist = re.sub("[^a-zA-Z0-9]","",artist) + + filename = artist + if filename != "": + filenames.append(filename) + filenames.append(filename.lower()) + filename = safeartist + if filename != "": + filenames.append(filename) + filenames.append(filename.lower()) + + filenames = list(set(filenames)) + if len(filenames) == 0: filenames.append(str(hash(artist))) + + return [superfolder + name for name in filenames] + +def local_files(artist=None,artists=None,title=None): + + + filenames = get_all_possible_filenames(artist,artists,title) + + images = [] + + for purename in filenames: + # direct files + for ext in ["png","jpg","jpeg","gif"]: + #for num in [""] + [str(n) for n in range(0,10)]: + if os.path.exists(data_dir['images'](purename + "." + ext)): + images.append("/images/" + purename + "." + ext) + + # folder + try: + for f in os.listdir(data_dir['images'](purename)): + if f.split(".")[-1] in ["png","jpg","jpeg","gif"]: + images.append("/images/" + purename + "/" + f) + except: + pass + + return images + + + +def set_image(b64,**keys): + track = "title" in keys + if track: + entity = {'artists':keys['artists'],'title':keys['title']} + id = database.sqldb.get_track_id(entity) + else: + entity = keys['artist'] + id = database.sqldb.get_artist_id(entity) + + log("Trying to set image, b64 string: " + str(b64[:30] + "..."),module="debug") + + regex = r"data:image/(\w+);base64,(.+)" + type,b64 = re.fullmatch(regex,b64).groups() + b64 = base64.b64decode(b64) + filename = "webupload" + str(int(datetime.datetime.now().timestamp())) + "." + type + for folder in get_all_possible_filenames(**keys): + if os.path.exists(data_dir['images'](folder)): + with open(data_dir['images'](folder,filename),"wb") as f: + f.write(b64) + break + else: + folder = get_all_possible_filenames(**keys)[0] + os.makedirs(data_dir['images'](folder)) + with open(data_dir['images'](folder,filename),"wb") as f: + f.write(b64) + + log("Saved image as " + data_dir['images'](folder,filename),module="debug") + + # set as current picture in rotation + if track: set_image_in_cache(id,'tracks',os.path.join("/images",folder,filename)) + else: set_image_in_cache(id,'artists',os.path.join("/images",folder,filename)) diff --git a/maloja/jinjaenv/context.py b/maloja/jinjaenv/context.py index f39755b..48356e7 100644 --- a/maloja/jinjaenv/context.py +++ b/maloja/jinjaenv/context.py @@ -1,8 +1,8 @@ -from .. import database_packed from . import filters from ..globalconf import malojaconfig -from .. import database, database_packed, malojatime, utilities, malojauri, thirdparty +from .. import database, malojatime, images, malojauri, thirdparty, __pkginfo__ +from ..database import jinjaview from doreah.regular import repeatdaily import urllib @@ -11,7 +11,7 @@ import math # templating from jinja2 import Environment, PackageLoader, select_autoescape -dbp = database_packed.DB() +#dbp = jinjaview.JinjaDBConnection() jinja_environment = Environment( loader=PackageLoader('maloja', "web/jinja"), @@ -25,13 +25,14 @@ def update_jinja_environment(): JINJA_CONTEXT = { # maloja - "db": database, - "dbp":dbp, + "db": database, #TODO: move these to connection manager as well + #"dbp":dbp, "malojatime": malojatime, - "utilities": utilities, + "images": images, "mlj_uri": malojauri, "settings": malojaconfig, "thirdparty": thirdparty, + "pkginfo": __pkginfo__, # external "urllib": urllib, "math":math, diff --git a/maloja/malojatime.py b/maloja/malojatime.py index 662ec85..5be3c72 100644 --- a/maloja/malojatime.py +++ b/maloja/malojatime.py @@ -58,10 +58,16 @@ class MTRangeGeneric: def unlimited(self): return False + def timestamps(self): + return (self.first_stamp(),self.last_stamp()) + # whether we currently live or will ever again live in this range def active(self): return (self.last_stamp() > datetime.utcnow().timestamp()) + def __contains__(self,timestamp): + return timestamp >= self.first_stamp() and timestamp <= self.last_stamp() + # Any range that has one defining base unit, whether week, year, etc. class MTRangeSingular(MTRangeGeneric): def fromstr(self): diff --git a/maloja/malojauri.py b/maloja/malojauri.py index 8b3f34b..d93caeb 100644 --- a/maloja/malojauri.py +++ b/maloja/malojauri.py @@ -36,7 +36,7 @@ def uri_to_internal(keys,forceTrack=False,forceArtist=False,api=False): limitkeys["timerange"] = get_range_object(since=since,to=to,within=within) #3 - delimitkeys = {"step":"month","stepn":1,"trail":1} + delimitkeys = {"step":"year","stepn":1,"trail":1} if "step" in keys: [delimitkeys["step"],delimitkeys["stepn"]] = (keys["step"].split("-") + [1])[:2] if "stepn" in keys: delimitkeys["stepn"] = keys["stepn"] #overwrite if explicitly given if "stepn" in delimitkeys: delimitkeys["stepn"] = int(delimitkeys["stepn"]) #in both cases, convert it here diff --git a/maloja/proccontrol/control.py b/maloja/proccontrol/control.py index ab45acf..fd2aeba 100644 --- a/maloja/proccontrol/control.py +++ b/maloja/proccontrol/control.py @@ -12,12 +12,13 @@ from .. import __pkginfo__ as info from .. import globalconf + def print_header_info(): print() - print("#####") - print("Maloja v" + info.VERSION) + #print("#####") + print(col['yellow']("Maloja"),"v" + info.VERSION) print(info.HOMEPAGE) - print("#####") + #print("#####") print() @@ -79,12 +80,16 @@ def stop(): print("Maloja stopped!") return True - +def onlysetup(): + print_header_info() + setup() + print("Setup complete!") def direct(): print_header_info() setup() from .. import server + server.run_server() def debug(): os.environ["MALOJA_DEV_MODE"] = 'true' @@ -93,29 +98,31 @@ def debug(): def print_info(): print_header_info() - print("Configuration Directory:",globalconf.dir_settings['config']) - print("Data Directory: ",globalconf.dir_settings['state']) - print("Log Directory: ",globalconf.dir_settings['logs']) - print("Network: ",f"IPv{ip_address(globalconf.malojaconfig['host']).version}, Port {globalconf.malojaconfig['port']}") - print("Timezone: ",f"UTC{globalconf.malojaconfig['timezone']:+d}") + print(col['lightblue']("Configuration Directory:"),globalconf.dir_settings['config']) + print(col['lightblue']("Data Directory: "),globalconf.dir_settings['state']) + print(col['lightblue']("Log Directory: "),globalconf.dir_settings['logs']) + print(col['lightblue']("Network: "),f"IPv{ip_address(globalconf.malojaconfig['host']).version}, Port {globalconf.malojaconfig['port']}") + print(col['lightblue']("Timezone: "),f"UTC{globalconf.malojaconfig['timezone']:+d}") print() - print("#####") print() -@mainfunction({"l":"level","v":"version","V":"version"},flags=['version'],shield=True) +@mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images'],shield=True) def main(*args,**kwargs): actions = { + # server "start":start, "restart":restart, "stop":stop, "run":direct, "debug":debug, - "import":tasks.loadexternal, - "backup":tasks.backuphere, - # "update":update, - "fix":tasks.fixdb, - "generate":tasks.generate_scrobbles, + "setup":onlysetup, + # admin scripts + "import":tasks.import_scrobbles, # maloja import /x/y.csv + "backup":tasks.backup, # maloja backup --targetfolder /x/y --include_images + "generate":tasks.generate, # maloja generate 400 + "export":tasks.export, # maloja export + # aux "info":print_info } @@ -124,8 +131,11 @@ def main(*args,**kwargs): else: try: action, *args = args - actions[action](*args,**kwargs) + action = actions[action] except (ValueError, KeyError): print("Valid commands: " + " ".join(a for a in actions)) + return + + return action(*args,**kwargs) return True diff --git a/maloja/proccontrol/profiler.py b/maloja/proccontrol/profiler.py new file mode 100644 index 0000000..3a98d35 --- /dev/null +++ b/maloja/proccontrol/profiler.py @@ -0,0 +1,35 @@ +import os + +import cProfile, pstats + + +from doreah.logging import log +from doreah.timing import Clock + +from ..globalconf import data_dir + + +profiler = cProfile.Profile() + +def profile(func): + def newfunc(*args,**kwargs): + + benchmarkfolder = data_dir['logs']("benchmarks") + os.makedirs(benchmarkfolder,exist_ok=True) + + clock = Clock() + clock.start() + + profiler.enable() + result = func(*args,**kwargs) + profiler.disable() + + log(f"Executed {func.__name__} ({args}, {kwargs}) in {clock.stop():.2f}s",module="debug_performance") + try: + pstats.Stats(profiler).dump_stats(os.path.join(benchmarkfolder,f"{func.__name__}.stats")) + except: + pass + + return result + + return newfunc diff --git a/maloja/proccontrol/setup.py b/maloja/proccontrol/setup.py index 630e9b4..8872357 100644 --- a/maloja/proccontrol/setup.py +++ b/maloja/proccontrol/setup.py @@ -1,10 +1,11 @@ -import pkg_resources +from importlib import resources from distutils import dir_util from doreah.io import col, ask, prompt from doreah import auth import os -from ..globalconf import data_dir, dir_settings, malojaconfig, apikeystore +from ..globalconf import data_dir, dir_settings, malojaconfig + # EXTERNAL API KEYS @@ -18,9 +19,9 @@ ext_apikeys = [ def copy_initial_local_files(): - folder = pkg_resources.resource_filename("maloja","data_files") - for cat in dir_settings: - dir_util.copy_tree(os.path.join(folder,cat),dir_settings[cat],update=False) + with resources.files("maloja") / 'data_files' as folder: + for cat in dir_settings: + dir_util.copy_tree(os.path.join(folder,cat),dir_settings[cat],update=False) charset = list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ") def randomstring(length=32): @@ -37,16 +38,17 @@ def setup(): keyname = malojaconfig.get_setting_info(k)['name'] key = malojaconfig[k] if key is False: - print("\t" + "Currently not using a " + col['red'](keyname) + " for image display.") + print(f"\tCurrently not using a {col['red'](keyname)} for image display.") elif key is None or key == "ASK": - print("\t" + "Please enter your " + col['gold'](keyname) + ". If you do not want to use one at this moment, simply leave this empty and press Enter.") - key = prompt("",types=(str,),default=False,skip=SKIP) + promptmsg = f"\tPlease enter your {col['gold'](keyname)}. If you do not want to use one at this moment, simply leave this empty and press Enter." + key = prompt(promptmsg,types=(str,),default=False,skip=SKIP) malojaconfig[k] = key else: - print("\t" + col['lawngreen'](keyname) + " found.") + print(f"\t{col['lawngreen'](keyname)} found.") # OWN API KEY + from ..apis import apikeystore if len(apikeystore) == 0: answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP) if answer: @@ -74,11 +76,3 @@ def setup(): if newpw != newpw_repeat: print("Passwords do not match!") else: break auth.defaultuser.setpw(newpw) - - if malojaconfig["SEND_STATS"] is None: - answer = ask("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)?",default=True,skip=SKIP) - if answer: - malojaconfig["SEND_STATS"] = True - malojaconfig["PUBLIC_URL"] = None - else: - malojaconfig["SEND_STATS"] = False diff --git a/maloja/proccontrol/supervisor.py b/maloja/proccontrol/supervisor.py index 0a81cee..f1ab668 100644 --- a/maloja/proccontrol/supervisor.py +++ b/maloja/proccontrol/supervisor.py @@ -14,17 +14,10 @@ from .control import getInstance setproctitle.setproctitle("maloja_supervisor") -def update(): - log("Updating...",module="supervisor") - try: - os.system("pip3 install maloja --upgrade --no-cache-dir") - except: - log("Could not update.",module="supervisor") - def start(): try: return subprocess.Popen( - ["python3", "-m", "maloja.server"], + ["python3", "-m", "maloja","run"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) @@ -35,8 +28,6 @@ def start(): while True: log("Maloja is not running, starting...",module="supervisor") - if malojaconfig["UPDATE_AFTER_CRASH"]: - update() process = start() process.wait() diff --git a/maloja/proccontrol/tasks/__init__.py b/maloja/proccontrol/tasks/__init__.py index 9fd1159..eda907d 100644 --- a/maloja/proccontrol/tasks/__init__.py +++ b/maloja/proccontrol/tasks/__init__.py @@ -1,49 +1,4 @@ -import os -from doreah.io import ask,col - -from ...globalconf import data_dir - - - -def loadexternal(filename): - - if not os.path.exists(filename): - print("File could not be found.") - return - - from .import_scrobbles import import_scrobbles - result = import_scrobbles(filename) - - msg = f"Successfully imported {result['CONFIDENT_IMPORT'] + result['UNCERTAIN_IMPORT']} scrobbles" - if result['UNCERTAIN_IMPORT'] > 0: - warningmsg = col['orange'](f"{result['UNCERTAIN_IMPORT']} Warning{'s' if result['UNCERTAIN_IMPORT'] != 1 else ''}!") - msg += f" ({warningmsg})" - print(msg) - - msg = f"Skipped {result['CONFIDENT_SKIP'] + result['UNCERTAIN_SKIP']} scrobbles" - if result['UNCERTAIN_SKIP'] > 0: - warningmsg = col['indianred'](f"{result['UNCERTAIN_SKIP']} Warning{'s' if result['UNCERTAIN_SKIP'] != 1 else ''}!") - msg += f" ({warningmsg})" - print(msg) - - if result['FAIL'] > 0: - print(col['red'](f"{result['FAIL']} Error{'s' if result['FAIL'] != 1 else ''}!")) - -def backuphere(): - from .backup import backup - backup(folder=os.getcwd()) - -def update(): - os.system("pip3 install malojaserver --upgrade --no-cache-dir") - from ..control import restart - restart() - -def fixdb(): - from .fixexisting import fix - fix() - -def generate_scrobbles(): - targetfile = data_dir['scrobbles']("randomgenerated.tsv") - - from .generate import generate - generate(targetfile) +from .import_scrobbles import import_scrobbles +from .backup import backup +from .generate import generate +from .export import export # read that line out loud diff --git a/maloja/proccontrol/tasks/backup.py b/maloja/proccontrol/tasks/backup.py index 7c7bbb5..73f797b 100644 --- a/maloja/proccontrol/tasks/backup.py +++ b/maloja/proccontrol/tasks/backup.py @@ -1,47 +1,51 @@ import tarfile -from datetime import datetime +import time import glob import os -from ...globalconf import data_dir +from ...globalconf import dir_settings from pathlib import PurePath from doreah.logging import log +from doreah.io import col -user_files = { - "minimal":{ - "rules":["*.tsv"], - "scrobbles":["*.tsv"] - }, - "full":{ - "clients":["apikeys.yml"], - "images":["artists","tracks"], - "settings":["settings.ini"] - } -} +basic_files = [ + ('config', ['rules/*.tsv','settings.ini','apikeys.yml','custom_css/*.css']), + ('state', ['auth/auth.ddb','malojadb.sqlite']) +] +expanded_files = [ + ('state', ['images']) +] -def backup(folder,level="full"): +def backup(targetfolder=None,include_images=False): - selected_files = user_files["minimal"] if level == "minimal" else {**user_files["minimal"], **user_files["full"]} - real_files = {cat:[] for cat in selected_files} - for cat in selected_files: - catfolder = data_dir[cat] - for g in selected_files[cat]: - real_files[cat] += glob.glob(catfolder(g)) + if targetfolder is None: + targetfolder = os.getcwd() + + if include_images: + file_patterns = basic_files + expanded_files + else: + file_patterns = basic_files + + real_files = {} + for category,patterns in file_patterns: + real_files.setdefault(category,[]) + for pattern in patterns: + real_files[category] += glob.glob(os.path.join(dir_settings[category],pattern)) log("Creating backup...") - now = datetime.utcnow() - timestr = now.strftime("%Y_%m_%d_%H_%M_%S") - filename = "maloja_backup_" + timestr + ".tar.gz" - archivefile = os.path.join(folder,filename) - assert not os.path.exists(archivefile) - with tarfile.open(name=archivefile,mode="x:gz") as archive: - for cat, value in real_files.items(): - for f in value: + timestr = time.strftime("%Y_%m_%d_%H_%M_%S") + filename = f"maloja_backup_{timestr}.tar.gz" + outputfile = os.path.join(targetfolder,filename) + assert not os.path.exists(outputfile) + + with tarfile.open(name=outputfile,mode="x:gz") as archive: + for category, filelist in real_files.items(): + for f in filelist: p = PurePath(f) - r = p.relative_to(data_dir[cat]()) - archive.add(f,arcname=os.path.join(cat,r)) - log("Backup created!") - return archivefile + r = p.relative_to(dir_settings[category]) + archive.add(f,arcname=os.path.join(category,r)) + log("Backup created: " + col['yellow'](outputfile)) + return outputfile diff --git a/maloja/proccontrol/tasks/export.py b/maloja/proccontrol/tasks/export.py new file mode 100644 index 0000000..a1e2a2b --- /dev/null +++ b/maloja/proccontrol/tasks/export.py @@ -0,0 +1,24 @@ +import json +import os +import time + +from doreah.io import col + +def export(targetfolder=None): + + from ...database.sqldb import get_scrobbles + + if targetfolder is None: + targetfolder = os.getcwd() + + timestr = time.strftime("%Y_%m_%d_%H_%M_%S") + filename = f"maloja_export_{timestr}.json" + outputfile = os.path.join(targetfolder,filename) + assert not os.path.exists(outputfile) + + data = {'scrobbles':get_scrobbles()} + with open(outputfile,'w') as outfd: + json.dump(data,outfd,indent=3) + + print(f"Exported {len(data['scrobbles'])} Scrobbles to {col['yellow'](outputfile)}") + return outputfile diff --git a/maloja/proccontrol/tasks/fixexisting.py b/maloja/proccontrol/tasks/fixexisting.py deleted file mode 100644 index b7f56f6..0000000 --- a/maloja/proccontrol/tasks/fixexisting.py +++ /dev/null @@ -1,70 +0,0 @@ -import os -from ...globalconf import data_dir -import re -from ...cleanup import CleanerAgent -from doreah.logging import log -import difflib -import datetime -from .backup import backup - -wendigo = CleanerAgent() - -exp = r"([0-9]*)(\t+)([^\t]+?)(\t+)([^\t]+)([^\n]*)\n" -# 1 2 3 4 5 6 -# groups: -# 1 - timestamp -# 2 - sep -# 3 - artists -# 4 - sep -# 5 - title -# 6 - rest - - - -def fix(): - - backup(level="minimal",folder=data_dir['backups']()) - - now = datetime.datetime.utcnow() - nowstr = now.strftime("%Y_%m_%d_%H_%M_%S") - datestr = now.strftime("%Y/%m/%d") - timestr = now.strftime("%H:%M:%S") - - patchfolder = data_dir['logs']("dbfix",nowstr) - os.makedirs(patchfolder) - - - log("Fixing database...") - for filename in os.listdir(data_dir['scrobbles']()): - if filename.endswith(".tsv"): - log("Fix file " + filename) - filename_new = filename + "_new" - - with open(data_dir['scrobbles'](filename_new),"w") as newfile: - with open(data_dir['scrobbles'](filename),"r") as oldfile: - - for l in oldfile: - - a,t = re.sub(exp,r"\3",l), re.sub(exp,r"\5",l) - r1,r2,r3 = re.sub(exp,r"\1\2",l),re.sub(exp,r"\4",l),re.sub(exp,r"\6",l) - - a = a.split("␟") - - (al,t) = wendigo.fullclean(a,t) - a = "␟".join(al) - newfile.write(r1 + a + r2 + t + r3 + "\n") - - - #os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn) - with open(data_dir['scrobbles'](filename_new),"r") as newfile, open(data_dir['scrobbles'](filename),"r") as oldfile: - - diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="",n=0) - diff = list(diff) - - with open(os.path.join(patchfolder,filename + ".diff"),"w") as patchfile: - patchfile.write("\n".join(diff)) - - os.replace(data_dir['scrobbles'](filename_new),data_dir['scrobbles'](filename)) - - - log("Database fixed!") diff --git a/maloja/proccontrol/tasks/generate.py b/maloja/proccontrol/tasks/generate.py index 4461d24..225b6ce 100644 --- a/maloja/proccontrol/tasks/generate.py +++ b/maloja/proccontrol/tasks/generate.py @@ -2,6 +2,7 @@ import random import datetime from doreah.io import ask + artists = [ "Chou Tzuyu","Jennie Kim","Kim Seolhyun","Nancy McDonie","Park Junghwa","Hirai Momo","Rosé Park","Laura Brehm","HyunA", "Jeremy Soule","Jerry Goldsmith","Howard Shore","Tilman Sillescu","James Newton Howard","Bear McCreary","David Newman", @@ -65,15 +66,28 @@ def generate_track(): -def generate(targetfile): - if ask("Generate random scrobbles?",default=False): - with open(targetfile,"a") as fd: - for _ in range(200): - track = generate_track() - for _ in range(random.randint(1, 50)): - timestamp = random.randint(1, int(datetime.datetime.now().timestamp())) +def generate(n=200): - entry = "\t".join([str(timestamp),"␟".join(track['artists']),track['title'],"-"]) - fd.write(entry) - fd.write("\n") - print("Done!") + from ...database.sqldb import add_scrobbles + + n = int(n) + + if ask("Generate random scrobbles?",default=False): + scrobbles = [] + for _ in range(n): + track = generate_track() + print("Generated",track) + for _ in range(random.randint(1, 50)): + timestamp = random.randint(1, int(datetime.datetime.now().timestamp())) + + scrobbles.append({ + "time":timestamp, + "track":{ + "artists":track['artists'], + "title":track['title'] + }, + "duration":None, + "origin":"generated" + }) + + add_scrobbles(scrobbles) diff --git a/maloja/proccontrol/tasks/import_scrobbles.py b/maloja/proccontrol/tasks/import_scrobbles.py index bfdb248..fec1f83 100644 --- a/maloja/proccontrol/tasks/import_scrobbles.py +++ b/maloja/proccontrol/tasks/import_scrobbles.py @@ -1,9 +1,9 @@ import os, datetime, re - import json, csv -from ...cleanup import * from doreah.io import col, ask, prompt + +from ...cleanup import * from ...globalconf import data_dir @@ -21,6 +21,8 @@ outputs = { def import_scrobbles(inputf): + from ...database.sqldb import add_scrobbles + result = { "CONFIDENT_IMPORT": 0, "UNCERTAIN_IMPORT": 0, @@ -32,76 +34,85 @@ def import_scrobbles(inputf): filename = os.path.basename(inputf) if re.match(".*\.csv",filename): - type = "Last.fm" - outputf = data_dir['scrobbles']("lastfmimport.tsv") + typeid,typedesc = "lastfm","Last.fm" importfunc = parse_lastfm elif re.match("endsong_[0-9]+\.json",filename): - type = "Spotify" - outputf = data_dir['scrobbles']("spotifyimport.tsv") + typeid,typedesc = "spotify","Spotify" importfunc = parse_spotify_full elif re.match("StreamingHistory[0-9]+\.json",filename): - type = "Spotify" - outputf = data_dir['scrobbles']("spotifyimport.tsv") + typeid,typedesc = "spotify","Spotify" importfunc = parse_spotify_lite + elif re.match("maloja_export_[0-9]+\.json",filename): + typeid,typedesc = "maloja","Maloja" + importfunc = parse_maloja + else: print("File",inputf,"could not be identified as a valid import source.") return result - print(f"Parsing {col['yellow'](inputf)} as {col['cyan'](type)} export") + print(f"Parsing {col['yellow'](inputf)} as {col['cyan'](typedesc)} export") + print("This could take a while...") + timestamps = set() + scrobblebuffer = [] - if os.path.exists(outputf): - while True: - action = prompt(f"Already imported {type} data. [O]verwrite, [A]ppend or [C]ancel?",default='c').lower()[0] - if action == 'c': - return result - elif action == 'a': - mode = 'a' - break - elif action == 'o': - mode = 'w' - break - else: - print("Could not understand response.") - else: - mode = 'w' + for status,scrobble,msg in importfunc(inputf): + result[status] += 1 + outputs[status](msg) + if status in ['CONFIDENT_IMPORT','UNCERTAIN_IMPORT']: + # prevent duplicate timestamps + while scrobble['scrobble_time'] in timestamps: + scrobble['scrobble_time'] += 1 + timestamps.add(scrobble['scrobble_time']) - with open(outputf,mode) as outputfd: + # clean up + (scrobble['track_artists'],scrobble['track_title']) = c.fullclean(scrobble['track_artists'],scrobble['track_title']) - timestamps = set() + # extra info + extrainfo = {} + if scrobble.get('album_name'): extrainfo['album_name'] = scrobble['album_name'] + # saving this in the scrobble instead of the track because for now it's not meant + # to be authorative information, just payload of the scrobble - for status,scrobble,msg in importfunc(inputf): - result[status] += 1 - outputs[status](msg) - if status in ['CONFIDENT_IMPORT','UNCERTAIN_IMPORT']: + scrobblebuffer.append({ + "time":scrobble['scrobble_time'], + "track":{ + "artists":scrobble['track_artists'], + "title":scrobble['track_title'], + "length":None + }, + "duration":scrobble['scrobble_duration'], + "origin":"import:" + typeid, + "extra":extrainfo + }) - while scrobble['timestamp'] in timestamps: - scrobble['timestamp'] += 1 - timestamps.add(scrobble['timestamp']) + if (result['CONFIDENT_IMPORT'] + result['UNCERTAIN_IMPORT']) % 1000 == 0: + print(f"Imported {result['CONFIDENT_IMPORT'] + result['UNCERTAIN_IMPORT']} scrobbles...") + add_scrobbles(scrobblebuffer) + scrobblebuffer = [] - # Format fields for tsv - scrobble['timestamp'] = str(scrobble['timestamp']) - scrobble['duration'] = str(scrobble['duration']) if scrobble['duration'] is not None else '-' - scrobble['album'] = scrobble['album'] if scrobble['album'] is not None else '-' - (artists,scrobble['title']) = c.fullclean(scrobble['artiststr'],scrobble['title']) - scrobble['artiststr'] = "␟".join(artists) + add_scrobbles(scrobblebuffer) - outputline = "\t".join([ - scrobble['timestamp'], - scrobble['artiststr'], - scrobble['title'], - scrobble['album'], - scrobble['duration'] - ]) - outputfd.write(outputline + '\n') + msg = f"Successfully imported {result['CONFIDENT_IMPORT'] + result['UNCERTAIN_IMPORT']} scrobbles" + if result['UNCERTAIN_IMPORT'] > 0: + warningmsg = col['orange'](f"{result['UNCERTAIN_IMPORT']} Warning{'s' if result['UNCERTAIN_IMPORT'] != 1 else ''}!") + msg += f" ({warningmsg})" + print(msg) + + msg = f"Skipped {result['CONFIDENT_SKIP'] + result['UNCERTAIN_SKIP']} scrobbles" + if result['UNCERTAIN_SKIP'] > 0: + warningmsg = col['indianred'](f"{result['UNCERTAIN_SKIP']} Warning{'s' if result['UNCERTAIN_SKIP'] != 1 else ''}!") + msg += f" ({warningmsg})" + print(msg) + + if result['FAIL'] > 0: + print(col['red'](f"{result['FAIL']} Error{'s' if result['FAIL'] != 1 else ''}!")) - if (result['CONFIDENT_IMPORT'] + result['UNCERTAIN_IMPORT']) % 100 == 0: - print(f"Imported {result['CONFIDENT_IMPORT'] + result['UNCERTAIN_IMPORT']} scrobbles...") return result @@ -136,11 +147,11 @@ def parse_spotify_lite(inputf): continue yield ("CONFIDENT_IMPORT",{ - 'title':title, - 'artiststr': artist, - 'timestamp': timestamp, - 'duration':played, - 'album': None + 'track_title':title, + 'track_artists': artist, + 'scrobble_time': timestamp, + 'scrobble_duration':played, + 'album_name': None },'') except Exception as e: yield ('FAIL',None,f"{entry} could not be parsed. Scrobble not imported. ({repr(e)})") @@ -240,11 +251,11 @@ def parse_spotify_full(inputf): yield (status,{ - 'title':title, - 'artiststr': artist, - 'album': album, - 'timestamp': timestamp, - 'duration':played + 'track_title':title, + 'track_artists': artist, + 'album_name': album, + 'scrobble_time': timestamp, + 'scrobble_duration':played },msg) except Exception as e: yield ('FAIL',None,f"{entry} could not be parsed. Scrobble not imported. ({repr(e)})") @@ -266,15 +277,36 @@ def parse_lastfm(inputf): try: yield ('CONFIDENT_IMPORT',{ - 'title': title, - 'artiststr': artist, - 'album': album, - 'timestamp': int(datetime.datetime.strptime( + 'track_title': title, + 'track_artists': artist, + 'album_name': album, + 'scrobble_time': int(datetime.datetime.strptime( time + '+0000', "%d %b %Y %H:%M%z" ).timestamp()), - 'duration':None + 'scrobble_duration':None },'') except Exception as e: yield ('FAIL',None,f"{entry} could not be parsed. Scrobble not imported. ({repr(e)})") continue + + +def parse_maloja(inputf): + + with open(inputf,'r') as inputfd: + data = json.load(inputfd) + + scrobbles = data['scrobbles'] + + for s in scrobbles: + try: + yield ('CONFIDENT_IMPORT',{ + 'track_title': s['track']['title'], + 'track_artists': s['track']['artists'], + 'album_name': s['track'].get('album',{}).get('name',''), + 'scrobble_time': s['time'], + 'scrobble_duration': s['duration'] + },'') + except Exception as e: + yield ('FAIL',None,f"{s} could not be parsed. Scrobble not imported. ({repr(e)})") + continue diff --git a/maloja/server.py b/maloja/server.py index f64e83f..5975a39 100644 --- a/maloja/server.py +++ b/maloja/server.py @@ -1,11 +1,11 @@ # technical import sys -import signal import os from threading import Thread import setproctitle -import pkg_resources +from importlib import resources from css_html_js_minify import html_minify, css_minify +import datauri # server stuff @@ -14,18 +14,19 @@ import waitress # doreah toolkit from doreah.logging import log -from doreah.timing import Clock from doreah import auth # rest of the project from . import database -from .utilities import resolveImage +from .database.jinjaview import JinjaDBConnection +from .images import resolve_track_image, resolve_artist_image from .malojauri import uri_to_internal, remove_identical -from .globalconf import malojaconfig, apikeystore, data_dir +from .globalconf import malojaconfig, data_dir from .jinjaenv.context import jinja_environment -from .apis import init_apis +from .apis import init_apis, apikeystore +from .proccontrol.profiler import profile ###### @@ -34,10 +35,10 @@ from .apis import init_apis PORT = malojaconfig["PORT"] HOST = malojaconfig["HOST"] -THREADS = 24 +THREADS = 12 BaseRequest.MEMFILE_MAX = 15 * 1024 * 1024 -STATICFOLDER = pkg_resources.resource_filename(__name__,"web/static") +#STATICFOLDER = importlib.resources.path(__name__,"web/static") webserver = Bottle() @@ -52,9 +53,12 @@ setproctitle.setproctitle("Maloja") def generate_css(): cssstr = "" - for file in os.listdir(os.path.join(STATICFOLDER,"css")): - with open(os.path.join(STATICFOLDER,"css",file),"r") as filed: - cssstr += filed.read() + with resources.files('maloja') / 'web' / 'static' as staticfolder: + + for file in os.listdir(os.path.join(staticfolder,"css")): + if file.endswith(".css"): + with open(os.path.join(staticfolder,"css",file),"r") as filed: + cssstr += filed.read() for file in os.listdir(data_dir['css']()): if file.endswith(".css"): @@ -158,10 +162,21 @@ def deprecated_api(pth): @webserver.route("/image") def dynamic_image(): keys = FormsDict.decode(request.query) - relevant, _, _, _, _ = uri_to_internal(keys) - result = resolveImage(**relevant) - if result == "": return "" - redirect(result,307) + if keys['type'] == 'track': + result = resolve_track_image(keys['id']) + elif keys['type'] == 'artist': + result = resolve_artist_image(keys['id']) + + if result is None or result['value'] in [None,'']: + return "" + if result['type'] == 'raw': + # data uris are directly served as image because a redirect to a data uri + # doesnt work + duri = datauri.DataURI(result['value']) + response.content_type = duri.mimetype + return duri.data + if result['type'] == 'url': + redirect(result['value'],307) @webserver.route("/images/") @webserver.route("/images/") @@ -172,39 +187,48 @@ def static_image(pth): ext = pth.split(".")[-1] small_pth = pth + "-small" if os.path.exists(data_dir['images'](small_pth)): - response = static_file(small_pth,root=data_dir['images']()) + resp = static_file(small_pth,root=data_dir['images']()) else: try: from pyvips import Image thumb = Image.thumbnail(data_dir['images'](pth),300) thumb.webpsave(data_dir['images'](small_pth)) - response = static_file(small_pth,root=data_dir['images']()) + resp = static_file(small_pth,root=data_dir['images']()) except Exception: - response = static_file(pth,root=data_dir['images']()) + resp = static_file(pth,root=data_dir['images']()) #response = static_file("images/" + pth,root="") - response.set_header("Cache-Control", "public, max-age=86400") - response.set_header("Content-Type", "image/" + ext) - return response + resp.set_header("Cache-Control", "public, max-age=86400") + resp.set_header("Content-Type", "image/" + ext) + return resp @webserver.route("/style.css") def get_css(): response.content_type = 'text/css' - global css - if malojaconfig["DEV_MODE"]: css = generate_css() - return css + if malojaconfig["DEV_MODE"]: return generate_css() + else: return css @webserver.route("/login") def login(): return auth.get_login_page() +# old @webserver.route("/.") @webserver.route("/media/.") def static(name,ext): - assert ext in ["txt","ico","jpeg","jpg","png","less","js"] - response = static_file(ext + "/" + name + "." + ext,root=STATICFOLDER) + assert ext in ["txt","ico","jpeg","jpg","png","less","js","ttf"] + with resources.files('maloja') / 'web' / 'static' as staticfolder: + response = static_file(ext + "/" + name + "." + ext,root=staticfolder) + response.set_header("Cache-Control", "public, max-age=3600") + return response + +# new, direct reference +@webserver.route("/static/") +def static(path): + with resources.files('maloja') / 'web' / 'static' as staticfolder: + response = static_file(path,root=staticfolder) response.set_header("Cache-Control", "public, max-age=3600") return response @@ -212,47 +236,46 @@ def static(name,ext): ### DYNAMIC -def static_html(name): +def jinja_page(name): if name in aliases: redirect(aliases[name]) keys = remove_identical(FormsDict.decode(request.query)) adminmode = request.cookies.get("adminmode") == "true" and auth.check(request) - clock = Clock() - clock.start() + with JinjaDBConnection() as conn: - loc_context = { - "adminmode":adminmode, - "apikey":request.cookies.get("apikey") if adminmode else None, - "apikeys":apikeystore, - "_urikeys":keys, #temporary! - } - loc_context["filterkeys"], loc_context["limitkeys"], loc_context["delimitkeys"], loc_context["amountkeys"], loc_context["specialkeys"] = uri_to_internal(keys) + loc_context = { + "dbc":conn, + "adminmode":adminmode, + "apikey":request.cookies.get("apikey") if adminmode else None, + "apikeys":apikeystore, + "_urikeys":keys, #temporary! + } + loc_context["filterkeys"], loc_context["limitkeys"], loc_context["delimitkeys"], loc_context["amountkeys"], loc_context["specialkeys"] = uri_to_internal(keys) - template = jinja_environment.get_template(name + '.jinja') - try: - res = template.render(**loc_context) - except (ValueError, IndexError): - abort(404,"This Artist or Track does not exist") + template = jinja_environment.get_template(name + '.jinja') + try: + res = template.render(**loc_context) + except (ValueError, IndexError): + abort(404,"This Artist or Track does not exist") if malojaconfig["DEV_MODE"]: jinja_environment.cache.clear() - log("Generated page {name} in {time:.5f}s".format(name=name,time=clock.stop()),module="debug_performance") return clean_html(res) @webserver.route("/") @auth.authenticated -def static_html_private(name): - return static_html(name) +def jinja_page_private(name): + return jinja_page(name) @webserver.route("/") -def static_html_public(name): - return static_html(name) +def jinja_page_public(name): + return jinja_page(name) @webserver.route("") @webserver.route("/") def mainpage(): - return static_html("start") + return jinja_page("start") # Shortlinks @@ -265,27 +288,6 @@ def redirect_track(artists,title): redirect("/track?title=" + title + "&" + "&".join("artist=" + artist for artist in artists.split("/"))) -###### -### SHUTDOWN -##### - - -def graceful_exit(sig=None,frame=None): - #urllib.request.urlopen("http://[::1]:" + str(DATABASE_PORT) + "/sync") - log("Received signal to shutdown") - try: - database.sync() - except Exception as e: - log("Error while shutting down!",e) - log("Server shutting down...") - sys.exit(0) - -#set graceful shutdown -signal.signal(signal.SIGINT, graceful_exit) -signal.signal(signal.SIGTERM, graceful_exit) - - - ###### @@ -302,11 +304,8 @@ def run_server(): try: #run(webserver, host=HOST, port=MAIN_PORT, server='waitress') + log(f"Listening on {HOST}:{PORT}") waitress.serve(webserver, host=HOST, port=PORT, threads=THREADS) except OSError: log("Error. Is another Maloja process already running?") raise - - - -run_server() diff --git a/maloja/thirdparty/__init__.py b/maloja/thirdparty/__init__.py index 68e221c..e38518e 100644 --- a/maloja/thirdparty/__init__.py +++ b/maloja/thirdparty/__init__.py @@ -11,6 +11,7 @@ import json import urllib.parse, urllib.request import base64 from doreah.logging import log +from threading import BoundedSemaphore from ..globalconf import malojaconfig from .. import database @@ -22,6 +23,11 @@ services = { "metadata":[] } +# have a limited number of worker threads so we don't completely hog the cpu with +# these requests. they are mostly network bound, so python will happily open up 200 new +# requests and then when all the responses come in we suddenly can't load pages anymore +thirdpartylock = BoundedSemaphore(4) + def import_scrobbles(identifier): for service in services['import']: @@ -34,27 +40,29 @@ def proxy_scrobble_all(artists,title,timestamp): service.scrobble(artists,title,timestamp) def get_image_track_all(track): - for service in services["metadata"]: - try: - res = service.get_image_track(track) - if res is not None: - log("Got track image for " + str(track) + " from " + service.name) - return res - else: - log("Could not get track image for " + str(track) + " from " + service.name) - except Exception as e: - log("Error getting track image from " + service.name + ": " + repr(e)) + with thirdpartylock: + for service in services["metadata"]: + try: + res = service.get_image_track(track) + if res is not None: + log("Got track image for " + str(track) + " from " + service.name) + return res + else: + log("Could not get track image for " + str(track) + " from " + service.name) + except Exception as e: + log("Error getting track image from " + service.name + ": " + repr(e)) def get_image_artist_all(artist): - for service in services["metadata"]: - try: - res = service.get_image_artist(artist) - if res is not None: - log("Got artist image for " + str(artist) + " from " + service.name) - return res - else: - log("Could not get artist image for " + str(artist) + " from " + service.name) - except Exception as e: - log("Error getting artist image from " + service.name + ": " + repr(e)) + with thirdpartylock: + for service in services["metadata"]: + try: + res = service.get_image_artist(artist) + if res is not None: + log("Got artist image for " + str(artist) + " from " + service.name) + return res + else: + log("Could not get artist image for " + str(artist) + " from " + service.name) + except Exception as e: + log("Error getting artist image from " + service.name + ": " + repr(e)) @@ -87,13 +95,13 @@ class GenericInterface: s = cls() if s.active_proxyscrobble(): services["proxyscrobble"].append(s) - log(cls.name + " registered as proxy scrobble target") + #log(cls.name + " registered as proxy scrobble target") if s.active_import(): services["import"].append(s) - log(cls.name + " registered as scrobble import source") + #log(cls.name + " registered as scrobble import source") if s.active_metadata(): services["metadata"].append(s) - log(cls.name + " registered as metadata provider") + #log(cls.name + " registered as metadata provider") def authorize(self): return True @@ -154,7 +162,7 @@ class ImportInterface(GenericInterface,abstract=True): def import_scrobbles(self): for scrobble in self.get_remote_scrobbles(): - database.createScrobble( + database.incoming_scrobble( artists=scrobble['artists'], title=scrobble['title'], time=scrobble['time'] diff --git a/maloja/upgrade.py b/maloja/upgrade.py new file mode 100644 index 0000000..7dff80e --- /dev/null +++ b/maloja/upgrade.py @@ -0,0 +1,90 @@ +# This module should take care of recognizing old install data and upgrading it before the actual server deals with it + +import os +import re +import csv + +from doreah.logging import log +from doreah.io import col + +from .globalconf import data_dir, dir_settings +from .apis import _apikeys + + +# Dealing with old style tsv files - these should be phased out everywhere +def read_tsvs(path,types): + result = [] + for f in os.listdir(path): + if f.split('.')[-1].lower() != 'tsv': continue + filepath = os.path.join(path,f) + result += read_tsv(filepath,types) + return result + +def read_tsv(filename,types): + with open(filename,'r') as filed: + reader = csv.reader(filed,delimiter="\t") + rawentries = [[col for col in entry if col] for entry in reader if len(entry)>0 and not entry[0].startswith('#')] + converted_entries = [[coltype(col) for col,coltype in zip(entry,types)] for entry in rawentries] + return converted_entries + + +def upgrade_apikeys(): + + oldfile = os.path.join(dir_settings['config'],"clients","authenticated_machines.tsv") + if os.path.exists(oldfile): + try: + entries = read_tsv(oldfile) + for key,identifier in entries: + _apikeys.apikeystore[identifier] = key + os.remove(oldfile) + except: + pass + + +def upgrade_db(callback_add_scrobbles): + + oldfolder = os.path.join(dir_settings['state'],"scrobbles") + newfolder = os.path.join(dir_settings['state'],".oldscrobbles") + os.makedirs(newfolder,exist_ok=True) + if os.path.exists(oldfolder): + scrobblefiles = [f for f in os.listdir(oldfolder) if f.endswith(".tsv")] + if len(scrobblefiles) > 0: + log("Upgrading v2 Database to v3 Database. This could take a while...",color='yellow') + idx = 0 + for sf in scrobblefiles: + idx += 1 + if re.match(r"[0-9]+_[0-9]+\.tsv",sf): + origin = 'legacy' + elif sf == "lastfmimport.tsv": + origin = 'import:lastfm' + elif sf == "spotifyimport.tsv": + origin = 'import:spotify' + else: + origin = 'unknown' + + scrobbles = read_tsv(os.path.join(oldfolder,sf),[int,str,str,str,str]) + #scrobbles = tsv.parse(os.path.join(oldfolder,sf),"int","string","string","string","string",comments=False) + scrobblelist = [] + log(f"\tImporting from {sf} ({idx}/{len(scrobblefiles)}) - {len(scrobbles)} Scrobbles") + for scrobble in scrobbles: + timestamp, artists, title, album, duration = scrobble + if album in ('-',''): album = None + if duration in ('-',''): duration = None + scrobblelist.append({ + "time":int(timestamp), + "track":{ + "artists":artists.split('␟'), + "title":title, + "length":None + }, + "duration":duration, + "origin":origin, + "extra":{ + "album_name":album + # saving this in the scrobble instead of the track because for now it's not meant + # to be authorative information, just payload of the scrobble + } + }) + callback_add_scrobbles(scrobblelist) + os.rename(os.path.join(oldfolder,sf),os.path.join(newfolder,sf)) + log("Done!",color='yellow') diff --git a/maloja/utilities/__init__.py b/maloja/utilities/__init__.py deleted file mode 100644 index 5fc1c9a..0000000 --- a/maloja/utilities/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .images import * -from .maintenance import * -from .utils import * diff --git a/maloja/utilities/images.py b/maloja/utilities/images.py deleted file mode 100644 index 0227225..0000000 --- a/maloja/utilities/images.py +++ /dev/null @@ -1,282 +0,0 @@ -from ..globalconf import data_dir, malojaconfig -from .. import thirdparty - -from doreah import caching -from doreah.logging import log - -import itertools -import os -import urllib -import random -import base64 -from threading import Thread, Timer -import re -import datetime - - - - - -### Caches - -cacheage = malojaconfig["CACHE_EXPIRE_POSITIVE"] * 24 * 3600 -cacheage_neg = malojaconfig["CACHE_EXPIRE_NEGATIVE"] * 24 * 3600 - -artist_cache = caching.Cache(name="imgcache_artists",maxage=cacheage,maxage_negative=cacheage_neg,persistent=True) -track_cache = caching.Cache(name="imgcache_tracks",maxage=cacheage,maxage_negative=cacheage_neg,persistent=True) - - -# removes emojis and weird shit from names -def clean(name): - return "".join(c for c in name if c.isalnum() or c in []).strip() - -def get_all_possible_filenames(artist=None,artists=None,title=None): - # check if we're dealing with a track or artist, then clean up names - # (only remove non-alphanumeric, allow korean and stuff) - - if title is not None and artists is not None: - track = True - title, artists = clean(title), [clean(a) for a in artists] - elif artist is not None: - track = False - artist = clean(artist) - else: return [] - - - superfolder = "tracks/" if track else "artists/" - - filenames = [] - - if track: - #unsafeartists = [artist.translate(None,"-_./\\") for artist in artists] - safeartists = [re.sub("[^a-zA-Z0-9]","",artist) for artist in artists] - #unsafetitle = title.translate(None,"-_./\\") - safetitle = re.sub("[^a-zA-Z0-9]","",title) - - if len(artists) < 4: - unsafeperms = itertools.permutations(artists) - safeperms = itertools.permutations(safeartists) - else: - unsafeperms = [sorted(artists)] - safeperms = [sorted(safeartists)] - - - for unsafeartistlist in unsafeperms: - filename = "-".join(unsafeartistlist) + "_" + title - if filename != "": - filenames.append(filename) - filenames.append(filename.lower()) - for safeartistlist in safeperms: - filename = "-".join(safeartistlist) + "_" + safetitle - if filename != "": - filenames.append(filename) - filenames.append(filename.lower()) - filenames = list(set(filenames)) - if len(filenames) == 0: filenames.append(str(hash((frozenset(artists),title)))) - else: - #unsafeartist = artist.translate(None,"-_./\\") - safeartist = re.sub("[^a-zA-Z0-9]","",artist) - - filename = artist - if filename != "": - filenames.append(filename) - filenames.append(filename.lower()) - filename = safeartist - if filename != "": - filenames.append(filename) - filenames.append(filename.lower()) - - filenames = list(set(filenames)) - if len(filenames) == 0: filenames.append(str(hash(artist))) - - return [superfolder + name for name in filenames] - -def local_files(artist=None,artists=None,title=None): - - - filenames = get_all_possible_filenames(artist,artists,title) - - images = [] - - for purename in filenames: - # direct files - for ext in ["png","jpg","jpeg","gif"]: - #for num in [""] + [str(n) for n in range(0,10)]: - if os.path.exists(data_dir['images'](purename + "." + ext)): - images.append("/images/" + purename + "." + ext) - - # folder - try: - for f in os.listdir(data_dir['images'](purename)): - if f.split(".")[-1] in ["png","jpg","jpeg","gif"]: - images.append("/images/" + purename + "/" + f) - except: - pass - - return images - - - -# these caches are there so we don't check all files every time, but return the same one -local_cache_age = malojaconfig["LOCAL_IMAGE_ROTATE"] -local_artist_cache = caching.Cache(maxage=local_cache_age) -local_track_cache = caching.Cache(maxage=local_cache_age) - -def getTrackImage(artists,title,fast=False): - - hashable_track = (frozenset(artists),title) - - # Prio 1: Local image - if malojaconfig["USE_LOCAL_IMAGES"]: - try: - return local_track_cache.get(hashable_track) - except: - images = local_files(artists=artists,title=title) - if len(images) != 0: - res = random.choice(images) - local_track_cache.add(hashable_track,res) - return urllib.parse.quote(res) - - - # Prio 2: Cached remote link - try: - result = track_cache.get(hashable_track) - if result is not None: return result - # if we have cached the nonexistence of that image, we immediately return - # the redirect to the artist and let the resolver handle it - # (even if we're not in a fast lookup right now) - for a in artists: - res = getArtistImage(artist=a,fast=True) - if res != "": return res - return "" - except: - pass - - - # fast request will not go further than this, but now generate redirect link - if fast: - return ("/image?title=" + urllib.parse.quote(title) + "&" + "&".join( - "artist=" + urllib.parse.quote(a) for a in artists)) - - - # Prio 3 (only non-fast lookup): actually call third parties - result = thirdparty.get_image_track_all((artists,title)) - # cache results (even negative ones) - track_cache.add(hashable_track,result) - # return either result or redirect to artist - if result is not None: return result - for a in artists: - res = getArtistImage(artist=a,fast=False) - if res != "": return res - return "" - - -def getArtistImage(artist,fast=False): - - # Prio 1: Local image - if malojaconfig["USE_LOCAL_IMAGES"]: - try: - return local_artist_cache.get(artist) - except: - images = local_files(artist=artist) - if len(images) != 0: - res = random.choice(images) - local_artist_cache.add(artist,res) - return urllib.parse.quote(res) - - - # Prio 2: Cached remote link - try: - result = artist_cache.get(artist) - if result is not None: return result - else: return "" - # none means non-existence is cached, return empty - except: - pass - # no cache entry, go on - - - # fast request will not go further than this, but now generate redirect link - if fast: return "/image?artist=" + urllib.parse.quote(artist) - - - # Prio 3 (only non-fast lookup): actually call third parties - result = thirdparty.get_image_artist_all(artist) - # cache results (even negative ones) - artist_cache.add(artist,result) #cache_artist(artist,result) - if result is not None: return result - else: return "" - -def getTrackImages(trackobjectlist,fast=False): - - threads = [] - - for track in trackobjectlist: - t = Thread(target=getTrackImage,args=(track["artists"],track["title"],),kwargs={"fast":fast}) - t.start() - threads.append(t) - - for t in threads: - t.join() - - - return [getTrackImage(t["artists"],t["title"]) for t in trackobjectlist] - -def getArtistImages(artistlist,fast=False): - - threads = [] - - for artist in artistlist: - t = Thread(target=getArtistImage,args=(artist,),kwargs={"fast":fast}) - t.start() - threads.append(t) - - for t in threads: - t.join() - - # async calls only cached results, now we need to get them - return [getArtistImage(a) for a in artistlist] - - - -# new way of serving images -# instead always generate a link locally, but redirect that on the fly -# this way the page can load faster and images will trickle in without having to resort to XHTTP requests - -def resolveImage(artist=None,track=None): - if track is not None: - return getTrackImage(track["artists"],track["title"]) - elif artist is not None: - return getArtistImage(artist) - - - - - - - -def set_image(b64,**keys): - track = "title" in keys - - log("Trying to set image, b64 string: " + str(b64[:30] + "..."),module="debug") - - regex = r"data:image/(\w+);base64,(.+)" - type,b64 = re.fullmatch(regex,b64).groups() - b64 = base64.b64decode(b64) - filename = "webupload" + str(int(datetime.datetime.now().timestamp())) + "." + type - for folder in get_all_possible_filenames(**keys): - if os.path.exists(data_dir['images'](folder)): - with open(data_dir['images'](folder,filename),"wb") as f: - f.write(b64) - break - else: - folder = get_all_possible_filenames(**keys)[0] - os.makedirs(data_dir['images'](folder)) - with open(data_dir['images'](folder,filename),"wb") as f: - f.write(b64) - - log("Saved image as " + data_dir['images'](folder,filename),module="debug") - - # set as current picture in rotation - if track: local_track_cache.add((frozenset(keys["artists"]),keys["title"]),os.path.join("/images",folder,filename)) - else: local_artist_cache.add(keys["artist"],os.path.join("/images",folder,filename)) diff --git a/maloja/utilities/maintenance.py b/maloja/utilities/maintenance.py deleted file mode 100644 index b8df8f4..0000000 --- a/maloja/utilities/maintenance.py +++ /dev/null @@ -1,114 +0,0 @@ -from ..__pkginfo__ import VERSION -from ..malojatime import ranges, thisweek, thisyear -from ..globalconf import malojaconfig - -from doreah.regular import yearly, daily -from doreah.logging import log - -import datetime -import json -import urllib -import itertools - - - -get_track = lambda x:(frozenset(x["track"]["artists"]),x["track"]["title"]) -get_artist = lambda x:x["artist"] - -def group_by_attribute(sequence,attribute): - grouped = itertools.groupby(sequence,key=lambda x:x[attribute]) - for attrvalue,members in grouped: - yield attrvalue,list(members) - -def collect_rankings(chart,identify,collection,iteration=None,count=True): - grouped = group_by_attribute(chart,"rank") - for rank, members in grouped: - if not count and rank not in rankmedals: break - if count and rank != 1: break - - for m in members: - # get the actual object that we're counting - entity = identify(m) - - # count no1 spots - if count: - collection[entity] = collection.setdefault(entity,0) + 1 - - # collect instances of top3 spots - else: - medal = rankmedals[rank] - collection.setdefault(entity,{}).setdefault(medal,[]).append(iteration) - - -rankmedals = { - 1:'gold', - 2:'silver', - 3:'bronze' -} - -@yearly -def update_medals(): - - - from ..database import MEDALS_ARTISTS, MEDALS_TRACKS, STAMPS, get_charts_artists, get_charts_tracks - - - MEDALS_ARTISTS.clear() - MEDALS_TRACKS.clear() - - for year in ranges(step="year"): - if year == thisyear(): break - - charts_artists = get_charts_artists(timerange=year) - charts_tracks = get_charts_tracks(timerange=year) - - collect_rankings(charts_artists,get_artist,MEDALS_ARTISTS,iteration=year,count=False) - collect_rankings(charts_tracks,get_track,MEDALS_TRACKS,iteration=year,count=False) - - -@daily -def update_weekly(): - - from ..database import WEEKLY_TOPTRACKS, WEEKLY_TOPARTISTS, get_charts_artists, get_charts_tracks - - - WEEKLY_TOPARTISTS.clear() - WEEKLY_TOPTRACKS.clear() - - for week in ranges(step="week"): - if week == thisweek(): break - - charts_artists = get_charts_artists(timerange=week) - charts_tracks = get_charts_tracks(timerange=week) - - collect_rankings(charts_artists,get_artist,WEEKLY_TOPARTISTS) - collect_rankings(charts_tracks,get_track,WEEKLY_TOPTRACKS) - - -@daily -def send_stats(): - if malojaconfig["SEND_STATS"]: - - log("Sending daily stats report...") - - from ..database import ARTISTS, TRACKS, SCROBBLES - - keys = { - "url":"https://myrcella.krateng.ch/malojastats", - "method":"POST", - "headers":{"Content-Type": "application/json"}, - "data":json.dumps({ - "name":malojaconfig["NAME"], - "url":malojaconfig["PUBLIC_URL"], - "version":VERSION, - "artists":len(ARTISTS), - "tracks":len(TRACKS), - "scrobbles":len(SCROBBLES) - }).encode("utf-8") - } - try: - req = urllib.request.Request(**keys) - response = urllib.request.urlopen(req) - log("Sent daily report!") - except: - log("Could not send daily report!") diff --git a/maloja/utilities/utils.py b/maloja/utilities/utils.py deleted file mode 100644 index 9795628..0000000 --- a/maloja/utilities/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -import json - - -##### -## SERIALIZE -##### - -def serialize(obj): - try: - return serialize(obj.hashable()) - except: - try: - return json.dumps(obj) - except: - if isinstance(obj, (list, tuple)): - return "[" + ",".join(serialize(o) for o in obj) + "]" - elif isinstance(obj,dict): - return "{" + ",".join(serialize(o) + ":" + serialize(obj[o]) for o in obj) + "}" - return json.dumps(obj.hashable()) - - - #if isinstance(obj,list) or if isinstance(obj,tuple): - # return "[" + ",".join(dumps(o) for o in obj) + "]" - #if isinstance(obj,str) diff --git a/maloja/web/jinja/about.jinja b/maloja/web/jinja/about.jinja new file mode 100644 index 0000000..0959be7 --- /dev/null +++ b/maloja/web/jinja/about.jinja @@ -0,0 +1,55 @@ +{% extends "abstracts/base.jinja" %} +{% block title %}Maloja - About{% endblock %} + +{% block content %} + + + + + +
+ +
+
+ 		+

Maloja

+
+ Version {{ pkginfo.VERSION }}
+ {# {% if adminmode %} + Python {{ platform.sys.version }}
+ {{ platform.system() }} {{ platform.release() }} ({{ platform.machine() }})
+
+ {% set pid = psutil.os.getpid() %} + {% set proc = psutil.Process(pid) %} + CPU: + {{ proc.cpu_percent() | int }}% Maloja, + {{ (psutil.getloadavg()[2]/psutil.os.cpu_count() * 100) | int }}% System +
+ RAM: + {{ (proc.memory_info().rss / (1024*1024)) | int }}MB Maloja ({{ proc.memory_percent() | int }}%), + {{ (psutil.virtual_memory().used / (1024*1024)) | int }}MB System ({{ psutil.virtual_memory().percent | int }}%) + {% endif %} #} + +

+ +

+ + + +
+ +Get your own Maloja server: +

+ + + + + + + + + +

+Maloja is released under the GNU General Public License v3.0. + +{% endblock %} diff --git a/maloja/web/jinja/abstracts/admin.jinja b/maloja/web/jinja/abstracts/admin.jinja index 9669445..dae7397 100644 --- a/maloja/web/jinja/abstracts/admin.jinja +++ b/maloja/web/jinja/abstracts/admin.jinja @@ -21,10 +21,12 @@ ['setup','Server Setup'], ['settings','Settings'], ['apikeys','API Keys'], - ['manual','Manual Scrobbling'], - ['issues','Database Maintenance'] + ['manual','Manual Scrobbling'] + ] %} - {# ['import','Scrobble Import'] hide page for now #} + {# ['import','Scrobble Import'], + ['issues','Database Maintenance'] + hide for now #} {% if page=='admin_' + tab_url %} {{ tab_name }} {% else %} diff --git a/maloja/web/jinja/abstracts/base.jinja b/maloja/web/jinja/abstracts/base.jinja index 1cab479..29cb70d 100644 --- a/maloja/web/jinja/abstracts/base.jinja +++ b/maloja/web/jinja/abstracts/base.jinja @@ -11,11 +11,14 @@ + + + {% block scripts %}{% endblock %} @@ -48,10 +51,12 @@
- Get your own charts on + + About
-
+
diff --git a/maloja/web/jinja/admin_issues.jinja b/maloja/web/jinja/admin_issues.jinja index 931bb4c..bda65bb 100644 --- a/maloja/web/jinja/admin_issues.jinja +++ b/maloja/web/jinja/admin_issues.jinja @@ -33,7 +33,7 @@ {% endblock %} -{% set issuedata = dbp.issues() %} +{% set issuedata = dbc.issues() %} {% block maincontent %} diff --git a/maloja/web/jinja/admin_overview.jinja b/maloja/web/jinja/admin_overview.jinja index 1c129cf..a0fc2f8 100644 --- a/maloja/web/jinja/admin_overview.jinja +++ b/maloja/web/jinja/admin_overview.jinja @@ -77,10 +77,13 @@

Download Data

- Download your data to have a backup.

+ Backup your data.

- + + + + @@ -89,7 +92,9 @@ Report Issue
Readme
- PyPi
+ PyPi | + Dockerhub | + Github
diff --git a/maloja/web/jinja/admin_setup.jinja b/maloja/web/jinja/admin_setup.jinja index 233da8e..e6313f1 100644 --- a/maloja/web/jinja/admin_setup.jinja +++ b/maloja/web/jinja/admin_setup.jinja @@ -47,14 +47,14 @@ {% endblock %} -{% set rulesets = dbp.get_predefined_rulesets() %} +{% set rulesets = dbc.get_predefined_rulesets() %} {% block maincontent %}

Start Scrobbling

- If you use Vivaldi, Brave, Iridium or any other Chromium-based browser and listen to music on Plex or YouTube Music, download the extension and simply enter the server URL as well as your API key in the relevant fields. They will turn green if the server is accessible. + If you use a Chromium-based browser and listen to music on Plex, Spotify, Soundcloud, Bandcamp or YouTube Music, download the extension and simply enter the server URL as well as your API key in the relevant fields. They will turn green if the server is accessible.

You can also use any standard-compliant scrobbler. For GNUFM (audioscrobbler) scrobblers, enter yourserver.tld/apis/audioscrobbler as your Gnukebox server and your API key as the password. For Listenbrainz scrobblers, use yourserver.tld/apis/listenbrainz as the API URL and your API key as token.

@@ -62,15 +62,20 @@ yourserver.tld/apis/mlj_1/newscrobble - (make sure to use the public URL) with the key-value-pairs + (make sure to use the public URL) with the following values encoded as JSON:

- - - - - + + + + + + + + + +
artist Artist String
title Title String
key API Key
time UNIX timestamp - optional, defaults to time of request
seconds Duration of Scrobble - optional and currently not used
artists List of artist names
title Song title
album Album title - optional
albumartists List of album artists - optional
duration Duration of play in seconds - optional
length Full length of the trackin seconds - optional
time UNIX timestamp - optional, defaults to time of request
fix Set this to false to skip server-side metadata fixing - optional
key API Key, see API Keys


Finally, you could always manually scrobble! @@ -79,7 +84,8 @@

Import your Last.FM data

- Switching from Last.fm? Download all your data and run the command maloja import (the file you just downloaded). + Switching from Last.fm? Download all your data and run the command maloja import (the file you just downloaded).
+ You can also try out Multi-Scrobbler to import scrobbles from a wider range of sources.

Set up some rules

diff --git a/maloja/web/jinja/artist.jinja b/maloja/web/jinja/artist.jinja index b59c59a..f738c8b 100644 --- a/maloja/web/jinja/artist.jinja +++ b/maloja/web/jinja/artist.jinja @@ -1,5 +1,5 @@ {% extends "abstracts/base.jinja" %} -{% block title %}Maloja - {{ artist }}{% endblock %} +{% block title %}Maloja - {{ info.artist }}{% endblock %} {% import 'snippets/links.jinja' as links %} {% import 'partials/awards_artist.jinja' as awards %} @@ -9,7 +9,7 @@ {% endblock %} {% set artist = filterkeys.artist %} -{% set info = db.artistInfo(artist) %} +{% set info = db.artist_info(artist=artist) %} {% set credited = info.get('replace') %} {% set included = info.get('associated') %} @@ -39,15 +39,15 @@ {% if adminmode %}
{% else %} -
+
{% endif %} -

{{ artist }}

+

{{ info.artist }}

{% if competes %}#{{ info.position }}{% endif %}
{% if competes and included %} diff --git a/maloja/web/jinja/charts_artists.jinja b/maloja/web/jinja/charts_artists.jinja index c5b6a13..87f51ff 100644 --- a/maloja/web/jinja/charts_artists.jinja +++ b/maloja/web/jinja/charts_artists.jinja @@ -5,11 +5,11 @@ {% endblock %} -{% set charts = dbp.get_charts_artists(filterkeys,limitkeys) %} +{% set charts = dbc.get_charts_artists(filterkeys,limitkeys) %} {% set pages = math.ceil(charts.__len__() / amountkeys.perpage) %} {% if charts[0] is defined %} {% set topartist = charts[0].artist %} - {% set img = utilities.getArtistImage(artist=topartist,fast=True) %} + {% set img = images.get_artist_image(topartist) %} {% else %} {% set img = "/favicon.png" %} {% endif %} diff --git a/maloja/web/jinja/charts_tracks.jinja b/maloja/web/jinja/charts_tracks.jinja index e4779f7..17b2f20 100644 --- a/maloja/web/jinja/charts_tracks.jinja +++ b/maloja/web/jinja/charts_tracks.jinja @@ -7,11 +7,11 @@ {% endblock %} -{% set charts = dbp.get_charts_tracks(filterkeys,limitkeys) %} +{% set charts = dbc.get_charts_tracks(filterkeys,limitkeys) %} {% set pages = math.ceil(charts.__len__() / amountkeys.perpage) %} {% if charts[0] is defined %} {% set toptrack = charts[0].track %} - {% set img = utilities.getTrackImage(artists=toptrack.artists,title=toptrack.title,fast=True) %} + {% set img = images.get_track_image(toptrack) %} {% else %} {% set img = "/favicon.png" %} {% endif %} diff --git a/maloja/web/jinja/compare.jinja b/maloja/web/jinja/compare.jinja deleted file mode 100644 index 9eaea42..0000000 --- a/maloja/web/jinja/compare.jinja +++ /dev/null @@ -1,112 +0,0 @@ -{% extends "abstracts/base.jinja" %} -{% block title %}Maloja - Compare{% endblock %} - -{% import 'snippets/links.jinja' as links %} - -{% block scripts %} - -{% endblock %} - -{% set data = db.compare(specialkeys.remote) %} -{% set comparedata = data.result %} -{% set info = data.info %} -{% set bestartist = data.commonartist %} - - - -{% set fullmatch = comparedata.common[1]*100 %} -{% set partialmatch = comparedata.more_self[1]*100 + comparedata.more_other[1]*100 %} - -{% set match = fullmatch + (partialmatch)/2 %} -{% set pixel_fullmatch = fullmatch * 2.5 %} -{% set pixel_partialmatch = (fullmatch+partialmatch) * 2.5 %} - -{% set match = [match,100] | min %} - -{% set r = [255*match/50,255] | min %} -{% set g = [255*match/50,255] | min %} -{% set b = [255*(match/50-1),0] | max %} - - -{% block content %} - - - - - - - - - - - - - - - - - -

{{ info.ownname }}

 - -
- {{ match | round(1) }}% - -
-

{{ info.remotename }}

- The size of the circle shows matching music taste. - The fuzziness of its border indicates differences in quantity. -
- Common Favorite -

{{ links.link(bestartist) }}

- -
- - -{% endblock %} diff --git a/maloja/web/jinja/partials/awards_artist.jinja b/maloja/web/jinja/partials/awards_artist.jinja index b47eb5e..10bfecc 100644 --- a/maloja/web/jinja/partials/awards_artist.jinja +++ b/maloja/web/jinja/partials/awards_artist.jinja @@ -55,13 +55,19 @@ {% macro certs(artist) %} -{% for track in db.get_tracks(artist=artist) -%} - {% set info = db.trackInfo(track) %} - {% if info.certification is not none -%} - + +{% set charts = db.get_charts_tracks(artist=artist,timerange=malojatime.alltime()) %} +{% for e in charts -%} + {%- if e.scrobbles >= settings.scrobbles_gold -%}{% set cert = 'gold' %}{%- endif -%} + {%- if e.scrobbles >= settings.scrobbles_platinum -%}{% set cert = 'platinum' %}{%- endif -%} + {%- if e.scrobbles >= settings.scrobbles_diamond -%}{% set cert = 'diamond' %}{%- endif -%} + + {%- if cert -%} + {%- endif %} + {%- endfor %} {%- endmacro %} diff --git a/maloja/web/jinja/partials/awards_track.jinja b/maloja/web/jinja/partials/awards_track.jinja index bdb8748..12a1948 100644 --- a/maloja/web/jinja/partials/awards_track.jinja +++ b/maloja/web/jinja/partials/awards_track.jinja @@ -53,7 +53,7 @@ -{% set info = db.trackInfo(track) %} +{% set info = db.track_info(track=track) %} {% if info.certification is not none %} -
+
#{{ rank }} {{ artist }}
diff --git a/maloja/web/jinja/partials/charts_tracks.jinja b/maloja/web/jinja/partials/charts_tracks.jinja index 24734c8..f9f3d70 100644 --- a/maloja/web/jinja/partials/charts_tracks.jinja +++ b/maloja/web/jinja/partials/charts_tracks.jinja @@ -2,7 +2,7 @@ {% import 'snippets/entityrow.jinja' as entityrow %} {% if charts is undefined %} - {% set charts = dbp.get_charts_tracks(filterkeys,limitkeys) %} + {% set charts = dbc.get_charts_tracks(filterkeys,limitkeys) %} {% endif %} {% if compare %} {% if compare is true %} @@ -10,7 +10,7 @@ {% if compare is none %}{% set compare = False %}{% endif %} {% endif %} {% if compare %} - {% set prevtracks = dbp.get_charts_tracks(filterkeys,{'timerange':compare}) %} + {% set prevtracks = dbc.get_charts_tracks(filterkeys,{'timerange':compare}) %} {% set lastranks = {} %} {% for t in prevtracks %} diff --git a/maloja/web/jinja/partials/charts_tracks_tiles.jinja b/maloja/web/jinja/partials/charts_tracks_tiles.jinja index ef110c4..6d597e7 100644 --- a/maloja/web/jinja/partials/charts_tracks_tiles.jinja +++ b/maloja/web/jinja/partials/charts_tracks_tiles.jinja @@ -2,7 +2,7 @@ {% if charts is undefined %} - {% set charts = dbp.get_charts_tracks(filterkeys,limitkeys) %} + {% set charts = dbc.get_charts_tracks(filterkeys,limitkeys) %} {% endif %} {% set charts_14 = charts | fixlength(14) %} @@ -23,7 +23,7 @@ {% set rank = entry.rank %} -
+
#{{ rank }} {{ track.title }}
diff --git a/maloja/web/jinja/partials/performance.jinja b/maloja/web/jinja/partials/performance.jinja index 6847a7d..6d1c20f 100644 --- a/maloja/web/jinja/partials/performance.jinja +++ b/maloja/web/jinja/partials/performance.jinja @@ -1,6 +1,6 @@ {% import 'snippets/links.jinja' as links %} -{% set ranges = dbp.get_performance(filterkeys,limitkeys,delimitkeys) %} +{% set ranges = dbc.get_performance(filterkeys,limitkeys,delimitkeys) %} {% set minrank = ranges|map(attribute="rank")|reject("none")|max|default(60) %} {% set minrank = minrank + 20 %} diff --git a/maloja/web/jinja/partials/pulse.jinja b/maloja/web/jinja/partials/pulse.jinja index fb1317d..68960e4 100644 --- a/maloja/web/jinja/partials/pulse.jinja +++ b/maloja/web/jinja/partials/pulse.jinja @@ -1,6 +1,6 @@ {% import 'snippets/links.jinja' as links %} -{% set ranges = dbp.get_pulse(filterkeys,limitkeys,delimitkeys) %} +{% set ranges = dbc.get_pulse(filterkeys,limitkeys,delimitkeys) %} {% set maxbar = ranges|map(attribute="scrobbles")|max|default(1) %} {% if maxbar < 1 %}{% set maxbar = 1 %}{% endif %} diff --git a/maloja/web/jinja/partials/scrobbles.jinja b/maloja/web/jinja/partials/scrobbles.jinja index c3086dc..1fb1cbd 100644 --- a/maloja/web/jinja/partials/scrobbles.jinja +++ b/maloja/web/jinja/partials/scrobbles.jinja @@ -1,4 +1,4 @@ -{% set scrobbles = dbp.get_scrobbles(filterkeys,limitkeys,amountkeys) %} +{% set scrobbles = dbc.get_scrobbles(filterkeys,limitkeys,amountkeys) %} {% set firstindex = amountkeys.page * amountkeys.perpage %} {% set lastindex = firstindex + amountkeys.perpage %} @@ -6,12 +6,33 @@ {% import 'snippets/entityrow.jinja' as entityrow %} + + + {% for s in scrobbles -%} {%- if loop.index0 >= firstindex and loop.index0 < lastindex -%} - {{ entityrow.row(s) }} + {{ entityrow.row(s.track) }} + {% if adminmode %} + + + {% endif %} {%- endif -%} {% endfor %} diff --git a/maloja/web/jinja/partials/top_artists.jinja b/maloja/web/jinja/partials/top_artists.jinja index 2b750a1..60c1481 100644 --- a/maloja/web/jinja/partials/top_artists.jinja +++ b/maloja/web/jinja/partials/top_artists.jinja @@ -1,7 +1,7 @@ {% import 'snippets/links.jinja' as links %} {% import 'snippets/entityrow.jinja' as entityrow %} -{% set ranges = dbp.get_top_artists(limitkeys,delimitkeys) %} +{% set ranges = dbc.get_top_artists(limitkeys,delimitkeys) %} {% set maxbar = ranges|map(attribute="scrobbles")|max|default(1) %} {% if maxbar < 1 %}{% set maxbar = 1 %}{% endif %} diff --git a/maloja/web/jinja/partials/top_tracks.jinja b/maloja/web/jinja/partials/top_tracks.jinja index 5af27eb..2c766f2 100644 --- a/maloja/web/jinja/partials/top_tracks.jinja +++ b/maloja/web/jinja/partials/top_tracks.jinja @@ -1,7 +1,7 @@ {% import 'snippets/links.jinja' as links %} {% import 'snippets/entityrow.jinja' as entityrow %} -{% set ranges = dbp.get_top_tracks(filterkeys,limitkeys,delimitkeys) %} +{% set ranges = dbc.get_top_tracks(filterkeys,limitkeys,delimitkeys) %} {% set maxbar = ranges|map(attribute="scrobbles")|max|default(1) %} {% if maxbar < 1 %}{% set maxbar = 1 %}{% endif %} diff --git a/maloja/web/jinja/performance.jinja b/maloja/web/jinja/performance.jinja index a30eb7d..5ab281d 100644 --- a/maloja/web/jinja/performance.jinja +++ b/maloja/web/jinja/performance.jinja @@ -4,9 +4,9 @@ {% import 'snippets/filterdescription.jinja' as filterdesc %} {% if filterkeys.get('track') is not none %} - {% set img = utilities.getTrackImage(artists=filterkeys.track.artists,title=filterkeys.track.title,fast=True) %} + {% set img = images.get_track_image(filterkeys.track) %} {% elif filterkeys.get('artist') is not none %} - {% set img = utilities.getArtistImage(filterkeys.artist,fast=True) %} + {% set img = images.get_artist_image(filterkeys.artist) %} {% else %} {% set img = "/favicon.png" %} {% endif %} diff --git a/maloja/web/jinja/pulse.jinja b/maloja/web/jinja/pulse.jinja index 08ac85f..79df94a 100644 --- a/maloja/web/jinja/pulse.jinja +++ b/maloja/web/jinja/pulse.jinja @@ -4,9 +4,9 @@ {% import 'snippets/filterdescription.jinja' as filterdesc %} {% if filterkeys.get('track') is not none %} - {% set img = utilities.getTrackImage(artists=filterkeys.track.artists,title=filterkeys.track.title,fast=True) %} + {% set img = images.get_track_image(filterkeys.track) %} {% elif filterkeys.get('artist') is not none %} - {% set img = utilities.getArtistImage(filterkeys.artist,fast=True) %} + {% set img = images.get_artist_image(filterkeys.artist) %} {% else %} {% set img = "/favicon.png" %} {% endif %} diff --git a/maloja/web/jinja/scrobbles.jinja b/maloja/web/jinja/scrobbles.jinja index 062cb90..0331bb9 100644 --- a/maloja/web/jinja/scrobbles.jinja +++ b/maloja/web/jinja/scrobbles.jinja @@ -4,15 +4,15 @@ {% import 'snippets/filterdescription.jinja' as filterdesc %} {% import 'snippets/pagination.jinja' as pagination %} -{% set scrobbles = dbp.get_scrobbles(filterkeys,limitkeys,amountkeys) %} +{% set scrobbles = dbc.get_scrobbles(filterkeys,limitkeys,amountkeys) %} {% set pages = math.ceil(scrobbles.__len__() / amountkeys.perpage) %} {% if filterkeys.get('track') is not none %} - {% set img = utilities.getTrackImage(artists=filterkeys.track.artists,title=filterkeys.track.title,fast=True) %} + {% set img = images.get_track_image(filterkeys.track) %} {% elif filterkeys.get('artist') is not none %} - {% set img = utilities.getArtistImage(filterkeys.artist,fast=True) %} + {% set img = images.get_artist_image(filterkeys.artist) %} {% elif scrobbles.__len__() > 0 %} - {% set img = utilities.getTrackImage(artists=scrobbles[0].artists,title=scrobbles[0].title,fast=True) %} + {% set img = images.get_track_image(scrobbles[0].track) %} {% else %} {% set img = "/favicon.png" %} {% endif %} diff --git a/maloja/web/jinja/snippets/entityrow.jinja b/maloja/web/jinja/snippets/entityrow.jinja index fb2668f..e44f7fd 100644 --- a/maloja/web/jinja/snippets/entityrow.jinja +++ b/maloja/web/jinja/snippets/entityrow.jinja @@ -3,9 +3,9 @@ {% import 'snippets/links.jinja' as links %} {% if 'artists' in entity %} - {% set img = utilities.getTrackImage(artists=entity.artists,title=entity.title,fast=True) %} + {% set img = images.get_track_image(entity) %} {% else %} - {% set img = utilities.getArtistImage(entity,fast=True) %} + {% set img = images.get_artist_image(entity) %} {% endif %} diff --git a/maloja/web/jinja/start.jinja b/maloja/web/jinja/start.jinja index 0c0d112..424ac86 100644 --- a/maloja/web/jinja/start.jinja +++ b/maloja/web/jinja/start.jinja @@ -68,7 +68,7 @@ {% for range in xcurrent %} {{ range.localisation }} - {{ db.get_scrobbles_num(timerange=range.range) }} + {{ dbc.get_scrobbles_num({'timerange':range.range}) }} {% endfor %}

diff --git a/maloja/web/jinja/top_artists.jinja b/maloja/web/jinja/top_artists.jinja index bb0c3d3..175d213 100644 --- a/maloja/web/jinja/top_artists.jinja +++ b/maloja/web/jinja/top_artists.jinja @@ -4,9 +4,9 @@ -{% set entries = dbp.get_top_artists(limitkeys,delimitkeys) %} +{% set entries = dbc.get_top_artists(limitkeys,delimitkeys) %} {% set repr = entries | find_representative('artist','scrobbles') %} -{% set img = "/favicon.png" if repr is none else utilities.getArtistImage(repr.artist) %} +{% set img = "/favicon.png" if repr is none else images.get_artist_image(repr.artist) %} {% block content %} diff --git a/maloja/web/jinja/top_tracks.jinja b/maloja/web/jinja/top_tracks.jinja index cdc66cd..42ae78d 100644 --- a/maloja/web/jinja/top_tracks.jinja +++ b/maloja/web/jinja/top_tracks.jinja @@ -4,9 +4,9 @@ -{% set entries = dbp.get_top_tracks(filterkeys,limitkeys,delimitkeys) %} +{% set entries = dbc.get_top_tracks(filterkeys,limitkeys,delimitkeys) %} {% set repr = entries | find_representative('track','scrobbles') %} -{% set img = "/favicon.png" if repr is none else utilities.getTrackImage(repr.track.artists,repr.track.title) %} +{% set img = "/favicon.png" if repr is none else images.get_track_image(repr.track) %} {% block content %} diff --git a/maloja/web/jinja/track.jinja b/maloja/web/jinja/track.jinja index 95076c6..03762c6 100644 --- a/maloja/web/jinja/track.jinja +++ b/maloja/web/jinja/track.jinja @@ -1,5 +1,5 @@ {% extends "abstracts/base.jinja" %} -{% block title %}Maloja - {{ track.title }}{% endblock %} +{% block title %}Maloja - {{ info.track.title }}{% endblock %} {% import 'snippets/links.jinja' as links %} @@ -13,7 +13,7 @@ {% endblock %} {% set track = filterkeys.track %} -{% set info = db.trackInfo(track) %} +{% set info = dbc.track_info({'track':track}) %} {% set initialrange ='month' %} @@ -33,16 +33,16 @@ {% if adminmode %}
{% else %} -
+
{% endif %}
{{ malojatime.timestamp_desc(s["time"],short=shortTimeDesc) }} + + + + + + +
+ + + +
+ + +
{{ links.links(track.artists) }}
-

{{ track.title }}

+

{{ info.track.title }}

{{ awards.certs(track) }} #{{ info.position }}
diff --git a/maloja/web/static/css/grisonsfont.css b/maloja/web/static/css/grisonsfont.css index e424fbf..ef4cb66 100644 --- a/maloja/web/static/css/grisonsfont.css +++ b/maloja/web/static/css/grisonsfont.css @@ -1,45 +1,7 @@ -/* cyrillic-ext */ + @font-face { font-family: 'Ubuntu'; font-style: normal; font-weight: 400; - src: local('Ubuntu Regular'), local('Ubuntu-Regular'), url(https://fonts.gstatic.com/s/ubuntu/v14/4iCs6KVjbNBYlgoKcg72j00.woff2) format('woff2'); -} -/* cyrillic */ -@font-face { - font-family: 'Ubuntu'; - font-style: normal; - font-weight: 400; - src: local('Ubuntu Regular'), local('Ubuntu-Regular'), url(https://fonts.gstatic.com/s/ubuntu/v14/4iCs6KVjbNBYlgoKew72j00.woff2) format('woff2'); - unicode-range: U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116; -} -/* greek-ext */ -@font-face { - font-family: 'Ubuntu'; - font-style: normal; - font-weight: 400; - src: local('Ubuntu Regular'), local('Ubuntu-Regular'), url(https://fonts.gstatic.com/s/ubuntu/v14/4iCs6KVjbNBYlgoKcw72j00.woff2) format('woff2'); - unicode-range: U+1F00-1FFF; -} -/* greek */ -@font-face { - font-family: 'Ubuntu'; - font-style: normal; - font-weight: 400; - src: local('Ubuntu Regular'), local('Ubuntu-Regular'), url(https://fonts.gstatic.com/s/ubuntu/v14/4iCs6KVjbNBYlgoKfA72j00.woff2) format('woff2'); - unicode-range: U+0370-03FF; -} -/* latin-ext */ -@font-face { - font-family: 'Ubuntu'; - font-style: normal; - font-weight: 400; - src: local('Ubuntu Regular'), local('Ubuntu-Regular'), url(https://fonts.gstatic.com/s/ubuntu/v14/4iCs6KVjbNBYlgoKcQ72j00.woff2) format('woff2'); -} -/* latin */ -@font-face { - font-family: 'Ubuntu'; - font-style: normal; - font-weight: 400; - src: local('Ubuntu Regular'), local('Ubuntu-Regular'), url(https://fonts.gstatic.com/s/ubuntu/v14/4iCs6KVjbNBYlgoKfw72.woff2) format('woff2'); + src: local('Ubuntu Regular'), local('Ubuntu-Regular'), url('/static/ttf/Ubuntu-Regular.ttf') format('woff2'); } diff --git a/maloja/web/static/css/maloja.css b/maloja/web/static/css/maloja.css index 75d287d..3056671 100644 --- a/maloja/web/static/css/maloja.css +++ b/maloja/web/static/css/maloja.css @@ -55,14 +55,22 @@ div.header h1 { settings icon **/ +div.clickable_icon { + display: inline-block; + fill: var(--text-color); + cursor: pointer; +} +div.clickable_icon:hover { + fill: var(--text-color-focus); +} +div.clickable_icon.danger:hover { + fill: red; +} + div#settingsicon { position:fixed; right:30px; top:30px; - fill: var(--text-color); -} -div#settingsicon:hover { - fill: var(--text-color-focus); } @@ -438,11 +446,21 @@ button:hover { background-color: var(--button-color-bg-focus); color: var(--button-color-fg-focus); } +button.warning:hover { + background-color:darkred; + color:white; +} button.locked { background-color:grey; color:black; cursor:not-allowed; } +button.smallbutton { + padding:1px; + padding-right:2px; + padding-left:2px; + font-size: 80%; +} @@ -471,6 +489,7 @@ table.list { table.list tr { background-color: var(--current-bg-color); border-color: var(--current-bg-color); + height: 1.4em; } @@ -492,10 +511,14 @@ table.list tr:nth-child(even) { /* make rows distinguishable with respect to any bg color */ } -table.list tr:nth-child(5n) td { - border-bottom: 1px solid rgba(120,120,120,0.2); +table.list tr:nth-child(4n) td { + border-bottom: 1px solid rgba(120,120,120,0.4); padding-bottom: 1px; } +table.list tr:nth-child(4n+1) td { + border-top: 1px solid rgba(120,120,120,0.4); + padding-top: 1px; +} table.list tr:hover { --current-bg-color: rgba(255,255,255,0.2); @@ -564,6 +587,37 @@ table.list td.searchProvider:hover { color: gold; } +table.list td.delete_area { + text-align: right; + width:7em; +} + +/* rows that can be deleted in some form + 'active' class on the delete area cell to toggle confirm prompt + 'removed' class on the whole row to delete +*/ +table.list tr td.delete_area span.confirmactions { + display: none; +} +table.list tr td.delete_area span.initializeactions { + display: initial; +} +table.list tr td.delete_area.active span.confirmactions { + display: initial; +} +table.list tr td.delete_area.active span.initializeactions { + display: none; +} +table.list tr.removed td.delete_area span.confirmactions { + display: none; +} +table.list tr.removed td.delete_area span.initializeactions { + display: none; +} +table.list tr.removed { + text-decoration: line-through; + opacity:0.4; +} /* diff --git a/maloja/web/static/css/themes/constantinople.css b/maloja/web/static/css/themes/constantinople.css new file mode 100644 index 0000000..617aa45 --- /dev/null +++ b/maloja/web/static/css/themes/constantinople.css @@ -0,0 +1,23 @@ +:root { + --base-color: #140c12; + --base-color-dark: #080507; + --base-color-light: #2c1b28; + --base-color-accent: #452a3e; + --base-color-accent-dark: #442b3e; + --base-color-accent-light: #dfcad9; + + --text-color: #d8b700; + --text-color-selected: fadeout(var(--text-color),40%); + --text-color-secondary: #6a7600; + --text-color-tertiary: #474f00; + --text-color-focus: #3D428B; + + --ctrl-element-color-bg: rgba(0,255,255,0.1); + --ctrl-element-color-main: rgba(103,85,0,0.7); + --ctrl-element-color-focus: gold; + + --button-color-bg: var(--text-color); + --button-color-bg-focus: var(--text-color-focus); + --button-color-fg: var(--base-color); + --button-color-fg-focus: var(--base-color); +} diff --git a/maloja/web/static/css/themes/kda.css b/maloja/web/static/css/themes/kda.css new file mode 100644 index 0000000..f778bd7 --- /dev/null +++ b/maloja/web/static/css/themes/kda.css @@ -0,0 +1,23 @@ +:root { + --base-color: #2f3493; + --base-color-dark: #000000; + --base-color-light: #59ebfc; + --base-color-accent: #9821a5; + --base-color-accent-dark: #740C7F; + --base-color-accent-light: #D41BE8; + + --text-color: #e5b48f; + --text-color-selected: fadeout(var(--text-color),40%); + --text-color-secondary: #f9f9f9; + --text-color-tertiary: #f7f7f7; + --text-color-focus: #59ebfc; + + --ctrl-element-color-bg: rgba(0,255,255,0.1); + --ctrl-element-color-main: rgba(103,85,0,0.7); + --ctrl-element-color-focus: gold; + + --button-color-bg: var(--text-color); + --button-color-bg-focus: var(--text-color-focus); + --button-color-fg: var(--base-color); + --button-color-fg-focus: var(--base-color); +} diff --git a/maloja/web/static/css/themes/maloja.css b/maloja/web/static/css/themes/maloja.css new file mode 100644 index 0000000..e69de29 diff --git a/maloja/web/static/js/edit.js b/maloja/web/static/js/edit.js new file mode 100644 index 0000000..0a415ba --- /dev/null +++ b/maloja/web/static/js/edit.js @@ -0,0 +1,12 @@ +// JS for all web interface editing / deletion of scrobble data + +function toggleDeleteConfirm(element) { + element.parentElement.parentElement.classList.toggle('active'); +} + +function deleteScrobble(id,element) { + element.parentElement.parentElement.parentElement.classList.add('removed'); + + neo.xhttpreq("/apis/mlj_1/delete_scrobble",data={'timestamp':id},method="POST",callback=(()=>null),json=true); + +} diff --git a/maloja/web/static/js/manualscrobble.js b/maloja/web/static/js/manualscrobble.js index 4816632..1ae55de 100644 --- a/maloja/web/static/js/manualscrobble.js +++ b/maloja/web/static/js/manualscrobble.js @@ -130,11 +130,12 @@ function searchresult_manualscrobbling() { track.innerHTML = "" + trackstr + ""; row = document.createElement("tr") col1 = document.createElement("td") - col1.className = "button" - col1.innerHTML = "Scrobble!" - col1.onclick = function(){ scrobble(t["artists"],t["title"])}; + button = document.createElement("button") + button.innerHTML = "Scrobble!" + button.onclick = function(){ scrobble(t["artists"],t["title"])}; col2 = document.createElement("td") row.appendChild(col1) + col1.appendChild(button) row.appendChild(col2) col2.appendChild(track) document.getElementById("searchresults").appendChild(row); diff --git a/maloja/web/static/ttf/Ubuntu-Bold.ttf b/maloja/web/static/ttf/Ubuntu-Bold.ttf new file mode 100644 index 0000000..c2293d5 Binary files /dev/null and b/maloja/web/static/ttf/Ubuntu-Bold.ttf differ diff --git a/maloja/web/static/ttf/Ubuntu-BoldItalic.ttf b/maloja/web/static/ttf/Ubuntu-BoldItalic.ttf new file mode 100644 index 0000000..ce6e784 Binary files /dev/null and b/maloja/web/static/ttf/Ubuntu-BoldItalic.ttf differ diff --git a/maloja/web/static/ttf/Ubuntu-Italic.ttf b/maloja/web/static/ttf/Ubuntu-Italic.ttf new file mode 100644 index 0000000..a599244 Binary files /dev/null and b/maloja/web/static/ttf/Ubuntu-Italic.ttf differ diff --git a/maloja/web/static/ttf/Ubuntu-Light.ttf b/maloja/web/static/ttf/Ubuntu-Light.ttf new file mode 100644 index 0000000..b310d15 Binary files /dev/null and b/maloja/web/static/ttf/Ubuntu-Light.ttf differ diff --git a/maloja/web/static/ttf/Ubuntu-LightItalic.ttf b/maloja/web/static/ttf/Ubuntu-LightItalic.ttf new file mode 100644 index 0000000..ad0741b Binary files /dev/null and b/maloja/web/static/ttf/Ubuntu-LightItalic.ttf differ diff --git a/maloja/web/static/ttf/Ubuntu-Medium.ttf b/maloja/web/static/ttf/Ubuntu-Medium.ttf new file mode 100644 index 0000000..7340a40 Binary files /dev/null and b/maloja/web/static/ttf/Ubuntu-Medium.ttf differ diff --git a/maloja/web/static/ttf/Ubuntu-MediumItalic.ttf b/maloja/web/static/ttf/Ubuntu-MediumItalic.ttf new file mode 100644 index 0000000..36ac1ae Binary files /dev/null and b/maloja/web/static/ttf/Ubuntu-MediumItalic.ttf differ diff --git a/maloja/web/static/ttf/Ubuntu-Regular.ttf b/maloja/web/static/ttf/Ubuntu-Regular.ttf new file mode 100644 index 0000000..f98a2da Binary files /dev/null and b/maloja/web/static/ttf/Ubuntu-Regular.ttf differ diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..a2596b8 --- /dev/null +++ b/pylintrc @@ -0,0 +1,41 @@ +# no linting is currently enforced, this is just for reference +[MASTER] +disable=C0114,C0115,C0116, # docstrings + W0703, # broad exception catching + W1514, # open without encoding + +[VARIABLES] +allow-global-unused-variables=no + +[BASIC] +argument-naming-style=snake_case +attr-naming-style=snake_case +class-naming-style=PascalCase +const-naming-style=UPPER_CASE +function-naming-style=snake_case +variable-naming-style=snake_case +indent-string='\t' +good-names=i,j,k, # loops + x,y, # dimensions + e # exceptions + +max-line-length=200 +max-module-lines=1000 + +[DESIGN] + +max-args=8 +max-attributes=7 +max-bool-expr=5 +max-branches=12 +max-locals=15 +max-parents=7 +max-public-methods=20 +max-returns=6 +max-statements=50 +min-public-methods=2 + +[EXCEPTIONS] + +overgeneral-exceptions=BaseException, + Exception diff --git a/pyproject.toml b/pyproject.toml index d3f69a2..a7ad1a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "malojaserver" -version = "2.14.10" +version = "3.0.0-beta.6" description = "Self-hosted music scrobble database" readme = "./README.md" requires-python = ">=3.6" @@ -21,14 +21,22 @@ classifiers = [ dependencies = [ "bottle>=0.12.16", "waitress>=1.3", - "doreah>=1.7.2, <1.8", - "nimrodel>=0.7.0", + "doreah>=1.9.0, <2", + "nimrodel>=0.8.0", "setproctitle>=1.1.10", #"pyvips>=2.1.16", "jinja2>=2.11", "lru-dict>=1.1.6", "css_html_js_minify>=2.5.5", - "psutil>=5.8.0" + "psutil>=5.8.0", + "sqlalchemy>=1.4", + "python-datauri>=1.1.0", + "requests>=2.27.1" +] + +[project.optional-dependencies] +full = [ + "pyvips>=2.1" ] [project.scripts] @@ -40,3 +48,32 @@ build-backend = "flit_core.buildapi" [tool.flit.module] name = "maloja" + +[tool.osreqs.alpine] +build =[ + "gcc", + "g++", + "python3-dev", + "libxml2-dev", + "libxslt-dev", + "libffi-dev", + "libc-dev", + "py3-pip", + "linux-headers" +] +run = [ + "python3", + "tzdata" +] +opt = [ + "vips" +] + +[tool.osreqs.debian] +build = [ + "python3-pip" +] +run = [ + "python3" +] +opt = [] diff --git a/requirements.txt b/requirements.txt index ffc101c..60aceb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,13 @@ bottle>=0.12.16 waitress>=1.3 -doreah>=1.7.2 -nimrodel>=0.7.0 +doreah>=1.9.0, <2 +nimrodel>=0.8.0 setproctitle>=1.1.10 jinja2>=2.11 lru-dict>=1.1.6 css_html_js_minify>=2.5.5 psutil>=5.8.0 +sqlalchemy>=1.4 +python-datauri>=1.1.0 +requests>=2.27.1 + diff --git a/requirements_extra.txt b/requirements_extra.txt index 82d08f0..d7b7dc7 100644 --- a/requirements_extra.txt +++ b/requirements_extra.txt @@ -1 +1,2 @@ pyvips>=2.1 + diff --git a/requirements_pre.txt b/requirements_pre.txt new file mode 100644 index 0000000..c4c0dbe --- /dev/null +++ b/requirements_pre.txt @@ -0,0 +1,3 @@ +# this is a more static file that enables container images to be cached +# it should contain packages that take long to build and don't change frequently +lxml diff --git a/screenshot.png b/screenshot.png new file mode 100644 index 0000000..5f31e70 Binary files /dev/null and b/screenshot.png differ