From ef2a2c817e3fba22128bb171abd92a5fc76030dc Mon Sep 17 00:00:00 2001 From: krateng Date: Mon, 1 Jun 2020 18:22:16 +0200 Subject: [PATCH 01/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dcba5bb..ba6578c 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ I can support you with issues best if you use **Alpine Linux**. In my experience 5) (Recommended) Until I have a proper service implemented, I would recommend setting two cronjobs for maloja: ``` -@reboot maloja start +@reboot sleep 15 && maloja start 42 0 * * * maloja restart ``` From 08fe4695f6d5ef09789688481db478d0decbd5df Mon Sep 17 00:00:00 2001 From: Krateng Date: Fri, 5 Jun 2020 13:20:54 +0200 Subject: [PATCH 02/10] High RAM usage affects all caches --- maloja/__pkginfo__.py | 2 +- maloja/database.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 561a9ae..8fb703b 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,9 +version = 2,4,10 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", diff --git a/maloja/database.py b/maloja/database.py index aefe140..4f62ab5 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -1173,8 +1173,8 @@ def invalidate_caches(): log("Database caches invalidated.") def reduce_caches(to=0.75): - global cache_query, cache_aggregate - for c in cache_query, cache_aggregate: + global cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm + for c in cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm: currentsize = len(c) targetsize = int(currentsize * to) c.set_size(targetsize) @@ -1183,7 +1183,7 @@ def reduce_caches(to=0.75): def reduce_caches_if_low_ram(): ramprct = psutil.virtual_memory().percent if ramprct > cmp: - log("{prct}% RAM usage, reducing temporary caches!".format(prct=ramprct),module="debug") + log("{prct}% RAM usage, reducing caches!".format(prct=ramprct),module="debug") ratio = (cmp / ramprct) ** 3 reduce_caches(to=ratio) From e531dc4007657ecdf2b3f11a88054a5b18fa10a0 Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 6 Jun 2020 11:53:42 +0200 Subject: [PATCH 03/10] Remove pip3 and dependencies after installing --- Dockerfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 46f6e41..44c6960 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,18 @@ -FROM python:3.6-alpine +FROM python:3-alpine WORKDIR /usr/src/app -RUN apk update -RUN apk add gcc libxml2-dev libxslt-dev py3-pip libc-dev linux-headers -RUN pip3 install psutil - -RUN pip3 install malojaserver +RUN apk add --no-cache --virtual .build-deps \ + gcc \ + libxml2-dev \ + libxslt-dev \ + py3-pip \ + libc-dev \ + linux-headers \ + && \ + pip3 install psutil && \ + pip3 install malojaserver && \ + apk del .build-deps EXPOSE 42010 From e73e047af9160a089216b71d37cb40cd64eb03a4 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 6 Jun 2020 16:38:45 +0200 Subject: [PATCH 04/10] Reduced disk access for cache settings --- maloja/database.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/maloja/database.py b/maloja/database.py index 4f62ab5..d275db6 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -1067,6 +1067,9 @@ cache_query_perm = lru.LRU(csz) cache_aggregate = lru.LRU(csz) cache_aggregate_perm = lru.LRU(csz) +perm_caching = settings.get_settings("CACHE_DATABASE_PERM") +temp_caching = settings.get_settings("CACHE_DATABASE_SHORT") + cachestats = { "cache_query":{ "hits_perm":0, @@ -1102,11 +1105,11 @@ def db_query_cached(**kwargs): eligible_permanent_caching = ( "timerange" in kwargs and not kwargs["timerange"].active() and - settings.get_settings("CACHE_DATABASE_PERM") + perm_caching ) eligible_temporary_caching = ( not eligible_permanent_caching and - settings.get_settings("CACHE_DATABASE_SHORT") + temp_caching ) # hit permanent cache for past timeranges @@ -1138,11 +1141,11 @@ def db_aggregate_cached(**kwargs): eligible_permanent_caching = ( "timerange" in kwargs and not kwargs["timerange"].active() and - settings.get_settings("CACHE_DATABASE_PERM") + perm_caching ) eligible_temporary_caching = ( not eligible_permanent_caching and - settings.get_settings("CACHE_DATABASE_SHORT") + temp_caching ) # hit permanent cache for past timeranges From 1828bd35bb8c942169cfd51d8f12b4d156eae7ec Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 6 Jun 2020 16:46:25 +0200 Subject: [PATCH 05/10] Can now use custom data directory with environment variable, close GH-18 --- maloja/__pkginfo__.py | 2 +- maloja/globalconf.py | 30 +++++++++++++++++++++--------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 8fb703b..807f5a6 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,10 +version = 2,4,11 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", diff --git a/maloja/globalconf.py b/maloja/globalconf.py index 4118189..3717d35 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -1,23 +1,34 @@ import os +from doreah.settings import get_settings +from doreah.settings import config as settingsconfig -# data folder -# must be determined first because getting settings relies on it -try: - DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] - assert os.path.exists(DATA_DIR) -except: - DATA_DIR = os.path.join(os.environ["HOME"],".local/share/") +# check environment variables for data directory +# otherwise, go with defaults +setting_datadir = get_settings("DATA_DIRECTORY",files=[],environ_prefix="MALOJA_") +if setting_datadir is not None and os.path.exists(setting_datadir): + DATA_DIR = setting_datadir +else: + try: + HOME_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] + assert os.path.exists(HOME_DIR) + except: + HOME_DIR = os.path.join(os.environ["HOME"],".local/share/") + + DATA_DIR = os.path.join(HOME_DIR,"maloja") -DATA_DIR = os.path.join(DATA_DIR,"maloja") os.makedirs(DATA_DIR,exist_ok=True) + + def datadir(*args): return os.path.join(DATA_DIR,*args) + + ### DOREAH CONFIGURATION from doreah import config @@ -44,9 +55,10 @@ config( } ) +# because we loaded a doreah module already before setting the config, we need to to that manually +settingsconfig._readpreconfig() -from doreah.settings import get_settings # thumbor From 5c6a901f5118be54ae44affbd6881b14bc30e04a Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 6 Jun 2020 16:47:59 +0200 Subject: [PATCH 06/10] Lowered recommended restart frequency --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba6578c..185e2df 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ I can support you with issues best if you use **Alpine Linux**. In my experience ``` @reboot sleep 15 && maloja start -42 0 * * * maloja restart +42 0 * * 2 maloja restart ``` From d551513733e8f1d18c8825305be302412ac9b924 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sun, 7 Jun 2020 15:01:05 +0200 Subject: [PATCH 07/10] Update install scripts --- install_alpine.sh | 3 ++- install_ubuntu.sh | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/install_alpine.sh b/install_alpine.sh index 75c9ac3..43705d7 100644 --- a/install_alpine.sh +++ b/install_alpine.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash -apk add python3 python3-dev gcc libxml2-dev libxslt-dev py3-pip libc-dev +apk add python3 python3-dev gcc libxml2-dev libxslt-dev py3-pip libc-dev linux-headers +pip3 install psutil pip3 install malojaserver diff --git a/install_ubuntu.sh b/install_ubuntu.sh index ff0bd66..d30bd7b 100644 --- a/install_ubuntu.sh +++ b/install_ubuntu.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash apt update apt install python3 python3-pip +pip3 install psutil pip3 install malojaserver From 6658165baedeee3939084ba4500de3de06bbc045 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 13 Jun 2020 17:34:30 +0200 Subject: [PATCH 08/10] Added setting for file logging (GH-19) --- maloja/__pkginfo__.py | 4 ++-- maloja/data_files/settings/default.ini | 1 + maloja/globalconf.py | 11 ++++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 807f5a6..299c8fc 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,11 +version = 2,4,12 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", @@ -15,7 +15,7 @@ links = { requires = [ "bottle>=0.12.16", "waitress>=1.3", - "doreah>=1.5.6", + "doreah>=1.6.3", "nimrodel>=0.6.3", "setproctitle>=1.1.10", "wand>=0.5.4", diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index 38f0bed..37d2ab4 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -70,3 +70,4 @@ FEDERATION = yes #does nothing yet UPDATE_AFTER_CRASH = no #update when server is automatically restarted DAILY_RESTART = 2 # hour of day. no / none means no daily restarts SKIP_SETUP = no +LOGGING = true diff --git a/maloja/globalconf.py b/maloja/globalconf.py index 3717d35..5ed516f 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -37,9 +37,6 @@ config( pyhp={ "version": 2 }, - logging={ - "logfolder": datadir("logs") - }, settings={ "files":[ datadir("settings/default.ini"), @@ -58,6 +55,14 @@ config( # because we loaded a doreah module already before setting the config, we need to to that manually settingsconfig._readpreconfig() +config( + logging={ + "logfolder": datadir("logs") if get_settings("LOGGING") else None + } +) + +settingsconfig._readpreconfig() + # thumbor From 4c6b40e42f695a5cf78548796a77a45ad77a48b0 Mon Sep 17 00:00:00 2001 From: Krateng Date: Sat, 13 Jun 2020 17:42:59 +0200 Subject: [PATCH 09/10] Added some sanity checks to cache reduction --- maloja/database.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/maloja/database.py b/maloja/database.py index d275db6..7fb9aa7 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -1179,9 +1179,10 @@ def reduce_caches(to=0.75): global cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm for c in cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm: currentsize = len(c) - targetsize = int(currentsize * to) - c.set_size(targetsize) - c.set_size(csz) + if currentsize > 100: + targetsize = max(int(currentsize * to),10) + c.set_size(targetsize) + c.set_size(csz) def reduce_caches_if_low_ram(): ramprct = psutil.virtual_memory().percent From 57403a89ab1d679523341d6a607d0b03e495ff35 Mon Sep 17 00:00:00 2001 From: Krateng Date: Thu, 18 Jun 2020 15:16:24 +0200 Subject: [PATCH 10/10] Updated database rebuild, should fix GH-16 --- maloja/__pkginfo__.py | 2 +- maloja/database.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 299c8fc..2fd02e5 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,12 +version = 2,4,13 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", diff --git a/maloja/database.py b/maloja/database.py index 7fb9aa7..84aada3 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -933,6 +933,7 @@ def build_db(): log("Building database...") global SCROBBLES, ARTISTS, TRACKS + global TRACKS_NORMALIZED_SET, TRACKS_NORMALIZED, ARTISTS_NORMALIZED_SET, ARTISTS_NORMALIZED global SCROBBLESDICT, STAMPS SCROBBLES = [] @@ -941,6 +942,11 @@ def build_db(): STAMPS = [] SCROBBLESDICT = {} + TRACKS_NORMALIZED = [] + ARTISTS_NORMALIZED = [] + ARTISTS_NORMALIZED_SET = set() + TRACKS_NORMALIZED_SET = set() + # parse files db = tsv.parse_all(datadir("scrobbles"),"int","string","string",comments=False)