mirror/maloja
1
0
Fork 0
mirror of https://github.com/krateng/maloja.git synced 2023-08-10 21:12:55 +03:00
Code Issues Projects Releases Wiki Activity

Added cache dump for high memory usage

Browse Source
This commit is contained in:
Krateng 2020-05-29 17:33:42 +02:00
parent c166620d5f
commit 98c1527f77
3 changed files with 18 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
maloja/__pkginfo__.py
View File

@ -5,7 +5,7 @@ author = {
"email":"maloja@krateng.dev",
"github": "krateng"
}
version = 2,4,7
version = 2,4,8
versionstr = ".".join(str(n) for n in version)
links = {
"pypi":"malojaserver",
@ -21,7 +21,8 @@ requires = [
"wand>=0.5.4",
"lesscpy>=0.13",
"jinja2>2.11",
"lru-dict>=1.1.6"
"lru-dict>=1.1.6",
"psutil>0.5.0"
]
resources = [
"web/*/*/*",

1
maloja/data_files/settings/default.ini
View File

@ -29,6 +29,7 @@ USE_DB_CACHE = yes
CACHE_DATABASE_SHORT = true
CACHE_DATABASE_PERM = true #more permanent cache for old timeranges
DB_CACHE_ENTRIES = 10000 #experiment with this depending on your RAM
DB_MAX_MEMORY = 75 # percentage of RAM utilization (whole container, not just maloja) that should trigger a flush
INVALID_ARTISTS = ["[Unknown Artist]","Unknown Artist","Spotify"]
REMOVE_FROM_TITLE = ["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)","(Bonus Track)"]
USE_PARSE_PLUGINS = no

15
maloja/database.py
View File

@ -34,6 +34,7 @@ from collections import namedtuple
from threading import Lock
import yaml
import lru
import psutil
# url handling
from importlib.machinery import SourceFileLoader
@ -1055,6 +1056,7 @@ else:
csz = settings.get_settings("DB_CACHE_ENTRIES")
cmp = settings.get_settings("DB_MAX_MEMORY")
cache_query = lru.LRU(csz)
cache_query_perm = lru.LRU(csz)
@ -1087,7 +1089,7 @@ def log_stats():
logstr = "{name}: {hitsperm} Perm Hits, {hitstmp} Tmp Hits, {misses} Misses; Current Size: {sizeperm}/{sizetmp}"
for s in (cachestats["cache_query"],cachestats["cache_aggregate"]):
log(logstr.format(name=s["name"],hitsperm=s["hits_perm"],hitstmp=s["hits_tmp"],misses=s["misses"],
sizeperm=len(s["objperm"]),sizetmp=len(s["objtmp"])))
sizeperm=len(s["objperm"]),sizetmp=len(s["objtmp"])),module="debug")
def db_query_cached(**kwargs):
global cache_query, cache_query_perm
@ -1118,6 +1120,12 @@ def db_query_cached(**kwargs):
result = db_query_full(**kwargs)
if eligible_permanent_caching: cache_query_perm[key] = result
elif eligible_temporary_caching: cache_query[key] = result
ramprct = psutil.virtual_memory().percent
if ramprct > cmp:
log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug")
invalidate_caches()
return result
@ -1151,6 +1159,11 @@ def db_aggregate_cached(**kwargs):
if eligible_permanent_caching: cache_aggregate_perm[key] = result
elif eligible_temporary_caching: cache_aggregate[key] = result
ramprct = psutil.virtual_memory().percent
if ramprct > cmp:
log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug")
invalidate_caches()
return result
def invalidate_caches():