mirror of
https://github.com/krateng/maloja.git
synced 2023-08-10 21:12:55 +03:00
Added cache dump for high memory usage
This commit is contained in:
parent
c166620d5f
commit
98c1527f77
|
@ -5,7 +5,7 @@ author = {
|
||||||
"email":"maloja@krateng.dev",
|
"email":"maloja@krateng.dev",
|
||||||
"github": "krateng"
|
"github": "krateng"
|
||||||
}
|
}
|
||||||
version = 2,4,7
|
version = 2,4,8
|
||||||
versionstr = ".".join(str(n) for n in version)
|
versionstr = ".".join(str(n) for n in version)
|
||||||
links = {
|
links = {
|
||||||
"pypi":"malojaserver",
|
"pypi":"malojaserver",
|
||||||
|
@ -21,7 +21,8 @@ requires = [
|
||||||
"wand>=0.5.4",
|
"wand>=0.5.4",
|
||||||
"lesscpy>=0.13",
|
"lesscpy>=0.13",
|
||||||
"jinja2>2.11",
|
"jinja2>2.11",
|
||||||
"lru-dict>=1.1.6"
|
"lru-dict>=1.1.6",
|
||||||
|
"psutil>0.5.0"
|
||||||
]
|
]
|
||||||
resources = [
|
resources = [
|
||||||
"web/*/*/*",
|
"web/*/*/*",
|
||||||
|
|
|
@ -29,6 +29,7 @@ USE_DB_CACHE = yes
|
||||||
CACHE_DATABASE_SHORT = true
|
CACHE_DATABASE_SHORT = true
|
||||||
CACHE_DATABASE_PERM = true #more permanent cache for old timeranges
|
CACHE_DATABASE_PERM = true #more permanent cache for old timeranges
|
||||||
DB_CACHE_ENTRIES = 10000 #experiment with this depending on your RAM
|
DB_CACHE_ENTRIES = 10000 #experiment with this depending on your RAM
|
||||||
|
DB_MAX_MEMORY = 75 # percentage of RAM utilization (whole container, not just maloja) that should trigger a flush
|
||||||
INVALID_ARTISTS = ["[Unknown Artist]","Unknown Artist","Spotify"]
|
INVALID_ARTISTS = ["[Unknown Artist]","Unknown Artist","Spotify"]
|
||||||
REMOVE_FROM_TITLE = ["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)","(Bonus Track)"]
|
REMOVE_FROM_TITLE = ["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)","(Bonus Track)"]
|
||||||
USE_PARSE_PLUGINS = no
|
USE_PARSE_PLUGINS = no
|
||||||
|
|
|
@ -34,6 +34,7 @@ from collections import namedtuple
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
import yaml
|
import yaml
|
||||||
import lru
|
import lru
|
||||||
|
import psutil
|
||||||
|
|
||||||
# url handling
|
# url handling
|
||||||
from importlib.machinery import SourceFileLoader
|
from importlib.machinery import SourceFileLoader
|
||||||
|
@ -1055,6 +1056,7 @@ else:
|
||||||
|
|
||||||
|
|
||||||
csz = settings.get_settings("DB_CACHE_ENTRIES")
|
csz = settings.get_settings("DB_CACHE_ENTRIES")
|
||||||
|
cmp = settings.get_settings("DB_MAX_MEMORY")
|
||||||
|
|
||||||
cache_query = lru.LRU(csz)
|
cache_query = lru.LRU(csz)
|
||||||
cache_query_perm = lru.LRU(csz)
|
cache_query_perm = lru.LRU(csz)
|
||||||
|
@ -1087,7 +1089,7 @@ def log_stats():
|
||||||
logstr = "{name}: {hitsperm} Perm Hits, {hitstmp} Tmp Hits, {misses} Misses; Current Size: {sizeperm}/{sizetmp}"
|
logstr = "{name}: {hitsperm} Perm Hits, {hitstmp} Tmp Hits, {misses} Misses; Current Size: {sizeperm}/{sizetmp}"
|
||||||
for s in (cachestats["cache_query"],cachestats["cache_aggregate"]):
|
for s in (cachestats["cache_query"],cachestats["cache_aggregate"]):
|
||||||
log(logstr.format(name=s["name"],hitsperm=s["hits_perm"],hitstmp=s["hits_tmp"],misses=s["misses"],
|
log(logstr.format(name=s["name"],hitsperm=s["hits_perm"],hitstmp=s["hits_tmp"],misses=s["misses"],
|
||||||
sizeperm=len(s["objperm"]),sizetmp=len(s["objtmp"])))
|
sizeperm=len(s["objperm"]),sizetmp=len(s["objtmp"])),module="debug")
|
||||||
|
|
||||||
def db_query_cached(**kwargs):
|
def db_query_cached(**kwargs):
|
||||||
global cache_query, cache_query_perm
|
global cache_query, cache_query_perm
|
||||||
|
@ -1118,6 +1120,12 @@ def db_query_cached(**kwargs):
|
||||||
result = db_query_full(**kwargs)
|
result = db_query_full(**kwargs)
|
||||||
if eligible_permanent_caching: cache_query_perm[key] = result
|
if eligible_permanent_caching: cache_query_perm[key] = result
|
||||||
elif eligible_temporary_caching: cache_query[key] = result
|
elif eligible_temporary_caching: cache_query[key] = result
|
||||||
|
|
||||||
|
ramprct = psutil.virtual_memory().percent
|
||||||
|
if ramprct > cmp:
|
||||||
|
log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug")
|
||||||
|
invalidate_caches()
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@ -1151,6 +1159,11 @@ def db_aggregate_cached(**kwargs):
|
||||||
if eligible_permanent_caching: cache_aggregate_perm[key] = result
|
if eligible_permanent_caching: cache_aggregate_perm[key] = result
|
||||||
elif eligible_temporary_caching: cache_aggregate[key] = result
|
elif eligible_temporary_caching: cache_aggregate[key] = result
|
||||||
|
|
||||||
|
ramprct = psutil.virtual_memory().percent
|
||||||
|
if ramprct > cmp:
|
||||||
|
log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug")
|
||||||
|
invalidate_caches()
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def invalidate_caches():
|
def invalidate_caches():
|
||||||
|
|
Loading…
Reference in New Issue
Block a user