mirror
/
maloja
mirror of https://github.com/krateng/maloja.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Reimplemented cache limitation

This commit is contained in:
krateng 2022-02-15 05:52:44 +01:00
parent cc060d650b
commit a1f8e96ae4
3 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
maloja/__pkginfo__.py
View File

@ -4,7 +4,7 @@
# you know what f*ck it # you know what f*ck it
# this is hardcoded for now because of that damn project / package name discrepancy # this is hardcoded for now because of that damn project / package name discrepancy
# i'll fix it one day # i'll fix it one day
VERSION = "3.0.0-alpha.1" VERSION = "3.0.0-alpha.2"
HOMEPAGE = "https://github.com/krateng/maloja" HOMEPAGE = "https://github.com/krateng/maloja"

25
maloja/database/dbcache.py
View File

@ -3,26 +3,37 @@
# that changes very infrequently or not at all # that changes very infrequently or not at all
import lru import lru
import psutil
import json import json
from doreah.regular import runhourly from doreah.regular import runhourly
from doreah.logging import log from doreah.logging import log
from ..globalconf import malojaconfig
USE_CACHE = True USE_CACHE = True
cache = lru.LRU(300000) HIGH_NUMBER = 1000000
cache = lru.LRU(HIGH_NUMBER)
hits, misses = 0, 0
@runhourly @runhourly
def print_stats(): def print_stats():
log(f"Cache Size: {len(cache)}") log(f"Cache Size: {len(cache)}, RAM Utilization: {psutil.virtual_memory().percent}%, Cache Hits: {hits}/{hits+misses}")
trim_cache()
def cached_wrapper(inner_func): def cached_wrapper(inner_func):
def outer_func(**kwargs): def outer_func(**kwargs):
global hits, misses
key = (serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to")) key = (serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to"))
if USE_CACHE and key in cache: if USE_CACHE and key in cache:
hits += 1
return cache.get(key) return cache.get(key)
else: else:
misses += 1
result = inner_func(**kwargs) result = inner_func(**kwargs)
cache[key] = result cache[key] = result
return result return result
@ -38,8 +49,14 @@ def invalidate_caches(scrobbletime):
def trim_cache():
ramprct = psutil.virtual_memory().percent
if ramprct > malojaconfig["DB_MAX_MEMORY"]:
log(f"{ramprct}% RAM usage, reducing caches!",module="debug")
ratio = (malojaconfig["DB_MAX_MEMORY"] / ramprct) ** 3
targetsize = max(int(len(cache) * ratio),100)
c.set_size(targetsize)
c.set_size(HIGH_NUMBER)
def serialize(obj): def serialize(obj):

3
maloja/globalconf.py
View File

@ -148,7 +148,8 @@ malojaconfig = Configuration(
}, },
"Technical":{ "Technical":{
"cache_expire_positive":(tp.Integer(), "Image Cache Expiration", 300, "Days until images are refetched"), "cache_expire_positive":(tp.Integer(), "Image Cache Expiration", 300, "Days until images are refetched"),
"cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 30, "Days until failed image fetches are reattempted") "cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 30, "Days until failed image fetches are reattempted"),
"db_max_memory":(tp.Integer(min=0,max=100), "Database Cache RAM Percentage soft limit", 80, "How much of your total memory Maloja should use for caching. If you don't run the application in a container or dedicated VM, you might want to set this lower.")
}, },
"Fluff":{ "Fluff":{
"scrobbles_gold":(tp.Integer(), "Scrobbles for Gold", 250, "How many scrobbles a track needs to be considered 'Gold' status"), "scrobbles_gold":(tp.Integer(), "Scrobbles for Gold", 250, "How many scrobbles a track needs to be considered 'Gold' status"),