mirror/maloja
1
0
Fork 0
mirror of https://github.com/krateng/maloja.git synced 2023-08-10 21:12:55 +03:00
Code Issues Projects Releases Wiki Activity

Ported cache cleanup from 3.1

Browse Source
This commit is contained in:
krateng 2022-04-24 20:55:07 +02:00
parent bfc83fdbb0
commit 342b8867d9
2 changed files with 106 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
maloja/database/dbcache.py
View File

@ -10,10 +10,13 @@ from doreah.logging import log
from ..globalconf import malojaconfig from ..globalconf import malojaconfig
HIGH_NUMBER = 1000000
CACHE_SIZE = 10000
ENTITY_CACHE_SIZE = 1000000
CACHE_ADJUST_STEP = 100
if malojaconfig['USE_GLOBAL_CACHE']:
CACHE_SIZE = 1000
ENTITY_CACHE_SIZE = 100000
cache = lru.LRU(CACHE_SIZE) cache = lru.LRU(CACHE_SIZE)
entitycache = lru.LRU(ENTITY_CACHE_SIZE) entitycache = lru.LRU(ENTITY_CACHE_SIZE)
@ -24,7 +27,6 @@ hits, misses = 0, 0
@runhourly @runhourly
def maintenance(): def maintenance():
if malojaconfig['USE_GLOBAL_CACHE']:
print_stats() print_stats()
trim_cache() trim_cache()
@ -38,8 +40,8 @@ def print_stats():
def cached_wrapper(inner_func): def cached_wrapper(inner_func):
if not malojaconfig['USE_GLOBAL_CACHE']: return inner_func
def outer_func(*args,**kwargs): def outer_func(*args,**kwargs):
if 'dbconn' in kwargs: if 'dbconn' in kwargs:
conn = kwargs.pop('dbconn') conn = kwargs.pop('dbconn')
else: else:
@ -65,7 +67,7 @@ def cached_wrapper(inner_func):
# cache that's aware of what we're calling # cache that's aware of what we're calling
def cached_wrapper_individual(inner_func): def cached_wrapper_individual(inner_func):
if not malojaconfig['USE_GLOBAL_CACHE']: return inner_func
def outer_func(set_arg,**kwargs): def outer_func(set_arg,**kwargs):
@ -94,12 +96,11 @@ def cached_wrapper_individual(inner_func):
return outer_func return outer_func
def invalidate_caches(scrobbletime): def invalidate_caches(scrobbletime=None):
if malojaconfig['USE_GLOBAL_CACHE']:
cleared, kept = 0, 0 cleared, kept = 0, 0
for k in cache.keys(): for k in cache.keys():
# VERY BIG TODO: differentiate between None as in 'unlimited timerange' and None as in 'time doesnt matter here'! # VERY BIG TODO: differentiate between None as in 'unlimited timerange' and None as in 'time doesnt matter here'!
if (k[3] is None or scrobbletime >= k[3]) and (k[4] is None or scrobbletime <= k[4]): if scrobbletime is None or (k[3] is None or scrobbletime >= k[3]) and (k[4] is None or scrobbletime <= k[4]):
cleared += 1 cleared += 1
del cache[k] del cache[k]
else: else:
@ -121,14 +122,28 @@ def trim_cache():
#cache.set_size(targetsize) #cache.set_size(targetsize)
#cache.set_size(HIGH_NUMBER) #cache.set_size(HIGH_NUMBER)
cache.clear() cache.clear()
if cache.get_size() > CACHE_ADJUST_STEP: #if cache.get_size() > CACHE_ADJUST_STEP:
cache.set_size(cache.get_size() - CACHE_ADJUST_STEP) # cache.set_size(cache.get_size() - CACHE_ADJUST_STEP)
#log(f"New RAM usage: {psutil.virtual_memory().percent}%") #log(f"New RAM usage: {psutil.virtual_memory().percent}%")
print_stats() print_stats()
else:
def cached_wrapper(func):
return func
def cached_wrapper_individual(func):
return func
def invalidate_caches(scrobbletime=None):
return None
def invalidate_entity_cache():
return None
def serialize(obj): def serialize(obj):
try: try:
return serialize(obj.hashable()) return serialize(obj.hashable())

4
maloja/globalconf.py
View File

@ -149,8 +149,8 @@ malojaconfig = Configuration(
"cache_expire_positive":(tp.Integer(), "Image Cache Expiration", 60, "Days until images are refetched"), "cache_expire_positive":(tp.Integer(), "Image Cache Expiration", 60, "Days until images are refetched"),
"cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 5, "Days until failed image fetches are reattempted"), "cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 5, "Days until failed image fetches are reattempted"),
"db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 80, "RAM Usage in percent at which Maloja should no longer increase its database cache."), "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 80, "RAM Usage in percent at which Maloja should no longer increase its database cache."),
"use_request_cache":(tp.Boolean(), "Use request-local DB Cache", True), "use_request_cache":(tp.Boolean(), "Use request-local DB Cache", False),
"use_global_cache":(tp.Boolean(), "Use global DB Cache", True) "use_global_cache":(tp.Boolean(), "Use global DB Cache", False)
}, },
"Fluff":{ "Fluff":{
"scrobbles_gold":(tp.Integer(), "Scrobbles for Gold", 250, "How many scrobbles a track needs to be considered 'Gold' status"), "scrobbles_gold":(tp.Integer(), "Scrobbles for Gold", 250, "How many scrobbles a track needs to be considered 'Gold' status"),