From ad50ee866c33aa8e100b509835240d2cdfffcd22 Mon Sep 17 00:00:00 2001 From: krateng Date: Mon, 25 Apr 2022 04:28:53 +0200 Subject: [PATCH] More cache organization --- dev/releases/3.0.yml | 1 + dev/releases/branch.yml | 3 +- dev/write_tags.py | 1 + maloja/database/dbcache.py | 74 ++++++++++++++++++++++---------------- maloja/database/sqldb.py | 2 ++ maloja/globalconf.py | 2 +- 6 files changed, 51 insertions(+), 32 deletions(-) diff --git a/dev/releases/3.0.yml b/dev/releases/3.0.yml index 9404100..099b2c9 100644 --- a/dev/releases/3.0.yml +++ b/dev/releases/3.0.yml @@ -43,6 +43,7 @@ minor_release_name: "Yeonhee" - "[Bugfix] No longer releasing database lock during scrobble creation" - "[Distribution] Experimental arm64 image" 3.0.7: + commit: "62abc319303a6cb6463f7c27b6ef09b76fc67f86" notes: - "[Bugix] Improved signal handling" - "[Bugix] Fixed constant re-caching of all-time stats, significantly increasing page load speed" diff --git a/dev/releases/branch.yml b/dev/releases/branch.yml index 8b13789..00203d9 100644 --- a/dev/releases/branch.yml +++ b/dev/releases/branch.yml @@ -1 +1,2 @@ - +- "[Performance] Adjusted cache sizes" +- "[Logging] Added cache memory use information" diff --git a/dev/write_tags.py b/dev/write_tags.py index 4a3cb42..7b467a7 100644 --- a/dev/write_tags.py +++ b/dev/write_tags.py @@ -6,6 +6,7 @@ FOLDER = "dev/releases" releases = {} for f in os.listdir(FOLDER): + if f == "branch.yml": continue #maj,min = (int(i) for i in f.split('.')[:2]) with open(os.path.join(FOLDER,f)) as fd: diff --git a/maloja/database/dbcache.py b/maloja/database/dbcache.py index 57117d3..6bc81df 100644 --- a/maloja/database/dbcache.py +++ b/maloja/database/dbcache.py @@ -5,6 +5,7 @@ import lru import psutil import json +import sys from doreah.regular import runhourly from doreah.logging import log @@ -12,16 +13,10 @@ from ..globalconf import malojaconfig - - if malojaconfig['USE_GLOBAL_CACHE']: - CACHE_SIZE = 1000 - ENTITY_CACHE_SIZE = 100000 - cache = lru.LRU(CACHE_SIZE) - entitycache = lru.LRU(ENTITY_CACHE_SIZE) - - hits, misses = 0, 0 + cache = lru.LRU(10000) + entitycache = lru.LRU(100000) @@ -31,11 +26,10 @@ if malojaconfig['USE_GLOBAL_CACHE']: trim_cache() def print_stats(): - log(f"Cache Size: {len(cache)} [{len(entitycache)} E], System RAM Utilization: {psutil.virtual_memory().percent}%, Cache Hits: {hits}/{hits+misses}") - #print("Full rundown:") - #import sys - #for k in cache.keys(): - # print(f"\t{k}\t{sys.getsizeof(cache[k])}") + for name,c in (('Cache',cache),('Entity Cache',entitycache)): + hits, misses = c.get_stats() + log(f"{name}: Size: {len(c)} | Hits: {hits}/{hits+misses} | Estimated Memory: {human_readable_size(c)}") + log(f"System RAM Utilization: {psutil.virtual_memory().percent}%") def cached_wrapper(inner_func): @@ -49,12 +43,9 @@ if malojaconfig['USE_GLOBAL_CACHE']: global hits, misses key = (serialize(args),serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to")) - if key in cache: - hits += 1 - return cache.get(key) - - else: - misses += 1 + try: + return cache[key] + except KeyError: result = inner_func(*args,**kwargs,dbconn=conn) cache[key] = result return result @@ -67,25 +58,18 @@ if malojaconfig['USE_GLOBAL_CACHE']: # cache that's aware of what we're calling def cached_wrapper_individual(inner_func): - def outer_func(set_arg,**kwargs): - - if 'dbconn' in kwargs: conn = kwargs.pop('dbconn') else: conn = None - #global hits, misses result = {} for id in set_arg: - if (inner_func,id) in entitycache: + try: result[id] = entitycache[(inner_func,id)] - #hits += 1 - else: + except KeyError: pass - #misses += 1 - remaining = inner_func(set(e for e in set_arg if e not in result),dbconn=conn) for id in remaining: @@ -115,13 +99,14 @@ if malojaconfig['USE_GLOBAL_CACHE']: def trim_cache(): ramprct = psutil.virtual_memory().percent if ramprct > malojaconfig["DB_MAX_MEMORY"]: - log(f"{ramprct}% RAM usage, clearing cache and adjusting size!") + log(f"{ramprct}% RAM usage, clearing cache!") + for c in (cache,entitycache): + c.clear() #ratio = 0.6 #targetsize = max(int(len(cache) * ratio),50) #log(f"Reducing to {targetsize} entries") #cache.set_size(targetsize) #cache.set_size(HIGH_NUMBER) - cache.clear() #if cache.get_size() > CACHE_ADJUST_STEP: # cache.set_size(cache.get_size() - CACHE_ADJUST_STEP) @@ -156,3 +141,32 @@ def serialize(obj): elif isinstance(obj,dict): return "{" + ",".join(serialize(o) + ":" + serialize(obj[o]) for o in obj) + "}" return json.dumps(obj.hashable()) + + + +def get_size_of(obj,counted=None): + if counted is None: + counted = set() + if id(obj) in counted: return 0 + size = sys.getsizeof(obj) + counted.add(id(obj)) + try: + for k,v in obj.items(): + size += get_size_of(v,counted=counted) + except: + try: + for i in obj: + size += get_size_of(i,counted=counted) + except: + pass + return size + +def human_readable_size(obj): + units = ['','K','M','G','T','P'] + idx = 0 + bytes = get_size_of(obj) + while bytes > 1024 and len(units) > idx+1: + bytes = bytes / 1024 + idx += 1 + + return f"{bytes:.2f} {units[idx]}B" diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py index 4ee49ca..496ab66 100644 --- a/maloja/database/sqldb.py +++ b/maloja/database/sqldb.py @@ -116,6 +116,8 @@ def connection_provider(func): with engine.connect() as connection: kwargs['dbconn'] = connection return func(*args,**kwargs) + + wrapper.__innerfunc__ = func return wrapper ##### DB <-> Dict translations diff --git a/maloja/globalconf.py b/maloja/globalconf.py index f150097..73515d6 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -148,7 +148,7 @@ malojaconfig = Configuration( "Technical":{ "cache_expire_positive":(tp.Integer(), "Image Cache Expiration", 60, "Days until images are refetched"), "cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 5, "Days until failed image fetches are reattempted"), - "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 80, "RAM Usage in percent at which Maloja should no longer increase its database cache."), + "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 50, "RAM Usage in percent at which Maloja should no longer increase its database cache."), "use_request_cache":(tp.Boolean(), "Use request-local DB Cache", False), "use_global_cache":(tp.Boolean(), "Use global DB Cache", True) },