Ported cache cleanup from 3.1

maloja/database/dbcache.py

@@ -10,96 +10,97 @@ from doreah.logging import log
 
 from ..globalconf import malojaconfig
 
-HIGH_NUMBER = 1000000
-CACHE_SIZE = 10000
-ENTITY_CACHE_SIZE = 1000000
-CACHE_ADJUST_STEP = 100
-
-cache = lru.LRU(CACHE_SIZE)
-entitycache = lru.LRU(ENTITY_CACHE_SIZE)
-
-hits, misses = 0, 0
 
 
 
-@runhourly
+
-def maintenance():
+if malojaconfig['USE_GLOBAL_CACHE']:
-	if malojaconfig['USE_GLOBAL_CACHE']:
+	CACHE_SIZE = 1000
+	ENTITY_CACHE_SIZE = 100000
+
+	cache = lru.LRU(CACHE_SIZE)
+	entitycache = lru.LRU(ENTITY_CACHE_SIZE)
+
+	hits, misses = 0, 0
+
+
+
+	@runhourly
+	def maintenance():
 		print_stats()
 		trim_cache()
 
-def print_stats():
+	def print_stats():
-	log(f"Cache Size: {len(cache)} [{len(entitycache)} E], System RAM Utilization: {psutil.virtual_memory().percent}%, Cache Hits: {hits}/{hits+misses}")
+		log(f"Cache Size: {len(cache)} [{len(entitycache)} E], System RAM Utilization: {psutil.virtual_memory().percent}%, Cache Hits: {hits}/{hits+misses}")
-	#print("Full rundown:")
+		#print("Full rundown:")
-	#import sys
+		#import sys
-	#for k in cache.keys():
+		#for k in cache.keys():
-	#	print(f"\t{k}\t{sys.getsizeof(cache[k])}")
+		#	print(f"\t{k}\t{sys.getsizeof(cache[k])}")
 
 
-def cached_wrapper(inner_func):
+	def cached_wrapper(inner_func):
 
-	if not malojaconfig['USE_GLOBAL_CACHE']: return inner_func
+		def outer_func(*args,**kwargs):
-	def outer_func(*args,**kwargs):
-		if 'dbconn' in kwargs:
-			conn = kwargs.pop('dbconn')
-		else:
-			conn = None
-		global hits, misses
-		key = (serialize(args),serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to"))
 
-		if key in cache:
+			if 'dbconn' in kwargs:
-			hits += 1
+				conn = kwargs.pop('dbconn')
-			return cache.get(key)
+			else:
+				conn = None
+			global hits, misses
+			key = (serialize(args),serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to"))
+
+			if key in cache:
+				hits += 1
+				return cache.get(key)
+
+			else:
+				misses += 1
+				result = inner_func(*args,**kwargs,dbconn=conn)
+				cache[key] = result
+				return result
+
+		return outer_func
+
+
+	# cache for functions that call with a whole list of entity ids
+	# we don't want a new cache entry for every single combination, but keep a common
+	# cache that's aware of what we're calling
+	def cached_wrapper_individual(inner_func):
+
+
+		def outer_func(set_arg,**kwargs):
+
+
+			if 'dbconn' in kwargs:
+				conn = kwargs.pop('dbconn')
+			else:
+				conn = None
+
+			#global hits, misses
+			result = {}
+			for id in set_arg:
+				if (inner_func,id) in entitycache:
+					result[id] = entitycache[(inner_func,id)]
+					#hits += 1
+				else:
+					pass
+					#misses += 1
+
+
+			remaining = inner_func(set(e for e in set_arg if e not in result),dbconn=conn)
+			for id in remaining:
+					entitycache[(inner_func,id)] = remaining[id]
+					result[id] = remaining[id]
 
-		else:
-			misses += 1
-			result = inner_func(*args,**kwargs,dbconn=conn)
-			cache[key] = result
 			return result
 
-	return outer_func
+		return outer_func
 
-
+	def invalidate_caches(scrobbletime=None):
-# cache for functions that call with a whole list of entity ids
-# we don't want a new cache entry for every single combination, but keep a common
-# cache that's aware of what we're calling
-def cached_wrapper_individual(inner_func):
-
-	if not malojaconfig['USE_GLOBAL_CACHE']: return inner_func
-	def outer_func(set_arg,**kwargs):
-
-
-		if 'dbconn' in kwargs:
-			conn = kwargs.pop('dbconn')
-		else:
-			conn = None
-
-		#global hits, misses
-		result = {}
-		for id in set_arg:
-			if (inner_func,id) in entitycache:
-				result[id] = entitycache[(inner_func,id)]
-				#hits += 1
-			else:
-				pass
-				#misses += 1
-
-
-		remaining = inner_func(set(e for e in set_arg if e not in result),dbconn=conn)
-		for id in remaining:
-				entitycache[(inner_func,id)] = remaining[id]
-				result[id] = remaining[id]
-
-		return result
-
-	return outer_func
-
-def invalidate_caches(scrobbletime):
-	if malojaconfig['USE_GLOBAL_CACHE']:
 		cleared, kept = 0, 0
 		for k in cache.keys():
 			# VERY BIG TODO: differentiate between None as in 'unlimited timerange' and None as in 'time doesnt matter here'!
-			if (k[3] is None or scrobbletime >= k[3]) and (k[4] is None or scrobbletime <= k[4]):
+			if scrobbletime is None or (k[3] is None or scrobbletime >= k[3]) and (k[4] is None or scrobbletime <= k[4]):
 				cleared += 1
 				del cache[k]
 			else:
@@ -107,28 +108,42 @@ def invalidate_caches(scrobbletime):
 		log(f"Invalidated {cleared} of {cleared+kept} DB cache entries")
 
 
-def invalidate_entity_cache():
+	def invalidate_entity_cache():
-	entitycache.clear()
+		entitycache.clear()
 
 
-def trim_cache():
+	def trim_cache():
-	ramprct = psutil.virtual_memory().percent
+		ramprct = psutil.virtual_memory().percent
-	if ramprct > malojaconfig["DB_MAX_MEMORY"]:
+		if ramprct > malojaconfig["DB_MAX_MEMORY"]:
-		log(f"{ramprct}% RAM usage, clearing cache and adjusting size!")
+			log(f"{ramprct}% RAM usage, clearing cache and adjusting size!")
-		#ratio = 0.6
+			#ratio = 0.6
-		#targetsize = max(int(len(cache) * ratio),50)
+			#targetsize = max(int(len(cache) * ratio),50)
-		#log(f"Reducing to {targetsize} entries")
+			#log(f"Reducing to {targetsize} entries")
-		#cache.set_size(targetsize)
+			#cache.set_size(targetsize)
-		#cache.set_size(HIGH_NUMBER)
+			#cache.set_size(HIGH_NUMBER)
-		cache.clear()
+			cache.clear()
-		if cache.get_size() > CACHE_ADJUST_STEP:
+			#if cache.get_size() > CACHE_ADJUST_STEP:
-			cache.set_size(cache.get_size() - CACHE_ADJUST_STEP)
+			#	cache.set_size(cache.get_size() - CACHE_ADJUST_STEP)
 
-		#log(f"New RAM usage: {psutil.virtual_memory().percent}%")
+			#log(f"New RAM usage: {psutil.virtual_memory().percent}%")
-		print_stats()
+			print_stats()
 
 
 
+
+
+
+else:
+	def cached_wrapper(func):
+		return func
+	def cached_wrapper_individual(func):
+		return func
+	def invalidate_caches(scrobbletime=None):
+		return None
+	def invalidate_entity_cache():
+		return None
+
+
 def serialize(obj):
 	try:
 		return serialize(obj.hashable())

maloja/globalconf.py

@@ -149,8 +149,8 @@ malojaconfig = Configuration(
 			"cache_expire_positive":(tp.Integer(),								"Image Cache Expiration", 				60,		"Days until images are refetched"),
 			"cache_expire_negative":(tp.Integer(),								"Image Cache Negative Expiration",		5,		"Days until failed image fetches are reattempted"),
 			"db_max_memory":(tp.Integer(min=0,max=100),							"RAM Percentage soft limit",			80,		"RAM Usage in percent at which Maloja should no longer increase its database cache."),
-			"use_request_cache":(tp.Boolean(),									"Use request-local DB Cache",			True),
+			"use_request_cache":(tp.Boolean(),									"Use request-local DB Cache",			False),
-			"use_global_cache":(tp.Boolean(),									"Use global DB Cache",					True)
+			"use_global_cache":(tp.Boolean(),									"Use global DB Cache",					False)
 		},
 		"Fluff":{
 			"scrobbles_gold":(tp.Integer(),										"Scrobbles for Gold",			250,				"How many scrobbles a track needs to be considered 'Gold' status"),