Merge branch 'master' of github.com:krateng/maloja

API.md

@@ -1,6 +1,7 @@
 # Scrobbling
 
-In order to scrobble from a wide selection of clients, you can use Maloja's standard-compliant APIs with the following settings:
+Scrobbling can be done with the native API, see [below](#submitting-a-scrobble).
+In order to scrobble from a wide selection of clients, you can also use Maloja's standard-compliant APIs with the following settings:
 
 GNU FM |  
 ------ | ---------
@@ -41,7 +42,7 @@ The user starts playing '(Fine Layers of) Slaysenflite', which is exactly 3:00 m
 * If the user ends the play after 1:22, no scrobble is submitted
 * If the user ends the play after 2:06, a scrobble with `"duration":126` is submitted
 * If the user jumps back several times and ends the play after 3:57, a scrobble with `"duration":237` is submitted
-* If the user jumps back several times and ends the play after 4:49, two scrobbles with `"duration":180` and `"duration":109` should be submitted
+* If the user jumps back several times and ends the play after 4:49, two scrobbles with `"duration":180` and `"duration":109` are submitted
     
   </td></tr>
 <table>
@@ -54,11 +55,26 @@ The native Maloja API is reachable at `/apis/mlj_1`. Endpoints are listed on `/a
 All endpoints return JSON data. POST request can be made with query string or form data arguments, but this is discouraged - JSON should be used whenever possible.
 
 No application should ever rely on the non-existence of fields in the JSON data - i.e., additional fields can be added at any time without this being considered a breaking change. Existing fields should usually not be removed or changed, but it is always a good idea to add basic handling for missing fields.
+  
+## Submitting a Scrobble
+
+The POST endpoint `/newscrobble` is used to submit new scrobbles. These use a flat JSON structure with the following fields:
+  
+| Key | Type | Description |
+| --- | --- | --- |
+| `artists` | List(String) | Track artists |
+| `title` | String | Track title |
+| `album` | String | Name of the album (Optional) |
+| `albumartists` | List(String) | Album artists (Optional) |
+| `duration` | Integer | How long the song was listened to in seconds (Optional) |
+| `length` | Integer | Actual length of the full song in seconds (Optional) |
+| `time` | Integer | Timestamp of the listen if it was not at the time of submitting (Optional) |
+| `nofix` | Boolean | Skip server-side metadata fixing (Optional) |
 
 ## General Structure
 
-  
-Most endpoints follow this structure:
+The API is not fully consistent in order to ensure backwards-compatibility. Refer to the individual endpoints.  
+Generally, most endpoints follow this structure:
 
 | Key | Type | Description |
 | --- | --- | --- |
@@ -66,7 +82,7 @@ Most endpoints follow this structure:
 | `error` | Mapping | Details about the error if one occured. |
 | `warnings` | List | Any warnings that did not result in failure, but should be noted. Field is omitted if there are no warnings! |
 | `desc` | String | Human-readable feedback. This can be shown directly to the user if desired. |
-| `list` | List | List of returned [entities](#Entity-Structure) |
+| `list` | List | List of returned [entities](#entity-structure) |
 
   
 Both errors and warnings have the following structure:
@@ -87,7 +103,7 @@ Whenever a list of entities is returned, they have the following fields:
 | Key | Type | Description |
 | --- | --- | --- |
 | `time` | Integer | Timestamp of the Scrobble in UTC |
-| `track` | Mapping | The [track](#Track) being scrobbled |
+| `track` | Mapping | The [track](#track) being scrobbled |
 | `duration` | Integer | How long the track was played for in seconds |
 | `origin` | String | Client that submitted the scrobble, or import source |
 
@@ -118,7 +134,7 @@ Whenever a list of entities is returned, they have the following fields:
 
 | Key | Type | Description |
 | --- | --- | --- |
-| `artists` | List | The [artists](#Artist) credited with the track |
+| `artists` | List | The [artists](#artist) credited with the track |
 | `title` | String | The title of the track |
 | `length` | Integer | The full length of the track in seconds |
 

dev/releases/3.0.yml

@@ -43,6 +43,7 @@ minor_release_name: "Yeonhee"
     - "[Bugfix] No longer releasing database lock during scrobble creation"
     - "[Distribution] Experimental arm64 image"
 3.0.7:
+    commit: "62abc319303a6cb6463f7c27b6ef09b76fc67f86"
     notes:
     - "[Bugix] Improved signal handling"
     - "[Bugix] Fixed constant re-caching of all-time stats, significantly increasing page load speed"

dev/releases/branch.yml

@@ -1 +1,2 @@
-
+- "[Performance] Adjusted cache sizes"
+- "[Logging] Added cache memory use information"

dev/write_tags.py

@@ -6,6 +6,7 @@ FOLDER = "dev/releases"
 
 releases = {}
 for f in os.listdir(FOLDER):
+	if f == "branch.yml": continue
 	#maj,min = (int(i) for i in f.split('.')[:2])
 
 	with open(os.path.join(FOLDER,f)) as fd:

maloja/database/dbcache.py

@@ -5,6 +5,7 @@
 import lru
 import psutil
 import json
+import sys
 from doreah.regular import runhourly
 from doreah.logging import log
 
@@ -12,16 +13,10 @@ from ..globalconf import malojaconfig
 
 
 
-
-
 if malojaconfig['USE_GLOBAL_CACHE']:
-	CACHE_SIZE = 1000
-	ENTITY_CACHE_SIZE = 100000
 
-	cache = lru.LRU(CACHE_SIZE)
-	entitycache = lru.LRU(ENTITY_CACHE_SIZE)
-
-	hits, misses = 0, 0
+	cache = lru.LRU(10000)
+	entitycache = lru.LRU(100000)
 
 
 
@@ -31,11 +26,10 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 		trim_cache()
 
 	def print_stats():
-		log(f"Cache Size: {len(cache)} [{len(entitycache)} E], System RAM Utilization: {psutil.virtual_memory().percent}%, Cache Hits: {hits}/{hits+misses}")
-		#print("Full rundown:")
-		#import sys
-		#for k in cache.keys():
-		#	print(f"\t{k}\t{sys.getsizeof(cache[k])}")
+		for name,c in (('Cache',cache),('Entity Cache',entitycache)):
+			hits, misses = c.get_stats()
+			log(f"{name}: Size: {len(c)} | Hits: {hits}/{hits+misses} | Estimated Memory: {human_readable_size(c)}")
+		log(f"System RAM Utilization: {psutil.virtual_memory().percent}%")
 
 
 	def cached_wrapper(inner_func):
@@ -49,12 +43,9 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 			global hits, misses
 			key = (serialize(args),serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to"))
 
-			if key in cache:
-				hits += 1
-				return cache.get(key)
-
-			else:
-				misses += 1
+			try:
+				return cache[key]
+			except KeyError:
 				result = inner_func(*args,**kwargs,dbconn=conn)
 				cache[key] = result
 				return result
@@ -67,25 +58,18 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 	# cache that's aware of what we're calling
 	def cached_wrapper_individual(inner_func):
 
-
 		def outer_func(set_arg,**kwargs):
-
-
 			if 'dbconn' in kwargs:
 				conn = kwargs.pop('dbconn')
 			else:
 				conn = None
 
-			#global hits, misses
 			result = {}
 			for id in set_arg:
-				if (inner_func,id) in entitycache:
+				try:
 					result[id] = entitycache[(inner_func,id)]
-					#hits += 1
-				else:
+				except KeyError:
 					pass
-					#misses += 1
-
 
 			remaining = inner_func(set(e for e in set_arg if e not in result),dbconn=conn)
 			for id in remaining:
@@ -115,13 +99,14 @@ if malojaconfig['USE_GLOBAL_CACHE']:
 	def trim_cache():
 		ramprct = psutil.virtual_memory().percent
 		if ramprct > malojaconfig["DB_MAX_MEMORY"]:
-			log(f"{ramprct}% RAM usage, clearing cache and adjusting size!")
+			log(f"{ramprct}% RAM usage, clearing cache!")
+			for c in (cache,entitycache):
+				c.clear()
 			#ratio = 0.6
 			#targetsize = max(int(len(cache) * ratio),50)
 			#log(f"Reducing to {targetsize} entries")
 			#cache.set_size(targetsize)
 			#cache.set_size(HIGH_NUMBER)
-			cache.clear()
 			#if cache.get_size() > CACHE_ADJUST_STEP:
 			#	cache.set_size(cache.get_size() - CACHE_ADJUST_STEP)
 
@@ -156,3 +141,32 @@ def serialize(obj):
 			elif isinstance(obj,dict):
 				return "{" + ",".join(serialize(o) + ":" + serialize(obj[o]) for o in obj) + "}"
 			return json.dumps(obj.hashable())
+
+
+
+def get_size_of(obj,counted=None):
+	if counted is None:
+		counted = set()
+	if id(obj) in counted: return 0
+	size = sys.getsizeof(obj)
+	counted.add(id(obj))
+	try:
+		for k,v in obj.items():
+			size += get_size_of(v,counted=counted)
+	except:
+		try:
+			for i in obj:
+				size += get_size_of(i,counted=counted)
+		except:
+			pass
+	return size
+
+def human_readable_size(obj):
+	units = ['','K','M','G','T','P']
+	idx = 0
+	bytes = get_size_of(obj)
+	while bytes > 1024 and len(units) > idx+1:
+		bytes = bytes / 1024
+		idx += 1
+
+	return f"{bytes:.2f} {units[idx]}B"

maloja/database/sqldb.py

@@ -116,6 +116,8 @@ def connection_provider(func):
 			with engine.connect() as connection:
 				kwargs['dbconn'] = connection
 				return func(*args,**kwargs)
+
+	wrapper.__innerfunc__ = func
 	return wrapper
 
 ##### DB <-> Dict translations

maloja/globalconf.py

@@ -148,7 +148,7 @@ malojaconfig = Configuration(
 		"Technical":{
 			"cache_expire_positive":(tp.Integer(),								"Image Cache Expiration", 				60,		"Days until images are refetched"),
 			"cache_expire_negative":(tp.Integer(),								"Image Cache Negative Expiration",		5,		"Days until failed image fetches are reattempted"),
-			"db_max_memory":(tp.Integer(min=0,max=100),							"RAM Percentage soft limit",			80,		"RAM Usage in percent at which Maloja should no longer increase its database cache."),
+			"db_max_memory":(tp.Integer(min=0,max=100),							"RAM Percentage soft limit",			50,		"RAM Usage in percent at which Maloja should no longer increase its database cache."),
 			"use_request_cache":(tp.Boolean(),									"Use request-local DB Cache",			False),
 			"use_global_cache":(tp.Boolean(),									"Use global DB Cache",					True)
 		},