Experimenting with more thread limitations

maloja/images.py

@@ -13,7 +13,7 @@ import base64
 import requests
 import datauri
 import io
-from threading import Thread, Timer
+from threading import Thread, Timer, BoundedSemaphore
 import re
 import datetime
 
@@ -118,61 +118,63 @@ def get_artist_image(artist=None,artist_id=None):
 
 
 
-
+resolve_semaphore = BoundedSemaphore(8)
 
 
 def resolve_track_image(track_id):
 
-	# check cache
+	with resolve_semaphore:
-	result = get_image_from_cache(track_id,'tracks')
+		# check cache
-	if result is not None:
+		result = get_image_from_cache(track_id,'tracks')
-		return result
+		if result is not None:
-
-	track = database.sqldb.get_track(track_id)
-
-	# local image
-	if malojaconfig["USE_LOCAL_IMAGES"]:
-		images = local_files(artists=track['artists'],title=track['title'])
-		if len(images) != 0:
-			result = random.choice(images)
-			result = urllib.parse.quote(result)
-			result = {'type':'url','value':result}
-			set_image_in_cache(track_id,'tracks',result['value'])
 			return result
 
-	# third party
+		track = database.sqldb.get_track(track_id)
-	result = thirdparty.get_image_track_all((track['artists'],track['title']))
-	result = {'type':'url','value':result}
-	set_image_in_cache(track_id,'tracks',result['value'])
 
-	return result
+		# local image
+		if malojaconfig["USE_LOCAL_IMAGES"]:
+			images = local_files(artists=track['artists'],title=track['title'])
+			if len(images) != 0:
+				result = random.choice(images)
+				result = urllib.parse.quote(result)
+				result = {'type':'url','value':result}
+				set_image_in_cache(track_id,'tracks',result['value'])
+				return result
+
+		# third party
+		result = thirdparty.get_image_track_all((track['artists'],track['title']))
+		result = {'type':'url','value':result}
+		set_image_in_cache(track_id,'tracks',result['value'])
+
+		return result
 
 
 def resolve_artist_image(artist_id):
 
-	# check cache
+	with resolve_semaphore:
-	result = get_image_from_cache(artist_id,'artists')
+		# check cache
-	if result is not None:
+		result = get_image_from_cache(artist_id,'artists')
-		return result
+		if result is not None:
-
-	artist = database.sqldb.get_artist(artist_id)
-
-	# local image
-	if malojaconfig["USE_LOCAL_IMAGES"]:
-		images = local_files(artist=artist)
-		if len(images) != 0:
-			result = random.choice(images)
-			result = urllib.parse.quote(result)
-			result = {'type':'url','value':result}
-			set_image_in_cache(artist_id,'artists',result['value'])
 			return result
 
-	# third party
+		artist = database.sqldb.get_artist(artist_id)
-	result = thirdparty.get_image_artist_all(artist)
-	result = {'type':'url','value':result}
-	set_image_in_cache(artist_id,'artists',result['value'])
 
-	return result
+		# local image
+		if malojaconfig["USE_LOCAL_IMAGES"]:
+			images = local_files(artist=artist)
+			if len(images) != 0:
+				result = random.choice(images)
+				result = urllib.parse.quote(result)
+				result = {'type':'url','value':result}
+				set_image_in_cache(artist_id,'artists',result['value'])
+				return result
+
+		# third party
+		result = thirdparty.get_image_artist_all(artist)
+		result = {'type':'url','value':result}
+		set_image_in_cache(artist_id,'artists',result['value'])
+
+		return result
 
 
 # removes emojis and weird shit from names

maloja/server.py

@@ -35,7 +35,7 @@ from .proccontrol.profiler import profile
 
 PORT = malojaconfig["PORT"]
 HOST = malojaconfig["HOST"]
-THREADS = 24
+THREADS = 8
 BaseRequest.MEMFILE_MAX = 15 * 1024 * 1024
 
 #STATICFOLDER = importlib.resources.path(__name__,"web/static")

maloja/thirdparty/__init__.py

@@ -11,7 +11,7 @@ import json
 import urllib.parse, urllib.request
 import base64
 from doreah.logging import log
-from threading import Semaphore
+from threading import BoundedSemaphore
 
 from ..globalconf import malojaconfig
 from .. import database
@@ -26,7 +26,7 @@ services = {
 # have a limited number of worker threads so we don't completely hog the cpu with
 # these requests. they are mostly network bound, so python will happily open up 200 new
 # requests and then when all the responses come in we suddenly can't load pages anymore
-thirdpartylock = Semaphore(4)
+thirdpartylock = BoundedSemaphore(4)
 
 
 def import_scrobbles(identifier):