diff --git a/maloja/thirdparty/__init__.py b/maloja/thirdparty/__init__.py index c065c0b..b00dae4 100644 --- a/maloja/thirdparty/__init__.py +++ b/maloja/thirdparty/__init__.py @@ -11,6 +11,7 @@ import json import urllib.parse, urllib.request import base64 from doreah.logging import log +from threading import Semaphore from ..globalconf import malojaconfig from .. import database @@ -22,6 +23,11 @@ services = { "metadata":[] } +# have a limited number of worker threads so we don't completely hog the cpu with +# these requests. they are mostly network bound, so python will happily open up 200 new +# requests and then when all the responses come in we suddenly can't load pages anymore +thirdpartylock = Semaphore(4) + def import_scrobbles(identifier): for service in services['import']: @@ -34,27 +40,29 @@ def proxy_scrobble_all(artists,title,timestamp): service.scrobble(artists,title,timestamp) def get_image_track_all(track): - for service in services["metadata"]: - try: - res = service.get_image_track(track) - if res is not None: - log("Got track image for " + str(track) + " from " + service.name) - return res - else: - log("Could not get track image for " + str(track) + " from " + service.name) - except Exception as e: - log("Error getting track image from " + service.name + ": " + repr(e)) + with thirdpartylock: + for service in services["metadata"]: + try: + res = service.get_image_track(track) + if res is not None: + log("Got track image for " + str(track) + " from " + service.name) + return res + else: + log("Could not get track image for " + str(track) + " from " + service.name) + except Exception as e: + log("Error getting track image from " + service.name + ": " + repr(e)) def get_image_artist_all(artist): - for service in services["metadata"]: - try: - res = service.get_image_artist(artist) - if res is not None: - log("Got artist image for " + str(artist) + " from " + service.name) - return res - else: - log("Could not get artist image for " + str(artist) + " from " + service.name) - except Exception as e: - log("Error getting artist image from " + service.name + ": " + repr(e)) + with thirdpartylock: + for service in services["metadata"]: + try: + res = service.get_image_artist(artist) + if res is not None: + log("Got artist image for " + str(artist) + " from " + service.name) + return res + else: + log("Could not get artist image for " + str(artist) + " from " + service.name) + except Exception as e: + log("Error getting artist image from " + service.name + ": " + repr(e))