1
0
mirror of https://github.com/krateng/maloja.git synced 2023-08-10 21:12:55 +03:00

Limited metadata requests

This commit is contained in:
krateng
2022-03-06 02:30:29 +01:00
parent 827b05da8f
commit 48d88b208f

View File

@ -11,6 +11,7 @@ import json
import urllib.parse, urllib.request import urllib.parse, urllib.request
import base64 import base64
from doreah.logging import log from doreah.logging import log
from threading import Semaphore
from ..globalconf import malojaconfig from ..globalconf import malojaconfig
from .. import database from .. import database
@ -22,6 +23,11 @@ services = {
"metadata":[] "metadata":[]
} }
# have a limited number of worker threads so we don't completely hog the cpu with
# these requests. they are mostly network bound, so python will happily open up 200 new
# requests and then when all the responses come in we suddenly can't load pages anymore
thirdpartylock = Semaphore(4)
def import_scrobbles(identifier): def import_scrobbles(identifier):
for service in services['import']: for service in services['import']:
@ -34,27 +40,29 @@ def proxy_scrobble_all(artists,title,timestamp):
service.scrobble(artists,title,timestamp) service.scrobble(artists,title,timestamp)
def get_image_track_all(track): def get_image_track_all(track):
for service in services["metadata"]: with thirdpartylock:
try: for service in services["metadata"]:
res = service.get_image_track(track) try:
if res is not None: res = service.get_image_track(track)
log("Got track image for " + str(track) + " from " + service.name) if res is not None:
return res log("Got track image for " + str(track) + " from " + service.name)
else: return res
log("Could not get track image for " + str(track) + " from " + service.name) else:
except Exception as e: log("Could not get track image for " + str(track) + " from " + service.name)
log("Error getting track image from " + service.name + ": " + repr(e)) except Exception as e:
log("Error getting track image from " + service.name + ": " + repr(e))
def get_image_artist_all(artist): def get_image_artist_all(artist):
for service in services["metadata"]: with thirdpartylock:
try: for service in services["metadata"]:
res = service.get_image_artist(artist) try:
if res is not None: res = service.get_image_artist(artist)
log("Got artist image for " + str(artist) + " from " + service.name) if res is not None:
return res log("Got artist image for " + str(artist) + " from " + service.name)
else: return res
log("Could not get artist image for " + str(artist) + " from " + service.name) else:
except Exception as e: log("Could not get artist image for " + str(artist) + " from " + service.name)
log("Error getting artist image from " + service.name + ": " + repr(e)) except Exception as e:
log("Error getting artist image from " + service.name + ": " + repr(e))