Merge pull request 'Use eventlet in deemix backend' (#42) from kermit/deemix:eventlet into main
Reviewed-on: https://codeberg.org/RemixDev/deemix/pulls/42
This commit is contained in:
commit
987254a127
|
@ -1,8 +1,9 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
import eventlet
|
||||||
import binascii
|
import binascii
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import requests
|
requests = eventlet.import_patched('requests')
|
||||||
from Cryptodome.Cipher import Blowfish, AES
|
from Cryptodome.Cipher import Blowfish, AES
|
||||||
from Cryptodome.Hash import MD5
|
from Cryptodome.Hash import MD5
|
||||||
from Cryptodome.Util.Padding import pad
|
from Cryptodome.Util.Padding import pad
|
||||||
|
@ -52,7 +53,7 @@ class Deezer:
|
||||||
headers=self.http_headers
|
headers=self.http_headers
|
||||||
)
|
)
|
||||||
except:
|
except:
|
||||||
time.sleep(2)
|
eventlet.sleep(2)
|
||||||
return self.get_track_filesizes(sng_id)
|
return self.get_track_filesizes(sng_id)
|
||||||
response = site.json()["results"]
|
response = site.json()["results"]
|
||||||
filesizes = {}
|
filesizes = {}
|
||||||
|
@ -80,7 +81,7 @@ class Deezer:
|
||||||
)
|
)
|
||||||
result_json = result.json()
|
result_json = result.json()
|
||||||
except:
|
except:
|
||||||
time.sleep(2)
|
eventlet.sleep(2)
|
||||||
return self.gw_api_call(method, args)
|
return self.gw_api_call(method, args)
|
||||||
if len(result_json['error']):
|
if len(result_json['error']):
|
||||||
raise APIError(json.dumps(result_json['error']))
|
raise APIError(json.dumps(result_json['error']))
|
||||||
|
@ -98,11 +99,11 @@ class Deezer:
|
||||||
)
|
)
|
||||||
result_json = result.json()
|
result_json = result.json()
|
||||||
except:
|
except:
|
||||||
time.sleep(2)
|
eventlet.sleep(2)
|
||||||
return self.api_call(method, args)
|
return self.api_call(method, args)
|
||||||
if 'error' in result_json.keys():
|
if 'error' in result_json.keys():
|
||||||
if 'code' in result_json['error'] and result_json['error']['code'] == 4:
|
if 'code' in result_json['error'] and result_json['error']['code'] == 4:
|
||||||
time.sleep(5)
|
eventlet.sleep(5)
|
||||||
return self.api_call(method, args)
|
return self.api_call(method, args)
|
||||||
raise APIError(json.dumps(result_json['error']))
|
raise APIError(json.dumps(result_json['error']))
|
||||||
return result_json
|
return result_json
|
||||||
|
@ -583,9 +584,10 @@ class Deezer:
|
||||||
try:
|
try:
|
||||||
request = requests.get(url, headers=self.http_headers, stream=True, timeout=30)
|
request = requests.get(url, headers=self.http_headers, stream=True, timeout=30)
|
||||||
except:
|
except:
|
||||||
time.sleep(2)
|
eventlet.sleep(2)
|
||||||
return self.stream_track(track_id, url, stream)
|
return self.stream_track(track_id, url, stream)
|
||||||
request.raise_for_status()
|
request.raise_for_status()
|
||||||
|
eventlet.sleep(0)
|
||||||
blowfish_key = str.encode(self._get_blowfish_key(str(track_id)))
|
blowfish_key = str.encode(self._get_blowfish_key(str(track_id)))
|
||||||
i = 0
|
i = 0
|
||||||
for chunk in request.iter_content(2048):
|
for chunk in request.iter_content(2048):
|
||||||
|
@ -594,6 +596,7 @@ class Deezer:
|
||||||
chunk)
|
chunk)
|
||||||
stream.write(chunk)
|
stream.write(chunk)
|
||||||
i += 1
|
i += 1
|
||||||
|
eventlet.sleep(0)
|
||||||
|
|
||||||
def _md5(self, data):
|
def _md5(self, data):
|
||||||
h = MD5.new()
|
h = MD5.new()
|
||||||
|
|
|
@ -1,14 +1,14 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
import eventlet
|
||||||
import os.path
|
import os.path
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from requests import get
|
requests = eventlet.import_patched('requests')
|
||||||
from requests import exceptions as request_exception
|
get = requests.get
|
||||||
|
request_exception = requests.exceptions
|
||||||
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from os import makedirs, remove, system as execute
|
from os import makedirs, remove, system as execute
|
||||||
from tempfile import gettempdir
|
from tempfile import gettempdir
|
||||||
from time import sleep
|
|
||||||
|
|
||||||
from deemix.app.queueitem import QISingle, QICollection
|
from deemix.app.queueitem import QISingle, QICollection
|
||||||
from deemix.app.track import Track
|
from deemix.app.track import Track
|
||||||
|
@ -64,12 +64,12 @@ def downloadImage(url, path, overwrite="n"):
|
||||||
pictureSize = int(pictureUrl[:pictureUrl.find("x")])
|
pictureSize = int(pictureUrl[:pictureUrl.find("x")])
|
||||||
if pictureSize > 1200:
|
if pictureSize > 1200:
|
||||||
logger.warn("Couldn't download "+str(pictureSize)+"x"+str(pictureSize)+" image, falling back to 1200x1200")
|
logger.warn("Couldn't download "+str(pictureSize)+"x"+str(pictureSize)+" image, falling back to 1200x1200")
|
||||||
sleep(1)
|
eventlet.sleep(1)
|
||||||
return downloadImage(urlBase+pictureUrl.replace(str(pictureSize)+"x"+str(pictureSize), '1200x1200'), path, overwrite)
|
return downloadImage(urlBase+pictureUrl.replace(str(pictureSize)+"x"+str(pictureSize), '1200x1200'), path, overwrite)
|
||||||
logger.error("Image not found: "+url)
|
logger.error("Image not found: "+url)
|
||||||
except (request_exception.ConnectionError, request_exception.ChunkedEncodingError) as e:
|
except (request_exception.ConnectionError, request_exception.ChunkedEncodingError) as e:
|
||||||
logger.error("Couldn't download Image, retrying in 5 seconds...: "+url+"\n")
|
logger.error("Couldn't download Image, retrying in 5 seconds...: "+url+"\n")
|
||||||
sleep(5)
|
eventlet.sleep(5)
|
||||||
return downloadImage(url, path, overwrite)
|
return downloadImage(url, path, overwrite)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception(f"Error while downloading an image, you should report this to the developers: {str(e)}")
|
logger.exception(f"Error while downloading an image, you should report this to the developers: {str(e)}")
|
||||||
|
@ -111,9 +111,9 @@ class DownloadJob:
|
||||||
self.singleAfterDownload(result)
|
self.singleAfterDownload(result)
|
||||||
elif isinstance(self.queueItem, QICollection):
|
elif isinstance(self.queueItem, QICollection):
|
||||||
tracks = [None] * len(self.queueItem.collection)
|
tracks = [None] * len(self.queueItem.collection)
|
||||||
with ThreadPoolExecutor(self.settings['queueConcurrency']) as executor:
|
pool = eventlet.GreenPool(size=self.settings['queueConcurrency'])
|
||||||
for pos, track in enumerate(self.queueItem.collection, start=0):
|
for pos, track in enumerate(self.queueItem.collection, start=0):
|
||||||
tracks[pos] = executor.submit(self.downloadWrapper, track)
|
tracks[pos] = pool.spawn(self.downloadWrapper, track)
|
||||||
self.collectionAfterDownload(tracks)
|
self.collectionAfterDownload(tracks)
|
||||||
if self.interface:
|
if self.interface:
|
||||||
if self.queueItem.cancel:
|
if self.queueItem.cancel:
|
||||||
|
@ -155,7 +155,7 @@ class DownloadJob:
|
||||||
searched = ""
|
searched = ""
|
||||||
|
|
||||||
for index in range(len(tracks)):
|
for index in range(len(tracks)):
|
||||||
result = tracks[index].result()
|
result = tracks[index].wait()
|
||||||
# Check if queue is cancelled
|
# Check if queue is cancelled
|
||||||
if not result:
|
if not result:
|
||||||
return None
|
return None
|
||||||
|
@ -476,7 +476,7 @@ class DownloadJob:
|
||||||
except (request_exception.ConnectionError, request_exception.ChunkedEncodingError) as e:
|
except (request_exception.ConnectionError, request_exception.ChunkedEncodingError) as e:
|
||||||
remove(writepath)
|
remove(writepath)
|
||||||
logger.warn(f"[{track.mainArtist['name']} - {track.title}] Error while downloading the track, trying again in 5s...")
|
logger.warn(f"[{track.mainArtist['name']} - {track.title}] Error while downloading the track, trying again in 5s...")
|
||||||
sleep(5)
|
eventlet.sleep(5)
|
||||||
return downloadMusic(track, trackAPI_gw)
|
return downloadMusic(track, trackAPI_gw)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
remove(writepath)
|
remove(writepath)
|
||||||
|
@ -575,9 +575,10 @@ class DownloadJob:
|
||||||
try:
|
try:
|
||||||
request = get(track.downloadUrl, headers=self.dz.http_headers, stream=True, timeout=30)
|
request = get(track.downloadUrl, headers=self.dz.http_headers, stream=True, timeout=30)
|
||||||
except request_exception.ConnectionError:
|
except request_exception.ConnectionError:
|
||||||
sleep(2)
|
eventlet.sleep(2)
|
||||||
return self.streamTrack(stream, track)
|
return self.streamTrack(stream, track)
|
||||||
request.raise_for_status()
|
request.raise_for_status()
|
||||||
|
eventlet.sleep(0)
|
||||||
blowfish_key = str.encode(self.dz._get_blowfish_key(str(track.id)))
|
blowfish_key = str.encode(self.dz._get_blowfish_key(str(track.id)))
|
||||||
complete = int(request.headers["Content-Length"])
|
complete = int(request.headers["Content-Length"])
|
||||||
if complete == 0:
|
if complete == 0:
|
||||||
|
@ -599,6 +600,7 @@ class DownloadJob:
|
||||||
self.downloadPercentage += chunkProgres
|
self.downloadPercentage += chunkProgres
|
||||||
self.updatePercentage()
|
self.updatePercentage()
|
||||||
i += 1
|
i += 1
|
||||||
|
eventlet.sleep(0)
|
||||||
|
|
||||||
def updatePercentage(self):
|
def updatePercentage(self):
|
||||||
if round(self.downloadPercentage) != self.lastPercentage and round(self.downloadPercentage) % 2 == 0:
|
if round(self.downloadPercentage) != self.lastPercentage and round(self.downloadPercentage) % 2 == 0:
|
||||||
|
|
|
@ -8,7 +8,6 @@ import logging
|
||||||
import os.path as path
|
import os.path as path
|
||||||
import json
|
import json
|
||||||
from os import remove
|
from os import remove
|
||||||
from time import sleep
|
|
||||||
from urllib.request import urlopen
|
from urllib.request import urlopen
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
|
@ -3,3 +3,4 @@ pycryptodomex
|
||||||
mutagen
|
mutagen
|
||||||
requests
|
requests
|
||||||
spotipy>=2.11.0
|
spotipy>=2.11.0
|
||||||
|
eventlet
|
||||||
|
|
Loading…
Reference in New Issue