Implemented socket.io

Started work on darkmode
Implemented queue system
Started work on download tab
This commit is contained in:
RemixDev 2020-04-10 16:12:21 +02:00
parent 4b2d9a15f9
commit b6956f6f6c
6 changed files with 228 additions and 125 deletions

View File

@ -268,7 +268,11 @@ class Deezer:
i += 1 i += 1
def stream_track(self, track_id, url, stream): def stream_track(self, track_id, url, stream):
request = requests.get(url, headers=self.http_headers, stream=True) try:
request = requests.get(url, headers=self.http_headers, stream=True, timeout=30)
except:
time.sleep(2)
return self.stream_track(track_id, url, stream)
request.raise_for_status() request.raise_for_status()
blowfish_key = str.encode(self._get_blowfish_key(str(track_id))) blowfish_key = str.encode(self._get_blowfish_key(str(track_id)))
i = 0 i = 0

View File

@ -2,7 +2,7 @@
from deemix.api.deezer import Deezer from deemix.api.deezer import Deezer
import deemix.utils.localpaths as localpaths import deemix.utils.localpaths as localpaths
from deemix.utils.misc import getIDFromLink, getTypeFromLink, getBitrateInt from deemix.utils.misc import getIDFromLink, getTypeFromLink, getBitrateInt
from deemix.app.downloader import download_track, download_album, download_playlist, download_artist, download_spotifytrack, download_spotifyalbum from deemix.app.queuemanager import addToQueue
from os import system as execute from os import system as execute
import os.path as path import os.path as path
from os import mkdir from os import mkdir
@ -31,26 +31,4 @@ def login():
f.write(arl) f.write(arl)
def downloadLink(url, settings, bitrate=None): def downloadLink(url, settings, bitrate=None):
forcedBitrate = getBitrateInt(bitrate) addToQueue(dz, url, settings, bitrate)
type = getTypeFromLink(url)
id = getIDFromLink(url, type)
folder = settings['downloadLocation']
if type == None or id == None:
print("URL not recognized")
if type == "track":
folder = download_track(dz, id, settings, forcedBitrate)
elif type == "album":
folder = download_album(dz, id, settings, forcedBitrate)
elif type == "playlist":
folder = download_playlist(dz, id, settings, forcedBitrate)
elif type == "artist":
download_artist(dz, id, settings, forcedBitrate)
elif type == "spotifytrack":
folder = download_spotifytrack(dz, id, settings, forcedBitrate)
elif type == "spotifyalbum":
folder = download_spotifyalbum(dz, id, settings, forcedBitrate)
else:
print("URL not supported yet")
return None
if settings['executeCommand'] != "":
execute(settings['executeCommand'].replace("%folder%", folder))

View File

@ -329,7 +329,7 @@ def getTrackData(dz, trackAPI_gw, trackAPI = None, albumAPI_gw = None, albumAPI
return track return track
def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack=None): def downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=None, socket=None):
result = {} result = {}
# Get the metadata # Get the metadata
if extraTrack: if extraTrack:
@ -348,7 +348,7 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack=
if not 'MD5_ORIGIN' in trackNew: if not 'MD5_ORIGIN' in trackNew:
trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID']) trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID'])
track = parseEssentialTrackData(track, trackNew) track = parseEssentialTrackData(track, trackNew)
return downloadTrackObj(dz, trackAPI, settings, extraTrack=track) return downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=track, socket=socket)
elif not 'searched' in track and settings['fallbackSearch']: elif not 'searched' in track and settings['fallbackSearch']:
print("Track not yet encoded, searching for alternative") print("Track not yet encoded, searching for alternative")
searchedId = dz.get_track_from_metadata(track['mainArtist']['name'], track['title'], track['album']['title']) searchedId = dz.get_track_from_metadata(track['mainArtist']['name'], track['title'], track['album']['title'])
@ -358,7 +358,7 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack=
trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID']) trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID'])
track = parseEssentialTrackData(track, trackNew) track = parseEssentialTrackData(track, trackNew)
track['searched'] = True track['searched'] = True
return downloadTrackObj(dz, trackAPI, settings, extraTrack=track) return downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=track, socket=socket)
else: else:
print("ERROR: Track not yet encoded and no alternative found!") print("ERROR: Track not yet encoded and no alternative found!")
result['error'] = { result['error'] = {
@ -375,7 +375,6 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack=
return result return result
# Get the selected bitrate # Get the selected bitrate
bitrate = overwriteBitrate if overwriteBitrate else settings['maxBitrate']
(format, filesize) = getPreferredBitrate(track['filesize'], bitrate, settings['fallbackBitrate']) (format, filesize) = getPreferredBitrate(track['filesize'], bitrate, settings['fallbackBitrate'])
if format == -100: if format == -100:
print("ERROR: Track not found at desired bitrate. Enable fallback to lower bitrates to fix this issue.") print("ERROR: Track not found at desired bitrate. Enable fallback to lower bitrates to fix this issue.")
@ -471,14 +470,14 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack=
if track['selectedFormat'] == 9 and settings['fallbackBitrate']: if track['selectedFormat'] == 9 and settings['fallbackBitrate']:
print("Track not available in flac, trying mp3") print("Track not available in flac, trying mp3")
track['filesize']['flac'] = 0 track['filesize']['flac'] = 0
return downloadTrackObj(dz, trackAPI, settings, extraTrack=track) return downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=track, socket=socket)
elif track['fallbackId'] != 0: elif track['fallbackId'] != 0:
print("Track not available, using fallback id") print("Track not available, using fallback id")
trackNew = dz.get_track_gw(track['fallbackId']) trackNew = dz.get_track_gw(track['fallbackId'])
if not 'MD5_ORIGIN' in trackNew: if not 'MD5_ORIGIN' in trackNew:
trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID']) trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID'])
track = parseEssentialTrackData(track, trackNew) track = parseEssentialTrackData(track, trackNew)
return downloadTrackObj(dz, trackAPI, settings, extraTrack=track) return downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=track, socket=socket)
elif not 'searched' in track and settings['fallbackSearch']: elif not 'searched' in track and settings['fallbackSearch']:
print("Track not available, searching for alternative") print("Track not available, searching for alternative")
searchedId = dz.get_track_from_metadata(track['mainArtist']['name'], track['title'], track['album']['title']) searchedId = dz.get_track_from_metadata(track['mainArtist']['name'], track['title'], track['album']['title'])
@ -488,7 +487,7 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack=
trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID']) trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID'])
track = parseEssentialTrackData(track, trackNew) track = parseEssentialTrackData(track, trackNew)
track['searched'] = True track['searched'] = True
return downloadTrackObj(dz, trackAPI, settings, extraTrack=track) return downloadTrackObj(dz, trackAPI, settings, bitrate, uuid, extraTrack=track, socket=socket)
else: else:
print("ERROR: Track not available on deezer's servers and no alternative found!") print("ERROR: Track not available on deezer's servers and no alternative found!")
result['error'] = { result['error'] = {
@ -510,8 +509,29 @@ def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack=
if 'searched' in track: if 'searched' in track:
result['searched'] = f'{track["mainArtist"]["name"]} - {track["title"]}' result['searched'] = f'{track["mainArtist"]["name"]} - {track["title"]}'
print("Done!") print("Done!")
if socket:
socket.emit("updateQueue", {'uuid': uuid, 'downloaded': True})
return result return result
def download(dz, queueItem, socket=None):
settings = queueItem['settings']
bitrate = queueItem['bitrate']
if 'single' in queueItem:
result = downloadTrackObj(dz, queueItem['single'], settings, bitrate, queueItem['uuid'], socket=socket)
download_path = after_download_single(result, settings)
elif 'collection' in queueItem:
print("Downloading collection")
playlist = [None] * len(queueItem['collection'])
with ThreadPoolExecutor(settings['queueConcurrency']) as executor:
for pos, track in enumerate(queueItem['collection'], start=0):
playlist[pos] = executor.submit(downloadTrackObj, dz, track, settings, bitrate, queueItem['uuid'], socket=socket)
download_path = after_download(playlist, settings)
return {
'dz': dz,
'socket': socket,
'download_path': download_path
}
def after_download(tracks, settings): def after_download(tracks, settings):
extrasPath = None extrasPath = None
playlist = [None] * len(tracks) playlist = [None] * len(tracks)
@ -558,72 +578,3 @@ def after_download_single(track, settings):
return track['extrasPath'] return track['extrasPath']
else: else:
return None return None
def download_track(dz, id, settings, overwriteBitrate=False):
trackAPI = dz.get_track_gw(id)
trackAPI['FILENAME_TEMPLATE'] = settings['tracknameTemplate']
trackAPI['SINGLE_TRACK'] = True
result = downloadTrackObj(dz, trackAPI, settings, overwriteBitrate)
return after_download_single(result, settings)
def download_spotifytrack(dz, id, settings, overwriteBitrate=False):
track_id = get_trackid_spotify(dz, id, settings['fallbackSearch'])
if track_id == "Not Enabled":
print("Spotify Features is not setted up correctly.")
if track_id != 0:
return download_track(dz, track_id, settings, overwriteBitrate)
else:
print("Track not found on deezer!")
return None
def download_album(dz, id, settings, overwriteBitrate=False):
albumAPI = dz.get_album(id)
albumAPI_gw = dz.get_album_gw(id)
albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK']
albumAPI['copyright'] = albumAPI_gw['COPYRIGHT']
if albumAPI['nb_tracks'] == 1:
trackAPI = dz.get_track_gw(albumAPI['tracks']['data'][0]['id'])
trackAPI['_EXTRA_ALBUM'] = albumAPI
trackAPI['FILENAME_TEMPLATE'] = settings['tracknameTemplate']
trackAPI['SINGLE_TRACK'] = True
result = downloadTrackObj(dz, trackAPI, settings, overwriteBitrate)
return after_download_single(result, settings)
else:
tracksArray = dz.get_album_tracks_gw(id)
playlist = [None] * len(tracksArray)
with ThreadPoolExecutor(settings['queueConcurrency']) as executor:
for pos, trackAPI in enumerate(tracksArray, start=1):
trackAPI['_EXTRA_ALBUM'] = albumAPI
trackAPI['POSITION'] = pos
trackAPI['FILENAME_TEMPLATE'] = settings['albumTracknameTemplate']
playlist[pos-1] = executor.submit(downloadTrackObj, dz, trackAPI, settings, overwriteBitrate)
return after_download(playlist, settings)
def download_spotifyalbum(dz, id, settings, overwriteBitrate=False):
album_id = get_albumid_spotify(dz, id)
if album_id == "Not Enabled":
print("Spotify Features is not setted up correctly.")
if album_id != 0:
return download_album(dz, album_id, settings, overwriteBitrate)
else:
print("Album not found on deezer!")
return None
def download_artist(dz, id, settings, overwriteBitrate=False):
artistAPI = dz.get_artist_albums(id)
for album in artistAPI['data']:
print(f"Album: {album['title']}")
download_album(dz, album['id'], settings, overwriteBitrate)
def download_playlist(dz, id, settings, overwriteBitrate=False):
playlistAPI = dz.get_playlist(id)
playlistTracksAPI = dz.get_playlist_tracks_gw(id)
playlist = [None] * len(playlistTracksAPI)
with ThreadPoolExecutor(settings['queueConcurrency']) as executor:
for pos, trackAPI in enumerate(playlistTracksAPI, start=1):
trackAPI['_EXTRA_PLAYLIST'] = playlistAPI
trackAPI['POSITION'] = pos
trackAPI['FILENAME_TEMPLATE'] = settings['playlistTracknameTemplate']
playlist[pos-1] = executor.submit(downloadTrackObj, dz, trackAPI, settings, overwriteBitrate)
return after_download(playlist, settings)

View File

@ -1,7 +1,6 @@
from deemix.api.deezer import Deezer from deemix.api.deezer import Deezer
import deemix.utils.localpaths as localpaths import deemix.utils.localpaths as localpaths
from deemix.utils.misc import getIDFromLink, getTypeFromLink, getBitrateInt, isValidLink from deemix.app.queuemanager import addToQueue
from deemix.app.downloader import download_track, download_album, download_playlist, download_artist, download_spotifytrack, download_spotifyalbum
from deemix.app.settings import initSettings from deemix.app.settings import initSettings
from os import system as execute from os import system as execute
import os.path as path import os.path as path
@ -43,28 +42,9 @@ def mainSearch(term):
def search(term, type, start, nb): def search(term, type, start, nb):
return dz.search_gw(term, type, start, nb) return dz.search_gw(term, type, start, nb)
def addToQueue_link(url, bitrate=None, socket=None):
addToQueue(dz, url, settings, bitrate, socket)
def downloadLink(url, bitrate=None): def downloadLink(url, bitrate=None):
global settings
forcedBitrate = getBitrateInt(bitrate)
type = getTypeFromLink(url)
id = getIDFromLink(url, type)
folder = settings['downloadLocation']
if type == None or id == None:
print("URL not recognized")
if type == "track":
folder = download_track(dz, id, settings, forcedBitrate)
elif type == "album":
folder = download_album(dz, id, settings, forcedBitrate)
elif type == "playlist":
folder = download_playlist(dz, id, settings, forcedBitrate)
elif type == "artist":
download_artist(dz, id, settings, forcedBitrate)
elif type == "spotifytrack":
folder = download_spotifytrack(dz, id, settings, forcedBitrate)
elif type == "spotifyalbum":
folder = download_spotifyalbum(dz, id, settings, forcedBitrate)
else:
print("URL not supported yet")
return None
if settings['executeCommand'] != "": if settings['executeCommand'] != "":
execute(settings['executeCommand'].replace("%folder%", folder)) execute(settings['executeCommand'].replace("%folder%", folder))

189
deemix/app/queuemanager.py Normal file
View File

@ -0,0 +1,189 @@
from deemix.utils.misc import getIDFromLink, getTypeFromLink, getBitrateInt
from deemix.utils.spotifyHelper import get_trackid_spotify, get_albumid_spotify
from concurrent.futures import ProcessPoolExecutor
from deemix.app.downloader import download
queue = []
queueList = {}
currentItem = ""
currentJob = None
"""
queueItem base structure
title
artist
cover
size
downloaded
failed
progress
type
id
bitrate
uuid: type+id+bitrate
if its a single track
single
if its an album/playlist
collection
"""
def generateQueueItem(dz, url, settings, bitrate=None, albumAPI=None):
forcedBitrate = getBitrateInt(bitrate)
bitrate = forcedBitrate if forcedBitrate else settings['maxBitrate']
type = getTypeFromLink(url)
id = getIDFromLink(url, type)
result = {}
if type == None or id == None:
print("URL not recognized")
result['error'] = "URL not recognized"
elif type == "track":
trackAPI = dz.get_track_gw(id)
if albumAPI:
trackAPI['_EXTRA_ALBUM'] = albumAPI
trackAPI['FILENAME_TEMPLATE'] = settings['tracknameTemplate']
trackAPI['SINGLE_TRACK'] = True
result['title'] = trackAPI['SNG_TITLE']
if 'VERSION' in trackAPI and trackAPI['VERSION']:
result['title'] += " " + trackAPI['VERSION']
result['artist'] = trackAPI['ART_NAME']
result['cover'] = f"https://e-cdns-images.dzcdn.net/images/cover/{trackAPI['ART_PICTURE']}/128x128-000000-80-0-0.jpg"
result['size'] = 1
result['downloaded'] = 0
result['failed'] = 0
result['progress'] = 0
result['type'] = 'track'
result['id'] = id
result['bitrate'] = bitrate
result['uuid'] = f"{result['type']}:{id}:{bitrate}"
result['settings'] = settings or {}
result['single'] = trackAPI
elif type == "album":
albumAPI = dz.get_album(id)
albumAPI_gw = dz.get_album_gw(id)
albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK']
albumAPI['copyright'] = albumAPI_gw['COPYRIGHT']
if albumAPI['nb_tracks'] == 1:
return generateQueueItem(dz, f"https://www.deezer.com/track/{albumAPI['tracks']['data'][0]['id']}", settings, bitrate, albumAPI)
tracksArray = dz.get_album_tracks_gw(id)
result['title'] = albumAPI['title']
result['artist'] = albumAPI['artist']['name']
result['cover'] = albumAPI['cover_small'][:-24]+'/128x128-000000-80-0-0.jpg'
result['size'] = albumAPI['nb_tracks']
result['downloaded'] = 0
result['failed'] = 0
result['progress'] = 0
result['type'] = 'album'
result['id'] = id
result['bitrate'] = bitrate
result['uuid'] = f"{result['type']}:{id}:{bitrate}"
result['settings'] = settings or {}
result['collection'] = []
for pos, trackAPI in enumerate(tracksArray, start=1):
trackAPI['_EXTRA_ALBUM'] = albumAPI
trackAPI['POSITION'] = pos
trackAPI['FILENAME_TEMPLATE'] = settings['albumTracknameTemplate']
result['collection'].append(trackAPI)
elif type == "playlist":
playlistAPI = dz.get_playlist(id)
playlistTracksAPI = dz.get_playlist_tracks_gw(id)
result['title'] = playlistAPI['title']
result['artist'] = playlistAPI['creator']['name']
result['cover'] = playlistAPI['picture_small'][:-24]+'/128x128-000000-80-0-0.jpg'
result['size'] = playlistAPI['nb_tracks']
result['downloaded'] = 0
result['failed'] = 0
result['progress'] = 0
result['type'] = 'playlist'
result['id'] = id
result['bitrate'] = bitrate
result['uuid'] = f"{result['type']}:{id}:{bitrate}"
result['settings'] = settings or {}
result['collection'] = []
for pos, trackAPI in enumerate(playlistTracksAPI, start=1):
trackAPI['_EXTRA_PLAYLIST'] = playlistAPI
trackAPI['POSITION'] = pos
trackAPI['FILENAME_TEMPLATE'] = settings['playlistTracknameTemplate']
result['collection'].append(trackAPI)
elif type == "artist":
artistAPI = dz.get_artist_albums(id)
albumList = []
for album in artistAPI['data']:
albumList.append(generateQueueItem(dz, album['link'], settings, bitrate))
return albumList
elif type == "spotifytrack":
track_id = get_trackid_spotify(dz, id, settings['fallbackSearch'])
result = {}
if track_id == "Not Enabled":
print("Spotify Features is not setted up correctly.")
result['error'] = "Spotify Features is not setted up correctly."
elif track_id != 0:
return generateQueueItem(dz, f'https://www.deezer.com/track/{track_id}', settings, bitrate)
else:
print("Track not found on deezer!")
result['error'] = "Track not found on deezer!"
elif type == "spotifyalbum":
album_id = get_albumid_spotify(dz, id)
if album_id == "Not Enabled":
print("Spotify Features is not setted up correctly.")
result['error'] = "Spotify Features is not setted up correctly."
elif album_id != 0:
return generateQueueItem(dz, f'https://www.deezer.com/album/{track_id}', settings, bitrate)
else:
print("Album not found on deezer!")
result['error'] = "Album not found on deezer!"
else:
print("URL not supported yet")
result['error'] = "URL not supported yet"
return result
def addToQueue(dz, url, settings, bitrate=None, socket=None):
global currentItem, currentJob, queueList, queue
queueItem = generateQueueItem(dz, url, settings, bitrate)
if 'error' in queueItem:
if socket:
socket.emit("message", queueItem['error'])
return None
if queueItem['uuid'] in list(queueList.keys()):
print("Already in queue!")
if socket:
socket.emit("message", "Already in queue!")
return None
if type(queueItem) is list:
for x in queueItem:
if socket:
socket.emit("addedToQueue", x)
queue.append(x['uuid'])
queueList[x['uuid']] = x
else:
if socket:
socket.emit("addedToQueue", queueItem)
queue.append(queueItem['uuid'])
queueList[queueItem['uuid']] = queueItem
nextItem(dz, socket)
def nextItem(dz, socket=None):
global currentItem, currentJob, queueList, queue
if currentItem != "":
return None
else:
if len(queue)>0:
currentItem = queue.pop(0)
else:
return None
if socket:
socket.emit("message", f"Started downloading {currentItem}")
result = download(dz, queueList[currentItem], socket)
callbackQueueDone(result)
def callbackQueueDone(result):
global currentItem, currentJob, queueList, queue
result['socket']
del queueList[currentItem]
currentItem = ""
nextItem(result['dz'], result['socket'])

View File

@ -5,3 +5,4 @@ requests
spotipy spotipy
pywebview pywebview
flask flask
flask-socketio