#!/usr/bin/env python3 import os.path import re from requests import get from requests.exceptions import HTTPError, ConnectionError from concurrent.futures import ThreadPoolExecutor from os import makedirs, remove, system as execute from tempfile import gettempdir from time import sleep from deemix.app.queueitem import QIConvertable, QISingle, QICollection from deemix.app.Track import Track from deemix.utils.misc import changeCase from deemix.utils.pathtemplates import generateFilename, generateFilepath, settingsRegexAlbum, settingsRegexArtist, settingsRegexPlaylistFile from deemix.api.deezer import USER_AGENT_HEADER from deemix.utils.taggers import tagID3, tagFLAC from Cryptodome.Cipher import Blowfish from mutagen.flac import FLACNoHeaderError import logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger('deemix') TEMPDIR = os.path.join(gettempdir(), 'deemix-imgs') if not os.path.isdir(TEMPDIR): makedirs(TEMPDIR) extensions = { 9: '.flac', 0: '.mp3', 3: '.mp3', 1: '.mp3', 8: '.mp3', 15: '.mp4', 14: '.mp4', 13: '.mp4' } errorMessages = { 'notOnDeezer': "Track not available on Deezer!", 'notEncoded': "Track not yet encoded!", 'notEncodedNoAlternative': "Track not yet encoded and no alternative found!", 'wrongBitrate': "Track not found at desired bitrate.", 'wrongBitrateNoAlternative': "Track not found at desired bitrate and no alternative found!", 'no360RA': "Track is not available in Reality Audio 360.", 'notAvailable': "Track not available on deezer's servers!" 'notAvailableNoAlternative': "Track not available on deezer's servers and no alternative found!", } def after_download(tracks, settings, queueItem): extrasPath = None playlist = [None] * len(tracks) playlistCover = None playlistURLs = [] errors = "" searched = "" for index in range(len(tracks)): result = tracks[index].result() if 'cancel' in result: return None if 'error' in result: if not 'data' in result['error']: result['error']['data'] = {'id': 0, 'title': 'Unknown', 'artist': 'Unknown'} errors += f"{result['error']['data']['id']} | {result['error']['data']['artist']} - {result['error']['data']['title']} | {result['error']['message']}\r\n" if 'searched' in result: searched += result['searched'] + "\r\n" if not extrasPath and 'extrasPath' in result: extrasPath = result['extrasPath'] if not playlistCover and 'playlistCover' in result: playlistCover = result['playlistCover'] playlistURLs = result['playlistURLs'] if settings['saveArtwork'] and 'albumPath' in result: for image in result['albumURLs']: downloadImage(image['url'], f"{result['albumPath']}.{image['ext']}", settings['overwriteFile']) if settings['saveArtworkArtist'] and 'artistPath' in result: for image in result['artistURLs']: downloadImage(image['url'], f"{result['artistPath']}.{image['ext']}", settings['overwriteFile']) if 'playlistPosition' in result: playlist[index] = result['playlistPosition'] else: playlist[index] = "" if not extrasPath: extrasPath = settings['downloadLocation'] if settings['logErrors'] and errors != "": with open(os.path.join(extrasPath, 'errors.txt'), 'wb') as f: f.write(errors.encode('utf-8')) if settings['saveArtwork'] and playlistCover and not settings['tags']['savePlaylistAsCompilation']: for image in playlistURLs: downloadImage(image['url'], os.path.join(extrasPath, playlistCover)+f".{image['ext']}", settings['overwriteFile']) if settings['logSearched'] and searched != "": with open(os.path.join(extrasPath, 'searched.txt'), 'wb') as f: f.write(searched.encode('utf-8')) if settings['createM3U8File']: filename = settingsRegexPlaylistFile(settings['playlistFilenameTemplate'], queueItem, settings) or "playlist" with open(os.path.join(extrasPath, filename+'.m3u8'), 'wb') as f: for line in playlist: f.write((line + "\n").encode('utf-8')) if settings['executeCommand'] != "": execute(settings['executeCommand'].replace("%folder%", extrasPath)) return extrasPath def after_download_single(track, settings): if 'cancel' in track: return None if 'extrasPath' not in track: track['extrasPath'] = settings['downloadLocation'] if settings['saveArtwork'] and 'albumPath' in track: for image in track['albumURLs']: downloadImage(image['url'], f"{track['albumPath']}.{image['ext']}", settings['overwriteFile']) if settings['saveArtworkArtist'] and 'artistPath' in track: for image in track['artistURLs']: downloadImage(image['url'], f"{track['artistPath']}.{image['ext']}", settings['overwriteFile']) if settings['logSearched'] and 'searched' in track: with open(os.path.join(track['extrasPath'], 'searched.txt'), 'wb+') as f: orig = f.read().decode('utf-8') if not track['searched'] in orig: if orig != "": orig += "\r\n" orig += track['searched'] + "\r\n" f.write(orig.encode('utf-8')) if settings['executeCommand'] != "": execute(settings['executeCommand'].replace("%folder%", track['extrasPath']).replace("%filename%", track['playlistPosition'])) return track['extrasPath'] def downloadImage(url, path, overwrite="n"): if not os.path.isfile(path) or overwrite in ['y', 't', 'b']: try: image = get(url, headers={'User-Agent': USER_AGENT_HEADER}, timeout=30) image.raise_for_status() with open(path, 'wb') as f: f.write(image.content) return path except HTTPError: if 'cdns-images.dzcdn.net' in url: urlBase = url[:url.rfind("/")+1] pictureUrl = url[len(urlBase):] pictureSize = int(pictureUrl[:pictureUrl.find("x")]) if pictureSize > 1200: logger.warn("Couldn't download "+str(pictureSize)+"x"+str(pictureSize)+" image, falling back to 1200x1200") sleep(1) return downloadImage(urlBase+pictureUrl.replace(str(pictureSize)+"x"+str(pictureSize), '1200x1200'), path, overwrite) logger.error("Couldn't download Image: "+url) except: sleep(1) return downloadImage(url, path, overwrite) remove(path) return None else: return path def formatDate(date, template): elements = { 'year': ['YYYY', 'YY', 'Y'], 'month': ['MM', 'M'], 'day': ['DD', 'D'] } for element, placeholders in elements.items(): for placeholder in placeholders: if placeholder in template: template = template.replace(placeholder, str(date[element])) return template class DownloadJob: def __init__(self, dz, sp, queueItem, interface=None): self.dz = dz self.sp = sp self.queueItem = queueItem self.interface = interface self.settings = queueItem.settings self.bitrate = queueItem.bitrate self.downloadPercentage = 0 self.lastPercentage = 0 self.extrasPath = self.settings['downloadLocation'] def start(self): if isinstance(self.queueItem, QIConvertable): self.sp.convert_spotify_playlist(self.dz, self.queueItem, self.settings, interface=self.interface) if isinstance(self.queueItem, QISingle): result = self.downloadWrapper(self.queueItem.single) if result: download_path = after_download_single(result, self.settings) elif isinstance(self.queueItem, QICollection): playlist = [None] * len(self.queueItem.collection) with ThreadPoolExecutor(self.settings['queueConcurrency']) as executor: for pos, track in enumerate(self.queueItem.collection, start=0): playlist[pos] = executor.submit(self.downloadWrapper, track) download_path = after_download(playlist, self.settings, self.queueItem) if self.interface: if self.queueItem.cancel: self.interface.send('currentItemCancelled', self.queueItem.uuid) self.interface.send("removedFromQueue", self.queueItem.uuid) else: self.interface.send("finishDownload", self.queueItem.uuid) return download_path def download(self, trackAPI_gw, track=None): result = {} if self.queueItem.cancel: raise DownloadCancelled if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer") # Create Track object if not track: logger.info(f"[{trackAPI_gw['ART_NAME']} - {trackAPI_gw['SNG_TITLE']}] Getting the tags") track = Track(self.dz, settings=self.settings, trackAPI_gw=trackAPI_gw, trackAPI=trackAPI_gw['_EXTRA_TRACK'] if '_EXTRA_TRACK' in trackAPI_gw else None, albumAPI=trackAPI_gw['_EXTRA_ALBUM'] if '_EXTRA_ALBUM' in trackAPI_gw else None ) if self.queueItem.cancel: raise DownloadCancelled if self.MD5 == '': if track.fallbackId != "0": logger.warn(f"[{track.mainArtist['name']} - {track.title}] Track not yet encoded, using fallback id") newTrack = self.dz.get_track_gw(track.fallbackId) track.parseEssentialData(self.dz, newTrack) return self.download(trackAPI_gw, track) elif not track.searched and self.settings['fallbackSearch']: logger.warn(f"[{self.mainArtist['name']} - {self.title}] Track not yet encoded, searching for alternative") searchedId = self.dz.get_track_from_metadata(self.mainArtist['name'], self.title, self.album['title']) if searchedId != 0: newTrack = self.dz.get_track_gw(searchedId) track.parseEssentialData(self.dz, newTrack) track.searched = True return self.download(trackAPI_gw, track) else: raise DownloadFailed("notEncodedNoAlternative") else: raise DownloadFailed("notEncoded") selectedFormat = self.getPreferredBitrate(track) if selectedFormat == -100: if track.fallbackId != "0": logger.warn(f"[{track.mainArtist['name']} - {track.title}] Track not found at desired bitrate, using fallback id") newTrack = self.dz.get_track_gw(track.fallbackId) track.parseEssentialData(self.dz, newTrack) return self.download(trackAPI_gw, track) elif not track.searched and self.settings['fallbackSearch']: logger.warn(f"[{self.mainArtist['name']} - {self.title}] Track not found at desired bitrate, searching for alternative") searchedId = self.dz.get_track_from_metadata(self.mainArtist['name'], self.title, self.album['title']) if searchedId != 0: newTrack = self.dz.get_track_gw(searchedId) track.parseEssentialData(self.dz, newTrack) track.searched = True return self.download(trackAPI_gw, track) else: raise DownloadFailed("wrongBitrateNoAlternative") else: raise DownloadFailed("wrongBitrate") elif selectedFormat == -200: raise DownloadFailed("no360RA") track.selectedFormat = selectedFormat if self.settings['tags']['savePlaylistAsCompilation'] and track.playlist: track.trackNumber = track.position track.discNumber = "1" track.album = {**track.album, **track.playlist} track.album['picPath'] = os.path.join(TEMPDIR, f"pl{trackAPI_gw['_EXTRA_PLAYLIST']['id']}_{settings['embeddedArtworkSize']}.jpg") else: if track.album['date']: track.date = track.album['date'] track.album['picUrl'] = "https://e-cdns-images.dzcdn.net/images/cover/{}/{}x{}-{}".format( track.album['pic'], self.settings['embeddedArtworkSize'], self.settings['embeddedArtworkSize'], f'000000-{self.settings["jpegImageQuality"]}-0-0.jpg' ) track.album['bitrate'] = selectedFormat track.dateString = formatDate(track.date, settings['dateFormat']) track.album['dateString'] = formatDate(track.album['date'], settings['dateFormat']) # Check if user wants the feat in the title # 0 => do not change # 1 => remove from title # 2 => add to title # 3 => remove from title and album title if self.settings['featuredToTitle'] == "1": track.title = track.getCleanTitle() elif self.settings['featuredToTitle'] == "2": track.title = track.getFeatTitle() elif self.settings['featuredToTitle'] == "3": track.title = track.getCleanTitle() track.album['title'] = track.getCleanAlbumTitle() # Remove (Album Version) from tracks that have that if self.settings['removeAlbumVersion']: if "Album Version" in track.title: track.title = re.sub(r' ?\(Album Version\)', "", track.title).strip() # Generate artist tag if needed if self.settings['tags']['multiArtistSeparator'] != "default": if self.settings['tags']['multiArtistSeparator'] == "andFeat": track.artistsString = track.mainArtistsString if track.featArtistsString and self.settings['featuredToTitle'] != "2": track.artistsString += " " + track.featArtistsString else: track.artistsString = self.settings['tags']['multiArtistSeparator'].join(track.artists) else: track.artistsString = ", ".join(track.artists) # Change Title and Artists casing if needed if self.settings['titleCasing'] != "nothing": track.title = changeCase(track.title, self.settings['titleCasing']) if self.settings['artistCasing'] != "nothing": track.artistsString = changeCase(track.artistsString, self.settings['artistCasing']) for i, artist in enumerate(track.artists): track.artists[i] = changeCase(artist, self.settings['artistCasing']) # Generate filename and filepath from metadata filename = generateFilename(track, trackAPI_gw, self.settings) (filepath, artistPath, coverPath, extrasPath) = generateFilepath(track, trackAPI_gw, self.settings) if self.queueItem.cancel: raise DownloadCancelled # Download and cache coverart if self.settings['tags']['savePlaylistAsCompilation'] and track.playlist: else: track.album['picPath'] = os.path.join(TEMPDIR, f"alb{track.album['id']}_{settings['embeddedArtworkSize']}.jpg") logger.info(f"[{track.mainArtist['name']} - {track.title}] Getting the album cover") track.album['picPath'] = downloadImage(track.album['picUrl'], track.album['picPath']) # Save local album art if coverPath: result['albumURLs'] = [] for format in self.settings['localArtworkFormat'].split(","): if format in ["png","jpg"]: url = track.album['picUrl'].replace( f"{self.settings['embeddedArtworkSize']}x{self.settings['embeddedArtworkSize']}", f"{self.settings['localArtworkSize']}x{self.settings['localArtworkSize']}") if format == "png": url = url[:url.find("000000-")]+"none-100-0-0.png" result['albumURLs'].append({'url': url, 'ext': format}) result['albumPath'] = os.path.join(coverPath, f"{settingsRegexAlbum(self.settings['coverImageTemplate'], track.album, self.settings, trackAPI_gw['_EXTRA_PLAYLIST'] if'_EXTRA_PLAYLIST' in trackAPI_gw else None)}") # Save artist art if artistPath: result['artistURLs'] = [] for format in self.settings['localArtworkFormat'].split(","): if format in ["png","jpg"]: url = "" if track.album['mainArtist']['pic'] != "": url = "https://e-cdns-images.dzcdn.net/images/artist/{}/{}x{}-{}".format( track.album['mainArtist']['pic'], self.settings['localArtworkSize'], self.settings['localArtworkSize'], 'none-100-0-0.png' if format == "png" else f'000000-{self.settings["jpegImageQuality"]}-0-0.jpg') elif format == "jpg": url = "https://e-cdns-images.dzcdn.net/images/artist//{}x{}-{}".format( self.settings['localArtworkSize'], self.settings['localArtworkSize'], f'000000-{self.settings["jpegImageQuality"]}-0-0.jpg') if url: result['artistURLs'].append({'url': url, 'ext': format}) result['artistPath'] = os.path.join(artistPath, f"{settingsRegexArtist(self.settings['artistImageTemplate'], track.album['mainArtist'], self.settings)}") # Remove subfolders from filename and add it to filepath if os.path.sep in filename: tempPath = filename[:filename.rfind(os.path.sep)] filepath = os.path.join(filepath, tempPath) filename = filename[filename.rfind(os.path.sep) + len(os.path.sep):] # Make sure the filepath exsists makedirs(filepath, exist_ok=True) writepath = os.path.join(filepath, filename + extensions[track.selectedFormat]) # Save lyrics in lrc file if self.settings['syncedLyrics'] and 'sync' in track.lyrics: if not os.path.isfile(os.path.join(filepath, filename + '.lrc')) or settings['overwriteFile'] in ['y', 't']: with open(os.path.join(filepath, filename + '.lrc'), 'wb') as f: f.write(track.lyrics['sync'].encode('utf-8')) trackAlreadyDownloaded = os.path.isfile(writepath) if trackAlreadyDownloaded and self.settings['overwriteFile'] == 'b': baseFilename = os.path.join(filepath, filename) i = 1 currentFilename = baseFilename+' ('+str(i)+')'+ extensions[track.selectedFormat] while os.path.isfile(currentFilename): i += 1 currentFilename = baseFilename+' ('+str(i)+')'+ extensions[track.selectedFormat] trackAlreadyDownloaded = False writepath = currentFilename if extrasPath: if not self.extrasPath: self.extrasPath = extrasPath result['extrasPath'] = extrasPath # Data for m3u file result['playlistPosition'] = writepath[len(extrasPath):] # Save playlist cover if track.playlist: result['playlistURLs'] = [] if 'dzcdn.net' in track.playlist['picUrl']: for format in self.settings['localArtworkFormat'].split(","): if format in ["png","jpg"]: url = track.playlist['picUrl'].replace( f"{self.settings['embeddedArtworkSize']}x{self.settings['embeddedArtworkSize']}", f"{self.settings['localArtworkSize']}x{self.settings['localArtworkSize']}") if format == "png": url = url[:url.find("000000-")]+"none-100-0-0.png" result['playlistURLs'].append({'url': url, 'ext': format}) else: result['playlistURLs'].append({'url': track.playlist['picUrl'], 'ext': 'jpg'}) track.playlist['id'] = "pl_" + str(trackAPI_gw['_EXTRA_PLAYLIST']['id']) track.playlist['genre'] = ["Compilation", ] track.playlist['bitrate'] = selectedFormat track.playlist['dateString'] = formatDate(track.playlist['date'], self.settings['dateFormat']) result['playlistCover'] = f"{settingsRegexAlbum(self.settings['coverImageTemplate'], track.playlist, self.settings, trackAPI_gw['_EXTRA_PLAYLIST'])}" if not trackAlreadyDownloaded or self.settings['overwriteFile'] == 'y': logger.info(f"[{track.mainArtist['name']} - {track.title}] Downloading the track") track.downloadUrl = dz.get_track_stream_url(track.id, track.MD5, track.mediaVersion, track.selectedFormat) def downloadMusic(track, trackAPI_gw): try: with open(writepath, 'wb') as stream: self.streamTrack(stream, track, trackAPI_gw) except DownloadCancelled: remove(writepath) raise DownloadCancelled except HTTPError: remove(writepath) if track.fallbackId != "0": logger.warn(f"[{track.mainArtist['name']} - {track.title}] Track not available, using fallback id") newTrack = self.dz.get_track_gw(track.fallbackId) track.parseEssentialData(self.dz, newTrack) return False elif not track.searched and self.settings['fallbackSearch']: logger.warn(f"[{self.mainArtist['name']} - {self.title}] Track not available, searching for alternative") searchedId = self.dz.get_track_from_metadata(self.mainArtist['name'], self.title, self.album['title']) if searchedId != 0: newTrack = self.dz.get_track_gw(searchedId) track.parseEssentialData(self.dz, newTrack) track.searched = True return False else: raise DownloadFailed("notAvailableNoAlternative") else: raise DownloadFailed("notAvailable") except ConnectionError as e: logger.exception(str(e)) logger.warn(f"[{track.mainArtist['name']} - {track.title}] Error while downloading the track, trying again in 5s...") sleep(5) return downloadMusic(track, trackAPI_gw) except Exception as e: logger.exception(str(e)) logger.warn(f"[{track.mainArtist['name']} - {track.title}] Error while downloading the track, you should report this to the developers") raise e return True try: trackDownloaded = downloadMusic(track, trackAPI_gw) except DownloadFailed as e: raise DownloadFailed except Exception as e: raise e if not trackDownloaded: return self.download(trackAPI_gw, track) else: logger.info(f"[{track.mainArtist['name']} - {track.title}] Skipping track as it's already downloaded") trackCompletePercentage(trackAPI, queueItem, interface) # Adding tags if (not trackAlreadyDownloaded or self.settings['overwriteFile'] in ['t', 'y']) and not track.localTrack: logger.info(f"[{track.mainArtist['name']} - {track.title}] Applying tags to the track") if track.selectedFormat in [3, 1, 8]: tagID3(writepath, track, self.settings['tags']) elif track.selectedFormat == 9: try: tagFLAC(writepath, track, self.settings['tags']) except FLACNoHeaderError: remove(writepath) logger.warn(f"[{track.mainArtist['name']} - {track.title}] Track not available in FLAC, falling back if necessary") self.removeTrackPercentage(trackAPI, queueItem, interface) track.formats['FILESIZE_FLAC'] = "0" return self.download(trackAPI_gw, track) if track.searched: result['searched'] = f'{track.mainArtist['name']} - {track.title}' logger.info(f"[{track.mainArtist['name']} - {track.title}] Track download completed") self.queueItem.downloaded += 1 if self.interface: self.interface.send("updateQueue", {'uuid': queueItem.uuid, 'downloaded': True, 'downloadPath': writepath}) return result def getPreferredBitrate(self, track): if track.localTrack: return 0 fallback = self.settings['fallbackBitrate'] formats_non_360 = { 9: "FLAC", 3: "MP3_320", 1: "MP3_128", } formats_360 = { 15: "MP4_RA3", 14: "MP4_RA2", 13: "MP4_RA1", } if not fallback: error_num = -100 formats = formats_360 formats.update(formats_non_360) elif int(self.bitrate) in formats_360: error_num = -200 formats = formats_360 else: error_num = 8 formats = formats_non_360 for format_num, format in formats.items(): if format_num <= int(self.bitrate): if f"FILESIZE_{format}" in track.filesizes and int(track.filesizes[f"FILESIZE_{format}"]) != 0: return format_num else: if fallback: continue else: return error_num return error_num # fallback is enabled and loop went through all formats def stream_track(self, stream, track, trackAPI): if self.queueItem.cancel: raise DownloadCancelled try: request = get(track.downloadUrl, headers=dz.http_headers, stream=True, timeout=30) except ConnectionError: sleep(2) return stream_track(dz, track, stream, trackAPI, queueItem, interface) request.raise_for_status() blowfish_key = str.encode(dz._get_blowfish_key(str(track.id))) complete = int(request.headers["Content-Length"]) chunkLength = 0 percentage = 0 i = 0 for chunk in request.iter_content(2048): if self.queueItem.cancel: raise DownloadCancelled if i % 3 == 0 and len(chunk) == 2048: chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk) stream.write(chunk) chunkLength += len(chunk) if 'SINGLE_TRACK' in trackAPI: percentage = (chunkLength / complete) * 100 self.downloadPercentage = percentage else: chunkProgres = (len(chunk) / complete) / trackAPI['SIZE'] * 100 self.downloadPercentage += chunkProgres self.updatePercentage() i += 1 def updatePercentage(self): if round(self.downloadPercentage) != self.lastPercentage and round(self.downloadPercentage) % 2 == 0: self.lastPercentage = round(self.downloadPercentage) self.queueItem.progress = self.lastPercentage if self.interface: self.interface.send("updateQueue", {'uuid': self.queueItem.uuid, 'progress': self.lastPercentage}) def completeTrackPercentage(self): if isinstance(self.queueItem, QISingle): self.downloadPercentage = 100 else: self.downloadPercentage += (1 / self.queueItem.size) * 100 self.updatePercentage() def removeTrackPercentage(self): if isinstance(self.queueItem, QISingle): self.downloadPercentage = 0 else: self.downloadPercentage -= (1 / self.queueItem.size) * 100 self.updatePercentage() def downloadWrapper(self, trackAPI_gw): track = { 'id': queueItem.single['SNG_ID'], 'title': queueItem.single['SNG_TITLE'] + (queueItem.single['VERSION'] if 'VERSION' in queueItem.single and queueItem.single['VERSION'] and not queueItem.single['VERSION'] in queueItem.single['SNG_TITLE'] else ""), 'mainArtist': {'name': queueItem.single['ART_NAME']} } try: result = self.download(trackAPI_gw) except DownloadCancelled: return None except DownloadFailed as error: logger.error(f"[{track['mainArtist']['name']} - {track['title']}] {error.message}") result = {'error': { 'message': error.message, 'errid': error.errid, 'data': track }} except Exception as e: logger.exception(str(e)) result = {'error': { 'message': str(e), 'data': track }} if 'error' in result: self.completeTrackPercentage() self.queueItem.failed += 1 self.queueItem.errors.append(error.message) if interface: error = result['error'] interface.send("updateQueue", { 'uuid': self.queueItem.uuid, 'failed': True, 'data': error['data'], 'error': error['message'], 'errid': error['errid'] if 'errid' in error else None }) return result class DownloadError(Exception): """Base class for exceptions in this module.""" pass class DownloadFailed(DownloadError): def __init__(self, errid): self.errid = errid self.message = errorMessages[self.errid] class DownloadCancelled(DownloadError): pass