From f530a4e89f930fb36db2ee2b9a4454bce825d3c9 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Sun, 27 Jun 2021 16:29:41 -0400 Subject: [PATCH] Merge refactoring (#4) Removed saveDownloadQueue and tagsLanguage from lib settings Revert embedded cover change Fixed bitrate fallback check Use overwriteFile setting when downloading embedded covers Fixed bitrate fallback not working Fixed some issues to make the lib work Implemented spotify plugin back Better handling of albums upcs Fixed queue item not cancelling correctly Code parity with deemix-js Code cleanup with pylint Even more rework on the library More work on the library (WIP) Total rework of the library (WIP) Some rework done on types Added start queue function Made nextitem work on a thread Removed dz as first parameter Started queuemanager refactoring Removed eventlet Co-authored-by: RemixDev Reviewed-on: https://git.freezer.life/RemixDev/deemix-py/pulls/4 Co-Authored-By: RemixDev Co-Committed-By: RemixDev --- .gitignore | 1 + .pylintrc | 2 + deemix/__init__.py | 79 ++- deemix/__main__.py | 59 +- deemix/app/__init__.py | 12 - deemix/app/cli.py | 40 -- deemix/app/downloadjob.py | 767 ------------------------- deemix/app/messageinterface.py | 4 - deemix/app/queueitem.py | 115 ---- deemix/app/queuemanager.py | 569 ------------------ deemix/app/settings.py | 220 ------- deemix/app/spotifyhelper.py | 349 ----------- deemix/decryption.py | 156 +++++ deemix/downloader.py | 564 ++++++++++++++++++ deemix/itemgen.py | 307 ++++++++++ deemix/plugins/__init__.py | 12 + deemix/plugins/spotify.py | 346 +++++++++++ deemix/settings.py | 137 +++++ deemix/{utils/taggers.py => tagger.py} | 10 +- deemix/types/Album.py | 83 +-- deemix/types/Artist.py | 10 +- deemix/types/Date.py | 8 +- deemix/types/DownloadObjects.py | 126 ++++ deemix/types/Lyrics.py | 16 +- deemix/types/Picture.py | 50 +- deemix/types/Playlist.py | 36 +- deemix/types/Track.py | 168 ++++-- deemix/types/__init__.py | 8 +- deemix/utils/__init__.py | 110 +--- deemix/utils/crypto.py | 26 + deemix/utils/decryption.py | 31 - deemix/utils/deezer.py | 32 ++ deemix/utils/localpaths.py | 88 ++- deemix/utils/pathtemplates.py | 144 +++-- requirements.txt | 1 - setup.py | 9 +- updatePyPi.sh | 4 +- 37 files changed, 2236 insertions(+), 2463 deletions(-) create mode 100644 .pylintrc delete mode 100644 deemix/app/__init__.py delete mode 100644 deemix/app/cli.py delete mode 100644 deemix/app/downloadjob.py delete mode 100644 deemix/app/messageinterface.py delete mode 100644 deemix/app/queueitem.py delete mode 100644 deemix/app/queuemanager.py delete mode 100644 deemix/app/settings.py delete mode 100644 deemix/app/spotifyhelper.py create mode 100644 deemix/decryption.py create mode 100644 deemix/downloader.py create mode 100644 deemix/itemgen.py create mode 100644 deemix/plugins/__init__.py create mode 100644 deemix/plugins/spotify.py create mode 100644 deemix/settings.py rename deemix/{utils/taggers.py => tagger.py} (98%) create mode 100644 deemix/types/DownloadObjects.py create mode 100644 deemix/utils/crypto.py delete mode 100644 deemix/utils/decryption.py create mode 100644 deemix/utils/deezer.py diff --git a/.gitignore b/.gitignore index d6fb3b0..3a64c4a 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ yarn-error.log* /build /*egg-info updatePyPi.sh +/deezer diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..cf11d0f --- /dev/null +++ b/.pylintrc @@ -0,0 +1,2 @@ +[MESSAGES CONTROL] +disable=C0301,C0103,R0902,R0903,C0321,R0911,R0912,R0913,R0914,R0915,R0916 diff --git a/deemix/__init__.py b/deemix/__init__.py index de69d03..d3aa8d6 100644 --- a/deemix/__init__.py +++ b/deemix/__init__.py @@ -1,6 +1,77 @@ #!/usr/bin/env python3 +import re +from urllib.request import urlopen -__version__ = "2.0.16" -USER_AGENT_HEADER = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) " \ - "Chrome/79.0.3945.130 Safari/537.36" -VARIOUS_ARTISTS = "5080" +from deemix.itemgen import generateTrackItem, \ + generateAlbumItem, \ + generatePlaylistItem, \ + generateArtistItem, \ + generateArtistDiscographyItem, \ + generateArtistTopItem, \ + LinkNotRecognized, \ + LinkNotSupported + +__version__ = "3.0.0" + +# Returns the Resolved URL, the Type and the ID +def parseLink(link): + if 'deezer.page.link' in link: link = urlopen(link).url # Resolve URL shortner + # Remove extra stuff + if '?' in link: link = link[:link.find('?')] + if '&' in link: link = link[:link.find('&')] + if link.endswith('/'): link = link[:-1] # Remove last slash if present + + link_type = None + link_id = None + + if not 'deezer' in link: return (link, link_type, link_id) # return if not a deezer link + + if '/track' in link: + link_type = 'track' + link_id = re.search(r"/track/(.+)", link).group(1) + elif '/playlist' in link: + link_type = 'playlist' + link_id = re.search(r"/playlist/(\d+)", link).group(1) + elif '/album' in link: + link_type = 'album' + link_id = re.search(r"/album/(.+)", link).group(1) + elif re.search(r"/artist/(\d+)/top_track", link): + link_type = 'artist_top' + link_id = re.search(r"/artist/(\d+)/top_track", link).group(1) + elif re.search(r"/artist/(\d+)/discography", link): + link_type = 'artist_discography' + link_id = re.search(r"/artist/(\d+)/discography", link).group(1) + elif '/artist' in link: + link_type = 'artist' + link_id = re.search(r"/artist/(\d+)", link).group(1) + + return (link, link_type, link_id) + +def generateDownloadObject(dz, link, bitrate, plugins=None, listener=None): + (link, link_type, link_id) = parseLink(link) + + if link_type is None or link_id is None: + if plugins is None: plugins = {} + plugin_names = plugins.keys() + current_plugin = None + item = None + for plugin in plugin_names: + current_plugin = plugins[plugin] + item = current_plugin.generateDownloadObject(dz, link, bitrate, listener) + if item: return item + raise LinkNotRecognized(link) + + if link_type == "track": + return generateTrackItem(dz, link_id, bitrate) + if link_type == "album": + return generateAlbumItem(dz, link_id, bitrate) + if link_type == "playlist": + return generatePlaylistItem(dz, link_id, bitrate) + if link_type == "artist": + return generateArtistItem(dz, link_id, bitrate, listener) + if link_type == "artist_discography": + return generateArtistDiscographyItem(dz, link_id, bitrate, listener) + if link_type == "artist_top": + return generateArtistTopItem(dz, link_id, bitrate) + + raise LinkNotSupported(link) diff --git a/deemix/__main__.py b/deemix/__main__.py index 35bb938..7885794 100644 --- a/deemix/__main__.py +++ b/deemix/__main__.py @@ -1,37 +1,76 @@ #!/usr/bin/env python3 import click - -from deemix.app.cli import cli from pathlib import Path +from deezer import Deezer +from deezer import TrackFormats + +from deemix import generateDownloadObject +from deemix.settings import load as loadSettings +from deemix.utils import getBitrateNumberFromText +import deemix.utils.localpaths as localpaths +from deemix.downloader import Downloader + @click.command() @click.option('--portable', is_flag=True, help='Creates the config folder in the same directory where the script is launched') @click.option('-b', '--bitrate', default=None, help='Overwrites the default bitrate selected') @click.option('-p', '--path', type=str, help='Downloads in the given folder') @click.argument('url', nargs=-1, required=True) def download(url, bitrate, portable, path): - + # Check for local configFolder localpath = Path('.') - configFolder = localpath / 'config' if portable else None + configFolder = localpath / 'config' if portable else localpaths.getConfigFolder() + + settings = loadSettings(configFolder) + dz = Deezer(settings.get('tagsLanguage', "")) + + def requestValidArl(): + while True: + arl = input("Paste here your arl:") + if dz.login_via_arl(arl.strip()): break + return arl + + if (configFolder / '.arl').is_file(): + with open(configFolder / '.arl', 'r') as f: + arl = f.readline().rstrip("\n").strip() + if not dz.login_via_arl(arl): arl = requestValidArl() + else: arl = requestValidArl() + with open(configFolder / '.arl', 'w') as f: + f.write(arl) + + def downloadLinks(url, bitrate=None): + if not bitrate: bitrate = settings.get("maxBitrate", TrackFormats.MP3_320) + links = [] + for link in url: + if ';' in link: + for l in link.split(";"): + links.append(l) + else: + links.append(link) + + for link in links: + downloadObject = generateDownloadObject(dz, link, bitrate) + Downloader(dz, downloadObject, settings).start() + if path is not None: if path == '': path = '.' path = Path(path) - - app = cli(path, configFolder) - app.login() + settings['downloadLocation'] = str(path) url = list(url) + if bitrate: bitrate = getBitrateNumberFromText(bitrate) + # If first url is filepath readfile and use them as URLs try: isfile = Path(url[0]).is_file() - except: + except Exception: isfile = False if isfile: filename = url[0] with open(filename) as f: url = f.readlines() - app.downloadLink(url, bitrate) + downloadLinks(url, bitrate) click.echo("All done!") if __name__ == '__main__': - download() + download() # pylint: disable=E1120 diff --git a/deemix/app/__init__.py b/deemix/app/__init__.py deleted file mode 100644 index 9b628bb..0000000 --- a/deemix/app/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from deezer import Deezer -from deemix.app.settings import Settings -from deemix.app.queuemanager import QueueManager -from deemix.app.spotifyhelper import SpotifyHelper - -class deemix: - def __init__(self, configFolder=None, overwriteDownloadFolder=None): - self.set = Settings(configFolder, overwriteDownloadFolder=overwriteDownloadFolder) - self.dz = Deezer() - self.dz.set_accept_language(self.set.settings.get('tagsLanguage')) - self.sp = SpotifyHelper(configFolder) - self.qm = QueueManager(self.sp) diff --git a/deemix/app/cli.py b/deemix/app/cli.py deleted file mode 100644 index 379e11f..0000000 --- a/deemix/app/cli.py +++ /dev/null @@ -1,40 +0,0 @@ -from pathlib import Path -from os import makedirs - -from deemix.app import deemix -from deemix.utils import checkFolder - -class cli(deemix): - def __init__(self, downloadpath, configFolder=None): - super().__init__(configFolder, overwriteDownloadFolder=downloadpath) - if downloadpath: - print("Using folder: "+self.set.settings['downloadLocation']) - - def downloadLink(self, url, bitrate=None): - for link in url: - if ';' in link: - for l in link.split(";"): - self.qm.addToQueue(self.dz, l, self.set.settings, bitrate) - else: - self.qm.addToQueue(self.dz, link, self.set.settings, bitrate) - - def requestValidArl(self): - while True: - arl = input("Paste here your arl:") - if self.dz.login_via_arl(arl): - break - return arl - - def login(self): - configFolder = Path(self.set.configFolder) - if not configFolder.is_dir(): - makedirs(configFolder, exist_ok=True) - if (configFolder / '.arl').is_file(): - with open(configFolder / '.arl', 'r') as f: - arl = f.readline().rstrip("\n") - if not self.dz.login_via_arl(arl): - arl = self.requestValidArl() - else: - arl = self.requestValidArl() - with open(configFolder / '.arl', 'w') as f: - f.write(arl) diff --git a/deemix/app/downloadjob.py b/deemix/app/downloadjob.py deleted file mode 100644 index f0e08b4..0000000 --- a/deemix/app/downloadjob.py +++ /dev/null @@ -1,767 +0,0 @@ -import eventlet -from eventlet.green.subprocess import call as execute -requests = eventlet.import_patched('requests') -get = requests.get -request_exception = requests.exceptions - -from os.path import sep as pathSep -from pathlib import Path -from shlex import quote -import re -import errno - -from ssl import SSLError -from os import makedirs -from tempfile import gettempdir -from urllib3.exceptions import SSLError as u3SSLError - -from deemix.app.queueitem import QISingle, QICollection -from deemix.types.Track import Track, AlbumDoesntExists -from deemix.utils import changeCase -from deemix.utils.pathtemplates import generateFilename, generateFilepath, settingsRegexAlbum, settingsRegexArtist, settingsRegexPlaylistFile -from deezer import TrackFormats -from deemix import USER_AGENT_HEADER -from deemix.utils.taggers import tagID3, tagFLAC -from deemix.utils.decryption import generateStreamURL, generateBlowfishKey -from deemix.app.settings import OverwriteOption, FeaturesOption - -from Cryptodome.Cipher import Blowfish -from mutagen.flac import FLACNoHeaderError, error as FLACError -import logging - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger('deemix') - -TEMPDIR = Path(gettempdir()) / 'deemix-imgs' -if not TEMPDIR.is_dir(): makedirs(TEMPDIR) - -extensions = { - TrackFormats.FLAC: '.flac', - TrackFormats.LOCAL: '.mp3', - TrackFormats.MP3_320: '.mp3', - TrackFormats.MP3_128: '.mp3', - TrackFormats.DEFAULT: '.mp3', - TrackFormats.MP4_RA3: '.mp4', - TrackFormats.MP4_RA2: '.mp4', - TrackFormats.MP4_RA1: '.mp4' -} - -errorMessages = { - 'notOnDeezer': "Track not available on Deezer!", - 'notEncoded': "Track not yet encoded!", - 'notEncodedNoAlternative': "Track not yet encoded and no alternative found!", - 'wrongBitrate': "Track not found at desired bitrate.", - 'wrongBitrateNoAlternative': "Track not found at desired bitrate and no alternative found!", - 'no360RA': "Track is not available in Reality Audio 360.", - 'notAvailable': "Track not available on deezer's servers!", - 'notAvailableNoAlternative': "Track not available on deezer's servers and no alternative found!", - 'noSpaceLeft': "No space left on target drive, clean up some space for the tracks", - 'albumDoesntExists': "Track's album does not exsist, failed to gather info" -} - -def downloadImage(url, path, overwrite=OverwriteOption.DONT_OVERWRITE): - if not path.is_file() or overwrite in [OverwriteOption.OVERWRITE, OverwriteOption.ONLY_TAGS, OverwriteOption.KEEP_BOTH]: - try: - image = get(url, headers={'User-Agent': USER_AGENT_HEADER}, timeout=30) - image.raise_for_status() - with open(path, 'wb') as f: - f.write(image.content) - return path - except request_exception.HTTPError: - if 'cdns-images.dzcdn.net' in url: - urlBase = url[:url.rfind("/")+1] - pictureUrl = url[len(urlBase):] - pictureSize = int(pictureUrl[:pictureUrl.find("x")]) - if pictureSize > 1200: - logger.warn("Couldn't download "+str(pictureSize)+"x"+str(pictureSize)+" image, falling back to 1200x1200") - eventlet.sleep(1) - return downloadImage(urlBase+pictureUrl.replace(str(pictureSize)+"x"+str(pictureSize), '1200x1200'), path, overwrite) - logger.error("Image not found: "+url) - except (request_exception.ConnectionError, request_exception.ChunkedEncodingError, u3SSLError) as e: - logger.error("Couldn't download Image, retrying in 5 seconds...: "+url+"\n") - eventlet.sleep(5) - return downloadImage(url, path, overwrite) - except OSError as e: - if e.errno == errno.ENOSPC: - raise DownloadFailed("noSpaceLeft") - else: - logger.exception(f"Error while downloading an image, you should report this to the developers: {str(e)}") - except Exception as e: - logger.exception(f"Error while downloading an image, you should report this to the developers: {str(e)}") - if path.is_file(): path.unlink() - return None - else: - return path - - -class DownloadJob: - def __init__(self, dz, queueItem, interface=None): - self.dz = dz - self.interface = interface - self.queueItem = queueItem - self.settings = queueItem.settings - self.bitrate = queueItem.bitrate - self.downloadPercentage = 0 - self.lastPercentage = 0 - self.extrasPath = None - self.playlistCoverName = None - self.playlistURLs = [] - - def start(self): - if not self.queueItem.cancel: - if isinstance(self.queueItem, QISingle): - result = self.downloadWrapper(self.queueItem.single) - if result: self.singleAfterDownload(result) - elif isinstance(self.queueItem, QICollection): - tracks = [None] * len(self.queueItem.collection) - pool = eventlet.GreenPool(size=self.settings['queueConcurrency']) - for pos, track in enumerate(self.queueItem.collection, start=0): - tracks[pos] = pool.spawn(self.downloadWrapper, track) - pool.waitall() - self.collectionAfterDownload(tracks) - if self.interface: - if self.queueItem.cancel: - self.interface.send('currentItemCancelled', self.queueItem.uuid) - self.interface.send("removedFromQueue", self.queueItem.uuid) - else: - self.interface.send("finishDownload", self.queueItem.uuid) - return self.extrasPath - - def singleAfterDownload(self, result): - if not self.extrasPath: self.extrasPath = Path(self.settings['downloadLocation']) - - # Save Album Cover - if self.settings['saveArtwork'] and 'albumPath' in result: - for image in result['albumURLs']: - downloadImage(image['url'], result['albumPath'] / f"{result['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) - - # Save Artist Artwork - if self.settings['saveArtworkArtist'] and 'artistPath' in result: - for image in result['artistURLs']: - downloadImage(image['url'], result['artistPath'] / f"{result['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) - - # Create searched logfile - if self.settings['logSearched'] and 'searched' in result: - with open(self.extrasPath / 'searched.txt', 'wb+') as f: - orig = f.read().decode('utf-8') - if not result['searched'] in orig: - if orig != "": orig += "\r\n" - orig += result['searched'] + "\r\n" - f.write(orig.encode('utf-8')) - # Execute command after download - if self.settings['executeCommand'] != "": - execute(self.settings['executeCommand'].replace("%folder%", quote(str(self.extrasPath))).replace("%filename%", quote(result['filename'])), shell=True) - - def collectionAfterDownload(self, tracks): - if not self.extrasPath: self.extrasPath = Path(self.settings['downloadLocation']) - playlist = [None] * len(tracks) - errors = "" - searched = "" - - for i in range(len(tracks)): - result = tracks[i].wait() - if not result: return None # Check if item is cancelled - - # Log errors to file - if result.get('error'): - if not result['error'].get('data'): result['error']['data'] = {'id': "0", 'title': 'Unknown', 'artist': 'Unknown'} - errors += f"{result['error']['data']['id']} | {result['error']['data']['artist']} - {result['error']['data']['title']} | {result['error']['message']}\r\n" - - # Log searched to file - if 'searched' in result: searched += result['searched'] + "\r\n" - - # Save Album Cover - if self.settings['saveArtwork'] and 'albumPath' in result: - for image in result['albumURLs']: - downloadImage(image['url'], result['albumPath'] / f"{result['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) - - # Save Artist Artwork - if self.settings['saveArtworkArtist'] and 'artistPath' in result: - for image in result['artistURLs']: - downloadImage(image['url'], result['artistPath'] / f"{result['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) - - # Save filename for playlist file - playlist[i] = result.get('filename', "") - - # Create errors logfile - if self.settings['logErrors'] and errors != "": - with open(self.extrasPath / 'errors.txt', 'wb') as f: - f.write(errors.encode('utf-8')) - - # Create searched logfile - if self.settings['logSearched'] and searched != "": - with open(self.extrasPath / 'searched.txt', 'wb') as f: - f.write(searched.encode('utf-8')) - - # Save Playlist Artwork - if self.settings['saveArtwork'] and self.playlistCoverName and not self.settings['tags']['savePlaylistAsCompilation']: - for image in self.playlistURLs: - downloadImage(image['url'], self.extrasPath / f"{self.playlistCoverName}.{image['ext']}", self.settings['overwriteFile']) - - # Create M3U8 File - if self.settings['createM3U8File']: - filename = settingsRegexPlaylistFile(self.settings['playlistFilenameTemplate'], self.queueItem, self.settings) or "playlist" - with open(self.extrasPath / f'{filename}.m3u8', 'wb') as f: - for line in playlist: - f.write((line + "\n").encode('utf-8')) - - # Execute command after download - if self.settings['executeCommand'] != "": - execute(self.settings['executeCommand'].replace("%folder%", quote(str(self.extrasPath))), shell=True) - - def download(self, trackAPI_gw, track=None): - result = {} - if self.queueItem.cancel: raise DownloadCancelled - if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer") - - # Create Track object - if not track: - logger.info(f"[{trackAPI_gw['ART_NAME']} - {trackAPI_gw['SNG_TITLE']}] Getting the tags") - try: - track = Track().parseData( - dz=self.dz, - trackAPI_gw=trackAPI_gw, - trackAPI=trackAPI_gw['_EXTRA_TRACK'] if '_EXTRA_TRACK' in trackAPI_gw else None, - albumAPI=trackAPI_gw['_EXTRA_ALBUM'] if '_EXTRA_ALBUM' in trackAPI_gw else None, - playlistAPI = trackAPI_gw['_EXTRA_PLAYLIST'] if '_EXTRA_PLAYLIST' in trackAPI_gw else None - ) - except AlbumDoesntExists: - raise DownloadError('albumDoesntExists') - if self.queueItem.cancel: raise DownloadCancelled - - # Check if track not yet encoded - if track.MD5 == '': - if track.fallbackId != "0": - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not yet encoded, using fallback id") - newTrack = self.dz.gw.get_track_with_fallback(track.fallbackId) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - return self.download(trackAPI_gw, track) - elif not track.searched and self.settings['fallbackSearch']: - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not yet encoded, searching for alternative") - searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) - if searchedId != "0": - newTrack = self.dz.gw.get_track_with_fallback(searchedId) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - track.searched = True - if self.interface: - self.interface.send('queueUpdate', { - 'uuid': self.queueItem.uuid, - 'searchFallback': True, - 'data': { - 'id': track.id, - 'title': track.title, - 'artist': track.mainArtist.name - }, - }) - return self.download(trackAPI_gw, track) - else: - raise DownloadFailed("notEncodedNoAlternative") - else: - raise DownloadFailed("notEncoded") - - # Choose the target bitrate - try: - selectedFormat = self.getPreferredBitrate(track) - except PreferredBitrateNotFound: - if track.fallbackId != "0": - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not found at desired bitrate, using fallback id") - newTrack = self.dz.gw.get_track_with_fallback(track.fallbackId) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - return self.download(trackAPI_gw, track) - elif not track.searched and self.settings['fallbackSearch']: - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not found at desired bitrate, searching for alternative") - searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) - if searchedId != "0": - newTrack = self.dz.gw.get_track_with_fallback(searchedId) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - track.searched = True - if self.interface: - self.interface.send('queueUpdate', { - 'uuid': self.queueItem.uuid, - 'searchFallback': True, - 'data': { - 'id': track.id, - 'title': track.title, - 'artist': track.mainArtist.name - }, - }) - return self.download(trackAPI_gw, track) - else: - raise DownloadFailed("wrongBitrateNoAlternative") - else: - raise DownloadFailed("wrongBitrate") - except TrackNot360: - raise DownloadFailed("no360RA") - track.selectedFormat = selectedFormat - track.album.bitrate = selectedFormat - - # Generate covers URLs - embeddedImageFormat = f'jpg-{self.settings["jpegImageQuality"]}' - if self.settings['embeddedArtworkPNG']: imageFormat = 'png' - - if self.settings['tags']['savePlaylistAsCompilation'] and track.playlist: - track.trackNumber = track.position - track.discNumber = "1" - track.album.makePlaylistCompilation(track.playlist) - track.album.embeddedCoverURL = track.playlist.pic.generatePictureURL(self.settings['embeddedArtworkSize'], embeddedImageFormat) - - ext = track.album.embeddedCoverURL[-4:] - if ext[0] != ".": ext = ".jpg" # Check for Spotify images - - track.album.embeddedCoverPath = TEMPDIR / f"pl{trackAPI_gw['_EXTRA_PLAYLIST']['id']}_{self.settings['embeddedArtworkSize']}{ext}" - else: - if track.album.date: track.date = track.album.date - track.album.embeddedCoverURL = track.album.pic.generatePictureURL(self.settings['embeddedArtworkSize'], embeddedImageFormat) - - ext = track.album.embeddedCoverURL[-4:] - track.album.embeddedCoverPath = TEMPDIR / f"alb{track.album.id}_{self.settings['embeddedArtworkSize']}{ext}" - - track.dateString = track.date.format(self.settings['dateFormat']) - track.album.dateString = track.album.date.format(self.settings['dateFormat']) - if track.playlist: track.playlist.dateString = track.playlist.date.format(self.settings['dateFormat']) - - # Check various artist option - if self.settings['albumVariousArtists'] and track.album.variousArtists: - artist = track.album.variousArtists - isMainArtist = artist.role == "Main" - - if artist.name not in track.album.artists: - track.album.artists.insert(0, artist.name) - - if isMainArtist or artist.name not in track.album.artist['Main'] and not isMainArtist: - if not artist.role in track.album.artist: - track.album.artist[artist.role] = [] - track.album.artist[artist.role].insert(0, artist.name) - track.album.mainArtist.save = not track.album.mainArtist.isVariousArtists() or self.settings['albumVariousArtists'] and track.album.mainArtist.isVariousArtists() - - # Check removeDuplicateArtists - if self.settings['removeDuplicateArtists']: track.removeDuplicateArtists() - - # Check if user wants the feat in the title - if str(self.settings['featuredToTitle']) == FeaturesOption.REMOVE_TITLE: - track.title = track.getCleanTitle() - elif str(self.settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: - track.title = track.getFeatTitle() - elif str(self.settings['featuredToTitle']) == FeaturesOption.REMOVE_TITLE_ALBUM: - track.title = track.getCleanTitle() - track.album.title = track.album.getCleanTitle() - - # Remove (Album Version) from tracks that have that - if self.settings['removeAlbumVersion']: - if "Album Version" in track.title: - track.title = re.sub(r' ?\(Album Version\)', "", track.title).strip() - - # Change Title and Artists casing if needed - if self.settings['titleCasing'] != "nothing": - track.title = changeCase(track.title, self.settings['titleCasing']) - if self.settings['artistCasing'] != "nothing": - track.mainArtist.name = changeCase(track.mainArtist.name, self.settings['artistCasing']) - for i, artist in enumerate(track.artists): - track.artists[i] = changeCase(artist, self.settings['artistCasing']) - for type in track.artist: - for i, artist in enumerate(track.artist[type]): - track.artist[type][i] = changeCase(artist, self.settings['artistCasing']) - track.generateMainFeatStrings() - - # Generate artist tag - if self.settings['tags']['multiArtistSeparator'] == "default": - if str(self.settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: - track.artistsString = ", ".join(track.artist['Main']) - else: - track.artistsString = ", ".join(track.artists) - elif self.settings['tags']['multiArtistSeparator'] == "andFeat": - track.artistsString = track.mainArtistsString - if track.featArtistsString and str(self.settings['featuredToTitle']) != FeaturesOption.MOVE_TITLE: - track.artistsString += " " + track.featArtistsString - else: - separator = self.settings['tags']['multiArtistSeparator'] - if str(self.settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: - track.artistsString = separator.join(track.artist['Main']) - else: - track.artistsString = separator.join(track.artists) - - # Generate filename and filepath from metadata - filename = generateFilename(track, self.settings, trackAPI_gw['FILENAME_TEMPLATE']) - (filepath, artistPath, coverPath, extrasPath) = generateFilepath(track, self.settings) - - if self.queueItem.cancel: raise DownloadCancelled - - # Download and cache coverart - logger.info(f"[{track.mainArtist.name} - {track.title}] Getting the album cover") - track.album.embeddedCoverPath = downloadImage(track.album.embeddedCoverURL, track.album.embeddedCoverPath) - - # Save local album art - if coverPath: - result['albumURLs'] = [] - for format in self.settings['localArtworkFormat'].split(","): - if format in ["png","jpg"]: - extendedFormat = format - if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" - url = track.album.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) - if self.settings['tags']['savePlaylistAsCompilation'] \ - and track.playlist \ - and track.playlist.pic.url \ - and not format.startswith("jpg"): - continue - result['albumURLs'].append({'url': url, 'ext': format}) - result['albumPath'] = coverPath - result['albumFilename'] = f"{settingsRegexAlbum(self.settings['coverImageTemplate'], track.album, self.settings, track.playlist)}" - - # Save artist art - if artistPath: - result['artistURLs'] = [] - for format in self.settings['localArtworkFormat'].split(","): - if format in ["png","jpg"]: - extendedFormat = format - if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" - url = track.album.mainArtist.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) - if track.album.mainArtist.pic.md5 == "" and not format.startswith("jpg"): continue - result['artistURLs'].append({'url': url, 'ext': format}) - result['artistPath'] = artistPath - result['artistFilename'] = f"{settingsRegexArtist(self.settings['artistImageTemplate'], track.album.mainArtist, self.settings, rootArtist=track.album.rootArtist)}" - - # Save playlist cover - if track.playlist: - if not len(self.playlistURLs): - for format in self.settings['localArtworkFormat'].split(","): - if format in ["png","jpg"]: - extendedFormat = format - if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" - url = track.playlist.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) - if track.playlist.pic.url and not format.startswith("jpg"): continue - self.playlistURLs.append({'url': url, 'ext': format}) - if not self.playlistCoverName: - track.playlist.bitrate = selectedFormat - track.playlist.dateString = track.playlist.date.format(self.settings['dateFormat']) - self.playlistCoverName = f"{settingsRegexAlbum(self.settings['coverImageTemplate'], track.playlist, self.settings, track.playlist)}" - - # Remove subfolders from filename and add it to filepath - if pathSep in filename: - tempPath = filename[:filename.rfind(pathSep)] - filepath = filepath / tempPath - filename = filename[filename.rfind(pathSep) + len(pathSep):] - - # Make sure the filepath exists - makedirs(filepath, exist_ok=True) - writepath = filepath / f"{filename}{extensions[track.selectedFormat]}" - - # Save lyrics in lrc file - if self.settings['syncedLyrics'] and track.lyrics.sync: - if not (filepath / f"{filename}.lrc").is_file() or self.settings['overwriteFile'] in [OverwriteOption.OVERWRITE, OverwriteOption.ONLY_TAGS]: - with open(filepath / f"{filename}.lrc", 'wb') as f: - f.write(track.lyrics.sync.encode('utf-8')) - - trackAlreadyDownloaded = writepath.is_file() - - # Don't overwrite and don't mind extension - if not trackAlreadyDownloaded and self.settings['overwriteFile'] == OverwriteOption.DONT_CHECK_EXT: - exts = ['.mp3', '.flac', '.opus', '.m4a'] - baseFilename = str(filepath / filename) - for ext in exts: - trackAlreadyDownloaded = Path(baseFilename+ext).is_file() - if trackAlreadyDownloaded: break - - # Don't overwrite and keep both files - if trackAlreadyDownloaded and self.settings['overwriteFile'] == OverwriteOption.KEEP_BOTH: - baseFilename = str(filepath / filename) - i = 1 - currentFilename = baseFilename+' ('+str(i)+')'+ extensions[track.selectedFormat] - while Path(currentFilename).is_file(): - i += 1 - currentFilename = baseFilename+' ('+str(i)+')'+ extensions[track.selectedFormat] - trackAlreadyDownloaded = False - writepath = Path(currentFilename) - - if extrasPath: - if not self.extrasPath: self.extrasPath = extrasPath - result['filename'] = str(writepath)[len(str(extrasPath))+ len(pathSep):] - - if not trackAlreadyDownloaded or self.settings['overwriteFile'] == OverwriteOption.OVERWRITE: - logger.info(f"[{track.mainArtist.name} - {track.title}] Downloading the track") - track.downloadUrl = generateStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat) - - def downloadMusic(track, trackAPI_gw): - try: - with open(writepath, 'wb') as stream: - self.streamTrack(stream, track) - except DownloadCancelled: - if writepath.is_file(): writepath.unlink() - raise DownloadCancelled - except (request_exception.HTTPError, DownloadEmpty): - if writepath.is_file(): writepath.unlink() - if track.fallbackId != "0": - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available, using fallback id") - newTrack = self.dz.gw.get_track_with_fallback(track.fallbackId) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - return False - elif not track.searched and self.settings['fallbackSearch']: - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available, searching for alternative") - searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) - if searchedId != "0": - newTrack = self.dz.gw.get_track_with_fallback(searchedId) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - track.searched = True - if self.interface: - self.interface.send('queueUpdate', { - 'uuid': self.queueItem.uuid, - 'searchFallback': True, - 'data': { - 'id': track.id, - 'title': track.title, - 'artist': track.mainArtist.name - }, - }) - return False - else: - raise DownloadFailed("notAvailableNoAlternative") - else: - raise DownloadFailed("notAvailable") - except (request_exception.ConnectionError, request_exception.ChunkedEncodingError) as e: - if writepath.is_file(): writepath.unlink() - logger.warn(f"[{track.mainArtist.name} - {track.title}] Error while downloading the track, trying again in 5s...") - eventlet.sleep(5) - return downloadMusic(track, trackAPI_gw) - except OSError as e: - if e.errno == errno.ENOSPC: - raise DownloadFailed("noSpaceLeft") - else: - if writepath.is_file(): writepath.unlink() - logger.exception(f"[{track.mainArtist.name} - {track.title}] Error while downloading the track, you should report this to the developers: {str(e)}") - raise e - except Exception as e: - if writepath.is_file(): writepath.unlink() - logger.exception(f"[{track.mainArtist.name} - {track.title}] Error while downloading the track, you should report this to the developers: {str(e)}") - raise e - return True - - try: - trackDownloaded = downloadMusic(track, trackAPI_gw) - except Exception as e: - raise e - - if not trackDownloaded: return self.download(trackAPI_gw, track) - else: - logger.info(f"[{track.mainArtist.name} - {track.title}] Skipping track as it's already downloaded") - self.completeTrackPercentage() - - # Adding tags - if (not trackAlreadyDownloaded or self.settings['overwriteFile'] in [OverwriteOption.ONLY_TAGS, OverwriteOption.OVERWRITE]) and not track.localTrack: - logger.info(f"[{track.mainArtist.name} - {track.title}] Applying tags to the track") - if track.selectedFormat in [TrackFormats.MP3_320, TrackFormats.MP3_128, TrackFormats.DEFAULT]: - tagID3(writepath, track, self.settings['tags']) - elif track.selectedFormat == TrackFormats.FLAC: - try: - tagFLAC(writepath, track, self.settings['tags']) - except (FLACNoHeaderError, FLACError): - if writepath.is_file(): writepath.unlink() - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available in FLAC, falling back if necessary") - self.removeTrackPercentage() - track.filesizes['FILESIZE_FLAC'] = "0" - track.filesizes['FILESIZE_FLAC_TESTED'] = True - return self.download(trackAPI_gw, track) - - if track.searched: result['searched'] = f"{track.mainArtist.name} - {track.title}" - logger.info(f"[{track.mainArtist.name} - {track.title}] Track download completed\n{str(writepath)}") - self.queueItem.downloaded += 1 - self.queueItem.files.append(str(writepath)) - self.queueItem.extrasPath = str(self.extrasPath) - if self.interface: - self.interface.send("updateQueue", {'uuid': self.queueItem.uuid, 'downloaded': True, 'downloadPath': str(writepath), 'extrasPath': str(self.extrasPath)}) - return result - - def getPreferredBitrate(self, track): - if track.localTrack: return TrackFormats.LOCAL - - shouldFallback = self.settings['fallbackBitrate'] - falledBack = False - - formats_non_360 = { - TrackFormats.FLAC: "FLAC", - TrackFormats.MP3_320: "MP3_320", - TrackFormats.MP3_128: "MP3_128", - } - formats_360 = { - TrackFormats.MP4_RA3: "MP4_RA3", - TrackFormats.MP4_RA2: "MP4_RA2", - TrackFormats.MP4_RA1: "MP4_RA1", - } - - is360format = int(self.bitrate) in formats_360 - - if not shouldFallback: - formats = formats_360 - formats.update(formats_non_360) - elif is360format: - formats = formats_360 - else: - formats = formats_non_360 - - for formatNumber, formatName in formats.items(): - if formatNumber <= int(self.bitrate): - if f"FILESIZE_{formatName}" in track.filesizes: - if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber - if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]: - request = requests.head( - generateStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), - headers={'User-Agent': USER_AGENT_HEADER}, - timeout=30 - ) - try: - request.raise_for_status() - return formatNumber - except request_exception.HTTPError: # if the format is not available, Deezer returns a 403 error - pass - if not shouldFallback: - raise PreferredBitrateNotFound - else: - if not falledBack: - falledBack = True - logger.info(f"[{track.mainArtist.name} - {track.title}] Fallback to lower bitrate") - if self.interface: - self.interface.send('queueUpdate', { - 'uuid': self.queueItem.uuid, - 'bitrateFallback': True, - 'data': { - 'id': track.id, - 'title': track.title, - 'artist': track.mainArtist.name - }, - }) - if is360format: raise TrackNot360 - return TrackFormats.DEFAULT - - def streamTrack(self, stream, track, start=0): - if self.queueItem.cancel: raise DownloadCancelled - - headers=dict(self.dz.http_headers) - if range != 0: headers['Range'] = f'bytes={start}-' - chunkLength = start - percentage = 0 - - itemName = f"[{track.mainArtist.name} - {track.title}]" - - try: - with self.dz.session.get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: - request.raise_for_status() - blowfish_key = str.encode(generateBlowfishKey(str(track.id))) - - complete = int(request.headers["Content-Length"]) - if complete == 0: raise DownloadEmpty - if start != 0: - responseRange = request.headers["Content-Range"] - logger.info(f'{itemName} downloading range {responseRange}') - else: - logger.info(f'{itemName} downloading {complete} bytes') - - for chunk in request.iter_content(2048 * 3): - if self.queueItem.cancel: raise DownloadCancelled - - if len(chunk) >= 2048: - chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk[0:2048]) + chunk[2048:] - - stream.write(chunk) - chunkLength += len(chunk) - - if isinstance(self.queueItem, QISingle): - percentage = (chunkLength / (complete + start)) * 100 - self.downloadPercentage = percentage - else: - chunkProgres = (len(chunk) / (complete + start)) / self.queueItem.size * 100 - self.downloadPercentage += chunkProgres - self.updatePercentage() - - except (SSLError, u3SSLError) as e: - logger.info(f'{itemName} retrying from byte {chunkLength}') - return self.streamTrack(stream, track, chunkLength) - except (request_exception.ConnectionError, requests.exceptions.ReadTimeout): - eventlet.sleep(2) - return self.streamTrack(stream, track, start) - - def updatePercentage(self): - if round(self.downloadPercentage) != self.lastPercentage and round(self.downloadPercentage) % 2 == 0: - self.lastPercentage = round(self.downloadPercentage) - self.queueItem.progress = self.lastPercentage - if self.interface: self.interface.send("updateQueue", {'uuid': self.queueItem.uuid, 'progress': self.lastPercentage}) - - def completeTrackPercentage(self): - if isinstance(self.queueItem, QISingle): - self.downloadPercentage = 100 - else: - self.downloadPercentage += (1 / self.queueItem.size) * 100 - self.updatePercentage() - - def removeTrackPercentage(self): - if isinstance(self.queueItem, QISingle): - self.downloadPercentage = 0 - else: - self.downloadPercentage -= (1 / self.queueItem.size) * 100 - self.updatePercentage() - - def downloadWrapper(self, trackAPI_gw): - track = { - 'id': trackAPI_gw['SNG_ID'], - 'title': trackAPI_gw['SNG_TITLE'].strip(), - 'artist': trackAPI_gw['ART_NAME'] - } - if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: - track['title'] += f" {trackAPI_gw['VERSION']}".strip() - - try: - result = self.download(trackAPI_gw) - except DownloadCancelled: - return None - except DownloadFailed as error: - logger.error(f"[{track['artist']} - {track['title']}] {error.message}") - result = {'error': { - 'message': error.message, - 'errid': error.errid, - 'data': track - }} - except Exception as e: - logger.exception(f"[{track['artist']} - {track['title']}] {str(e)}") - result = {'error': { - 'message': str(e), - 'data': track - }} - - if 'error' in result: - self.completeTrackPercentage() - self.queueItem.failed += 1 - self.queueItem.errors.append(result['error']) - if self.interface: - error = result['error'] - self.interface.send("updateQueue", { - 'uuid': self.queueItem.uuid, - 'failed': True, - 'data': error['data'], - 'error': error['message'], - 'errid': error['errid'] if 'errid' in error else None - }) - return result - -class DownloadError(Exception): - """Base class for exceptions in this module.""" - pass - -class DownloadFailed(DownloadError): - def __init__(self, errid): - self.errid = errid - self.message = errorMessages[self.errid] - -class DownloadCancelled(DownloadError): - pass - -class DownloadEmpty(DownloadError): - pass - -class PreferredBitrateNotFound(DownloadError): - pass - -class TrackNot360(DownloadError): - pass diff --git a/deemix/app/messageinterface.py b/deemix/app/messageinterface.py deleted file mode 100644 index ef910c2..0000000 --- a/deemix/app/messageinterface.py +++ /dev/null @@ -1,4 +0,0 @@ -class MessageInterface: - def send(self, message, value=None): - """Implement this class to process updates and messages from the core""" - pass diff --git a/deemix/app/queueitem.py b/deemix/app/queueitem.py deleted file mode 100644 index 49e223b..0000000 --- a/deemix/app/queueitem.py +++ /dev/null @@ -1,115 +0,0 @@ -class QueueItem: - def __init__(self, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, type=None, settings=None, queueItemDict=None): - if queueItemDict: - self.title = queueItemDict['title'] - self.artist = queueItemDict['artist'] - self.cover = queueItemDict['cover'] - self.explicit = queueItemDict.get('explicit', False) - self.size = queueItemDict['size'] - self.type = queueItemDict['type'] - self.id = queueItemDict['id'] - self.bitrate = queueItemDict['bitrate'] - self.extrasPath = queueItemDict.get('extrasPath', '') - self.files = queueItemDict['files'] - self.downloaded = queueItemDict['downloaded'] - self.failed = queueItemDict['failed'] - self.errors = queueItemDict['errors'] - self.progress = queueItemDict['progress'] - self.settings = queueItemDict.get('settings') - else: - self.title = title - self.artist = artist - self.cover = cover - self.explicit = explicit - self.size = size - self.type = type - self.id = id - self.bitrate = bitrate - self.extrasPath = None - self.files = [] - self.settings = settings - self.downloaded = 0 - self.failed = 0 - self.errors = [] - self.progress = 0 - self.uuid = f"{self.type}_{self.id}_{self.bitrate}" - self.cancel = False - self.ack = None - - def toDict(self): - return { - 'title': self.title, - 'artist': self.artist, - 'cover': self.cover, - 'explicit': self.explicit, - 'size': self.size, - 'extrasPath': self.extrasPath, - 'files': self.files, - 'downloaded': self.downloaded, - 'failed': self.failed, - 'errors': self.errors, - 'progress': self.progress, - 'type': self.type, - 'id': self.id, - 'bitrate': self.bitrate, - 'uuid': self.uuid, - 'ack': self.ack - } - - def getResettedItem(self): - item = self.toDict() - item['downloaded'] = 0 - item['failed'] = 0 - item['progress'] = 0 - item['errors'] = [] - return item - - def getSlimmedItem(self): - light = self.toDict() - propertiesToDelete = ['single', 'collection', '_EXTRA', 'settings'] - for property in propertiesToDelete: - if property in light: - del light[property] - return light - -class QISingle(QueueItem): - def __init__(self, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, type=None, settings=None, single=None, queueItemDict=None): - if queueItemDict: - super().__init__(queueItemDict=queueItemDict) - self.single = queueItemDict['single'] - else: - super().__init__(id, bitrate, title, artist, cover, explicit, 1, type, settings) - self.single = single - - def toDict(self): - queueItem = super().toDict() - queueItem['single'] = self.single - return queueItem - -class QICollection(QueueItem): - def __init__(self, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, type=None, settings=None, collection=None, queueItemDict=None): - if queueItemDict: - super().__init__(queueItemDict=queueItemDict) - self.collection = queueItemDict['collection'] - else: - super().__init__(id, bitrate, title, artist, cover, explicit, size, type, settings) - self.collection = collection - - def toDict(self): - queueItem = super().toDict() - queueItem['collection'] = self.collection - return queueItem - -class QIConvertable(QICollection): - def __init__(self, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, type=None, settings=None, extra=None, queueItemDict=None): - if queueItemDict: - super().__init__(queueItemDict=queueItemDict) - self.extra = queueItemDict['_EXTRA'] - else: - super().__init__(id, bitrate, title, artist, cover, explicit, size, type, settings, []) - self.extra = extra - - def toDict(self): - queueItem = super().toDict() - queueItem['_EXTRA'] = self.extra - return queueItem diff --git a/deemix/app/queuemanager.py b/deemix/app/queuemanager.py deleted file mode 100644 index 4547bf8..0000000 --- a/deemix/app/queuemanager.py +++ /dev/null @@ -1,569 +0,0 @@ -from deemix.app.downloadjob import DownloadJob -from deemix.utils import getIDFromLink, getTypeFromLink, getBitrateInt -from deezer.gw import APIError as gwAPIError, LyricsStatus -from deezer.api import APIError -from deezer.utils import map_user_playlist -from spotipy.exceptions import SpotifyException -from deemix.app.queueitem import QueueItem, QISingle, QICollection, QIConvertable -import logging -from pathlib import Path -import json -from os import remove -import eventlet -import uuid -urlopen = eventlet.import_patched('urllib.request').urlopen - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger('deemix') - -class QueueManager: - def __init__(self, spotifyHelper=None): - self.queue = [] - self.queueList = {} - self.queueComplete = [] - self.currentItem = "" - self.sp = spotifyHelper - - def generateTrackQueueItem(self, dz, id, settings, bitrate, trackAPI=None, albumAPI=None): - # Check if is an isrc: url - if str(id).startswith("isrc"): - try: - trackAPI = dz.api.get_track(id) - except APIError as e: - e = str(e) - return QueueError("https://deezer.com/track/"+str(id), f"Wrong URL: {e}") - if 'id' in trackAPI and 'title' in trackAPI: - id = trackAPI['id'] - else: - return QueueError("https://deezer.com/track/"+str(id), "Track ISRC is not available on deezer", "ISRCnotOnDeezer") - - # Get essential track info - try: - trackAPI_gw = dz.gw.get_track_with_fallback(id) - except gwAPIError as e: - e = str(e) - message = "Wrong URL" - if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}" - return QueueError("https://deezer.com/track/"+str(id), message) - - if albumAPI: trackAPI_gw['_EXTRA_ALBUM'] = albumAPI - if trackAPI: trackAPI_gw['_EXTRA_TRACK'] = trackAPI - - if settings['createSingleFolder']: - trackAPI_gw['FILENAME_TEMPLATE'] = settings['albumTracknameTemplate'] - else: - trackAPI_gw['FILENAME_TEMPLATE'] = settings['tracknameTemplate'] - - trackAPI_gw['SINGLE_TRACK'] = True - - title = trackAPI_gw['SNG_TITLE'].strip() - if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: - title += f" {trackAPI_gw['VERSION']}".strip() - explicit = bool(int(trackAPI_gw.get('EXPLICIT_LYRICS', 0))) - - return QISingle( - id=id, - bitrate=bitrate, - title=title, - artist=trackAPI_gw['ART_NAME'], - cover=f"https://e-cdns-images.dzcdn.net/images/cover/{trackAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg", - explicit=explicit, - type='track', - settings=settings, - single=trackAPI_gw, - ) - - def generateAlbumQueueItem(self, dz, id, settings, bitrate, rootArtist=None): - # Get essential album info - try: - albumAPI = dz.api.get_album(id) - except APIError as e: - e = str(e) - return QueueError("https://deezer.com/album/"+str(id), f"Wrong URL: {e}") - - if str(id).startswith('upc'): id = albumAPI['id'] - - # Get extra info about album - # This saves extra api calls when downloading - albumAPI_gw = dz.gw.get_album(id) - albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK'] - albumAPI['copyright'] = albumAPI_gw['COPYRIGHT'] - albumAPI['root_artist'] = rootArtist - - # If the album is a single download as a track - if albumAPI['nb_tracks'] == 1: - return self.generateTrackQueueItem(dz, albumAPI['tracks']['data'][0]['id'], settings, bitrate, albumAPI=albumAPI) - - tracksArray = dz.gw.get_album_tracks(id) - - if albumAPI['cover_small'] != None: - cover = albumAPI['cover_small'][:-24] + '/75x75-000000-80-0-0.jpg' - else: - cover = f"https://e-cdns-images.dzcdn.net/images/cover/{albumAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg" - - totalSize = len(tracksArray) - albumAPI['nb_tracks'] = totalSize - collection = [] - for pos, trackAPI in enumerate(tracksArray, start=1): - trackAPI['_EXTRA_ALBUM'] = albumAPI - trackAPI['POSITION'] = pos - trackAPI['SIZE'] = totalSize - trackAPI['FILENAME_TEMPLATE'] = settings['albumTracknameTemplate'] - collection.append(trackAPI) - - explicit = albumAPI_gw.get('EXPLICIT_ALBUM_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT] - - return QICollection( - id=id, - bitrate=bitrate, - title=albumAPI['title'], - artist=albumAPI['artist']['name'], - cover=cover, - explicit=explicit, - size=totalSize, - type='album', - settings=settings, - collection=collection, - ) - - def generatePlaylistQueueItem(self, dz, id, settings, bitrate): - # Get essential playlist info - try: - playlistAPI = dz.api.get_playlist(id) - except: - playlistAPI = None - # Fallback to gw api if the playlist is private - if not playlistAPI: - try: - userPlaylist = dz.gw.get_playlist_page(id) - playlistAPI = map_user_playlist(userPlaylist['DATA']) - except gwAPIError as e: - e = str(e) - message = "Wrong URL" - if "DATA_ERROR" in e: - message += f": {e['DATA_ERROR']}" - return QueueError("https://deezer.com/playlist/"+str(id), message) - - # Check if private playlist and owner - if not playlistAPI.get('public', False) and playlistAPI['creator']['id'] != str(dz.current_user['id']): - logger.warning("You can't download others private playlists.") - return QueueError("https://deezer.com/playlist/"+str(id), "You can't download others private playlists.", "notYourPrivatePlaylist") - - playlistTracksAPI = dz.gw.get_playlist_tracks(id) - playlistAPI['various_artist'] = dz.api.get_artist(5080) # Useful for save as compilation - - totalSize = len(playlistTracksAPI) - playlistAPI['nb_tracks'] = totalSize - collection = [] - for pos, trackAPI in enumerate(playlistTracksAPI, start=1): - if trackAPI.get('EXPLICIT_TRACK_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT]: - playlistAPI['explicit'] = True - trackAPI['_EXTRA_PLAYLIST'] = playlistAPI - trackAPI['POSITION'] = pos - trackAPI['SIZE'] = totalSize - trackAPI['FILENAME_TEMPLATE'] = settings['playlistTracknameTemplate'] - collection.append(trackAPI) - if not 'explicit' in playlistAPI: - playlistAPI['explicit'] = False - - return QICollection( - id=id, - bitrate=bitrate, - title=playlistAPI['title'], - artist=playlistAPI['creator']['name'], - cover=playlistAPI['picture_small'][:-24] + '/75x75-000000-80-0-0.jpg', - explicit=playlistAPI['explicit'], - size=totalSize, - type='playlist', - settings=settings, - collection=collection, - ) - - def generateArtistQueueItem(self, dz, id, settings, bitrate, interface=None): - # Get essential artist info - try: - artistAPI = dz.api.get_artist(id) - except APIError as e: - e = str(e) - return QueueError("https://deezer.com/artist/"+str(id), f"Wrong URL: {e}") - - if interface: interface.send("startAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) - rootArtist = { - 'id': artistAPI['id'], - 'name': artistAPI['name'] - } - - artistDiscographyAPI = dz.gw.get_artist_discography_tabs(id, 100) - allReleases = artistDiscographyAPI.pop('all', []) - albumList = [] - for album in allReleases: - albumList.append(self.generateAlbumQueueItem(dz, album['id'], settings, bitrate, rootArtist=rootArtist)) - - if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) - return albumList - - def generateArtistDiscographyQueueItem(self, dz, id, settings, bitrate, interface=None): - # Get essential artist info - try: - artistAPI = dz.api.get_artist(id) - except APIError as e: - e = str(e) - return QueueError("https://deezer.com/artist/"+str(id)+"/discography", f"Wrong URL: {e}") - - if interface: interface.send("startAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) - rootArtist = { - 'id': artistAPI['id'], - 'name': artistAPI['name'] - } - - artistDiscographyAPI = dz.gw.get_artist_discography_tabs(id, 100) - artistDiscographyAPI.pop('all', None) # all contains albums and singles, so its all duplicates. This removes them - albumList = [] - for type in artistDiscographyAPI: - for album in artistDiscographyAPI[type]: - albumList.append(self.generateAlbumQueueItem(dz, album['id'], settings, bitrate, rootArtist=rootArtist)) - - if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) - return albumList - - def generateArtistTopQueueItem(self, dz, id, settings, bitrate, interface=None): - # Get essential artist info - try: - artistAPI = dz.api.get_artist(id) - except APIError as e: - e = str(e) - return QueueError("https://deezer.com/artist/"+str(id)+"/top_track", f"Wrong URL: {e}") - - # Emulate the creation of a playlist - # Can't use generatePlaylistQueueItem as this is not a real playlist - playlistAPI = { - 'id': str(artistAPI['id'])+"_top_track", - 'title': artistAPI['name']+" - Top Tracks", - 'description': "Top Tracks for "+artistAPI['name'], - 'duration': 0, - 'public': True, - 'is_loved_track': False, - 'collaborative': False, - 'nb_tracks': 0, - 'fans': artistAPI['nb_fan'], - 'link': "https://www.deezer.com/artist/"+str(artistAPI['id'])+"/top_track", - 'share': None, - 'picture': artistAPI['picture'], - 'picture_small': artistAPI['picture_small'], - 'picture_medium': artistAPI['picture_medium'], - 'picture_big': artistAPI['picture_big'], - 'picture_xl': artistAPI['picture_xl'], - 'checksum': None, - 'tracklist': "https://api.deezer.com/artist/"+str(artistAPI['id'])+"/top", - 'creation_date': "XXXX-00-00", - 'creator': { - 'id': "art_"+str(artistAPI['id']), - 'name': artistAPI['name'], - 'type': "user" - }, - 'type': "playlist" - } - - artistTopTracksAPI_gw = dz.gw.get_artist_toptracks(id) - playlistAPI['various_artist'] = dz.api.get_artist(5080) # Useful for save as compilation - - totalSize = len(artistTopTracksAPI_gw) - playlistAPI['nb_tracks'] = totalSize - collection = [] - for pos, trackAPI in enumerate(artistTopTracksAPI_gw, start=1): - if trackAPI.get('EXPLICIT_TRACK_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT]: - playlistAPI['explicit'] = True - trackAPI['_EXTRA_PLAYLIST'] = playlistAPI - trackAPI['POSITION'] = pos - trackAPI['SIZE'] = totalSize - trackAPI['FILENAME_TEMPLATE'] = settings['playlistTracknameTemplate'] - collection.append(trackAPI) - if not 'explicit' in playlistAPI: - playlistAPI['explicit'] = False - - return QICollection( - id=id, - bitrate=bitrate, - title=playlistAPI['title'], - artist=playlistAPI['creator']['name'], - cover=playlistAPI['picture_small'][:-24] + '/75x75-000000-80-0-0.jpg', - explicit=playlistAPI['explicit'], - size=totalSize, - type='playlist', - settings=settings, - collection=collection, - ) - - def generateQueueItem(self, dz, url, settings, bitrate=None, interface=None): - bitrate = getBitrateInt(bitrate) or settings['maxBitrate'] - if 'deezer.page.link' in url: url = urlopen(url).url - if 'link.tospotify.com' in url: url = urlopen(url).url - - type = getTypeFromLink(url) - id = getIDFromLink(url, type) - if type == None or id == None: - logger.warn("URL not recognized") - return QueueError(url, "URL not recognized", "invalidURL") - - if type == "track": - return self.generateTrackQueueItem(dz, id, settings, bitrate) - elif type == "album": - return self.generateAlbumQueueItem(dz, id, settings, bitrate) - elif type == "playlist": - return self.generatePlaylistQueueItem(dz, id, settings, bitrate) - elif type == "artist": - return self.generateArtistQueueItem(dz, id, settings, bitrate, interface=interface) - elif type == "artistdiscography": - return self.generateArtistDiscographyQueueItem(dz, id, settings, bitrate, interface=interface) - elif type == "artisttop": - return self.generateArtistTopQueueItem(dz, id, settings, bitrate, interface=interface) - elif type.startswith("spotify") and self.sp: - if not self.sp.spotifyEnabled: - logger.warn("Spotify Features is not setted up correctly.") - return QueueError(url, "Spotify Features is not setted up correctly.", "spotifyDisabled") - - if type == "spotifytrack": - try: - (track_id, trackAPI, _) = self.sp.get_trackid_spotify(dz, id, settings['fallbackSearch']) - except SpotifyException as e: - return QueueError(url, "Wrong URL: "+e.msg[e.msg.find('\n')+2:]) - except Exception as e: - return QueueError(url, "Something went wrong: "+str(e)) - - if track_id != "0": - return self.generateTrackQueueItem(dz, track_id, settings, bitrate, trackAPI=trackAPI) - else: - logger.warn("Track not found on deezer!") - return QueueError(url, "Track not found on deezer!", "trackNotOnDeezer") - - elif type == "spotifyalbum": - try: - album_id = self.sp.get_albumid_spotify(dz, id) - except SpotifyException as e: - return QueueError(url, "Wrong URL: "+e.msg[e.msg.find('\n')+2:]) - except Exception as e: - return QueueError(url, "Something went wrong: "+str(e)) - - if album_id != "0": - return self.generateAlbumQueueItem(dz, album_id, settings, bitrate) - else: - logger.warn("Album not found on deezer!") - return QueueError(url, "Album not found on deezer!", "albumNotOnDeezer") - - elif type == "spotifyplaylist": - try: - return self.sp.generate_playlist_queueitem(dz, id, bitrate, settings) - except SpotifyException as e: - return QueueError(url, "Wrong URL: "+e.msg[e.msg.find('\n')+2:]) - except Exception as e: - return QueueError(url, "Something went wrong: "+str(e)) - logger.warn("URL not supported yet") - return QueueError(url, "URL not supported yet", "unsupportedURL") - - def addToQueue(self, dz, url, settings, bitrate=None, interface=None, ack=None): - if not dz.logged_in: - if interface: interface.send("loginNeededToDownload") - return False - - def parseLink(link): - link = link.strip() - if link == "": return False - logger.info("Generating queue item for: "+link) - item = self.generateQueueItem(dz, link, settings, bitrate, interface=interface) - - # Add ack to all items - if type(item) is list: - for i in item: - if isinstance(i, QueueItem): - i.ack = ack - elif isinstance(item, QueueItem): - item.ack = ack - return item - - if type(url) is list: - queueItem = [] - request_uuid = str(uuid.uuid4()) - if interface: interface.send("startGeneratingItems", {'uuid': request_uuid, 'total': len(url)}) - for link in url: - item = parseLink(link) - if not item: continue - if type(item) is list: - queueItem += item - else: - queueItem.append(item) - if interface: interface.send("finishGeneratingItems", {'uuid': request_uuid, 'total': len(queueItem)}) - if not len(queueItem): - return False - else: - queueItem = parseLink(url) - if not queueItem: - return False - - def processQueueItem(item, silent=False): - if isinstance(item, QueueError): - logger.error(f"[{item.link}] {item.message}") - if interface: interface.send("queueError", item.toDict()) - return False - if item.uuid in list(self.queueList.keys()): - logger.warn(f"[{item.uuid}] Already in queue, will not be added again.") - if interface and not silent: interface.send("alreadyInQueue", {'uuid': item.uuid, 'title': item.title}) - return False - self.queue.append(item.uuid) - self.queueList[item.uuid] = item - logger.info(f"[{item.uuid}] Added to queue.") - return True - - if type(queueItem) is list: - slimmedItems = [] - for item in queueItem: - if processQueueItem(item, silent=True): - slimmedItems.append(item.getSlimmedItem()) - else: - continue - if not len(slimmedItems): - return False - if interface: interface.send("addedToQueue", slimmedItems) - else: - if processQueueItem(queueItem): - if interface: interface.send("addedToQueue", queueItem.getSlimmedItem()) - else: - return False - self.nextItem(dz, interface) - return True - - def nextItem(self, dz, interface=None): - # Check that nothing is already downloading and - # that the queue is not empty - if self.currentItem != "": return None - if not len(self.queue): return None - - self.currentItem = self.queue.pop(0) - - if isinstance(self.queueList[self.currentItem], QIConvertable) and self.queueList[self.currentItem].extra: - logger.info(f"[{self.currentItem}] Converting tracks to deezer.") - self.sp.convert_spotify_playlist(dz, self.queueList[self.currentItem], interface=interface) - logger.info(f"[{self.currentItem}] Tracks converted.") - - if interface: interface.send("startDownload", self.currentItem) - logger.info(f"[{self.currentItem}] Started downloading.") - - DownloadJob(dz, self.queueList[self.currentItem], interface).start() - - if self.queueList[self.currentItem].cancel: - del self.queueList[self.currentItem] - else: - self.queueComplete.append(self.currentItem) - logger.info(f"[{self.currentItem}] Finished downloading.") - self.currentItem = "" - self.nextItem(dz, interface) - - def getQueue(self): - return (self.queue, self.queueComplete, self.slimQueueList(), self.currentItem) - - def saveQueue(self, configFolder): - if len(self.queueList) > 0: - if self.currentItem != "": - self.queue.insert(0, self.currentItem) - with open(Path(configFolder) / 'queue.json', 'w') as f: - json.dump({ - 'queue': self.queue, - 'queueComplete': self.queueComplete, - 'queueList': self.exportQueueList() - }, f) - - def exportQueueList(self): - queueList = {} - for uuid in self.queueList: - if uuid in self.queue: - queueList[uuid] = self.queueList[uuid].getResettedItem() - else: - queueList[uuid] = self.queueList[uuid].toDict() - return queueList - - def slimQueueList(self): - queueList = {} - for uuid in self.queueList: - queueList[uuid] = self.queueList[uuid].getSlimmedItem() - return queueList - - def loadQueue(self, configFolder, settings, interface=None): - configFolder = Path(configFolder) - if (configFolder / 'queue.json').is_file() and not len(self.queue): - if interface: interface.send('restoringQueue') - with open(configFolder / 'queue.json', 'r') as f: - try: - qd = json.load(f) - except json.decoder.JSONDecodeError: - logger.warn("Saved queue is corrupted, resetting it") - qd = { - 'queue': [], - 'queueComplete': [], - 'queueList': {} - } - remove(configFolder / 'queue.json') - self.restoreQueue(qd['queue'], qd['queueComplete'], qd['queueList'], settings) - if interface: - interface.send('init_downloadQueue', { - 'queue': self.queue, - 'queueComplete': self.queueComplete, - 'queueList': self.slimQueueList(), - 'restored': True - }) - - def restoreQueue(self, queue, queueComplete, queueList, settings): - self.queue = queue - self.queueComplete = queueComplete - self.queueList = {} - for uuid in queueList: - if 'single' in queueList[uuid]: - self.queueList[uuid] = QISingle(queueItemDict = queueList[uuid]) - if 'collection' in queueList[uuid]: - self.queueList[uuid] = QICollection(queueItemDict = queueList[uuid]) - if '_EXTRA' in queueList[uuid]: - self.queueList[uuid] = QIConvertable(queueItemDict = queueList[uuid]) - self.queueList[uuid].settings = settings - - def removeFromQueue(self, uuid, interface=None): - if uuid == self.currentItem: - if interface: interface.send("cancellingCurrentItem", uuid) - self.queueList[uuid].cancel = True - return - if uuid in self.queue: - self.queue.remove(uuid) - elif uuid in self.queueComplete: - self.queueComplete.remove(uuid) - else: - return - del self.queueList[uuid] - if interface: interface.send("removedFromQueue", uuid) - - - def cancelAllDownloads(self, interface=None): - self.queue = [] - self.queueComplete = [] - if self.currentItem != "": - if interface: interface.send("cancellingCurrentItem", self.currentItem) - self.queueList[self.currentItem].cancel = True - for uuid in list(self.queueList.keys()): - if uuid != self.currentItem: del self.queueList[uuid] - if interface: interface.send("removedAllDownloads", self.currentItem) - - - def removeFinishedDownloads(self, interface=None): - for uuid in self.queueComplete: - del self.queueList[uuid] - self.queueComplete = [] - if interface: interface.send("removedFinishedDownloads") - -class QueueError: - def __init__(self, link, message, errid=None): - self.link = link - self.message = message - self.errid = errid - - def toDict(self): - return { - 'link': self.link, - 'error': self.message, - 'errid': self.errid - } diff --git a/deemix/app/settings.py b/deemix/app/settings.py deleted file mode 100644 index 2390048..0000000 --- a/deemix/app/settings.py +++ /dev/null @@ -1,220 +0,0 @@ -import json -from pathlib import Path -from os import makedirs, listdir -from deemix import __version__ as deemixVersion -from deezer import TrackFormats -from deemix.utils import checkFolder -import logging -import datetime -import platform - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger('deemix') - -import deemix.utils.localpaths as localpaths - -class OverwriteOption(): - """Should the lib overwrite files?""" - - OVERWRITE = 'y' - """Yes, overwrite the file""" - - DONT_OVERWRITE = 'n' - """No, don't overwrite the file""" - - DONT_CHECK_EXT = 'e' - """No, and don't check for extensions""" - - KEEP_BOTH = 'b' - """No, and keep both files""" - - ONLY_TAGS = 't' - """Overwrite only the tags""" - -class FeaturesOption(): - """What should I do with featured artists?""" - - NO_CHANGE = "0" - """Do nothing""" - - REMOVE_TITLE = "1" - """Remove from track title""" - - REMOVE_TITLE_ALBUM = "3" - """Remove from track title and album title""" - - MOVE_TITLE = "2" - """Move to track title""" - -DEFAULT_SETTINGS = { - "downloadLocation": str(localpaths.getMusicFolder()), - "tracknameTemplate": "%artist% - %title%", - "albumTracknameTemplate": "%tracknumber% - %title%", - "playlistTracknameTemplate": "%position% - %artist% - %title%", - "createPlaylistFolder": True, - "playlistNameTemplate": "%playlist%", - "createArtistFolder": False, - "artistNameTemplate": "%artist%", - "createAlbumFolder": True, - "albumNameTemplate": "%artist% - %album%", - "createCDFolder": True, - "createStructurePlaylist": False, - "createSingleFolder": False, - "padTracks": True, - "paddingSize": "0", - "illegalCharacterReplacer": "_", - "queueConcurrency": 3, - "maxBitrate": str(TrackFormats.MP3_320), - "fallbackBitrate": True, - "fallbackSearch": False, - "logErrors": True, - "logSearched": False, - "saveDownloadQueue": False, - "overwriteFile": OverwriteOption.DONT_OVERWRITE, - "createM3U8File": False, - "playlistFilenameTemplate": "playlist", - "syncedLyrics": False, - "embeddedArtworkSize": 800, - "embeddedArtworkPNG": False, - "localArtworkSize": 1400, - "localArtworkFormat": "jpg", - "saveArtwork": True, - "coverImageTemplate": "cover", - "saveArtworkArtist": False, - "artistImageTemplate": "folder", - "jpegImageQuality": 80, - "dateFormat": "Y-M-D", - "albumVariousArtists": True, - "removeAlbumVersion": False, - "removeDuplicateArtists": False, - "tagsLanguage": "", - "featuredToTitle": FeaturesOption.NO_CHANGE, - "titleCasing": "nothing", - "artistCasing": "nothing", - "executeCommand": "", - "tags": { - "title": True, - "artist": True, - "album": True, - "cover": True, - "trackNumber": True, - "trackTotal": False, - "discNumber": True, - "discTotal": False, - "albumArtist": True, - "genre": True, - "year": True, - "date": True, - "explicit": False, - "isrc": True, - "length": True, - "barcode": True, - "bpm": True, - "replayGain": False, - "label": True, - "lyrics": False, - "syncedLyrics": False, - "copyright": False, - "composer": False, - "involvedPeople": False, - "source": False, - "savePlaylistAsCompilation": False, - "useNullSeparator": False, - "saveID3v1": True, - "multiArtistSeparator": "default", - "singleAlbumArtist": False, - "coverDescriptionUTF8": False - } -} - -class Settings: - def __init__(self, configFolder=None, overwriteDownloadFolder=None): - self.settings = {} - self.configFolder = Path(configFolder or localpaths.getConfigFolder()) - - # Create config folder if it doesn't exsist - makedirs(self.configFolder, exist_ok=True) - - # Create config file if it doesn't exsist - if not (self.configFolder / 'config.json').is_file(): - with open(self.configFolder / 'config.json', 'w') as f: - json.dump(DEFAULT_SETTINGS, f, indent=2) - - # Read config file - with open(self.configFolder / 'config.json', 'r') as configFile: - self.settings = json.load(configFile) - - # Check for overwriteDownloadFolder - # This prevents the creation of the original download folder when - # using overwriteDownloadFolder - originalDownloadFolder = self.settings['downloadLocation'] - if overwriteDownloadFolder: - overwriteDownloadFolder = str(overwriteDownloadFolder) - self.settings['downloadLocation'] = overwriteDownloadFolder - - # Make sure the download path exsits, fallback to default - invalidDownloadFolder = False - if self.settings['downloadLocation'] == "" or not checkFolder(self.settings['downloadLocation']): - self.settings['downloadLocation'] = DEFAULT_SETTINGS['downloadLocation'] - originalDownloadFolder = self.settings['downloadLocation'] - invalidDownloadFolder = True - - # Check the settings and save them if something changed - if self.settingsCheck() > 0 or invalidDownloadFolder: - makedirs(self.settings['downloadLocation'], exist_ok=True) - self.settings['downloadLocation'] = originalDownloadFolder # Prevents the saving of the overwritten path - self.saveSettings() - self.settings['downloadLocation'] = overwriteDownloadFolder or originalDownloadFolder # Restores the correct path - - # LOGFILES - - # Create logfile name and path - logspath = self.configFolder / 'logs' - now = datetime.datetime.now() - logfile = now.strftime("%Y-%m-%d_%H%M%S")+".log" - makedirs(logspath, exist_ok=True) - - # Add handler for logging - fh = logging.FileHandler(logspath / logfile, 'w', 'utf-8') - fh.setLevel(logging.DEBUG) - fh.setFormatter(logging.Formatter('%(asctime)s - [%(levelname)s] %(message)s')) - logger.addHandler(fh) - logger.info(f"{platform.platform(True, True)} - Python {platform.python_version()}, deemix {deemixVersion}") - - # Only keep last 5 logfiles (to preserve disk space) - logslist = listdir(logspath) - logslist.sort() - if len(logslist)>5: - for i in range(len(logslist)-5): - (logspath / logslist[i]).unlink() - - # Saves the settings - def saveSettings(self, newSettings=None, dz=None): - if newSettings: - if dz and newSettings.get('tagsLanguage') != self.settings.get('tagsLanguage'): dz.set_accept_language(newSettings.get('tagsLanguage')) - if newSettings.get('downloadLocation') != self.settings.get('downloadLocation') and not checkFolder(newSettings.get('downloadLocation')): - newSettings['downloadLocation'] = DEFAULT_SETTINGS['downloadLocation'] - makedirs(newSettings['downloadLocation'], exist_ok=True) - self.settings = newSettings - with open(self.configFolder / 'config.json', 'w') as configFile: - json.dump(self.settings, configFile, indent=2) - - # Checks if the default settings have changed - def settingsCheck(self): - changes = 0 - for set in DEFAULT_SETTINGS: - if not set in self.settings or type(self.settings[set]) != type(DEFAULT_SETTINGS[set]): - self.settings[set] = DEFAULT_SETTINGS[set] - changes += 1 - for set in DEFAULT_SETTINGS['tags']: - if not set in self.settings['tags'] or type(self.settings['tags'][set]) != type(DEFAULT_SETTINGS['tags'][set]): - self.settings['tags'][set] = DEFAULT_SETTINGS['tags'][set] - changes += 1 - if self.settings['downloadLocation'] == "": - self.settings['downloadLocation'] = DEFAULT_SETTINGS['downloadLocation'] - changes += 1 - for template in ['tracknameTemplate', 'albumTracknameTemplate', 'playlistTracknameTemplate', 'playlistNameTemplate', 'artistNameTemplate', 'albumNameTemplate', 'playlistFilenameTemplate', 'coverImageTemplate', 'artistImageTemplate', 'paddingSize']: - if self.settings[template] == "": - self.settings[template] = DEFAULT_SETTINGS[template] - changes += 1 - return changes diff --git a/deemix/app/spotifyhelper.py b/deemix/app/spotifyhelper.py deleted file mode 100644 index 0eaf907..0000000 --- a/deemix/app/spotifyhelper.py +++ /dev/null @@ -1,349 +0,0 @@ -import eventlet -import json -from pathlib import Path - -eventlet.import_patched('requests.adapters') - -spotipy = eventlet.import_patched('spotipy') -SpotifyClientCredentials = spotipy.oauth2.SpotifyClientCredentials -from deemix.utils.localpaths import getConfigFolder -from deemix.app.queueitem import QIConvertable - -emptyPlaylist = { - 'collaborative': False, - 'description': "", - 'external_urls': {'spotify': None}, - 'followers': {'total': 0, 'href': None}, - 'id': None, - 'images': [], - 'name': "Something went wrong", - 'owner': { - 'display_name': "Error", - 'id': None - }, - 'public': True, - 'tracks' : [], - 'type': 'playlist', - 'uri': None -} - -class SpotifyHelper: - def __init__(self, configFolder=None): - self.credentials = {} - self.spotifyEnabled = False - self.sp = None - self.configFolder = configFolder - - # Make sure config folder exists - if not self.configFolder: - self.configFolder = getConfigFolder() - self.configFolder = Path(self.configFolder) - if not self.configFolder.is_dir(): - self.configFolder.mkdir() - - # Make sure authCredentials exsits - if not (self.configFolder / 'authCredentials.json').is_file(): - with open(self.configFolder / 'authCredentials.json', 'w') as f: - json.dump({'clientId': "", 'clientSecret': ""}, f, indent=2) - - # Load spotify id and secret and check if they are usable - with open(self.configFolder / 'authCredentials.json', 'r') as credentialsFile: - self.credentials = json.load(credentialsFile) - self.checkCredentials() - self.checkValidCache() - - def checkValidCache(self): - if (self.configFolder / 'spotifyCache.json').is_file(): - with open(self.configFolder / 'spotifyCache.json', 'r') as spotifyCache: - try: - cache = json.load(spotifyCache) - except Exception as e: - print(str(e)) - (self.configFolder / 'spotifyCache.json').unlink() - return - # Remove old versions of cache - if len(cache['tracks'].values()) and isinstance(list(cache['tracks'].values())[0], int) or \ - len(cache['albums'].values()) and isinstance(list(cache['albums'].values())[0], int): - (self.configFolder / 'spotifyCache.json').unlink() - - def checkCredentials(self): - if self.credentials['clientId'] == "" or self.credentials['clientSecret'] == "": - spotifyEnabled = False - else: - try: - client_credentials_manager = SpotifyClientCredentials(client_id=self.credentials['clientId'], - client_secret=self.credentials['clientSecret']) - self.sp = spotipy.Spotify(client_credentials_manager=client_credentials_manager) - self.sp.user_playlists('spotify') - self.spotifyEnabled = True - except Exception as e: - self.spotifyEnabled = False - return self.spotifyEnabled - - def getCredentials(self): - return self.credentials - - def setCredentials(self, spotifyCredentials): - # Remove extra spaces, just to be sure - spotifyCredentials['clientId'] = spotifyCredentials['clientId'].strip() - spotifyCredentials['clientSecret'] = spotifyCredentials['clientSecret'].strip() - - # Save them to disk - with open(self.configFolder / 'authCredentials.json', 'w') as f: - json.dump(spotifyCredentials, f, indent=2) - - # Check if they are usable - self.credentials = spotifyCredentials - self.checkCredentials() - - # Converts spotify API playlist structure to deezer's playlist structure - def _convert_playlist_structure(self, spotify_obj): - if len(spotify_obj['images']): - url = spotify_obj['images'][0]['url'] - else: - url = False - deezer_obj = { - 'checksum': spotify_obj['snapshot_id'], - 'collaborative': spotify_obj['collaborative'], - 'creation_date': "XXXX-00-00", - 'creator': { - 'id': spotify_obj['owner']['id'], - 'name': spotify_obj['owner']['display_name'], - 'tracklist': spotify_obj['owner']['href'], - 'type': "user" - }, - 'description': spotify_obj['description'], - 'duration': 0, - 'fans': spotify_obj['followers']['total'] if 'followers' in spotify_obj else 0, - 'id': spotify_obj['id'], - 'is_loved_track': False, - 'link': spotify_obj['external_urls']['spotify'], - 'nb_tracks': spotify_obj['tracks']['total'], - 'picture': url, - 'picture_small': url, - 'picture_medium': url, - 'picture_big': url, - 'picture_xl': url, - 'public': spotify_obj['public'], - 'share': spotify_obj['external_urls']['spotify'], - 'title': spotify_obj['name'], - 'tracklist': spotify_obj['tracks']['href'], - 'type': "playlist" - } - if not url: - deezer_obj['picture_small'] = "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/56x56-000000-80-0-0.jpg" - deezer_obj['picture_medium'] = "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/250x250-000000-80-0-0.jpg" - deezer_obj['picture_big'] = "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/500x500-000000-80-0-0.jpg" - deezer_obj['picture_xl'] = "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/1000x1000-000000-80-0-0.jpg" - return deezer_obj - - # Returns deezer song_id from spotify track_id or track dict - def get_trackid_spotify(self, dz, track_id, fallbackSearch, spotifyTrack=None): - if not self.spotifyEnabled: - raise spotifyFeaturesNotEnabled - singleTrack = False - if not spotifyTrack: - if (self.configFolder / 'spotifyCache.json').is_file(): - with open(self.configFolder / 'spotifyCache.json', 'r') as spotifyCache: - cache = json.load(spotifyCache) - else: - cache = {'tracks': {}, 'albums': {}} - if str(track_id) in cache['tracks']: - dz_track = None - if cache['tracks'][str(track_id)]['isrc']: - dz_track = dz.api.get_track_by_ISRC(cache['tracks'][str(track_id)]['isrc']) - dz_id = dz_track['id'] if 'id' in dz_track and 'title' in dz_track else "0" - cache['tracks'][str(track_id)]['id'] = dz_id - return (cache['tracks'][str(track_id)]['id'], dz_track, cache['tracks'][str(track_id)]['isrc']) - singleTrack = True - spotify_track = self.sp.track(track_id) - else: - spotify_track = spotifyTrack - dz_id = "0" - dz_track = None - isrc = None - if 'external_ids' in spotify_track and 'isrc' in spotify_track['external_ids']: - try: - dz_track = dz.api.get_track_by_ISRC(spotify_track['external_ids']['isrc']) - dz_id = dz_track['id'] if 'id' in dz_track and 'title' in dz_track else "0" - isrc = spotify_track['external_ids']['isrc'] - except: - dz_id = dz.api.get_track_id_from_metadata( - artist=spotify_track['artists'][0]['name'], - track=spotify_track['name'], - album=spotify_track['album']['name'] - ) if fallbackSearch else "0" - elif fallbackSearch: - dz_id = dz.api.get_track_id_from_metadata( - artist=spotify_track['artists'][0]['name'], - track=spotify_track['name'], - album=spotify_track['album']['name'] - ) - if singleTrack: - cache['tracks'][str(track_id)] = {'id': dz_id, 'isrc': isrc} - with open(self.configFolder / 'spotifyCache.json', 'w') as spotifyCache: - json.dump(cache, spotifyCache) - return (dz_id, dz_track, isrc) - - # Returns deezer album_id from spotify album_id - def get_albumid_spotify(self, dz, album_id): - if not self.spotifyEnabled: - raise spotifyFeaturesNotEnabled - if (self.configFolder / 'spotifyCache.json').is_file(): - with open(self.configFolder / 'spotifyCache.json', 'r') as spotifyCache: - cache = json.load(spotifyCache) - else: - cache = {'tracks': {}, 'albums': {}} - if str(album_id) in cache['albums']: - return cache['albums'][str(album_id)]['id'] - spotify_album = self.sp.album(album_id) - dz_album = "0" - upc = None - if 'external_ids' in spotify_album and 'upc' in spotify_album['external_ids']: - try: - dz_album = dz.api.get_album_by_UPC(spotify_album['external_ids']['upc']) - dz_album = dz_album['id'] if 'id' in dz_album else "0" - upc = spotify_album['external_ids']['upc'] - except: - try: - dz_album = dz.api.get_album_by_UPC(int(spotify_album['external_ids']['upc'])) - dz_album = dz_album['id'] if 'id' in dz_album else "0" - except: - dz_album = "0" - cache['albums'][str(album_id)] = {'id': dz_album, 'upc': upc} - with open(self.configFolder / 'spotifyCache.json', 'w') as spotifyCache: - json.dump(cache, spotifyCache) - return dz_album - - - def generate_playlist_queueitem(self, dz, playlist_id, bitrate, settings): - if not self.spotifyEnabled: - raise spotifyFeaturesNotEnabled - spotify_playlist = self.sp.playlist(playlist_id) - - if len(spotify_playlist['images']): - cover = spotify_playlist['images'][0]['url'] - else: - cover = "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/75x75-000000-80-0-0.jpg" - - playlistAPI = self._convert_playlist_structure(spotify_playlist) - playlistAPI['various_artist'] = dz.api.get_artist(5080) - - extra = {} - extra['unconverted'] = [] - - tracklistTmp = spotify_playlist['tracks']['items'] - while spotify_playlist['tracks']['next']: - spotify_playlist['tracks'] = self.sp.next(spotify_playlist['tracks']) - tracklistTmp += spotify_playlist['tracks']['items'] - for item in tracklistTmp: - if item['track']: - if item['track']['explicit']: - playlistAPI['explicit'] = True - extra['unconverted'].append(item['track']) - - totalSize = len(extra['unconverted']) - if not 'explicit' in playlistAPI: - playlistAPI['explicit'] = False - extra['playlistAPI'] = playlistAPI - return QIConvertable( - playlist_id, - bitrate, - spotify_playlist['name'], - spotify_playlist['owner']['display_name'], - cover, - playlistAPI['explicit'], - totalSize, - 'spotify_playlist', - settings, - extra, - ) - - def convert_spotify_playlist(self, dz, queueItem, interface=None): - convertPercentage = 0 - lastPercentage = 0 - if (self.configFolder / 'spotifyCache.json').is_file(): - with open(self.configFolder / 'spotifyCache.json', 'r') as spotifyCache: - cache = json.load(spotifyCache) - else: - cache = {'tracks': {}, 'albums': {}} - if interface: - interface.send("startConversion", queueItem.uuid) - collection = [] - for pos, track in enumerate(queueItem.extra['unconverted'], start=1): - if queueItem.cancel: - return - if str(track['id']) in cache['tracks']: - trackID = cache['tracks'][str(track['id'])]['id'] - trackAPI = None - if cache['tracks'][str(track['id'])]['isrc']: - trackAPI = dz.api.get_track_by_ISRC(cache['tracks'][str(track['id'])]['isrc']) - else: - (trackID, trackAPI, isrc) = self.get_trackid_spotify(dz, "0", queueItem.settings['fallbackSearch'], track) - cache['tracks'][str(track['id'])] = { - 'id': trackID, - 'isrc': isrc - } - if str(trackID) == "0": - deezerTrack = { - 'SNG_ID': "0", - 'SNG_TITLE': track['name'], - 'DURATION': 0, - 'MD5_ORIGIN': 0, - 'MEDIA_VERSION': 0, - 'FILESIZE': 0, - 'ALB_TITLE': track['album']['name'], - 'ALB_PICTURE': "", - 'ART_ID': 0, - 'ART_NAME': track['artists'][0]['name'] - } - else: - deezerTrack = dz.gw.get_track_with_fallback(trackID) - deezerTrack['_EXTRA_PLAYLIST'] = queueItem.extra['playlistAPI'] - if trackAPI: - deezerTrack['_EXTRA_TRACK'] = trackAPI - deezerTrack['POSITION'] = pos - deezerTrack['SIZE'] = queueItem.size - deezerTrack['FILENAME_TEMPLATE'] = queueItem.settings['playlistTracknameTemplate'] - collection.append(deezerTrack) - - convertPercentage = (pos / queueItem.size) * 100 - if round(convertPercentage) != lastPercentage and round(convertPercentage) % 5 == 0: - lastPercentage = round(convertPercentage) - if interface: - interface.send("updateQueue", {'uuid': queueItem.uuid, 'conversion': lastPercentage}) - - queueItem.extra = None - queueItem.collection = collection - - with open(self.configFolder / 'spotifyCache.json', 'w') as spotifyCache: - json.dump(cache, spotifyCache) - - def get_user_playlists(self, user): - if not self.spotifyEnabled: - raise spotifyFeaturesNotEnabled - result = [] - playlists = self.sp.user_playlists(user) - while playlists: - for playlist in playlists['items']: - result.append(self._convert_playlist_structure(playlist)) - if playlists['next']: - playlists = self.sp.next(playlists) - else: - playlists = None - return result - - def get_playlist_tracklist(self, id): - if not self.spotifyEnabled: - raise spotifyFeaturesNotEnabled - playlist = self.sp.playlist(id) - tracklist = playlist['tracks']['items'] - while playlist['tracks']['next']: - playlist['tracks'] = self.sp.next(playlist['tracks']) - tracklist += playlist['tracks']['items'] - playlist['tracks'] = tracklist - return playlist - - -class spotifyFeaturesNotEnabled(Exception): - pass diff --git a/deemix/decryption.py b/deemix/decryption.py new file mode 100644 index 0000000..51bef59 --- /dev/null +++ b/deemix/decryption.py @@ -0,0 +1,156 @@ +from ssl import SSLError +from time import sleep +import logging + +from requests import get +from requests.exceptions import ConnectionError as RequestsConnectionError, ReadTimeout, ChunkedEncodingError +from urllib3.exceptions import SSLError as u3SSLError + +from deemix.utils.crypto import _md5, _ecbCrypt, _ecbDecrypt, generateBlowfishKey, decryptChunk + +from deemix.utils import USER_AGENT_HEADER +from deemix.types.DownloadObjects import Single + +logger = logging.getLogger('deemix') + +def generateStreamPath(sng_id, md5, media_version, media_format): + urlPart = b'\xa4'.join( + [md5.encode(), str(media_format).encode(), str(sng_id).encode(), str(media_version).encode()]) + md5val = _md5(urlPart) + step2 = md5val.encode() + b'\xa4' + urlPart + b'\xa4' + step2 = step2 + (b'.' * (16 - (len(step2) % 16))) + urlPart = _ecbCrypt('jo6aey6haid2Teih', step2) + return urlPart.decode("utf-8") + +def reverseStreamPath(urlPart): + step2 = _ecbDecrypt('jo6aey6haid2Teih', urlPart) + (_, md5, media_format, sng_id, media_version, _) = step2.split(b'\xa4') + return (sng_id.decode('utf-8'), md5.decode('utf-8'), media_version.decode('utf-8'), media_format.decode('utf-8')) + +def generateCryptedStreamURL(sng_id, md5, media_version, media_format): + urlPart = generateStreamPath(sng_id, md5, media_version, media_format) + return "https://e-cdns-proxy-" + md5[0] + ".dzcdn.net/mobile/1/" + urlPart + +def generateStreamURL(sng_id, md5, media_version, media_format): + urlPart = generateStreamPath(sng_id, md5, media_version, media_format) + return "https://e-cdns-proxy-" + md5[0] + ".dzcdn.net/api/1/" + urlPart + +def reverseStreamURL(url): + urlPart = url[url.find("/1/")+3:] + return reverseStreamPath(urlPart) + +def streamTrack(outputStream, track, start=0, downloadObject=None, listener=None): + if downloadObject.isCanceled: raise DownloadCanceled + headers= {'User-Agent': USER_AGENT_HEADER} + chunkLength = start + + itemName = f"[{track.mainArtist.name} - {track.title}]" + + try: + with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: + request.raise_for_status() + + complete = int(request.headers["Content-Length"]) + if complete == 0: raise DownloadEmpty + if start != 0: + responseRange = request.headers["Content-Range"] + if listener: + listener.send('downloadInfo', { + 'uuid': downloadObject.uuid, + 'itemName': itemName, + 'state': "downloading", + 'alreadyStarted': True, + 'value': responseRange + }) + else: + if listener: + listener.send('downloadInfo', { + 'uuid': downloadObject.uuid, + 'itemName': itemName, + 'state': "downloading", + 'alreadyStarted': False, + 'value': complete + }) + + for chunk in request.iter_content(2048 * 3): + outputStream.write(chunk) + chunkLength += len(chunk) + + if downloadObject: + if isinstance(downloadObject, Single): + chunkProgres = (chunkLength / (complete + start)) * 100 + downloadObject.progressNext = chunkProgres + else: + chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100 + downloadObject.progressNext += chunkProgres + downloadObject.updateProgress(listener) + + except (SSLError, u3SSLError): + logger.info('%s retrying from byte %s', itemName, chunkLength) + streamTrack(outputStream, track, chunkLength, downloadObject, listener) + except (RequestsConnectionError, ReadTimeout, ChunkedEncodingError): + sleep(2) + streamTrack(outputStream, track, start, downloadObject, listener) + +def streamCryptedTrack(outputStream, track, start=0, downloadObject=None, listener=None): + if downloadObject.isCanceled: raise DownloadCanceled + headers= {'User-Agent': USER_AGENT_HEADER} + chunkLength = start + + itemName = f"[{track.mainArtist.name} - {track.title}]" + + try: + with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: + request.raise_for_status() + blowfish_key = str.encode(generateBlowfishKey(str(track.id))) + + complete = int(request.headers["Content-Length"]) + if complete == 0: raise DownloadEmpty + if start != 0: + responseRange = request.headers["Content-Range"] + if listener: + listener.send('downloadInfo', { + 'uuid': downloadObject.uuid, + 'itemName': itemName, + 'state': "downloading", + 'alreadyStarted': True, + 'value': responseRange + }) + else: + if listener: + listener.send('downloadInfo', { + 'uuid': downloadObject.uuid, + 'itemName': itemName, + 'state': "downloading", + 'alreadyStarted': False, + 'value': complete + }) + + for chunk in request.iter_content(2048 * 3): + if len(chunk) >= 2048: + chunk = decryptChunk(blowfish_key, chunk[0:2048]) + chunk[2048:] + + outputStream.write(chunk) + chunkLength += len(chunk) + + if downloadObject: + if isinstance(downloadObject, Single): + chunkProgres = (chunkLength / (complete + start)) * 100 + downloadObject.progressNext = chunkProgres + else: + chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100 + downloadObject.progressNext += chunkProgres + downloadObject.updateProgress(listener) + + except (SSLError, u3SSLError): + logger.info('%s retrying from byte %s', itemName, chunkLength) + streamCryptedTrack(outputStream, track, chunkLength, downloadObject, listener) + except (RequestsConnectionError, ReadTimeout, ChunkedEncodingError): + sleep(2) + streamCryptedTrack(outputStream, track, start, downloadObject, listener) + +class DownloadCanceled(Exception): + pass + +class DownloadEmpty(Exception): + pass diff --git a/deemix/downloader.py b/deemix/downloader.py new file mode 100644 index 0000000..a9c7247 --- /dev/null +++ b/deemix/downloader.py @@ -0,0 +1,564 @@ +from concurrent.futures import ThreadPoolExecutor +from time import sleep + +from os.path import sep as pathSep +from os import makedirs, system as execute +from pathlib import Path +from shlex import quote +import errno + +import logging +from tempfile import gettempdir + +import requests +from requests import get + +from urllib3.exceptions import SSLError as u3SSLError + +from mutagen.flac import FLACNoHeaderError, error as FLACError + +from deezer import TrackFormats +from deemix.types.DownloadObjects import Single, Collection +from deemix.types.Track import Track, AlbumDoesntExists, MD5NotFound +from deemix.types.Picture import StaticPicture +from deemix.utils import USER_AGENT_HEADER +from deemix.utils.pathtemplates import generatePath, generateAlbumName, generateArtistName, generateDownloadObjectName +from deemix.tagger import tagID3, tagFLAC +from deemix.decryption import generateStreamURL, streamTrack, DownloadCanceled +from deemix.settings import OverwriteOption + +logger = logging.getLogger('deemix') + +extensions = { + TrackFormats.FLAC: '.flac', + TrackFormats.LOCAL: '.mp3', + TrackFormats.MP3_320: '.mp3', + TrackFormats.MP3_128: '.mp3', + TrackFormats.DEFAULT: '.mp3', + TrackFormats.MP4_RA3: '.mp4', + TrackFormats.MP4_RA2: '.mp4', + TrackFormats.MP4_RA1: '.mp4' +} + +TEMPDIR = Path(gettempdir()) / 'deemix-imgs' +if not TEMPDIR.is_dir(): makedirs(TEMPDIR) + +def downloadImage(url, path, overwrite=OverwriteOption.DONT_OVERWRITE): + if path.is_file() and overwrite not in [OverwriteOption.OVERWRITE, OverwriteOption.ONLY_TAGS, OverwriteOption.KEEP_BOTH]: return path + + try: + image = get(url, headers={'User-Agent': USER_AGENT_HEADER}, timeout=30) + image.raise_for_status() + with open(path, 'wb') as f: + f.write(image.content) + return path + except requests.exceptions.HTTPError: + if path.is_file(): path.unlink() + if 'cdns-images.dzcdn.net' in url: + urlBase = url[:url.rfind("/")+1] + pictureUrl = url[len(urlBase):] + pictureSize = int(pictureUrl[:pictureUrl.find("x")]) + if pictureSize > 1200: + return downloadImage(urlBase+pictureUrl.replace(f"{pictureSize}x{pictureSize}", '1200x1200'), path, overwrite) + except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError, u3SSLError) as e: + if path.is_file(): path.unlink() + sleep(5) + return downloadImage(url, path, overwrite) + except OSError as e: + if path.is_file(): path.unlink() + if e.errno == errno.ENOSPC: raise DownloadFailed("noSpaceLeft") from e + logger.exception("Error while downloading an image, you should report this to the developers: %s", e) + return None + +def getPreferredBitrate(track, bitrate, shouldFallback, uuid=None, listener=None): + bitrate = int(bitrate) + if track.local: return TrackFormats.LOCAL + + falledBack = False + + formats_non_360 = { + TrackFormats.FLAC: "FLAC", + TrackFormats.MP3_320: "MP3_320", + TrackFormats.MP3_128: "MP3_128", + } + formats_360 = { + TrackFormats.MP4_RA3: "MP4_RA3", + TrackFormats.MP4_RA2: "MP4_RA2", + TrackFormats.MP4_RA1: "MP4_RA1", + } + + is360format = bitrate in formats_360.keys() + + if not shouldFallback: + formats = formats_360 + formats.update(formats_non_360) + elif is360format: + formats = formats_360 + else: + formats = formats_non_360 + + def testBitrate(track, formatNumber, formatName): + request = requests.head( + generateStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), + headers={'User-Agent': USER_AGENT_HEADER}, + timeout=30 + ) + try: + request.raise_for_status() + track.filesizes[f"FILESIZE_{formatName}"] = int(request.headers["Content-Length"]) + track.filesizes[f"FILESIZE_{formatName}_TESTED"] = True + if track.filesizes[f"FILESIZE_{formatName}"] == 0: return None + return formatNumber + except requests.exceptions.HTTPError: # if the format is not available, Deezer returns a 403 error + return None + + for formatNumber, formatName in formats.items(): + if formatNumber > bitrate: continue + if f"FILESIZE_{formatName}" in track.filesizes: + if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber + if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]: + testedBitrate = testBitrate(track, formatNumber, formatName) + if testedBitrate: return testedBitrate + + if not shouldFallback: + raise PreferredBitrateNotFound + if not falledBack: + falledBack = True + logger.info("%s Fallback to lower bitrate", f"[{track.mainArtist.name} - {track.title}]") + if listener and uuid: + listener.send('queueUpdate', { + 'uuid': uuid, + 'bitrateFallback': True, + 'data': { + 'id': track.id, + 'title': track.title, + 'artist': track.mainArtist.name + }, + }) + if is360format: raise TrackNot360 + return TrackFormats.DEFAULT + +class Downloader: + def __init__(self, dz, downloadObject, settings, listener=None): + self.dz = dz + self.downloadObject = downloadObject + self.settings = settings + self.bitrate = downloadObject.bitrate + self.listener = listener + + self.extrasPath = None + self.playlistCoverName = None + self.playlistURLs = [] + + def start(self): + if not self.downloadObject.isCanceled: + if isinstance(self.downloadObject, Single): + track = self.downloadWrapper({ + 'trackAPI_gw': self.downloadObject.single['trackAPI_gw'], + 'trackAPI': self.downloadObject.single.get('trackAPI'), + 'albumAPI': self.downloadObject.single.get('albumAPI') + }) + if track: self.afterDownloadSingle(track) + elif isinstance(self.downloadObject, Collection): + tracks = [None] * len(self.downloadObject.collection['tracks_gw']) + with ThreadPoolExecutor(self.settings['queueConcurrency']) as executor: + for pos, track in enumerate(self.downloadObject.collection['tracks_gw'], start=0): + tracks[pos] = executor.submit(self.downloadWrapper, { + 'trackAPI_gw': track, + 'albumAPI': self.downloadObject.collection.get('albumAPI'), + 'playlistAPI': self.downloadObject.collection.get('playlistAPI') + }) + self.afterDownloadCollection(tracks) + + if self.listener: + if self.listener: + self.listener.send('currentItemCancelled', self.downloadObject.uuid) + self.listener.send("removedFromQueue", self.downloadObject.uuid) + else: + self.listener.send("finishDownload", self.downloadObject.uuid) + + def download(self, extraData, track=None): + returnData = {} + trackAPI_gw = extraData['trackAPI_gw'] + trackAPI = extraData.get('trackAPI') + albumAPI = extraData.get('albumAPI') + playlistAPI = extraData.get('playlistAPI') + if self.downloadObject.isCanceled: raise DownloadCanceled + if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer") + + itemName = f"[{trackAPI_gw['ART_NAME']} - {trackAPI_gw['SNG_TITLE']}]" + + # Create Track object + if not track: + logger.info("%s Getting the tags", itemName) + try: + track = Track().parseData( + dz=self.dz, + trackAPI_gw=trackAPI_gw, + trackAPI=trackAPI, + albumAPI=albumAPI, + playlistAPI=playlistAPI + ) + except AlbumDoesntExists as e: + raise DownloadError('albumDoesntExists') from e + except MD5NotFound as e: + raise DownloadError('notLoggedIn') from e + + itemName = f"[{track.mainArtist.name} - {track.title}]" + + # Check if track not yet encoded + if track.MD5 == '': raise DownloadFailed("notEncoded", track) + + # Choose the target bitrate + try: + selectedFormat = getPreferredBitrate( + track, + self.bitrate, + self.settings['fallbackBitrate'], + self.downloadObject.uuid, self.listener + ) + except PreferredBitrateNotFound as e: + raise DownloadFailed("wrongBitrate", track) from e + except TrackNot360 as e: + raise DownloadFailed("no360RA") from e + track.bitrate = selectedFormat + track.album.bitrate = selectedFormat + + # Apply settings + track.applySettings(self.settings) + + # Generate filename and filepath from metadata + (filename, filepath, artistPath, coverPath, extrasPath) = generatePath(track, self.downloadObject, self.settings) + + # Make sure the filepath exists + makedirs(filepath, exist_ok=True) + extension = extensions[track.bitrate] + writepath = filepath / f"{filename}{extension}" + + # Save extrasPath + if extrasPath and not self.extrasPath: self.extrasPath = extrasPath + + # Generate covers URLs + embeddedImageFormat = f'jpg-{self.settings["jpegImageQuality"]}' + if self.settings['embeddedArtworkPNG']: embeddedImageFormat = 'png' + + track.album.embeddedCoverURL = track.album.pic.getURL(self.settings['embeddedArtworkSize'], embeddedImageFormat) + ext = track.album.embeddedCoverURL[-4:] + if ext[0] != ".": ext = ".jpg" # Check for Spotify images + track.album.embeddedCoverPath = TEMPDIR / ((f"pl{track.playlist.id}" if track.album.isPlaylist else f"alb{track.album.id}") + f"_{self.settings['embeddedArtworkSize']}{ext}") + + # Download and cache coverart + logger.info("%s Getting the album cover", itemName) + track.album.embeddedCoverPath = downloadImage(track.album.embeddedCoverURL, track.album.embeddedCoverPath) + + # Save local album art + if coverPath: + returnData['albumURLs'] = [] + for pic_format in self.settings['localArtworkFormat'].split(","): + if pic_format in ["png","jpg"]: + extendedFormat = pic_format + if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" + url = track.album.pic.getURL(self.settings['localArtworkSize'], extendedFormat) + # Skip non deezer pictures at the wrong format + if isinstance(track.album.pic, StaticPicture) and pic_format != "jpg": + continue + returnData['albumURLs'].append({'url': url, 'ext': pic_format}) + returnData['albumPath'] = coverPath + returnData['albumFilename'] = generateAlbumName(self.settings['coverImageTemplate'], track.album, self.settings, track.playlist) + + # Save artist art + if artistPath: + returnData['artistURLs'] = [] + for pic_format in self.settings['localArtworkFormat'].split(","): + # Deezer doesn't support png artist images + if pic_format == "jpg": + extendedFormat = f"{pic_format}-{self.settings['jpegImageQuality']}" + url = track.album.mainArtist.pic.getURL(self.settings['localArtworkSize'], extendedFormat) + if track.album.mainArtist.pic.md5 == "": continue + returnData['artistURLs'].append({'url': url, 'ext': pic_format}) + returnData['artistPath'] = artistPath + returnData['artistFilename'] = generateArtistName(self.settings['artistImageTemplate'], track.album.mainArtist, self.settings, rootArtist=track.album.rootArtist) + + # Save playlist art + if track.playlist: + if len(self.playlistURLs) == 0: + for pic_format in self.settings['localArtworkFormat'].split(","): + if pic_format in ["png","jpg"]: + extendedFormat = pic_format + if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" + url = track.playlist.pic.getURL(self.settings['localArtworkSize'], extendedFormat) + if isinstance(track.playlist.pic, StaticPicture) and pic_format != "jpg": continue + self.playlistURLs.append({'url': url, 'ext': pic_format}) + if not self.playlistCoverName: + track.playlist.bitrate = selectedFormat + track.playlist.dateString = track.playlist.date.format(self.settings['dateFormat']) + self.playlistCoverName = generateAlbumName(self.settings['coverImageTemplate'], track.playlist, self.settings, track.playlist) + + # Save lyrics in lrc file + if self.settings['syncedLyrics'] and track.lyrics.sync: + if not (filepath / f"{filename}.lrc").is_file() or self.settings['overwriteFile'] in [OverwriteOption.OVERWRITE, OverwriteOption.ONLY_TAGS]: + with open(filepath / f"{filename}.lrc", 'wb') as f: + f.write(track.lyrics.sync.encode('utf-8')) + + # Check for overwrite settings + trackAlreadyDownloaded = writepath.is_file() + + # Don't overwrite and don't mind extension + if not trackAlreadyDownloaded and self.settings['overwriteFile'] == OverwriteOption.DONT_CHECK_EXT: + exts = ['.mp3', '.flac', '.opus', '.m4a'] + baseFilename = str(filepath / filename) + for ext in exts: + trackAlreadyDownloaded = Path(baseFilename+ext).is_file() + if trackAlreadyDownloaded: break + # Don't overwrite and keep both files + if trackAlreadyDownloaded and self.settings['overwriteFile'] == OverwriteOption.KEEP_BOTH: + baseFilename = str(filepath / filename) + c = 1 + currentFilename = baseFilename+' ('+str(c)+')'+ extension + while Path(currentFilename).is_file(): + c += 1 + currentFilename = baseFilename+' ('+str(c)+')'+ extension + trackAlreadyDownloaded = False + writepath = Path(currentFilename) + + if not trackAlreadyDownloaded or self.settings['overwriteFile'] == OverwriteOption.OVERWRITE: + logger.info("%s Downloading the track", itemName) + track.downloadUrl = generateStreamURL(track.id, track.MD5, track.mediaVersion, track.bitrate) + + try: + with open(writepath, 'wb') as stream: + streamTrack(stream, track, downloadObject=self.downloadObject, listener=self.listener) + except requests.exceptions.HTTPError as e: + raise DownloadFailed('notAvailable', track) from e + except OSError as e: + if writepath.is_file(): writepath.unlink() + if e.errno == errno.ENOSPC: raise DownloadFailed("noSpaceLeft") from e + raise e + + else: + logger.info("%s Skipping track as it's already downloaded", itemName) + self.downloadObject.completeTrackProgress(self.listener) + + # Adding tags + if (not trackAlreadyDownloaded or self.settings['overwriteFile'] in [OverwriteOption.ONLY_TAGS, OverwriteOption.OVERWRITE]) and not track.local: + logger.info("%s Applying tags to the track", itemName) + if extension == '.mp3': + tagID3(writepath, track, self.settings['tags']) + elif extension == '.flac': + try: + tagFLAC(writepath, track, self.settings['tags']) + except (FLACNoHeaderError, FLACError): + writepath.unlink() + logger.warning("%s Track not available in FLAC, falling back if necessary", itemName) + self.downloadObject.removeTrackProgress(self.listener) + track.filesizes['FILESIZE_FLAC'] = "0" + track.filesizes['FILESIZE_FLAC_TESTED'] = True + return self.download(trackAPI_gw, track=track) + + if track.searched: returnData['searched'] = True + self.downloadObject.downloaded += 1 + self.downloadObject.files.append(str(writepath)) + self.downloadObject.extrasPath = str(self.extrasPath) + logger.info("%s Track download completed\n%s", itemName, writepath) + if self.listener: self.listener.send("updateQueue", { + 'uuid': self.downloadObject.uuid, + 'downloaded': True, + 'downloadPath': str(writepath), + 'extrasPath': str(self.extrasPath) + }) + returnData['filename'] = str(writepath)[len(str(extrasPath))+ len(pathSep):] + returnData['data'] = { + 'id': track.id, + 'title': track.title, + 'artist': track.mainArtist.name + } + return returnData + + def downloadWrapper(self, extraData, track=None): + trackAPI_gw = extraData['trackAPI_gw'] + if ('_EXTRA_TRACK' in trackAPI_gw): + extraData['trackAPI'] = trackAPI_gw['_EXTRA_TRACK'].copy() + del extraData['trackAPI_gw']['_EXTRA_TRACK'] + del trackAPI_gw['_EXTRA_TRACK'] + # Temp metadata to generate logs + tempTrack = { + 'id': trackAPI_gw['SNG_ID'], + 'title': trackAPI_gw['SNG_TITLE'].strip(), + 'artist': trackAPI_gw['ART_NAME'] + } + if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: + tempTrack['title'] += f" {trackAPI_gw['VERSION']}".strip() + + itemName = f"[{tempTrack['artist']} - {tempTrack['title']}]" + + try: + result = self.download(extraData, track) + except DownloadFailed as error: + if error.track: + track = error.track + if track.fallbackID != "0": + logger.warning("%s %s Using fallback id", itemName, error.message) + newTrack = self.dz.gw.get_track_with_fallback(track.fallbackID) + track.parseEssentialData(newTrack) + track.retriveFilesizes(self.dz) + return self.downloadWrapper(extraData, track) + if not track.searched and self.settings['fallbackSearch']: + logger.warning("%s %s Searching for alternative", itemName, error.message) + searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) + if searchedId != "0": + newTrack = self.dz.gw.get_track_with_fallback(searchedId) + track.parseEssentialData(newTrack) + track.retriveFilesizes(self.dz) + track.searched = True + if self.listener: self.listener.send('queueUpdate', { + 'uuid': self.downloadObject.uuid, + 'searchFallback': True, + 'data': { + 'id': track.id, + 'title': track.title, + 'artist': track.mainArtist.name + }, + }) + return self.downloadWrapper(extraData, track) + error.errid += "NoAlternative" + error.message = errorMessages[error.errid] + logger.error("%s %s", itemName, error.message) + result = {'error': { + 'message': error.message, + 'errid': error.errid, + 'data': tempTrack + }} + except Exception as e: + logger.exception("%s %s", itemName, e) + result = {'error': { + 'message': str(e), + 'data': tempTrack + }} + + if 'error' in result: + self.downloadObject.completeTrackProgress(self.listener) + self.downloadObject.failed += 1 + self.downloadObject.errors.append(result['error']) + if self.listener: + error = result['error'] + self.listener.send("updateQueue", { + 'uuid': self.downloadObject.uuid, + 'failed': True, + 'data': error['data'], + 'error': error['message'], + 'errid': error['errid'] if 'errid' in error else None + }) + return result + + def afterDownloadSingle(self, track): + if not self.extrasPath: self.extrasPath = Path(self.settings['downloadLocation']) + + # Save Album Cover + if self.settings['saveArtwork'] and 'albumPath' in track: + for image in track['albumURLs']: + downloadImage(image['url'], track['albumPath'] / f"{track['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) + + # Save Artist Artwork + if self.settings['saveArtworkArtist'] and 'artistPath' in track: + for image in track['artistURLs']: + downloadImage(image['url'], track['artistPath'] / f"{track['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) + + # Create searched logfile + if self.settings['logSearched'] and 'searched' in track: + filename = f"{track.data.artist} - {track.data.title}" + with open(self.extrasPath / 'searched.txt', 'wb+') as f: + searchedFile = f.read().decode('utf-8') + if not filename in searchedFile: + if searchedFile != "": searchedFile += "\r\n" + searchedFile += filename + "\r\n" + f.write(searchedFile.encode('utf-8')) + + # Execute command after download + if self.settings['executeCommand'] != "": + execute(self.settings['executeCommand'].replace("%folder%", quote(str(self.extrasPath))).replace("%filename%", quote(track['filename'])), shell=True) + + def afterDownloadCollection(self, tracks): + if not self.extrasPath: self.extrasPath = Path(self.settings['downloadLocation']) + playlist = [None] * len(tracks) + errors = "" + searched = "" + + for i, track in enumerate(tracks): + track = track.result() + if not track: return # Check if item is cancelled + + # Log errors to file + if track.get('error'): + if not track['error'].get('data'): track['error']['data'] = {'id': "0", 'title': 'Unknown', 'artist': 'Unknown'} + errors += f"{track['error']['data']['id']} | {track['error']['data']['artist']} - {track['error']['data']['title']} | {track['error']['message']}\r\n" + + # Log searched to file + if 'searched' in track: searched += track['searched'] + "\r\n" + + # Save Album Cover + if self.settings['saveArtwork'] and 'albumPath' in track: + for image in track['albumURLs']: + downloadImage(image['url'], track['albumPath'] / f"{track['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) + + # Save Artist Artwork + if self.settings['saveArtworkArtist'] and 'artistPath' in track: + for image in track['artistURLs']: + downloadImage(image['url'], track['artistPath'] / f"{track['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) + + # Save filename for playlist file + playlist[i] = track.get('filename', "") + + # Create errors logfile + if self.settings['logErrors'] and errors != "": + with open(self.extrasPath / 'errors.txt', 'wb') as f: + f.write(errors.encode('utf-8')) + + # Create searched logfile + if self.settings['logSearched'] and searched != "": + with open(self.extrasPath / 'searched.txt', 'wb') as f: + f.write(searched.encode('utf-8')) + + # Save Playlist Artwork + if self.settings['saveArtwork'] and self.playlistCoverName and not self.settings['tags']['savePlaylistAsCompilation']: + for image in self.playlistURLs: + downloadImage(image['url'], self.extrasPath / f"{self.playlistCoverName}.{image['ext']}", self.settings['overwriteFile']) + + # Create M3U8 File + if self.settings['createM3U8File']: + filename = generateDownloadObjectName(self.settings['playlistFilenameTemplate'], self.downloadObject, self.settings) or "playlist" + with open(self.extrasPath / f'{filename}.m3u8', 'wb') as f: + for line in playlist: + f.write((line + "\n").encode('utf-8')) + + # Execute command after download + if self.settings['executeCommand'] != "": + execute(self.settings['executeCommand'].replace("%folder%", quote(str(self.extrasPath))), shell=True) + +class DownloadError(Exception): + """Base class for exceptions in this module.""" + +errorMessages = { + 'notOnDeezer': "Track not available on Deezer!", + 'notEncoded': "Track not yet encoded!", + 'notEncodedNoAlternative': "Track not yet encoded and no alternative found!", + 'wrongBitrate': "Track not found at desired bitrate.", + 'wrongBitrateNoAlternative': "Track not found at desired bitrate and no alternative found!", + 'no360RA': "Track is not available in Reality Audio 360.", + 'notAvailable': "Track not available on deezer's servers!", + 'notAvailableNoAlternative': "Track not available on deezer's servers and no alternative found!", + 'noSpaceLeft': "No space left on target drive, clean up some space for the tracks", + 'albumDoesntExists': "Track's album does not exsist, failed to gather info" +} + +class DownloadFailed(DownloadError): + def __init__(self, errid, track=None): + super().__init__() + self.errid = errid + self.message = errorMessages[self.errid] + self.track = track + +class PreferredBitrateNotFound(DownloadError): + pass + +class TrackNot360(DownloadError): + pass diff --git a/deemix/itemgen.py b/deemix/itemgen.py new file mode 100644 index 0000000..fd5f6b8 --- /dev/null +++ b/deemix/itemgen.py @@ -0,0 +1,307 @@ +import logging + +from deemix.types.DownloadObjects import Single, Collection +from deezer.gw import GWAPIError, LyricsStatus +from deezer.api import APIError +from deezer.utils import map_user_playlist + +logger = logging.getLogger('deemix') + +def generateTrackItem(dz, link_id, bitrate, trackAPI=None, albumAPI=None): + # Check if is an isrc: url + if str(link_id).startswith("isrc"): + try: + trackAPI = dz.api.get_track(link_id) + except APIError as e: + raise GenerationError(f"https://deezer.com/track/{link_id}", str(e)) from e + + if 'id' in trackAPI and 'title' in trackAPI: + link_id = trackAPI['id'] + else: + raise ISRCnotOnDeezer(f"https://deezer.com/track/{link_id}") + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/track/{link_id}") + + # Get essential track info + try: + trackAPI_gw = dz.gw.get_track_with_fallback(link_id) + except GWAPIError as e: + raise GenerationError(f"https://deezer.com/track/{link_id}", str(e)) from e + + title = trackAPI_gw['SNG_TITLE'].strip() + if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: + title += f" {trackAPI_gw['VERSION']}".strip() + explicit = bool(int(trackAPI_gw.get('EXPLICIT_LYRICS', 0))) + + return Single({ + 'type': 'track', + 'id': link_id, + 'bitrate': bitrate, + 'title': title, + 'artist': trackAPI_gw['ART_NAME'], + 'cover': f"https://e-cdns-images.dzcdn.net/images/cover/{trackAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg", + 'explicit': explicit, + 'single': { + 'trackAPI_gw': trackAPI_gw, + 'trackAPI': trackAPI, + 'albumAPI': albumAPI + } + }) + +def generateAlbumItem(dz, link_id, bitrate, rootArtist=None): + # Get essential album info + if str(link_id).startswith('upc'): + upcs = [link_id[4:],] + upcs.append(int(upcs[0])) + lastError = None + for upc in upcs: + try: + albumAPI = dz.api.get_album(f"upc:{upc}") + except APIError as e: + lastError = e + albumAPI = None + if not albumAPI: + raise GenerationError(f"https://deezer.com/album/{link_id}", str(lastError)) from lastError + link_id = albumAPI['id'] + else: + try: + albumAPI = dz.api.get_album(link_id) + except APIError as e: + raise GenerationError(f"https://deezer.com/album/{link_id}", str(e)) from e + + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/album/{link_id}") + + # Get extra info about album + # This saves extra api calls when downloading + albumAPI_gw = dz.gw.get_album(link_id) + albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK'] + albumAPI['copyright'] = albumAPI_gw['COPYRIGHT'] + albumAPI['release_date'] = albumAPI_gw['PHYSICAL_RELEASE_DATE'] + albumAPI['root_artist'] = rootArtist + + # If the album is a single download as a track + if albumAPI['nb_tracks'] == 1: + if len(albumAPI['tracks']['data']): + return generateTrackItem(dz, albumAPI['tracks']['data'][0]['id'], bitrate, albumAPI=albumAPI) + raise GenerationError(f"https://deezer.com/album/{link_id}", "Single has no tracks.") + + tracksArray = dz.gw.get_album_tracks(link_id) + + if albumAPI['cover_small'] is not None: + cover = albumAPI['cover_small'][:-24] + '/75x75-000000-80-0-0.jpg' + else: + cover = f"https://e-cdns-images.dzcdn.net/images/cover/{albumAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg" + + totalSize = len(tracksArray) + albumAPI['nb_tracks'] = totalSize + collection = [] + for pos, trackAPI in enumerate(tracksArray, start=1): + trackAPI['POSITION'] = pos + trackAPI['SIZE'] = totalSize + collection.append(trackAPI) + + explicit = albumAPI_gw.get('EXPLICIT_ALBUM_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT] + + return Collection({ + 'type': 'album', + 'id': link_id, + 'bitrate': bitrate, + 'title': albumAPI['title'], + 'artist': albumAPI['artist']['name'], + 'cover': cover, + 'explicit': explicit, + 'size': totalSize, + 'collection': { + 'tracks_gw': collection, + 'albumAPI': albumAPI + } + }) + +def generatePlaylistItem(dz, link_id, bitrate, playlistAPI=None, playlistTracksAPI=None): + if not playlistAPI: + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/playlist/{link_id}") + # Get essential playlist info + try: + playlistAPI = dz.api.get_playlist(link_id) + except APIError: + playlistAPI = None + # Fallback to gw api if the playlist is private + if not playlistAPI: + try: + userPlaylist = dz.gw.get_playlist_page(link_id) + playlistAPI = map_user_playlist(userPlaylist['DATA']) + except GWAPIError as e: + raise GenerationError(f"https://deezer.com/playlist/{link_id}", str(e)) from e + + # Check if private playlist and owner + if not playlistAPI.get('public', False) and playlistAPI['creator']['id'] != str(dz.current_user['id']): + logger.warning("You can't download others private playlists.") + raise NotYourPrivatePlaylist(f"https://deezer.com/playlist/{link_id}") + + if not playlistTracksAPI: + playlistTracksAPI = dz.gw.get_playlist_tracks(link_id) + playlistAPI['various_artist'] = dz.api.get_artist(5080) # Useful for save as compilation + + totalSize = len(playlistTracksAPI) + playlistAPI['nb_tracks'] = totalSize + collection = [] + for pos, trackAPI in enumerate(playlistTracksAPI, start=1): + if trackAPI.get('EXPLICIT_TRACK_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT]: + playlistAPI['explicit'] = True + trackAPI['POSITION'] = pos + trackAPI['SIZE'] = totalSize + collection.append(trackAPI) + + if 'explicit' not in playlistAPI: playlistAPI['explicit'] = False + + return Collection({ + 'type': 'playlist', + 'id': link_id, + 'bitrate': bitrate, + 'title': playlistAPI['title'], + 'artist': playlistAPI['creator']['name'], + 'cover': playlistAPI['picture_small'][:-24] + '/75x75-000000-80-0-0.jpg', + 'explicit': playlistAPI['explicit'], + 'size': totalSize, + 'collection': { + 'tracks_gw': collection, + 'playlistAPI': playlistAPI + } + }) + +def generateArtistItem(dz, link_id, bitrate, listener=None): + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/artist/{link_id}") + # Get essential artist info + try: + artistAPI = dz.api.get_artist(link_id) + except APIError as e: + raise GenerationError(f"https://deezer.com/artist/{link_id}", str(e)) from e + + rootArtist = { + 'id': artistAPI['id'], + 'name': artistAPI['name'], + 'picture_small': artistAPI['picture_small'] + } + if listener: listener.send("startAddingArtist", rootArtist) + + artistDiscographyAPI = dz.gw.get_artist_discography_tabs(link_id, 100) + allReleases = artistDiscographyAPI.pop('all', []) + albumList = [] + for album in allReleases: + try: + albumList.append(generateAlbumItem(dz, album['id'], bitrate, rootArtist=rootArtist)) + except GenerationError as e: + logger.warning("Album %s has no data: %s", str(album['id']), str(e)) + + if listener: listener.send("finishAddingArtist", rootArtist) + return albumList + +def generateArtistDiscographyItem(dz, link_id, bitrate, listener=None): + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/artist/{link_id}/discography") + # Get essential artist info + try: + artistAPI = dz.api.get_artist(link_id) + except APIError as e: + raise GenerationError(f"https://deezer.com/artist/{link_id}/discography", str(e)) from e + + rootArtist = { + 'id': artistAPI['id'], + 'name': artistAPI['name'], + 'picture_small': artistAPI['picture_small'] + } + if listener: listener.send("startAddingArtist", rootArtist) + + artistDiscographyAPI = dz.gw.get_artist_discography_tabs(link_id, 100) + artistDiscographyAPI.pop('all', None) # all contains albums and singles, so its all duplicates. This removes them + albumList = [] + for releaseType in artistDiscographyAPI: + for album in artistDiscographyAPI[releaseType]: + try: + albumList.append(generateAlbumItem(dz, album['id'], bitrate, rootArtist=rootArtist)) + except GenerationError as e: + logger.warning("Album %s has no data: %s", str(album['id']), str(e)) + + if listener: listener.send("finishAddingArtist", rootArtist) + return albumList + +def generateArtistTopItem(dz, link_id, bitrate): + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/artist/{link_id}/top_track") + # Get essential artist info + try: + artistAPI = dz.api.get_artist(link_id) + except APIError as e: + raise GenerationError(f"https://deezer.com/artist/{link_id}/top_track", str(e)) from e + + # Emulate the creation of a playlist + # Can't use generatePlaylistItem directly as this is not a real playlist + playlistAPI = { + 'id':f"{artistAPI['id']}_top_track", + 'title': f"{artistAPI['name']} - Top Tracks", + 'description': f"Top Tracks for {artistAPI['name']}", + 'duration': 0, + 'public': True, + 'is_loved_track': False, + 'collaborative': False, + 'nb_tracks': 0, + 'fans': artistAPI['nb_fan'], + 'link': f"https://www.deezer.com/artist/{artistAPI['id']}/top_track", + 'share': None, + 'picture': artistAPI['picture'], + 'picture_small': artistAPI['picture_small'], + 'picture_medium': artistAPI['picture_medium'], + 'picture_big': artistAPI['picture_big'], + 'picture_xl': artistAPI['picture_xl'], + 'checksum': None, + 'tracklist': f"https://api.deezer.com/artist/{artistAPI['id']}/top", + 'creation_date': "XXXX-00-00", + 'creator': { + 'id': f"art_{artistAPI['id']}", + 'name': artistAPI['name'], + 'type': "user" + }, + 'type': "playlist" + } + + artistTopTracksAPI_gw = dz.gw.get_artist_toptracks(link_id) + return generatePlaylistItem(dz, playlistAPI['id'], bitrate, playlistAPI=playlistAPI, playlistTracksAPI=artistTopTracksAPI_gw) + +class GenerationError(Exception): + def __init__(self, link, message, errid=None): + super().__init__() + self.link = link + self.message = message + self.errid = errid + + def toDict(self): + return { + 'link': self.link, + 'error': self.message, + 'errid': self.errid + } + +class ISRCnotOnDeezer(GenerationError): + def __init__(self, link): + super().__init__(link, "Track ISRC is not available on deezer", "ISRCnotOnDeezer") + +class NotYourPrivatePlaylist(GenerationError): + def __init__(self, link): + super().__init__(link, "You can't download others private playlists.", "notYourPrivatePlaylist") + +class TrackNotOnDeezer(GenerationError): + def __init__(self, link): + super().__init__(link, "Track not found on deezer!", "trackNotOnDeezer") + +class AlbumNotOnDeezer(GenerationError): + def __init__(self, link): + super().__init__(link, "Album not found on deezer!", "albumNotOnDeezer") + +class InvalidID(GenerationError): + def __init__(self, link): + super().__init__(link, "Link ID is invalid!", "invalidID") + +class LinkNotSupported(GenerationError): + def __init__(self, link): + super().__init__(link, "Link is not supported.", "unsupportedURL") + +class LinkNotRecognized(GenerationError): + def __init__(self, link): + super().__init__(link, "Link is not recognized.", "invalidURL") diff --git a/deemix/plugins/__init__.py b/deemix/plugins/__init__.py new file mode 100644 index 0000000..59bfeea --- /dev/null +++ b/deemix/plugins/__init__.py @@ -0,0 +1,12 @@ +class Plugin: + def __init__(self): + pass + + def setup(self): + pass + + def parseLink(self, link): + pass + + def generateDownloadObject(self, dz, link, bitrate, listener): + pass diff --git a/deemix/plugins/spotify.py b/deemix/plugins/spotify.py new file mode 100644 index 0000000..b36eb17 --- /dev/null +++ b/deemix/plugins/spotify.py @@ -0,0 +1,346 @@ +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import re +from urllib.request import urlopen +from deemix.plugins import Plugin +from deemix.utils.localpaths import getConfigFolder +from deemix.itemgen import generateTrackItem, generateAlbumItem, GenerationError, TrackNotOnDeezer, AlbumNotOnDeezer +from deemix.types.DownloadObjects import Convertable + +import spotipy +SpotifyClientCredentials = spotipy.oauth2.SpotifyClientCredentials + +class Spotify(Plugin): + def __init__(self, configFolder=None): + super().__init__() + self.credentials = {'clientId': "", 'clientSecret': ""} + self.settings = { + 'fallbackSearch': False + } + self.enabled = False + self.sp = None + self.configFolder = Path(configFolder or getConfigFolder()) + self.configFolder /= 'spotify' + + def setup(self): + if not self.configFolder.is_dir(): self.configFolder.mkdir() + + self.loadSettings() + return self + + @classmethod + def parseLink(cls, link): + if 'link.tospotify.com' in link: link = urlopen(link).url + # Remove extra stuff + if '?' in link: link = link[:link.find('?')] + if '&' in link: link = link[:link.find('&')] + if link.endswith('/'): link = link[:-1] # Remove last slash if present + + link_type = None + link_id = None + + if not 'spotify' in link: return (link, link_type, link_id) # return if not a spotify link + + if re.search(r"[/:]track[/:](.+)", link): + link_type = 'track' + link_id = re.search(r"[/:]track[/:](.+)", link).group(1) + elif re.search(r"[/:]album[/:](.+)", link): + link_type = 'album' + link_id = re.search(r"[/:]album[/:](.+)", link).group(1) + elif re.search(r"[/:]playlist[/:](.+)", link): + link_type = 'playlist' + link_id = re.search(r"[/:]playlist[/:](.+)", link).group(1) + + return (link, link_type, link_id) + + def generateDownloadObject(self, dz, link, bitrate): + (link, link_type, link_id) = self.parseLink(link) + + if link_type is None or link_id is None: return None + + if link_type == "track": + return self.generateTrackItem(dz, link_id, bitrate) + if link_type == "album": + return self.generateAlbumItem(dz, link_id, bitrate) + if link_type == "playlist": + return self.generatePlaylistItem(dz, link_id, bitrate) + return None + + def generateTrackItem(self, dz, link_id, bitrate): + cache = self.loadCache() + + if link_id in cache['tracks']: + cachedTrack = cache['tracks'][link_id] + else: + cachedTrack = self.getTrack(link_id) + cache['tracks'][link_id] = cachedTrack + self.saveCache(cache) + + if 'isrc' in cachedTrack: + try: return generateTrackItem(dz, f"isrc:{cachedTrack['isrc']}", bitrate) + except GenerationError: pass + if self.settings['fallbackSearch']: + if 'id' not in cachedTrack or cachedTrack['id'] == "0": + trackID = dz.api.get_track_id_from_metadata( + cachedTrack['data']['artist'], + cachedTrack['data']['title'], + cachedTrack['data']['album'], + ) + if trackID != "0": + cachedTrack['id'] = trackID + cache['tracks'][link_id] = cachedTrack + self.saveCache(cache) + if cachedTrack['id'] != "0": return generateTrackItem(dz, cachedTrack['id'], bitrate) + raise TrackNotOnDeezer(f"https://open.spotify.com/track/{link_id}") + + def generateAlbumItem(self, dz, link_id, bitrate): + cache = self.loadCache() + + if link_id in cache['albums']: + cachedAlbum = cache['albums'][link_id] + else: + cachedAlbum = self.getAlbum(link_id) + cache['albums'][link_id] = cachedAlbum + self.saveCache(cache) + + try: return generateAlbumItem(dz, f"upc:{cachedAlbum['upc']}", bitrate) + except GenerationError as e: raise AlbumNotOnDeezer(f"https://open.spotify.com/album/{link_id}") from e + + def generatePlaylistItem(self, dz, link_id, bitrate): + if not self.enabled: raise Exception("Spotify plugin not enabled") + spotifyPlaylist = self.sp.playlist(link_id) + + playlistAPI = self._convertPlaylistStructure(spotifyPlaylist) + playlistAPI.various_artist = dz.api.get_artist(5080) # Useful for save as compilation + + tracklistTemp = spotifyPlaylist.track.items + while spotifyPlaylist['tracks']['next']: + spotifyPlaylist['tracks'] = self.sp.next(spotifyPlaylist['tracks']) + tracklistTemp += spotifyPlaylist['tracks']['items'] + + tracklist = [] + for item in tracklistTemp: + if item['track']: + if item['track']['explicit']: + playlistAPI['explicit'] = True + tracklist.append(item['track']) + if 'explicit' not in playlistAPI: playlistAPI['explicit'] = False + + return Convertable({ + 'type': 'spotify_playlist', + 'id': link_id, + 'bitrate': bitrate, + 'title': spotifyPlaylist['name'], + 'artist': spotifyPlaylist['owner']['display_name'], + 'cover': playlistAPI['picture_thumbnail'], + 'explicit': playlistAPI['explicit'], + 'size': len(tracklist), + 'collection': { + 'tracks_gw': [], + 'playlistAPI': playlistAPI + }, + 'plugin': 'spotify', + 'conversion_data': tracklist + }) + + def getTrack(self, track_id, spotifyTrack=None): + if not self.enabled: raise Exception("Spotify plugin not enabled") + cachedTrack = { + 'isrc': None, + 'data': None + } + + if not spotifyTrack: + spotifyTrack = self.sp.track(track_id) + if 'isrc' in spotifyTrack.get('external_ids', {}): + cachedTrack['isrc'] = spotifyTrack['external_ids']['isrc'] + cachedTrack['data'] = { + 'title': spotifyTrack['name'], + 'artist': spotifyTrack['artists'][0]['name'], + 'album': spotifyTrack['album']['name'] + } + return cachedTrack + + def getAlbum(self, album_id, spotifyAlbum=None): + if not self.enabled: raise Exception("Spotify plugin not enabled") + cachedAlbum = { + 'upc': None, + 'data': None + } + + if not spotifyAlbum: + spotifyAlbum = self.sp.album(album_id) + if 'upc' in spotifyAlbum.get('external_ids', {}): + cachedAlbum['upc'] = spotifyAlbum['external_ids']['upc'] + cachedAlbum['data'] = { + 'title': spotifyAlbum['name'], + 'artist': spotifyAlbum['artists'][0]['name'] + } + return cachedAlbum + + def convertTrack(self, dz, downloadObject, track, pos, conversion, conversionNext, cache, listener): + if downloadObject.isCanceled: return + + if track['id'] in cache['tracks']: + cachedTrack = cache['tracks'][track['id']] + else: + cachedTrack = self.getTrack(track['id'], track) + cache['tracks'][track['id']] = cachedTrack + self.saveCache(cache) + + if 'isrc' in cachedTrack: + try: + trackAPI = dz.api.get_track_by_ISRC(cachedTrack['isrc']) + if 'id' not in trackAPI or 'title' not in trackAPI: trackAPI = None + except GenerationError: pass + if self.settings['fallbackSearch'] and not trackAPI: + if 'id' not in cachedTrack or cachedTrack['id'] == "0": + trackID = dz.api.get_track_id_from_metadata( + cachedTrack['data']['artist'], + cachedTrack['data']['title'], + cachedTrack['data']['album'], + ) + if trackID != "0": + cachedTrack['id'] = trackID + cache['tracks'][track['id']] = cachedTrack + self.saveCache(cache) + if cachedTrack['id'] != "0": trackAPI = dz.api.get_track(cachedTrack['id']) + + if not trackAPI: + deezerTrack = { + 'SNG_ID': "0", + 'SNG_TITLE': track['name'], + 'DURATION': 0, + 'MD5_ORIGIN': 0, + 'MEDIA_VERSION': 0, + 'FILESIZE': 0, + 'ALB_TITLE': track['album']['name'], + 'ALB_PICTURE': "", + 'ART_ID': 0, + 'ART_NAME': track['artists'][0]['name'] + } + else: + deezerTrack = dz.gw.get_track_with_fallback(trackAPI['id']) + deezerTrack['_EXTRA_TRACK'] = trackAPI + deezerTrack['POSITION'] = pos+1 + + conversionNext += (1 / downloadObject.size) * 100 + if round(conversionNext) != conversion and round(conversionNext) % 2 == 0: + conversion = round(conversionNext) + if listener: listener.send("updateQueue", {'uuid': downloadObject.uuid, 'conversion': conversion}) + + def convert(self, dz, downloadObject, settings, listener=None): + cache = self.loadCache() + + conversion = 0 + conversionNext = 0 + + collection = [None] * len(downloadObject.conversion_data) + with ThreadPoolExecutor(settings['queueConcurrency']) as executor: + for pos, track in enumerate(downloadObject.conversion_data, start=0): + collection[pos] = executor.submit(self.convertTrack, + dz, downloadObject, + track, pos, + conversion, conversionNext, + cache, listener + ) + + @classmethod + def _convertPlaylistStructure(cls, spotifyPlaylist): + cover = None + if len(spotifyPlaylist['images']): cover = spotifyPlaylist['images'][0]['url'] + + deezerPlaylist = { + 'checksum': spotifyPlaylist['snapshot_id'], + 'collaborative': spotifyPlaylist['collaborative'], + 'creation_date': "XXXX-00-00", + 'creator': { + 'id': spotifyPlaylist['owner']['id'], + 'name': spotifyPlaylist['owner']['display_name'], + 'tracklist': spotifyPlaylist['owner']['href'], + 'type': "user" + }, + 'description': spotifyPlaylist['description'], + 'duration': 0, + 'fans': spotifyPlaylist['followers']['total'] if 'followers' in spotifyPlaylist else 0, + 'id': spotifyPlaylist['id'], + 'is_loved_track': False, + 'link': spotifyPlaylist['external_urls']['spotify'], + 'nb_tracks': spotifyPlaylist['tracks']['total'], + 'picture': cover, + 'picture_small': cover or "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/56x56-000000-80-0-0.jpg", + 'picture_medium': cover or "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/250x250-000000-80-0-0.jpg", + 'picture_big': cover or "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/500x500-000000-80-0-0.jpg", + 'picture_xl': cover or "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/1000x1000-000000-80-0-0.jpg", + 'public': spotifyPlaylist['public'], + 'share': spotifyPlaylist['external_urls']['spotify'], + 'title': spotifyPlaylist['name'], + 'tracklist': spotifyPlaylist['tracks']['href'], + 'type': "playlist" + } + return deezerPlaylist + + def loadSettings(self): + if not (self.configFolder / 'settings.json').is_file(): + with open(self.configFolder / 'settings.json', 'w') as f: + json.dump({**self.credentials, **self.settings}, f, indent=2) + + with open(self.configFolder / 'settings.json', 'r') as settingsFile: + settings = json.load(settingsFile) + self.setSettings(settings) + self.checkCredentials() + + def saveSettings(self, newSettings=None): + if newSettings: self.setSettings(newSettings) + self.checkCredentials() + with open(self.configFolder / 'settings.json', 'w') as f: + json.dump({**self.credentials, **self.settings}, f, indent=2) + + def getSettings(self): + return {**self.credentials, **self.settings} + + def setSettings(self, newSettings): + self.credentials = { 'clientId': newSettings['clientId'], 'clientSecret': newSettings['clientSecret'] } + settings = {**newSettings} + del settings['clientId'] + del settings['clientSecret'] + self.settings = settings + + def loadCache(self): + if (self.configFolder / 'cache.json').is_file(): + with open(self.configFolder / 'cache.json', 'r') as f: + cache = json.load(f) + else: + cache = {'tracks': {}, 'albums': {}} + return cache + + def saveCache(self, newCache): + with open(self.configFolder / 'cache.json', 'w') as spotifyCache: + json.dump(newCache, spotifyCache) + + def checkCredentials(self): + if self.credentials['clientId'] == "" or self.credentials['clientSecret'] == "": + self.enabled = False + return + + try: + client_credentials_manager = SpotifyClientCredentials(client_id=self.credentials['clientId'], + client_secret=self.credentials['clientSecret']) + self.sp = spotipy.Spotify(client_credentials_manager=client_credentials_manager) + self.sp.user_playlists('spotify') + self.enabled = True + except Exception: + self.enabled = False + + def getCredentials(self): + return self.credentials + + def setCredentials(self, clientId, clientSecret): + # Remove extra spaces, just to be sure + clientId = clientId.strip() + clientSecret = clientSecret.strip() + + # Save them to disk + self.credentials = { 'clientId': clientId, 'clientSecret': clientSecret} + self.saveSettings() diff --git a/deemix/settings.py b/deemix/settings.py new file mode 100644 index 0000000..d1c6d5e --- /dev/null +++ b/deemix/settings.py @@ -0,0 +1,137 @@ +import json +from pathlib import Path +from os import makedirs +from deezer import TrackFormats +import deemix.utils.localpaths as localpaths + +class OverwriteOption(): + """Should the lib overwrite files?""" + OVERWRITE = 'y' # Yes, overwrite the file + DONT_OVERWRITE = 'n' # No, don't overwrite the file + DONT_CHECK_EXT = 'e' # No, and don't check for extensions + KEEP_BOTH = 'b' # No, and keep both files + ONLY_TAGS = 't' # Overwrite only the tags + +class FeaturesOption(): + """What should I do with featured artists?""" + NO_CHANGE = "0" # Do nothing + REMOVE_TITLE = "1" # Remove from track title + REMOVE_TITLE_ALBUM = "3" # Remove from track title and album title + MOVE_TITLE = "2" # Move to track title + +DEFAULTS = { + "downloadLocation": str(localpaths.getMusicFolder()), + "tracknameTemplate": "%artist% - %title%", + "albumTracknameTemplate": "%tracknumber% - %title%", + "playlistTracknameTemplate": "%position% - %artist% - %title%", + "createPlaylistFolder": True, + "playlistNameTemplate": "%playlist%", + "createArtistFolder": False, + "artistNameTemplate": "%artist%", + "createAlbumFolder": True, + "albumNameTemplate": "%artist% - %album%", + "createCDFolder": True, + "createStructurePlaylist": False, + "createSingleFolder": False, + "padTracks": True, + "paddingSize": "0", + "illegalCharacterReplacer": "_", + "queueConcurrency": 3, + "maxBitrate": str(TrackFormats.MP3_320), + "fallbackBitrate": True, + "fallbackSearch": False, + "logErrors": True, + "logSearched": False, + "overwriteFile": OverwriteOption.DONT_OVERWRITE, + "createM3U8File": False, + "playlistFilenameTemplate": "playlist", + "syncedLyrics": False, + "embeddedArtworkSize": 800, + "embeddedArtworkPNG": False, + "localArtworkSize": 1400, + "localArtworkFormat": "jpg", + "saveArtwork": True, + "coverImageTemplate": "cover", + "saveArtworkArtist": False, + "artistImageTemplate": "folder", + "jpegImageQuality": 80, + "dateFormat": "Y-M-D", + "albumVariousArtists": True, + "removeAlbumVersion": False, + "removeDuplicateArtists": False, + "featuredToTitle": FeaturesOption.NO_CHANGE, + "titleCasing": "nothing", + "artistCasing": "nothing", + "executeCommand": "", + "tags": { + "title": True, + "artist": True, + "album": True, + "cover": True, + "trackNumber": True, + "trackTotal": False, + "discNumber": True, + "discTotal": False, + "albumArtist": True, + "genre": True, + "year": True, + "date": True, + "explicit": False, + "isrc": True, + "length": True, + "barcode": True, + "bpm": True, + "replayGain": False, + "label": True, + "lyrics": False, + "syncedLyrics": False, + "copyright": False, + "composer": False, + "involvedPeople": False, + "source": False, + "savePlaylistAsCompilation": False, + "useNullSeparator": False, + "saveID3v1": True, + "multiArtistSeparator": "default", + "singleAlbumArtist": False, + "coverDescriptionUTF8": False + } +} + +def save(settings, configFolder=None): + configFolder = Path(configFolder or localpaths.getConfigFolder()) + makedirs(configFolder, exist_ok=True) # Create config folder if it doesn't exsist + + with open(configFolder / 'config.json', 'w') as configFile: + json.dump(settings, configFile, indent=2) + +def load(configFolder=None): + configFolder = Path(configFolder or localpaths.getConfigFolder()) + makedirs(configFolder, exist_ok=True) # Create config folder if it doesn't exsist + if not (configFolder / 'config.json').is_file(): save(DEFAULTS, configFolder) # Create config file if it doesn't exsist + + # Read config file + with open(configFolder / 'config.json', 'r') as configFile: + settings = json.load(configFile) + + if check(settings) > 0: save(settings, configFolder) # Check the settings and save them if something changed + return settings + +def check(settings): + changes = 0 + for i_set in DEFAULTS: + if not i_set in settings or not isinstance(settings[i_set], type(DEFAULTS[i_set])): + settings[i_set] = DEFAULTS[i_set] + changes += 1 + for i_set in DEFAULTS['tags']: + if not i_set in settings['tags'] or not isinstance(settings['tags'][i_set], type(DEFAULTS['tags'][i_set])): + settings['tags'][i_set] = DEFAULTS['tags'][i_set] + changes += 1 + if settings['downloadLocation'] == "": + settings['downloadLocation'] = DEFAULTS['downloadLocation'] + changes += 1 + for template in ['tracknameTemplate', 'albumTracknameTemplate', 'playlistTracknameTemplate', 'playlistNameTemplate', 'artistNameTemplate', 'albumNameTemplate', 'playlistFilenameTemplate', 'coverImageTemplate', 'artistImageTemplate', 'paddingSize']: + if settings[template] == "": + settings[template] = DEFAULTS[template] + changes += 1 + return changes diff --git a/deemix/utils/taggers.py b/deemix/tagger.py similarity index 98% rename from deemix/utils/taggers.py rename to deemix/tagger.py index b363752..e3cdbf6 100644 --- a/deemix/utils/taggers.py +++ b/deemix/tagger.py @@ -4,10 +4,10 @@ from mutagen.id3 import ID3, ID3NoHeaderError, \ TPUB, TSRC, USLT, SYLT, APIC, IPLS, TCOM, TCOP, TCMP, Encoding, PictureType # Adds tags to a MP3 file -def tagID3(stream, track, save): +def tagID3(path, track, save): # Delete exsisting tags try: - tag = ID3(stream) + tag = ID3(path) tag.delete() except ID3NoHeaderError: tag = ID3() @@ -111,15 +111,15 @@ def tagID3(stream, track, save): with open(track.album.embeddedCoverPath, 'rb') as f: tag.add(APIC(descEncoding, mimeType, PictureType.COVER_FRONT, desc='cover', data=f.read())) - tag.save( stream, + tag.save( path, v1=2 if save['saveID3v1'] else 0, v2_version=3, v23_sep=None if save['useNullSeparator'] else '/' ) # Adds tags to a FLAC file -def tagFLAC(stream, track, save): +def tagFLAC(path, track, save): # Delete exsisting tags - tag = FLAC(stream) + tag = FLAC(path) tag.delete() tag.clear_pictures() diff --git a/deemix/types/Album.py b/deemix/types/Album.py index a6472cb..d1e0fda 100644 --- a/deemix/types/Album.py +++ b/deemix/types/Album.py @@ -4,47 +4,57 @@ from deemix.utils import removeDuplicateArtists, removeFeatures from deemix.types.Artist import Artist from deemix.types.Date import Date from deemix.types.Picture import Picture -from deemix import VARIOUS_ARTISTS +from deemix.types import VARIOUS_ARTISTS class Album: - def __init__(self, id="0", title="", pic_md5=""): - self.id = id + def __init__(self, alb_id="0", title="", pic_md5=""): + self.id = alb_id self.title = title - self.pic = Picture(md5=pic_md5, type="cover") + self.pic = Picture(pic_md5, "cover") self.artist = {"Main": []} self.artists = [] self.mainArtist = None - self.dateString = None - self.barcode = "Unknown" - self.date = None + self.date = Date() + self.dateString = "" + self.trackTotal = "0" self.discTotal = "0" - self.embeddedCoverPath = None - self.embeddedCoverURL = None + self.embeddedCoverPath = "" + self.embeddedCoverURL = "" self.explicit = False self.genre = [] + self.barcode = "Unknown" self.label = "Unknown" + self.copyright = "" self.recordType = "album" - self.rootArtist = None - self.trackTotal = "0" self.bitrate = 0 + self.rootArtist = None self.variousArtists = None + self.playlistId = None + self.owner = None + self.isPlaylist = False + def parseAlbum(self, albumAPI): self.title = albumAPI['title'] # Getting artist image ID # ex: https://e-cdns-images.dzcdn.net/images/artist/f2bc007e9133c946ac3c3907ddc5d2ea/56x56-000000-80-0-0.jpg - artistPicture = albumAPI['artist']['picture_small'] - artistPicture = artistPicture[artistPicture.find('artist/') + 7:-24] + art_pic = albumAPI['artist']['picture_small'] + art_pic = art_pic[art_pic.find('artist/') + 7:-24] self.mainArtist = Artist( - id = albumAPI['artist']['id'], - name = albumAPI['artist']['name'], - pic_md5 = artistPicture + albumAPI['artist']['id'], + albumAPI['artist']['name'], + "Main", + art_pic ) if albumAPI.get('root_artist'): + art_pic = albumAPI['root_artist']['picture_small'] + art_pic = art_pic[art_pic.find('artist/') + 7:-24] self.rootArtist = Artist( - id = albumAPI['root_artist']['id'], - name = albumAPI['root_artist']['name'] + albumAPI['root_artist']['id'], + albumAPI['root_artist']['name'], + "Root", + art_pic ) for artist in albumAPI['contributors']: @@ -53,7 +63,7 @@ class Album: if isVariousArtists: self.variousArtists = Artist( - id = artist['id'], + art_id = artist['id'], name = artist['name'], role = artist['role'] ) @@ -74,18 +84,19 @@ class Album: self.label = albumAPI.get('label', self.label) self.explicit = bool(albumAPI.get('explicit_lyrics', False)) if 'release_date' in albumAPI: - day = albumAPI["release_date"][8:10] - month = albumAPI["release_date"][5:7] - year = albumAPI["release_date"][0:4] - self.date = Date(year, month, day) + self.date.day = albumAPI["release_date"][8:10] + self.date.month = albumAPI["release_date"][5:7] + self.date.year = albumAPI["release_date"][0:4] + self.date.fixDayMonth() self.discTotal = albumAPI.get('nb_disk') self.copyright = albumAPI.get('copyright') - if not self.pic.md5: + if self.pic.md5 == "": # Getting album cover MD5 # ex: https://e-cdns-images.dzcdn.net/images/cover/2e018122cb56986277102d2041a592c8/56x56-000000-80-0-0.jpg - self.pic.md5 = albumAPI['cover_small'][albumAPI['cover_small'].find('cover/') + 6:-24] + alb_pic = albumAPI['cover_small'] + self.pic.md5 = alb_pic[alb_pic.find('cover/') + 6:-24] if albumAPI.get('genres') and len(albumAPI['genres'].get('data', [])) > 0: for genre in albumAPI['genres']['data']: @@ -94,8 +105,9 @@ class Album: def parseAlbumGW(self, albumAPI_gw): self.title = albumAPI_gw['ALB_TITLE'] self.mainArtist = Artist( - id = albumAPI_gw['ART_ID'], - name = albumAPI_gw['ART_NAME'] + art_id = albumAPI_gw['ART_ID'], + name = albumAPI_gw['ART_NAME'], + role = "Main" ) self.artists = [albumAPI_gw['ART_NAME']] @@ -106,13 +118,16 @@ class Album: explicitLyricsStatus = albumAPI_gw.get('EXPLICIT_ALBUM_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) self.explicit = explicitLyricsStatus in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT] - if not self.pic.md5: + self.addExtraAlbumGWData(albumAPI_gw) + + def addExtraAlbumGWData(self, albumAPI_gw): + if self.pic.md5 == "": self.pic.md5 = albumAPI_gw['ALB_PICTURE'] if 'PHYSICAL_RELEASE_DATE' in albumAPI_gw: - day = albumAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] - month = albumAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] - year = albumAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] - self.date = Date(year, month, day) + self.date.day = albumAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] + self.date.month = albumAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] + self.date.year = albumAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] + self.date.fixDayMonth() def makePlaylistCompilation(self, playlist): self.variousArtists = playlist.variousArtists @@ -131,10 +146,12 @@ class Album: self.playlistId = playlist.playlistId self.owner = playlist.owner self.pic = playlist.pic + self.isPlaylist = True def removeDuplicateArtists(self): + """Removes duplicate artists for both artist array and artists dict""" (self.artist, self.artists) = removeDuplicateArtists(self.artist, self.artists) - # Removes featuring from the album name def getCleanTitle(self): + """Removes featuring from the album name""" return removeFeatures(self.title) diff --git a/deemix/types/Artist.py b/deemix/types/Artist.py index 42cb573..9b18262 100644 --- a/deemix/types/Artist.py +++ b/deemix/types/Artist.py @@ -1,12 +1,12 @@ from deemix.types.Picture import Picture -from deemix import VARIOUS_ARTISTS +from deemix.types import VARIOUS_ARTISTS class Artist: - def __init__(self, id="0", name="", pic_md5="", role=""): - self.id = str(id) + def __init__(self, art_id="0", name="", role="", pic_md5=""): + self.id = str(art_id) self.name = name - self.pic = Picture(md5=pic_md5, type="artist") - self.role = "" + self.pic = Picture(md5=pic_md5, pic_type="artist") + self.role = role self.save = True def isVariousArtists(self): diff --git a/deemix/types/Date.py b/deemix/types/Date.py index b74e04f..e0c73d3 100644 --- a/deemix/types/Date.py +++ b/deemix/types/Date.py @@ -1,8 +1,8 @@ -class Date(object): - def __init__(self, year="XXXX", month="00", day="00"): - self.year = year - self.month = month +class Date: + def __init__(self, day="00", month="00", year="XXXX"): self.day = day + self.month = month + self.year = year self.fixDayMonth() # Fix incorrect day month when detectable diff --git a/deemix/types/DownloadObjects.py b/deemix/types/DownloadObjects.py new file mode 100644 index 0000000..a2f54ac --- /dev/null +++ b/deemix/types/DownloadObjects.py @@ -0,0 +1,126 @@ +class IDownloadObject: + """DownloadObject Interface""" + def __init__(self, obj): + self.type = obj['type'] + self.id = obj['id'] + self.bitrate = obj['bitrate'] + self.title = obj['title'] + self.artist = obj['artist'] + self.cover = obj['cover'] + self.explicit = obj.get('explicit', False) + self.size = obj.get('size', 0) + self.downloaded = obj.get('downloaded', 0) + self.failed = obj.get('failed', 0) + self.progress = obj.get('progress', 0) + self.errors = obj.get('errors', []) + self.files = obj.get('files', []) + self.progressNext = 0 + self.uuid = f"{self.type}_{self.id}_{self.bitrate}" + self.isCanceled = False + self.__type__ = None + + def toDict(self): + return { + 'type': self.type, + 'id': self.id, + 'bitrate': self.bitrate, + 'uuid': self.uuid, + 'title': self.title, + 'artist': self.artist, + 'cover': self.cover, + 'explicit': self.explicit, + 'size': self.size, + 'downloaded': self.downloaded, + 'failed': self.failed, + 'progress': self.progress, + 'errors': self.errors, + 'files': self.files, + '__type__': self.__type__ + } + + def getResettedDict(self): + item = self.toDict() + item['downloaded'] = 0 + item['failed'] = 0 + item['progress'] = 0 + item['errors'] = [] + item['files'] = [] + return item + + def getSlimmedDict(self): + light = self.toDict() + propertiesToDelete = ['single', 'collection', 'plugin', 'conversion_data'] + for prop in propertiesToDelete: + if prop in light: + del light[prop] + return light + + def getEssentialDict(self): + return { + 'type': self.type, + 'id': self.id, + 'bitrate': self.bitrate, + 'uuid': self.uuid, + 'title': self.title, + 'artist': self.artist, + 'cover': self.cover, + 'explicit': self.explicit, + 'size': self.size + } + + def updateProgress(self, listener=None): + if round(self.progressNext) != self.progress and round(self.progressNext) % 2 == 0: + self.progress = round(self.progressNext) + if listener: listener.send("updateQueue", {'uuid': self.uuid, 'progress': self.progress}) + +class Single(IDownloadObject): + def __init__(self, obj): + super().__init__(obj) + self.size = 1 + self.single = obj['single'] + self.__type__ = "Single" + + def toDict(self): + item = super().toDict() + item['single'] = self.single + return item + + def completeTrackProgress(self, listener=None): + self.progressNext = 100 + self.updateProgress(listener) + + def removeTrackProgress(self, listener=None): + self.progressNext = 0 + self.updateProgress(listener) + +class Collection(IDownloadObject): + def __init__(self, obj): + super().__init__(obj) + self.collection = obj['collection'] + self.__type__ = "Collection" + + def toDict(self): + item = super().toDict() + item['collection'] = self.collection + return item + + def completeTrackProgress(self, listener=None): + self.progressNext += (1 / self.size) * 100 + self.updateProgress(listener) + + def removeTrackProgress(self, listener=None): + self.progressNext -= (1 / self.size) * 100 + self.updateProgress(listener) + +class Convertable(Collection): + def __init__(self, obj): + super().__init__(obj) + self.plugin = obj['plugin'] + self.conversion_data = obj['conversion_data'] + self.__type__ = "Convertable" + + def toDict(self): + item = super().toDict() + item['plugin'] = self.plugin + item['conversion_data'] = self.conversion_data + return item diff --git a/deemix/types/Lyrics.py b/deemix/types/Lyrics.py index a21beb1..f16e960 100644 --- a/deemix/types/Lyrics.py +++ b/deemix/types/Lyrics.py @@ -1,19 +1,17 @@ class Lyrics: - def __init__(self, id="0"): - self.id = id - self.sync = None - self.unsync = None - self.syncID3 = None + def __init__(self, lyr_id="0"): + self.id = lyr_id + self.sync = "" + self.unsync = "" + self.syncID3 = [] def parseLyrics(self, lyricsAPI): self.unsync = lyricsAPI.get("LYRICS_TEXT") if "LYRICS_SYNC_JSON" in lyricsAPI: syncLyricsJson = lyricsAPI["LYRICS_SYNC_JSON"] - self.sync = "" - self.syncID3 = [] timestamp = "" milliseconds = 0 - for line in range(len(syncLyricsJson)): + for line, _ in enumerate(syncLyricsJson): if syncLyricsJson[line]["line"] != "": timestamp = syncLyricsJson[line]["lrc_timestamp"] milliseconds = int(syncLyricsJson[line]["milliseconds"]) @@ -21,6 +19,6 @@ class Lyrics: else: notEmptyLine = line + 1 while syncLyricsJson[notEmptyLine]["line"] == "": - notEmptyLine = notEmptyLine + 1 + notEmptyLine += 1 timestamp = syncLyricsJson[notEmptyLine]["lrc_timestamp"] self.sync += timestamp + syncLyricsJson[line]["line"] + "\r\n" diff --git a/deemix/types/Picture.py b/deemix/types/Picture.py index ca00f49..b3fed1b 100644 --- a/deemix/types/Picture.py +++ b/deemix/types/Picture.py @@ -1,27 +1,29 @@ class Picture: - def __init__(self, md5="", type=None, url=None): + def __init__(self, md5="", pic_type=""): self.md5 = md5 - self.type = type - self.url = url + self.type = pic_type - def generatePictureURL(self, size, format): - if self.url: return self.url - if format.startswith("jpg"): - if '-' in format: - quality = format[4:] - else: - quality = 80 - format = 'jpg' - return "https://e-cdns-images.dzcdn.net/images/{}/{}/{}x{}-{}".format( - self.type, - self.md5, - size, size, - f'000000-{quality}-0-0.jpg' - ) - if format == 'png': - return "https://e-cdns-images.dzcdn.net/images/{}/{}/{}x{}-{}".format( - self.type, - self.md5, - size, size, - 'none-100-0-0.png' - ) + def getURL(self, size, pic_format): + url = "https://e-cdns-images.dzcdn.net/images/{}/{}/{size}x{size}".format( + self.type, + self.md5, + size=size + ) + + if pic_format.startswith("jpg"): + quality = 80 + if '-' in pic_format: + quality = pic_format[4:] + pic_format = 'jpg' + return url + f'-000000-{quality}-0-0.jpg' + if pic_format == 'png': + return url + '-none-100-0-0.png' + + return url+'.jpg' + +class StaticPicture: + def __init__(self, url): + self.staticURL = url + + def getURL(self): + return self.staticURL diff --git a/deemix/types/Playlist.py b/deemix/types/Playlist.py index 9625719..f936f44 100644 --- a/deemix/types/Playlist.py +++ b/deemix/types/Playlist.py @@ -1,20 +1,9 @@ from deemix.types.Artist import Artist from deemix.types.Date import Date -from deemix.types.Picture import Picture +from deemix.types.Picture import Picture, StaticPicture class Playlist: def __init__(self, playlistAPI): - if 'various_artist' in playlistAPI: - playlistAPI['various_artist']['role'] = "Main" - self.variousArtists = Artist( - id = playlistAPI['various_artist']['id'], - name = playlistAPI['various_artist']['name'], - pic_md5 = playlistAPI['various_artist']['picture_small'][ - playlistAPI['various_artist']['picture_small'].find('artist/') + 7:-24], - role = playlistAPI['various_artist']['role'] - ) - self.mainArtist = self.variousArtists - self.id = "pl_" + str(playlistAPI['id']) self.title = playlistAPI['title'] self.rootArtist = None @@ -30,19 +19,28 @@ class Playlist: year = playlistAPI["creation_date"][0:4] month = playlistAPI["creation_date"][5:7] day = playlistAPI["creation_date"][8:10] - self.date = Date(year, month, day) + self.date = Date(day, month, year) self.discTotal = "1" - self.playlistId = playlistAPI['id'] + self.playlistID = playlistAPI['id'] self.owner = playlistAPI['creator'] + if 'dzcdn.net' in playlistAPI['picture_small']: url = playlistAPI['picture_small'] picType = url[url.find('images/')+7:] picType = picType[:picType.find('/')] md5 = url[url.find(picType+'/') + len(picType)+1:-24] - self.pic = Picture( - md5 = md5, - type = picType - ) + self.pic = Picture(md5, picType) else: - self.pic = Picture(url = playlistAPI['picture_xl']) + self.pic = StaticPicture(playlistAPI['picture_xl']) + + if 'various_artist' in playlistAPI: + pic_md5 = playlistAPI['various_artist']['picture_small'] + pic_md5 = pic_md5[pic_md5.find('artist/') + 7:-24] + self.variousArtists = Artist( + playlistAPI['various_artist']['id'], + playlistAPI['various_artist']['name'], + "Main", + pic_md5 + ) + self.mainArtist = self.variousArtists diff --git a/deemix/types/Track.py b/deemix/types/Track.py index 523384f..85e359e 100644 --- a/deemix/types/Track.py +++ b/deemix/types/Track.py @@ -1,38 +1,39 @@ -import eventlet -requests = eventlet.import_patched('requests') +from time import sleep +import re +import requests -import logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger('deemix') - -from deezer.gw import APIError as gwAPIError +from deezer.gw import GWAPIError from deezer.api import APIError -from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString + +from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString, changeCase + from deemix.types.Album import Album from deemix.types.Artist import Artist from deemix.types.Date import Date from deemix.types.Picture import Picture from deemix.types.Playlist import Playlist from deemix.types.Lyrics import Lyrics -from deemix import VARIOUS_ARTISTS +from deemix.types import VARIOUS_ARTISTS + +from deemix.settings import FeaturesOption class Track: - def __init__(self, id="0", name=""): - self.id = id + def __init__(self, sng_id="0", name=""): + self.id = sng_id self.title = name self.MD5 = "" self.mediaVersion = "" self.duration = 0 - self.fallbackId = "0" + self.fallbackID = "0" self.filesizes = {} - self.localTrack = False + self.local = False self.mainArtist = None self.artist = {"Main": []} self.artists = [] self.album = None self.trackNumber = "0" self.discNumber = "0" - self.date = None + self.date = Date() self.lyrics = None self.bpm = 0 self.contributors = {} @@ -45,7 +46,7 @@ class Track: self.searched = False self.selectedFormat = 0 self.singleDownload = False - self.dateString = None + self.dateString = "" self.artistsString = "" self.mainArtistsString = "" self.featArtistsString = "" @@ -60,14 +61,14 @@ class Track: else: raise MD5NotFound self.mediaVersion = trackAPI_gw['MEDIA_VERSION'] - self.fallbackId = "0" + self.fallbackID = "0" if 'FALLBACK' in trackAPI_gw: - self.fallbackId = trackAPI_gw['FALLBACK']['SNG_ID'] - self.localTrack = int(self.id) < 0 + self.fallbackID = trackAPI_gw['FALLBACK']['SNG_ID'] + self.local = int(self.id) < 0 def retriveFilesizes(self, dz): + guest_sid = dz.session.cookies.get('sid') try: - guest_sid = dz.session.cookies.get('sid') site = requests.post( "https://api.deezer.com/1.0/gateway.php", params={ @@ -83,21 +84,20 @@ class Track: ) result_json = site.json() except: - eventlet.sleep(2) - return self.retriveFilesizes(dz) + sleep(2) + self.retriveFilesizes(dz) if len(result_json['error']): - raise APIError(json.dumps(result_json['error'])) - response = result_json.get("results") + raise TrackError(result_json.dumps(result_json['error'])) + response = result_json.get("results", {}) filesizes = {} for key, value in response.items(): if key.startswith("FILESIZE_"): - filesizes[key] = value + filesizes[key] = int(value) filesizes[key+"_TESTED"] = False self.filesizes = filesizes - def parseData(self, dz, id=None, trackAPI_gw=None, trackAPI=None, albumAPI_gw=None, albumAPI=None, playlistAPI=None): - if id: - if not trackAPI_gw: trackAPI_gw = dz.gw.get_track_with_fallback(id) + def parseData(self, dz, track_id=None, trackAPI_gw=None, trackAPI=None, albumAPI_gw=None, albumAPI=None, playlistAPI=None): + if track_id and not trackAPI_gw: trackAPI_gw = dz.gw.get_track_with_fallback(track_id) elif not trackAPI_gw: raise NoDataToParse if not trackAPI: try: trackAPI = dz.api.get_track(trackAPI_gw['SNG_ID']) @@ -105,21 +105,21 @@ class Track: self.parseEssentialData(trackAPI_gw, trackAPI) - if self.localTrack: + if self.local: self.parseLocalTrackData(trackAPI_gw) else: self.retriveFilesizes(dz) - self.parseTrackGW(trackAPI_gw) + # Get Lyrics data if not "LYRICS" in trackAPI_gw and self.lyrics.id != "0": try: trackAPI_gw["LYRICS"] = dz.gw.get_track_lyrics(self.id) - except gwAPIError: self.lyrics.id = "0" + except GWAPIError: self.lyrics.id = "0" if self.lyrics.id != "0": self.lyrics.parseLyrics(trackAPI_gw["LYRICS"]) - # Parse Album data + # Parse Album Data self.album = Album( - id = trackAPI_gw['ALB_ID'], + alb_id = trackAPI_gw['ALB_ID'], title = trackAPI_gw['ALB_TITLE'], pic_md5 = trackAPI_gw.get('ALB_PICTURE') ) @@ -132,7 +132,7 @@ class Track: # Get album_gw Data if not albumAPI_gw: try: albumAPI_gw = dz.gw.get_album(self.album.id) - except gwAPIError: albumAPI_gw = None + except GWAPIError: albumAPI_gw = None if albumAPI: self.album.parseAlbum(albumAPI) @@ -147,6 +147,7 @@ class Track: raise AlbumDoesntExists # Fill missing data + if albumAPI_gw: self.album.addExtraAlbumGWData(albumAPI_gw) if self.album.date and not self.date: self.date = self.album.date if not self.album.discTotal: self.album.discTotal = albumAPI_gw.get('NUMBER_DISK', "1") if not self.copyright: self.copyright = albumAPI_gw['COPYRIGHT'] @@ -157,10 +158,9 @@ class Track: self.title = ' '.join(self.title.split()) # Make sure there is at least one artist - if not len(self.artist['Main']): + if len(self.artist['Main']) == 0: self.artist['Main'] = [self.mainArtist['name']] - self.singleDownload = trackAPI_gw.get('SINGLE_TRACK', False) self.position = trackAPI_gw.get('POSITION') # Add playlist data if track is in a playlist @@ -176,9 +176,9 @@ class Track: self.album = Album(title=trackAPI_gw['ALB_TITLE']) self.album.pic = Picture( md5 = trackAPI_gw.get('ALB_PICTURE', ""), - type = "cover" + pic_type = "cover" ) - self.mainArtist = Artist(name=trackAPI_gw['ART_NAME']) + self.mainArtist = Artist(name=trackAPI_gw['ART_NAME'], role="Main") self.artists = [trackAPI_gw['ART_NAME']] self.artist = { 'Main': [trackAPI_gw['ART_NAME']] @@ -187,12 +187,11 @@ class Track: self.album.artists = self.artists self.album.date = self.date self.album.mainArtist = self.mainArtist - self.date = Date() def parseTrackGW(self, trackAPI_gw): self.title = trackAPI_gw['SNG_TITLE'].strip() - if trackAPI_gw.get('VERSION') and not trackAPI_gw['VERSION'] in trackAPI_gw['SNG_TITLE']: - self.title += " " + trackAPI_gw['VERSION'].strip() + if trackAPI_gw.get('VERSION') and not trackAPI_gw['VERSION'].strip() in self.title: + self.title += f" {trackAPI_gw['VERSION'].strip()}" self.discNumber = trackAPI_gw.get('DISK_NUMBER') self.explicit = bool(int(trackAPI_gw.get('EXPLICIT_LYRICS', "0"))) @@ -205,16 +204,17 @@ class Track: self.lyrics = Lyrics(trackAPI_gw.get('LYRICS_ID', "0")) self.mainArtist = Artist( - id = trackAPI_gw['ART_ID'], + art_id = trackAPI_gw['ART_ID'], name = trackAPI_gw['ART_NAME'], + role = "Main", pic_md5 = trackAPI_gw.get('ART_PICTURE') ) if 'PHYSICAL_RELEASE_DATE' in trackAPI_gw: - day = trackAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] - month = trackAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] - year = trackAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] - self.date = Date(year, month, day) + self.date.day = trackAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] + self.date.month = trackAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] + self.date.year = trackAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] + self.date.fixDayMonth() def parseTrack(self, trackAPI): self.bpm = trackAPI['bpm'] @@ -249,8 +249,8 @@ class Track: return removeFeatures(self.title) def getFeatTitle(self): - if self.featArtistsString and not "(feat." in self.title.lower(): - return self.title + " ({})".format(self.featArtistsString) + if self.featArtistsString and "feat." not in self.title.lower(): + return f"{self.title} ({self.featArtistsString})" return self.title def generateMainFeatStrings(self): @@ -259,9 +259,81 @@ class Track: if 'Featured' in self.artist: self.featArtistsString = "feat. "+andCommaConcat(self.artist['Featured']) + def applySettings(self, settings): + + # Check if should save the playlist as a compilation + if self.playlist and settings['tags']['savePlaylistAsCompilation']: + self.trackNumber = self.position + self.discNumber = "1" + self.album.makePlaylistCompilation(self.playlist) + else: + if self.album.date: self.date = self.album.date + + self.dateString = self.date.format(settings['dateFormat']) + self.album.dateString = self.album.date.format(settings['dateFormat']) + if self.playlist: self.playlist.dateString = self.playlist.date.format(settings['dateFormat']) + + # Check various artist option + if settings['albumVariousArtists'] and self.album.variousArtists: + artist = self.album.variousArtists + isMainArtist = artist.role == "Main" + + if artist.name not in self.album.artists: + self.album.artists.insert(0, artist.name) + + if isMainArtist or artist.name not in self.album.artist['Main'] and not isMainArtist: + if artist.role not in self.album.artist: + self.album.artist[artist.role] = [] + self.album.artist[artist.role].insert(0, artist.name) + self.album.mainArtist.save = not self.album.mainArtist.isVariousArtists() or settings['albumVariousArtists'] and self.album.mainArtist.isVariousArtists() + + # Check removeDuplicateArtists + if settings['removeDuplicateArtists']: self.removeDuplicateArtists() + + # Check if user wants the feat in the title + if str(settings['featuredToTitle']) == FeaturesOption.REMOVE_TITLE: + self.title = self.getCleanTitle() + elif str(settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: + self.title = self.getFeatTitle() + elif str(settings['featuredToTitle']) == FeaturesOption.REMOVE_TITLE_ALBUM: + self.title = self.getCleanTitle() + self.album.title = self.album.getCleanTitle() + + # Remove (Album Version) from tracks that have that + if settings['removeAlbumVersion'] and "Album Version" in self.title: + self.title = re.sub(r' ?\(Album Version\)', "", self.title).strip() + + # Change Title and Artists casing if needed + if settings['titleCasing'] != "nothing": + self.title = changeCase(self.title, settings['titleCasing']) + if settings['artistCasing'] != "nothing": + self.mainArtist.name = changeCase(self.mainArtist.name, settings['artistCasing']) + for i, artist in enumerate(self.artists): + self.artists[i] = changeCase(artist, settings['artistCasing']) + for art_type in self.artist: + for i, artist in enumerate(self.artist[art_type]): + self.artist[art_type][i] = changeCase(artist, settings['artistCasing']) + self.generateMainFeatStrings() + + # Generate artist tag + if settings['tags']['multiArtistSeparator'] == "default": + if str(settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: + self.artistsString = ", ".join(self.artist['Main']) + else: + self.artistsString = ", ".join(self.artists) + elif settings['tags']['multiArtistSeparator'] == "andFeat": + self.artistsString = self.mainArtistsString + if self.featArtistsString and str(settings['featuredToTitle']) != FeaturesOption.MOVE_TITLE: + self.artistsString += " " + self.featArtistsString + else: + separator = settings['tags']['multiArtistSeparator'] + if str(settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: + self.artistsString = separator.join(self.artist['Main']) + else: + self.artistsString = separator.join(self.artists) + class TrackError(Exception): """Base class for exceptions in this module.""" - pass class AlbumDoesntExists(TrackError): pass diff --git a/deemix/types/__init__.py b/deemix/types/__init__.py index 9db5426..3c0325c 100644 --- a/deemix/types/__init__.py +++ b/deemix/types/__init__.py @@ -1,7 +1 @@ -from deemix.types.Date import Date -from deemix.types.Picture import Picture -from deemix.types.Lyrics import Lyrics -from deemix.types.Album import Album -from deemix.types.Artist import Artist -from deemix.types.Playlist import Playlist -from deemix.types.Track import Track +VARIOUS_ARTISTS = "5080" diff --git a/deemix/utils/__init__.py b/deemix/utils/__init__.py index 5936119..3b79e97 100644 --- a/deemix/utils/__init__.py +++ b/deemix/utils/__init__.py @@ -1,41 +1,42 @@ -import re import string from deezer import TrackFormats import os +USER_AGENT_HEADER = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) " \ + "Chrome/79.0.3945.130 Safari/537.36" + +def canWrite(folder): + return os.access(folder, os.W_OK) + def generateReplayGainString(trackGain): return "{0:.2f} dB".format((float(trackGain) + 18.4) * -1) -def getBitrateInt(txt): +def getBitrateNumberFromText(txt): txt = str(txt).lower() if txt in ['flac', 'lossless', '9']: return TrackFormats.FLAC - elif txt in ['mp3', '320', '3']: + if txt in ['mp3', '320', '3']: return TrackFormats.MP3_320 - elif txt in ['128', '1']: + if txt in ['128', '1']: return TrackFormats.MP3_128 - elif txt in ['360', '360_hq', '15']: + if txt in ['360', '360_hq', '15']: return TrackFormats.MP4_RA3 - elif txt in ['360_mq', '14']: + if txt in ['360_mq', '14']: return TrackFormats.MP4_RA2 - elif txt in ['360_lq', '13']: + if txt in ['360_lq', '13']: return TrackFormats.MP4_RA1 - else: - return None - - -def changeCase(str, type): - if type == "lower": - return str.lower() - elif type == "upper": - return str.upper() - elif type == "start": - return string.capwords(str) - elif type == "sentence": - return str.capitalize() - else: - return str + return None +def changeCase(txt, case_type): + if case_type == "lower": + return txt.lower() + if case_type == "upper": + return txt.upper() + if case_type == "start": + return string.capwords(txt) + if case_type == "sentence": + return txt.capitalize() + return str def removeFeatures(title): clean = title @@ -48,7 +49,6 @@ def removeFeatures(title): clean = ' '.join(clean.split()) return clean - def andCommaConcat(lst): tot = len(lst) result = "" @@ -61,62 +61,6 @@ def andCommaConcat(lst): result += ", " return result - -def getIDFromLink(link, type): - if '?' in link: - link = link[:link.find('?')] - if link.endswith("/"): - link = link[:-1] - - if link.startswith("http") and 'open.spotify.com/' in link: - if '&' in link: link = link[:link.find('&')] - if type == "spotifyplaylist": - return link[link.find("/playlist/") + 10:] - if type == "spotifytrack": - return link[link.find("/track/") + 7:] - if type == "spotifyalbum": - return link[link.find("/album/") + 7:] - elif link.startswith("spotify:"): - if type == "spotifyplaylist": - return link[link.find("playlist:") + 9:] - if type == "spotifytrack": - return link[link.find("track:") + 6:] - if type == "spotifyalbum": - return link[link.find("album:") + 6:] - elif type == "artisttop": - return re.search(r"\/artist\/(\d+)\/top_track", link)[1] - elif type == "artistdiscography": - return re.search(r"\/artist\/(\d+)\/discography", link)[1] - else: - return link[link.rfind("/") + 1:] - - -def getTypeFromLink(link): - type = '' - if 'spotify' in link: - type = 'spotify' - if 'playlist' in link: - type += 'playlist' - elif 'track' in link: - type += 'track' - elif 'album' in link: - type += 'album' - elif 'deezer' in link: - if '/track' in link: - type = 'track' - elif '/playlist' in link: - type = 'playlist' - elif '/album' in link: - type = 'album' - elif re.search("\/artist\/(\d+)\/top_track", link): - type = 'artisttop' - elif re.search("\/artist\/(\d+)\/discography", link): - type = 'artistdiscography' - elif '/artist' in link: - type = 'artist' - return type - - def uniqueArray(arr): for iPrinc, namePrinc in enumerate(arr): for iRest, nRest in enumerate(arr): @@ -129,11 +73,3 @@ def removeDuplicateArtists(artist, artists): for role in artist.keys(): artist[role] = uniqueArray(artist[role]) return (artist, artists) - -def checkFolder(folder): - try: - os.makedirs(folder, exist_ok=True) - except Exception as e: - print(str(e)) - return False - return os.access(folder, os.W_OK) diff --git a/deemix/utils/crypto.py b/deemix/utils/crypto.py new file mode 100644 index 0000000..6edb49b --- /dev/null +++ b/deemix/utils/crypto.py @@ -0,0 +1,26 @@ +import binascii + +from Cryptodome.Cipher import Blowfish, AES +from Cryptodome.Hash import MD5 + +def _md5(data): + h = MD5.new() + h.update(data.encode() if isinstance(data, str) else data) + return h.hexdigest() + +def _ecbCrypt(key, data): + return binascii.hexlify(AES.new(key.encode(), AES.MODE_ECB).encrypt(data)) + +def _ecbDecrypt(key, data): + return AES.new(key.encode(), AES.MODE_ECB).decrypt(binascii.unhexlify(data.encode("utf-8"))) + +def generateBlowfishKey(trackId): + SECRET = 'g4el58wc0zvf9na1' + idMd5 = _md5(trackId) + bfKey = "" + for i in range(16): + bfKey += chr(ord(idMd5[i]) ^ ord(idMd5[i + 16]) ^ ord(SECRET[i])) + return bfKey + +def decryptChunk(key, data): + return Blowfish.new(key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(data) diff --git a/deemix/utils/decryption.py b/deemix/utils/decryption.py deleted file mode 100644 index 616bbac..0000000 --- a/deemix/utils/decryption.py +++ /dev/null @@ -1,31 +0,0 @@ -import binascii -from Cryptodome.Cipher import Blowfish, AES -from Cryptodome.Hash import MD5 - -def _md5(data): - h = MD5.new() - h.update(str.encode(data) if isinstance(data, str) else data) - return h.hexdigest() - -def generateBlowfishKey(trackId): - SECRET = 'g4el58wc' + '0zvf9na1' - idMd5 = _md5(trackId) - bfKey = "" - for i in range(16): - bfKey += chr(ord(idMd5[i]) ^ ord(idMd5[i + 16]) ^ ord(SECRET[i])) - return bfKey - -def generateStreamURL(sng_id, md5, media_version, format): - urlPart = b'\xa4'.join( - [str.encode(md5), str.encode(str(format)), str.encode(str(sng_id)), str.encode(str(media_version))]) - md5val = _md5(urlPart) - step2 = str.encode(md5val) + b'\xa4' + urlPart + b'\xa4' - step2 = step2 + (b'.' * (16 - (len(step2) % 16))) - urlPart = binascii.hexlify(AES.new(b'jo6aey6haid2Teih', AES.MODE_ECB).encrypt(step2)) - return "https://e-cdns-proxy-" + md5[0] + ".dzcdn.net/mobile/1/" + urlPart.decode("utf-8") - -def reverseStreamURL(url): - urlPart = url[42:] - step2 = AES.new(b'jo6aey6haid2Teih', AES.MODE_ECB).decrypt(binascii.unhexlify(urlPart.encode("utf-8"))) - (md5val, md5, format, sng_id, media_version, _) = step2.split(b'\xa4') - return (sng_id.decode('utf-8'), md5.decode('utf-8'), media_version.decode('utf-8'), format.decode('utf-8')) diff --git a/deemix/utils/deezer.py b/deemix/utils/deezer.py new file mode 100644 index 0000000..7841a5e --- /dev/null +++ b/deemix/utils/deezer.py @@ -0,0 +1,32 @@ +import requests +from deemix.utils.crypto import _md5 +from deemix.utils import USER_AGENT_HEADER +CLIENT_ID = "172365" +CLIENT_SECRET = "fb0bec7ccc063dab0417eb7b0d847f34" + +def getAccessToken(email, password): + password = _md5(password) + request_hash = _md5(''.join([CLIENT_ID, email, password, CLIENT_SECRET])) + response = requests.get( + 'https://api.deezer.com/auth/token', + params={ + 'app_id': CLIENT_ID, + 'login': email, + 'password': password, + 'hash': request_hash + }, + headers={"User-Agent": USER_AGENT_HEADER} + ).json() + return response.get('access_token') + +def getArtFromAccessToken(accessToken): + session = requests.Session() + session.get( + "https://api.deezer.com/platform/generic/track/3135556", + headers={"Authorization": f"Bearer {accessToken}", "User-Agent": USER_AGENT_HEADER} + ) + response = session.get( + 'https://www.deezer.com/ajax/gw-light.php?method=user.getArl&input=3&api_version=1.0&api_token=null', + headers={"User-Agent": USER_AGENT_HEADER} + ).json() + return response.get('results') diff --git a/deemix/utils/localpaths.py b/deemix/utils/localpaths.py index e9a39b0..2e7670c 100644 --- a/deemix/utils/localpaths.py +++ b/deemix/utils/localpaths.py @@ -1,44 +1,72 @@ from pathlib import Path import sys import os +import re +from deemix.utils import canWrite homedata = Path.home() userdata = "" musicdata = "" - -if os.getenv("DEEMIX_DATA_DIR"): - userdata = Path(os.getenv("DEEMIX_DATA_DIR")) -elif os.getenv("XDG_CONFIG_HOME"): - userdata = Path(os.getenv("XDG_CONFIG_HOME")) / 'deemix' -elif os.getenv("APPDATA"): - userdata = Path(os.getenv("APPDATA")) / "deemix" -elif sys.platform.startswith('darwin'): - userdata = homedata / 'Library' / 'Application Support' / 'deemix' -else: - userdata = homedata / '.config' / 'deemix' - -if os.getenv("DEEMIX_MUSIC_DIR"): - musicdata = Path(os.getenv("DEEMIX_MUSIC_DIR")) -elif os.getenv("XDG_MUSIC_DIR"): - musicdata = Path(os.getenv("XDG_MUSIC_DIR")) / "deemix Music" -elif os.name == 'nt': - import winreg - sub_key = r'SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders' - music_guid = '{4BD8D571-6D19-48D3-BE97-422220080E43}' - with winreg.OpenKey(winreg.HKEY_CURRENT_USER, sub_key) as key: - location = None - try: location = winreg.QueryValueEx(key, music_guid)[0] - except: pass - try: location = winreg.QueryValueEx(key, 'My Music')[0] - except: pass - if not location: location = homedata / "Music" - musicdata = Path(location) / "deemix Music" -else: - musicdata = homedata / "Music" / "deemix Music" +def checkPath(path): + if path == "": return "" + if not path.is_dir(): return "" + if not canWrite(path): return "" + return path def getConfigFolder(): + global userdata + if userdata != "": return userdata + if os.getenv("XDG_CONFIG_HOME") and userdata == "": + userdata = Path(os.getenv("XDG_CONFIG_HOME")) + userdata = checkPath(userdata) + if os.getenv("APPDATA") and userdata == "": + userdata = Path(os.getenv("APPDATA")) + userdata = checkPath(userdata) + if sys.platform.startswith('darwin') and userdata == "": + userdata = homedata / 'Library' / 'Application Support' + userdata = checkPath(userdata) + if userdata == "": + userdata = homedata / '.config' + userdata = checkPath(userdata) + + if userdata == "": userdata = Path(os.getcwd()) / 'config' + else: userdata = userdata / 'deemix' + + if os.getenv("DEEMIX_DATA_DIR"): + userdata = Path(os.getenv("DEEMIX_DATA_DIR")) return userdata def getMusicFolder(): + global musicdata + if musicdata != "": return musicdata + if os.getenv("XDG_MUSIC_DIR") and musicdata == "": + musicdata = Path(os.getenv("XDG_MUSIC_DIR")) + musicdata = checkPath(musicdata) + if (homedata / '.config' / 'user-dirs.dirs').is_file() and musicdata == "": + with open(homedata / '.config' / 'user-dirs.dirs', 'r') as f: + userDirs = f.read() + musicdata = re.search(r"XDG_MUSIC_DIR=\"(.*)\"", userDirs).group(1) + musicdata = Path(os.path.expandvars(musicdata)) + musicdata = checkPath(musicdata) + if os.name == 'nt' and musicdata == "": + musicKeys = ['My Music', '{4BD8D571-6D19-48D3-BE97-422220080E43}'] + regData = os.popen(r'reg.exe query "HKCU\SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"').read().split('\r\n') + for i, line in enumerate(regData): + if line == "": continue + if i == 1: continue + line = line.split(' ') + if line[1] in musicKeys: + musicdata = Path(line[3]) + break + musicdata = checkPath(musicdata) + if musicdata == "": + musicdata = homedata / 'Music' + musicdata = checkPath(musicdata) + + if musicdata == "": musicdata = Path(os.getcwd()) / 'music' + else: musicdata = musicdata / 'deemix Music' + + if os.getenv("DEEMIX_MUSIC_DIR"): + musicdata = Path(os.getenv("DEEMIX_MUSIC_DIR")) return musicdata diff --git a/deemix/utils/pathtemplates.py b/deemix/utils/pathtemplates.py index 3d04dce..b654428 100644 --- a/deemix/utils/pathtemplates.py +++ b/deemix/utils/pathtemplates.py @@ -21,14 +21,13 @@ def fixName(txt, char='_'): txt = normalize("NFC", txt) return txt -def fixEndOfData(bString): - try: - bString.decode() - return True - except: - return False - def fixLongName(name): + def fixEndOfData(bString): + try: + bString.decode() + return True + except Exception: + return False if pathSep in name: sepName = name.split(pathSep) name = "" @@ -52,30 +51,40 @@ def antiDot(string): return string -def pad(num, max, settings): +def pad(num, max_val, settings): if int(settings['paddingSize']) == 0: - paddingSize = len(str(max)) + paddingSize = len(str(max_val)) else: paddingSize = len(str(10 ** (int(settings['paddingSize']) - 1))) if paddingSize == 1: paddingSize = 2 if settings['padTracks']: return str(num).zfill(paddingSize) + return str(num) + +def generatePath(track, downloadObject, settings): + filenameTemplate = "%artist% - %title%" + singleTrack = False + if downloadObject.type == "track": + if settings['createSingleFolder']: + filenameTemplate = settings['albumTracknameTemplate'] + else: + filenameTemplate = settings['tracknameTemplate'] + singleTrack = True + elif downloadObject.type == "album": + filenameTemplate = settings['albumTracknameTemplate'] else: - return str(num) + filenameTemplate = settings['playlistTracknameTemplate'] -def generateFilename(track, settings, template): - filename = template or "%artist% - %title%" - return settingsRegex(filename, track, settings) + filename = generateTrackName(filenameTemplate, track, settings) -def generateFilepath(track, settings): - filepath = Path(settings['downloadLocation']) + filepath = Path(settings['downloadLocation'] or '.') artistPath = None coverPath = None extrasPath = None if settings['createPlaylistFolder'] and track.playlist and not settings['tags']['savePlaylistAsCompilation']: - filepath = filepath / settingsRegexPlaylist(settings['playlistNameTemplate'], track.playlist, settings) + filepath = filepath / generatePlaylistName(settings['playlistNameTemplate'], track.playlist, settings) if track.playlist and not settings['tags']['savePlaylistAsCompilation']: extrasPath = filepath @@ -85,61 +94,66 @@ def generateFilepath(track, settings): (settings['createArtistFolder'] and track.playlist and settings['tags']['savePlaylistAsCompilation']) or (settings['createArtistFolder'] and track.playlist and settings['createStructurePlaylist']) ): - filepath = filepath / settingsRegexArtist(settings['artistNameTemplate'], track.album.mainArtist, settings, rootArtist=track.album.rootArtist) + filepath = filepath / generateArtistName(settings['artistNameTemplate'], track.album.mainArtist, settings, rootArtist=track.album.rootArtist) artistPath = filepath if (settings['createAlbumFolder'] and - (not track.singleDownload or (track.singleDownload and settings['createSingleFolder'])) and + (not singleTrack or (singleTrack and settings['createSingleFolder'])) and (not track.playlist or (track.playlist and settings['tags']['savePlaylistAsCompilation']) or (track.playlist and settings['createStructurePlaylist']) ) ): - filepath = filepath / settingsRegexAlbum(settings['albumNameTemplate'], track.album, settings, track.playlist) + filepath = filepath / generateAlbumName(settings['albumNameTemplate'], track.album, settings, track.playlist) coverPath = filepath - if not (track.playlist and not settings['tags']['savePlaylistAsCompilation']): - extrasPath = filepath + if not extrasPath: extrasPath = filepath if ( - int(track.album.discTotal) > 1 and ( + int(track.album.discTotal) > 1 and ( (settings['createAlbumFolder'] and settings['createCDFolder']) and - (not track.singleDownload or (track.singleDownload and settings['createSingleFolder'])) and + (not singleTrack or (singleTrack and settings['createSingleFolder'])) and (not track.playlist or (track.playlist and settings['tags']['savePlaylistAsCompilation']) or (track.playlist and settings['createStructurePlaylist']) - ) + ) )): - filepath = filepath / f'CD{str(track.discNumber)}' + filepath = filepath / f'CD{track.discNumber}' - return (filepath, artistPath, coverPath, extrasPath) + # Remove subfolders from filename and add it to filepath + if pathSep in filename: + tempPath = filename[:filename.rfind(pathSep)] + filepath = filepath / tempPath + filename = filename[filename.rfind(pathSep) + len(pathSep):] + + return (filename, filepath, artistPath, coverPath, extrasPath) -def settingsRegex(filename, track, settings): - filename = filename.replace("%title%", fixName(track.title, settings['illegalCharacterReplacer'])) - filename = filename.replace("%artist%", fixName(track.mainArtist.name, settings['illegalCharacterReplacer'])) - filename = filename.replace("%artists%", fixName(", ".join(track.artists), settings['illegalCharacterReplacer'])) - filename = filename.replace("%allartists%", fixName(track.artistsString, settings['illegalCharacterReplacer'])) - filename = filename.replace("%mainartists%", fixName(track.mainArtistsString, settings['illegalCharacterReplacer'])) +def generateTrackName(filename, track, settings): + c = settings['illegalCharacterReplacer'] + filename = filename.replace("%title%", fixName(track.title, c)) + filename = filename.replace("%artist%", fixName(track.mainArtist.name, c)) + filename = filename.replace("%artists%", fixName(", ".join(track.artists), c)) + filename = filename.replace("%allartists%", fixName(track.artistsString, c)) + filename = filename.replace("%mainartists%", fixName(track.mainArtistsString, c)) if track.featArtistsString: - filename = filename.replace("%featartists%", fixName('('+track.featArtistsString+')', settings['illegalCharacterReplacer'])) + filename = filename.replace("%featartists%", fixName('('+track.featArtistsString+')', c)) else: filename = filename.replace("%featartists%", '') - filename = filename.replace("%album%", fixName(track.album.title, settings['illegalCharacterReplacer'])) - filename = filename.replace("%albumartist%", fixName(track.album.mainArtist.name, settings['illegalCharacterReplacer'])) + filename = filename.replace("%album%", fixName(track.album.title, c)) + filename = filename.replace("%albumartist%", fixName(track.album.mainArtist.name, c)) filename = filename.replace("%tracknumber%", pad(track.trackNumber, track.album.trackTotal, settings)) filename = filename.replace("%tracktotal%", str(track.album.trackTotal)) filename = filename.replace("%discnumber%", str(track.discNumber)) filename = filename.replace("%disctotal%", str(track.album.discTotal)) if len(track.album.genre) > 0: - filename = filename.replace("%genre%", - fixName(track.album.genre[0], settings['illegalCharacterReplacer'])) + filename = filename.replace("%genre%", fixName(track.album.genre[0], c)) else: filename = filename.replace("%genre%", "Unknown") filename = filename.replace("%year%", str(track.date.year)) filename = filename.replace("%date%", track.dateString) filename = filename.replace("%bpm%", str(track.bpm)) - filename = filename.replace("%label%", fixName(track.album.label, settings['illegalCharacterReplacer'])) + filename = filename.replace("%label%", fixName(track.album.label, c)) filename = filename.replace("%isrc%", track.ISRC) filename = filename.replace("%upc%", track.album.barcode) filename = filename.replace("%explicit%", "(Explicit)" if track.explicit else "") @@ -148,40 +162,41 @@ def settingsRegex(filename, track, settings): filename = filename.replace("%album_id%", str(track.album.id)) filename = filename.replace("%artist_id%", str(track.mainArtist.id)) if track.playlist: - filename = filename.replace("%playlist_id%", str(track.playlist.playlistId)) + filename = filename.replace("%playlist_id%", str(track.playlist.playlistID)) filename = filename.replace("%position%", pad(track.position, track.playlist.trackTotal, settings)) else: filename = filename.replace("%playlist_id%", '') - filename = filename.replace("%position%", pad(track.trackNumber, track.album.trackTotal, settings)) + filename = filename.replace("%position%", pad(track.position, track.album.trackTotal, settings)) filename = filename.replace('\\', pathSep).replace('/', pathSep) return antiDot(fixLongName(filename)) -def settingsRegexAlbum(foldername, album, settings, playlist=None): +def generateAlbumName(foldername, album, settings, playlist=None): + c = settings['illegalCharacterReplacer'] if playlist and settings['tags']['savePlaylistAsCompilation']: - foldername = foldername.replace("%album_id%", "pl_" + str(playlist.playlistId)) + foldername = foldername.replace("%album_id%", "pl_" + str(playlist.playlistID)) foldername = foldername.replace("%genre%", "Compile") else: foldername = foldername.replace("%album_id%", str(album.id)) if len(album.genre) > 0: - foldername = foldername.replace("%genre%", fixName(album.genre[0], settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%genre%", fixName(album.genre[0], c)) else: foldername = foldername.replace("%genre%", "Unknown") - foldername = foldername.replace("%album%", fixName(album.title, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%artist%", fixName(album.mainArtist.name, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%album%", fixName(album.title, c)) + foldername = foldername.replace("%artist%", fixName(album.mainArtist.name, c)) foldername = foldername.replace("%artist_id%", str(album.mainArtist.id)) if album.rootArtist: - foldername = foldername.replace("%root_artist%", fixName(album.rootArtist.name, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%root_artist%", fixName(album.rootArtist.name, c)) foldername = foldername.replace("%root_artist_id%", str(album.rootArtist.id)) else: - foldername = foldername.replace("%root_artist%", fixName(album.mainArtist.name, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%root_artist%", fixName(album.mainArtist.name, c)) foldername = foldername.replace("%root_artist_id%", str(album.mainArtist.id)) foldername = foldername.replace("%tracktotal%", str(album.trackTotal)) foldername = foldername.replace("%disctotal%", str(album.discTotal)) - foldername = foldername.replace("%type%", fixName(album.recordType.capitalize(), settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%type%", fixName(album.recordType.capitalize(), c)) foldername = foldername.replace("%upc%", album.barcode) foldername = foldername.replace("%explicit%", "(Explicit)" if album.explicit else "") - foldername = foldername.replace("%label%", fixName(album.label, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%label%", fixName(album.label, c)) foldername = foldername.replace("%year%", str(album.date.year)) foldername = foldername.replace("%date%", album.dateString) foldername = foldername.replace("%bitrate%", bitrateLabels[int(album.bitrate)]) @@ -190,23 +205,25 @@ def settingsRegexAlbum(foldername, album, settings, playlist=None): return antiDot(fixLongName(foldername)) -def settingsRegexArtist(foldername, artist, settings, rootArtist=None): - foldername = foldername.replace("%artist%", fixName(artist.name, settings['illegalCharacterReplacer'])) +def generateArtistName(foldername, artist, settings, rootArtist=None): + c = settings['illegalCharacterReplacer'] + foldername = foldername.replace("%artist%", fixName(artist.name, c)) foldername = foldername.replace("%artist_id%", str(artist.id)) if rootArtist: - foldername = foldername.replace("%root_artist%", fixName(rootArtist.name, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%root_artist%", fixName(rootArtist.name, c)) foldername = foldername.replace("%root_artist_id%", str(rootArtist.id)) else: - foldername = foldername.replace("%root_artist%", fixName(artist.name, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%root_artist%", fixName(artist.name, c)) foldername = foldername.replace("%root_artist_id%", str(artist.id)) foldername = foldername.replace('\\', pathSep).replace('/', pathSep) return antiDot(fixLongName(foldername)) -def settingsRegexPlaylist(foldername, playlist, settings): - foldername = foldername.replace("%playlist%", fixName(playlist.title, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistId, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%owner%", fixName(playlist.owner['name'], settings['illegalCharacterReplacer'])) +def generatePlaylistName(foldername, playlist, settings): + c = settings['illegalCharacterReplacer'] + foldername = foldername.replace("%playlist%", fixName(playlist.title, c)) + foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistID, c)) + foldername = foldername.replace("%owner%", fixName(playlist.owner['name'], c)) foldername = foldername.replace("%owner_id%", str(playlist.owner['id'])) foldername = foldername.replace("%year%", str(playlist.date.year)) foldername = foldername.replace("%date%", str(playlist.dateString)) @@ -214,12 +231,13 @@ def settingsRegexPlaylist(foldername, playlist, settings): foldername = foldername.replace('\\', pathSep).replace('/', pathSep) return antiDot(fixLongName(foldername)) -def settingsRegexPlaylistFile(foldername, queueItem, settings): - foldername = foldername.replace("%title%", fixName(queueItem.title, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%artist%", fixName(queueItem.artist, settings['illegalCharacterReplacer'])) +def generateDownloadObjectName(foldername, queueItem, settings): + c = settings['illegalCharacterReplacer'] + foldername = foldername.replace("%title%", fixName(queueItem.title, c)) + foldername = foldername.replace("%artist%", fixName(queueItem.artist, c)) foldername = foldername.replace("%size%", str(queueItem.size)) - foldername = foldername.replace("%type%", fixName(queueItem.type, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%id%", fixName(queueItem.id, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%type%", fixName(queueItem.type, c)) + foldername = foldername.replace("%id%", fixName(queueItem.id, c)) foldername = foldername.replace("%bitrate%", bitrateLabels[int(queueItem.bitrate)]) - foldername = foldername.replace('\\', pathSep).replace('/', pathSep).replace(pathSep, settings['illegalCharacterReplacer']) + foldername = foldername.replace('\\', pathSep).replace('/', pathSep).replace(pathSep, c) return antiDot(fixLongName(foldername)) diff --git a/requirements.txt b/requirements.txt index 3242ce0..6f5ae55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,4 @@ pycryptodomex mutagen requests spotipy>=2.11.0 -eventlet deezer-py diff --git a/setup.py b/setup.py index f14b4e9..e853813 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ README = (HERE / "README.md").read_text() setup( name="deemix", - version="2.0.16", + version="3.0.0", description="A barebone deezer downloader library", long_description=README, long_description_content_type="text/markdown", @@ -16,15 +16,14 @@ setup( license="GPL3", classifiers=[ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - "Development Status :: 4 - Beta", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Operating System :: OS Independent", ], - python_requires='>=3.6', + python_requires='>=3.7', packages=find_packages(exclude=("tests",)), include_package_data=True, - install_requires=["click", "pycryptodomex", "mutagen", "requests", "spotipy>=2.11.0", "eventlet", "deezer-py"], + install_requires=["click", "pycryptodomex", "mutagen", "requests", "spotipy>=2.11.0", "deezer-py"], entry_points={ "console_scripts": [ "deemix=deemix.__main__:download", diff --git a/updatePyPi.sh b/updatePyPi.sh index 9c400e0..767fd01 100755 --- a/updatePyPi.sh +++ b/updatePyPi.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash rm -rd build rm -rd dist -python -m bump -python -m bump deemix/__init__.py +#python -m bump +#python -m bump deemix/__init__.py python3 setup.py sdist bdist_wheel python3 -m twine upload dist/*