More work on the library (WIP)

deemix/__init__.py

@@ -26,19 +26,19 @@ def parseLink(link):
         id = link[link.rfind("/") + 1:]
     elif '/playlist' in link:
         type = 'playlist'
-        id = re.search("\/playlist\/(\d+)", link)[0]
+        id = re.search("\/playlist\/(\d+)", link).group(1)
     elif '/album' in link:
         type = 'album'
         id = link[link.rfind("/") + 1:]
     elif re.search("\/artist\/(\d+)\/top_track", link):
         type = 'artist_top'
-        id = re.search("\/artist\/(\d+)\/top_track", link)[0]
+        id = re.search("\/artist\/(\d+)\/top_track", link).group(1)
     elif re.search("\/artist\/(\d+)\/discography", link):
         type = 'artist_discography'
-        id = re.search("\/artist\/(\d+)\/discography", link)[0]
+        id = re.search("\/artist\/(\d+)\/discography", link).group(1)
     elif '/artist' in link:
         type = 'artist'
-        id = re.search("\/artist\/(\d+)", link)[0]
+        id = re.search("\/artist\/(\d+)", link).group(1)
 
     return (link, type, id)
 

deemix/decryption.py

@@ -2,6 +2,18 @@ import binascii
 from Cryptodome.Cipher import Blowfish, AES
 from Cryptodome.Hash import MD5
 
+from deemix import USER_AGENT_HEADER
+from deemix.types.DownloadObjects import Single, Collection
+
+from requests import get
+
+from requests.exceptions import ConnectionError, ReadTimeout
+from ssl import SSLError
+from urllib3.exceptions import SSLError as u3SSLError
+
+import logging
+logger = logging.getLogger('deemix')
+
 def _md5(data):
     h = MD5.new()
     h.update(str.encode(data) if isinstance(data, str) else data)
@@ -40,3 +52,127 @@ def generateUnencryptedStreamURL(sng_id, md5, media_version, format):
 def reverseStreamURL(url):
     urlPart = url[url.find("/1/")+3:]
     return generateStreamPath(urlPart)
+
+def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, interface=None):
+    headers= {'User-Agent': USER_AGENT_HEADER}
+    chunkLength = start
+    percentage = 0
+
+    itemName = f"[{track.mainArtist.name} - {track.title}]"
+
+    try:
+        with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request:
+            request.raise_for_status()
+
+            complete = int(request.headers["Content-Length"])
+            if complete == 0: raise DownloadEmpty
+            if start != 0:
+                responseRange = request.headers["Content-Range"]
+                logger.info(f'{itemName} downloading range {responseRange}')
+            else:
+                logger.info(f'{itemName} downloading {complete} bytes')
+
+            for chunk in request.iter_content(2048 * 3):
+                outputStream.write(chunk)
+                chunkLength += len(chunk)
+
+                if downloadObject:
+                    if isinstance(downloadObject, Single):
+                        percentage = (chunkLength / (complete + start)) * 100
+                        downloadObject.progressNext = percentage
+                    else:
+                        chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100
+                        downloadObject.progressNext += chunkProgres
+                    downloadObject.updateProgress(interface)
+
+    except (SSLError, u3SSLError) as e:
+        logger.info(f'{itemName} retrying from byte {chunkLength}')
+        return streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface)
+    except (ConnectionError, ReadTimeout):
+        sleep(2)
+        return streamUnencryptedTrack(outputStream, track, start, downloadObject, interface)
+
+def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, interface=None):
+    headers= {'User-Agent': USER_AGENT_HEADER}
+    chunkLength = start
+    percentage = 0
+
+    itemName = f"[{track.mainArtist.name} - {track.title}]"
+
+    try:
+        with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request:
+            request.raise_for_status()
+
+            complete = int(request.headers["Content-Length"])
+            if complete == 0: raise DownloadEmpty
+            if start != 0:
+                responseRange = request.headers["Content-Range"]
+                logger.info(f'{itemName} downloading range {responseRange}')
+            else:
+                logger.info(f'{itemName} downloading {complete} bytes')
+
+            for chunk in request.iter_content(2048 * 3):
+                outputStream.write(chunk)
+                chunkLength += len(chunk)
+
+                if downloadObject:
+                    if isinstance(downloadObject, Single):
+                        percentage = (chunkLength / (complete + start)) * 100
+                        downloadObject.progressNext = percentage
+                    else:
+                        chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100
+                        downloadObject.progressNext += chunkProgres
+                    downloadObject.updateProgress(interface)
+
+    except (SSLError, u3SSLError) as e:
+        logger.info(f'{itemName} retrying from byte {chunkLength}')
+        return streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface)
+    except (ConnectionError, ReadTimeout):
+        sleep(2)
+        return streamUnencryptedTrack(outputStream, track, start, downloadObject, interface)
+
+def streamTrack(outputStream, track, start=0, downloadObject=None, interface=None):
+    headers= {'User-Agent': USER_AGENT_HEADER}
+    chunkLength = start
+    percentage = 0
+
+    itemName = f"[{track.mainArtist.name} - {track.title}]"
+
+    try:
+        with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request:
+            request.raise_for_status()
+            blowfish_key = str.encode(generateBlowfishKey(str(track.id)))
+
+            complete = int(request.headers["Content-Length"])
+            if complete == 0: raise DownloadEmpty
+            if start != 0:
+                responseRange = request.headers["Content-Range"]
+                logger.info(f'{itemName} downloading range {responseRange}')
+            else:
+                logger.info(f'{itemName} downloading {complete} bytes')
+
+            for chunk in request.iter_content(2048 * 3):
+                if len(chunk) >= 2048:
+                    chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk[0:2048]) + chunk[2048:]
+
+                outputStream.write(chunk)
+                chunkLength += len(chunk)
+
+                if downloadObject:
+                    if isinstance(downloadObject, Single):
+                        percentage = (chunkLength / (complete + start)) * 100
+                        downloadObject.progressNext = percentage
+                    else:
+                        chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100
+                        downloadObject.progressNext += chunkProgres
+                    downloadObject.updateProgress(interface)
+
+    except (SSLError, u3SSLError) as e:
+        logger.info(f'{itemName} retrying from byte {chunkLength}')
+        return streamTrack(outputStream, track, chunkLength, downloadObject, interface)
+    except (ConnectionError, ReadTimeout):
+        sleep(2)
+        return streamTrack(outputStream, track, start, downloadObject, interface)
+
+class DownloadEmpty(Exception):
+    pass

deemix/downloader.py

@@ -11,24 +11,21 @@ import re
 import errno
 
 from ssl import SSLError
-from os import makedirs
 from urllib3.exceptions import SSLError as u3SSLError
+from os import makedirs
 
 from deemix.types.DownloadObjects import Single, Collection
 from deemix.types.Track import Track, AlbumDoesntExists
-from deemix.utils import changeCase
 from deemix.utils.pathtemplates import generateFilename, generateFilepath, settingsRegexAlbum, settingsRegexArtist, settingsRegexPlaylistFile
 from deezer import TrackFormats
 from deemix import USER_AGENT_HEADER
 from deemix.taggers import tagID3, tagFLAC
-from deemix.decryption import generateStreamURL, generateBlowfishKey
+from deemix.decryption import generateUnencryptedStreamURL, streamUnencryptedTrack
 from deemix.settings import OverwriteOption
 
-from Cryptodome.Cipher import Blowfish
 from mutagen.flac import FLACNoHeaderError, error as FLACError
-import logging
 
-logging.basicConfig(level=logging.INFO)
+import logging
 logger = logging.getLogger('deemix')
 
 from tempfile import gettempdir
@@ -124,7 +121,7 @@ def getPreferredBitrate(track, preferredBitrate, shouldFallback, downloadObjectU
                 if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber
                 if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]:
                     request = requests.head(
-                        generateStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber),
+                        generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber),
                         headers={'User-Agent': USER_AGENT_HEADER},
                         timeout=30
                     )
@@ -159,8 +156,6 @@ class Downloader:
         self.settings = settings
         self.bitrate = downloadObject.bitrate
         self.interface = interface
-        self.downloadPercentage = 0
-        self.lastPercentage = 0
         self.extrasPath = None
         self.playlistCoverName = None
         self.playlistURLs = []
@@ -184,7 +179,6 @@ class Downloader:
         if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer")
 
         # Create Track object
-        print(track)
         if not track:
             logger.info(f"[{trackAPI_gw['ART_NAME']} - {trackAPI_gw['SNG_TITLE']}] Getting the tags")
             try:
@@ -252,7 +246,7 @@ class Downloader:
                     url = track.album.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat)
                     if self.settings['tags']['savePlaylistAsCompilation'] \
                         and track.playlist \
-                        and track.playlist.pic.url \
+                        and track.playlist.pic.staticUrl \
                         and not format.startswith("jpg"):
                             continue
                     result['albumURLs'].append({'url': url, 'ext': format})
@@ -280,7 +274,7 @@ class Downloader:
                         extendedFormat = format
                         if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}"
                         url = track.playlist.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat)
-                        if track.playlist.pic.url and not format.startswith("jpg"): continue
+                        if track.playlist.pic.staticUrl and not format.startswith("jpg"): continue
                         self.playlistURLs.append({'url': url, 'ext': format})
             if not self.playlistCoverName:
                 track.playlist.bitrate = selectedFormat
@@ -316,12 +310,12 @@ class Downloader:
 
         if not trackAlreadyDownloaded or self.settings['overwriteFile'] == OverwriteOption.OVERWRITE:
             logger.info(f"[{track.mainArtist.name} - {track.title}] Downloading the track")
-            track.downloadUrl = generateStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat)
+            track.downloadUrl = generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat)
 
             def downloadMusic(track, trackAPI_gw):
                 try:
                     with open(writepath, 'wb') as stream:
-                        self.streamTrack(stream, track)
+                        streamUnencryptedTrack(stream, track, downloadObject=self.downloadObject, interface=self.interface)
                 except DownloadCancelled:
                     if writepath.is_file(): writepath.unlink()
                     raise DownloadCancelled
@@ -382,7 +376,7 @@ class Downloader:
             if not trackDownloaded: return self.download(trackAPI_gw, track=track)
         else:
             logger.info(f"[{track.mainArtist.name} - {track.title}] Skipping track as it's already downloaded")
-            self.completeTrackPercentage()
+            self.downloadObject.completeTrackProgress(self.interface)
 
         # Adding tags
         if (not trackAlreadyDownloaded or self.settings['overwriteFile'] in [OverwriteOption.ONLY_TAGS, OverwriteOption.OVERWRITE]) and not track.localTrack:
@@ -395,7 +389,7 @@ class Downloader:
                 except (FLACNoHeaderError, FLACError):
                     if writepath.is_file(): writepath.unlink()
                     logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available in FLAC, falling back if necessary")
-                    self.removeTrackPercentage()
+                    self.downloadObject.removeTrackProgress(self.interface)
                     track.filesizes['FILESIZE_FLAC'] = "0"
                     track.filesizes['FILESIZE_FLAC_TESTED'] = True
                     return self.download(trackAPI_gw, track=track)
@@ -409,71 +403,6 @@ class Downloader:
             self.interface.send("updateQueue", {'uuid': self.downloadObject.uuid, 'downloaded': True, 'downloadPath': str(writepath), 'extrasPath': str(self.extrasPath)})
         return result
 
-    def streamTrack(self, stream, track, start=0):
-
-        headers=dict(self.dz.http_headers)
-        if range != 0: headers['Range'] = f'bytes={start}-'
-        chunkLength = start
-        percentage = 0
-
-        itemName = f"[{track.mainArtist.name} - {track.title}]"
-
-        try:
-            with self.dz.session.get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request:
-                request.raise_for_status()
-                blowfish_key = str.encode(generateBlowfishKey(str(track.id)))
-
-                complete = int(request.headers["Content-Length"])
-                if complete == 0: raise DownloadEmpty
-                if start != 0:
-                    responseRange = request.headers["Content-Range"]
-                    logger.info(f'{itemName} downloading range {responseRange}')
-                else:
-                    logger.info(f'{itemName} downloading {complete} bytes')
-
-                for chunk in request.iter_content(2048 * 3):
-
-                    if len(chunk) >= 2048:
-                        chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk[0:2048]) + chunk[2048:]
-
-                    stream.write(chunk)
-                    chunkLength += len(chunk)
-
-                    if isinstance(self.downloadObject, Single):
-                        percentage = (chunkLength / (complete + start)) * 100
-                        self.downloadPercentage = percentage
-                    else:
-                        chunkProgres = (len(chunk) / (complete + start)) / self.downloadObject.size * 100
-                        self.downloadPercentage += chunkProgres
-                    self.updatePercentage()
-
-        except (SSLError, u3SSLError) as e:
-            logger.info(f'{itemName} retrying from byte {chunkLength}')
-            return self.streamTrack(stream, track, chunkLength)
-        except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
-            sleep(2)
-            return self.streamTrack(stream, track, start)
-
-    def updatePercentage(self):
-        if round(self.downloadPercentage) != self.lastPercentage and round(self.downloadPercentage) % 2 == 0:
-            self.lastPercentage = round(self.downloadPercentage)
-            self.downloadObject.progress = self.lastPercentage
-            if self.interface: self.interface.send("updateQueue", {'uuid': self.downloadObject.uuid, 'progress': self.lastPercentage})
-
-    def completeTrackPercentage(self):
-        if isinstance(self.downloadObject, Single):
-            self.downloadPercentage = 100
-        else:
-            self.downloadPercentage += (1 / self.downloadObject.size) * 100
-        self.updatePercentage()
-
-    def removeTrackPercentage(self):
-        if isinstance(self.downloadObject, Single):
-            self.downloadPercentage = 0
-        else:
-            self.downloadPercentage -= (1 / self.downloadObject.size) * 100
-        self.updatePercentage()
-
     def downloadWrapper(self, trackAPI_gw, trackAPI=None, albumAPI=None, playlistAPI=None, track=None):
         # Temp metadata to generate logs
         tempTrack = {
@@ -531,7 +460,7 @@ class Downloader:
                     }}
 
         if 'error' in result:
-            self.completeTrackPercentage()
+            self.downloadObject.completeTrackProgress(self.interface)
             self.downloadObject.failed += 1
             self.downloadObject.errors.append(result['error'])
             if self.interface:
@@ -640,9 +569,6 @@ class DownloadFailed(DownloadError):
 class DownloadCancelled(DownloadError):
     pass
 
-class DownloadEmpty(DownloadError):
-    pass
-
 class PreferredBitrateNotFound(DownloadError):
     pass
 

deemix/itemgen.py

@@ -1,4 +1,6 @@
 from deemix.types.DownloadObjects import Single, Collection
+from deezer.api import APIError
+from deezer.gw import GWAPIError, LyricsStatus
 
 class GenerationError(Exception):
     def __init__(self, link, message, errid=None):
@@ -29,7 +31,7 @@ def generateTrackItem(dz, id, bitrate, trackAPI=None, albumAPI=None):
     # Get essential track info
     try:
         trackAPI_gw = dz.gw.get_track_with_fallback(id)
-    except gwAPIError as e:
+    except GWAPIError as e:
         e = str(e)
         message = "Wrong URL"
         if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}"
@@ -116,7 +118,7 @@ def generatePlaylistItem(dz, id, bitrate, playlistAPI=None, playlistTracksAPI=No
             try:
                 userPlaylist = dz.gw.get_playlist_page(id)
                 playlistAPI = map_user_playlist(userPlaylist['DATA'])
-            except gwAPIError as e:
+            except GWAPIError as e:
                 e = str(e)
                 message = "Wrong URL"
                 if "DATA_ERROR" in e:

deemix/types/DownloadObjects.py

@@ -28,6 +28,7 @@ class IDownloadObject:
             self.progress = 0
             self.errors = []
             self.files = []
+        self.progressNext = 0
         self.uuid = f"{self.type}_{self.id}_{self.bitrate}"
         self.ack = None
         self.__type__ = None
@@ -69,6 +70,11 @@ class IDownloadObject:
                 del light[property]
         return light
 
+    def updateProgress(self, interface=None):
+        if round(self.progressNext) != self.progress and round(self.progressNext) % 2 == 0:
+            self.progress = round(self.progressNext)
+            if interface: interface.send("updateQueue", {'uuid': self.uuid, 'progress': self.progress})
+
 class Single(IDownloadObject):
     def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, trackAPI_gw=None, trackAPI=None, albumAPI=None, dictItem=None):
         if dictItem:
@@ -88,6 +94,14 @@ class Single(IDownloadObject):
         item['single'] = self.single
         return item
 
+    def completeTrackProgress(self, interface=None):
+        self.progressNext = 100
+        self.updateProgress(interface)
+
+    def removeTrackProgress(self, interface=None):
+        self.progressNext = 0
+        self.updateProgress(interface)
+
 class Collection(IDownloadObject):
     def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, tracks_gw=None, albumAPI=None, playlistAPI=None, dictItem=None):
         if dictItem:
@@ -107,6 +121,14 @@ class Collection(IDownloadObject):
         item['collection'] = self.collection
         return item
 
+    def completeTrackProgress(self, interface=None):
+        self.progressNext += (1 / self.size) * 100
+        self.updateProgress(interface)
+
+    def removeTrackProgress(self, interface=None):
+        self.progressNext -= (1 / self.size) * 100
+        self.updateProgress(interface)
+
 class Convertable(Collection):
     def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, plugin=None, conversion_data=None, dictItem=None):
         if dictItem:

deemix/types/Playlist.py

@@ -39,7 +39,7 @@ class Playlist:
 
         if 'various_artist' in playlistAPI:
             pic_md5 = playlistAPI['various_artist']['picture_small']
-            pic_md5 = pic_md5[pic_md5.indexOf('artist/') + 7:-24]
+            pic_md5 = pic_md5[pic_md5.find('artist/') + 7:-24]
             self.variousArtists = Artist(
                 id = playlistAPI['various_artist']['id'],
                 name = playlistAPI['various_artist']['name'],

deemix/types/Track.py

@@ -5,9 +5,10 @@ import logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger('deemix')
 
-from deezer.gw import APIError as gwAPIError
+from deezer.gw import GWAPIError
 from deezer.api import APIError
-from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString
+from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString, changeCase
+
 from deemix.types.Album import Album
 from deemix.types.Artist import Artist
 from deemix.types.Date import Date
@@ -114,7 +115,7 @@ class Track:
             # Get Lyrics data
             if not "LYRICS" in trackAPI_gw and self.lyrics.id != "0":
                 try: trackAPI_gw["LYRICS"] = dz.gw.get_track_lyrics(self.id)
-                except gwAPIError: self.lyrics.id = "0"
+                except GWAPIError: self.lyrics.id = "0"
             if self.lyrics.id != "0": self.lyrics.parseLyrics(trackAPI_gw["LYRICS"])
 
             # Parse Album data
@@ -132,7 +133,7 @@ class Track:
             # Get album_gw Data
             if not albumAPI_gw:
                 try: albumAPI_gw = dz.gw.get_album(self.album.id)
-                except gwAPIError: albumAPI_gw = None
+                except GWAPIError: albumAPI_gw = None
 
             if albumAPI:
                 self.album.parseAlbum(albumAPI)
@@ -261,7 +262,7 @@ class Track:
 
     def applySettings(self, settings, TEMPDIR, embeddedImageFormat):
         from deemix.settings import FeaturesOption
-        
+
         # Check if should save the playlist as a compilation
         if self.playlist and settings['tags']['savePlaylistAsCompilation']:
             self.trackNumber = self.position

deemix/utils/pathtemplates.py

@@ -148,7 +148,7 @@ def settingsRegex(filename, track, settings):
     filename = filename.replace("%album_id%", str(track.album.id))
     filename = filename.replace("%artist_id%", str(track.mainArtist.id))
     if track.playlist:
-        filename = filename.replace("%playlist_id%", str(track.playlist.playlistId))
+        filename = filename.replace("%playlist_id%", str(track.playlist.playlistID))
         filename = filename.replace("%position%", pad(track.position, track.playlist.trackTotal, settings))
     else:
         filename = filename.replace("%playlist_id%", '')
@@ -159,7 +159,7 @@ def settingsRegex(filename, track, settings):
 
 def settingsRegexAlbum(foldername, album, settings, playlist=None):
     if playlist and settings['tags']['savePlaylistAsCompilation']:
-        foldername = foldername.replace("%album_id%", "pl_" + str(playlist.playlistId))
+        foldername = foldername.replace("%album_id%", "pl_" + str(playlist.playlistID))
         foldername = foldername.replace("%genre%", "Compile")
     else:
         foldername = foldername.replace("%album_id%", str(album.id))
@@ -205,7 +205,7 @@ def settingsRegexArtist(foldername, artist, settings, rootArtist=None):
 
 def settingsRegexPlaylist(foldername, playlist, settings):
     foldername = foldername.replace("%playlist%", fixName(playlist.title, settings['illegalCharacterReplacer']))
-    foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistId, settings['illegalCharacterReplacer']))
+    foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistID, settings['illegalCharacterReplacer']))
     foldername = foldername.replace("%owner%", fixName(playlist.owner['name'], settings['illegalCharacterReplacer']))
     foldername = foldername.replace("%owner_id%", str(playlist.owner['id']))
     foldername = foldername.replace("%year%", str(playlist.date.year))