deemix-py/deemix/app/downloader.py

#!/usr/bin/env python3
from deemix.api.deezer import APIError, USER_AGENT_HEADER
from deemix.utils.taggers import tagID3, tagFLAC
from deemix.utils.pathtemplates import generateFilename, generateFilepath, settingsRegexAlbum, settingsRegexArtist
import os.path
from os import makedirs, remove
from requests import get
from requests.exceptions import HTTPError
from tempfile import gettempdir
from concurrent.futures import ThreadPoolExecutor
import json

TEMPDIR = os.path.join(gettempdir(), 'deezloader-imgs')
if not os.path.isdir(TEMPDIR):
	makedirs(TEMPDIR)

extensions = {
	9: '.flac',
	3: '.mp3',
	1: '.mp3',
	8: '.mp3',
	15: '.mp4',
	14: '.mp4',
	13: '.mp4'
}

def downloadImage(url, path):
	if not os.path.isfile(path):
		with open(path, 'wb') as f:
			try:
				f.write(get(url, headers={'User-Agent': USER_AGENT_HEADER}).content)
				return path
			except HTTPError:
				print("Couldn't download Image")
		remove(path)
		return None
	else:
		return path

def formatDate(date, template):
	if 'YYYY' in template:
		template = template.replace('YYYY', str(date['year']))
	if 'YY' in template:
		template = template.replace('YY', str(date['year']))
	if 'Y' in template:
		template = template.replace('Y', str(date['year']))
	if 'MM' in template:
		template = template.replace('MM', str(date['month']))
	if 'M' in template:
		template = template.replace('M', str(date['month']))
	if 'DD' in template:
		template = template.replace('DD', str(date['day']))
	if 'D' in template:
		template = template.replace('D', str(date['day']))
	return template


def getPreferredBitrate(filesize, bitrate):
	formats = {'flac': 9, 'mp3_320': 3, 'mp3_128': 1}
	selectedFormat = 8
	selectedFilesize = filesize['default']
	for format, formatNum in formats.items():
		if formatNum <= int(bitrate) and filesize[format] > 0:
			selectedFormat = formatNum
			selectedFilesize = filesize[format]
			break
	return (selectedFormat, selectedFilesize)

def parseEssentialTrackData(track, trackAPI):
	track['id'] = trackAPI['SNG_ID']
	track['duration'] = trackAPI['DURATION']
	track['MD5'] = trackAPI['MD5_ORIGIN']
	track['mediaVersion'] = trackAPI['MEDIA_VERSION']
	if 'FALLBACK' in trackAPI:
		track['fallbackId'] = trackAPI['FALLBACK']['SNG_ID']
	else:
		track['fallbackId'] = 0
	track['filesize'] = {}
	track['filesize']['default'] = int(trackAPI['FILESIZE']) if 'FILESIZE' in trackAPI else None
	track['filesize']['mp3_128'] = int(trackAPI['FILESIZE_MP3_128']) if 'FILESIZE_MP3_128' in trackAPI else None
	track['filesize']['mp3_320'] = int(trackAPI['FILESIZE_MP3_320']) if 'FILESIZE_MP3_320' in trackAPI else None
	track['filesize']['flac'] = int(trackAPI['FILESIZE_FLAC']) if 'FILESIZE_FLAC' in trackAPI else None
	track['filesize']['mp4_ra1'] = int(trackAPI['FILESIZE_MP4_RA1']) if 'FILESIZE_MP4_RA1' in trackAPI else None
	track['filesize']['mp4_ra2'] = int(trackAPI['FILESIZE_MP4_RA2']) if 'FILESIZE_MP4_RA2' in trackAPI else None
	track['filesize']['mp4_ra3'] = int(trackAPI['FILESIZE_MP4_RA3']) if 'FILESIZE_MP4_RA3' in trackAPI else None

	return track

def getTrackData(dz, trackAPI_gw, trackAPI = None, albumAPI_gw = None, albumAPI = None):
	if not 'MD5_ORIGIN' in trackAPI_gw:
		trackAPI_gw['MD5_ORIGIN'] = dz.get_track_md5(trackAPI_gw['SNG_ID'])

	track = {}
	track['title'] = trackAPI_gw['SNG_TITLE']
	if 'VERSION' in trackAPI_gw and trackAPI_gw['VERSION']:
		track['title'] += " " + trackAPI_gw['VERSION']

	track = parseEssentialTrackData(track, trackAPI_gw)

	if int(track['id']) < 0:
		track['filesize'] = trackAPI_gw['FILESIZE']
		track['album'] = {}
		track['album']['id'] = 0
		track['album']['title'] = trackAPI_gw['ALB_TITLE']
		if 'ALB_PICTURE' in trackAPI_gw:
			track['album']['pic'] = trackAPI_gw['ALB_PICTURE']
		track['mainArtist'] = {}
		track['mainArtist']['id'] = 0
		track['mainArtist']['name'] = trackAPI_gw['ART_NAME']
		track['artists'] = [trackAPI_gw['ART_NAME']]
		track['aritst'] = {
			'Main': [trackAPI_gw['ART_NAME']]
		}
		track['date'] = {
			'day': 0,
			'month': 0,
			'year': 0
		}
		track['localTrack'] = True
		return track

	if 'DISK_NUMBER' in trackAPI_gw:
		track['discNumber'] = trackAPI_gw['DISK_NUMBER']
	if 'EXPLICIT_LYRICS' in trackAPI_gw:
		track['explicit'] = trackAPI_gw['EXPLICIT_LYRICS'] != "0"
	if 'COPYRIGHT' in trackAPI_gw:
		track['copyright'] = trackAPI_gw['COPYRIGHT']
	track['replayGain'] = "{0:.2f} dB".format((float(trackAPI_gw['GAIN']) + 18.4) * -1) if 'GAIN' in trackAPI_gw else None
	track['ISRC'] = trackAPI_gw['ISRC']
	track['trackNumber'] = trackAPI_gw['TRACK_NUMBER']
	track['contributors'] = trackAPI_gw['SNG_CONTRIBUTORS']
	if 'POSITION' in trackAPI_gw:
		track['position'] = trackAPI_gw['POSITION']

	track['lyrics'] = {}
	if 'LYRICS_ID' in trackAPI_gw:
		track['lyrics']['id'] = trackAPI_gw['LYRICS_ID']
	if not "LYRICS" in trackAPI_gw and int(track['lyrics']['id']) != 0:
		trackAPI_gw["LYRICS"] = dz.get_lyrics_gw(track['id'])
	if int(track['lyrics']['id']) != 0:
		if "LYRICS_TEXT" in trackAPI_gw["LYRICS"]:
			track['lyrics']['unsync'] = trackAPI_gw["LYRICS"]["LYRICS_TEXT"]
		if "LYRICS_SYNC_JSON" in trackAPI_gw["LYRICS"]:
			track['lyrics']['sync'] = ""
			for i in range(len(trackAPI_gw["LYRICS"]["LYRICS_SYNC_JSON"])):
				if "lrc_timestamp" in trackAPI_gw["LYRICS"]["LYRICS_SYNC_JSON"][i]:
					track['lyrics']['sync'] += trackAPI_gw["LYRICS"]["LYRICS_SYNC_JSON"][i]["lrc_timestamp"] + \
											   trackAPI_gw["LYRICS"]["LYRICS_SYNC_JSON"][i]["line"] + "\r\n"
				elif i + 1 < len(trackAPI_gw["LYRICS"]["LYRICS_SYNC_JSON"]):
					track['lyrics']['sync'] += trackAPI_gw["LYRICS"]["LYRICS_SYNC_JSON"][i + 1]["lrc_timestamp"] + \
											   trackAPI_gw["LYRICS"]["LYRICS_SYNC_JSON"][i]["line"] + "\r\n"

	track['mainArtist'] = {}
	track['mainArtist']['id'] = trackAPI_gw['ART_ID']
	track['mainArtist']['name'] = trackAPI_gw['ART_NAME']
	if 'ART_PICTURE' in trackAPI_gw:
		track['mainArtist']['pic'] = trackAPI_gw['ART_PICTURE']

	if 'PHYSICAL_RELEASE_DATE' in trackAPI_gw:
		track['date'] = {
			'day': trackAPI_gw["PHYSICAL_RELEASE_DATE"][8:10],
			'month': trackAPI_gw["PHYSICAL_RELEASE_DATE"][5:7],
			'year': trackAPI_gw["PHYSICAL_RELEASE_DATE"][0:4]
		}

	track['album'] = {}
	track['album']['id'] = trackAPI_gw['ALB_ID']
	track['album']['title'] = trackAPI_gw['ALB_TITLE']
	if 'ALB_PICTURE' in trackAPI_gw:
		track['album']['pic'] = trackAPI_gw['ALB_PICTURE']

	try:
		if not albumAPI:
			albumAPI = dz.get_album(track['album']['id'])
		track['album']['mainArtist'] = {
			'id': albumAPI['artist']['id'],
			'name': albumAPI['artist']['name'],
			'pic': albumAPI['artist']['picture_small'][albumAPI['artist']['picture_small'].find('artist/')+7:-24]
		}
		track['album']['artist'] = {}
		track['album']['artists'] = []
		for artist in albumAPI['contributors']:
			if artist['id'] != 5080:
				track['album']['artists'].append(artist['name'])
				if not artist['role'] in track['album']['artist']:
					track['album']['artist'][artist['role']] = []
				track['album']['artist'][artist['role']].append(artist['name'])
		track['album']['trackTotal'] = albumAPI['nb_tracks']
		track['album']['recordType'] = albumAPI['record_type']
		track['album']['barcode'] = albumAPI['upc'] if 'upc' in albumAPI else "Unknown"
		track['album']['label'] = albumAPI['label'] if 'label' in albumAPI else "Unknown"
		if not 'pic' in track['album']:
			track['album']['pic'] = albumAPI['cover_small'][albumAPI['cover_small'].find('cover/')+6:-24]
		if 'release_date' in albumAPI:
			track['album']['date'] = {
				'day': albumAPI["release_date"][8:10],
				'month': albumAPI["release_date"][5:7],
				'year': albumAPI["release_date"][0:4]
			}
		track['album']['discTotal'] = albumAPI['nb_disk'] if 'nb_disk' in albumAPI else None
		track['copyright'] = albumAPI['copyright'] if 'copyright' in albumAPI else None
		track['album']['genre'] = []
		if 'genres' in albumAPI and 'data' in albumAPI['genres'] and len(albumAPI['genres']['data']) > 0:
			for genre in albumAPI['genres']['data']:
				track['album']['genre'].append(genre['name'])
	except APIError:
		if not albumAPI_gw:
			albumAPI_gw = dz.get_album_gw(track['album']['id'])
		track['album']['artist'] = {
			'id': albumAPI_gw['ART_ID'],
			'name': albumAPI_gw['ART_NAME']
		}
		artistAPI = dz.get_artist(track['album']['artist']['id'])
		track['album']['artist']['pic'] = artistAPI['picture_small'][artistAPI['picture_small'].find('artist/')+7:-24]
		track['album']['trackTotal'] = albumAPI_gw['NUMBER_TRACK']
		track['album']['discTotal'] = albumAPI_gw['NUMBER_DISK']
		track['album']['recordType'] = "Album"
		track['album']['barcode'] = "Unknown"
		track['album']['label'] = albumAPI_gw['LABEL_NAME'] if 'LABEL_NAME' in albumAPI_gw else "Unknown"
		if not 'pic' in track['album']:
			track['album']['pic'] = albumAPI_gw['ALB_PICTURE']
		if 'PHYSICAL_RELEASE_DATE' in albumAPI_gw:
			track['album']['date'] = {
				'day': albumAPI_gw["PHYSICAL_RELEASE_DATE"][8:10],
				'month': albumAPI_gw["PHYSICAL_RELEASE_DATE"][5:7],
				'year': albumAPI_gw["PHYSICAL_RELEASE_DATE"][0:4]
			}
		track['album']['genre'] = []

	if 'date' in track['album']:
		track['date'] = track['album']['date']

	if not trackAPI:
		trackAPI = dz.get_track(track['id'])
	track['bpm'] = trackAPI['bpm']
	if not 'replayGain' in track:
		track['replayGain'] = "{0:.2f} dB".format((float(trackAPI['gain']) + 18.4) * -1) if 'gain' in trackAPI else ""
	if not 'explicit' in track:
		track['explicit'] = trackAPI['explicit_lyrics']
	if not 'discNumber' in track:
		track['discNumber'] = trackAPI['disk_number']
	track['artist'] = {}
	track['artists'] = []
	for artist in trackAPI['contributors']:
		if artist['id'] != 5080:
			track['artists'].append(artist['name'])
			if not artist['role'] in track['artist']:
				track['artist'][artist['role']] = []
			track['artist'][artist['role']].append(artist['name'])

	if not 'discTotal' in track['album'] or not track['album']['discTotal']:
		if not albumAPI_gw:
			albumAPI_gw = dz.get_album_gw(track['album']['id'])
		track['album']['discTotal'] = albumAPI_gw['NUMBER_DISK']
	if not 'copyright' in track or not track['copyright']:
		if not albumAPI_gw:
			albumAPI_gw = dz.get_album_gw(track['album']['id'])
		track['copyright'] = albumAPI_gw['COPYRIGHT']

	# Fix incorrect day month when detectable
	if int(track['date']['month']) > 12:
		monthTemp = track['date']['month']
		track['date']['month'] = track['date']['day']
		track['date']['day'] = monthTemp
	if int(track['album']['date']['month']) > 12:
		monthTemp = track['album']['date']['month']
		track['album']['date']['month'] = track['album']['date']['day']
		track['album']['date']['day'] = monthTemp
	return track

def downloadTrackObj(dz, trackAPI, settings, overwriteBitrate=False, extraTrack=None):
	result = {}
	# Get the metadata
	if extraTrack:
		track = extraTrack
	else:
		track = getTrackData(dz,
			trackAPI_gw = trackAPI,
			trackAPI =  trackAPI['_EXTRA_TRACK'] if '_EXTRA_TRACK' in trackAPI else None,
			albumAPI = trackAPI['_EXTRA_ALBUM'] if '_EXTRA_ALBUM' in trackAPI else None
		)
	print('Downloading: {} - {}'.format(track['mainArtist']['name'], track['title']))
	if track['MD5'] == '':
		if track['fallbackId'] != 0:
			print("Track not available, using fallback id")
			trackNew = dz.get_track_gw(track['fallbackId'])
			if not 'MD5_ORIGIN' in trackNew:
				trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID'])
			track = parseEssentialTrackData(track, trackNew)
			return downloadTrackObj(dz, trackAPI, settings, extraTrack=track)
		else:
			print("ERROR: Track not yet encoded!")
			return result

	# Get the selected bitrate
	bitrate = overwriteBitrate if overwriteBitrate else settings['maxBitrate']
	(format, filesize) = getPreferredBitrate(track['filesize'], bitrate)
	track['selectedFormat'] = format
	track['selectedFilesize'] = filesize
	track['album']['bitrate'] = format
	track['album']['picUrl'] = "https://e-cdns-images.dzcdn.net/images/cover/{}/{}x{}-000000-80-0-0.{}".format(track['album']['pic'], settings['embeddedArtworkSize'], settings['embeddedArtworkSize'], 'png' if settings['PNGcovers'] else 'jpg')
	track['dateString'] = formatDate(track['date'], settings['dateFormat'])
	track['album']['dateString'] = formatDate(track['album']['date'], settings['dateFormat'])

	# Generate filename and filepath from metadata
	filename = generateFilename(track, trackAPI, settings)
	(filepath, artistPath, coverPath, extrasPath) = generateFilepath(track, trackAPI, settings)

	# Download and cache coverart
	track['album']['picPath'] = os.path.join(TEMPDIR, f"alb{track['album']['id']}_{settings['embeddedArtworkSize']}.{'png' if settings['PNGcovers'] else 'jpg'}")
	track['album']['picPath'] = downloadImage(track['album']['picUrl'], track['album']['picPath'])

	makedirs(filepath, exist_ok=True)
	writepath = os.path.join(filepath, filename + extensions[track['selectedFormat']])

	# Save lyrics in lrc file
	if settings['syncedLyrics'] and 'sync' in track['lyrics']:
		with open(os.path.join(filepath, filename + '.lrc'), 'w') as f:
			f.write(track['lyrics']['sync'])

	# Save local album art
	if coverPath:
		result['albumURL'] = track['album']['picUrl'].replace(f"{settings['embeddedArtworkSize']}x{settings['embeddedArtworkSize']}", f"{settings['localArtworkSize']}x{settings['localArtworkSize']}")
		result['albumPath'] = os.path.join(coverPath, f"{settingsRegexAlbum(settings['coverImageTemplate'], track['album'], settings)}.{'png' if settings['PNGcovers'] else 'jpg'}")

	# Save artist art
	if artistPath:
		result['artistURL'] = "https://e-cdns-images.dzcdn.net/images/artist/{}/{}x{}-000000-80-0-0.{}".format(track['album']['artist']['pic'], settings['localArtworkSize'], settings['localArtworkSize'], 'png' if settings['PNGcovers'] else 'jpg')
		result['artistPath'] = os.path.join(artistPath, f"{settingsRegexArtist(settings['artistImageTemplate'], track['album']['artist'], settings)}.{'png' if settings['PNGcovers'] else 'jpg'}")

	# Data for m3u file
	if extrasPath:
		result['extrasPath'] = extrasPath
		result['playlistPosition'] = writepath[len(extrasPath):]

	track['downloadUrl'] = dz.get_track_stream_url(track['id'], track['MD5'], track['mediaVersion'], track['selectedFormat'])
	try:
		with open(writepath, 'wb') as stream:
			dz.stream_track(track['id'], track['downloadUrl'], stream)
	except HTTPError:
		remove(writepath)
		if track['selectedFormat'] == 9:
			print("Track not available in flac, trying mp3")
			track['filesize']['flac'] = 0
			return downloadTrackObj(dz, trackAPI, settings, extraTrack=track)
		elif track['fallbackId'] != 0:
			print("Track not available, using fallback id")
			trackNew = dz.get_track_gw(track['fallbackId'])
			if not 'MD5_ORIGIN' in trackNew:
				trackNew['MD5_ORIGIN'] = dz.get_track_md5(trackNew['SNG_ID'])
			track = parseEssentialTrackData(track, trackNew)
			return downloadTrackObj(dz, trackAPI, settings, extraTrack=track)
		else:
			print("ERROR: Track not available on deezer's servers!")
			return result
	if track['selectedFormat'] in [3, 1, 8]:
		tagID3(writepath, track, settings['tags'], settings['saveID3v1'], settings['useNullSeparator'])
	elif track['selectedFormat'] == 9:
		tagFLAC(writepath, track, settings['tags'])
	print("Done!")
	return result

def download_track(dz, id, settings, overwriteBitrate=False):
	trackAPI = dz.get_track_gw(id)
	trackAPI['FILENAME_TEMPLATE'] = settings['tracknameTemplate']
	trackAPI['SINGLE_TRACK'] = True
	downloadTrackObj(dz, trackAPI, settings, overwriteBitrate)

def download_album(dz, id, settings, overwriteBitrate=False):
	albumAPI = dz.get_album(id)
	albumAPI_gw = dz.get_album_gw(id)
	albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK']
	albumAPI['copyright'] = albumAPI_gw['COPYRIGHT']
	if albumAPI['nb_tracks'] == 1:
		trackAPI = dz.get_track_gw(albumAPI['tracks']['data'][0]['id'])
		trackAPI['_EXTRA_ALBUM'] = albumAPI
		trackAPI['FILENAME_TEMPLATE'] = settings['tracknameTemplate']
		trackAPI['SINGLE_TRACK'] = True
		downloadTrackObj(dz, trackAPI, settings, overwriteBitrate)
	else:
		tracksArray = dz.get_album_tracks_gw(id)
		playlist = [None] * len(tracksArray)
		with ThreadPoolExecutor(settings['queueConcurrency']) as executor:
			for pos, trackAPI in enumerate(tracksArray, start=1):
				trackAPI['_EXTRA_ALBUM'] = albumAPI
				trackAPI['POSITION'] = pos
				trackAPI['FILENAME_TEMPLATE'] = settings['albumTracknameTemplate']
				playlist[pos-1] = executor.submit(downloadTrackObj, dz, trackAPI, settings, overwriteBitrate)

		extrasPath = None
		for index in range(len(playlist)):
			result = playlist[index].result()
			if not extrasPath:
				extrasPath = result['extrasPath']
			if settings['saveArtwork'] and result['albumPath']:
				downloadImage(result['albumURL'], result['albumPath'])
			if settings['saveArtworkArtist'] and result['artistPath']:
				downloadImage(result['artistURL'], result['artistPath'])
			if 'playlistPosition' in result:
				playlist[index] = result['playlistPosition']
			else:
				playlist[index] = ""
		if settings['createM3U8File'] and extrasPath:
			with open(os.path.join(extrasPath, 'playlist.m3u8'), 'w') as f:
				for line in playlist:
					f.write(line+"\n")

def download_artist(dz, id, settings, overwriteBitrate=False):
	artistAPI = dz.get_artist_albums(id)
	for album in artistAPI['data']:
		print(f"Album: {album['title']}")
		download_album(dz, album['id'], settings, overwriteBitrate)

def download_playlist(dz, id, settings, overwriteBitrate=False):
	playlistAPI = dz.get_playlist(id)
	playlistTracksAPI = dz.get_playlist_tracks_gw(id)
	playlist = [None] * len(playlistTracksAPI)
	with ThreadPoolExecutor(settings['queueConcurrency']) as executor:
		for pos, trackAPI in enumerate(playlistTracksAPI, start=1):
			trackAPI['_EXTRA_PLAYLIST'] = playlistAPI
			trackAPI['POSITION'] = pos
			trackAPI['FILENAME_TEMPLATE'] = settings['playlistTracknameTemplate']
			playlist[pos-1] = executor.submit(downloadTrackObj, dz, trackAPI, settings, overwriteBitrate)

	extrasPath = None
	for index in range(len(playlist)):
		result = playlist[index].result()
		if not extrasPath:
			extrasPath = result['extrasPath']
		if settings['saveArtwork'] and result['albumPath']:
			downloadImage(result['albumURL'], result['albumPath'])
		if settings['saveArtworkArtist'] and result['artistPath']:
			downloadImage(result['artistURL'], result['artistPath'])
		if 'playlistPosition' in result:
			playlist[index] = result['playlistPosition']
		else:
			playlist[index] = ""
	if settings['createM3U8File'] and extrasPath:
		with open(os.path.join(extrasPath, 'playlist.m3u8'), 'w') as f:
			for line in playlist:
				f.write(line+"\n")