mirrors/youtube-dl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Use a dictionary for storing the subtitles

Browse Source 
The errors while getting the subtitles are reported as warnings, if no subtitles are found return and empty dict.
This commit is contained in:
Jaime Marquínez Ferrándiz
2013-06-26 11:03:44 +02:00
parent c4a91be726
commit 5d51a883c2
4 changed files with 59 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
test/test_youtube_subtitles.py
View File

@@ -35,47 +35,47 @@ class TestYoutubeSubtitles(unittest.TestCase):
DL.params['writesubtitles'] = True DL.params['writesubtitles'] = True
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['en']
self.assertEqual(md5(sub[2]), '4cd9278a35ba2305f47354ee13472260') self.assertEqual(md5(sub), '4cd9278a35ba2305f47354ee13472260')
def test_youtube_subtitles_it(self): def test_youtube_subtitles_it(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['writesubtitles'] = True DL.params['writesubtitles'] = True
DL.params['subtitleslang'] = 'it' DL.params['subtitleslang'] = 'it'
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['it']
self.assertEqual(md5(sub[2]), '164a51f16f260476a05b50fe4c2f161d') self.assertEqual(md5(sub), '164a51f16f260476a05b50fe4c2f161d')
def test_youtube_onlysubtitles(self): def test_youtube_onlysubtitles(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['writesubtitles'] = True DL.params['writesubtitles'] = True
DL.params['onlysubtitles'] = True DL.params['onlysubtitles'] = True
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['en']
self.assertEqual(md5(sub[2]), '4cd9278a35ba2305f47354ee13472260') self.assertEqual(md5(sub), '4cd9278a35ba2305f47354ee13472260')
def test_youtube_allsubtitles(self): def test_youtube_allsubtitles(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['allsubtitles'] = True DL.params['allsubtitles'] = True
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
subtitles = info_dict[0]['subtitles'] subtitles = info_dict[0]['subtitles']
self.assertEqual(len(subtitles), 13) self.assertEqual(len(subtitles.keys()), 13)
def test_youtube_subtitles_sbv_format(self): def test_youtube_subtitles_sbv_format(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['writesubtitles'] = True DL.params['writesubtitles'] = True
DL.params['subtitlesformat'] = 'sbv' DL.params['subtitlesformat'] = 'sbv'
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['en']
self.assertEqual(md5(sub[2]), '13aeaa0c245a8bed9a451cb643e3ad8b') self.assertEqual(md5(sub), '13aeaa0c245a8bed9a451cb643e3ad8b')
def test_youtube_subtitles_vtt_format(self): def test_youtube_subtitles_vtt_format(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['writesubtitles'] = True DL.params['writesubtitles'] = True
DL.params['subtitlesformat'] = 'vtt' DL.params['subtitlesformat'] = 'vtt'
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('QRS8MkLhQmM') info_dict = IE.extract('QRS8MkLhQmM')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['en']
self.assertEqual(md5(sub[2]), '356cdc577fde0c6783b9b822e7206ff7') self.assertEqual(md5(sub), '356cdc577fde0c6783b9b822e7206ff7')
def test_youtube_list_subtitles(self): def test_youtube_list_subtitles(self):
DL = FakeYDL() DL = FakeYDL()
DL.params['listsubtitles'] = True DL.params['listsubtitles'] = True
@@ -88,8 +88,8 @@ class TestYoutubeSubtitles(unittest.TestCase):
DL.params['subtitleslang'] = 'it' DL.params['subtitleslang'] = 'it'
IE = YoutubeIE(DL) IE = YoutubeIE(DL)
info_dict = IE.extract('8YoUxe5ncPo') info_dict = IE.extract('8YoUxe5ncPo')
sub = info_dict[0]['subtitles'][0] sub = info_dict[0]['subtitles']['it']
self.assertTrue(sub[2] is not None) self.assertTrue(sub is not None)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

23
youtube_dl/YoutubeDL.py
View File

@@ -492,19 +492,16 @@ class YoutubeDL(object):
# that way it will silently go on when used with unsupporting IE # that way it will silently go on when used with unsupporting IE
subtitles = info_dict['subtitles'] subtitles = info_dict['subtitles']
sub_format = self.params.get('subtitlesformat') sub_format = self.params.get('subtitlesformat')
for subtitle in subtitles: for sub_lang in subtitles.keys():
(sub_error, sub_lang, sub) = subtitle sub = subtitles[sub_lang]
if sub_error: try:
self.report_warning("Some error while getting the subtitles") sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
else: self.report_writesubtitles(sub_filename)
try: with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format subfile.write(sub)
self.report_writesubtitles(sub_filename) except (OSError, IOError):
with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: self.report_error(u'Cannot write subtitles file ' + descfn)
subfile.write(sub) return
except (OSError, IOError):
self.report_error(u'Cannot write subtitles file ' + descfn)
return
if self.params.get('writeinfojson', False): if self.params.get('writeinfojson', False):
infofn = filename + u'.info.json' infofn = filename + u'.info.json'

3
youtube_dl/extractor/common.py
View File

@@ -47,7 +47,8 @@ class InfoExtractor(object):
uploader_id: Nickname or id of the video uploader. uploader_id: Nickname or id of the video uploader.
location: Physical location of the video. location: Physical location of the video.
player_url: SWF Player URL (used for rtmpdump). player_url: SWF Player URL (used for rtmpdump).
subtitles: The subtitle file contents. subtitles: The subtitle file contents as a dictionary in the format
{language: subtitles}.
view_count: How many users have watched the video on the platform. view_count: How many users have watched the video on the platform.
urlhandle: [internal] The urlHandle to be used to download the file, urlhandle: [internal] The urlHandle to be used to download the file,
like returned by urllib.request.urlopen like returned by urllib.request.urlopen

67
youtube_dl/extractor/youtube.py
View File

@@ -209,11 +209,13 @@ class YoutubeIE(InfoExtractor):
try: try:
sub_list = compat_urllib_request.urlopen(request).read().decode('utf-8') sub_list = compat_urllib_request.urlopen(request).read().decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
return (u'unable to download video subtitles: %s' % compat_str(err), None) self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
return {}
sub_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list) sub_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
sub_lang_list = dict((l[1], l[0]) for l in sub_lang_list) sub_lang_list = dict((l[1], l[0]) for l in sub_lang_list)
if not sub_lang_list: if not sub_lang_list:
return (u'video doesn\'t have subtitles', None) self._downloader.report_warning(u'video doesn\'t have subtitles')
return {}
return sub_lang_list return sub_lang_list
def _list_available_subtitles(self, video_id): def _list_available_subtitles(self, video_id):
@@ -222,8 +224,7 @@ class YoutubeIE(InfoExtractor):
def _request_subtitle(self, sub_lang, sub_name, video_id, format): def _request_subtitle(self, sub_lang, sub_name, video_id, format):
""" """
Return tuple: Return the subtitle as a string or None if they are not found
(error_message, sub_lang, sub)
""" """
self.report_video_subtitles_request(video_id, sub_lang, format) self.report_video_subtitles_request(video_id, sub_lang, format)
params = compat_urllib_parse.urlencode({ params = compat_urllib_parse.urlencode({
@@ -236,10 +237,12 @@ class YoutubeIE(InfoExtractor):
try: try:
sub = compat_urllib_request.urlopen(url).read().decode('utf-8') sub = compat_urllib_request.urlopen(url).read().decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
return (u'unable to download video subtitles: %s' % compat_str(err), None, None) self._downloader.report_warning(u'unable to download video subtitles for %s: %s' % (sub_lang, compat_str(err)))
return
if not sub: if not sub:
return (u'Did not fetch video subtitles', None, None) self._downloader.report_warning(u'Did not fetch video subtitles')
return (None, sub_lang, sub) return
return sub
def _request_automatic_caption(self, video_id, webpage): def _request_automatic_caption(self, video_id, webpage):
"""We need the webpage for getting the captions url, pass it as an """We need the webpage for getting the captions url, pass it as an
@@ -250,7 +253,8 @@ class YoutubeIE(InfoExtractor):
mobj = re.search(r';ytplayer.config = ({.*?});', webpage) mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
err_msg = u'Couldn\'t find automatic captions for "%s"' % sub_lang err_msg = u'Couldn\'t find automatic captions for "%s"' % sub_lang
if mobj is None: if mobj is None:
return [(err_msg, None, None)] self._downloader.report_warning(err_msg)
return {}
player_config = json.loads(mobj.group(1)) player_config = json.loads(mobj.group(1))
try: try:
args = player_config[u'args'] args = player_config[u'args']
@@ -265,19 +269,20 @@ class YoutubeIE(InfoExtractor):
}) })
subtitles_url = caption_url + '&' + params subtitles_url = caption_url + '&' + params
sub = self._download_webpage(subtitles_url, video_id, u'Downloading automatic captions') sub = self._download_webpage(subtitles_url, video_id, u'Downloading automatic captions')
return [(None, sub_lang, sub)] return {sub_lang: sub}
except KeyError: except KeyError:
return [(err_msg, None, None)] self._downloader.report_warning(err_msg)
return {}
def _extract_subtitle(self, video_id): def _extract_subtitle(self, video_id):
""" """
Return a list with a tuple: Return a dictionary: {language: subtitles} or {} if the subtitles
[(error_message, sub_lang, sub)] couldn't be found
""" """
sub_lang_list = self._get_available_subtitles(video_id) sub_lang_list = self._get_available_subtitles(video_id)
sub_format = self._downloader.params.get('subtitlesformat') sub_format = self._downloader.params.get('subtitlesformat')
if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles if not sub_lang_list: #There was some error, it didn't get the available subtitles
return [(sub_lang_list[0], None, None)] return {}
if self._downloader.params.get('subtitleslang', False): if self._downloader.params.get('subtitleslang', False):
sub_lang = self._downloader.params.get('subtitleslang') sub_lang = self._downloader.params.get('subtitleslang')
elif 'en' in sub_lang_list: elif 'en' in sub_lang_list:
@@ -285,20 +290,28 @@ class YoutubeIE(InfoExtractor):
else: else:
sub_lang = list(sub_lang_list.keys())[0] sub_lang = list(sub_lang_list.keys())[0]
if not sub_lang in sub_lang_list: if not sub_lang in sub_lang_list:
return [(u'no closed captions found in the specified language "%s"' % sub_lang, None, None)] self._downloader.report_warning(u'no closed captions found in the specified language "%s"' % sub_lang)
return {}
subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format) subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format)
return [subtitle] if subtitle:
self.to_screen('sub %s' % subtitle[:20])
return {sub_lang: subtitle}
else:
return {}
def _extract_all_subtitles(self, video_id): def _extract_all_subtitles(self, video_id):
"""
Return a dicitonary: {language: subtitles} or {} if the subtitles
couldn't be found
"""
sub_lang_list = self._get_available_subtitles(video_id) sub_lang_list = self._get_available_subtitles(video_id)
sub_format = self._downloader.params.get('subtitlesformat') sub_format = self._downloader.params.get('subtitlesformat')
if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles subtitles = {}
return [(sub_lang_list[0], None, None)]
subtitles = []
for sub_lang in sub_lang_list: for sub_lang in sub_lang_list:
subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format) subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format)
subtitles.append(subtitle) if subtitle:
subtitles[sub_lang] = subtitle
return subtitles return subtitles
def _print_formats(self, formats): def _print_formats(self, formats):
@@ -523,23 +536,11 @@ class YoutubeIE(InfoExtractor):
if self._downloader.params.get('writesubtitles', False): if self._downloader.params.get('writesubtitles', False):
video_subtitles = self._extract_subtitle(video_id) video_subtitles = self._extract_subtitle(video_id)
if video_subtitles: elif self._downloader.params.get('writeautomaticsub', False):
(sub_error, sub_lang, sub) = video_subtitles[0]
if sub_error:
self._downloader.report_warning(sub_error)
if self._downloader.params.get('writeautomaticsub', False):
video_subtitles = self._request_automatic_caption(video_id, video_webpage) video_subtitles = self._request_automatic_caption(video_id, video_webpage)
(sub_error, sub_lang, sub) = video_subtitles[0]
if sub_error:
self._downloader.report_warning(sub_error)
if self._downloader.params.get('allsubtitles', False): if self._downloader.params.get('allsubtitles', False):
video_subtitles = self._extract_all_subtitles(video_id) video_subtitles = self._extract_all_subtitles(video_id)
for video_subtitle in video_subtitles:
(sub_error, sub_lang, sub) = video_subtitle
if sub_error:
self._downloader.report_warning(sub_error)
if self._downloader.params.get('listsubtitles', False): if self._downloader.params.get('listsubtitles', False):
self._list_available_subtitles(video_id) self._list_available_subtitles(video_id)