release 2014.04.21

It uses the same system as discovery.com

test/helper.py

@@ -74,13 +74,19 @@ class FakeYDL(YoutubeDL):
             old_report_warning(message)
         self.report_warning = types.MethodType(report_warning, self)
 
-def gettestcases():
+
+def gettestcases(include_onlymatching=False):
     for ie in youtube_dl.extractor.gen_extractors():
         t = getattr(ie, '_TEST', None)
         if t:
-            t['name'] = type(ie).__name__[:-len('IE')]
-            yield t
-        for t in getattr(ie, '_TESTS', []):
+            assert not hasattr(ie, '_TESTS'), \
+                '%s has _TEST and _TESTS' % type(ie).__name__
+            tests = [t]
+        else:
+            tests = getattr(ie, '_TESTS', [])
+        for t in tests:
+            if not include_onlymatching and getattr(t, 'only_matching', False):
+                continue
             t['name'] = type(ie).__name__[:-len('IE')]
             yield t
 

test/test_all_urls.py

@@ -77,20 +77,20 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
 
     def test_justin_tv_channelid_matching(self):
-        self.assertTrue(JustinTVIE.suitable(u"justin.tv/vanillatv"))
-        self.assertTrue(JustinTVIE.suitable(u"twitch.tv/vanillatv"))
-        self.assertTrue(JustinTVIE.suitable(u"www.justin.tv/vanillatv"))
-        self.assertTrue(JustinTVIE.suitable(u"www.twitch.tv/vanillatv"))
-        self.assertTrue(JustinTVIE.suitable(u"http://www.justin.tv/vanillatv"))
-        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv"))
-        self.assertTrue(JustinTVIE.suitable(u"http://www.justin.tv/vanillatv/"))
-        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv/"))
+        self.assertTrue(JustinTVIE.suitable('justin.tv/vanillatv'))
+        self.assertTrue(JustinTVIE.suitable('twitch.tv/vanillatv'))
+        self.assertTrue(JustinTVIE.suitable('www.justin.tv/vanillatv'))
+        self.assertTrue(JustinTVIE.suitable('www.twitch.tv/vanillatv'))
+        self.assertTrue(JustinTVIE.suitable('http://www.justin.tv/vanillatv'))
+        self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv'))
+        self.assertTrue(JustinTVIE.suitable('http://www.justin.tv/vanillatv/'))
+        self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv/'))
 
     def test_justintv_videoid_matching(self):
-        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv/b/328087483"))
+        self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv/b/328087483'))
 
     def test_justin_tv_chapterid_matching(self):
-        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/tsm_theoddone/c/2349361"))
+        self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361'))
 
     def test_youtube_extract(self):
         assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
@@ -106,7 +106,7 @@ class TestAllURLsMatching(unittest.TestCase):
 
     def test_no_duplicates(self):
         ies = gen_extractors()
-        for tc in gettestcases():
+        for tc in gettestcases(include_onlymatching=True):
             url = tc['url']
             for ie in ies:
                 if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'):
@@ -176,5 +176,6 @@ class TestAllURLsMatching(unittest.TestCase):
             'https://screen.yahoo.com/smartwatches-latest-wearable-gadgets-163745379-cbs.html',
             ['Yahoo'])
 
+
 if __name__ == '__main__':
     unittest.main()

test/test_playlists.py

@@ -192,8 +192,8 @@ class TestPlaylists(unittest.TestCase):
         self.assertIsPlaylist(result)
         self.assertEqual(result['id'], 'dezhurnyi_angel')
         self.assertEqual(result['title'], 'Дежурный ангел (2010 - 2012)')
-        self.assertTrue(len(result['entries']) >= 36)
-        
+        self.assertTrue(len(result['entries']) >= 23)
+
     def test_ivi_compilation_season(self):
         dl = FakeYDL()
         ie = IviCompilationIE(dl)
@@ -201,7 +201,7 @@ class TestPlaylists(unittest.TestCase):
         self.assertIsPlaylist(result)
         self.assertEqual(result['id'], 'dezhurnyi_angel/season2')
         self.assertEqual(result['title'], 'Дежурный ангел (2010 - 2012) 2 сезон')
-        self.assertTrue(len(result['entries']) >= 20)
+        self.assertTrue(len(result['entries']) >= 7)
         
     def test_imdb_list(self):
         dl = FakeYDL()

youtube_dl/extractor/__init__.py

@@ -181,6 +181,7 @@ from .nfb import NFBIE
 from .nhl import NHLIE, NHLVideocenterIE
 from .niconico import NiconicoIE
 from .ninegag import NineGagIE
+from .noco import NocoIE
 from .normalboots import NormalbootsIE
 from .novamov import NovaMovIE
 from .nowness import NownessIE
@@ -251,6 +252,7 @@ from .tf1 import TF1IE
 from .theplatform import ThePlatformIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
+from .tlc import TlcIE, TlcDeIE
 from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE

youtube_dl/extractor/canalplus.py

@@ -1,4 +1,6 @@
 # encoding: utf-8
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
@@ -8,46 +10,56 @@ from ..utils import unified_strdate
 class CanalplusIE(InfoExtractor):
     _VALID_URL = r'https?://(www\.canalplus\.fr/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>\d+))'
     _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s'
-    IE_NAME = u'canalplus.fr'
+    IE_NAME = 'canalplus.fr'
 
     _TEST = {
-        u'url': u'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
-        u'file': u'922470.flv',
-        u'info_dict': {
-            u'title': u'Zapping - 26/08/13',
-            u'description': u'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
-            u'upload_date': u'20130826',
-        },
-        u'params': {
-            u'skip_download': True,
+        'url': 'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
+        'md5': '60c29434a416a83c15dae2587d47027d',
+        'info_dict': {
+            'id': '922470',
+            'ext': 'flv',
+            'title': 'Zapping - 26/08/13',
+            'description': 'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
+            'upload_date': '20130826',
         },
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.groupdict().get('id')
+        video_id = mobj.group('id')
+
         if video_id is None:
             webpage = self._download_webpage(url, mobj.group('path'))
-            video_id = self._search_regex(r'<canal:player videoId="(\d+)"', webpage, u'video id')
+            video_id = self._search_regex(r'<canal:player videoId="(\d+)"', webpage, 'video id')
+
         info_url = self._VIDEO_INFO_TEMPLATE % video_id
-        doc = self._download_xml(info_url,video_id, 
-                                           u'Downloading video info')
+        doc = self._download_xml(info_url, video_id, 'Downloading video XML')
 
-        self.report_extraction(video_id)
         video_info = [video for video in doc if video.find('ID').text == video_id][0]
-        infos = video_info.find('INFOS')
         media = video_info.find('MEDIA')
-        formats = [media.find('VIDEOS/%s' % format)
-            for format in ['BAS_DEBIT', 'HAUT_DEBIT', 'HD']]
-        video_url = [format.text for format in formats if format is not None][-1]
+        infos = video_info.find('INFOS')
 
-        return {'id': video_id,
-                'title': u'%s - %s' % (infos.find('TITRAGE/TITRE').text,
-                                       infos.find('TITRAGE/SOUS_TITRE').text),
-                'url': video_url,
-                'ext': 'flv',
-                'upload_date': unified_strdate(infos.find('PUBLICATION/DATE').text),
-                'thumbnail': media.find('IMAGES/GRAND').text,
-                'description': infos.find('DESCRIPTION').text,
-                'view_count': int(infos.find('NB_VUES').text),
-                }
+        preferences = ['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS']
+
+        formats = [
+            {
+                'url': fmt.text + '?hdcore=2.11.3' if fmt.tag == 'HDS' else fmt.text,
+                'format_id': fmt.tag,
+                'ext': 'mp4' if fmt.tag == 'HLS' else 'flv',
+                'preference': preferences.index(fmt.tag) if fmt.tag in preferences else -1,
+            } for fmt in media.find('VIDEOS') if fmt.text
+        ]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': '%s - %s' % (infos.find('TITRAGE/TITRE').text,
+                                  infos.find('TITRAGE/SOUS_TITRE').text),
+            'upload_date': unified_strdate(infos.find('PUBLICATION/DATE').text),
+            'thumbnail': media.find('IMAGES/GRAND').text,
+            'description': infos.find('DESCRIPTION').text,
+            'view_count': int(infos.find('NB_VUES').text),
+            'like_count': int(infos.find('NB_LIKES').text),
+            'comment_count': int(infos.find('NB_COMMENTS').text),
+            'formats': formats,
+        }

youtube_dl/extractor/dailymotion.py

@@ -8,7 +8,6 @@ from .subtitles import SubtitlesInfoExtractor
 from ..utils import (
     compat_urllib_request,
     compat_str,
-    get_element_by_id,
     orderedSet,
     str_to_int,
     int_or_none,

youtube_dl/extractor/extremetube.py

@@ -1,4 +1,5 @@
-import os
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
@@ -8,18 +9,23 @@ from ..utils import (
     compat_urllib_parse,
 )
 
+
 class ExtremeTubeIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>extremetube\.com/video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
-    _TEST = {
-        u'url': u'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
-        u'file': u'652431.mp4',
-        u'md5': u'1fb9228f5e3332ec8c057d6ac36f33e0',
-        u'info_dict': {
-            u"title": u"Music Video 14 british euro brit european cumshots swallow",
-            u"uploader": u"unknown",
-            u"age_limit": 18,
+    _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>extremetube\.com/.*?video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
+    _TESTS = [{
+        'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
+        'md5': '1fb9228f5e3332ec8c057d6ac36f33e0',
+        'info_dict': {
+            'id': '652431',
+            'ext': 'mp4',
+            'title': 'Music Video 14 british euro brit european cumshots swallow',
+            'uploader': 'unknown',
+            'age_limit': 18,
         }
-    }
+    }, {
+        'url': 'http://www.extremetube.com/gay/video/abcde-1234',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -30,11 +36,14 @@ class ExtremeTubeIE(InfoExtractor):
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
-        video_title = self._html_search_regex(r'<h1 [^>]*?title="([^"]+)"[^>]*>\1<', webpage, u'title')
-        uploader = self._html_search_regex(r'>Posted by:(?=<)(?:\s|<[^>]*>)*(.+?)\|', webpage, u'uploader', fatal=False)
-        video_url = compat_urllib_parse.unquote(self._html_search_regex(r'video_url=(.+?)&amp;', webpage, u'video_url'))
+        video_title = self._html_search_regex(
+            r'<h1 [^>]*?title="([^"]+)"[^>]*>\1<', webpage, 'title')
+        uploader = self._html_search_regex(
+            r'>Posted by:(?=<)(?:\s|<[^>]*>)*(.+?)\|', webpage, 'uploader',
+            fatal=False)
+        video_url = compat_urllib_parse.unquote(self._html_search_regex(
+            r'video_url=(.+?)&amp;', webpage, 'video_url'))
         path = compat_urllib_parse_urlparse(video_url).path
-        extension = os.path.splitext(path)[1][1:]
         format = path.split('/')[5].split('_')[:2]
         format = "-".join(format)
 
@@ -43,7 +52,6 @@ class ExtremeTubeIE(InfoExtractor):
             'title': video_title,
             'uploader': uploader,
             'url': video_url,
-            'ext': extension,
             'format': format,
             'format_id': format,
             'age_limit': 18,

youtube_dl/extractor/fivemin.py

@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     compat_str,
+    compat_urllib_parse,
 )
 
 
@@ -16,16 +17,28 @@ class FiveMinIE(InfoExtractor):
         (?P<id>\d+)
         '''
 
-    _TEST = {
-        # From http://www.engadget.com/2013/11/15/ipad-mini-retina-display-review/
-        'url': 'http://pshared.5min.com/Scripts/PlayerSeed.js?sid=281&width=560&height=345&playList=518013791',
-        'md5': '4f7b0b79bf1a470e5004f7112385941d',
-        'info_dict': {
-            'id': '518013791',
-            'ext': 'mp4',
-            'title': 'iPad Mini with Retina Display Review',
+    _TESTS = [
+        {
+            # From http://www.engadget.com/2013/11/15/ipad-mini-retina-display-review/
+            'url': 'http://pshared.5min.com/Scripts/PlayerSeed.js?sid=281&width=560&height=345&playList=518013791',
+            'md5': '4f7b0b79bf1a470e5004f7112385941d',
+            'info_dict': {
+                'id': '518013791',
+                'ext': 'mp4',
+                'title': 'iPad Mini with Retina Display Review',
+            },
         },
-    }
+        {
+            # From http://on.aol.com/video/how-to-make-a-next-level-fruit-salad-518086247
+            'url': '5min:518086247',
+            'md5': 'e539a9dd682c288ef5a498898009f69e',
+            'info_dict': {
+                'id': '518086247',
+                'ext': 'mp4',
+                'title': 'How to Make a Next-Level Fruit Salad',
+            },
+        },
+    ]
 
     @classmethod
     def _build_result(cls, video_id):
@@ -34,9 +47,19 @@ class FiveMinIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        embed_url = 'https://embed.5min.com/playerseed/?playList=%s' % video_id
+        embed_page = self._download_webpage(embed_url, video_id,
+            'Downloading embed page')
+        sid = self._search_regex(r'sid=(\d+)', embed_page, 'sid')
+        query = compat_urllib_parse.urlencode({
+            'func': 'GetResults',
+            'playlist': video_id,
+            'sid': sid,
+            'isPlayerSeed': 'true',
+            'url': embed_url,
+        })
         info = self._download_json(
-            'https://syn.5min.com/handlers/SenseHandler.ashx?func=GetResults&'
-            'playlist=%s&url=https' % video_id,
+            'https://syn.5min.com/handlers/SenseHandler.ashx?' + query,
             video_id)['binding'][0]
 
         second_id = compat_str(int(video_id[:-2]) + 1)

youtube_dl/extractor/generic.py

@@ -35,9 +35,10 @@ class GenericIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
-            'file': '13601338388002.mp4',
-            'md5': '6e15c93721d7ec9e9ca3fdbf07982cfd',
+            'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
             'info_dict': {
+                'id': '13601338388002',
+                'ext': 'mp4',
                 'uploader': 'www.hodiho.fr',
                 'title': 'R\u00e9gis plante sa Jeep',
             }
@@ -46,8 +47,9 @@ class GenericIE(InfoExtractor):
         {
             'add_ie': ['Bandcamp'],
             'url': 'http://bronyrock.com/track/the-pony-mash',
-            'file': '3235767654.mp3',
             'info_dict': {
+                'id': '3235767654',
+                'ext': 'mp3',
                 'title': 'The Pony Mash',
                 'uploader': 'M_Pallante',
             },
@@ -73,9 +75,10 @@ class GenericIE(InfoExtractor):
         {
             # https://github.com/rg3/youtube-dl/issues/2253
             'url': 'http://bcove.me/i6nfkrc3',
-            'file': '3101154703001.mp4',
             'md5': '0ba9446db037002366bab3b3eb30c88c',
             'info_dict': {
+                'id': '3101154703001',
+                'ext': 'mp4',
                 'title': 'Still no power',
                 'uploader': 'thestar.com',
                 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',

youtube_dl/extractor/ign.py

@@ -106,7 +106,7 @@ class OneUPIE(IGNIE):
 
     _DESCRIPTION_RE = r'<div id="vid_summary">(.+?)</div>'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://gamevideos.1up.com/video/id/34976',
         'md5': '68a54ce4ebc772e4b71e3123d413163d',
         'info_dict': {
@@ -115,10 +115,7 @@ class OneUPIE(IGNIE):
             'title': 'Sniper Elite V2 - Trailer',
             'description': 'md5:5d289b722f5a6d940ca3136e9dae89cf',
         }
-    }
-
-    # Override IGN tests
-    _TESTS = []
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

youtube_dl/extractor/ninegag.py

@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
 import re
+import json
 
 from .common import InfoExtractor
+from ..utils import str_to_int
 
 
 class NineGagIE(InfoExtractor):
@@ -44,23 +46,14 @@ class NineGagIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        youtube_id = self._html_search_regex(
-            r'(?s)id="jsid-video-post-container".*?data-external-id="([^"]+)"',
-            webpage, 'video ID')
-        title = self._html_search_regex(
-            r'(?s)id="jsid-video-post-container".*?data-title="([^"]+)"',
-            webpage, 'title', default=None)
-        if not title:
-            title = self._og_search_title(webpage)
-        description = self._html_search_regex(
-            r'(?s)<div class="video-caption">.*?<p>(.*?)</p>', webpage,
-            'description', fatal=False)
-        view_count_str = self._html_search_regex(
-            r'<p><b>([0-9][0-9,]*)</b> views</p>', webpage, 'view count',
-            fatal=False)
-        view_count = (
-            None if view_count_str is None
-            else int(view_count_str.replace(',', '')))
+        post_view = json.loads(self._html_search_regex(
+            r'var postView = new app\.PostView\({ post: ({.+?}),', webpage, 'post view'))
+
+        youtube_id = post_view['videoExternalId']
+        title = post_view['title']
+        description = post_view['description']
+        view_count = str_to_int(post_view['externalView'])
+        thumbnail = post_view.get('thumbnail_700w') or post_view.get('ogImageUrl') or post_view.get('thumbnail_300w')
 
         return {
             '_type': 'url_transparent',
@@ -71,5 +64,5 @@ class NineGagIE(InfoExtractor):
             'title': title,
             'description': description,
             'view_count': view_count,
-            'thumbnail': self._og_search_thumbnail(webpage),
+            'thumbnail': thumbnail,
         }

youtube_dl/extractor/noco.py

@@ -0,0 +1,105 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    unified_strdate,
+    compat_str,
+)
+
+
+class NocoIE(InfoExtractor):
+    _VALID_URL = r'http://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://noco.tv/emission/11538/nolife/ami-ami-idol-hello-france/',
+        'md5': '0a993f0058ddbcd902630b2047ef710e',
+        'info_dict': {
+            'id': '11538',
+            'ext': 'mp4',
+            'title': 'Ami Ami Idol - Hello! France',
+            'description': 'md5:4eaab46ab68fa4197a317a88a53d3b86',
+            'upload_date': '20140412',
+            'uploader': 'Nolife',
+            'uploader_id': 'NOL',
+            'duration': 2851.2,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        medias = self._download_json(
+            'http://api.noco.tv/1.0/video/medias/%s' % video_id, video_id, 'Downloading video JSON')
+
+        formats = []
+
+        for fmt in medias['fr']['video_list']['default']['quality_list']:
+            format_id = fmt['quality_key']
+
+            file = self._download_json(
+                'http://api.noco.tv/1.0/video/file/%s/fr/%s' % (format_id.lower(), video_id),
+                video_id, 'Downloading %s video JSON' % format_id)
+
+            file_url = file['file']
+            if not file_url:
+                continue
+
+            if file_url == 'forbidden':
+                raise ExtractorError(
+                    '%s returned error: %s - %s' % (
+                        self.IE_NAME, file['popmessage']['title'], file['popmessage']['message']),
+                    expected=True)
+
+            formats.append({
+                'url': file_url,
+                'format_id': format_id,
+                'width': fmt['res_width'],
+                'height': fmt['res_lines'],
+                'abr': fmt['audiobitrate'],
+                'vbr': fmt['videobitrate'],
+                'filesize': fmt['filesize'],
+                'format_note': fmt['quality_name'],
+                'preference': fmt['priority'],
+            })
+
+        self._sort_formats(formats)
+
+        show = self._download_json(
+            'http://api.noco.tv/1.0/shows/show/%s' % video_id, video_id, 'Downloading show JSON')[0]
+
+        upload_date = unified_strdate(show['indexed'])
+        uploader = show['partner_name']
+        uploader_id = show['partner_key']
+        duration = show['duration_ms'] / 1000.0
+        thumbnail = show['screenshot']
+
+        episode = show.get('show_TT') or show.get('show_OT')
+        family = show.get('family_TT') or show.get('family_OT')
+        episode_number = show.get('episode_number')
+
+        title = ''
+        if family:
+            title += family
+        if episode_number:
+            title += ' #' + compat_str(episode_number)
+        if episode:
+            title += ' - ' + episode
+
+        description = show.get('show_resume') or show.get('family_resume')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'duration': duration,
+            'formats': formats,
+        }

youtube_dl/extractor/podomatic.py

@@ -6,22 +6,36 @@ import re
 from .common import InfoExtractor
 from ..utils import int_or_none
 
-
 class PodomaticIE(InfoExtractor):
     IE_NAME = 'podomatic'
     _VALID_URL = r'^(?P<proto>https?)://(?P<channel>[^.]+)\.podomatic\.com/entry/(?P<id>[^?]+)'
 
-    _TEST = {
-        "url": "http://scienceteachingtips.podomatic.com/entry/2009-01-02T16_03_35-08_00",
-        "file": "2009-01-02T16_03_35-08_00.mp3",
-        "md5": "84bb855fcf3429e6bf72460e1eed782d",
-        "info_dict": {
-            "uploader": "Science Teaching Tips",
-            "uploader_id": "scienceteachingtips",
-            "title": "64.  When the Moon Hits Your Eye",
-            "duration": 446,
-        }
-    }
+    _TESTS = [
+        {
+            'url': 'http://scienceteachingtips.podomatic.com/entry/2009-01-02T16_03_35-08_00',
+            'md5': '84bb855fcf3429e6bf72460e1eed782d',
+            'info_dict': {
+                'id': '2009-01-02T16_03_35-08_00',
+                'ext': 'mp3',
+                'uploader': 'Science Teaching Tips',
+                'uploader_id': 'scienceteachingtips',
+                'title': '64.  When the Moon Hits Your Eye',
+                'duration': 446,
+            }
+        },
+        {
+            'url': 'http://ostbahnhof.podomatic.com/entry/2013-11-15T16_31_21-08_00',
+            'md5': 'd2cf443931b6148e27638650e2638297',
+            'info_dict': {
+                'id': '2013-11-15T16_31_21-08_00',
+                'ext': 'mp3',
+                'uploader': 'Ostbahnhof / Techno Mix',
+                'uploader_id': 'ostbahnhof',
+                'title': 'Einunddreizig',
+                'duration': 3799,
+            }
+        },
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -32,10 +46,12 @@ class PodomaticIE(InfoExtractor):
                      '?permalink=true&rtmp=0') %
                     (mobj.group('proto'), channel, video_id))
         data_json = self._download_webpage(
-            json_url, video_id, note=u'Downloading video info')
+            json_url, video_id, 'Downloading video info')
         data = json.loads(data_json)
 
         video_url = data['downloadLink']
+        if not video_url:
+            video_url = '%s/%s' % (data['streamer'].replace('rtmp', 'http'), data['mediaLocation'])
         uploader = data['podcast']
         title = data['title']
         thumbnail = data['imageLocation']

youtube_dl/extractor/rutube.py

@@ -43,13 +43,14 @@ class RutubeIE(InfoExtractor):
             'http://rutube.ru/api/video/%s/?format=json' % video_id,
             video_id, 'Downloading video JSON')
 
-        trackinfo = self._download_json(
-            'http://rutube.ru/api/play/trackinfo/%s/?format=json' % video_id,
-            video_id, 'Downloading trackinfo JSON')
-
         # Some videos don't have the author field
-        author = trackinfo.get('author') or {}
-        m3u8_url = trackinfo['video_balancer'].get('m3u8')
+        author = video.get('author') or {}
+
+        options = self._download_json(
+            'http://rutube.ru/api/play/options/%s/?format=json' %video_id,
+            video_id, 'Downloading options JSON')
+
+        m3u8_url = options['video_balancer'].get('m3u8')
         if m3u8_url is None:
             raise ExtractorError('Couldn\'t find m3u8 manifest url')
 

youtube_dl/extractor/steam.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
@@ -8,78 +10,114 @@ from ..utils import (
 
 
 class SteamIE(InfoExtractor):
-    _VALID_URL = r"""http://store\.steampowered\.com/
-                (agecheck/)?
-                (?P<urltype>video|app)/ #If the page is only for videos or for a game
-                (?P<gameID>\d+)/?
-                (?P<videoID>\d*)(?P<extra>\??) #For urltype == video we sometimes get the videoID
-                """
+    _VALID_URL = r"""(?x)
+        https?://store\.steampowered\.com/
+            (agecheck/)?
+            (?P<urltype>video|app)/ #If the page is only for videos or for a game
+            (?P<gameID>\d+)/?
+            (?P<videoID>\d*)(?P<extra>\??) # For urltype == video we sometimes get the videoID
+        |
+        https?://(?:www\.)?steamcommunity\.com/sharedfiles/filedetails/\?id=(?P<fileID>[0-9]+)
+    """
     _VIDEO_PAGE_TEMPLATE = 'http://store.steampowered.com/video/%s/'
     _AGECHECK_TEMPLATE = 'http://store.steampowered.com/agecheck/video/%s/?snr=1_agecheck_agecheck__age-gate&ageDay=1&ageMonth=January&ageYear=1970'
-    _TEST = {
-        u"url": u"http://store.steampowered.com/video/105600/",
-        u"playlist": [
+    _TESTS = [{
+        "url": "http://store.steampowered.com/video/105600/",
+        "playlist": [
             {
-                u"file": u"81300.flv",
-                u"md5": u"f870007cee7065d7c76b88f0a45ecc07",
-                u"info_dict": {
-                        u"title": u"Terraria 1.1 Trailer",
-                        u'playlist_index': 1,
+                "md5": "f870007cee7065d7c76b88f0a45ecc07",
+                "info_dict": {
+                    'id': '81300',
+                    'ext': 'flv',
+                    "title": "Terraria 1.1 Trailer",
+                    'playlist_index': 1,
                 }
             },
             {
-                u"file": u"80859.flv",
-                u"md5": u"61aaf31a5c5c3041afb58fb83cbb5751",
-                u"info_dict": {
-                    u"title": u"Terraria Trailer",
-                    u'playlist_index': 2,
+                "md5": "61aaf31a5c5c3041afb58fb83cbb5751",
+                "info_dict": {
+                    'id': '80859',
+                    'ext': 'flv',
+                    "title": "Terraria Trailer",
+                    'playlist_index': 2,
                 }
             }
-        ]
-    }
-
-
-    @classmethod
-    def suitable(cls, url):
-        """Receives a URL and returns True if suitable for this IE."""
-        return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
+        ],
+        'params': {
+            'playlistend': 2,
+        }
+    }, {
+        'url': 'http://steamcommunity.com/sharedfiles/filedetails/?id=242472205',
+        'info_dict': {
+            'id': 'WB5DvDOOvAY',
+            'ext': 'mp4',
+            'upload_date': '20140329',
+            'title': 'FRONTIERS - Final Greenlight Trailer',
+            'description': "The final trailer for the Steam Greenlight launch. Hooray, progress! Here's the official Greenlight page: http://steamcommunity.com/sharedfiles/filedetails/?id=242472205",
+            'uploader': 'AAD Productions',
+            'uploader_id': 'AtomicAgeDogGames',
+        }
+    }]
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url, re.VERBOSE)
-        gameID = m.group('gameID')
-
-        videourl = self._VIDEO_PAGE_TEMPLATE % gameID
-        webpage = self._download_webpage(videourl, gameID)
+        m = re.match(self._VALID_URL, url)
+        fileID = m.group('fileID')
+        if fileID:
+            videourl = url
+            playlist_id = fileID
+        else:
+            gameID = m.group('gameID')
+            playlist_id = gameID
+            videourl = self._VIDEO_PAGE_TEMPLATE % playlist_id
+        webpage = self._download_webpage(videourl, playlist_id)
 
         if re.search('<h2>Please enter your birth date to continue:</h2>', webpage) is not None:
-            videourl = self._AGECHECK_TEMPLATE % gameID
+            videourl = self._AGECHECK_TEMPLATE % playlist_id
             self.report_age_confirmation()
-            webpage = self._download_webpage(videourl, gameID)
+            webpage = self._download_webpage(videourl, playlist_id)
 
-        self.report_extraction(gameID)
-        game_title = self._html_search_regex(r'<h2 class="pageheader">(.*?)</h2>',
-                                             webpage, 'game title')
+        if fileID:
+            playlist_title = self._html_search_regex(
+                r'<div class="workshopItemTitle">(.+)</div>', webpage, 'title')
+            mweb = re.finditer(r'''(?x)
+                'movie_(?P<videoID>[0-9]+)':\s*\{\s*
+                YOUTUBE_VIDEO_ID:\s*"(?P<youtube_id>[^"]+)",
+                ''', webpage)
+            videos = [{
+                '_type': 'url',
+                'url': vid.group('youtube_id'),
+                'ie_key': 'Youtube',
+            } for vid in mweb]
+        else:
+            playlist_title = self._html_search_regex(
+                r'<h2 class="pageheader">(.*?)</h2>', webpage, 'game title')
 
-        urlRE = r"'movie_(?P<videoID>\d+)': \{\s*FILENAME: \"(?P<videoURL>[\w:/\.\?=]+)\"(,\s*MOVIE_NAME: \"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},"
-        mweb = re.finditer(urlRE, webpage)
-        namesRE = r'<span class="title">(?P<videoName>.+?)</span>'
-        titles = re.finditer(namesRE, webpage)
-        thumbsRE = r'<img class="movie_thumb" src="(?P<thumbnail>.+?)">'
-        thumbs = re.finditer(thumbsRE, webpage)
-        videos = []
-        for vid,vtitle,thumb in zip(mweb,titles,thumbs):
-            video_id = vid.group('videoID')
-            title = vtitle.group('videoName')
-            video_url = vid.group('videoURL')
-            video_thumb = thumb.group('thumbnail')
-            if not video_url:
-                raise ExtractorError(u'Cannot find video url for %s' % video_id)
-            info = {
-                'id':video_id,
-                'url':video_url,
-                'ext': 'flv',
-                'title': unescapeHTML(title),
-                'thumbnail': video_thumb
-                  }
-            videos.append(info)
-        return [self.playlist_result(videos, gameID, game_title)]
+            mweb = re.finditer(r'''(?x)
+                'movie_(?P<videoID>[0-9]+)':\s*\{\s*
+                FILENAME:\s*"(?P<videoURL>[\w:/\.\?=]+)"
+                (,\s*MOVIE_NAME:\s*\"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},
+                ''', webpage)
+            titles = re.finditer(
+                r'<span class="title">(?P<videoName>.+?)</span>', webpage)
+            thumbs = re.finditer(
+                r'<img class="movie_thumb" src="(?P<thumbnail>.+?)">', webpage)
+            videos = []
+
+            for vid, vtitle, thumb in zip(mweb, titles, thumbs):
+                video_id = vid.group('videoID')
+                title = vtitle.group('videoName')
+                video_url = vid.group('videoURL')
+                video_thumb = thumb.group('thumbnail')
+                if not video_url:
+                    raise ExtractorError('Cannot find video url for %s' % video_id)
+                videos.append({
+                    'id': video_id,
+                    'url': video_url,
+                    'ext': 'flv',
+                    'title': unescapeHTML(title),
+                    'thumbnail': video_thumb
+                })
+        if not videos:
+            raise ExtractorError('Could not find any videos')
+
+        return self.playlist_result(videos, playlist_id, playlist_title)

youtube_dl/extractor/teamcoco.py

@@ -3,9 +3,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
 
 
 class TeamcocoIE(InfoExtractor):

youtube_dl/extractor/ted.py

@@ -49,6 +49,19 @@ class TEDIE(SubtitlesInfoExtractor):
             'thumbnail': 're:^https?://.+\.jpg',
             'description': 'Adaptive, intelligent, and consistent, algorithms are emerging as the ultimate app for everything from matching consumers to products to assessing medical diagnoses. Vishal Sikka shares his appreciation for the algorithm, charting both its inherent beauty and its growing power.',
         }
+    }, {
+        'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
+        'info_dict': {
+            'id': '1972',
+            'ext': 'flv',
+            'title': 'Be passionate. Be courageous. Be your best.',
+            'uploader': 'Gabby Giffords and Mark Kelly',
+            'description': 'md5:d89e1d8ebafdac8e55df4c219ecdbfe9',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }]
 
     _NATIVE_FORMATS = {
@@ -102,11 +115,23 @@ class TEDIE(SubtitlesInfoExtractor):
             'url': format_url,
             'format_id': format_id,
             'format': format_id,
-        } for (format_id, format_url) in talk_info['nativeDownloads'].items()]
-        for f in formats:
-            finfo = self._NATIVE_FORMATS.get(f['format_id'])
-            if finfo:
-                f.update(finfo)
+        } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
+        if formats:
+            for f in formats:
+                finfo = self._NATIVE_FORMATS.get(f['format_id'])
+                if finfo:
+                    f.update(finfo)
+        else:
+            # Use rtmp downloads
+            formats = [{
+                'format_id': f['name'],
+                'url': talk_info['streamer'],
+                'play_path': f['file'],
+                'ext': 'flv',
+                'width': f['width'],
+                'height': f['height'],
+                'tbr': f['bitrate'],
+            } for f in talk_info['resources']['rtmp']]
         self._sort_formats(formats)
 
         video_id = compat_str(talk_info['id'])

youtube_dl/extractor/tlc.py

@@ -0,0 +1,60 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+import re
+
+from .common import InfoExtractor
+from .brightcove import BrightcoveIE
+from .discovery import DiscoveryIE
+
+
+class TlcIE(DiscoveryIE):
+    IE_NAME = 'tlc.com'
+    _VALID_URL = r'http://www\.tlc\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9\-]*)(.htm)?'
+
+    _TEST = {
+        'url': 'http://www.tlc.com/tv-shows/cake-boss/videos/too-big-to-fly.htm',
+        'md5': 'c4038f4a9b44d0b5d74caaa64ed2a01a',
+        'info_dict': {
+            'id': '853232',
+            'ext': 'mp4',
+            'title': 'Cake Boss: Too Big to Fly',
+            'description': 'Buddy has taken on a high flying task.',
+            'duration': 119,
+        },
+    }
+
+
+class TlcDeIE(InfoExtractor):
+    IE_NAME = 'tlc.de'
+    _VALID_URL = r'http://www\.tlc\.de/sendungen/[^/]+/videos/(?P<title>[^/?]+)'
+
+    _TEST = {
+        'url': 'http://www.tlc.de/sendungen/breaking-amish/videos/#3235167922001',
+        'info_dict': {
+            'id': '3235167922001',
+            'ext': 'mp4',
+            'title': 'Breaking Amish: Die Welt da draußen',
+            'uploader': 'Discovery Networks - Germany',
+            'description': 'Vier Amische und eine Mennonitin wagen in New York'
+                '  den Sprung in ein komplett anderes Leben. Begleitet sie auf'
+                ' ihrem spannenden Weg.',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        title = mobj.group('title')
+        webpage = self._download_webpage(url, title)
+        iframe_url = self._search_regex(
+            '<iframe src="(http://www\.tlc\.de/wp-content/.+?)"', webpage,
+            'iframe url')
+        # Otherwise we don't get the correct 'BrightcoveExperience' element,
+        # example: http://www.tlc.de/sendungen/cake-boss/videos/cake-boss-cannoli-drama/
+        iframe_url = iframe_url.replace('.htm?', '.php?')
+        iframe = self._download_webpage(iframe_url, title)
+
+        return {
+            '_type': 'url',
+            'url': BrightcoveIE._extract_brightcove_url(iframe),
+            'ie': BrightcoveIE.ie_key(),
+        }

youtube_dl/extractor/yahoo.py

@@ -104,7 +104,7 @@ class YahooNewsIE(YahooIE):
     IE_NAME = 'yahoo:news'
     _VALID_URL = r'http://news\.yahoo\.com/video/.*?-(?P<id>\d*?)\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
         'md5': '67010fdf3a08d290e060a4dd96baa07b',
         'info_dict': {
@@ -113,10 +113,7 @@ class YahooNewsIE(YahooIE):
             'title': 'China Moses Is Crazy About the Blues',
             'description': 'md5:9900ab8cd5808175c7b3fe55b979bed0',
         },
-    }
-
-    # Overwrite YahooIE properties we don't want
-    _TESTS = []
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

youtube_dl/extractor/youtube.py

@@ -1419,7 +1419,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                 self.to_screen(u'Downloading just video %s because of --no-playlist' % video_id)
                 return self.url_result(video_id, 'Youtube', video_id=video_id)
             else:
-                self.to_screen(u'Downloading playlist PL%s - add --no-playlist to just download video %s' % (playlist_id, video_id))
+                self.to_screen(u'Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
 
         if playlist_id.startswith('RD'):
             # Mixes require a custom extraction process

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2014.04.13'
+__version__ = '2014.04.21'