release 2013.12.20

[imdb] Add support for mobile site URLs

test/test_utils.py

@@ -188,6 +188,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(url_basename(u'http://foo.de/bar/baz?x=y'), u'baz')
         self.assertEqual(url_basename(u'http://foo.de/bar/baz#x=y'), u'baz')
         self.assertEqual(url_basename(u'http://foo.de/bar/baz/'), u'baz')
+        self.assertEqual(
+            url_basename(u'http://media.w3.org/2010/05/sintel/trailer.mp4'),
+            u'trailer.mp4')
 
 if __name__ == '__main__':
     unittest.main()

youtube_dl/extractor/__init__.py

@@ -1,6 +1,7 @@
 from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
 from .anitube import AnitubeIE
+from .aparat import AparatIE
 from .appletrailers import AppleTrailersIE
 from .archiveorg import ArchiveOrgIE
 from .ard import ARDIE

youtube_dl/extractor/academicearth.py

@@ -1,11 +1,6 @@
-import datetime
-import json
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    remove_start,
-)
 
 
 class AcademicEarthCourseIE(InfoExtractor):

youtube_dl/extractor/aparat.py

@@ -0,0 +1,56 @@
+#coding: utf-8
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    HEADRequest,
+)
+
+
+class AparatIE(InfoExtractor):
+    _VALID_URL = r'^https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P<id>[a-zA-Z0-9]+)'
+
+    _TEST = {
+        u'url': u'http://www.aparat.com/v/wP8On',
+        u'file': u'wP8On.mp4',
+        u'md5': u'6714e0af7e0d875c5a39c4dc4ab46ad1',
+        u'info_dict': {
+            u"title": u"تیم گلکسی 11 - زومیت",
+        },
+        #u'skip': u'Extremely unreliable',
+    }
+
+    def _real_extract(self, url):
+        m = re.match(self._VALID_URL, url)
+        video_id = m.group('id')
+
+        # Note: There is an easier-to-parse configuration at
+        # http://www.aparat.com/video/video/config/videohash/%video_id
+        # but the URL in there does not work
+        embed_url = (u'http://www.aparat.com/video/video/embed/videohash/' +
+                     video_id + u'/vt/frame')
+        webpage = self._download_webpage(embed_url, video_id)
+
+        video_urls = re.findall(r'fileList\[[0-9]+\]\s*=\s*"([^"]+)"', webpage)
+        for i, video_url in enumerate(video_urls):
+            req = HEADRequest(video_url)
+            res = self._request_webpage(
+                req, video_id, note=u'Testing video URL %d' % i, errnote=False)
+            if res:
+                break
+        else:
+            raise ExtractorError(u'No working video URLs found')
+
+        title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, u'title')
+        thumbnail = self._search_regex(
+            r'\s+image:\s*"([^"]+)"', webpage, u'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'ext': 'mp4',
+            'thumbnail': thumbnail,
+        }

youtube_dl/extractor/blinkx.py

@@ -54,6 +54,10 @@ class BlinkxIE(InfoExtractor):
                 })
             elif m['type'] == 'original':
                 duration = m['d']
+            elif m['type'] == 'youtube':
+                yt_id = m['link']
+                self.to_screen(u'Youtube video detected: %s' % yt_id)
+                return self.url_result(yt_id, 'Youtube', video_id=yt_id)
             elif m['type'] in ('flv', 'mp4'):
                 vcodec = remove_start(m['vcodec'], 'ff')
                 acodec = remove_start(m['acodec'], 'ff')

youtube_dl/extractor/common.py

@@ -170,6 +170,8 @@ class InfoExtractor(object):
         try:
             return self._downloader.urlopen(url_or_request)
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+            if errnote is False:
+                return False
             if errnote is None:
                 errnote = u'Unable to download webpage'
             errmsg = u'%s: %s' % (errnote, compat_str(err))
@@ -263,7 +265,8 @@ class InfoExtractor(object):
         self.to_screen(u'Logging in')
 
     #Methods for following #608
-    def url_result(self, url, ie=None, video_id=None):
+    @staticmethod
+    def url_result(url, ie=None, video_id=None):
         """Returns a url that points to a page that should be processed"""
         #TODO: ie should be the class used for getting the info
         video_info = {'_type': 'url',
@@ -272,7 +275,8 @@ class InfoExtractor(object):
         if video_id is not None:
             video_info['id'] = video_id
         return video_info
-    def playlist_result(self, entries, playlist_id=None, playlist_title=None):
+    @staticmethod
+    def playlist_result(entries, playlist_id=None, playlist_title=None):
         """Returns a playlist"""
         video_info = {'_type': 'playlist',
                       'entries': entries}

youtube_dl/extractor/generic.py

@@ -11,10 +11,14 @@ from ..utils import (
     compat_urlparse,
 
     ExtractorError,
+    HEADRequest,
     smuggle_url,
     unescapeHTML,
+    unified_strdate,
+    url_basename,
 )
 from .brightcove import BrightcoveIE
+from .ooyala import OoyalaIE
 
 
 class GenericIE(InfoExtractor):
@@ -71,6 +75,27 @@ class GenericIE(InfoExtractor):
                 u'skip_download': True,
             },
         },
+        # Direct link to a video
+        {
+            u'url': u'http://media.w3.org/2010/05/sintel/trailer.mp4',
+            u'file': u'trailer.mp4',
+            u'md5': u'67d406c2bcb6af27fa886f31aa934bbe',
+            u'info_dict': {
+                u'id': u'trailer',
+                u'title': u'trailer',
+                u'upload_date': u'20100513',
+            }
+        },
+        # ooyala video
+        {
+            u'url': u'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
+            u'md5': u'5644c6ca5d5782c1d0d350dad9bd840c',
+            u'info_dict': {
+                u'id': u'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
+                u'ext': u'mp4',
+                u'title': u'2cc213299525360.mov', #that's what we get
+            },
+        },
     ]
 
     def report_download_webpage(self, video_id):
@@ -83,23 +108,20 @@ class GenericIE(InfoExtractor):
         """Report information extraction."""
         self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
 
-    def _test_redirect(self, url):
+    def _send_head(self, url):
         """Check if it is a redirect, like url shorteners, in case return the new url."""
-        class HeadRequest(compat_urllib_request.Request):
-            def get_method(self):
-                return "HEAD"
 
         class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
             """
             Subclass the HTTPRedirectHandler to make it use our
-            HeadRequest also on the redirected URL
+            HEADRequest also on the redirected URL
             """
             def redirect_request(self, req, fp, code, msg, headers, newurl):
                 if code in (301, 302, 303, 307):
                     newurl = newurl.replace(' ', '%20')
                     newheaders = dict((k,v) for k,v in req.headers.items()
                                       if k.lower() not in ("content-length", "content-type"))
-                    return HeadRequest(newurl,
+                    return HEADRequest(newurl,
                                        headers=newheaders,
                                        origin_req_host=req.get_origin_req_host(),
                                        unverifiable=True)
@@ -128,32 +150,49 @@ class GenericIE(InfoExtractor):
                         compat_urllib_request.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]:
             opener.add_handler(handler())
 
-        response = opener.open(HeadRequest(url))
+        response = opener.open(HEADRequest(url))
         if response is None:
             raise ExtractorError(u'Invalid URL protocol')
-        new_url = response.geturl()
-
-        if url == new_url:
-            return False
-
-        self.report_following_redirect(new_url)
-        return new_url
+        return response
 
     def _real_extract(self, url):
         parsed_url = compat_urlparse.urlparse(url)
         if not parsed_url.scheme:
             self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
             return self.url_result('http://' + url)
+        video_id = os.path.splitext(url.split('/')[-1])[0]
 
         try:
-            new_url = self._test_redirect(url)
-            if new_url:
-                return [self.url_result(new_url)]
+            response = self._send_head(url)
+
+            # Check for redirect
+            new_url = response.geturl()
+            if url != new_url:
+                self.report_following_redirect(new_url)
+                return self.url_result(new_url)
+
+            # Check for direct link to a video
+            content_type = response.headers.get('Content-Type', '')
+            m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
+            if m:
+                upload_date = response.headers.get('Last-Modified')
+                if upload_date:
+                    upload_date = unified_strdate(upload_date)
+                return {
+                    'id': video_id,
+                    'title': os.path.splitext(url_basename(url))[0],
+                    'formats': [{
+                        'format_id': m.group('format_id'),
+                        'url': url,
+                        'vcodec': u'none' if m.group('type') == 'audio' else None
+                    }],
+                    'upload_date': upload_date,
+                }
+
         except compat_urllib_error.HTTPError:
             # This may be a stupid server that doesn't like HEAD, our UA, or so
             pass
 
-        video_id = url.split('/')[-1]
         try:
             webpage = self._download_webpage(url, video_id)
         except ValueError:
@@ -192,8 +231,11 @@ class GenericIE(InfoExtractor):
             return self.url_result(surl, 'Vimeo')
 
         # Look for embedded YouTube player
-        matches = re.findall(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?youtube\.com/embed/.+?)\1', webpage)
+        matches = re.findall(r'''(?x)
+            (?:<iframe[^>]+?src=|embedSWF\(\s*)
+            (["\'])(?P<url>(?:https?:)?//(?:www\.)?youtube\.com/
+                (?:embed|v)/.+?)
+            \1''', webpage)
         if matches:
             urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube')
                      for tuppl in matches]
@@ -247,6 +289,16 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
+        # Look for Ooyala videos
+        mobj = re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=([^"&]+)', webpage)
+        if mobj is not None:
+            return OoyalaIE._build_url_result(mobj.group(1))
+
+        # Look for Aparat videos
+        mobj = re.search(r'<iframe src="(http://www.aparat.com/video/[^"]+)"', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group(1), 'Aparat')
+
         # Start with something easy: JW Player in SWFObject
         mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
         if mobj is None:

youtube_dl/extractor/imdb.py

@@ -11,7 +11,7 @@ from ..utils import (
 class ImdbIE(InfoExtractor):
     IE_NAME = u'imdb'
     IE_DESC = u'Internet Movie Database trailers'
-    _VALID_URL = r'http://www\.imdb\.com/video/imdb/vi(?P<id>\d+)'
+    _VALID_URL = r'http://(?:www|m)\.imdb\.com/video/imdb/vi(?P<id>\d+)'
 
     _TEST = {
         u'url': u'http://www.imdb.com/video/imdb/vi2524815897',
@@ -27,7 +27,7 @@ class ImdbIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        webpage = self._download_webpage(url,video_id)
+        webpage = self._download_webpage('http://www.imdb.com/video/imdb/vi%s' % video_id, video_id)
         descr = get_element_by_attribute('itemprop', 'description', webpage)
         available_formats = re.findall(
             r'case \'(?P<f_id>.*?)\' :$\s+url = \'(?P<path>.*?)\'', webpage,

youtube_dl/extractor/ooyala.py

@@ -22,6 +22,11 @@ class OoyalaIE(InfoExtractor):
     def _url_for_embed_code(embed_code):
         return 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code
 
+    @classmethod
+    def _build_url_result(cls, embed_code):
+        return cls.url_result(cls._url_for_embed_code(embed_code),
+            ie=cls.ie_key())
+
     def _extract_result(self, info, more_info):
         return {'id': info['embedCode'],
                 'ext': 'mp4',

youtube_dl/extractor/radiofrance.py

@@ -1,12 +1,7 @@
 # coding: utf-8
-import datetime
-import json
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    remove_start,
-)
 
 
 class RadioFranceIE(InfoExtractor):
@@ -42,11 +37,11 @@ class RadioFranceIE(InfoExtractor):
             webpage, u'audio URLs')
         formats = [
             {
-                'format_id': m[0],
-                'url': m[1],
+                'format_id': fm[0],
+                'url': fm[1],
                 'vcodec': 'none',
             }
-            for m in
+            for fm in
             re.findall(r"([a-z0-9]+)\s*:\s*'([^']+)'", formats_str)
         ]
         # No sorting, we don't know any more about these formats

youtube_dl/extractor/smotri.py

@@ -202,7 +202,7 @@ class SmotriIE(InfoExtractor):
             'uploader': video_uploader,
             'upload_date': video_upload_date,
             'uploader_id': video_uploader_id,
-            'video_duration': video_duration,
+            'duration': video_duration,
             'view_count': video_view_count,
             'age_limit': 18 if adult_content else 0,
             'video_page_url': video_page_url

youtube_dl/extractor/soundcloud.py

@@ -24,7 +24,7 @@ class SoundcloudIE(InfoExtractor):
      """
 
     _VALID_URL = r'''^(?:https?://)?
-                    (?:(?:(?:www\.)?soundcloud\.com/
+                    (?:(?:(?:www\.|m\.)?soundcloud\.com/
                             (?P<uploader>[\w\d-]+)/
                             (?!sets/)(?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)

youtube_dl/utils.py

@@ -767,6 +767,10 @@ def unified_strdate(date_str):
             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
         except:
             pass
+    if upload_date is None:
+        timetuple = email.utils.parsedate_tz(date_str)
+        if timetuple:
+            upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
     return upload_date
 
 def determine_ext(url, default_ext=u'unknown_video'):
@@ -1087,7 +1091,10 @@ def remove_start(s, start):
 
 
 def url_basename(url):
-    m = re.match(r'(?:https?:|)//[^/]+/(?:[^/?#]+/)?([^/?#]+)/?(?:[?#]|$)', url)
-    if not m:
-        return u''
-    return m.group(1)
+    path = compat_urlparse.urlparse(url).path
+    return path.strip(u'/').split(u'/')[-1]
+
+
+class HEADRequest(compat_urllib_request.Request):
+    def get_method(self):
+        return "HEAD"

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2013.12.17.1'
+__version__ = '2013.12.20'