release 2013.04.30

[ci skip]

.travis.yml

@@ -8,6 +8,7 @@ notifications:
   email:
     - filippo.valsorda@gmail.com
     - phihag@phihag.de
+    - jaime.marquinez.ferrandiz+travis@gmail.com
 #  irc:
 #    channels:
 #      - "irc.freenode.org#youtube-dl"

README.md

@@ -18,17 +18,20 @@ which means you can modify it, redistribute it or use it however you like.
     --version                  print program version and exit
     -U, --update               update this program to latest version
     -i, --ignore-errors        continue on download errors
-    -r, --rate-limit LIMIT   download rate limit (e.g. 50k or 44.6m)
+    -r, --rate-limit LIMIT     maximum download rate (e.g. 50k or 44.6m)
     -R, --retries RETRIES      number of retries (default is 10)
-    --buffer-size SIZE       size of download buffer (e.g. 1024 or 16k) (default
-                             is 1024)
+    --buffer-size SIZE         size of download buffer (e.g. 1024 or 16k)
+                               (default is 1024)
     --no-resize-buffer         do not automatically adjust the buffer size. By
                                default, the buffer size is automatically resized
                                from an initial value of SIZE.
     --dump-user-agent          display the current browser identification
     --user-agent UA            specify a custom user agent
+    --referer REF              specify a custom referer, use if the video access
+                               is restricted to one domain
     --list-extractors          List all supported extractors and the URLs they
                                would handle
+    --proxy None               Use the specified HTTP/HTTPS proxy
 
 ## Video Selection:
     --playlist-start NUMBER    playlist video to start at (default is 1)
@@ -38,28 +41,36 @@ which means you can modify it, redistribute it or use it however you like.
     --reject-title REGEX       skip download for matching titles (regex or
                                caseless sub-string)
     --max-downloads NUMBER     Abort after downloading NUMBER files
-    --min-filesize SIZE      Do not download any videos smaller than SIZE (e.g.
-                             50k or 44.6m)
+    --min-filesize SIZE        Do not download any videos smaller than SIZE
+                               (e.g. 50k or 44.6m)
     --max-filesize SIZE        Do not download any videos larger than SIZE (e.g.
                                50k or 44.6m)
+    --date DATE                download only videos uploaded in this date
+    --datebefore DATE          download only videos uploaded before this date
+    --dateafter DATE           download only videos uploaded after this date
 
 ## Filesystem Options:
-    -t, --title              use title in file name
-    --id                     use video ID in file name
+    -t, --title                use title in file name (default)
+    --id                       use only video ID in file name
     -l, --literal              [deprecated] alias of --title
     -A, --auto-number          number downloaded files starting from 00000
-    -o, --output TEMPLATE    output filename template. Use %(title)s to get the
-                             title, %(uploader)s for the uploader name,
+    -o, --output TEMPLATE      output filename template. Use %(title)s to get
+                               the title, %(uploader)s for the uploader name,
                                %(uploader_id)s for the uploader nickname if
                                different, %(autonumber)s to get an automatically
                                incremented number, %(ext)s for the filename
                                extension, %(upload_date)s for the upload date
                                (YYYYMMDD), %(extractor)s for the provider
                                (youtube, metacafe, etc), %(id)s for the video id
-                             and %% for a literal percent. Use - to output to
-                             stdout. Can also be used to download to a different
-                             directory, for example with -o '/my/downloads/%(upl
-                             oader)s/%(title)s-%(id)s.%(ext)s' .
+                               , %(playlist)s for the playlist the video is in,
+                               %(playlist_index)s for the position in the
+                               playlist and %% for a literal percent. Use - to
+                               output to stdout. Can also be used to download to
+                               a different directory, for example with -o '/my/d
+                               ownloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
+    --autonumber-size NUMBER   Specifies the number of digits in %(autonumber)s
+                               when it is present in output filename template or
+                               --autonumber option is given
     --restrict-filenames       Restrict filenames to only ASCII characters, and
                                avoid "&" and spaces in filenames
     -a, --batch-file FILE      file containing URLs to download ('-' for stdin)
@@ -69,15 +80,15 @@ which means you can modify it, redistribute it or use it however you like.
                                from beginning)
     --cookies FILE             file to read cookies from and dump cookie jar in
     --no-part                  do not use .part files
-    --no-mtime               do not use the Last-modified header to set the file
-                             modification time
+    --no-mtime                 do not use the Last-modified header to set the
+                               file modification time
     --write-description        write video description to a .description file
     --write-info-json          write video metadata to a .info.json file
 
 ## Verbosity / Simulation Options:
     -q, --quiet                activates quiet mode
-    -s, --simulate           do not download the video and do not write anything
-                             to disk
+    -s, --simulate             do not download the video and do not write
+                               anything to disk
     --skip-download            do not download the video
     -g, --get-url              simulate, quiet but print URL
     -e, --get-title            simulate, quiet but print title
@@ -89,18 +100,28 @@ which means you can modify it, redistribute it or use it however you like.
     --no-progress              do not print progress bar
     --console-title            display progress in console titlebar
     -v, --verbose              print various debugging information
+    --dump-intermediate-pages  print downloaded pages to debug problems(very
+                               verbose)
 
 ## Video Format Options:
-    -f, --format FORMAT      video format code
+    -f, --format FORMAT        video format code, specifiy the order of
+                               preference using slashes: "-f 22/17/18"
     --all-formats              download all available video formats
-    --prefer-free-formats    prefer free video formats unless a specific one is
-                             requested
+    --prefer-free-formats      prefer free video formats unless a specific one
+                               is requested
     --max-quality FORMAT       highest quality format to download
-    -F, --list-formats       list all available formats (currently youtube only)
-    --write-srt              write video closed captions to a .srt file
+    -F, --list-formats         list all available formats (currently youtube
+                               only)
+    --write-sub                write subtitle file (currently youtube only)
+    --only-sub                 downloads only the subtitles (no video)
+    --all-subs                 downloads all the available subtitles of the
+                               video (currently youtube only)
+    --list-subs                lists all available subtitles for the video
                                (currently youtube only)
-    --srt-lang LANG          language of the closed captions to download
-                             (optional) use IETF language tags like 'en'
+    --sub-format LANG          subtitle format [srt/sbv] (default=srt)
+                               (currently youtube only)
+    --sub-lang LANG            language of the subtitles to download (optional)
+                               use IETF language tags like 'en'
 
 ## Authentication Options:
     -u, --username USERNAME    account username
@@ -112,9 +133,9 @@ which means you can modify it, redistribute it or use it however you like.
                                ffmpeg or avconv and ffprobe or avprobe)
     --audio-format FORMAT      "best", "aac", "vorbis", "mp3", "m4a", "opus", or
                                "wav"; best by default
-    --audio-quality QUALITY  ffmpeg/avconv audio quality specification, insert a
-                             value between 0 (better) and 9 (worse) for VBR or a
-                             specific bitrate like 128K (default 5)
+    --audio-quality QUALITY    ffmpeg/avconv audio quality specification, insert
+                               a value between 0 (better) and 9 (worse) for VBR
+                               or a specific bitrate like 128K (default 5)
     --recode-video FORMAT      Encode the video to another format if necessary
                                (currently supported: mp4|flv|ogg|webm)
     -k, --keep-video           keeps the video file on disk after the post-
@@ -138,6 +159,8 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `ext`: The sequence will be replaced by the appropriate extension (like flv or mp4).
  - `epoch`: The sequence will be replaced by the Unix epoch when creating the file.
  - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
+ - `playlist`: The name or the id of the playlist that contains the video.
+ - `playlist_index`: The index of the video in the playlist, a five-digit number.
 
 The current default template is `%(id)s.%(ext)s`, but that will be switchted to `%(title)s-%(id)s.%(ext)s` (which can be requested with `-t` at the moment).
 
@@ -148,6 +171,19 @@ In some cases, you don't want special characters such as 中, spaces, or &, such
     $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
     youtube-dl_test_video_.mp4          # A simple file name
 
+# VIDEO SELECTION
+
+Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`, they accept dates in two formats:
+
+ - Absolute dates: Dates in the format `YYYYMMDD`.
+ - Relative dates: Dates in the format `(now|today)[+-][0-9](day|week|month|year)(s)?`
+ 
+Examples:
+
+	$ youtube-dl --dateafter now-6months #will only download the videos uploaded in the last 6 months
+	$ youtube-dl --date 19700101 #will only download the videos uploaded in January 1, 1970
+	$ youtube-dl --dateafter 20000101 --datebefore 20100101 #will only download the videos uploaded between 2000 and 2010
+
 # FAQ
 
 ### Can you please put the -b option back?

devscripts/gh-pages/update-feed.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import datetime
+
+import textwrap
+
+import json
+
+atom_template=textwrap.dedent("""\
+								<?xml version='1.0' encoding='utf-8'?>
+								<atom:feed xmlns:atom="http://www.w3.org/2005/Atom">
+									<atom:title>youtube-dl releases</atom:title>
+									<atom:id>youtube-dl-updates-feed</atom:id>
+									<atom:updated>@TIMESTAMP@</atom:updated>
+									@ENTRIES@
+								</atom:feed>""")
+
+entry_template=textwrap.dedent("""
+								<atom:entry>
+									<atom:id>youtube-dl-@VERSION@</atom:id>
+									<atom:title>New version @VERSION@</atom:title>
+									<atom:link href="http://rg3.github.io/youtube-dl" />
+									<atom:content type="xhtml">
+										<div xmlns="http://www.w3.org/1999/xhtml">
+											Downloads available at <a href="http://youtube-dl.org/downloads/@VERSION@/">http://youtube-dl.org/downloads/@VERSION@/</a>
+										</div>
+									</atom:content>
+									<atom:author>
+										<atom:name>The youtube-dl maintainers</atom:name>
+									</atom:author>
+									<atom:updated>@TIMESTAMP@</atom:updated>
+								</atom:entry>
+								""")
+
+now = datetime.datetime.now()
+now_iso = now.isoformat()
+
+atom_template = atom_template.replace('@TIMESTAMP@',now_iso)
+
+entries=[]
+
+versions_info = json.load(open('update/versions.json'))
+versions = list(versions_info['versions'].keys())
+versions.sort()
+
+for v in versions:
+	entry = entry_template.replace('@TIMESTAMP@',v.replace('.','-'))
+	entry = entry.replace('@VERSION@',v)
+	entries.append(entry)
+
+entries_str = textwrap.indent(''.join(entries), '\t')
+atom_template = atom_template.replace('@ENTRIES@', entries_str)
+
+with open('update/releases.atom','w',encoding='utf-8') as atom_file:
+	atom_file.write(atom_template)
+
+

devscripts/release.sh

@@ -22,7 +22,7 @@ if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit
 
 /bin/echo -e "\n### First of all, testing..."
 make cleanall
-nosetests --with-coverage --cover-package=youtube_dl --cover-html test || exit 1
+nosetests --with-coverage --cover-package=youtube_dl --cover-html test --stop || exit 1
 
 /bin/echo -e "\n### Changing version in version.py..."
 sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
@@ -69,6 +69,7 @@ ROOT=$(pwd)
     ORIGIN_URL=$(git config --get remote.origin.url)
     cd build/gh-pages
     "$ROOT/devscripts/gh-pages/add-version.py" $version
+    "$ROOT/devscripts/gh-pages/update-feed.py"
     "$ROOT/devscripts/gh-pages/sign-versions.py" < "$ROOT/updates_key.pem"
     "$ROOT/devscripts/gh-pages/generate-download.py"
     "$ROOT/devscripts/gh-pages/update-copyright.py"

devscripts/transition_helper_exe/youtube-dl.py

@@ -40,7 +40,7 @@ raw_input()
 
 filename = sys.argv[0]
 
-UPDATE_URL = "http://rg3.github.com/youtube-dl/update/"
+UPDATE_URL = "http://rg3.github.io/youtube-dl/update/"
 VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
 JSON_URL = UPDATE_URL + 'versions.json'
 UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)

test/parameters.json

@@ -29,6 +29,7 @@
     "simulate": false, 
     "skip_download": false, 
     "subtitleslang": null, 
+    "subtitlesformat": "srt",
     "test": true, 
     "updatetime": true, 
     "usenetrc": false, 
@@ -36,5 +37,8 @@
     "verbose": true, 
     "writedescription": false, 
     "writeinfojson": true, 
-    "writesubtitles": false
+    "writesubtitles": false,
+    "onlysubtitles": false,
+    "allsubtitles": false,
+    "listssubtitles": false
 }

test/test_all_urls.py

@@ -7,16 +7,27 @@ import unittest
 import os
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE
+from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE, YoutubeChannelIE
 
 class TestAllURLsMatching(unittest.TestCase):
     def test_youtube_playlist_matching(self):
-        self.assertTrue(YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8'))
-        self.assertTrue(YoutubePlaylistIE().suitable(u'PL63F0C78739B09958'))
-        self.assertFalse(YoutubePlaylistIE().suitable(u'PLtS2H6bU1M'))
+        self.assertTrue(YoutubePlaylistIE.suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8'))
+        self.assertTrue(YoutubePlaylistIE.suitable(u'UUBABnxM4Ar9ten8Mdjj1j0Q')) #585
+        self.assertTrue(YoutubePlaylistIE.suitable(u'PL63F0C78739B09958'))
+        self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q'))
+        self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8'))
+        self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC'))
+        self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')) #668
+        self.assertFalse(YoutubePlaylistIE.suitable(u'PLtS2H6bU1M'))
 
     def test_youtube_matching(self):
-        self.assertTrue(YoutubeIE().suitable(u'PLtS2H6bU1M'))
+        self.assertTrue(YoutubeIE.suitable(u'PLtS2H6bU1M'))
+        self.assertFalse(YoutubeIE.suitable(u'https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')) #668
+
+    def test_youtube_channel_matching(self):
+        self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM'))
+        self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM?feature=gb_ch_rec'))
+        self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM/videos'))
 
     def test_youtube_extract(self):
         self.assertEqual(YoutubeIE()._extract_id('http://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')

test/test_download.py

@@ -20,6 +20,8 @@ from youtube_dl.utils import *
 DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json')
 PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
 
+RETRIES = 3
+
 # General configuration (from __init__, not very elegant...)
 jar = compat_cookiejar.CookieJar()
 cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
@@ -56,6 +58,7 @@ with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
 
 
 class TestDownload(unittest.TestCase):
+    maxDiff = None
     def setUp(self):
         self.parameters = parameters
         self.defs = defs
@@ -64,7 +67,7 @@ class TestDownload(unittest.TestCase):
 def generator(test_case):
 
     def test_template(self):
-        ie = getattr(youtube_dl.InfoExtractors, test_case['name'] + 'IE')
+        ie = youtube_dl.InfoExtractors.get_info_extractor(test_case['name'])
         if not ie._WORKING:
             print('Skipping: IE marked as not _WORKING')
             return
@@ -79,9 +82,8 @@ def generator(test_case):
         params.update(test_case.get('params', {}))
 
         fd = FileDownloader(params)
-        fd.add_info_extractor(ie())
-        for ien in test_case.get('add_ie', []):
-            fd.add_info_extractor(getattr(youtube_dl.InfoExtractors, ien + 'IE')())
+        for ie in youtube_dl.InfoExtractors.gen_extractors():
+            fd.add_info_extractor(ie)
         finished_hook_called = set()
         def _hook(status):
             if status['status'] == 'finished':
@@ -93,8 +95,20 @@ def generator(test_case):
             _try_rm(tc['file'])
             _try_rm(tc['file'] + '.part')
             _try_rm(tc['file'] + '.info.json')
+        try:
+            for retry in range(1, RETRIES + 1):
                 try:
                     fd.download([test_case['url']])
+                except (DownloadError, ExtractorError) as err:
+                    if retry == RETRIES: raise
+
+                    # Check if the exception is not a network related one
+                    if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
+                        raise
+
+                    print('Retrying: {0} failed tries\n\n##########\n\n'.format(retry))
+                else:
+                    break
 
             for tc in test_cases:
                 if not test_case.get('params', {}).get('skip_download', False):

test/test_utils.py

@@ -14,6 +14,8 @@ from youtube_dl.utils import timeconvert
 from youtube_dl.utils import sanitize_filename
 from youtube_dl.utils import unescapeHTML
 from youtube_dl.utils import orderedSet
+from youtube_dl.utils import DateRange
+from youtube_dl.utils import unified_strdate
 
 if sys.version_info < (3, 0):
     _compat_str = lambda b: b.decode('unicode-escape')
@@ -96,5 +98,19 @@ class TestUtil(unittest.TestCase):
     def test_unescape_html(self):
         self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;'))
         
+    def test_daterange(self):
+        _20century = DateRange("19000101","20000101")
+        self.assertFalse("17890714" in _20century)
+        _ac = DateRange("00010101")
+        self.assertTrue("19690721" in _ac)
+        _firstmilenium = DateRange(end="10000101")
+        self.assertTrue("07110427" in _firstmilenium)
+
+    def test_unified_dates(self):
+        self.assertEqual(unified_strdate('December 21, 2010'), '20101221')
+        self.assertEqual(unified_strdate('8/7/2009'), '20090708')
+        self.assertEqual(unified_strdate('Dec 14, 2012'), '20121214')
+        self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011')
+
 if __name__ == '__main__':
     unittest.main()

test/test_youtube_lists.py

@@ -8,8 +8,9 @@ import json
 import os
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.InfoExtractors import YoutubeUserIE,YoutubePlaylistIE
+from youtube_dl.InfoExtractors import YoutubeUserIE, YoutubePlaylistIE, YoutubeIE, YoutubeChannelIE
 from youtube_dl.utils import *
+from youtube_dl.FileDownloader import FileDownloader
 
 PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
 with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
@@ -22,7 +23,7 @@ proxy_handler = compat_urllib_request.ProxyHandler()
 opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
 compat_urllib_request.install_opener(opener)
 
-class FakeDownloader(object):
+class FakeDownloader(FileDownloader):
     def __init__(self):
         self.result = []
         self.params = parameters
@@ -30,44 +31,79 @@ class FakeDownloader(object):
         print(s)
     def trouble(self, s):
         raise Exception(s)
-    def download(self, x):
-        self.result.append(x)
+    def extract_info(self, url):
+        self.result.append(url)
+        return url
 
 class TestYoutubeLists(unittest.TestCase):
+    def assertIsPlaylist(self,info):
+        """Make sure the info has '_type' set to 'playlist'"""
+        self.assertEqual(info['_type'], 'playlist')
+
     def test_youtube_playlist(self):
-        DL = FakeDownloader()
-        IE = YoutubePlaylistIE(DL)
-        IE.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
-        self.assertEqual(DL.result, [
-            ['http://www.youtube.com/watch?v=bV9L5Ht9LgY'],
-            ['http://www.youtube.com/watch?v=FXxLjLQi3Fg'],
-            ['http://www.youtube.com/watch?v=tU3Bgo5qJZE']
-        ])
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')[0]
+        self.assertIsPlaylist(result)
+        self.assertEqual(result['title'], 'ytdl test PL')
+        ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']]
+        self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE'])
+
+    def test_issue_673(self):
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('PLBB231211A4F62143')[0]
+        self.assertEqual(result['title'], 'Team Fortress 2')
+        self.assertTrue(len(result['entries']) > 40)
 
     def test_youtube_playlist_long(self):
-        DL = FakeDownloader()
-        IE = YoutubePlaylistIE(DL)
-        IE.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
-        self.assertTrue(len(DL.result) >= 799)
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')[0]
+        self.assertIsPlaylist(result)
+        self.assertTrue(len(result['entries']) >= 799)
+
+    def test_youtube_playlist_with_deleted(self):
+        #651
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')[0]
+        ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']]
+        self.assertFalse('pElCt5oNDuI' in ytie_results)
+        self.assertFalse('KdPEApIVdWM' in ytie_results)
+        
+    def test_youtube_playlist_empty(self):
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx')[0]
+        self.assertIsPlaylist(result)
+        self.assertEqual(len(result['entries']), 0)
 
     def test_youtube_course(self):
-        DL = FakeDownloader()
-        IE = YoutubePlaylistIE(DL)
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
         # TODO find a > 100 (paginating?) videos course
-        IE.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
-        self.assertEqual(DL.result[0], ['http://www.youtube.com/watch?v=j9WZyLZCBzs'])
-        self.assertEqual(len(DL.result), 25)
-        self.assertEqual(DL.result[-1], ['http://www.youtube.com/watch?v=rYefUsYuEp0'])
+        result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')[0]
+        entries = result['entries']
+        self.assertEqual(YoutubeIE()._extract_id(entries[0]['url']), 'j9WZyLZCBzs')
+        self.assertEqual(len(entries), 25)
+        self.assertEqual(YoutubeIE()._extract_id(entries[-1]['url']), 'rYefUsYuEp0')
 
     def test_youtube_channel(self):
-        # I give up, please find a channel that does paginate and test this like test_youtube_playlist_long
-        pass # TODO
+        dl = FakeDownloader()
+        ie = YoutubeChannelIE(dl)
+        #test paginated channel
+        result = ie.extract('https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w')[0]
+        self.assertTrue(len(result['entries']) > 90)
+        #test autogenerated channel
+        result = ie.extract('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')[0]
+        self.assertTrue(len(result['entries']) >= 18)
 
     def test_youtube_user(self):
-        DL = FakeDownloader()
-        IE = YoutubeUserIE(DL)
-        IE.extract('https://www.youtube.com/user/TheLinuxFoundation')
-        self.assertTrue(len(DL.result) >= 320)
+        dl = FakeDownloader()
+        ie = YoutubeUserIE(dl)
+        result = ie.extract('https://www.youtube.com/user/TheLinuxFoundation')[0]
+        self.assertTrue(len(result['entries']) >= 320)
 
 if __name__ == '__main__':
     unittest.main()

test/test_youtube_subtitles.py

@@ -38,20 +38,63 @@ class FakeDownloader(object):
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 class TestYoutubeSubtitles(unittest.TestCase):
+    def setUp(self):
+        DL = FakeDownloader()
+        DL.params['allsubtitles'] = False
+        DL.params['writesubtitles'] = False
+        DL.params['subtitlesformat'] = 'srt'
+        DL.params['listsubtitles'] = False
+    def test_youtube_no_subtitles(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = False
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        subtitles = info_dict[0]['subtitles']
+        self.assertEqual(subtitles, None)
     def test_youtube_subtitles(self):
         DL = FakeDownloader()
         DL.params['writesubtitles'] = True
         IE = YoutubeIE(DL)
         info_dict = IE.extract('QRS8MkLhQmM')
-        self.assertEqual(md5(info_dict[0]['subtitles']), 'c3228550d59116f3c29fba370b55d033')
-
+        sub = info_dict[0]['subtitles'][0]
+        self.assertEqual(md5(sub[2]), '4cd9278a35ba2305f47354ee13472260')
     def test_youtube_subtitles_it(self):
         DL = FakeDownloader()
         DL.params['writesubtitles'] = True
         DL.params['subtitleslang'] = 'it'
         IE = YoutubeIE(DL)
         info_dict = IE.extract('QRS8MkLhQmM')
-        self.assertEqual(md5(info_dict[0]['subtitles']), '132a88a0daf8e1520f393eb58f1f646a')
+        sub = info_dict[0]['subtitles'][0]
+        self.assertEqual(md5(sub[2]), '164a51f16f260476a05b50fe4c2f161d')
+    def test_youtube_onlysubtitles(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        DL.params['onlysubtitles'] = True
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        sub = info_dict[0]['subtitles'][0]
+        self.assertEqual(md5(sub[2]), '4cd9278a35ba2305f47354ee13472260')
+    def test_youtube_allsubtitles(self):
+        DL = FakeDownloader()
+        DL.params['allsubtitles'] = True
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        subtitles = info_dict[0]['subtitles']
+        self.assertEqual(len(subtitles), 13)
+    def test_youtube_subtitles_format(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        DL.params['subtitlesformat'] = 'sbv'
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        sub = info_dict[0]['subtitles'][0]
+        self.assertEqual(md5(sub[2]), '13aeaa0c245a8bed9a451cb643e3ad8b')
+    def test_youtube_list_subtitles(self):
+        DL = FakeDownloader()
+        DL.params['listsubtitles'] = True
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        self.assertEqual(info_dict, None)
 
 if __name__ == '__main__':
     unittest.main()

test/tests.json

@@ -76,8 +76,7 @@
     "name": "StanfordOpenClassroom",
     "md5":  "544a9468546059d4e80d76265b0443b8",
     "url":  "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100",
-    "file":  "PracticalUnix_intro-environment.mp4",
-    "skip": "Currently offline"
+    "file":  "PracticalUnix_intro-environment.mp4"
   },
   {
     "name": "XNXX",
@@ -128,18 +127,6 @@
     "file": "0732f586d7.mp4",
     "md5": "f647e9e90064b53b6e046e75d0241fbd"
   },
-  {
-    "name": "TweetReel",
-    "url": "http://tweetreel.com/?77smq",
-    "file": "77smq.mov",
-    "md5": "56b4d9ca9de467920f3f99a6d91255d6",
-    "info_dict": {
-        "uploader": "itszero",
-        "uploader_id": "itszero",
-        "upload_date": "20091225",
-        "description": "Installing Gentoo Linux on Powerbook G4, it turns out the sleep indicator becomes HDD activity indicator :D"
-    }
-  },
   {
     "name": "Steam",
     "url": "http://store.steampowered.com/video/105600/",
@@ -293,7 +280,8 @@
     "file": "102.mp4",
     "md5": "7bc087e71d16f18f9b8ab9fa62a8a031",
     "info_dict": {
-        "title": "Dan Dennett: The illusion of consciousness"
+        "title": "Dan Dennett: The illusion of consciousness",
+        "thumbnail": "http://images.ted.com/images/ted/488_389x292.jpg"
     }
   },
   {
@@ -304,5 +292,60 @@
     "info_dict": {
         "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2"
     }
+  },
+  {
+    "name": "Generic",
+    "url": "http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html",
+    "file": "13601338388002.mp4",
+    "md5": "85b90ccc9d73b4acd9138d3af4c27f89"
+  },
+  {
+    "name": "Spiegel",
+    "url": "http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html",
+    "file": "1259285.mp4",
+    "md5": "2c2754212136f35fb4b19767d242f66e",
+    "info_dict": {
+        "title": "Vulkanausbruch in Ecuador: Der \"Feuerschlund\" ist wieder aktiv"
     }
+  },
+  {
+    "name": "LiveLeak",
+    "md5":  "0813c2430bea7a46bf13acf3406992f4",
+    "url":  "http://www.liveleak.com/view?i=757_1364311680",
+    "file":  "757_1364311680.mp4",
+    "info_dict": {
+        "title": "Most unlucky car accident",
+        "description": "extremely bad day for this guy..!",
+        "uploader": "ljfriel2"
+    }
+  },
+  {
+    "name": "WorldStarHipHop",
+    "url": "http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO",
+    "file": "wshh6a7q1ny0G34ZwuIO.mp4",
+    "md5": "9d04de741161603bf7071bbf4e883186",
+    "info_dict": {
+        "title": "Video: KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick! "
+    }
+  },
+  {
+    "name": "ARD",
+    "url": "http://www.ardmediathek.de/das-erste/tagesschau-in-100-sek?documentId=14077640",
+    "file": "14077640.mp4",
+    "md5": "6ca8824255460c787376353f9e20bbd8",
+    "info_dict": {
+        "title": "11.04.2013 09:23 Uhr - Tagesschau in 100 Sekunden"
+    },
+    "skip": "Requires rtmpdump"
+  },
+  {
+    "name": "Tumblr",
+    "url": "http://birthdayproject2012.tumblr.com/post/17258355236/a-sample-video-from-leeann-if-you-need-an-idea",
+    "file": "17258355236.mp4",
+    "md5": "7c6a514d691b034ccf8567999e9e88a3",
+    "info_dict": {
+        "title": "A sample video from LeeAnn. (If you need an idea..."
+    }
+  }
+
 ]

youtube_dl/FileDownloader.py

@@ -17,6 +17,7 @@ if os.name == 'nt':
     import ctypes
 
 from .utils import *
+from .InfoExtractors import get_info_extractor
 
 
 class FileDownloader(object):
@@ -78,12 +79,17 @@ class FileDownloader(object):
     updatetime:        Use the Last-modified header to set output file timestamps.
     writedescription:  Write the video description to a .description file
     writeinfojson:     Write the video description to a .info.json file
-    writesubtitles:    Write the video subtitles to a .srt file
+    writesubtitles:    Write the video subtitles to a file
+    onlysubtitles:     Downloads only the subtitles of the video
+    allsubtitles:      Downloads all the subtitles of the video
+    listsubtitles:     Lists all available subtitles for the video
+    subtitlesformat:   Subtitle format [sbv/srt] (default=srt)
     subtitleslang:     Language of the subtitles to download
     test:              Download only first bytes to test the downloader.
     keepvideo:         Keep the video file after post-processing
     min_filesize:      Skip files smaller than this size
     max_filesize:      Skip files larger than this size
+    daterange:         A DateRange object, download only if the upload_date is in the range.
     """
 
     params = None
@@ -104,7 +110,7 @@ class FileDownloader(object):
         self.params = params
 
         if '%(stitle)s' in self.params['outtmpl']:
-            self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
+            self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
 
     @staticmethod
     def format_bytes(bytes):
@@ -116,7 +122,7 @@ class FileDownloader(object):
             exponent = 0
         else:
             exponent = int(math.log(bytes, 1024.0))
-        suffix = 'bkMGTPEZY'[exponent]
+        suffix = ['B','KiB','MiB','GiB','TiB','PiB','EiB','ZiB','YiB'][exponent]
         converted = float(bytes) / float(1024 ** exponent)
         return '%.2f%s' % (converted, suffix)
 
@@ -227,13 +233,47 @@ class FileDownloader(object):
             self.to_stderr(message)
         if self.params.get('verbose'):
             if tb is None:
+                if sys.exc_info()[0]:  # if .trouble has been called from an except block
+                    tb = u''
+                    if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+                        tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
+                    tb += compat_str(traceback.format_exc())
+                else:
                     tb_data = traceback.format_list(traceback.extract_stack())
                     tb = u''.join(tb_data)
             self.to_stderr(tb)
         if not self.params.get('ignoreerrors', False):
-            raise DownloadError(message)
+            if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+                exc_info = sys.exc_info()[1].exc_info
+            else:
+                exc_info = sys.exc_info()
+            raise DownloadError(message, exc_info)
         self._download_retcode = 1
 
+    def report_warning(self, message):
+        '''
+        Print the message to stderr, it will be prefixed with 'WARNING:'
+        If stderr is a tty file the 'WARNING:' will be colored
+        '''
+        if sys.stderr.isatty() and os.name != 'nt':
+            _msg_header=u'\033[0;33mWARNING:\033[0m'
+        else:
+            _msg_header=u'WARNING:'
+        warning_message=u'%s %s' % (_msg_header,message)
+        self.to_stderr(warning_message)
+
+    def report_error(self, message, tb=None):
+        '''
+        Do the same as trouble, but prefixes the message with 'ERROR:', colored
+        in red if stderr is a tty file.
+        '''
+        if sys.stderr.isatty() and os.name != 'nt':
+            _msg_header = u'\033[0;31mERROR:\033[0m'
+        else:
+            _msg_header = u'ERROR:'
+        error_message = u'%s %s' % (_msg_header, message)
+        self.trouble(error_message, tb)
+
     def slow_down(self, start_time, byte_counter):
         """Sleep if the download speed is over the rate limit."""
         rate_limit = self.params.get('ratelimit', None)
@@ -265,7 +305,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
-            self.trouble(u'ERROR: unable to rename file')
+            self.report_error(u'unable to rename file')
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
@@ -289,9 +329,9 @@ class FileDownloader(object):
         """ Report that the description file is being written """
         self.to_screen(u'[info] Writing video description to: ' + descfn)
 
-    def report_writesubtitles(self, srtfn):
+    def report_writesubtitles(self, sub_filename):
         """ Report that the subtitles file is being written """
-        self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
+        self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename)
 
     def report_writeinfojson(self, infofn):
         """ Report that the metadata file has been written """
@@ -350,7 +390,13 @@ class FileDownloader(object):
             template_dict = dict(info_dict)
 
             template_dict['epoch'] = int(time.time())
-            template_dict['autonumber'] = u'%05d' % self._num_downloads
+            autonumber_size = self.params.get('autonumber_size')
+            if autonumber_size is None:
+                autonumber_size = 5
+            autonumber_templ = u'%0' + str(autonumber_size) + u'd'
+            template_dict['autonumber'] = autonumber_templ % self._num_downloads
+            if template_dict['playlist_index'] is not None:
+                template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']
 
             sanitize = lambda k,v: sanitize_filename(
                 u'NA' if v is None else compat_str(v),
@@ -360,8 +406,11 @@ class FileDownloader(object):
 
             filename = self.params['outtmpl'] % template_dict
             return filename
-        except (ValueError, KeyError) as err:
-            self.trouble(u'ERROR: invalid system charset or erroneous output template')
+        except KeyError as err:
+            self.report_error(u'Erroneous output template')
+            return None
+        except ValueError as err:
+            self.report_error(u'Insufficient system charset ' + repr(preferredencoding()))
             return None
 
     def _match_entry(self, info_dict):
@@ -370,19 +419,138 @@ class FileDownloader(object):
         title = info_dict['title']
         matchtitle = self.params.get('matchtitle', False)
         if matchtitle:
-            matchtitle = matchtitle.decode('utf8')
             if not re.search(matchtitle, title, re.IGNORECASE):
                 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
         rejecttitle = self.params.get('rejecttitle', False)
         if rejecttitle:
-            rejecttitle = rejecttitle.decode('utf8')
             if re.search(rejecttitle, title, re.IGNORECASE):
                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
+        date = info_dict.get('upload_date', None)
+        if date is not None:
+            dateRange = self.params.get('daterange', DateRange())
+            if date not in dateRange:
+                return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
         return None
         
+    def extract_info(self, url, download = True, ie_name = None):
+        '''
+        Returns a list with a dictionary for each video we find.
+        If 'download', also downloads the videos.
+         '''
+        suitable_found = False
+        
+        #We copy the original list
+        ies = list(self._ies)
+
+        if ie_name is not None:
+            #We put in the first place the given info extractor
+            first_ie = get_info_extractor(ie_name)()
+            first_ie.set_downloader(self)
+            ies.insert(0, first_ie)
+
+        for ie in ies:
+            # Go to next InfoExtractor if not suitable
+            if not ie.suitable(url):
+                continue
+
+            # Warn if the _WORKING attribute is False
+            if not ie.working():
+                self.report_warning(u'the program functionality for this site has been marked as broken, '
+                               u'and will probably not work. If you want to go on, use the -i option.')
+
+            # Suitable InfoExtractor found
+            suitable_found = True
+
+            # Extract information from URL and process it
+            try:
+                ie_results = ie.extract(url)
+                if ie_results is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
+                    break
+                results = []
+                for ie_result in ie_results:
+                    if not 'extractor' in ie_result:
+                        #The extractor has already been set somewhere else
+                        ie_result['extractor'] = ie.IE_NAME
+                    results.append(self.process_ie_result(ie_result, download))
+                return results
+            except ExtractorError as de: # An error we somewhat expected
+                self.report_error(compat_str(de), de.format_traceback())
+                break
+            except Exception as e:
+                if self.params.get('ignoreerrors', False):
+                    self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
+                    break
+                else:
+                    raise
+        if not suitable_found:
+                self.report_error(u'no suitable InfoExtractor: %s' % url)
+        
+    def process_ie_result(self, ie_result, download = True):
+        """
+        Take the result of the ie and return a list of videos.
+        For url elements it will search the suitable ie and get the videos
+        For playlist elements it will process each of the elements of the 'entries' key
+        
+        It will also download the videos if 'download'.
+        """
+        result_type = ie_result.get('_type', 'video') #If not given we suppose it's a video, support the dafault old system
+        if result_type == 'video':
+            if 'playlist' not in ie_result:
+                #It isn't part of a playlist
+                ie_result['playlist'] = None
+                ie_result['playlist_index'] = None
+            if download:
+                #Do the download:
+                self.process_info(ie_result)
+            return ie_result
+        elif result_type == 'url':
+            #We get the video pointed by the url
+            result = self.extract_info(ie_result['url'], download, ie_name = ie_result['ie_key'])[0]
+            return result
+        elif result_type == 'playlist':
+            #We process each entry in the playlist
+            playlist = ie_result.get('title', None) or ie_result.get('id', None)
+            self.to_screen(u'[download] Downloading playlist: %s'  % playlist)
+
+            playlist_results = []
+
+            n_all_entries = len(ie_result['entries'])
+            playliststart = self.params.get('playliststart', 1) - 1
+            playlistend = self.params.get('playlistend', -1)
+
+            if playlistend == -1:
+                entries = ie_result['entries'][playliststart:]
+            else:
+                entries = ie_result['entries'][playliststart:playlistend]
+
+            n_entries = len(entries)
+
+            self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
+                (ie_result['extractor'], playlist, n_all_entries, n_entries))
+
+            for i,entry in enumerate(entries,1):
+                self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
+                entry_result = self.process_ie_result(entry, False)
+                entry_result['playlist'] = playlist
+                entry_result['playlist_index'] = i + playliststart
+                #We must do the download here to correctly set the 'playlist' key
+                if download:
+                    self.process_info(entry_result)
+                playlist_results.append(entry_result)
+            result = ie_result.copy()
+            result['entries'] = playlist_results
+            return result
+
     def process_info(self, info_dict):
         """Process a single dictionary returned by an InfoExtractor."""
 
+        #We increment the download the download count here to match the previous behaviour.
+        self.increment_downloads()
+        
+        info_dict['fulltitle'] = info_dict['title']
+        if len(info_dict['title']) > 200:
+            info_dict['title'] = info_dict['title'][:197] + u'...'
+
         # Keep for backwards compatibility
         info_dict['stitle'] = info_dict['title']
 
@@ -427,7 +595,7 @@ class FileDownloader(object):
             if dn != '' and not os.path.exists(dn): # dn is already encoded
                 os.makedirs(dn)
         except (OSError, IOError) as err:
-            self.trouble(u'ERROR: unable to create directory ' + compat_str(err))
+            self.report_error(u'unable to create directory ' + compat_str(err))
             return
 
         if self.params.get('writedescription', False):
@@ -437,19 +605,46 @@ class FileDownloader(object):
                 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
                     descfile.write(info_dict['description'])
             except (OSError, IOError):
-                self.trouble(u'ERROR: Cannot write description file ' + descfn)
+                self.report_error(u'Cannot write description file ' + descfn)
                 return
 
         if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
             # subtitles download errors are already managed as troubles in relevant IE
             # that way it will silently go on when used with unsupporting IE
+            subtitle = info_dict['subtitles'][0]
+            (sub_error, sub_lang, sub) = subtitle
+            sub_format = self.params.get('subtitlesformat')
+            if sub_error:
+                self.report_warning("Some error while getting the subtitles")
+            else:
                 try:
-                srtfn = filename.rsplit('.', 1)[0] + u'.srt'
-                self.report_writesubtitles(srtfn)
-                with io.open(encodeFilename(srtfn), 'w', encoding='utf-8') as srtfile:
-                    srtfile.write(info_dict['subtitles'])
+                    sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
+                    self.report_writesubtitles(sub_filename)
+                    with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
+                        subfile.write(sub)
                 except (OSError, IOError):
-                self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
+                    self.report_error(u'Cannot write subtitles file ' + descfn)
+                    return
+            if self.params.get('onlysubtitles', False):
+                return 
+
+        if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
+            subtitles = info_dict['subtitles']
+            sub_format = self.params.get('subtitlesformat')
+            for subtitle in subtitles:
+                (sub_error, sub_lang, sub) = subtitle
+                if sub_error:
+                    self.report_warning("Some error while getting the subtitles")
+                else:
+                    try:
+                        sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
+                        self.report_writesubtitles(sub_filename)
+                        with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
+                                subfile.write(sub)
+                    except (OSError, IOError):
+                        self.report_error(u'Cannot write subtitles file ' + descfn)
+                        return
+            if self.params.get('onlysubtitles', False):
                 return 
 
         if self.params.get('writeinfojson', False):
@@ -459,7 +654,7 @@ class FileDownloader(object):
                 json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
                 write_json_file(json_info_dict, encodeFilename(infofn))
             except (OSError, IOError):
-                self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
+                self.report_error(u'Cannot write metadata to JSON file ' + infofn)
                 return
 
         if not self.params.get('skip_download', False):
@@ -471,17 +666,17 @@ class FileDownloader(object):
                 except (OSError, IOError) as err:
                     raise UnavailableVideoError()
                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                    self.trouble(u'ERROR: unable to download video data: %s' % str(err))
+                    self.report_error(u'unable to download video data: %s' % str(err))
                     return
                 except (ContentTooShortError, ) as err:
-                    self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+                    self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
                     return
 
             if success:
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
-                    self.trouble(u'ERROR: postprocessing: %s' % str(err))
+                    self.report_error(u'postprocessing: %s' % str(err))
                     return
 
     def download(self, url_list):
@@ -490,49 +685,14 @@ class FileDownloader(object):
             raise SameFileError(self.params['outtmpl'])
 
         for url in url_list:
-            suitable_found = False
-            for ie in self._ies:
-                # Go to next InfoExtractor if not suitable
-                if not ie.suitable(url):
-                    continue
-
-                # Warn if the _WORKING attribute is False
-                if not ie.working():
-                    self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, '
-                                   u'and will probably not work. If you want to go on, use the -i option.')
-
-                # Suitable InfoExtractor found
-                suitable_found = True
-
-                # Extract information from URL and process it
             try:
-                    videos = ie.extract(url)
-                except ExtractorError as de: # An error we somewhat expected
-                    self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback())
-                    break
-                except Exception as e:
-                    if self.params.get('ignoreerrors', False):
-                        self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc()))
-                        break
-                    else:
-                        raise
-
-                if len(videos or []) > 1 and self.fixed_template():
-                    raise SameFileError(self.params['outtmpl'])
-
-                for video in videos or []:
-                    video['extractor'] = ie.IE_NAME
-                    try:
-                        self.increment_downloads()
-                        self.process_info(video)
+                #It also downloads the videos
+                videos = self.extract_info(url)
             except UnavailableVideoError:
-                        self.trouble(u'\nERROR: unable to download video')
-
-                # Suitable InfoExtractor had been found; go to next URL
-                break
-
-            if not suitable_found:
-                self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
+                self.report_error(u'unable to download video')
+            except MaxDownloadsReached:
+                self.to_screen(u'[info] Maximum number of downloaded files reached.')
+                raise
 
         return self._download_retcode
 
@@ -554,20 +714,20 @@ class FileDownloader(object):
                 self.to_stderr(u'ERROR: ' + e.msg)
         if keep_video is False and not self.params.get('keepvideo', False):
             try:
-                self.to_stderr(u'Deleting original file %s (pass -k to keep)' % filename)
+                self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename)
                 os.remove(encodeFilename(filename))
             except (IOError, OSError):
-                self.to_stderr(u'WARNING: Unable to remove downloaded video file')
+                self.report_warning(u'Unable to remove downloaded video file')
 
-    def _download_with_rtmpdump(self, filename, url, player_url, page_url):
+    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
         # Check for rtmpdump first
         try:
-            subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+            subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
         except (OSError, IOError):
-            self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
+            self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
             return False
 
         # Download using rtmpdump. rtmpdump returns exit code 2 when
@@ -578,6 +738,8 @@ class FileDownloader(object):
             basic_args += ['-W', player_url]
         if page_url is not None:
             basic_args += ['--pageUrl', page_url]
+        if play_path is not None:
+            basic_args += ['-y', play_path]
         args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:
@@ -612,7 +774,8 @@ class FileDownloader(object):
             })
             return True
         else:
-            self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
+            self.to_stderr(u"\n")
+            self.report_error(u'rtmpdump exited with code %d' % retval)
             return False
 
     def _do_download(self, filename, info_dict):
@@ -631,7 +794,8 @@ class FileDownloader(object):
         if url.startswith('rtmp'):
             return self._download_with_rtmpdump(filename, url,
                                                 info_dict.get('player_url', None),
-                                                info_dict.get('page_url', None))
+                                                info_dict.get('page_url', None),
+                                                info_dict.get('play_path', None))
 
         tmpfilename = self.temp_name(filename)
         stream = None
@@ -712,7 +876,7 @@ class FileDownloader(object):
                 self.report_retry(count, retries)
 
         if count > retries:
-            self.trouble(u'ERROR: giving up after %s retries' % retries)
+            self.report_error(u'giving up after %s retries' % retries)
             return False
 
         data_len = data.info().get('Content-length', None)
@@ -748,12 +912,13 @@ class FileDownloader(object):
                     filename = self.undo_temp_name(tmpfilename)
                     self.report_destination(filename)
                 except (OSError, IOError) as err:
-                    self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
+                    self.report_error(u'unable to open for writing: %s' % str(err))
                     return False
             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:
-                self.trouble(u'\nERROR: unable to write data: %s' % str(err))
+                self.to_stderr(u"\n")
+                self.report_error(u'unable to write data: %s' % str(err))
                 return False
             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
@@ -779,7 +944,8 @@ class FileDownloader(object):
             self.slow_down(start, byte_counter - resume_len)
 
         if stream is None:
-            self.trouble(u'\nERROR: Did not get any data blocks')
+            self.to_stderr(u"\n")
+            self.report_error(u'Did not get any data blocks')
             return False
         stream.close()
         self.report_finish()

youtube_dl/__init__.py

@@ -24,6 +24,7 @@ __authors__  = (
     'Jaime Marquínez Ferrándiz',
     'Jeff Crouse',
     'Osama Khalid',
+    'Michael Walter',
     )
 
 __license__ = 'Public Domain'
@@ -46,7 +47,7 @@ from .FileDownloader import *
 from .InfoExtractors import gen_extractors
 from .PostProcessor import *
 
-def parseOpts():
+def parseOpts(overrideArguments=None):
     def _readOptions(filename_bytes):
         try:
             optionf = open(filename_bytes)
@@ -126,7 +127,7 @@ def parseOpts():
     general.add_option('-i', '--ignore-errors',
             action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
     general.add_option('-r', '--rate-limit',
-            dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
+            dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)')
     general.add_option('-R', '--retries',
             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
     general.add_option('--buffer-size',
@@ -139,9 +140,13 @@ def parseOpts():
             help='display the current browser identification', default=False)
     general.add_option('--user-agent',
             dest='user_agent', help='specify a custom user agent', metavar='UA')
+    general.add_option('--referer',
+            dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
+            metavar='REF', default=None)
     general.add_option('--list-extractors',
             action='store_true', dest='list_extractors',
             help='List all supported extractors and the URLs they would handle', default=False)
+    general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy')
     general.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 
     selection.add_option('--playlist-start',
@@ -153,6 +158,9 @@ def parseOpts():
     selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
+    selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
+    selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
+    selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
 
 
     authentication.add_option('-u', '--username',
@@ -164,7 +172,8 @@ def parseOpts():
 
 
     video_format.add_option('-f', '--format',
-            action='store', dest='format', metavar='FORMAT', help='video format code')
+            action='store', dest='format', metavar='FORMAT',
+            help='video format code, specifiy the order of preference using slashes: "-f 22/17/18"')
     video_format.add_option('--all-formats',
             action='store_const', dest='format', help='download all available video formats', const='all')
     video_format.add_option('--prefer-free-formats',
@@ -173,12 +182,24 @@ def parseOpts():
             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
     video_format.add_option('-F', '--list-formats',
             action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
-    video_format.add_option('--write-srt',
+    video_format.add_option('--write-sub', '--write-srt',
             action='store_true', dest='writesubtitles',
-            help='write video closed captions to a .srt file (currently youtube only)', default=False)
-    video_format.add_option('--srt-lang',
+            help='write subtitle file (currently youtube only)', default=False)
+    video_format.add_option('--only-sub',
+            action='store_true', dest='onlysubtitles',
+            help='downloads only the subtitles (no video)', default=False)
+    video_format.add_option('--all-subs',
+            action='store_true', dest='allsubtitles',
+            help='downloads all the available subtitles of the video (currently youtube only)', default=False)
+    video_format.add_option('--list-subs',
+            action='store_true', dest='listsubtitles',
+            help='lists all available subtitles for the video (currently youtube only)', default=False)
+    video_format.add_option('--sub-format',
+            action='store', dest='subtitlesformat', metavar='LANG',
+            help='subtitle format [srt/sbv] (default=srt) (currently youtube only)', default='srt')
+    video_format.add_option('--sub-lang', '--srt-lang',
             action='store', dest='subtitleslang', metavar='LANG',
-            help='language of the closed captions to download (optional) use IETF language tags like \'en\'')
+            help='language of the subtitles to download (optional) use IETF language tags like \'en\'')
 
     verbosity.add_option('-q', '--quiet',
             action='store_true', dest='quiet', help='activates quiet mode', default=False)
@@ -211,18 +232,33 @@ def parseOpts():
             help='display progress in console titlebar', default=False)
     verbosity.add_option('-v', '--verbose',
             action='store_true', dest='verbose', help='print various debugging information', default=False)
+    verbosity.add_option('--dump-intermediate-pages',
+            action='store_true', dest='dump_intermediate_pages', default=False,
+            help='print downloaded pages to debug problems(very verbose)')
 
     filesystem.add_option('-t', '--title',
-            action='store_true', dest='usetitle', help='use title in file name', default=False)
+            action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
     filesystem.add_option('--id',
-            action='store_true', dest='useid', help='use video ID in file name', default=False)
+            action='store_true', dest='useid', help='use only video ID in file name', default=False)
     filesystem.add_option('-l', '--literal',
             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
     filesystem.add_option('-A', '--auto-number',
             action='store_true', dest='autonumber',
             help='number downloaded files starting from 00000', default=False)
     filesystem.add_option('-o', '--output',
-            dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
+            dest='outtmpl', metavar='TEMPLATE',
+            help=('output filename template. Use %(title)s to get the title, '
+                  '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
+                  '%(autonumber)s to get an automatically incremented number, '
+                  '%(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), '
+                  '%(extractor)s for the provider (youtube, metacafe, etc), '
+                  '%(id)s for the video id , %(playlist)s for the playlist the video is in, '
+                  '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
+                  'Use - to output to stdout. Can also be used to download to a different directory, '
+                  'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
+    filesystem.add_option('--autonumber-size',
+            dest='autonumber_size', metavar='NUMBER',
+            help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given')
     filesystem.add_option('--restrict-filenames',
             action='store_true', dest='restrictfilenames',
             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
@@ -272,18 +308,30 @@ def parseOpts():
     parser.add_option_group(authentication)
     parser.add_option_group(postproc)
 
+    if overrideArguments is not None:
+        opts, args = parser.parse_args(overrideArguments)
+        if opts.verbose:
+            print(u'[debug] Override config: ' + repr(overrideArguments))
+    else:
         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
         if xdg_config_home:
-        userConf = os.path.join(xdg_config_home, 'youtube-dl.conf')
+            userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
         else:
-        userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
-    argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:]
+            userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
+        systemConf = _readOptions('/etc/youtube-dl.conf')
+        userConf = _readOptions(userConfFile)
+        commandLineConf = sys.argv[1:] 
+        argv = systemConf + userConf + commandLineConf
         opts, args = parser.parse_args(argv)
+        if opts.verbose:
+            print(u'[debug] System config: ' + repr(systemConf))
+            print(u'[debug] User config: ' + repr(userConf))
+            print(u'[debug] Command-line args: ' + repr(commandLineConf))
 
     return parser, opts, args
 
-def _real_main():
-    parser, opts, args = parseOpts()
+def _real_main(argv=None):
+    parser, opts, args = parseOpts(argv)
 
     # Open appropriate CookieJar
     if opts.cookiefile is None:
@@ -302,6 +350,10 @@ def _real_main():
     if opts.user_agent is not None:
         std_headers['User-Agent'] = opts.user_agent
     
+    # Set referer
+    if opts.referer is not None:
+        std_headers['Referer'] = opts.referer
+
     # Dump user agent
     if opts.dump_user_agent:
         print(std_headers['User-Agent'])
@@ -325,7 +377,14 @@ def _real_main():
 
     # General configuration
     cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
-    proxy_handler = compat_urllib_request.ProxyHandler()
+    if opts.proxy:
+        proxies = {'http': opts.proxy, 'https': opts.proxy}
+    else:
+        proxies = compat_urllib_request.getproxies()
+        # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
+        if 'http' in proxies and 'https' not in proxies:
+            proxies['https'] = proxies['http']
+    proxy_handler = compat_urllib_request.ProxyHandler(proxies)
     opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
     compat_urllib_request.install_opener(opener)
     socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
@@ -399,6 +458,10 @@ def _real_main():
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
             parser.error(u'invalid video recode format specified')
+    if opts.date is not None:
+        date = DateRange.day(opts.date)
+    else:
+        date = DateRange(opts.dateafter, opts.datebefore)
 
     if sys.version_info < (3,):
         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
@@ -411,7 +474,8 @@ def _real_main():
             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
             or (opts.useid and u'%(id)s.%(ext)s')
             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
-            or u'%(id)s.%(ext)s')
+            or u'%(title)s-%(id)s.%(ext)s')
+
     # File downloader
     fd = FileDownloader({
         'usenetrc': opts.usenetrc,
@@ -430,6 +494,7 @@ def _real_main():
         'format_limit': opts.format_limit,
         'listformats': opts.listformats,
         'outtmpl': outtmpl,
+        'autonumber_size': opts.autonumber_size,
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
         'ratelimit': opts.ratelimit,
@@ -449,16 +514,22 @@ def _real_main():
         'writedescription': opts.writedescription,
         'writeinfojson': opts.writeinfojson,
         'writesubtitles': opts.writesubtitles,
+        'onlysubtitles': opts.onlysubtitles,
+        'allsubtitles': opts.allsubtitles,
+        'listsubtitles': opts.listsubtitles,
+        'subtitlesformat': opts.subtitlesformat,
         'subtitleslang': opts.subtitleslang,
-        'matchtitle': opts.matchtitle,
-        'rejecttitle': opts.rejecttitle,
+        'matchtitle': decodeOption(opts.matchtitle),
+        'rejecttitle': decodeOption(opts.rejecttitle),
         'max_downloads': opts.max_downloads,
         'prefer_free_formats': opts.prefer_free_formats,
         'verbose': opts.verbose,
+        'dump_intermediate_pages': opts.dump_intermediate_pages,
         'test': opts.test,
         'keepvideo': opts.keepvideo,
         'min_filesize': opts.min_filesize,
-        'max_filesize': opts.max_filesize
+        'max_filesize': opts.max_filesize,
+        'daterange': date
         })
 
     if opts.verbose:
@@ -510,9 +581,9 @@ def _real_main():
 
     sys.exit(retcode)
 
-def main():
+def main(argv=None):
     try:
-        _real_main()
+        _real_main(argv)
     except DownloadError:
         sys.exit(1)
     except SameFileError:

youtube_dl/__main__.py

@@ -9,7 +9,8 @@ import sys
 if __package__ is None and not hasattr(sys, "frozen"):
     # direct call of __main__.py
     import os.path
-    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    path = os.path.realpath(os.path.abspath(__file__))
+    sys.path.append(os.path.dirname(os.path.dirname(path)))
 
 import youtube_dl
 

youtube_dl/update.py

@@ -37,7 +37,7 @@ def rsa_verify(message, signature, key):
 def update_self(to_screen, verbose, filename):
     """Update the program file with the latest version from the repository"""
 
-    UPDATE_URL = "http://rg3.github.com/youtube-dl/update/"
+    UPDATE_URL = "http://rg3.github.io/youtube-dl/update/"
     VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
     JSON_URL = UPDATE_URL + 'versions.json'
     UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
@@ -77,10 +77,8 @@ def update_self(to_screen, verbose, filename):
 
     to_screen(u'Updating to version ' + versions_info['latest'] + '...')
     version = versions_info['versions'][versions_info['latest']]
-    if version.get('notes'):
-        to_screen(u'PLEASE NOTE:')
-        for note in version['notes']:
-            to_screen(note)
+
+    print_notes(to_screen, versions_info['versions'])
 
     if not os.access(filename, os.W_OK):
         to_screen(u'ERROR: no write permissions on %s' % filename)
@@ -158,3 +156,17 @@ del "%s"
             return
 
     to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
+
+def get_notes(versions, fromVersion):
+    notes = []
+    for v,vdata in sorted(versions.items()):
+        if v > fromVersion:
+            notes.extend(vdata.get('notes', []))
+    return notes
+
+def print_notes(to_screen, versions, fromVersion=__version__):
+    notes = get_notes(versions, fromVersion)
+    if notes:
+        to_screen(u'PLEASE NOTE:')
+        for note in notes:
+            to_screen(note)

youtube_dl/utils.py

@@ -12,6 +12,7 @@ import traceback
 import zlib
 import email.utils
 import json
+import datetime
 
 try:
     import urllib.request as compat_urllib_request
@@ -311,7 +312,7 @@ def clean_html(html):
     html = re.sub('<.*?>', '', html)
     # Replace html entities
     html = unescapeHTML(html)
-    return html
+    return html.strip()
 
 
 def sanitize_open(filename, open_mode):
@@ -329,7 +330,7 @@ def sanitize_open(filename, open_mode):
             if sys.platform == 'win32':
                 import msvcrt
                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
-            return (sys.stdout, filename)
+            return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
         stream = open(encodeFilename(filename), open_mode)
         return (stream, filename)
     except (IOError, OSError) as err:
@@ -420,6 +421,14 @@ def encodeFilename(s):
             encoding = 'utf-8'
         return s.encode(encoding, 'ignore')
 
+def decodeOption(optval):
+    if optval is None:
+        return optval
+    if isinstance(optval, bytes):
+        optval = optval.decode(preferredencoding())
+
+    assert isinstance(optval, compat_str)
+    return optval
 
 class ExtractorError(Exception):
     """Error during info extraction."""
@@ -427,6 +436,7 @@ class ExtractorError(Exception):
         """ tb, if given, is the original traceback (so that it can be printed out). """
         super(ExtractorError, self).__init__(msg)
         self.traceback = tb
+        self.exc_info = sys.exc_info()  # preserve original exception
 
     def format_traceback(self):
         if self.traceback is None:
@@ -441,7 +451,10 @@ class DownloadError(Exception):
     configured to continue on errors. They will contain the appropriate
     error message.
     """
-    pass
+    def __init__(self, msg, exc_info=None):
+        """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
+        super(DownloadError, self).__init__(msg)
+        self.exc_info = exc_info
 
 
 class SameFileError(Exception):
@@ -556,3 +569,70 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
     https_request = http_request
     https_response = http_response
+
+def unified_strdate(date_str):
+    """Return a string with the date in the format YYYYMMDD"""
+    upload_date = None
+    #Replace commas
+    date_str = date_str.replace(',',' ')
+    # %z (UTC offset) is only supported in python>=3.2
+    date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
+    format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y', '%Y-%m-%d', '%d/%m/%Y', '%Y/%m/%d %H:%M:%S']
+    for expression in format_expressions:
+        try:
+            upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
+        except:
+            pass
+    return upload_date
+
+def date_from_str(date_str):
+    """
+    Return a datetime object from a string in the format YYYYMMDD or
+    (now|today)[+-][0-9](day|week|month|year)(s)?"""
+    today = datetime.date.today()
+    if date_str == 'now'or date_str == 'today':
+        return today
+    match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
+    if match is not None:
+        sign = match.group('sign')
+        time = int(match.group('time'))
+        if sign == '-':
+            time = -time
+        unit = match.group('unit')
+        #A bad aproximation?
+        if unit == 'month':
+            unit = 'day'
+            time *= 30
+        elif unit == 'year':
+            unit = 'day'
+            time *= 365
+        unit += 's'
+        delta = datetime.timedelta(**{unit: time})
+        return today + delta
+    return datetime.datetime.strptime(date_str, "%Y%m%d").date()
+    
+class DateRange(object):
+    """Represents a time interval between two dates"""
+    def __init__(self, start=None, end=None):
+        """start and end must be strings in the format accepted by date"""
+        if start is not None:
+            self.start = date_from_str(start)
+        else:
+            self.start = datetime.datetime.min.date()
+        if end is not None:
+            self.end = date_from_str(end)
+        else:
+            self.end = datetime.datetime.max.date()
+        if self.start > self.end:
+            raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
+    @classmethod
+    def day(cls, day):
+        """Returns a range that only contains the given day"""
+        return cls(day,day)
+    def __contains__(self, date):
+        """Check if the date is in the range"""
+        if not isinstance(date, datetime.date):
+            date = date_from_str(date)
+        return self.start <= date <= self.end
+    def __str__(self):
+        return '%s - %s' % ( self.start.isoformat(), self.end.isoformat())

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2013.02.19'
+__version__ = '2013.04.30'