release 2013.01.11

justin.tv tweaks

.gitignore

@@ -1,6 +1,19 @@
 *.pyc
 *.pyo
 *~
+*.DS_Store
 wine-py2exe/
 py2exe.log
 *.kate-swp
+build/
+dist/
+MANIFEST
+README.txt
+youtube-dl.1
+youtube-dl.bash-completion
+youtube-dl
+youtube-dl.exe
+youtube-dl.tar.gz
+.coverage
+cover/
+updates_key.pem

.travis.yml

@@ -1,9 +1,14 @@
 language: python
-#specify the python version
 python:
   - "2.6"
   - "2.7"
-#command to install the setup
-install:
-# command to run tests
-script: nosetests test --nocapture
+  - "3.3"
+script: nosetests test --verbose
+notifications:
+  email:
+    - filippo.valsorda@gmail.com
+    - phihag@phihag.de
+#  irc:
+#    channels:
+#      - "irc.freenode.org#youtube-dl"
+#    skip_join: true

CHANGELOG

@@ -0,0 +1,14 @@
+2013.01.02  Codename: GIULIA
+
+    * Add support for ComedyCentral clips <nto>
+    * Corrected Vimeo description fetching <Nick Daniels>
+    * Added the --no-post-overwrites argument <Barbu Paul - Gheorghe>
+    * --verbose offers more environment info
+    * New info_dict field: uploader_id
+    * New updates system, with signature checking
+    * New IEs: NBA, JustinTV, FunnyOrDie, TweetReel, Steam, Ustream
+    * Fixed IEs: BlipTv
+    * Fixed for Python 3 IEs: Xvideo, Youku, XNXX, Dailymotion, Vimeo, InfoQ
+    * Simplified IEs and test code
+    * Various (Python 3 and other) fixes
+    * Revamped and expanded tests

LATEST_VERSION

@@ -1 +1 @@
-2012.11.29
+9999.99.99

LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include test/*.py
+include test/*.json

Makefile

@@ -1,8 +1,7 @@
-all: youtube-dl README.md youtube-dl.1 youtube-dl.bash-completion LATEST_VERSION
-# TODO: re-add youtube-dl.exe, and make sure it's 1. safe and 2. doesn't need sudo
+all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
 
 clean:
-	rm -f youtube-dl youtube-dl.exe youtube-dl.1 LATEST_VERSION
+	rm -rf youtube-dl youtube-dl.exe youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz
 
 PREFIX=/usr/local
 BINDIR=$(PREFIX)/bin
@@ -18,40 +17,44 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
 	install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl
 
 test:
-	nosetests2 --nocapture test
+	#nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test
+	nosetests --verbose test
 
-.PHONY: all clean install test README.md youtube-dl.bash-completion
-# TODO un-phony README.md and youtube-dl.bash_completion by reading from .in files and generating from them
+tar: youtube-dl.tar.gz
+
+.PHONY: all clean install test tar
 
 youtube-dl: youtube_dl/*.py
-	zip --quiet --junk-paths youtube-dl youtube_dl/*.py
+	zip --quiet youtube-dl youtube_dl/*.py
+	zip --quiet --junk-paths youtube-dl youtube_dl/__main__.py
 	echo '#!/usr/bin/env python' > youtube-dl
 	cat youtube-dl.zip >> youtube-dl
 	rm youtube-dl.zip
 	chmod a+x youtube-dl
 
-youtube-dl.exe: youtube_dl/*.py
-	bash devscripts/wine-py2exe.sh build_exe.py
-
 README.md: youtube_dl/*.py
-	@options=$$(COLUMNS=80 python -m youtube_dl --help | sed -e '1,/.*General Options.*/ d' -e 's/^\W\{2\}\(\w\)/## \1/') && \
-		header=$$(sed -e '/.*# OPTIONS/,$$ d' README.md) && \
-		footer=$$(sed -e '1,/.*# CONFIGURATION/ d' README.md) && \
-		echo "$${header}" > README.md && \
-		echo >> README.md && \
-		echo '# OPTIONS' >> README.md && \
-		echo "$${options}" >> README.md&& \
-		echo >> README.md && \
-		echo '# CONFIGURATION' >> README.md && \
-		echo "$${footer}" >> README.md
+	COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py
+
+README.txt: README.md
+	pandoc -f markdown -t plain README.md -o README.txt
 
 youtube-dl.1: README.md
-	pandoc -s -w man README.md -o youtube-dl.1
+	pandoc -s -f markdown -t man README.md -o youtube-dl.1
 
-youtube-dl.bash-completion: README.md
-	@options=`egrep -o '(--[a-z-]+) ' README.md | sort -u | xargs echo` && \
-		content=`sed "s/opts=\"[^\"]*\"/opts=\"$${options}\"/g" youtube-dl.bash-completion` && \
-		echo "$${content}" > youtube-dl.bash-completion
+youtube-dl.bash-completion: youtube_dl/*.py devscripts/bash-completion.in
+	python devscripts/bash-completion.py
 
-LATEST_VERSION: youtube_dl/__init__.py
-	python -m youtube_dl --version > LATEST_VERSION
+youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
+	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
+		--exclude '*.DS_Store' \
+		--exclude '*.kate-swp' \
+		--exclude '*.pyc' \
+		--exclude '*.pyo' \
+		--exclude '*~' \
+		--exclude '__pycache' \
+		--exclude '.git' \
+		-- \
+		bin devscripts test youtube_dl \
+		CHANGELOG LICENSE README.md README.txt \
+		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion setup.py \
+		youtube-dl

README.md

@@ -9,8 +9,8 @@ youtube-dl
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
 YouTube.com and a few more sites. It requires the Python interpreter, version
-2.x (x being at least 6), and it is not platform specific. It should work in
-your Unix box, in Windows or in Mac OS X. It is released to the public domain,
+2.6, 2.7, or 3.3+, and it is not platform specific. It should work on
+your Unix box, on Windows or on Mac OS X. It is released to the public domain,
 which means you can modify it, redistribute it or use it however you like.
 
 # OPTIONS
@@ -46,12 +46,16 @@ which means you can modify it, redistribute it or use it however you like.
     -A, --auto-number        number downloaded files starting from 00000
     -o, --output TEMPLATE    output filename template. Use %(title)s to get the
                              title, %(uploader)s for the uploader name,
-                             %(autonumber)s to get an automatically incremented
-                             number, %(ext)s for the filename extension,
-                             %(upload_date)s for the upload date (YYYYMMDD),
-                             %(extractor)s for the provider (youtube, metacafe,
-                             etc), %(id)s for the video id and %% for a literal
-                             percent. Use - to output to stdout.
+                             %(uploader_id)s for the uploader nickname if
+                             different, %(autonumber)s to get an automatically
+                             incremented number, %(ext)s for the filename
+                             extension, %(upload_date)s for the upload date
+                             (YYYYMMDD), %(extractor)s for the provider
+                             (youtube, metacafe, etc), %(id)s for the video id
+                             and %% for a literal percent. Use - to output to
+                             stdout. Can also be used to download to a different
+                             directory, for example with -o '/my/downloads/%(upl
+                             oader)s/%(title)s-%(id)s.%(ext)s' .
     --restrict-filenames     Restrict filenames to only ASCII characters, and
                              avoid "&" and spaces in filenames
     -a, --batch-file FILE    file containing URLs to download ('-' for stdin)
@@ -101,17 +105,19 @@ which means you can modify it, redistribute it or use it however you like.
 ## Post-processing Options:
     -x, --extract-audio      convert video files to audio-only files (requires
                              ffmpeg or avconv and ffprobe or avprobe)
-    --audio-format FORMAT    "best", "aac", "vorbis", "mp3", "m4a", or "wav";
-                             best by default
+    --audio-format FORMAT    "best", "aac", "vorbis", "mp3", "m4a", "opus", or
+                             "wav"; best by default
     --audio-quality QUALITY  ffmpeg/avconv audio quality specification, insert a
                              value between 0 (better) and 9 (worse) for VBR or a
                              specific bitrate like 128K (default 5)
     -k, --keep-video         keeps the video file on disk after the post-
                              processing; the video is erased by default
+    --no-post-overwrites     do not overwrite post-processed files; the post-
+                             processed files are overwritten by default
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.local/config/youtube-dl.conf`.
+You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl.conf`.
 
 # OUTPUT TEMPLATE
 
@@ -170,14 +176,6 @@ The error
 
 means you're using an outdated version of Python. Please update to Python 2.6 or 2.7.
 
-To run youtube-dl under Python 2.5, you'll have to manually check it out like this:
-
-	git clone git://github.com/rg3/youtube-dl.git
-	cd youtube-dl
-	python -m youtube_dl --help
-
-Please note that Python 2.5 is not supported anymore.
-
 ### What is this binary file? Where has the code gone?
 
 Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
@@ -199,6 +197,9 @@ Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/i
 Please include:
 
 * Your exact command line, like `youtube-dl -t "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"`. A common mistake is not to escape the `&`. Putting URLs in quotes should solve this problem.
+* If possible re-run the command with `--verbose`, and include the full output, it is really helpful to us.
 * The output of `youtube-dl --version`
 * The output of `python --version`
 * The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).
+
+For discussions, join us in the irc channel #youtube-dl on freenode.

bin/youtube-dl

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import youtube_dl
+
+if __name__ == '__main__':
+    youtube_dl.main()

build_exe.py

@@ -1,48 +0,0 @@
-from distutils.core import setup
-import py2exe
-import sys, os
-
-"""This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""
-
-# If run without args, build executables
-if len(sys.argv) == 1:
-    sys.argv.append("py2exe")
-
-# os.chdir(os.path.dirname(os.path.abspath(sys.argv[0]))) # conflict with wine-py2exe.sh
-sys.path.append('./youtube_dl')
-
-options = {
-    "bundle_files": 1,
-    "compressed": 1,
-    "optimize": 2,
-    "dist_dir": '.',
-    "dll_excludes": ['w9xpopen.exe']
-}
-
-console = [{
-    "script":"./youtube_dl/__main__.py",
-    "dest_base": "youtube-dl",
-}]
-
-init_file = open('./youtube_dl/__init__.py')
-for line in init_file.readlines():
-    if line.startswith('__version__'):
-        version = line[11:].strip(" ='\n")
-        break
-else:
-    version = ''
-
-setup(name='youtube-dl',
-      version=version,
-      description='Small command-line program to download videos from YouTube.com and other video sites',
-      url='https://github.com/rg3/youtube-dl',
-      packages=['youtube_dl'],
-      
-      console = console,
-      options = {"py2exe": options},
-      zipfile = None,
-)
-
-import shutil
-shutil.rmtree("build")
-

devscripts/bash-completion.in

@@ -0,0 +1,14 @@
+__youtube-dl()
+{
+    local cur prev opts
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    opts="{{flags}}"
+
+    if [[ ${cur} == * ]] ; then
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+        return 0
+    fi
+}
+
+complete -F __youtube-dl youtube-dl

devscripts/bash-completion.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+import os
+from os.path import dirname as dirn
+import sys
+
+sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+import youtube_dl
+
+BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
+BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
+
+def build_completion(opt_parser):
+    opts_flag = []
+    for group in opt_parser.option_groups:
+        for option in group.option_list:
+            #for every long flag
+            opts_flag.append(option.get_opt_string())
+    with open(BASH_COMPLETION_TEMPLATE) as f:
+        template = f.read()
+    with open(BASH_COMPLETION_FILE, "w") as f:
+        #just using the special char
+        filled_template = template.replace("{{flags}}", " ".join(opts_flag))
+        f.write(filled_template)
+
+parser = youtube_dl.parseOpts()[0]
+build_completion(parser)

devscripts/gh-pages/add-version.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+import hashlib
+import urllib.request
+
+if len(sys.argv) <= 1:
+	print('Specify the version number as parameter')
+	sys.exit()
+version = sys.argv[1]
+
+with open('update/LATEST_VERSION', 'w') as f:
+	f.write(version)
+
+versions_info = json.load(open('update/versions.json'))
+if 'signature' in versions_info:
+	del versions_info['signature']
+
+new_version = {}
+
+filenames = {'bin': 'youtube-dl', 'exe': 'youtube-dl.exe', 'tar': 'youtube-dl-%s.tar.gz' % version}
+for key, filename in filenames.items():
+	print('Downloading and checksumming %s...' %filename)
+	url = 'http://youtube-dl.org/downloads/%s/%s' % (version, filename)
+	data = urllib.request.urlopen(url).read()
+	sha256sum = hashlib.sha256(data).hexdigest()
+	new_version[key] = (url, sha256sum)
+
+versions_info['versions'][version] = new_version
+versions_info['latest'] = version
+
+json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True)

devscripts/gh-pages/generate-download.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+import hashlib
+import shutil
+import subprocess
+import tempfile
+import urllib.request
+import json
+
+versions_info = json.load(open('update/versions.json'))
+version = versions_info['latest']
+URL = versions_info['versions'][version]['bin'][0]
+
+data = urllib.request.urlopen(URL).read()
+
+# Read template page
+with open('download.html.in', 'r', encoding='utf-8') as tmplf:
+    template = tmplf.read()
+
+md5sum = hashlib.md5(data).hexdigest()
+sha1sum = hashlib.sha1(data).hexdigest()
+sha256sum = hashlib.sha256(data).hexdigest()
+template = template.replace('@PROGRAM_VERSION@', version)
+template = template.replace('@PROGRAM_URL@', URL)
+template = template.replace('@PROGRAM_MD5SUM@', md5sum)
+template = template.replace('@PROGRAM_SHA1SUM@', sha1sum)
+template = template.replace('@PROGRAM_SHA256SUM@', sha256sum)
+template = template.replace('@EXE_URL@', versions_info['versions'][version]['exe'][0])
+template = template.replace('@EXE_SHA256SUM@', versions_info['versions'][version]['exe'][1])
+template = template.replace('@TAR_URL@', versions_info['versions'][version]['tar'][0])
+template = template.replace('@TAR_SHA256SUM@', versions_info['versions'][version]['tar'][1])
+with open('download.html', 'w', encoding='utf-8') as dlf:
+    dlf.write(template)

devscripts/gh-pages/sign-versions.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+import rsa
+import json
+from binascii import hexlify
+
+versions_info = json.load(open('update/versions.json'))
+if 'signature' in versions_info:
+	del versions_info['signature']
+
+print('Enter the PKCS1 private key, followed by a blank line:')
+privkey = ''
+while True:
+	try:
+		line = input()
+	except EOFError:
+		break
+	if line == '':
+		break
+	privkey += line + '\n'
+privkey = bytes(privkey, 'ascii')
+privkey = rsa.PrivateKey.load_pkcs1(privkey)
+
+signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).encode('utf-8'), privkey, 'SHA-256')).decode()
+print('signature: ' + signature)
+
+versions_info['signature'] = signature
+json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True)

devscripts/gh-pages/update-copyright.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import with_statement
+
+import datetime
+import glob
+import io # For Python 2 compatibilty
+import os
+import re
+
+year = str(datetime.datetime.now().year)
+for fn in glob.glob('*.html*'):
+    with io.open(fn, encoding='utf-8') as f:
+        content = f.read()
+    newc = re.sub(u'(?P<copyright>Copyright © 2006-)(?P<year>[0-9]{4})', u'Copyright © 2006-' + year, content)
+    if content != newc:
+        tmpFn = fn + '.part'
+        with io.open(tmpFn, 'wt', encoding='utf-8') as outf:
+            outf.write(newc)
+        os.rename(tmpFn, fn)

devscripts/make_readme.py

@@ -0,0 +1,20 @@
+import sys
+import re
+
+README_FILE = 'README.md'
+helptext = sys.stdin.read()
+
+with open(README_FILE) as f:
+    oldreadme = f.read()
+
+header = oldreadme[:oldreadme.index('# OPTIONS')]
+footer = oldreadme[oldreadme.index('# CONFIGURATION'):]
+
+options = helptext[helptext.index('  General Options:')+19:]
+options = re.sub(r'^  (\w.+)$', r'## \1', options, flags=re.M)
+options = '# OPTIONS\n' + options + '\n'
+
+with open(README_FILE, 'w') as f:
+    f.write(header)
+    f.write(options)
+    f.write(footer)

devscripts/release.sh

@@ -1,11 +1,83 @@
-#!/bin/sh
+#!/bin/bash
+
+# IMPORTANT: the following assumptions are made
+# * the GH repo is on the origin remote
+# * the gh-pages branch is named so locally
+# * the git config user.signingkey is properly set
+
+# You will need
+# pip install coverage nose rsa
+
+# TODO
+# release notes
+# make hash on local files
+
+set -e
 
 if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi
 version="$1"
 if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
-if [ ! -z "`git status --porcelain`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
-sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/__init__.py
-make all
-git add -A
+if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
+if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
+
+echo "\n### First of all, testing..."
+make clean
+nosetests --with-coverage --cover-package=youtube_dl --cover-html test || exit 1
+
+echo "\n### Changing version in version.py..."
+sed -i~ "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
+
+echo "\n### Committing CHANGELOG README.md and youtube_dl/version.py..."
+make README.md
+git add CHANGELOG README.md youtube_dl/version.py
 git commit -m "release $version"
-git tag -m "Release $version" "$version"
+
+echo "\n### Now tagging, signing and pushing..."
+git tag -s -m "Release $version" "$version"
+git show "$version"
+read -p "Is it good, can I push? (y/n) " -n 1
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
+echo
+MASTER=$(git rev-parse --abbrev-ref HEAD)
+git push origin $MASTER:master
+git push origin "$version"
+
+echo "\n### OK, now it is time to build the binaries..."
+REV=$(git rev-parse HEAD)
+make youtube-dl youtube-dl.tar.gz
+wget "http://jeromelaheurte.net:8142/download/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe || \
+	wget "http://jeromelaheurte.net:8142/build/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe
+mkdir -p "update_staging/$version"
+mv youtube-dl youtube-dl.exe "update_staging/$version"
+mv youtube-dl.tar.gz "update_staging/$version/youtube-dl-$version.tar.gz"
+RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz"
+(cd update_staging/$version/ && md5sum $RELEASE_FILES > MD5SUMS)
+(cd update_staging/$version/ && sha1sum $RELEASE_FILES > SHA1SUMS)
+(cd update_staging/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS)
+(cd update_staging/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS)
+git checkout HEAD -- youtube-dl youtube-dl.exe
+
+echo "\n### Signing and uploading the new binaries to youtube-dl.org..."
+for f in $RELEASE_FILES; do gpg --detach-sig "update_staging/$version/$f"; done
+scp -r "update_staging/$version" ytdl@youtube-dl.org:html/downloads/
+rm -r update_staging
+
+echo "\n### Now switching to gh-pages..."
+git checkout gh-pages
+git checkout "$MASTER" -- devscripts/gh-pages/
+git reset devscripts/gh-pages/
+devscripts/gh-pages/add-version.py $version
+devscripts/gh-pages/sign-versions.py < updates_key.pem
+devscripts/gh-pages/generate-download.py
+devscripts/gh-pages/update-copyright.py
+git add *.html *.html.in update
+git commit -m "release $version"
+git show HEAD
+read -p "Is it good, can I push? (y/n) " -n 1
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
+echo
+git push origin gh-pages
+
+echo "\n### DONE!"
+rm -r devscripts
+git checkout $MASTER

devscripts/transition_helper.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+import sys, os
+
+try:
+    import urllib.request as compat_urllib_request
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_request
+
+sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
+sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
+sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n')
+
+try:
+	raw_input()
+except NameError: # Python 3
+	input()
+
+filename = sys.argv[0]
+
+API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads"
+BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl"
+
+if not os.access(filename, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % filename)
+
+try:
+    urlh = compat_urllib_request.urlopen(BIN_URL)
+    newcontent = urlh.read()
+    urlh.close()
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to download latest version')
+
+try:
+    with open(filename, 'wb') as outf:
+        outf.write(newcontent)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to overwrite current version')
+
+sys.stderr.write(u'Done! Now you can run youtube-dl.\n')

devscripts/transition_helper_exe/setup.py

@@ -0,0 +1,12 @@
+from distutils.core import setup
+import py2exe
+
+py2exe_options = {
+    "bundle_files": 1,
+    "compressed": 1,
+    "optimize": 2,
+    "dist_dir": '.',
+    "dll_excludes": ['w9xpopen.exe']
+}
+
+setup(console=['youtube-dl.py'], options={ "py2exe": py2exe_options }, zipfile=None)

devscripts/transition_helper_exe/youtube-dl.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+
+import sys, os
+import urllib2
+import json, hashlib
+
+def rsa_verify(message, signature, key):
+    from struct import pack
+    from hashlib import sha256
+    from sys import version_info
+    def b(x):
+        if version_info[0] == 2: return x
+        else: return x.encode('latin1')
+    assert(type(message) == type(b('')))
+    block_size = 0
+    n = key[0]
+    while n:
+        block_size += 1
+        n >>= 8
+    signature = pow(int(signature, 16), key[1], key[0])
+    raw_bytes = []
+    while signature:
+        raw_bytes.insert(0, pack("B", signature & 0xFF))
+        signature >>= 8
+    signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
+    if signature[0:2] != b('\x00\x01'): return False
+    signature = signature[2:]
+    if not b('\x00') in signature: return False
+    signature = signature[signature.index(b('\x00'))+1:]
+    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
+    signature = signature[19:]
+    if signature != sha256(message).digest(): return False
+    return True
+
+sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
+sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
+sys.stderr.write(u'From now on, get the binaries from http://rg3.github.com/youtube-dl/download.html, not from the git repository.\n\n')
+
+raw_input()
+
+filename = sys.argv[0]
+
+UPDATE_URL = "http://rg3.github.com/youtube-dl/update/"
+VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
+JSON_URL = UPDATE_URL + 'versions.json'
+UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
+
+if not os.access(filename, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % filename)
+
+exe = os.path.abspath(filename)
+directory = os.path.dirname(exe)
+if not os.access(directory, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % directory)
+
+try:
+    versions_info = urllib2.urlopen(JSON_URL).read().decode('utf-8')
+    versions_info = json.loads(versions_info)
+except:
+    sys.exit(u'ERROR: can\'t obtain versions info. Please try again later.')
+if not 'signature' in versions_info:
+    sys.exit(u'ERROR: the versions file is not signed or corrupted. Aborting.')
+signature = versions_info['signature']
+del versions_info['signature']
+if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY):
+    sys.exit(u'ERROR: the versions file signature is invalid. Aborting.')
+
+version = versions_info['versions'][versions_info['latest']]
+
+try:
+    urlh = urllib2.urlopen(version['exe'][0])
+    newcontent = urlh.read()
+    urlh.close()
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to download latest version')
+
+newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+if newcontent_hash != version['exe'][1]:
+    sys.exit(u'ERROR: the downloaded file hash does not match. Aborting.')
+
+try:
+    with open(exe + '.new', 'wb') as outf:
+        outf.write(newcontent)
+except (IOError, OSError) as err:
+    sys.exit(u'ERROR: unable to write the new version')
+
+try:
+    bat = os.path.join(directory, 'youtube-dl-updater.bat')
+    b = open(bat, 'w')
+    b.write("""
+echo Updating youtube-dl...
+ping 127.0.0.1 -n 5 -w 1000 > NUL
+move /Y "%s.new" "%s"
+del "%s"
+    \n""" %(exe, exe, bat))
+    b.close()
+
+    os.startfile(bat)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to overwrite current version')
+
+sys.stderr.write(u'Done! Now you can run youtube-dl.\n')

setup.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+from distutils.core import setup
+import pkg_resources
+import sys
+
+try:
+    import py2exe
+    """This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""
+except ImportError:
+    if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
+        print("Cannot import py2exe", file=sys.stderr)
+        exit(1)
+
+py2exe_options = {
+    "bundle_files": 1,
+    "compressed": 1,
+    "optimize": 2,
+    "dist_dir": '.',
+    "dll_excludes": ['w9xpopen.exe']
+}
+py2exe_console = [{
+    "script": "./youtube_dl/__main__.py",
+    "dest_base": "youtube-dl",
+}]
+py2exe_params = {
+    'console': py2exe_console,
+    'options': { "py2exe": py2exe_options },
+    'zipfile': None
+}
+
+if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
+    params = py2exe_params
+else:
+    params = {
+        'scripts': ['bin/youtube-dl'],
+        'data_files': [('etc/bash_completion.d', ['youtube-dl.bash-completion']), # Installing system-wide would require sudo...
+                       ('share/doc/youtube_dl', ['README.txt']),
+                       ('share/man/man1/', ['youtube-dl.1'])]
+    }
+
+# Get the version from youtube_dl/version.py without importing the package
+exec(compile(open('youtube_dl/version.py').read(), 'youtube_dl/version.py', 'exec'))
+
+setup(
+    name = 'youtube_dl',
+    version = __version__,
+    description = 'YouTube video downloader',
+    long_description = 'Small command-line program to download videos from YouTube.com and other video sites.',
+    url = 'https://github.com/rg3/youtube-dl',
+    author = 'Ricardo Garcia',
+    maintainer = 'Philipp Hagemeister',
+    maintainer_email = 'phihag@phihag.de',
+    packages = ['youtube_dl'],
+
+    # Provokes warning on most systems (why?!)
+    #test_suite = 'nose.collector',
+    #test_requires = ['nosetest'],
+
+    classifiers = [
+        "Topic :: Multimedia :: Video",
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Console",
+        "License :: Public Domain",
+        "Programming Language :: Python :: 2.6",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.3"
+    ],
+
+    **params
+)

test/parameters.json

@@ -1 +1,40 @@
-{"username": null, "listformats": null, "skip_download": false, "usenetrc": false, "max_downloads": null, "noprogress": false, "forcethumbnail": false, "forceformat": false, "format_limit": null, "ratelimit": null, "nooverwrites": false, "forceurl": false, "writeinfojson": false, "simulate": false, "playliststart": 1, "continuedl": true, "password": null, "prefer_free_formats": false, "nopart": false, "retries": 10, "updatetime": true, "consoletitle": false, "verbose": true, "forcefilename": false, "ignoreerrors": false, "logtostderr": false, "format": null, "subtitleslang": null, "quiet": false, "outtmpl": "%(id)s.%(ext)s", "rejecttitle": null, "playlistend": -1, "writedescription": false, "forcetitle": false, "forcedescription": false, "writesubtitles": false, "matchtitle": null}
+{
+    "consoletitle": false, 
+    "continuedl": true, 
+    "forcedescription": false, 
+    "forcefilename": false, 
+    "forceformat": false, 
+    "forcethumbnail": false, 
+    "forcetitle": false, 
+    "forceurl": false, 
+    "format": null, 
+    "format_limit": null, 
+    "ignoreerrors": false, 
+    "listformats": null, 
+    "logtostderr": false, 
+    "matchtitle": null, 
+    "max_downloads": null, 
+    "nooverwrites": false, 
+    "nopart": false, 
+    "noprogress": false, 
+    "outtmpl": "%(id)s.%(ext)s", 
+    "password": null, 
+    "playlistend": -1, 
+    "playliststart": 1, 
+    "prefer_free_formats": false, 
+    "quiet": false, 
+    "ratelimit": null, 
+    "rejecttitle": null, 
+    "retries": 10, 
+    "simulate": false, 
+    "skip_download": false, 
+    "subtitleslang": null, 
+    "test": true, 
+    "updatetime": true, 
+    "usenetrc": false, 
+    "username": null, 
+    "verbose": true, 
+    "writedescription": false, 
+    "writeinfojson": true, 
+    "writesubtitles": false
+}

test/test_all_urls.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE
+
+class TestAllURLsMatching(unittest.TestCase):
+    def test_youtube_playlist_matching(self):
+        self.assertTrue(YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8'))
+        self.assertTrue(YoutubePlaylistIE().suitable(u'PL63F0C78739B09958'))
+        self.assertFalse(YoutubePlaylistIE().suitable(u'PLtS2H6bU1M'))
+
+    def test_youtube_matching(self):
+        self.assertTrue(YoutubeIE().suitable(u'PLtS2H6bU1M'))
+
+    def test_youtube_extract(self):
+        self.assertEqual(YoutubeIE()._extract_id('http://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
+        self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
+        self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc'), 'BaW_jenozKc')
+
+if __name__ == '__main__':
+    unittest.main()

test/test_download.py

@@ -1,198 +1,125 @@
-#!/usr/bin/env python2
-import unittest
+#!/usr/bin/env python
+
+import errno
 import hashlib
+import io
 import os
 import json
+import unittest
+import sys
+import hashlib
+import socket
 
-from youtube_dl.FileDownloader import FileDownloader
-from youtube_dl.InfoExtractors  import YoutubeIE, DailymotionIE
-from youtube_dl.InfoExtractors import  MetacafeIE, BlipTVIE
-from youtube_dl.InfoExtractors import  XVideosIE, VimeoIE
-from youtube_dl.InfoExtractors import  SoundcloudIE, StanfordOpenClassroomIE
-from youtube_dl.InfoExtractors import  CollegeHumorIE, XNXXIE
+# Allow direct execution
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl.FileDownloader
+import youtube_dl.InfoExtractors
+from youtube_dl.utils import *
+
+DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json')
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+def _try_rm(filename):
+    """ Remove a file if it exists """
+    try:
+        os.remove(filename)
+    except OSError as ose:
+        if ose.errno != errno.ENOENT:
+            raise
+
+class FileDownloader(youtube_dl.FileDownloader):
+    def __init__(self, *args, **kwargs):
+        self.to_stderr = self.to_screen
+        self.processed_info_dicts = []
+        return youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
+    def process_info(self, info_dict):
+        self.processed_info_dicts.append(info_dict)
+        return youtube_dl.FileDownloader.process_info(self, info_dict)
+
+def _file_md5(fn):
+    with open(fn, 'rb') as f:
+        return hashlib.md5(f.read()).hexdigest()
+
+with io.open(DEF_FILE, encoding='utf-8') as deff:
+    defs = json.load(deff)
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)
 
 
-class DownloadTest(unittest.TestCase):
-	PARAMETERS_FILE = "test/parameters.json"
-	#calculated with md5sum:
-	#md5sum (GNU coreutils) 8.19
+class TestDownload(unittest.TestCase):
+    def setUp(self):
+        self.parameters = parameters
+        self.defs = defs
 
-	YOUTUBE_SIZE = 1993883
-	YOUTUBE_URL = "http://www.youtube.com/watch?v=BaW_jenozKc"
-	YOUTUBE_FILE = "BaW_jenozKc.mp4"
+### Dynamically generate tests
+def generator(test_case):
 
-	DAILYMOTION_MD5 = "d363a50e9eb4f22ce90d08d15695bb47"
-	DAILYMOTION_URL = "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech"
-	DAILYMOTION_FILE = "x33vw9.mp4"
+    def test_template(self):
+        ie = getattr(youtube_dl.InfoExtractors, test_case['name'] + 'IE')
+        if not ie._WORKING:
+            print('Skipping: IE marked as not _WORKING')
+            return
+        if 'playlist' not in test_case and not test_case['file']:
+            print('Skipping: No output file specified')
+            return
+        if 'skip' in test_case:
+            print('Skipping: {0}'.format(test_case['skip']))
+            return
 
-	METACAFE_SIZE = 5754305
-	METACAFE_URL = "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/"
-	METACAFE_FILE = "_aUehQsCQtM.flv"
+        params = self.parameters.copy()
+        params.update(test_case.get('params', {}))
 
-	BLIP_MD5 = "93c24d2f4e0782af13b8a7606ea97ba7"
-	BLIP_URL = "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352"
-	BLIP_FILE = "5779306.m4v"
+        fd = FileDownloader(params)
+        fd.add_info_extractor(ie())
+        for ien in test_case.get('add_ie', []):
+            fd.add_info_extractor(getattr(youtube_dl.InfoExtractors, ien + 'IE')())
 
-	XVIDEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
-	XVIDEO_URL = "http://www.xvideos.com/video939581/funny_porns_by_s_-1"
-	XVIDEO_FILE = "939581.flv"
+        test_cases = test_case.get('playlist', [test_case])
+        for tc in test_cases:
+            _try_rm(tc['file'])
+            _try_rm(tc['file'] + '.part')
+            _try_rm(tc['file'] + '.info.json')
+        try:
+            fd.download([test_case['url']])
 
-	VIMEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
-	VIMEO_URL = "http://vimeo.com/14160053"
-	VIMEO_FILE = ""
+            for tc in test_cases:
+                if not test_case.get('params', {}).get('skip_download', False):
+                    self.assertTrue(os.path.exists(tc['file']))
+                self.assertTrue(os.path.exists(tc['file'] + '.info.json'))
+                if 'md5' in tc:
+                    md5_for_file = _file_md5(tc['file'])
+                    self.assertEqual(md5_for_file, tc['md5'])
+                with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof:
+                    info_dict = json.load(infof)
+                for (info_field, value) in tc.get('info_dict', {}).items():
+                    if value.startswith('md5:'):
+                        md5_info_value = hashlib.md5(info_dict.get(info_field, '')).hexdigest()
+                        self.assertEqual(value[3:], md5_info_value)
+                    else:
+                        self.assertEqual(value, info_dict.get(info_field))
+        finally:
+            for tc in test_cases:
+                _try_rm(tc['file'])
+                _try_rm(tc['file'] + '.part')
+                _try_rm(tc['file'] + '.info.json')
 
-	VIMEO2_MD5 = ""
-	VIMEO2_URL = "http://player.vimeo.com/video/47019590"
-	VIMEO2_FILE = ""
+    return test_template
 
-	SOUNDCLOUD_MD5 = "ce3775768ebb6432fa8495d446a078ed"
-	SOUNDCLOUD_URL = "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy"
-	SOUNDCLOUD_FILE = "n6FLbx6ZzMiu.mp3"
+### And add them to TestDownload
+for test_case in defs:
+    test_method = generator(test_case)
+    test_method.__name__ = "test_{0}".format(test_case["name"])
+    setattr(TestDownload, test_method.__name__, test_method)
+    del test_method
 
-	STANDFORD_MD5 = "22c8206291368c4e2c9c1a307f0ea0f4"
-	STANDFORD_URL = "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100"
-	STANDFORD_FILE = "PracticalUnix_intro-environment.mp4"
 
-	COLLEGEHUMOR_MD5 = ""
-	COLLEGEHUMOR_URL = "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody"
-	COLLEGEHUMOR_FILE = ""
-
-	XNXX_MD5 = "5f0469c8d1dfd1bc38c8e6deb5e0a21d"
-	XNXX_URL = "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_"
-	XNXX_FILE = "1135332.flv"
-
-	def test_youtube(self):
-		#let's download a file from youtube
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(YoutubeIE())
-		fd.download([DownloadTest.YOUTUBE_URL])
-		self.assertTrue(os.path.exists(DownloadTest.YOUTUBE_FILE))
-		self.assertEqual(os.path.getsize(DownloadTest.YOUTUBE_FILE), DownloadTest.YOUTUBE_SIZE)
-
-	def test_dailymotion(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(DailymotionIE())
-		fd.download([DownloadTest.DAILYMOTION_URL])
-		self.assertTrue(os.path.exists(DownloadTest.DAILYMOTION_FILE))
-		md5_down_file = md5_for_file(DownloadTest.DAILYMOTION_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.DAILYMOTION_MD5)
-
-	def test_metacafe(self):
-		#this emulate a skip,to be 2.6 compatible
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(MetacafeIE())
-		fd.add_info_extractor(YoutubeIE())
-		fd.download([DownloadTest.METACAFE_URL])
-		self.assertTrue(os.path.exists(DownloadTest.METACAFE_FILE))
-		self.assertEqual(os.path.getsize(DownloadTest.METACAFE_FILE), DownloadTest.METACAFE_SIZE)
-
-	def test_blip(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(BlipTVIE())
-		fd.download([DownloadTest.BLIP_URL])
-		self.assertTrue(os.path.exists(DownloadTest.BLIP_FILE))
-		md5_down_file = md5_for_file(DownloadTest.BLIP_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.BLIP_MD5)
-
-	def test_xvideo(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(XVideosIE())
-		fd.download([DownloadTest.XVIDEO_URL])
-		self.assertTrue(os.path.exists(DownloadTest.XVIDEO_FILE))
-		md5_down_file = md5_for_file(DownloadTest.XVIDEO_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.XVIDEO_MD5)
-
-	def test_vimeo(self):
-		#skipped for the moment produce an error
-		return
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(VimeoIE())
-		fd.download([DownloadTest.VIMEO_URL])
-		self.assertTrue(os.path.exists(DownloadTest.VIMEO_FILE))
-		md5_down_file = md5_for_file(DownloadTest.VIMEO_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.VIMEO_MD5)
-
-	def test_vimeo2(self):
-		#skipped for the moment produce an error
-		return
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(VimeoIE())
-		fd.download([DownloadTest.VIMEO2_URL])
-		self.assertTrue(os.path.exists(DownloadTest.VIMEO2_FILE))
-		md5_down_file = md5_for_file(DownloadTest.VIMEO2_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.VIMEO2_MD5)
-
-	def test_soundcloud(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(SoundcloudIE())
-		fd.download([DownloadTest.SOUNDCLOUD_URL])
-		self.assertTrue(os.path.exists(DownloadTest.SOUNDCLOUD_FILE))
-		md5_down_file = md5_for_file(DownloadTest.SOUNDCLOUD_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.SOUNDCLOUD_MD5)
-
-	def test_standford(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(StanfordOpenClassroomIE())
-		fd.download([DownloadTest.STANDFORD_URL])
-		self.assertTrue(os.path.exists(DownloadTest.STANDFORD_FILE))
-		md5_down_file = md5_for_file(DownloadTest.STANDFORD_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.STANDFORD_MD5)
-
-	def test_collegehumor(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(CollegeHumorIE())
-		fd.download([DownloadTest.COLLEGEHUMOR_URL])
-		self.assertTrue(os.path.exists(DownloadTest.COLLEGEHUMOR_FILE))
-		md5_down_file = md5_for_file(DownloadTest.COLLEGEHUMOR_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.COLLEGEHUMOR_MD5)
-
-	def test_xnxx(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(XNXXIE())
-		fd.download([DownloadTest.XNXX_URL])
-		self.assertTrue(os.path.exists(DownloadTest.XNXX_FILE))
-		md5_down_file = md5_for_file(DownloadTest.XNXX_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.XNXX_MD5)
-
-	def tearDown(self):
-		if os.path.exists(DownloadTest.YOUTUBE_FILE):
-			os.remove(DownloadTest.YOUTUBE_FILE)
-		if os.path.exists(DownloadTest.DAILYMOTION_FILE):
-			os.remove(DownloadTest.DAILYMOTION_FILE)
-		if os.path.exists(DownloadTest.METACAFE_FILE):
-			os.remove(DownloadTest.METACAFE_FILE)
-		if os.path.exists(DownloadTest.BLIP_FILE):
-			os.remove(DownloadTest.BLIP_FILE)
-		if os.path.exists(DownloadTest.XVIDEO_FILE):
-			os.remove(DownloadTest.XVIDEO_FILE)
-		if os.path.exists(DownloadTest.VIMEO_FILE):
-			os.remove(DownloadTest.VIMEO_FILE)
-		if os.path.exists(DownloadTest.SOUNDCLOUD_FILE):
-			os.remove(DownloadTest.SOUNDCLOUD_FILE)
-		if os.path.exists(DownloadTest.STANDFORD_FILE):
-			os.remove(DownloadTest.STANDFORD_FILE)
-		if os.path.exists(DownloadTest.COLLEGEHUMOR_FILE):
-			os.remove(DownloadTest.COLLEGEHUMOR_FILE)
-		if os.path.exists(DownloadTest.XNXX_FILE):
-			os.remove(DownloadTest.XNXX_FILE)
-
-def md5_for_file(filename, block_size=2**20):
-	with open(filename) as f:
-		md5 = hashlib.md5()
-		while True:
-			data = f.read(block_size)
-			if not data:
-				break
-			md5.update(data)
-			return md5.hexdigest()
+if __name__ == '__main__':
+    unittest.main()

test/test_execution.py

@@ -0,0 +1,26 @@
+import unittest
+
+import sys
+import os
+import subprocess
+
+rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+try:
+    _DEV_NULL = subprocess.DEVNULL
+except AttributeError:
+    _DEV_NULL = open(os.devnull, 'wb')
+
+class TestExecution(unittest.TestCase):
+    def test_import(self):
+        subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
+
+    def test_module_exec(self):
+        if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution
+            subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+    def test_main_exec(self):
+        subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+if __name__ == '__main__':
+    unittest.main()

test/test_utils.py

@@ -1,79 +1,100 @@
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python
 
 # Various small unit tests
 
+import sys
 import unittest
 
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 #from youtube_dl.utils import htmlentity_transform
 from youtube_dl.utils import timeconvert
 from youtube_dl.utils import sanitize_filename
 from youtube_dl.utils import unescapeHTML
 from youtube_dl.utils import orderedSet
 
+if sys.version_info < (3, 0):
+    _compat_str = lambda b: b.decode('unicode-escape')
+else:
+    _compat_str = lambda s: s
+
 
 class TestUtil(unittest.TestCase):
-	def test_timeconvert(self):
-		self.assertTrue(timeconvert('') is None)
-		self.assertTrue(timeconvert('bougrg') is None)
+    def test_timeconvert(self):
+        self.assertTrue(timeconvert('') is None)
+        self.assertTrue(timeconvert('bougrg') is None)
 
-	def test_sanitize_filename(self):
-		self.assertEqual(sanitize_filename(u'abc'), u'abc')
-		self.assertEqual(sanitize_filename(u'abc_d-e'), u'abc_d-e')
+    def test_sanitize_filename(self):
+        self.assertEqual(sanitize_filename('abc'), 'abc')
+        self.assertEqual(sanitize_filename('abc_d-e'), 'abc_d-e')
 
-		self.assertEqual(sanitize_filename(u'123'), u'123')
+        self.assertEqual(sanitize_filename('123'), '123')
 
-		self.assertEqual(u'abc_de', sanitize_filename(u'abc/de'))
-		self.assertFalse(u'/' in sanitize_filename(u'abc/de///'))
+        self.assertEqual('abc_de', sanitize_filename('abc/de'))
+        self.assertFalse('/' in sanitize_filename('abc/de///'))
 
-		self.assertEqual(u'abc_de', sanitize_filename(u'abc/<>\\*|de'))
-		self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|'))
-		self.assertEqual(u'yes no', sanitize_filename(u'yes? no'))
-		self.assertEqual(u'this - that', sanitize_filename(u'this: that'))
+        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de'))
+        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|'))
+        self.assertEqual('yes no', sanitize_filename('yes? no'))
+        self.assertEqual('this - that', sanitize_filename('this: that'))
 
-		self.assertEqual(sanitize_filename(u'AT&T'), u'AT&T')
-		self.assertEqual(sanitize_filename(u'ä'), u'ä')
-		self.assertEqual(sanitize_filename(u'кириллица'), u'кириллица')
+        self.assertEqual(sanitize_filename('AT&T'), 'AT&T')
+        aumlaut = _compat_str('\xe4')
+        self.assertEqual(sanitize_filename(aumlaut), aumlaut)
+        tests = _compat_str('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430')
+        self.assertEqual(sanitize_filename(tests), tests)
 
-		forbidden = u'"\0\\/'
-		for fc in forbidden:
-			for fbc in forbidden:
-				self.assertTrue(fbc not in sanitize_filename(fc))
+        forbidden = '"\0\\/'
+        for fc in forbidden:
+            for fbc in forbidden:
+                self.assertTrue(fbc not in sanitize_filename(fc))
 
-	def test_sanitize_filename_restricted(self):
-		self.assertEqual(sanitize_filename(u'abc', restricted=True), u'abc')
-		self.assertEqual(sanitize_filename(u'abc_d-e', restricted=True), u'abc_d-e')
+    def test_sanitize_filename_restricted(self):
+        self.assertEqual(sanitize_filename('abc', restricted=True), 'abc')
+        self.assertEqual(sanitize_filename('abc_d-e', restricted=True), 'abc_d-e')
 
-		self.assertEqual(sanitize_filename(u'123', restricted=True), u'123')
+        self.assertEqual(sanitize_filename('123', restricted=True), '123')
 
-		self.assertEqual(u'abc_de', sanitize_filename(u'abc/de', restricted=True))
-		self.assertFalse(u'/' in sanitize_filename(u'abc/de///', restricted=True))
+        self.assertEqual('abc_de', sanitize_filename('abc/de', restricted=True))
+        self.assertFalse('/' in sanitize_filename('abc/de///', restricted=True))
 
-		self.assertEqual(u'abc_de', sanitize_filename(u'abc/<>\\*|de', restricted=True))
-		self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|', restricted=True))
-		self.assertEqual(u'yes_no', sanitize_filename(u'yes? no', restricted=True))
-		self.assertEqual(u'this_-_that', sanitize_filename(u'this: that', restricted=True))
+        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de', restricted=True))
+        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|', restricted=True))
+        self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True))
+        self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True))
 
-		self.assertEqual(sanitize_filename(u'aäb中国的c', restricted=True), u'a_b_c')
-		self.assertTrue(sanitize_filename(u'ö', restricted=True) != u'') # No empty filename
+        tests = _compat_str('a\xe4b\u4e2d\u56fd\u7684c')
+        self.assertEqual(sanitize_filename(tests, restricted=True), 'a_b_c')
+        self.assertTrue(sanitize_filename(_compat_str('\xf6'), restricted=True) != '')  # No empty filename
 
-		forbidden = u'"\0\\/&!: \'\t\n'
-		for fc in forbidden:
-			for fbc in forbidden:
-				self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
+        forbidden = '"\0\\/&!: \'\t\n()[]{}$;`^,#'
+        for fc in forbidden:
+            for fbc in forbidden:
+                self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
 
-		# Handle a common case more neatly
-		self.assertEqual(sanitize_filename(u'大声带 - Song', restricted=True), u'Song')
-		self.assertEqual(sanitize_filename(u'总统: Speech', restricted=True), u'Speech')
-		# .. but make sure the file name is never empty
-		self.assertTrue(sanitize_filename(u'-', restricted=True) != u'')
-		self.assertTrue(sanitize_filename(u':', restricted=True) != u'')
+        # Handle a common case more neatly
+        self.assertEqual(sanitize_filename(_compat_str('\u5927\u58f0\u5e26 - Song'), restricted=True), 'Song')
+        self.assertEqual(sanitize_filename(_compat_str('\u603b\u7edf: Speech'), restricted=True), 'Speech')
+        # .. but make sure the file name is never empty
+        self.assertTrue(sanitize_filename('-', restricted=True) != '')
+        self.assertTrue(sanitize_filename(':', restricted=True) != '')
 
-	def test_ordered_set(self):
-		self.assertEqual(orderedSet([1,1,2,3,4,4,5,6,7,3,5]), [1,2,3,4,5,6,7])
-		self.assertEqual(orderedSet([]), [])
-		self.assertEqual(orderedSet([1]), [1])
-		#keep the list ordered
-		self.assertEqual(orderedSet([135,1,1,1]), [135,1])
+    def test_sanitize_ids(self):
+        self.assertEqual(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
+        self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
+        self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
 
-	def test_unescape_html(self):
-		self.assertEqual(unescapeHTML(u"%20;"), u"%20;")
+    def test_ordered_set(self):
+        self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
+        self.assertEqual(orderedSet([]), [])
+        self.assertEqual(orderedSet([1]), [1])
+        #keep the list ordered
+        self.assertEqual(orderedSet([135, 1, 1, 1]), [135, 1])
+
+    def test_unescape_html(self):
+        self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;'))
+
+if __name__ == '__main__':
+    unittest.main()

test/test_write_info_json.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import json
+import os
+import sys
+import unittest
+
+# Allow direct execution
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl.FileDownloader
+import youtube_dl.InfoExtractors
+from youtube_dl.utils import *
+
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+class FileDownloader(youtube_dl.FileDownloader):
+    def __init__(self, *args, **kwargs):
+        youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
+        self.to_stderr = self.to_screen
+
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    params = json.load(pf)
+params['writeinfojson'] = True
+params['skip_download'] = True
+params['writedescription'] = True
+
+TEST_ID = 'BaW_jenozKc'
+INFO_JSON_FILE = TEST_ID + '.mp4.info.json'
+DESCRIPTION_FILE = TEST_ID + '.mp4.description'
+EXPECTED_DESCRIPTION = u'''test chars:  "'/\ä↭𝕐
+
+This is a test video for youtube-dl.
+
+For more information, contact phihag@phihag.de .'''
+
+class TestInfoJSON(unittest.TestCase):
+    def setUp(self):
+        # Clear old files
+        self.tearDown()
+
+    def test_info_json(self):
+        ie = youtube_dl.InfoExtractors.YoutubeIE()
+        fd = FileDownloader(params)
+        fd.add_info_extractor(ie)
+        fd.download([TEST_ID])
+        self.assertTrue(os.path.exists(INFO_JSON_FILE))
+        with io.open(INFO_JSON_FILE, 'r', encoding='utf-8') as jsonf:
+            jd = json.load(jsonf)
+        self.assertEqual(jd['upload_date'], u'20121002')
+        self.assertEqual(jd['description'], EXPECTED_DESCRIPTION)
+        self.assertEqual(jd['id'], TEST_ID)
+        self.assertEqual(jd['extractor'], 'youtube')
+        self.assertEqual(jd['title'], u'''youtube-dl test video "'/\ä↭𝕐''')
+        self.assertEqual(jd['uploader'], 'Philipp Hagemeister')
+
+        self.assertTrue(os.path.exists(DESCRIPTION_FILE))
+        with io.open(DESCRIPTION_FILE, 'r', encoding='utf-8') as descf:
+            descr = descf.read()
+        self.assertEqual(descr, EXPECTED_DESCRIPTION)
+
+    def tearDown(self):
+        if os.path.exists(INFO_JSON_FILE):
+            os.remove(INFO_JSON_FILE)
+        if os.path.exists(DESCRIPTION_FILE):
+            os.remove(DESCRIPTION_FILE)
+
+if __name__ == '__main__':
+    unittest.main()

test/test_youtube_lists.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+import json
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeUserIE,YoutubePlaylistIE
+from youtube_dl.utils import *
+
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+class FakeDownloader(object):
+    def __init__(self):
+        self.result = []
+        self.params = parameters
+    def to_screen(self, s):
+        print(s)
+    def trouble(self, s):
+        raise Exception(s)
+    def download(self, x):
+        self.result.append(x)
+
+class TestYoutubeLists(unittest.TestCase):
+    def test_youtube_playlist(self):
+        DL = FakeDownloader()
+        IE = YoutubePlaylistIE(DL)
+        IE.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
+        self.assertEqual(DL.result, [
+            ['http://www.youtube.com/watch?v=bV9L5Ht9LgY'],
+            ['http://www.youtube.com/watch?v=FXxLjLQi3Fg'],
+            ['http://www.youtube.com/watch?v=tU3Bgo5qJZE']
+        ])
+
+    def test_youtube_playlist_long(self):
+        DL = FakeDownloader()
+        IE = YoutubePlaylistIE(DL)
+        IE.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
+        self.assertTrue(len(DL.result) >= 799)
+
+    def test_youtube_course(self):
+        DL = FakeDownloader()
+        IE = YoutubePlaylistIE(DL)
+        # TODO find a > 100 (paginating?) videos course
+        IE.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
+        self.assertEqual(DL.result[0], ['http://www.youtube.com/watch?v=j9WZyLZCBzs'])
+        self.assertEqual(len(DL.result), 25)
+        self.assertEqual(DL.result[-1], ['http://www.youtube.com/watch?v=rYefUsYuEp0'])
+
+    def test_youtube_channel(self):
+        # I give up, please find a channel that does paginate and test this like test_youtube_playlist_long
+        pass # TODO
+
+    def test_youtube_user(self):
+        DL = FakeDownloader()
+        IE = YoutubeUserIE(DL)
+        IE.extract('https://www.youtube.com/user/TheLinuxFoundation')
+        self.assertTrue(len(DL.result) >= 320)
+
+if __name__ == '__main__':
+    unittest.main()

test/test_youtube_subtitles.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+import json
+import io
+import hashlib
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeIE
+from youtube_dl.utils import *
+
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+class FakeDownloader(object):
+    def __init__(self):
+        self.result = []
+        self.params = parameters
+    def to_screen(self, s):
+        print(s)
+    def trouble(self, s):
+        raise Exception(s)
+    def download(self, x):
+        self.result.append(x)
+
+md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+
+class TestYoutubeSubtitles(unittest.TestCase):
+    def test_youtube_subtitles(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        self.assertEqual(md5(info_dict[0]['subtitles']), 'c3228550d59116f3c29fba370b55d033')
+
+    def test_youtube_subtitles_it(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        DL.params['subtitleslang'] = 'it'
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        self.assertEqual(md5(info_dict[0]['subtitles']), '132a88a0daf8e1520f393eb58f1f646a')
+
+if __name__ == '__main__':
+    unittest.main()

test/tests.json

@@ -0,0 +1,198 @@
+[
+  {
+    "name": "Youtube",
+    "url":  "http://www.youtube.com/watch?v=BaW_jenozKc",
+    "file":  "BaW_jenozKc.mp4",
+    "info_dict": {
+      "title": "youtube-dl test video \"'/\\ä↭𝕐",
+      "uploader": "Philipp Hagemeister",
+      "uploader_id": "phihag",
+      "upload_date": "20121002",
+      "description": "test chars:  \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
+    }
+  },
+  {
+    "name": "Dailymotion",
+    "md5":  "392c4b85a60a90dc4792da41ce3144eb",
+    "url":  "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech",
+    "file":  "x33vw9.mp4"
+  },
+  {
+    "name": "Metacafe",
+    "add_ie": ["Youtube"],
+    "url":  "http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
+    "file":  "_aUehQsCQtM.flv"
+  },
+  {
+    "name": "BlipTV",
+    "md5":  "b2d849efcf7ee18917e4b4d9ff37cafe",
+    "url":  "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352",
+    "file":  "5779306.m4v"
+  },
+  {
+    "name": "XVideos",
+    "md5":  "1d0c835822f0a71a7bf011855db929d0",
+    "url":  "http://www.xvideos.com/video939581/funny_porns_by_s_-1",
+    "file":  "939581.flv"
+  },
+  {
+    "name": "Vimeo",
+    "md5":  "8879b6cc097e987f02484baf890129e5",
+    "url":  "http://vimeo.com/56015672",
+    "file": "56015672.mp4",
+    "info_dict": {
+      "title": "youtube-dl test video - ★ \" ' 幸 / \\ ä ↭ 𝕐",
+      "uploader": "Filippo Valsorda",
+      "uploader_id": "user7108434",
+      "upload_date": "20121220",
+      "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: ★ \" ' 幸 / \\ ä ↭ 𝕐"
+    }
+  },
+  {
+    "name": "Soundcloud",
+    "md5":  "ebef0a451b909710ed1d7787dddbf0d7",
+    "url":  "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy",
+    "file":  "62986583.mp3"
+  },
+  {
+    "name": "StanfordOpenClassroom",
+    "md5":  "544a9468546059d4e80d76265b0443b8",
+    "url":  "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100",
+    "file":  "PracticalUnix_intro-environment.mp4"
+  },
+  {
+    "name": "XNXX",
+    "md5":  "0831677e2b4761795f68d417e0b7b445",
+    "url":  "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_",
+    "file":  "1135332.flv"
+  },
+  {
+    "name": "Youku",
+    "url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
+    "file": "XNDgyMDQ2NTQw_part00.flv",
+    "md5": "ffe3f2e435663dc2d1eea34faeff5b5b",
+    "params": { "test": false }
+  },
+  {
+    "name": "NBA",
+    "url": "http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html",
+    "file": "0021200253-okc-bkn-recap.nba.mp4",
+    "md5": "c0edcfc37607344e2ff8f13c378c88a4"
+  },
+  {
+    "name": "JustinTV",
+    "url": "http://www.twitch.tv/thegamedevhub/b/296128360",
+    "file": "296128360.flv",
+    "md5": "ecaa8a790c22a40770901460af191c9a"
+  },
+  {
+    "name": "MyVideo",
+    "url": "http://www.myvideo.de/watch/8229274/bowling_fail_or_win",
+    "file": "8229274.flv",
+    "md5": "2d2753e8130479ba2cb7e0a37002053e"
+  },
+  {
+    "name": "Escapist",
+    "url": "http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate",
+    "file": "6618-Breaking-Down-Baldurs-Gate.flv",
+    "md5": "c6793dbda81388f4264c1ba18684a74d",
+    "skip": "Fails with timeout on Travis"
+  },
+  {
+    "name": "GooglePlus",
+    "url": "https://plus.google.com/u/0/108897254135232129896/posts/ZButuJc6CtH",
+    "file": "ZButuJc6CtH.flv"
+  },
+  {
+    "name": "FunnyOrDie",
+    "url": "http://www.funnyordie.com/videos/0732f586d7/heart-shaped-box-literal-video-version",
+    "file": "0732f586d7.mp4",
+    "md5": "f647e9e90064b53b6e046e75d0241fbd"
+  },
+  {
+    "name": "TweetReel",
+    "url": "http://tweetreel.com/?77smq",
+    "file": "77smq.mov",
+    "md5": "56b4d9ca9de467920f3f99a6d91255d6",
+    "info_dict": {
+        "uploader": "itszero",
+        "uploader_id": "itszero",
+        "upload_date": "20091225",
+        "description": "Installing Gentoo Linux on Powerbook G4, it turns out the sleep indicator becomes HDD activity indicator :D"
+    }
+  },
+  {
+    "name": "Steam",
+    "url": "http://store.steampowered.com/video/105600/",
+    "playlist": [
+      {
+        "file": "81300.flv",
+        "md5": "f870007cee7065d7c76b88f0a45ecc07",
+        "info_dict": {
+            "title": "Terraria 1.1 Trailer"
+        }
+      },
+      {
+        "file": "80859.flv",
+        "md5": "61aaf31a5c5c3041afb58fb83cbb5751",
+        "info_dict": {
+          "title": "Terraria Trailer"
+        }
+      }
+    ]
+  },
+  {
+    "name": "Ustream",
+    "url": "http://www.ustream.tv/recorded/20274954",
+    "file": "20274954.flv",
+    "md5": "088f151799e8f572f84eb62f17d73e5c",
+    "info_dict": {
+        "title": "Young Americans for Liberty February 7, 2012 2:28 AM"
+    }
+  },
+  {
+    "name": "InfoQ",
+    "url": "http://www.infoq.com/presentations/A-Few-of-My-Favorite-Python-Things",
+    "file": "12-jan-pythonthings.mp4",
+    "info_dict": {
+      "title": "A Few of My Favorite [Python] Things"
+    },
+    "params": {
+      "skip_download": true
+    }
+  },
+  {
+    "name": "ComedyCentral",
+    "url": "http://www.thedailyshow.com/full-episodes/thu-december-13-2012-kristen-stewart",
+    "playlist": [
+      {
+        "file": "422204.mp4",
+        "md5": "7a7abe068b31ff03e7b8a37596e72380",
+        "info_dict": {
+            "title": "thedailyshow-thu-december-13-2012-kristen-stewart part 1"
+        }
+      },
+      {
+        "file": "422205.mp4",
+        "md5": "30552b7274c94dbb933f64600eadddd2",
+        "info_dict": {
+            "title": "thedailyshow-thu-december-13-2012-kristen-stewart part 2"
+        }
+      },
+      {
+        "file": "422206.mp4",
+        "md5": "1f4c0664b352cb8e8fe85d5da4fbee91",
+        "info_dict": {
+            "title": "thedailyshow-thu-december-13-2012-kristen-stewart part 3"
+        }
+      },
+      {
+        "file": "422207.mp4",
+        "md5": "f61ee8a4e6bd1308438e03badad78554",
+        "info_dict": {
+            "title": "thedailyshow-thu-december-13-2012-kristen-stewart part 4"
+        }
+      }
+    ]
+  }
+]

youtube-dl.1

@@ -1,306 +0,0 @@
-.TH YOUTUBE-DL 1 "" 
-.SH NAME
-.PP
-youtube-dl
-.SH SYNOPSIS
-.PP
-\f[B]youtube-dl\f[] [OPTIONS] URL [URL...]
-.SH DESCRIPTION
-.PP
-\f[B]youtube-dl\f[] is a small command-line program to download videos
-from YouTube.com and a few more sites.
-It requires the Python interpreter, version 2.x (x being at least 6),
-and it is not platform specific.
-It should work in your Unix box, in Windows or in Mac OS X.
-It is released to the public domain, which means you can modify it,
-redistribute it or use it however you like.
-.SH OPTIONS
-.IP
-.nf
-\f[C]
--h,\ --help\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ this\ help\ text\ and\ exit
---version\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ program\ version\ and\ exit
--U,\ --update\ \ \ \ \ \ \ \ \ \ \ \ \ update\ this\ program\ to\ latest\ version
--i,\ --ignore-errors\ \ \ \ \ \ continue\ on\ download\ errors
--r,\ --rate-limit\ LIMIT\ \ \ download\ rate\ limit\ (e.g.\ 50k\ or\ 44.6m)
--R,\ --retries\ RETRIES\ \ \ \ number\ of\ retries\ (default\ is\ 10)
---buffer-size\ SIZE\ \ \ \ \ \ \ size\ of\ download\ buffer\ (e.g.\ 1024\ or\ 16k)\ (default
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ is\ 1024)
---no-resize-buffer\ \ \ \ \ \ \ do\ not\ automatically\ adjust\ the\ buffer\ size.\ By
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ default,\ the\ buffer\ size\ is\ automatically\ resized
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ from\ an\ initial\ value\ of\ SIZE.
---dump-user-agent\ \ \ \ \ \ \ \ display\ the\ current\ browser\ identification
---user-agent\ UA\ \ \ \ \ \ \ \ \ \ specify\ a\ custom\ user\ agent
---list-extractors\ \ \ \ \ \ \ \ List\ all\ supported\ extractors\ and\ the\ URLs\ they
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ would\ handle
-\f[]
-.fi
-.SS Video Selection:
-.IP
-.nf
-\f[C]
---playlist-start\ NUMBER\ \ playlist\ video\ to\ start\ at\ (default\ is\ 1)
---playlist-end\ NUMBER\ \ \ \ playlist\ video\ to\ end\ at\ (default\ is\ last)
---match-title\ REGEX\ \ \ \ \ \ download\ only\ matching\ titles\ (regex\ or\ caseless
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ sub-string)
---reject-title\ REGEX\ \ \ \ \ skip\ download\ for\ matching\ titles\ (regex\ or
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ caseless\ sub-string)
---max-downloads\ NUMBER\ \ \ Abort\ after\ downloading\ NUMBER\ files
-\f[]
-.fi
-.SS Filesystem Options:
-.IP
-.nf
-\f[C]
--t,\ --title\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ title\ in\ file\ name
---id\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ video\ ID\ in\ file\ name
--l,\ --literal\ \ \ \ \ \ \ \ \ \ \ \ [deprecated]\ alias\ of\ --title
--A,\ --auto-number\ \ \ \ \ \ \ \ number\ downloaded\ files\ starting\ from\ 00000
--o,\ --output\ TEMPLATE\ \ \ \ output\ filename\ template.\ Use\ %(title)s\ to\ get\ the
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ title,\ %(uploader)s\ for\ the\ uploader\ name,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(autonumber)s\ to\ get\ an\ automatically\ incremented
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ number,\ %(ext)s\ for\ the\ filename\ extension,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(upload_date)s\ for\ the\ upload\ date\ (YYYYMMDD),
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(extractor)s\ for\ the\ provider\ (youtube,\ metacafe,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ etc),\ %(id)s\ for\ the\ video\ id\ and\ %%\ for\ a\ literal
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ percent.\ Use\ -\ to\ output\ to\ stdout.
---restrict-filenames\ \ \ \ \ Restrict\ filenames\ to\ only\ ASCII\ characters,\ and
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ avoid\ "&"\ and\ spaces\ in\ filenames
--a,\ --batch-file\ FILE\ \ \ \ file\ containing\ URLs\ to\ download\ (\[aq]-\[aq]\ for\ stdin)
--w,\ --no-overwrites\ \ \ \ \ \ do\ not\ overwrite\ files
--c,\ --continue\ \ \ \ \ \ \ \ \ \ \ resume\ partially\ downloaded\ files
---no-continue\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ resume\ partially\ downloaded\ files\ (restart
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ from\ beginning)
---cookies\ FILE\ \ \ \ \ \ \ \ \ \ \ file\ to\ read\ cookies\ from\ and\ dump\ cookie\ jar\ in
---no-part\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ .part\ files
---no-mtime\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ the\ Last-modified\ header\ to\ set\ the\ file
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ modification\ time
---write-description\ \ \ \ \ \ write\ video\ description\ to\ a\ .description\ file
---write-info-json\ \ \ \ \ \ \ \ write\ video\ metadata\ to\ a\ .info.json\ file
-\f[]
-.fi
-.SS Verbosity / Simulation Options:
-.IP
-.nf
-\f[C]
--q,\ --quiet\ \ \ \ \ \ \ \ \ \ \ \ \ \ activates\ quiet\ mode
--s,\ --simulate\ \ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video\ and\ do\ not\ write\ anything
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ to\ disk
---skip-download\ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video
--g,\ --get-url\ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ URL
--e,\ --get-title\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ title
---get-thumbnail\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ thumbnail\ URL
---get-description\ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ video\ description
---get-filename\ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ filename
---get-format\ \ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ format
---no-progress\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ print\ progress\ bar
---console-title\ \ \ \ \ \ \ \ \ \ display\ progress\ in\ console\ titlebar
--v,\ --verbose\ \ \ \ \ \ \ \ \ \ \ \ print\ various\ debugging\ information
-\f[]
-.fi
-.SS Video Format Options:
-.IP
-.nf
-\f[C]
--f,\ --format\ FORMAT\ \ \ \ \ \ video\ format\ code
---all-formats\ \ \ \ \ \ \ \ \ \ \ \ download\ all\ available\ video\ formats
---prefer-free-formats\ \ \ \ prefer\ free\ video\ formats\ unless\ a\ specific\ one\ is
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ requested
---max-quality\ FORMAT\ \ \ \ \ highest\ quality\ format\ to\ download
--F,\ --list-formats\ \ \ \ \ \ \ list\ all\ available\ formats\ (currently\ youtube\ only)
---write-srt\ \ \ \ \ \ \ \ \ \ \ \ \ \ write\ video\ closed\ captions\ to\ a\ .srt\ file
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (currently\ youtube\ only)
---srt-lang\ LANG\ \ \ \ \ \ \ \ \ \ language\ of\ the\ closed\ captions\ to\ download
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (optional)\ use\ IETF\ language\ tags\ like\ \[aq]en\[aq]
-\f[]
-.fi
-.SS Authentication Options:
-.IP
-.nf
-\f[C]
--u,\ --username\ USERNAME\ \ account\ username
--p,\ --password\ PASSWORD\ \ account\ password
--n,\ --netrc\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ .netrc\ authentication\ data
-\f[]
-.fi
-.SS Post-processing Options:
-.IP
-.nf
-\f[C]
--x,\ --extract-audio\ \ \ \ \ \ convert\ video\ files\ to\ audio-only\ files\ (requires
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ffmpeg\ or\ avconv\ and\ ffprobe\ or\ avprobe)
---audio-format\ FORMAT\ \ \ \ "best",\ "aac",\ "vorbis",\ "mp3",\ "m4a",\ or\ "wav";
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ best\ by\ default
---audio-quality\ QUALITY\ \ ffmpeg/avconv\ audio\ quality\ specification,\ insert\ a
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ value\ between\ 0\ (better)\ and\ 9\ (worse)\ for\ VBR\ or\ a
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ specific\ bitrate\ like\ 128K\ (default\ 5)
--k,\ --keep-video\ \ \ \ \ \ \ \ \ keeps\ the\ video\ file\ on\ disk\ after\ the\ post-
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ processing;\ the\ video\ is\ erased\ by\ default
-\f[]
-.fi
-.SH CONFIGURATION
-.PP
-You can configure youtube-dl by placing default arguments (such as
-\f[C]--extract-audio\ --no-mtime\f[] to always extract the audio and not
-copy the mtime) into \f[C]/etc/youtube-dl.conf\f[] and/or
-\f[C]~/.local/config/youtube-dl.conf\f[].
-.SH OUTPUT TEMPLATE
-.PP
-The \f[C]-o\f[] option allows users to indicate a template for the
-output file names.
-The basic usage is not to set any template arguments when downloading a
-single file, like in
-\f[C]youtube-dl\ -o\ funny_video.flv\ "http://some/video"\f[].
-However, it may contain special sequences that will be replaced when
-downloading each video.
-The special sequences have the format \f[C]%(NAME)s\f[].
-To clarify, that is a percent symbol followed by a name in parenthesis,
-followed by a lowercase S.
-Allowed names are:
-.IP \[bu] 2
-\f[C]id\f[]: The sequence will be replaced by the video identifier.
-.IP \[bu] 2
-\f[C]url\f[]: The sequence will be replaced by the video URL.
-.IP \[bu] 2
-\f[C]uploader\f[]: The sequence will be replaced by the nickname of the
-person who uploaded the video.
-.IP \[bu] 2
-\f[C]upload_date\f[]: The sequence will be replaced by the upload date
-in YYYYMMDD format.
-.IP \[bu] 2
-\f[C]title\f[]: The sequence will be replaced by the video title.
-.IP \[bu] 2
-\f[C]ext\f[]: The sequence will be replaced by the appropriate extension
-(like flv or mp4).
-.IP \[bu] 2
-\f[C]epoch\f[]: The sequence will be replaced by the Unix epoch when
-creating the file.
-.IP \[bu] 2
-\f[C]autonumber\f[]: The sequence will be replaced by a five-digit
-number that will be increased with each download, starting at zero.
-.PP
-The current default template is \f[C]%(id)s.%(ext)s\f[], but that will
-be switchted to \f[C]%(title)s-%(id)s.%(ext)s\f[] (which can be
-requested with \f[C]-t\f[] at the moment).
-.PP
-In some cases, you don\[aq]t want special characters such as 中, spaces,
-or &, such as when transferring the downloaded filename to a Windows
-system or the filename through an 8bit-unsafe channel.
-In these cases, add the \f[C]--restrict-filenames\f[] flag to get a
-shorter title:
-.IP
-.nf
-\f[C]
-$\ youtube-dl\ --get-filename\ -o\ "%(title)s.%(ext)s"\ BaW_jenozKc
-youtube-dl\ test\ video\ \[aq]\[aq]_ä↭𝕐.mp4\ \ \ \ #\ All\ kinds\ of\ weird\ characters
-$\ youtube-dl\ --get-filename\ -o\ "%(title)s.%(ext)s"\ BaW_jenozKc\ --restrict-filenames
-youtube-dl_test_video_.mp4\ \ \ \ \ \ \ \ \ \ #\ A\ simple\ file\ name
-\f[]
-.fi
-.SH FAQ
-.SS Can you please put the -b option back?
-.PP
-Most people asking this question are not aware that youtube-dl now
-defaults to downloading the highest available quality as reported by
-YouTube, which will be 1080p or 720p in some cases, so you no longer
-need the -b option.
-For some specific videos, maybe YouTube does not report them to be
-available in a specific high quality format you\[aq]\[aq]re interested
-in.
-In that case, simply request it with the -f option and youtube-dl will
-try to download it.
-.SS I get HTTP error 402 when trying to download a video. What\[aq]s
-this?
-.PP
-Apparently YouTube requires you to pass a CAPTCHA test if you download
-too much.
-We\[aq]\[aq]re considering to provide a way to let you solve the
-CAPTCHA (https://github.com/rg3/youtube-dl/issues/154), but at the
-moment, your best course of action is pointing a webbrowser to the
-youtube URL, solving the CAPTCHA, and restart youtube-dl.
-.SS I have downloaded a video but how can I play it?
-.PP
-Once the video is fully downloaded, use any video player, such as
-vlc (http://www.videolan.org) or mplayer (http://www.mplayerhq.hu/).
-.SS The links provided by youtube-dl -g are not working anymore
-.PP
-The URLs youtube-dl outputs require the downloader to have the correct
-cookies.
-Use the \f[C]--cookies\f[] option to write the required cookies into a
-file, and advise your downloader to read cookies from that file.
-Some sites also require a common user agent to be used, use
-\f[C]--dump-user-agent\f[] to see the one in use by youtube-dl.
-.SS ERROR: no fmt_url_map or conn information found in video info
-.PP
-youtube has switched to a new video info format in July 2011 which is
-not supported by old versions of youtube-dl.
-You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
-.SS ERROR: unable to download video
-.PP
-youtube requires an additional signature since September 2012 which is
-not supported by old versions of youtube-dl.
-You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
-.SS SyntaxError: Non-ASCII character
-.PP
-The error
-.IP
-.nf
-\f[C]
-File\ "youtube-dl",\ line\ 2
-SyntaxError:\ Non-ASCII\ character\ \[aq]\\x93\[aq]\ ...
-\f[]
-.fi
-.PP
-means you\[aq]re using an outdated version of Python.
-Please update to Python 2.6 or 2.7.
-.PP
-To run youtube-dl under Python 2.5, you\[aq]ll have to manually check it
-out like this:
-.IP
-.nf
-\f[C]
-git\ clone\ git://github.com/rg3/youtube-dl.git
-cd\ youtube-dl
-python\ -m\ youtube_dl\ --help
-\f[]
-.fi
-.PP
-Please note that Python 2.5 is not supported anymore.
-.SS What is this binary file? Where has the code gone?
-.PP
-Since June 2012 (#342) youtube-dl is packed as an executable zipfile,
-simply unzip it (might need renaming to \f[C]youtube-dl.zip\f[] first on
-some systems) or clone the git repository, as laid out above.
-If you modify the code, you can run it by executing the
-\f[C]__main__.py\f[] file.
-To recompile the executable, run \f[C]make\ youtube-dl\f[].
-.SS The exe throws a \f[I]Runtime error from Visual C++\f[]
-.PP
-To run the exe you need to install first the Microsoft Visual C++ 2008
-Redistributable
-Package (http://www.microsoft.com/en-us/download/details.aspx?id=29).
-.SH COPYRIGHT
-.PP
-youtube-dl is released into the public domain by the copyright holders.
-.PP
-This README file was originally written by Daniel Bolton
-(<https://github.com/dbbolton>) and is likewise released into the public
-domain.
-.SH BUGS
-.PP
-Bugs and suggestions should be reported at:
-<https://github.com/rg3/youtube-dl/issues>
-.PP
-Please include:
-.IP \[bu] 2
-Your exact command line, like
-\f[C]youtube-dl\ -t\ "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"\f[].
-A common mistake is not to escape the \f[C]&\f[].
-Putting URLs in quotes should solve this problem.
-.IP \[bu] 2
-The output of \f[C]youtube-dl\ --version\f[]
-.IP \[bu] 2
-The output of \f[C]python\ --version\f[]
-.IP \[bu] 2
-The name and version of your Operating System ("Ubuntu 11.04 x64" or
-"Windows 7 x64" is usually enough).

youtube-dl.bash-completion

@@ -1,14 +0,0 @@
-__youtube-dl()
-{
-    local cur prev opts
-    COMPREPLY=()
-    cur="${COMP_WORDS[COMP_CWORD]}"
-    opts="--all-formats --audio-format --audio-quality --auto-number --batch-file --buffer-size --console-title --continue --cookies --dump-user-agent --extract-audio --format --get-description --get-filename --get-format --get-thumbnail --get-title --get-url --help --id --ignore-errors --keep-video --list-extractors --list-formats --literal --match-title --max-downloads --max-quality --netrc --no-continue --no-mtime --no-overwrites --no-part --no-progress --no-resize-buffer --output --password --playlist-end --playlist-start --prefer-free-formats --quiet --rate-limit --reject-title --restrict-filenames --retries --simulate --skip-download --srt-lang --title --update --user-agent --username --verbose --version --write-description --write-info-json --write-srt"
-
-    if [[ ${cur} == * ]] ; then
-        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
-        return 0
-    fi
-}
-
-complete -F __youtube-dl youtube-dl

youtube_dl/PostProcessor.py

@@ -1,198 +1,211 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
+
 import os
 import subprocess
 import sys
 import time
 
-from utils import *
+from .utils import *
 
 
 class PostProcessor(object):
-	"""Post Processor class.
+    """Post Processor class.
 
-	PostProcessor objects can be added to downloaders with their
-	add_post_processor() method. When the downloader has finished a
-	successful download, it will take its internal chain of PostProcessors
-	and start calling the run() method on each one of them, first with
-	an initial argument and then with the returned value of the previous
-	PostProcessor.
+    PostProcessor objects can be added to downloaders with their
+    add_post_processor() method. When the downloader has finished a
+    successful download, it will take its internal chain of PostProcessors
+    and start calling the run() method on each one of them, first with
+    an initial argument and then with the returned value of the previous
+    PostProcessor.
 
-	The chain will be stopped if one of them ever returns None or the end
-	of the chain is reached.
+    The chain will be stopped if one of them ever returns None or the end
+    of the chain is reached.
 
-	PostProcessor objects follow a "mutual registration" process similar
-	to InfoExtractor objects.
-	"""
+    PostProcessor objects follow a "mutual registration" process similar
+    to InfoExtractor objects.
+    """
 
-	_downloader = None
+    _downloader = None
 
-	def __init__(self, downloader=None):
-		self._downloader = downloader
+    def __init__(self, downloader=None):
+        self._downloader = downloader
 
-	def set_downloader(self, downloader):
-		"""Sets the downloader for this PP."""
-		self._downloader = downloader
+    def set_downloader(self, downloader):
+        """Sets the downloader for this PP."""
+        self._downloader = downloader
 
-	def run(self, information):
-		"""Run the PostProcessor.
+    def run(self, information):
+        """Run the PostProcessor.
 
-		The "information" argument is a dictionary like the ones
-		composed by InfoExtractors. The only difference is that this
-		one has an extra field called "filepath" that points to the
-		downloaded file.
+        The "information" argument is a dictionary like the ones
+        composed by InfoExtractors. The only difference is that this
+        one has an extra field called "filepath" that points to the
+        downloaded file.
 
-		When this method returns None, the postprocessing chain is
-		stopped. However, this method may return an information
-		dictionary that will be passed to the next postprocessing
-		object in the chain. It can be the one it received after
-		changing some fields.
+        When this method returns None, the postprocessing chain is
+        stopped. However, this method may return an information
+        dictionary that will be passed to the next postprocessing
+        object in the chain. It can be the one it received after
+        changing some fields.
 
-		In addition, this method may raise a PostProcessingError
-		exception that will be taken into account by the downloader
-		it was called from.
-		"""
-		return information # by default, do nothing
+        In addition, this method may raise a PostProcessingError
+        exception that will be taken into account by the downloader
+        it was called from.
+        """
+        return information # by default, do nothing
 
 class AudioConversionError(BaseException):
-	def __init__(self, message):
-		self.message = message
+    def __init__(self, message):
+        self.message = message
 
 class FFmpegExtractAudioPP(PostProcessor):
-	def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
-		PostProcessor.__init__(self, downloader)
-		if preferredcodec is None:
-			preferredcodec = 'best'
-		self._preferredcodec = preferredcodec
-		self._preferredquality = preferredquality
-		self._keepvideo = keepvideo
-		self._exes = self.detect_executables()
+    def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False, nopostoverwrites=False):
+        PostProcessor.__init__(self, downloader)
+        if preferredcodec is None:
+            preferredcodec = 'best'
+        self._preferredcodec = preferredcodec
+        self._preferredquality = preferredquality
+        self._keepvideo = keepvideo
+        self._nopostoverwrites = nopostoverwrites
+        self._exes = self.detect_executables()
 
-	@staticmethod
-	def detect_executables():
-		def executable(exe):
-			try:
-				subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-			except OSError:
-				return False
-			return exe
-		programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-		return dict((program, executable(program)) for program in programs)
+    @staticmethod
+    def detect_executables():
+        def executable(exe):
+            try:
+                subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+            except OSError:
+                return False
+            return exe
+        programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
+        return dict((program, executable(program)) for program in programs)
 
-	def get_audio_codec(self, path):
-		if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
-		try:
-			cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)]
-			handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
-			output = handle.communicate()[0]
-			if handle.wait() != 0:
-				return None
-		except (IOError, OSError):
-			return None
-		audio_codec = None
-		for line in output.split('\n'):
-			if line.startswith('codec_name='):
-				audio_codec = line.split('=')[1].strip()
-			elif line.strip() == 'codec_type=audio' and audio_codec is not None:
-				return audio_codec
-		return None
+    def get_audio_codec(self, path):
+        if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
+        try:
+            cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', encodeFilename(self._ffmpeg_filename_argument(path))]
+            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
+            output = handle.communicate()[0]
+            if handle.wait() != 0:
+                return None
+        except (IOError, OSError):
+            return None
+        audio_codec = None
+        for line in output.decode('ascii', 'ignore').split('\n'):
+            if line.startswith('codec_name='):
+                audio_codec = line.split('=')[1].strip()
+            elif line.strip() == 'codec_type=audio' and audio_codec is not None:
+                return audio_codec
+        return None
 
-	def run_ffmpeg(self, path, out_path, codec, more_opts):
-		if not self._exes['ffmpeg'] and not self._exes['avconv']:
-			raise AudioConversionError('ffmpeg or avconv not found. Please install one.')	
-		if codec is None:
-			acodec_opts = []
-		else:
-			acodec_opts = ['-acodec', codec]
-		cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn']
-			   + acodec_opts + more_opts +
-			   ['--', encodeFilename(out_path)])
-		p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-		stdout,stderr = p.communicate()
-		if p.returncode != 0:
-			msg = stderr.strip().split('\n')[-1]
-			raise AudioConversionError(msg)
+    def run_ffmpeg(self, path, out_path, codec, more_opts):
+        if not self._exes['ffmpeg'] and not self._exes['avconv']:
+            raise AudioConversionError('ffmpeg or avconv not found. Please install one.')
+        if codec is None:
+            acodec_opts = []
+        else:
+            acodec_opts = ['-acodec', codec]
+        cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn']
+               + acodec_opts + more_opts +
+               [encodeFilename(self._ffmpeg_filename_argument(out_path))])
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout,stderr = p.communicate()
+        if p.returncode != 0:
+            msg = stderr.strip().split('\n')[-1]
+            raise AudioConversionError(msg)
 
-	def run(self, information):
-		path = information['filepath']
+    def run(self, information):
+        path = information['filepath']
 
-		filecodec = self.get_audio_codec(path)
-		if filecodec is None:
-			self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
-			return None
+        filecodec = self.get_audio_codec(path)
+        if filecodec is None:
+            self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
+            return None
 
-		more_opts = []
-		if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
-			if self._preferredcodec == 'm4a' and filecodec == 'aac':
-				# Lossless, but in another container
-				acodec = 'copy'
-				extension = self._preferredcodec
-				more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
-			elif filecodec in ['aac', 'mp3', 'vorbis']:
-				# Lossless if possible
-				acodec = 'copy'
-				extension = filecodec
-				if filecodec == 'aac':
-					more_opts = ['-f', 'adts']
-				if filecodec == 'vorbis':
-					extension = 'ogg'
-			else:
-				# MP3 otherwise.
-				acodec = 'libmp3lame'
-				extension = 'mp3'
-				more_opts = []
-				if self._preferredquality is not None:
-					if int(self._preferredquality) < 10:
-						more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
-					else:
-						more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
-		else:
-			# We convert the audio (lossy)
-			acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
-			extension = self._preferredcodec
-			more_opts = []
-			if self._preferredquality is not None:
-				if int(self._preferredquality) < 10:
-					more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
-				else:
-					more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
-			if self._preferredcodec == 'aac':
-				more_opts += ['-f', 'adts']
-			if self._preferredcodec == 'm4a':
-				more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
-			if self._preferredcodec == 'vorbis':
-				extension = 'ogg'
-			if self._preferredcodec == 'wav':
-				extension = 'wav'
-				more_opts += ['-f', 'wav']
+        more_opts = []
+        if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
+            if self._preferredcodec == 'm4a' and filecodec == 'aac':
+                # Lossless, but in another container
+                acodec = 'copy'
+                extension = self._preferredcodec
+                more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
+            elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
+                # Lossless if possible
+                acodec = 'copy'
+                extension = filecodec
+                if filecodec == 'aac':
+                    more_opts = ['-f', 'adts']
+                if filecodec == 'vorbis':
+                    extension = 'ogg'
+            else:
+                # MP3 otherwise.
+                acodec = 'libmp3lame'
+                extension = 'mp3'
+                more_opts = []
+                if self._preferredquality is not None:
+                    if int(self._preferredquality) < 10:
+                        more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+                    else:
+                        more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
+        else:
+            # We convert the audio (lossy)
+            acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
+            extension = self._preferredcodec
+            more_opts = []
+            if self._preferredquality is not None:
+                if int(self._preferredquality) < 10:
+                    more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+                else:
+                    more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
+            if self._preferredcodec == 'aac':
+                more_opts += ['-f', 'adts']
+            if self._preferredcodec == 'm4a':
+                more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
+            if self._preferredcodec == 'vorbis':
+                extension = 'ogg'
+            if self._preferredcodec == 'wav':
+                extension = 'wav'
+                more_opts += ['-f', 'wav']
 
-		prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
-		new_path = prefix + sep + extension
-		self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
-		try:
-			self.run_ffmpeg(path, new_path, acodec, more_opts)
-		except:
-			etype,e,tb = sys.exc_info()
-			if isinstance(e, AudioConversionError):
-				self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message)
-			else:
-				self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg'))
-			return None
+        prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
+        new_path = prefix + sep + extension
+        try:
+            if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
+                self._downloader.to_screen(u'[youtube] Post-process file %s exists, skipping' % new_path)
+            else:
+                self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
+                self.run_ffmpeg(path, new_path, acodec, more_opts)
+        except:
+            etype,e,tb = sys.exc_info()
+            if isinstance(e, AudioConversionError):
+                self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message)
+            else:
+                self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg'))
+            return None
 
- 		# Try to update the date time for extracted audio file.
-		if information.get('filetime') is not None:
-			try:
-				os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
-			except:
-				self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
+        # Try to update the date time for extracted audio file.
+        if information.get('filetime') is not None:
+            try:
+                os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
+            except:
+                self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
 
-		if not self._keepvideo:
-			try:
-				os.remove(encodeFilename(path))
-			except (IOError, OSError):
-				self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
-				return None
+        if not self._keepvideo:
+            try:
+                os.remove(encodeFilename(path))
+            except (IOError, OSError):
+                self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
+                return None
+
+        information['filepath'] = new_path
+        return information
+
+    def _ffmpeg_filename_argument(self, fn):
+        # ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details
+        if fn.startswith(u'-'):
+            return u'./' + fn
+        return fn
 
-		information['filepath'] = new_path
-		return information

youtube_dl/__init__.py

@@ -2,34 +2,30 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import with_statement
+from __future__ import absolute_import
 
 __authors__  = (
-	'Ricardo Garcia Gonzalez',
-	'Danny Colligan',
-	'Benjamin Johnson',
-	'Vasyl\' Vavrychuk',
-	'Witold Baryluk',
-	'Paweł Paprota',
-	'Gergely Imreh',
-	'Rogério Brito',
-	'Philipp Hagemeister',
-	'Sören Schulze',
-	'Kevin Ngo',
-	'Ori Avtalion',
-	'shizeeg',
-	'Filippo Valsorda',
-	'Christian Albrecht',
-	)
+    'Ricardo Garcia Gonzalez',
+    'Danny Colligan',
+    'Benjamin Johnson',
+    'Vasyl\' Vavrychuk',
+    'Witold Baryluk',
+    'Paweł Paprota',
+    'Gergely Imreh',
+    'Rogério Brito',
+    'Philipp Hagemeister',
+    'Sören Schulze',
+    'Kevin Ngo',
+    'Ori Avtalion',
+    'shizeeg',
+    'Filippo Valsorda',
+    'Christian Albrecht',
+    'Dave Vasilevsky',
+    'Jaime Marquínez Ferrándiz',
+    )
 
 __license__ = 'Public Domain'
-__version__ = '2012.11.29'
 
-UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
-UPDATE_URL_VERSION = 'https://raw.github.com/rg3/youtube-dl/master/LATEST_VERSION'
-UPDATE_URL_EXE = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl.exe'
-
-
-import cookielib
 import getpass
 import optparse
 import os
@@ -38,535 +34,459 @@ import shlex
 import socket
 import subprocess
 import sys
-import urllib2
 import warnings
+import platform
 
-from utils import *
-from FileDownloader import *
-from InfoExtractors import *
-from PostProcessor import *
-
-def updateSelf(downloader, filename):
-	''' Update the program file with the latest version from the repository '''
-	# Note: downloader only used for options
-
-	if not os.access(filename, os.W_OK):
-		sys.exit('ERROR: no write permissions on %s' % filename)
-
-	downloader.to_screen(u'Updating to latest version...')
-
-	urlv = urllib2.urlopen(UPDATE_URL_VERSION)
-	newversion = urlv.read().strip()
-	if newversion == __version__:
-		downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
-		return
-	urlv.close()
-
-	if hasattr(sys, "frozen"): #py2exe
-		exe = os.path.abspath(filename)
-		directory = os.path.dirname(exe)
-		if not os.access(directory, os.W_OK):
-			sys.exit('ERROR: no write permissions on %s' % directory)
-
-		try:
-			urlh = urllib2.urlopen(UPDATE_URL_EXE)
-			newcontent = urlh.read()
-			urlh.close()
-			with open(exe + '.new', 'wb') as outf:
-				outf.write(newcontent)
-		except (IOError, OSError), err:
-			sys.exit('ERROR: unable to download latest version')
-
-		try:
-			bat = os.path.join(directory, 'youtube-dl-updater.bat')
-			b = open(bat, 'w')
-			b.write("""
-echo Updating youtube-dl...
-ping 127.0.0.1 -n 5 -w 1000 > NUL
-move /Y "%s.new" "%s"
-del "%s"
-			\n""" %(exe, exe, bat))
-			b.close()
-
-			os.startfile(bat)
-		except (IOError, OSError), err:
-			sys.exit('ERROR: unable to overwrite current version')
-
-	else:
-		try:
-			urlh = urllib2.urlopen(UPDATE_URL)
-			newcontent = urlh.read()
-			urlh.close()
-		except (IOError, OSError), err:
-			sys.exit('ERROR: unable to download latest version')
-
-		try:
-			with open(filename, 'wb') as outf:
-				outf.write(newcontent)
-		except (IOError, OSError), err:
-			sys.exit('ERROR: unable to overwrite current version')
-
-	downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
+from .utils import *
+from .update import update_self
+from .version import __version__
+from .FileDownloader import *
+from .InfoExtractors import gen_extractors
+from .PostProcessor import *
 
 def parseOpts():
-	def _readOptions(filename_bytes):
-		try:
-			optionf = open(filename_bytes)
-		except IOError:
-			return [] # silently skip if file is not present
-		try:
-			res = []
-			for l in optionf:
-				res += shlex.split(l, comments=True)
-		finally:
-			optionf.close()
-		return res
+    def _readOptions(filename_bytes):
+        try:
+            optionf = open(filename_bytes)
+        except IOError:
+            return [] # silently skip if file is not present
+        try:
+            res = []
+            for l in optionf:
+                res += shlex.split(l, comments=True)
+        finally:
+            optionf.close()
+        return res
 
-	def _format_option_string(option):
-		''' ('-o', '--option') -> -o, --format METAVAR'''
+    def _format_option_string(option):
+        ''' ('-o', '--option') -> -o, --format METAVAR'''
 
-		opts = []
+        opts = []
 
-		if option._short_opts: opts.append(option._short_opts[0])
-		if option._long_opts: opts.append(option._long_opts[0])
-		if len(opts) > 1: opts.insert(1, ', ')
+        if option._short_opts:
+            opts.append(option._short_opts[0])
+        if option._long_opts:
+            opts.append(option._long_opts[0])
+        if len(opts) > 1:
+            opts.insert(1, ', ')
 
-		if option.takes_value(): opts.append(' %s' % option.metavar)
+        if option.takes_value(): opts.append(' %s' % option.metavar)
 
-		return "".join(opts)
+        return "".join(opts)
 
-	def _find_term_columns():
-		columns = os.environ.get('COLUMNS', None)
-		if columns:
-			return int(columns)
+    def _find_term_columns():
+        columns = os.environ.get('COLUMNS', None)
+        if columns:
+            return int(columns)
 
-		try:
-			sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-			out,err = sp.communicate()
-			return int(out.split()[1])
-		except:
-			pass
-		return None
+        try:
+            sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            out,err = sp.communicate()
+            return int(out.split()[1])
+        except:
+            pass
+        return None
 
-	max_width = 80
-	max_help_position = 80
+    max_width = 80
+    max_help_position = 80
 
-	# No need to wrap help messages if we're on a wide console
-	columns = _find_term_columns()
-	if columns: max_width = columns
+    # No need to wrap help messages if we're on a wide console
+    columns = _find_term_columns()
+    if columns: max_width = columns
 
-	fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
-	fmt.format_option_strings = _format_option_string
+    fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
+    fmt.format_option_strings = _format_option_string
 
-	kw = {
-		'version'   : __version__,
-		'formatter' : fmt,
-		'usage' : '%prog [options] url [url...]',
-		'conflict_handler' : 'resolve',
-	}
+    kw = {
+        'version'   : __version__,
+        'formatter' : fmt,
+        'usage' : '%prog [options] url [url...]',
+        'conflict_handler' : 'resolve',
+    }
 
-	parser = optparse.OptionParser(**kw)
+    parser = optparse.OptionParser(**kw)
 
-	# option groups
-	general        = optparse.OptionGroup(parser, 'General Options')
-	selection      = optparse.OptionGroup(parser, 'Video Selection')
-	authentication = optparse.OptionGroup(parser, 'Authentication Options')
-	video_format   = optparse.OptionGroup(parser, 'Video Format Options')
-	postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
-	filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
-	verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
+    # option groups
+    general        = optparse.OptionGroup(parser, 'General Options')
+    selection      = optparse.OptionGroup(parser, 'Video Selection')
+    authentication = optparse.OptionGroup(parser, 'Authentication Options')
+    video_format   = optparse.OptionGroup(parser, 'Video Format Options')
+    postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
+    filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
+    verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 
-	general.add_option('-h', '--help',
-			action='help', help='print this help text and exit')
-	general.add_option('-v', '--version',
-			action='version', help='print program version and exit')
-	general.add_option('-U', '--update',
-			action='store_true', dest='update_self', help='update this program to latest version')
-	general.add_option('-i', '--ignore-errors',
-			action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
-	general.add_option('-r', '--rate-limit',
-			dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
-	general.add_option('-R', '--retries',
-			dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
-	general.add_option('--buffer-size',
-			dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024")
-	general.add_option('--no-resize-buffer',
-			action='store_true', dest='noresizebuffer',
-			help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
-	general.add_option('--dump-user-agent',
-			action='store_true', dest='dump_user_agent',
-			help='display the current browser identification', default=False)
-	general.add_option('--user-agent',
-			dest='user_agent', help='specify a custom user agent', metavar='UA')
-	general.add_option('--list-extractors',
-			action='store_true', dest='list_extractors',
-			help='List all supported extractors and the URLs they would handle', default=False)
+    general.add_option('-h', '--help',
+            action='help', help='print this help text and exit')
+    general.add_option('-v', '--version',
+            action='version', help='print program version and exit')
+    general.add_option('-U', '--update',
+            action='store_true', dest='update_self', help='update this program to latest version')
+    general.add_option('-i', '--ignore-errors',
+            action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
+    general.add_option('-r', '--rate-limit',
+            dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
+    general.add_option('-R', '--retries',
+            dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
+    general.add_option('--buffer-size',
+            dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024")
+    general.add_option('--no-resize-buffer',
+            action='store_true', dest='noresizebuffer',
+            help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
+    general.add_option('--dump-user-agent',
+            action='store_true', dest='dump_user_agent',
+            help='display the current browser identification', default=False)
+    general.add_option('--user-agent',
+            dest='user_agent', help='specify a custom user agent', metavar='UA')
+    general.add_option('--list-extractors',
+            action='store_true', dest='list_extractors',
+            help='List all supported extractors and the URLs they would handle', default=False)
+    general.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 
-	selection.add_option('--playlist-start',
-			dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
-	selection.add_option('--playlist-end',
-			dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
-	selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
-	selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
-	selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
+    selection.add_option('--playlist-start',
+            dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
+    selection.add_option('--playlist-end',
+            dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
+    selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
+    selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
+    selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
 
-	authentication.add_option('-u', '--username',
-			dest='username', metavar='USERNAME', help='account username')
-	authentication.add_option('-p', '--password',
-			dest='password', metavar='PASSWORD', help='account password')
-	authentication.add_option('-n', '--netrc',
-			action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
+    authentication.add_option('-u', '--username',
+            dest='username', metavar='USERNAME', help='account username')
+    authentication.add_option('-p', '--password',
+            dest='password', metavar='PASSWORD', help='account password')
+    authentication.add_option('-n', '--netrc',
+            action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 
 
-	video_format.add_option('-f', '--format',
-			action='store', dest='format', metavar='FORMAT', help='video format code')
-	video_format.add_option('--all-formats',
-			action='store_const', dest='format', help='download all available video formats', const='all')
-	video_format.add_option('--prefer-free-formats',
-			action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
-	video_format.add_option('--max-quality',
-			action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
-	video_format.add_option('-F', '--list-formats',
-			action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
-	video_format.add_option('--write-srt',
-			action='store_true', dest='writesubtitles',
-			help='write video closed captions to a .srt file (currently youtube only)', default=False)
-	video_format.add_option('--srt-lang',
-			action='store', dest='subtitleslang', metavar='LANG',
-			help='language of the closed captions to download (optional) use IETF language tags like \'en\'')
+    video_format.add_option('-f', '--format',
+            action='store', dest='format', metavar='FORMAT', help='video format code')
+    video_format.add_option('--all-formats',
+            action='store_const', dest='format', help='download all available video formats', const='all')
+    video_format.add_option('--prefer-free-formats',
+            action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
+    video_format.add_option('--max-quality',
+            action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
+    video_format.add_option('-F', '--list-formats',
+            action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
+    video_format.add_option('--write-srt',
+            action='store_true', dest='writesubtitles',
+            help='write video closed captions to a .srt file (currently youtube only)', default=False)
+    video_format.add_option('--srt-lang',
+            action='store', dest='subtitleslang', metavar='LANG',
+            help='language of the closed captions to download (optional) use IETF language tags like \'en\'')
 
 
-	verbosity.add_option('-q', '--quiet',
-			action='store_true', dest='quiet', help='activates quiet mode', default=False)
-	verbosity.add_option('-s', '--simulate',
-			action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
-	verbosity.add_option('--skip-download',
-			action='store_true', dest='skip_download', help='do not download the video', default=False)
-	verbosity.add_option('-g', '--get-url',
-			action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
-	verbosity.add_option('-e', '--get-title',
-			action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
-	verbosity.add_option('--get-thumbnail',
-			action='store_true', dest='getthumbnail',
-			help='simulate, quiet but print thumbnail URL', default=False)
-	verbosity.add_option('--get-description',
-			action='store_true', dest='getdescription',
-			help='simulate, quiet but print video description', default=False)
-	verbosity.add_option('--get-filename',
-			action='store_true', dest='getfilename',
-			help='simulate, quiet but print output filename', default=False)
-	verbosity.add_option('--get-format',
-			action='store_true', dest='getformat',
-			help='simulate, quiet but print output format', default=False)
-	verbosity.add_option('--no-progress',
-			action='store_true', dest='noprogress', help='do not print progress bar', default=False)
-	verbosity.add_option('--console-title',
-			action='store_true', dest='consoletitle',
-			help='display progress in console titlebar', default=False)
-	verbosity.add_option('-v', '--verbose',
-			action='store_true', dest='verbose', help='print various debugging information', default=False)
+    verbosity.add_option('-q', '--quiet',
+            action='store_true', dest='quiet', help='activates quiet mode', default=False)
+    verbosity.add_option('-s', '--simulate',
+            action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
+    verbosity.add_option('--skip-download',
+            action='store_true', dest='skip_download', help='do not download the video', default=False)
+    verbosity.add_option('-g', '--get-url',
+            action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
+    verbosity.add_option('-e', '--get-title',
+            action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
+    verbosity.add_option('--get-thumbnail',
+            action='store_true', dest='getthumbnail',
+            help='simulate, quiet but print thumbnail URL', default=False)
+    verbosity.add_option('--get-description',
+            action='store_true', dest='getdescription',
+            help='simulate, quiet but print video description', default=False)
+    verbosity.add_option('--get-filename',
+            action='store_true', dest='getfilename',
+            help='simulate, quiet but print output filename', default=False)
+    verbosity.add_option('--get-format',
+            action='store_true', dest='getformat',
+            help='simulate, quiet but print output format', default=False)
+    verbosity.add_option('--no-progress',
+            action='store_true', dest='noprogress', help='do not print progress bar', default=False)
+    verbosity.add_option('--console-title',
+            action='store_true', dest='consoletitle',
+            help='display progress in console titlebar', default=False)
+    verbosity.add_option('-v', '--verbose',
+            action='store_true', dest='verbose', help='print various debugging information', default=False)
 
 
-	filesystem.add_option('-t', '--title',
-			action='store_true', dest='usetitle', help='use title in file name', default=False)
-	filesystem.add_option('--id',
-			action='store_true', dest='useid', help='use video ID in file name', default=False)
-	filesystem.add_option('-l', '--literal',
-			action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
-	filesystem.add_option('-A', '--auto-number',
-			action='store_true', dest='autonumber',
-			help='number downloaded files starting from 00000', default=False)
-	filesystem.add_option('-o', '--output',
-			dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
-	filesystem.add_option('--restrict-filenames',
-			action='store_true', dest='restrictfilenames',
-			help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
-	filesystem.add_option('-a', '--batch-file',
-			dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
-	filesystem.add_option('-w', '--no-overwrites',
-			action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
-	filesystem.add_option('-c', '--continue',
-			action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
-	filesystem.add_option('--no-continue',
-			action='store_false', dest='continue_dl',
-			help='do not resume partially downloaded files (restart from beginning)')
-	filesystem.add_option('--cookies',
-			dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
-	filesystem.add_option('--no-part',
-			action='store_true', dest='nopart', help='do not use .part files', default=False)
-	filesystem.add_option('--no-mtime',
-			action='store_false', dest='updatetime',
-			help='do not use the Last-modified header to set the file modification time', default=True)
-	filesystem.add_option('--write-description',
-			action='store_true', dest='writedescription',
-			help='write video description to a .description file', default=False)
-	filesystem.add_option('--write-info-json',
-			action='store_true', dest='writeinfojson',
-			help='write video metadata to a .info.json file', default=False)
+    filesystem.add_option('-t', '--title',
+            action='store_true', dest='usetitle', help='use title in file name', default=False)
+    filesystem.add_option('--id',
+            action='store_true', dest='useid', help='use video ID in file name', default=False)
+    filesystem.add_option('-l', '--literal',
+            action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
+    filesystem.add_option('-A', '--auto-number',
+            action='store_true', dest='autonumber',
+            help='number downloaded files starting from 00000', default=False)
+    filesystem.add_option('-o', '--output',
+            dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
+    filesystem.add_option('--restrict-filenames',
+            action='store_true', dest='restrictfilenames',
+            help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
+    filesystem.add_option('-a', '--batch-file',
+            dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
+    filesystem.add_option('-w', '--no-overwrites',
+            action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
+    filesystem.add_option('-c', '--continue',
+            action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
+    filesystem.add_option('--no-continue',
+            action='store_false', dest='continue_dl',
+            help='do not resume partially downloaded files (restart from beginning)')
+    filesystem.add_option('--cookies',
+            dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
+    filesystem.add_option('--no-part',
+            action='store_true', dest='nopart', help='do not use .part files', default=False)
+    filesystem.add_option('--no-mtime',
+            action='store_false', dest='updatetime',
+            help='do not use the Last-modified header to set the file modification time', default=True)
+    filesystem.add_option('--write-description',
+            action='store_true', dest='writedescription',
+            help='write video description to a .description file', default=False)
+    filesystem.add_option('--write-info-json',
+            action='store_true', dest='writeinfojson',
+            help='write video metadata to a .info.json file', default=False)
 
 
-	postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
-			help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
-	postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
-			help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default')
-	postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
-			help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
-	postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
-			help='keeps the video file on disk after the post-processing; the video is erased by default')
+    postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
+            help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
+    postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
+            help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
+    postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
+            help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
+    postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
+            help='keeps the video file on disk after the post-processing; the video is erased by default')
+    postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
+            help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 
 
-	parser.add_option_group(general)
-	parser.add_option_group(selection)
-	parser.add_option_group(filesystem)
-	parser.add_option_group(verbosity)
-	parser.add_option_group(video_format)
-	parser.add_option_group(authentication)
-	parser.add_option_group(postproc)
+    parser.add_option_group(general)
+    parser.add_option_group(selection)
+    parser.add_option_group(filesystem)
+    parser.add_option_group(verbosity)
+    parser.add_option_group(video_format)
+    parser.add_option_group(authentication)
+    parser.add_option_group(postproc)
 
-	xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
-	if xdg_config_home:
-		userConf = os.path.join(xdg_config_home, 'youtube-dl.conf')
-	else:
-		userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
-	argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:]
-	opts, args = parser.parse_args(argv)
+    xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
+    if xdg_config_home:
+        userConf = os.path.join(xdg_config_home, 'youtube-dl.conf')
+    else:
+        userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
+    argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:]
+    opts, args = parser.parse_args(argv)
 
-	return parser, opts, args
-
-def gen_extractors():
-	""" Return a list of an instance of every supported extractor.
-	The order does matter; the first extractor matched is the one handling the URL.
-	"""
-	return [
-		YoutubePlaylistIE(),
-		YoutubeChannelIE(),
-		YoutubeUserIE(),
-		YoutubeSearchIE(),
-		YoutubeIE(),
-		MetacafeIE(),
-		DailymotionIE(),
-		GoogleIE(),
-		GoogleSearchIE(),
-		PhotobucketIE(),
-		YahooIE(),
-		YahooSearchIE(),
-		DepositFilesIE(),
-		FacebookIE(),
-		BlipTVUserIE(),
-		BlipTVIE(),
-		VimeoIE(),
-		MyVideoIE(),
-		ComedyCentralIE(),
-		EscapistIE(),
-		CollegeHumorIE(),
-		XVideosIE(),
-		SoundcloudIE(),
-		InfoQIE(),
-		MixcloudIE(),
-		StanfordOpenClassroomIE(),
-		MTVIE(),
-		YoukuIE(),
-		XNXXIE(),
-		GooglePlusIE(),
-		ArteTvIE(),
-		GenericIE()
-	]
+    return parser, opts, args
 
 def _real_main():
-	parser, opts, args = parseOpts()
+    parser, opts, args = parseOpts()
 
-	# Open appropriate CookieJar
-	if opts.cookiefile is None:
-		jar = cookielib.CookieJar()
-	else:
-		try:
-			jar = cookielib.MozillaCookieJar(opts.cookiefile)
-			if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
-				jar.load()
-		except (IOError, OSError), err:
-			sys.exit(u'ERROR: unable to open cookie file')
-	# Set user agent
-	if opts.user_agent is not None:
-		std_headers['User-Agent'] = opts.user_agent
+    # Open appropriate CookieJar
+    if opts.cookiefile is None:
+        jar = compat_cookiejar.CookieJar()
+    else:
+        try:
+            jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile)
+            if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
+                jar.load()
+        except (IOError, OSError) as err:
+            sys.exit(u'ERROR: unable to open cookie file')
+    # Set user agent
+    if opts.user_agent is not None:
+        std_headers['User-Agent'] = opts.user_agent
 
-	# Dump user agent
-	if opts.dump_user_agent:
-		print std_headers['User-Agent']
-		sys.exit(0)
+    # Dump user agent
+    if opts.dump_user_agent:
+        print(std_headers['User-Agent'])
+        sys.exit(0)
 
-	# Batch file verification
-	batchurls = []
-	if opts.batchfile is not None:
-		try:
-			if opts.batchfile == '-':
-				batchfd = sys.stdin
-			else:
-				batchfd = open(opts.batchfile, 'r')
-			batchurls = batchfd.readlines()
-			batchurls = [x.strip() for x in batchurls]
-			batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
-		except IOError:
-			sys.exit(u'ERROR: batch file could not be read')
-	all_urls = batchurls + args
-	all_urls = map(lambda url: url.strip(), all_urls)
+    # Batch file verification
+    batchurls = []
+    if opts.batchfile is not None:
+        try:
+            if opts.batchfile == '-':
+                batchfd = sys.stdin
+            else:
+                batchfd = open(opts.batchfile, 'r')
+            batchurls = batchfd.readlines()
+            batchurls = [x.strip() for x in batchurls]
+            batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
+        except IOError:
+            sys.exit(u'ERROR: batch file could not be read')
+    all_urls = batchurls + args
+    all_urls = [url.strip() for url in all_urls]
 
-	# General configuration
-	cookie_processor = urllib2.HTTPCookieProcessor(jar)
-	proxy_handler = urllib2.ProxyHandler()
-	opener = urllib2.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
-	urllib2.install_opener(opener)
-	socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
+    # General configuration
+    cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+    proxy_handler = compat_urllib_request.ProxyHandler()
+    opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+    compat_urllib_request.install_opener(opener)
+    socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
 
-	extractors = gen_extractors()
+    extractors = gen_extractors()
 
-	if opts.list_extractors:
-		for ie in extractors:
-			print(ie.IE_NAME)
-			matchedUrls = filter(lambda url: ie.suitable(url), all_urls)
-			all_urls = filter(lambda url: url not in matchedUrls, all_urls)
-			for mu in matchedUrls:
-				print(u'  ' + mu)
-		sys.exit(0)
+    if opts.list_extractors:
+        for ie in extractors:
+            print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
+            matchedUrls = [url for url in all_urls if ie.suitable(url)]
+            all_urls = [url for url in all_urls if url not in matchedUrls]
+            for mu in matchedUrls:
+                print(u'  ' + mu)
+        sys.exit(0)
 
-	# Conflicting, missing and erroneous options
-	if opts.usenetrc and (opts.username is not None or opts.password is not None):
-		parser.error(u'using .netrc conflicts with giving username/password')
-	if opts.password is not None and opts.username is None:
-		parser.error(u'account username missing')
-	if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
-		parser.error(u'using output template conflicts with using title, video ID or auto number')
-	if opts.usetitle and opts.useid:
-		parser.error(u'using title conflicts with using video ID')
-	if opts.username is not None and opts.password is None:
-		opts.password = getpass.getpass(u'Type account password and press return:')
-	if opts.ratelimit is not None:
-		numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
-		if numeric_limit is None:
-			parser.error(u'invalid rate limit specified')
-		opts.ratelimit = numeric_limit
-	if opts.retries is not None:
-		try:
-			opts.retries = long(opts.retries)
-		except (TypeError, ValueError), err:
-			parser.error(u'invalid retry count specified')
-	if opts.buffersize is not None:
-		numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
-		if numeric_buffersize is None:
-			parser.error(u'invalid buffer size specified')
-		opts.buffersize = numeric_buffersize
-	try:
-		opts.playliststart = int(opts.playliststart)
-		if opts.playliststart <= 0:
-			raise ValueError(u'Playlist start must be positive')
-	except (TypeError, ValueError), err:
-		parser.error(u'invalid playlist start number specified')
-	try:
-		opts.playlistend = int(opts.playlistend)
-		if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
-			raise ValueError(u'Playlist end must be greater than playlist start')
-	except (TypeError, ValueError), err:
-		parser.error(u'invalid playlist end number specified')
-	if opts.extractaudio:
-		if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']:
-			parser.error(u'invalid audio format specified')
-	if opts.audioquality:
-		opts.audioquality = opts.audioquality.strip('k').strip('K')
-		if not opts.audioquality.isdigit():
-			parser.error(u'invalid audio quality specified')
+    # Conflicting, missing and erroneous options
+    if opts.usenetrc and (opts.username is not None or opts.password is not None):
+        parser.error(u'using .netrc conflicts with giving username/password')
+    if opts.password is not None and opts.username is None:
+        parser.error(u'account username missing')
+    if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
+        parser.error(u'using output template conflicts with using title, video ID or auto number')
+    if opts.usetitle and opts.useid:
+        parser.error(u'using title conflicts with using video ID')
+    if opts.username is not None and opts.password is None:
+        opts.password = getpass.getpass(u'Type account password and press return:')
+    if opts.ratelimit is not None:
+        numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
+        if numeric_limit is None:
+            parser.error(u'invalid rate limit specified')
+        opts.ratelimit = numeric_limit
+    if opts.retries is not None:
+        try:
+            opts.retries = int(opts.retries)
+        except (TypeError, ValueError) as err:
+            parser.error(u'invalid retry count specified')
+    if opts.buffersize is not None:
+        numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
+        if numeric_buffersize is None:
+            parser.error(u'invalid buffer size specified')
+        opts.buffersize = numeric_buffersize
+    try:
+        opts.playliststart = int(opts.playliststart)
+        if opts.playliststart <= 0:
+            raise ValueError(u'Playlist start must be positive')
+    except (TypeError, ValueError) as err:
+        parser.error(u'invalid playlist start number specified')
+    try:
+        opts.playlistend = int(opts.playlistend)
+        if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
+            raise ValueError(u'Playlist end must be greater than playlist start')
+    except (TypeError, ValueError) as err:
+        parser.error(u'invalid playlist end number specified')
+    if opts.extractaudio:
+        if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
+            parser.error(u'invalid audio format specified')
+    if opts.audioquality:
+        opts.audioquality = opts.audioquality.strip('k').strip('K')
+        if not opts.audioquality.isdigit():
+            parser.error(u'invalid audio quality specified')
 
-	# File downloader
-	fd = FileDownloader({
-		'usenetrc': opts.usenetrc,
-		'username': opts.username,
-		'password': opts.password,
-		'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
-		'forceurl': opts.geturl,
-		'forcetitle': opts.gettitle,
-		'forcethumbnail': opts.getthumbnail,
-		'forcedescription': opts.getdescription,
-		'forcefilename': opts.getfilename,
-		'forceformat': opts.getformat,
-		'simulate': opts.simulate,
-		'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
-		'format': opts.format,
-		'format_limit': opts.format_limit,
-		'listformats': opts.listformats,
-		'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
-			or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
-			or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
-			or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
-			or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
-			or (opts.useid and u'%(id)s.%(ext)s')
-			or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
-			or u'%(id)s.%(ext)s'),
-		'restrictfilenames': opts.restrictfilenames,
-		'ignoreerrors': opts.ignoreerrors,
-		'ratelimit': opts.ratelimit,
-		'nooverwrites': opts.nooverwrites,
-		'retries': opts.retries,
-		'buffersize': opts.buffersize,
-		'noresizebuffer': opts.noresizebuffer,
-		'continuedl': opts.continue_dl,
-		'noprogress': opts.noprogress,
-		'playliststart': opts.playliststart,
-		'playlistend': opts.playlistend,
-		'logtostderr': opts.outtmpl == '-',
-		'consoletitle': opts.consoletitle,
-		'nopart': opts.nopart,
-		'updatetime': opts.updatetime,
-		'writedescription': opts.writedescription,
-		'writeinfojson': opts.writeinfojson,
-		'writesubtitles': opts.writesubtitles,
-		'subtitleslang': opts.subtitleslang,
-		'matchtitle': opts.matchtitle,
-		'rejecttitle': opts.rejecttitle,
-		'max_downloads': opts.max_downloads,
-		'prefer_free_formats': opts.prefer_free_formats,
-		'verbose': opts.verbose,
-		})
+    if sys.version_info < (3,):
+        # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
+        if opts.outtmpl is not None:
+            opts.outtmpl = opts.outtmpl.decode(preferredencoding())
+    outtmpl =((opts.outtmpl is not None and opts.outtmpl)
+            or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
+            or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
+            or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
+            or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
+            or (opts.useid and u'%(id)s.%(ext)s')
+            or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
+            or u'%(id)s.%(ext)s')
+    # File downloader
+    fd = FileDownloader({
+        'usenetrc': opts.usenetrc,
+        'username': opts.username,
+        'password': opts.password,
+        'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
+        'forceurl': opts.geturl,
+        'forcetitle': opts.gettitle,
+        'forcethumbnail': opts.getthumbnail,
+        'forcedescription': opts.getdescription,
+        'forcefilename': opts.getfilename,
+        'forceformat': opts.getformat,
+        'simulate': opts.simulate,
+        'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
+        'format': opts.format,
+        'format_limit': opts.format_limit,
+        'listformats': opts.listformats,
+        'outtmpl': outtmpl,
+        'restrictfilenames': opts.restrictfilenames,
+        'ignoreerrors': opts.ignoreerrors,
+        'ratelimit': opts.ratelimit,
+        'nooverwrites': opts.nooverwrites,
+        'retries': opts.retries,
+        'buffersize': opts.buffersize,
+        'noresizebuffer': opts.noresizebuffer,
+        'continuedl': opts.continue_dl,
+        'noprogress': opts.noprogress,
+        'playliststart': opts.playliststart,
+        'playlistend': opts.playlistend,
+        'logtostderr': opts.outtmpl == '-',
+        'consoletitle': opts.consoletitle,
+        'nopart': opts.nopart,
+        'updatetime': opts.updatetime,
+        'writedescription': opts.writedescription,
+        'writeinfojson': opts.writeinfojson,
+        'writesubtitles': opts.writesubtitles,
+        'subtitleslang': opts.subtitleslang,
+        'matchtitle': opts.matchtitle,
+        'rejecttitle': opts.rejecttitle,
+        'max_downloads': opts.max_downloads,
+        'prefer_free_formats': opts.prefer_free_formats,
+        'verbose': opts.verbose,
+        'test': opts.test,
+        })
 
-	if opts.verbose:
-		fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies))
+    if opts.verbose:
+        fd.to_screen(u'[debug] youtube-dl version ' + __version__)
+        try:
+            sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                  cwd=os.path.dirname(os.path.abspath(__file__)))
+            out, err = sp.communicate()
+            out = out.decode().strip()
+            if re.match('[0-9a-f]+', out):
+                fd.to_screen(u'[debug] Git HEAD: ' + out)
+        except:
+            pass
+        fd.to_screen(u'[debug] Python version %s - %s' %(platform.python_version(), platform.platform()))
+        fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies))
 
-	for extractor in extractors:
-		fd.add_info_extractor(extractor)
+    for extractor in extractors:
+        fd.add_info_extractor(extractor)
 
-	# PostProcessors
-	if opts.extractaudio:
-		fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo))
+    # PostProcessors
+    if opts.extractaudio:
+        fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo, nopostoverwrites=opts.nopostoverwrites))
 
-	# Update version
-	if opts.update_self:
-		updateSelf(fd, sys.argv[0])
+    # Update version
+    if opts.update_self:
+        update_self(fd.to_screen, opts.verbose, sys.argv[0])
 
-	# Maybe do nothing
-	if len(all_urls) < 1:
-		if not opts.update_self:
-			parser.error(u'you must provide at least one URL')
-		else:
-			sys.exit()
+    # Maybe do nothing
+    if len(all_urls) < 1:
+        if not opts.update_self:
+            parser.error(u'you must provide at least one URL')
+        else:
+            sys.exit()
 
-	try:
-		retcode = fd.download(all_urls)
-	except MaxDownloadsReached:
-		fd.to_screen(u'--max-download limit reached, aborting.')
-		retcode = 101
+    try:
+        retcode = fd.download(all_urls)
+    except MaxDownloadsReached:
+        fd.to_screen(u'--max-download limit reached, aborting.')
+        retcode = 101
 
-	# Dump cookie jar if requested
-	if opts.cookiefile is not None:
-		try:
-			jar.save()
-		except (IOError, OSError), err:
-			sys.exit(u'ERROR: unable to save cookie jar')
+    # Dump cookie jar if requested
+    if opts.cookiefile is not None:
+        try:
+            jar.save()
+        except (IOError, OSError) as err:
+            sys.exit(u'ERROR: unable to save cookie jar')
 
-	sys.exit(retcode)
+    sys.exit(retcode)
 
 def main():
-	try:
-		_real_main()
-	except DownloadError:
-		sys.exit(1)
-	except SameFileError:
-		sys.exit(u'ERROR: fixed output name but more than one file to download')
-	except KeyboardInterrupt:
-		sys.exit(u'\nERROR: Interrupted by user')
+    try:
+        _real_main()
+    except DownloadError:
+        sys.exit(1)
+    except SameFileError:
+        sys.exit(u'ERROR: fixed output name but more than one file to download')
+    except KeyboardInterrupt:
+        sys.exit(u'\nERROR: Interrupted by user')

youtube_dl/__main__.py

@@ -1,7 +1,17 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 
-import __init__
+# Execute with
+# $ python youtube_dl/__main__.py (2.6+)
+# $ python -m youtube_dl          (2.7+)
+
+import sys
+
+if __package__ is None and not hasattr(sys, "frozen"):
+    # direct call of __main__.py
+    import os.path
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl
 
 if __name__ == '__main__':
-	__init__.main()
+    youtube_dl.main()

youtube_dl/update.py

@@ -0,0 +1,160 @@
+import json
+import traceback
+import hashlib
+from zipimport import zipimporter
+
+from .utils import *
+from .version import __version__
+
+def rsa_verify(message, signature, key):
+    from struct import pack
+    from hashlib import sha256
+    from sys import version_info
+    def b(x):
+        if version_info[0] == 2: return x
+        else: return x.encode('latin1')
+    assert(type(message) == type(b('')))
+    block_size = 0
+    n = key[0]
+    while n:
+        block_size += 1
+        n >>= 8
+    signature = pow(int(signature, 16), key[1], key[0])
+    raw_bytes = []
+    while signature:
+        raw_bytes.insert(0, pack("B", signature & 0xFF))
+        signature >>= 8
+    signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
+    if signature[0:2] != b('\x00\x01'): return False
+    signature = signature[2:]
+    if not b('\x00') in signature: return False
+    signature = signature[signature.index(b('\x00'))+1:]
+    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
+    signature = signature[19:]
+    if signature != sha256(message).digest(): return False
+    return True
+
+def update_self(to_screen, verbose, filename):
+    """Update the program file with the latest version from the repository"""
+
+    UPDATE_URL = "http://rg3.github.com/youtube-dl/update/"
+    VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
+    JSON_URL = UPDATE_URL + 'versions.json'
+    UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
+
+
+    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"):
+        to_screen(u'It looks like you installed youtube-dl with pip, setup.py or a tarball. Please use that to update.')
+        return
+
+    # Check if there is a new version
+    try:
+        newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip()
+    except:
+        if verbose: to_screen(compat_str(traceback.format_exc()))
+        to_screen(u'ERROR: can\'t find the current version. Please try again later.')
+        return
+    if newversion == __version__:
+        to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
+        return
+
+    # Download and check versions info
+    try:
+        versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
+        versions_info = json.loads(versions_info)
+    except:
+        if verbose: to_screen(compat_str(traceback.format_exc()))
+        to_screen(u'ERROR: can\'t obtain versions info. Please try again later.')
+        return
+    if not 'signature' in versions_info:
+        to_screen(u'ERROR: the versions file is not signed or corrupted. Aborting.')
+        return
+    signature = versions_info['signature']
+    del versions_info['signature']
+    if not rsa_verify(json.dumps(versions_info, sort_keys=True).encode('utf-8'), signature, UPDATES_RSA_KEY):
+        to_screen(u'ERROR: the versions file signature is invalid. Aborting.')
+        return
+
+    to_screen(u'Updating to version ' + versions_info['latest'] + '...')
+    version = versions_info['versions'][versions_info['latest']]
+    if version.get('notes'):
+        to_screen(u'PLEASE NOTE:')
+        for note in version['notes']:
+            to_screen(note)
+
+    if not os.access(filename, os.W_OK):
+        to_screen(u'ERROR: no write permissions on %s' % filename)
+        return
+
+    # Py2EXE
+    if hasattr(sys, "frozen"):
+        exe = os.path.abspath(filename)
+        directory = os.path.dirname(exe)
+        if not os.access(directory, os.W_OK):
+            to_screen(u'ERROR: no write permissions on %s' % directory)
+            return
+
+        try:
+            urlh = compat_urllib_request.urlopen(version['exe'][0])
+            newcontent = urlh.read()
+            urlh.close()
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to download latest version')
+            return
+
+        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+        if newcontent_hash != version['exe'][1]:
+            to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
+            return
+
+        try:
+            with open(exe + '.new', 'wb') as outf:
+                outf.write(newcontent)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to write the new version')
+            return
+
+        try:
+            bat = os.path.join(directory, 'youtube-dl-updater.bat')
+            b = open(bat, 'w')
+            b.write("""
+echo Updating youtube-dl...
+ping 127.0.0.1 -n 5 -w 1000 > NUL
+move /Y "%s.new" "%s"
+del "%s"
+            \n""" %(exe, exe, bat))
+            b.close()
+
+            os.startfile(bat)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to overwrite current version')
+            return
+
+    # Zip unix package
+    elif isinstance(globals().get('__loader__'), zipimporter):
+        try:
+            urlh = compat_urllib_request.urlopen(version['bin'][0])
+            newcontent = urlh.read()
+            urlh.close()
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to download latest version')
+            return
+
+        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+        if newcontent_hash != version['bin'][1]:
+            to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
+            return
+
+        try:
+            with open(filename, 'wb') as outf:
+                outf.write(newcontent)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to overwrite current version')
+            return
+
+    to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')

youtube_dl/utils.py

@@ -2,379 +2,542 @@
 # -*- coding: utf-8 -*-
 
 import gzip
-import htmlentitydefs
-import HTMLParser
+import io
+import json
 import locale
 import os
 import re
 import sys
+import traceback
 import zlib
-import urllib2
 import email.utils
 import json
 
 try:
-	import cStringIO as StringIO
-except ImportError:
-	import StringIO
+    import urllib.request as compat_urllib_request
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_request
 
-std_headers = {
-	'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
-	'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
-	'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-	'Accept-Encoding': 'gzip, deflate',
-	'Accept-Language': 'en-us,en;q=0.5',
-}
+try:
+    import urllib.error as compat_urllib_error
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_error
+
+try:
+    import urllib.parse as compat_urllib_parse
+except ImportError: # Python 2
+    import urllib as compat_urllib_parse
+
+try:
+    from urllib.parse import urlparse as compat_urllib_parse_urlparse
+except ImportError: # Python 2
+    from urlparse import urlparse as compat_urllib_parse_urlparse
+
+try:
+    import http.cookiejar as compat_cookiejar
+except ImportError: # Python 2
+    import cookielib as compat_cookiejar
+
+try:
+    import html.entities as compat_html_entities
+except ImportError: # Python 2
+    import htmlentitydefs as compat_html_entities
+
+try:
+    import html.parser as compat_html_parser
+except ImportError: # Python 2
+    import HTMLParser as compat_html_parser
+
+try:
+    import http.client as compat_http_client
+except ImportError: # Python 2
+    import httplib as compat_http_client
+
+try:
+    from subprocess import DEVNULL
+    compat_subprocess_get_DEVNULL = lambda: DEVNULL
+except ImportError:
+    compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
+
+try:
+    from urllib.parse import parse_qs as compat_parse_qs
+except ImportError: # Python 2
+    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
+    # Python 2's version is apparently totally broken
+    def _unquote(string, encoding='utf-8', errors='replace'):
+        if string == '':
+            return string
+        res = string.split('%')
+        if len(res) == 1:
+            return string
+        if encoding is None:
+            encoding = 'utf-8'
+        if errors is None:
+            errors = 'replace'
+        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
+        pct_sequence = b''
+        string = res[0]
+        for item in res[1:]:
+            try:
+                if not item:
+                    raise ValueError
+                pct_sequence += item[:2].decode('hex')
+                rest = item[2:]
+                if not rest:
+                    # This segment was just a single percent-encoded character.
+                    # May be part of a sequence of code units, so delay decoding.
+                    # (Stored in pct_sequence).
+                    continue
+            except ValueError:
+                rest = '%' + item
+            # Encountered non-percent-encoded characters. Flush the current
+            # pct_sequence.
+            string += pct_sequence.decode(encoding, errors) + rest
+            pct_sequence = b''
+        if pct_sequence:
+            # Flush the final pct_sequence
+            string += pct_sequence.decode(encoding, errors)
+        return string
+
+    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        qs, _coerce_result = qs, unicode
+        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
+        r = []
+        for name_value in pairs:
+            if not name_value and not strict_parsing:
+                continue
+            nv = name_value.split('=', 1)
+            if len(nv) != 2:
+                if strict_parsing:
+                    raise ValueError("bad query field: %r" % (name_value,))
+                # Handle case of a control-name with no equal sign
+                if keep_blank_values:
+                    nv.append('')
+                else:
+                    continue
+            if len(nv[1]) or keep_blank_values:
+                name = nv[0].replace('+', ' ')
+                name = _unquote(name, encoding=encoding, errors=errors)
+                name = _coerce_result(name)
+                value = nv[1].replace('+', ' ')
+                value = _unquote(value, encoding=encoding, errors=errors)
+                value = _coerce_result(value)
+                r.append((name, value))
+        return r
+
+    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        parsed_result = {}
+        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
+                        encoding=encoding, errors=errors)
+        for name, value in pairs:
+            if name in parsed_result:
+                parsed_result[name].append(value)
+            else:
+                parsed_result[name] = [value]
+        return parsed_result
 
 try:
     compat_str = unicode # Python 2
 except NameError:
     compat_str = str
 
+try:
+    compat_chr = unichr # Python 2
+except NameError:
+    compat_chr = chr
+
+std_headers = {
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
+    'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Encoding': 'gzip, deflate',
+    'Accept-Language': 'en-us,en;q=0.5',
+}
+
 def preferredencoding():
-	"""Get preferred encoding.
+    """Get preferred encoding.
 
-	Returns the best encoding scheme for the system, based on
-	locale.getpreferredencoding() and some further tweaks.
-	"""
-	def yield_preferredencoding():
-		try:
-			pref = locale.getpreferredencoding()
-			u'TEST'.encode(pref)
-		except:
-			pref = 'UTF-8'
-		while True:
-			yield pref
-	return yield_preferredencoding().next()
+    Returns the best encoding scheme for the system, based on
+    locale.getpreferredencoding() and some further tweaks.
+    """
+    try:
+        pref = locale.getpreferredencoding()
+        u'TEST'.encode(pref)
+    except:
+        pref = 'UTF-8'
 
+    return pref
+
+if sys.version_info < (3,0):
+    def compat_print(s):
+        print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
+else:
+    def compat_print(s):
+        assert type(s) == type(u'')
+        print(s)
+
+# In Python 2.x, json.dump expects a bytestream.
+# In Python 3.x, it writes to a character stream
+if sys.version_info < (3,0):
+    def write_json_file(obj, fn):
+        with open(fn, 'wb') as f:
+            json.dump(obj, f)
+else:
+    def write_json_file(obj, fn):
+        with open(fn, 'w', encoding='utf-8') as f:
+            json.dump(obj, f)
 
 def htmlentity_transform(matchobj):
-	"""Transforms an HTML entity to a Unicode character.
+    """Transforms an HTML entity to a character.
 
-	This function receives a match object and is intended to be used with
-	the re.sub() function.
-	"""
-	entity = matchobj.group(1)
+    This function receives a match object and is intended to be used with
+    the re.sub() function.
+    """
+    entity = matchobj.group(1)
 
-	# Known non-numeric HTML entity
-	if entity in htmlentitydefs.name2codepoint:
-		return unichr(htmlentitydefs.name2codepoint[entity])
+    # Known non-numeric HTML entity
+    if entity in compat_html_entities.name2codepoint:
+        return compat_chr(compat_html_entities.name2codepoint[entity])
 
-	# Unicode character
-	mobj = re.match(ur'(?u)#(x?\d+)', entity)
-	if mobj is not None:
-		numstr = mobj.group(1)
-		if numstr.startswith(u'x'):
-			base = 16
-			numstr = u'0%s' % numstr
-		else:
-			base = 10
-		return unichr(long(numstr, base))
+    mobj = re.match(u'(?u)#(x?\\d+)', entity)
+    if mobj is not None:
+        numstr = mobj.group(1)
+        if numstr.startswith(u'x'):
+            base = 16
+            numstr = u'0%s' % numstr
+        else:
+            base = 10
+        return compat_chr(int(numstr, base))
 
-	# Unknown entity in name, return its literal representation
-	return (u'&%s;' % entity)
+    # Unknown entity in name, return its literal representation
+    return (u'&%s;' % entity)
 
-HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
-class IDParser(HTMLParser.HTMLParser):
-	"""Modified HTMLParser that isolates a tag with the specified id"""
-	def __init__(self, id):
-		self.id = id
-		self.result = None
-		self.started = False
-		self.depth = {}
-		self.html = None
-		self.watch_startpos = False
-		self.error_count = 0
-		HTMLParser.HTMLParser.__init__(self)
+compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
+class AttrParser(compat_html_parser.HTMLParser):
+    """Modified HTMLParser that isolates a tag with the specified attribute"""
+    def __init__(self, attribute, value):
+        self.attribute = attribute
+        self.value = value
+        self.result = None
+        self.started = False
+        self.depth = {}
+        self.html = None
+        self.watch_startpos = False
+        self.error_count = 0
+        compat_html_parser.HTMLParser.__init__(self)
 
-	def error(self, message):
-		if self.error_count > 10 or self.started:
-			raise HTMLParser.HTMLParseError(message, self.getpos())
-		self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
-		self.error_count += 1
-		self.goahead(1)
+    def error(self, message):
+        if self.error_count > 10 or self.started:
+            raise compat_html_parser.HTMLParseError(message, self.getpos())
+        self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
+        self.error_count += 1
+        self.goahead(1)
 
-	def loads(self, html):
-		self.html = html
-		self.feed(html)
-		self.close()
+    def loads(self, html):
+        self.html = html
+        self.feed(html)
+        self.close()
 
-	def handle_starttag(self, tag, attrs):
-		attrs = dict(attrs)
-		if self.started:
-			self.find_startpos(None)
-		if 'id' in attrs and attrs['id'] == self.id:
-			self.result = [tag]
-			self.started = True
-			self.watch_startpos = True
-		if self.started:
-			if not tag in self.depth: self.depth[tag] = 0
-			self.depth[tag] += 1
+    def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        if self.started:
+            self.find_startpos(None)
+        if self.attribute in attrs and attrs[self.attribute] == self.value:
+            self.result = [tag]
+            self.started = True
+            self.watch_startpos = True
+        if self.started:
+            if not tag in self.depth: self.depth[tag] = 0
+            self.depth[tag] += 1
 
-	def handle_endtag(self, tag):
-		if self.started:
-			if tag in self.depth: self.depth[tag] -= 1
-			if self.depth[self.result[0]] == 0:
-				self.started = False
-				self.result.append(self.getpos())
+    def handle_endtag(self, tag):
+        if self.started:
+            if tag in self.depth: self.depth[tag] -= 1
+            if self.depth[self.result[0]] == 0:
+                self.started = False
+                self.result.append(self.getpos())
 
-	def find_startpos(self, x):
-		"""Needed to put the start position of the result (self.result[1])
-		after the opening tag with the requested id"""
-		if self.watch_startpos:
-			self.watch_startpos = False
-			self.result.append(self.getpos())
-	handle_entityref = handle_charref = handle_data = handle_comment = \
-	handle_decl = handle_pi = unknown_decl = find_startpos
+    def find_startpos(self, x):
+        """Needed to put the start position of the result (self.result[1])
+        after the opening tag with the requested id"""
+        if self.watch_startpos:
+            self.watch_startpos = False
+            self.result.append(self.getpos())
+    handle_entityref = handle_charref = handle_data = handle_comment = \
+    handle_decl = handle_pi = unknown_decl = find_startpos
 
-	def get_result(self):
-		if self.result == None: return None
-		if len(self.result) != 3: return None
-		lines = self.html.split('\n')
-		lines = lines[self.result[1][0]-1:self.result[2][0]]
-		lines[0] = lines[0][self.result[1][1]:]
-		if len(lines) == 1:
-			lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
-		lines[-1] = lines[-1][:self.result[2][1]]
-		return '\n'.join(lines).strip()
+    def get_result(self):
+        if self.result is None:
+            return None
+        if len(self.result) != 3:
+            return None
+        lines = self.html.split('\n')
+        lines = lines[self.result[1][0]-1:self.result[2][0]]
+        lines[0] = lines[0][self.result[1][1]:]
+        if len(lines) == 1:
+            lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
+        lines[-1] = lines[-1][:self.result[2][1]]
+        return '\n'.join(lines).strip()
 
 def get_element_by_id(id, html):
-	"""Return the content of the tag with the specified id in the passed HTML document"""
-	parser = IDParser(id)
-	try:
-		parser.loads(html)
-	except HTMLParser.HTMLParseError:
-		pass
-	return parser.get_result()
+    """Return the content of the tag with the specified ID in the passed HTML document"""
+    return get_element_by_attribute("id", id, html)
+
+def get_element_by_attribute(attribute, value, html):
+    """Return the content of the tag with the specified attribute in the passed HTML document"""
+    parser = AttrParser(attribute, value)
+    try:
+        parser.loads(html)
+    except compat_html_parser.HTMLParseError:
+        pass
+    return parser.get_result()
 
 
 def clean_html(html):
-	"""Clean an HTML snippet into a readable string"""
-	# Newline vs <br />
-	html = html.replace('\n', ' ')
-	html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
-	# Strip html tags
-	html = re.sub('<.*?>', '', html)
-	# Replace html entities
-	html = unescapeHTML(html)
-	return html
+    """Clean an HTML snippet into a readable string"""
+    # Newline vs <br />
+    html = html.replace('\n', ' ')
+    html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
+    html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
+    # Strip html tags
+    html = re.sub('<.*?>', '', html)
+    # Replace html entities
+    html = unescapeHTML(html)
+    return html
 
 
 def sanitize_open(filename, open_mode):
-	"""Try to open the given filename, and slightly tweak it if this fails.
+    """Try to open the given filename, and slightly tweak it if this fails.
 
-	Attempts to open the given filename. If this fails, it tries to change
-	the filename slightly, step by step, until it's either able to open it
-	or it fails and raises a final exception, like the standard open()
-	function.
+    Attempts to open the given filename. If this fails, it tries to change
+    the filename slightly, step by step, until it's either able to open it
+    or it fails and raises a final exception, like the standard open()
+    function.
 
-	It returns the tuple (stream, definitive_file_name).
-	"""
-	try:
-		if filename == u'-':
-			if sys.platform == 'win32':
-				import msvcrt
-				msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
-			return (sys.stdout, filename)
-		stream = open(encodeFilename(filename), open_mode)
-		return (stream, filename)
-	except (IOError, OSError), err:
-		# In case of error, try to remove win32 forbidden chars
-		filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
+    It returns the tuple (stream, definitive_file_name).
+    """
+    try:
+        if filename == u'-':
+            if sys.platform == 'win32':
+                import msvcrt
+                msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+            return (sys.stdout, filename)
+        stream = open(encodeFilename(filename), open_mode)
+        return (stream, filename)
+    except (IOError, OSError) as err:
+        # In case of error, try to remove win32 forbidden chars
+        filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
 
-		# An exception here should be caught in the caller
-		stream = open(encodeFilename(filename), open_mode)
-		return (stream, filename)
+        # An exception here should be caught in the caller
+        stream = open(encodeFilename(filename), open_mode)
+        return (stream, filename)
 
 
 def timeconvert(timestr):
-	"""Convert RFC 2822 defined time string into system timestamp"""
-	timestamp = None
-	timetuple = email.utils.parsedate_tz(timestr)
-	if timetuple is not None:
-		timestamp = email.utils.mktime_tz(timetuple)
-	return timestamp
+    """Convert RFC 2822 defined time string into system timestamp"""
+    timestamp = None
+    timetuple = email.utils.parsedate_tz(timestr)
+    if timetuple is not None:
+        timestamp = email.utils.mktime_tz(timetuple)
+    return timestamp
 
-def sanitize_filename(s, restricted=False):
-	"""Sanitizes a string so it could be used as part of a filename.
-	If restricted is set, use a stricter subset of allowed characters.
-	"""
-	def replace_insane(char):
-		if char == '?' or ord(char) < 32 or ord(char) == 127:
-			return ''
-		elif char == '"':
-			return '' if restricted else '\''
-		elif char == ':':
-			return '_-' if restricted else ' -'
-		elif char in '\\/|*<>':
-			return '_'
-		if restricted and (char in '!&\'' or char.isspace()):
-			return '_'
-		if restricted and ord(char) > 127:
-			return '_'
-		return char
+def sanitize_filename(s, restricted=False, is_id=False):
+    """Sanitizes a string so it could be used as part of a filename.
+    If restricted is set, use a stricter subset of allowed characters.
+    Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
+    """
+    def replace_insane(char):
+        if char == '?' or ord(char) < 32 or ord(char) == 127:
+            return ''
+        elif char == '"':
+            return '' if restricted else '\''
+        elif char == ':':
+            return '_-' if restricted else ' -'
+        elif char in '\\/|*<>':
+            return '_'
+        if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
+            return '_'
+        if restricted and ord(char) > 127:
+            return '_'
+        return char
 
-	result = u''.join(map(replace_insane, s))
-	while '__' in result:
-		result = result.replace('__', '_')
-	result = result.strip('_')
-	# Common case of "Foreign band name - English song title"
-	if restricted and result.startswith('-_'):
-		result = result[2:]
-	if not result:
-		result = '_'
-	return result
+    result = u''.join(map(replace_insane, s))
+    if not is_id:
+        while '__' in result:
+            result = result.replace('__', '_')
+        result = result.strip('_')
+        # Common case of "Foreign band name - English song title"
+        if restricted and result.startswith('-_'):
+            result = result[2:]
+        if not result:
+            result = '_'
+    return result
 
 def orderedSet(iterable):
-	""" Remove all duplicates from the input iterable """
-	res = []
-	for el in iterable:
-		if el not in res:
-			res.append(el)
-	return res
+    """ Remove all duplicates from the input iterable """
+    res = []
+    for el in iterable:
+        if el not in res:
+            res.append(el)
+    return res
 
 def unescapeHTML(s):
-	"""
-	@param s a string (of type unicode)
-	"""
-	assert type(s) == type(u'')
+    """
+    @param s a string
+    """
+    assert type(s) == type(u'')
 
-	result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s)
-	return result
+    result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
+    return result
 
 def encodeFilename(s):
-	"""
-	@param s The name of the file (of type unicode)
-	"""
+    """
+    @param s The name of the file
+    """
 
-	assert type(s) == type(u'')
+    assert type(s) == type(u'')
+
+    # Python 3 has a Unicode API
+    if sys.version_info >= (3, 0):
+        return s
+
+    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
+        # Pass u'' directly to use Unicode APIs on Windows 2000 and up
+        # (Detecting Windows NT 4 is tricky because 'major >= 4' would
+        # match Windows 9x series as well. Besides, NT 4 is obsolete.)
+        return s
+    else:
+        return s.encode(sys.getfilesystemencoding(), 'ignore')
+
+
+class ExtractorError(Exception):
+    """Error during info extraction."""
+    def __init__(self, msg, tb=None):
+        """ tb, if given, is the original traceback (so that it can be printed out). """
+        super(ExtractorError, self).__init__(msg)
+        self.traceback = tb
+
+    def format_traceback(self):
+        if self.traceback is None:
+            return None
+        return u''.join(traceback.format_tb(self.traceback))
 
-	if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
-		# Pass u'' directly to use Unicode APIs on Windows 2000 and up
-		# (Detecting Windows NT 4 is tricky because 'major >= 4' would
-		# match Windows 9x series as well. Besides, NT 4 is obsolete.)
-		return s
-	else:
-		return s.encode(sys.getfilesystemencoding(), 'ignore')
 
 class DownloadError(Exception):
-	"""Download Error exception.
+    """Download Error exception.
 
-	This exception may be thrown by FileDownloader objects if they are not
-	configured to continue on errors. They will contain the appropriate
-	error message.
-	"""
-	pass
+    This exception may be thrown by FileDownloader objects if they are not
+    configured to continue on errors. They will contain the appropriate
+    error message.
+    """
+    pass
 
 
 class SameFileError(Exception):
-	"""Same File exception.
+    """Same File exception.
 
-	This exception will be thrown by FileDownloader objects if they detect
-	multiple files would have to be downloaded to the same file on disk.
-	"""
-	pass
+    This exception will be thrown by FileDownloader objects if they detect
+    multiple files would have to be downloaded to the same file on disk.
+    """
+    pass
 
 
 class PostProcessingError(Exception):
-	"""Post Processing exception.
+    """Post Processing exception.
 
-	This exception may be raised by PostProcessor's .run() method to
-	indicate an error in the postprocessing task.
-	"""
-	pass
+    This exception may be raised by PostProcessor's .run() method to
+    indicate an error in the postprocessing task.
+    """
+    pass
 
 class MaxDownloadsReached(Exception):
-	""" --max-downloads limit has been reached. """
-	pass
+    """ --max-downloads limit has been reached. """
+    pass
 
 
 class UnavailableVideoError(Exception):
-	"""Unavailable Format exception.
+    """Unavailable Format exception.
 
-	This exception will be thrown when a video is requested
-	in a format that is not available for that video.
-	"""
-	pass
+    This exception will be thrown when a video is requested
+    in a format that is not available for that video.
+    """
+    pass
 
 
 class ContentTooShortError(Exception):
-	"""Content Too Short exception.
+    """Content Too Short exception.
 
-	This exception may be raised by FileDownloader objects when a file they
-	download is too small for what the server announced first, indicating
-	the connection was probably interrupted.
-	"""
-	# Both in bytes
-	downloaded = None
-	expected = None
+    This exception may be raised by FileDownloader objects when a file they
+    download is too small for what the server announced first, indicating
+    the connection was probably interrupted.
+    """
+    # Both in bytes
+    downloaded = None
+    expected = None
 
-	def __init__(self, downloaded, expected):
-		self.downloaded = downloaded
-		self.expected = expected
+    def __init__(self, downloaded, expected):
+        self.downloaded = downloaded
+        self.expected = expected
 
+class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
+    """Handler for HTTP requests and responses.
 
-class Trouble(Exception):
-	"""Trouble helper exception
+    This class, when installed with an OpenerDirector, automatically adds
+    the standard headers to every HTTP request and handles gzipped and
+    deflated responses from web servers. If compression is to be avoided in
+    a particular request, the original request in the program code only has
+    to include the HTTP header "Youtubedl-No-Compression", which will be
+    removed before making the real request.
 
-	This is an exception to be handled with
-	FileDownloader.trouble
-	"""
+    Part of this code was copied from:
 
-class YoutubeDLHandler(urllib2.HTTPHandler):
-	"""Handler for HTTP requests and responses.
+    http://techknack.net/python-urllib2-handlers/
 
-	This class, when installed with an OpenerDirector, automatically adds
-	the standard headers to every HTTP request and handles gzipped and
-	deflated responses from web servers. If compression is to be avoided in
-	a particular request, the original request in the program code only has
-	to include the HTTP header "Youtubedl-No-Compression", which will be
-	removed before making the real request.
+    Andrew Rowls, the author of that code, agreed to release it to the
+    public domain.
+    """
 
-	Part of this code was copied from:
+    @staticmethod
+    def deflate(data):
+        try:
+            return zlib.decompress(data, -zlib.MAX_WBITS)
+        except zlib.error:
+            return zlib.decompress(data)
 
-	http://techknack.net/python-urllib2-handlers/
+    @staticmethod
+    def addinfourl_wrapper(stream, headers, url, code):
+        if hasattr(compat_urllib_request.addinfourl, 'getcode'):
+            return compat_urllib_request.addinfourl(stream, headers, url, code)
+        ret = compat_urllib_request.addinfourl(stream, headers, url)
+        ret.code = code
+        return ret
 
-	Andrew Rowls, the author of that code, agreed to release it to the
-	public domain.
-	"""
+    def http_request(self, req):
+        for h in std_headers:
+            if h in req.headers:
+                del req.headers[h]
+            req.add_header(h, std_headers[h])
+        if 'Youtubedl-no-compression' in req.headers:
+            if 'Accept-encoding' in req.headers:
+                del req.headers['Accept-encoding']
+            del req.headers['Youtubedl-no-compression']
+        return req
 
-	@staticmethod
-	def deflate(data):
-		try:
-			return zlib.decompress(data, -zlib.MAX_WBITS)
-		except zlib.error:
-			return zlib.decompress(data)
+    def http_response(self, req, resp):
+        old_resp = resp
+        # gzip
+        if resp.headers.get('Content-encoding', '') == 'gzip':
+            gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
+            resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp.msg = old_resp.msg
+        # deflate
+        if resp.headers.get('Content-encoding', '') == 'deflate':
+            gz = io.BytesIO(self.deflate(resp.read()))
+            resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp.msg = old_resp.msg
+        return resp
 
-	@staticmethod
-	def addinfourl_wrapper(stream, headers, url, code):
-		if hasattr(urllib2.addinfourl, 'getcode'):
-			return urllib2.addinfourl(stream, headers, url, code)
-		ret = urllib2.addinfourl(stream, headers, url)
-		ret.code = code
-		return ret
-
-	def http_request(self, req):
-		for h in std_headers:
-			if h in req.headers:
-				del req.headers[h]
-			req.add_header(h, std_headers[h])
-		if 'Youtubedl-no-compression' in req.headers:
-			if 'Accept-encoding' in req.headers:
-				del req.headers['Accept-encoding']
-			del req.headers['Youtubedl-no-compression']
-		return req
-
-	def http_response(self, req, resp):
-		old_resp = resp
-		# gzip
-		if resp.headers.get('Content-encoding', '') == 'gzip':
-			gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
-			resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
-			resp.msg = old_resp.msg
-		# deflate
-		if resp.headers.get('Content-encoding', '') == 'deflate':
-			gz = StringIO.StringIO(self.deflate(resp.read()))
-			resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
-			resp.msg = old_resp.msg
-		return resp
+    https_request = http_request
+    https_response = http_response

youtube_dl/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '2013.01.11'