Release 2011.10.19

Merge branch 'vimeo' of https://github.com/rbrito/youtube-dl
vimeo: Add the ability to detect if a video is available in HD. (Closes: #194 )
2011-10-19 00:40:13 +02:00 · 2011-10-19 00:38:41 +02:00 · 2011-10-19 00:37:45 +02:00 · 2011-10-18 19:47:19 -02:00 · 2011-10-18 13:49:21 -07:00 · 2011-10-18 18:42:01 -02:00
2 changed files with 181 additions and 5 deletions
--- a/2
+++ b/2
@@ -1 +1 @@
-2011.09.30
+2011.10.19
--- a/184
+++ b/184
@@ -15,7 +15,7 @@ __author__  = (
 	)

 __license__ = 'Public Domain'
-__version__ = '2011.09.30'
+__version__ = '2011.10.19'

 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'

@@ -1236,7 +1236,7 @@ class YoutubeIE(InfoExtractor):

 		# Get video webpage
 		self.report_video_webpage_download(video_id)
-		request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&amp;has_verified=1' % video_id)
+		request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
 		try:
 			video_webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
@@ -2059,6 +2059,18 @@ class VimeoIE(InfoExtractor):
 			return
 		sig = mobj.group(1).decode('utf-8')

+		# Vimeo specific: extract video quality information
+		mobj = re.search(r'<isHD>(\d+)</isHD>', webpage)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: unable to extract video quality information')
+			return
+		quality = mobj.group(1).decode('utf-8')
+
+		if int(quality) == 1:
+			quality = 'hd'
+		else:
+			quality = 'sd'
+
 		# Vimeo specific: Extract request signature expiration
 		mobj = re.search(r'<request_signature_expires>(.*?)</request_signature_expires>', webpage)
 		if mobj is None:
@@ -2066,7 +2078,7 @@ class VimeoIE(InfoExtractor):
 			return
 		sig_exp = mobj.group(1).decode('utf-8')

-		video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s" % (video_id, sig, sig_exp)
+		video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s/?q=%s" % (video_id, sig, sig_exp, quality)

 		try:
 			# Process video information
@@ -2458,7 +2470,7 @@ class YahooSearchIE(InfoExtractor):
 class YoutubePlaylistIE(InfoExtractor):
 	"""Information Extractor for YouTube playlists."""

-	_VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)([0-9A-Za-z]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
+	_VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
 	_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
 	_VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
 	_MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
@@ -3306,6 +3318,168 @@ class EscapistIE(InfoExtractor):
 			self._downloader.trouble(u'\nERROR: unable to download ' + videoId)


+class CollegeHumorIE(InfoExtractor):
+	"""Information extractor for collegehumor.com"""
+
+	_VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
+	IE_NAME = u'collegehumor'
+
+	def report_webpage(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+	def report_extraction(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+	def _simplify_title(self, title):
+		res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
+		res = res.strip(ur'_')
+		return res
+
+	def _real_extract(self, url):
+		htmlParser = HTMLParser.HTMLParser()
+
+		mobj = re.match(self._VALID_URL, url)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+			return
+		video_id = mobj.group('videoid')
+
+		self.report_webpage(video_id)
+		request = urllib2.Request(url)
+		try:
+			webpage = urllib2.urlopen(request).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			return
+
+		m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
+		if m is None:
+			self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
+			return
+		internal_video_id = m.group('internalvideoid')
+
+		info = {
+			'id': video_id,
+			'internal_id': internal_video_id,
+		}
+
+		self.report_extraction(video_id)
+		xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
+		try:
+			metaXml = urllib2.urlopen(xmlUrl).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % str(err))
+			return
+
+		mdoc = xml.etree.ElementTree.fromstring(metaXml)
+		try:
+			videoNode = mdoc.findall('./video')[0]
+			info['description'] = videoNode.findall('./description')[0].text
+			info['title'] = videoNode.findall('./caption')[0].text
+			info['stitle'] = self._simplify_title(info['title'])
+			info['url'] = videoNode.findall('./file')[0].text
+			info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
+			info['ext'] = info['url'].rpartition('.')[2]
+			info['format'] = info['ext']
+		except IndexError:
+			self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
+			return
+
+		self._downloader.increment_downloads()
+
+		try:
+			self._downloader.process_info(info)
+		except UnavailableVideoError, err:
+			self._downloader.trouble(u'\nERROR: unable to download video')
+
+
+class XVideosIE(InfoExtractor):
+	"""Information extractor for xvideos.com"""
+
+	_VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
+	IE_NAME = u'xvideos'
+
+	def report_webpage(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+	def report_extraction(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+	def _simplify_title(self, title):
+		res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
+		res = res.strip(ur'_')
+		return res
+
+	def _real_extract(self, url):
+		htmlParser = HTMLParser.HTMLParser()
+
+		mobj = re.match(self._VALID_URL, url)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+			return
+		video_id = mobj.group(1).decode('utf-8')
+
+		self.report_webpage(video_id)
+
+		request = urllib2.Request(r'http://www.xvideos.com/video' + video_id)
+		try:
+			webpage = urllib2.urlopen(request).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			return
+
+		self.report_extraction(video_id)
+
+
+		# Extract video URL
+		mobj = re.search(r'flv_url=(.+?)&', webpage)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: unable to extract video url')
+			return
+		video_url = urllib2.unquote(mobj.group(1).decode('utf-8'))
+
+
+		# Extract title
+		mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: unable to extract video title')
+			return
+		video_title = mobj.group(1).decode('utf-8')
+
+
+		# Extract video thumbnail
+		mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]/[a-fA-F0-9]/[a-fA-F0-9]/([a-fA-F0-9.]+jpg)', webpage)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
+			return
+		video_thumbnail = mobj.group(1).decode('utf-8')
+
+
+
+		self._downloader.increment_downloads()
+		info = {
+			'id': video_id,
+			'url': video_url,
+			'uploader': None,
+			'upload_date': None,
+			'title': video_title,
+			'stitle': self._simplify_title(video_title),
+			'ext': 'flv',
+			'format': 'flv',
+			'thumbnail': video_thumbnail,
+			'description': None,
+			'player_url': None,
+		}
+
+		try:
+			self._downloader.process_info(info)
+		except UnavailableVideoError, err:
+			self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
+

 class PostProcessor(object):
 	"""Post Processor class.
@@ -3701,6 +3875,8 @@ def gen_extractors():
 		MyVideoIE(),
 		ComedyCentralIE(),
 		EscapistIE(),
+		CollegeHumorIE(),
+		XVideosIE(),

 		GenericIE()
 	]
Author	SHA1	Message	Date
Philipp Hagemeister	45aa690868	Release 2011.10.19	2011-10-19 00:40:13 +02:00
Philipp Hagemeister	beb245e92f	Merge branch 'vimeo' of https://github.com/rbrito/youtube-dl	2011-10-19 00:38:41 +02:00
Rogério Brito	c424df0d2f	vimeo: Add the ability to detect if a video is available in HD. (Closes: #194 )	2011-10-19 00:37:45 +02:00
Rogério Brito	87929e4b35	vimeo: Add the ability to detect if a video is available in HD.	2011-10-18 19:47:19 -02:00
Philipp Hagemeister	d76736fc5e	Merge pull request #195 from rbrito/xvideos Fixes for the xvideos IE	2011-10-18 13:49:21 -07:00
Rogério Brito	0f9b77223e	xvideos: Capture only the video title, not the name of the site.	2011-10-18 18:42:01 -02:00
Rogério Brito	9f47175a40	xvideos: Fix misleading error message when extracting the URL. We said that we were trying to extract the title of the video.	2011-10-18 18:41:02 -02:00
Rogério Brito	a1a8713aad	xvideos: Normalize the URL or it will fail with some inputs. For instance, if we give it <www.xvideos.com/video1384059>, we would end up passing that to urllib2, which would complain about an unknown URL type.	2011-10-18 18:38:17 -02:00
Rogério Brito	6501a06d46	Quick and dirty IE for xvideos.com.	2011-10-13 16:44:20 -03:00
Philipp Hagemeister	8d89fbae5a	CollegeHumor IE	2011-10-12 21:13:43 +02:00
Philipp Hagemeister	7a2cf5455c	Fix recognition of http://www.youtube.com/course?list=PL41FDABC6AA085E78&category=University/Mathematics/Topology%20%26%20Geometry	2011-10-04 03:25:20 +02:00
Rogério Brito	7125a7ca8b	Support "newstyle" Youtube playlist IDs. Many playlists reported by Youtube now have the form like in: http://www.youtube.com/playlist?list=PL520044A3524F5E5D where `PL` is prefixed to what youtube-dl used to use as playlist IDs. So, while matching it, we adapt the regular expression so as to discard the `PL` and only get the `520044A3524F5E5D` in the case of the playlist of the example.	2011-10-03 21:41:33 -03:00
@@ -1 +1 @@
 .09.30
 .10.19