2016-10-02 11:39:18 +00:00
# coding: utf-8
2014-06-05 16:48:03 +00:00
from __future__ import unicode_literals
import re
from . common import InfoExtractor
2016-04-30 22:38:46 +00:00
from . . utils import (
determine_ext ,
2016-05-01 00:44:59 +00:00
js_to_json ,
parse_iso8601 ,
2016-04-30 22:38:46 +00:00
parse_filesize ,
)
2014-06-05 16:48:03 +00:00
2016-05-01 00:44:59 +00:00
class TagesschauPlayerIE ( InfoExtractor ) :
IE_NAME = ' tagesschau:player '
_VALID_URL = r ' https?://(?:www \ .)?tagesschau \ .de/multimedia/(?P<kind>audio|video)/(?P=kind)-(?P<id> \ d+)~player(?:_[^/?#&]+)? \ .html '
_TESTS = [ {
' url ' : ' http://www.tagesschau.de/multimedia/video/video-179517~player.html ' ,
' md5 ' : ' 8d09548d5c15debad38bee3a4d15ca21 ' ,
' info_dict ' : {
' id ' : ' 179517 ' ,
' ext ' : ' mp4 ' ,
' title ' : ' Marie Kristin Boese, ARD Berlin, über den zukünftigen Kurs der AfD ' ,
2017-01-02 12:08:07 +00:00
' thumbnail ' : r ' re:^https?:.* \ .jpg$ ' ,
2016-05-01 00:44:59 +00:00
' formats ' : ' mincount:6 ' ,
} ,
} , {
' url ' : ' https://www.tagesschau.de/multimedia/audio/audio-29417~player.html ' ,
' md5 ' : ' 76e6eec6ebd40740671cf0a2c88617e5 ' ,
' info_dict ' : {
' id ' : ' 29417 ' ,
' ext ' : ' mp3 ' ,
' title ' : ' Trabi - Bye, bye Rennpappe ' ,
2017-01-02 12:08:07 +00:00
' thumbnail ' : r ' re:^https?:.* \ .jpg$ ' ,
2016-05-01 00:44:59 +00:00
' formats ' : ' mincount:2 ' ,
} ,
} , {
' url ' : ' http://www.tagesschau.de/multimedia/audio/audio-29417~player_autoplay-true.html ' ,
' only_matching ' : True ,
} ]
_FORMATS = {
' xs ' : { ' quality ' : 0 } ,
' s ' : { ' width ' : 320 , ' height ' : 180 , ' quality ' : 1 } ,
' m ' : { ' width ' : 512 , ' height ' : 288 , ' quality ' : 2 } ,
' l ' : { ' width ' : 960 , ' height ' : 540 , ' quality ' : 3 } ,
' xl ' : { ' width ' : 1280 , ' height ' : 720 , ' quality ' : 4 } ,
' xxl ' : { ' quality ' : 5 } ,
}
def _extract_via_api ( self , kind , video_id ) :
info = self . _download_json (
' https://www.tagesschau.de/api/multimedia/ {0} / {0} - {1} .json ' . format ( kind , video_id ) ,
video_id )
title = info [ ' headline ' ]
formats = [ ]
for media in info [ ' mediadata ' ] :
for format_id , format_url in media . items ( ) :
if determine_ext ( format_url ) == ' m3u8 ' :
formats . extend ( self . _extract_m3u8_formats (
format_url , video_id , ' mp4 ' ,
entry_protocol = ' m3u8_native ' , m3u8_id = ' hls ' ) )
else :
formats . append ( {
' url ' : format_url ,
' format_id ' : format_id ,
' vcodec ' : ' none ' if kind == ' audio ' else None ,
} )
self . _sort_formats ( formats )
timestamp = parse_iso8601 ( info . get ( ' date ' ) )
return {
' id ' : video_id ,
' title ' : title ,
' timestamp ' : timestamp ,
' formats ' : formats ,
}
def _real_extract ( self , url ) :
mobj = re . match ( self . _VALID_URL , url )
video_id = mobj . group ( ' id ' )
# kind = mobj.group('kind').lower()
# if kind == 'video':
# return self._extract_via_api(kind, video_id)
# JSON api does not provide some audio formats (e.g. ogg) thus
# extractiong audio via webpage
webpage = self . _download_webpage ( url , video_id )
title = self . _og_search_title ( webpage ) . strip ( )
formats = [ ]
for media_json in re . findall ( r ' ( { src \ s*: \ s*[ " \' ]http[^}]+type \ s*:[^}]+}) ' , webpage ) :
media = self . _parse_json ( js_to_json ( media_json ) , video_id , fatal = False )
if not media :
continue
src = media . get ( ' src ' )
if not src :
return
quality = media . get ( ' quality ' )
kind = media . get ( ' type ' , ' ' ) . split ( ' / ' ) [ 0 ]
ext = determine_ext ( src )
f = {
' url ' : src ,
' format_id ' : ' %s _ %s ' % ( quality , ext ) if quality else ext ,
' ext ' : ext ,
' vcodec ' : ' none ' if kind == ' audio ' else None ,
}
f . update ( self . _FORMATS . get ( quality , { } ) )
formats . append ( f )
self . _sort_formats ( formats )
thumbnail = self . _og_search_thumbnail ( webpage )
return {
' id ' : video_id ,
' title ' : title ,
' thumbnail ' : thumbnail ,
' formats ' : formats ,
}
2014-06-05 16:48:03 +00:00
class TagesschauIE ( InfoExtractor ) :
2016-05-01 01:01:55 +00:00
_VALID_URL = r ' https?://(?:www \ .)?tagesschau \ .de/(?P<path>[^/]+/(?:[^/]+/)*?(?P<id>[^/#?]+?(?:-?[0-9]+)?))(?:~_?[^/#?]+?)? \ .html '
2014-06-05 16:48:03 +00:00
_TESTS = [ {
2015-07-19 01:52:54 +00:00
' url ' : ' http://www.tagesschau.de/multimedia/video/video-102143.html ' ,
2016-04-30 22:18:56 +00:00
' md5 ' : ' f7c27a0eff3bfe8c7727e65f8fe1b1e6 ' ,
2014-06-05 16:48:03 +00:00
' info_dict ' : {
2016-05-01 01:01:55 +00:00
' id ' : ' video-102143 ' ,
2014-06-05 16:48:03 +00:00
' ext ' : ' mp4 ' ,
2015-07-19 01:52:54 +00:00
' title ' : ' Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt ' ,
2016-05-01 00:44:59 +00:00
' description ' : ' 18.07.2015 20:10 Uhr ' ,
2017-01-02 12:08:07 +00:00
' thumbnail ' : r ' re:^https?:.* \ .jpg$ ' ,
2014-06-05 16:48:03 +00:00
} ,
2014-12-05 23:42:39 +00:00
} , {
' url ' : ' http://www.tagesschau.de/multimedia/sendung/ts-5727.html ' ,
' md5 ' : ' 3c54c1f6243d279b706bde660ceec633 ' ,
' info_dict ' : {
2016-05-01 01:01:55 +00:00
' id ' : ' ts-5727 ' ,
2014-12-05 23:42:39 +00:00
' ext ' : ' mp4 ' ,
' title ' : ' Sendung: tagesschau \t 04.12.2014 20:00 Uhr ' ,
2016-05-01 00:44:59 +00:00
' description ' : ' md5:695c01bfd98b7e313c501386327aea59 ' ,
2017-01-02 12:08:07 +00:00
' thumbnail ' : r ' re:^https?:.* \ .jpg$ ' ,
2016-05-01 00:44:59 +00:00
} ,
} , {
# exclusive audio
' url ' : ' http://www.tagesschau.de/multimedia/audio/audio-29417.html ' ,
' md5 ' : ' 76e6eec6ebd40740671cf0a2c88617e5 ' ,
' info_dict ' : {
2016-05-01 01:01:55 +00:00
' id ' : ' audio-29417 ' ,
2016-05-01 00:44:59 +00:00
' ext ' : ' mp3 ' ,
' title ' : ' Trabi - Bye, bye Rennpappe ' ,
' description ' : ' md5:8687dda862cbbe2cfb2df09b56341317 ' ,
2017-01-02 12:08:07 +00:00
' thumbnail ' : r ' re:^https?:.* \ .jpg$ ' ,
2015-07-22 18:30:48 +00:00
} ,
2015-07-22 19:01:04 +00:00
} , {
2016-05-01 00:44:59 +00:00
# audio in article
' url ' : ' http://www.tagesschau.de/inland/bnd-303.html ' ,
' md5 ' : ' e0916c623e85fc1d2b26b78f299d3958 ' ,
2015-07-22 19:01:04 +00:00
' info_dict ' : {
2016-05-01 01:01:55 +00:00
' id ' : ' bnd-303 ' ,
2015-07-22 19:01:04 +00:00
' ext ' : ' mp3 ' ,
2016-05-01 00:44:59 +00:00
' title ' : ' Viele Baustellen für neuen BND-Chef ' ,
' description ' : ' md5:1e69a54be3e1255b2b07cdbce5bcd8b4 ' ,
2017-01-02 12:08:07 +00:00
' thumbnail ' : r ' re:^https?:.* \ .jpg$ ' ,
2015-07-22 19:01:04 +00:00
} ,
2016-04-30 22:18:56 +00:00
} , {
' url ' : ' http://www.tagesschau.de/inland/afd-parteitag-135.html ' ,
' info_dict ' : {
2016-05-01 01:01:55 +00:00
' id ' : ' afd-parteitag-135 ' ,
2016-04-30 22:18:56 +00:00
' title ' : ' Möchtegern-Underdog mit Machtanspruch ' ,
} ,
' playlist_count ' : 2 ,
2015-07-19 01:34:22 +00:00
} , {
' url ' : ' http://www.tagesschau.de/multimedia/sendung/tsg-3771.html ' ,
2015-07-22 18:30:48 +00:00
' only_matching ' : True ,
2015-07-19 01:34:22 +00:00
} , {
' url ' : ' http://www.tagesschau.de/multimedia/sendung/tt-3827.html ' ,
2015-07-22 18:30:48 +00:00
' only_matching ' : True ,
2015-07-19 01:34:22 +00:00
} , {
' url ' : ' http://www.tagesschau.de/multimedia/sendung/nm-3475.html ' ,
2015-07-22 18:30:48 +00:00
' only_matching ' : True ,
} , {
' url ' : ' http://www.tagesschau.de/multimedia/sendung/weltspiegel-3167.html ' ,
' only_matching ' : True ,
2015-07-19 01:34:22 +00:00
} , {
' url ' : ' http://www.tagesschau.de/multimedia/tsvorzwanzig-959.html ' ,
2015-07-22 18:30:48 +00:00
' only_matching ' : True ,
2015-07-19 02:27:58 +00:00
} , {
' url ' : ' http://www.tagesschau.de/multimedia/sendung/bab/bab-3299~_bab-sendung-209.html ' ,
2015-07-22 18:30:48 +00:00
' only_matching ' : True ,
} , {
' url ' : ' http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html ' ,
' only_matching ' : True ,
2016-05-01 00:57:19 +00:00
} , {
' url ' : ' http://www.tagesschau.de/100sekunden/index.html ' ,
' only_matching ' : True ,
2016-05-01 01:15:23 +00:00
} , {
# playlist article with collapsing sections
' url ' : ' http://www.tagesschau.de/wirtschaft/faq-freihandelszone-eu-usa-101.html ' ,
' only_matching ' : True ,
2014-06-07 13:14:20 +00:00
} ]
2016-05-01 00:44:59 +00:00
@classmethod
def suitable ( cls , url ) :
return False if TagesschauPlayerIE . suitable ( url ) else super ( TagesschauIE , cls ) . suitable ( url )
2014-06-05 16:48:03 +00:00
2016-04-30 22:38:46 +00:00
def _extract_formats ( self , download_text , media_kind ) :
2016-04-30 22:18:56 +00:00
links = re . finditer (
r ' <div class= " button " title= " (?P<title>[^ " ]*) " ><a href= " (?P<url>[^ " ]+) " >(?P<name>.+?)</a></div> ' ,
download_text )
formats = [ ]
for l in links :
2016-04-30 22:38:46 +00:00
link_url = l . group ( ' url ' )
if not link_url :
continue
2016-04-30 22:18:56 +00:00
format_id = self . _search_regex (
2016-04-30 22:38:46 +00:00
r ' .*/[^/.]+ \ .([^/]+) \ .[^/.]+$ ' , link_url , ' format ID ' ,
default = determine_ext ( link_url ) )
2016-04-30 22:18:56 +00:00
format = {
' format_id ' : format_id ,
' url ' : l . group ( ' url ' ) ,
' format_name ' : l . group ( ' name ' ) ,
}
2016-04-30 22:38:46 +00:00
title = l . group ( ' title ' )
if title :
if media_kind . lower ( ) == ' video ' :
m = re . match (
r ''' (?x)
Video : \s * ( ? P < vcodec > [ a - zA - Z0 - 9 / . _ - ] + ) \s * & \#10;
( ? P < width > [ 0 - 9 ] + ) x ( ? P < height > [ 0 - 9 ] + ) px & \#10;
( ? P < vbr > [ 0 - 9 ] + ) kbps & \#10;
Audio : \s * ( ? P < abr > [ 0 - 9 ] + ) kbps , \s * ( ? P < audio_desc > [ A - Za - z \.0 - 9 ] + ) & \#10;
Gr & ouml ; & szlig ; e : \s * ( ? P < filesize_approx > [ 0 - 9. , ] + \s + [ a - zA - Z ] * B ) ''' ,
title )
if m :
format . update ( {
' format_note ' : m . group ( ' audio_desc ' ) ,
' vcodec ' : m . group ( ' vcodec ' ) ,
' width ' : int ( m . group ( ' width ' ) ) ,
' height ' : int ( m . group ( ' height ' ) ) ,
' abr ' : int ( m . group ( ' abr ' ) ) ,
' vbr ' : int ( m . group ( ' vbr ' ) ) ,
' filesize_approx ' : parse_filesize ( m . group ( ' filesize_approx ' ) ) ,
} )
else :
m = re . match (
r ' (?P<format>.+?)-Format \ s*: \ s*(?P<abr> \ d+)kbps \ s*, \ s*(?P<note>.+) ' ,
title )
if m :
format . update ( {
' format_note ' : ' %s , %s ' % ( m . group ( ' format ' ) , m . group ( ' note ' ) ) ,
' vcodec ' : ' none ' ,
' abr ' : int ( m . group ( ' abr ' ) ) ,
} )
2016-04-30 22:18:56 +00:00
formats . append ( format )
self . _sort_formats ( formats )
return formats
2014-06-05 16:48:03 +00:00
def _real_extract ( self , url ) :
2016-05-01 00:57:19 +00:00
mobj = re . match ( self . _VALID_URL , url )
video_id = mobj . group ( ' id ' ) or mobj . group ( ' path ' )
2014-12-05 09:59:55 +00:00
display_id = video_id . lstrip ( ' - ' )
2016-05-01 00:44:59 +00:00
2014-06-05 16:48:03 +00:00
webpage = self . _download_webpage ( url , display_id )
2016-05-01 00:44:59 +00:00
title = self . _html_search_regex (
r ' <span[^>]*class= " headline " [^>]*>(.+?)</span> ' ,
webpage , ' title ' , default = None ) or self . _og_search_title ( webpage )
DOWNLOAD_REGEX = r ' (?s)<p>Wir bieten dieses (?P<kind>Video|Audio) in folgenden Formaten zum Download an:</p> \ s*<div class= " controls " >(?P<links>.*?)</div> \ s*<p> '
webpage_type = self . _og_search_property ( ' type ' , webpage , default = None )
if webpage_type == ' website ' : # Article
entries = [ ]
for num , ( entry_title , media_kind , download_text ) in enumerate ( re . findall (
2016-05-01 01:15:23 +00:00
r ' (?s)<p[^>]+class= " infotext " [^>]*> \ s*(?:<a[^>]+>)? \ s*<strong>(.+?)</strong>.*?</p>.*? %s ' % DOWNLOAD_REGEX ,
2016-05-01 00:44:59 +00:00
webpage ) , 1 ) :
entries . append ( {
' id ' : ' %s - %d ' % ( display_id , num ) ,
' title ' : ' %s ' % entry_title ,
' formats ' : self . _extract_formats ( download_text , media_kind ) ,
} )
if len ( entries ) > 1 :
2016-04-30 22:18:56 +00:00
return self . playlist_result ( entries , display_id , title )
2016-05-01 00:44:59 +00:00
formats = entries [ 0 ] [ ' formats ' ]
else : # Assume single video
download_text = self . _search_regex (
DOWNLOAD_REGEX , webpage , ' download links ' , group = ' links ' )
media_kind = self . _search_regex (
DOWNLOAD_REGEX , webpage , ' media kind ' , default = ' Video ' , group = ' kind ' )
formats = self . _extract_formats ( download_text , media_kind )
thumbnail = self . _og_search_thumbnail ( webpage )
description = self . _html_search_regex (
r ' (?s)<p class= " teasertext " >(.*?)</p> ' ,
webpage , ' description ' , default = None )
2016-04-30 22:18:56 +00:00
2014-06-05 16:48:03 +00:00
self . _sort_formats ( formats )
return {
' id ' : display_id ,
2014-12-05 23:42:39 +00:00
' title ' : title ,
' thumbnail ' : thumbnail ,
2014-06-05 16:48:03 +00:00
' formats ' : formats ,
2014-12-05 23:42:39 +00:00
' description ' : description ,
2014-06-05 16:48:03 +00:00
}