1
0
mirror of https://github.com/TeamNewPipe/NewPipe synced 2024-12-24 00:50:32 +00:00

quick and dirty solution for download/audio bug

Youtube has changed (again) this time it was a bit harder,
therfor the changes are deeper. May be still a bit unstable.
This commit is contained in:
Christian Schabesberger 2016-02-12 01:29:14 +01:00
parent 60d636940d
commit 751ffb9de9
15 changed files with 308 additions and 130 deletions

View File

@ -37,6 +37,7 @@ public class YoutubeSearchEngineTest extends AndroidTestCase {
public void setUp() throws Exception{
super.setUp();
SearchEngine engine = new YoutubeSearchEngine();
result = engine.search("https://www.youtube.com/results?search_query=bla",
0, "de", new Downloader());
suggestionReply = engine.suggestionList("hello", new Downloader());

View File

@ -1,12 +1,11 @@
package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import android.util.Log;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.services.youtube.YoutubeStreamExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import java.io.IOException;
@ -31,11 +30,15 @@ import java.io.IOException;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
private YoutubeVideoExtractor extractor;
public class YoutubeStreamExtractorDefaultTest extends AndroidTestCase {
private YoutubeStreamExtractor extractor;
public void setUp() throws IOException, CrawlingException {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys",
/* some anonymus video test
extractor = new YoutubeStreamExtractor("https://www.youtube.com/watch?v=FmG385_uUys",
new Downloader()); */
/* some vevo video (suggested to test against) */
extractor = new YoutubeStreamExtractor("https://www.youtube.com/watch?v=YQHsXMglC9A",
new Downloader());
}
@ -45,8 +48,8 @@ public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
}
public void testGetValidTimeStamp() throws CrawlingException, IOException {
YoutubeVideoExtractor extractor =
new YoutubeVideoExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader());
YoutubeStreamExtractor extractor =
new YoutubeStreamExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader());
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() == 174);
}
@ -86,7 +89,7 @@ public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
}
public void testGetAudioStreams() throws ParsingException {
assertTrue(extractor.getAudioStreams() == null);
assertTrue(!extractor.getAudioStreams().isEmpty());
}
public void testGetVideoStreams() throws ParsingException {
@ -98,4 +101,9 @@ public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
0 <= s.format && s.format <= 4);
}
}
public void testGetDashMpd() throws ParsingException {
assertTrue(extractor.getDashMpdUrl(),
!extractor.getDashMpdUrl().isEmpty());
}
}

View File

@ -4,9 +4,7 @@ import android.test.AndroidTestCase;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.services.youtube.YoutubeStreamExtractor;
import java.io.IOException;
@ -32,7 +30,7 @@ import java.io.IOException;
// This class only works in Germany.
public class YoutubeVideoExtractorGemaTest extends AndroidTestCase {
public class YoutubeStreamExtractorGemaTest extends AndroidTestCase {
// Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail.
private static final boolean testActive = false;
@ -40,10 +38,10 @@ public class YoutubeVideoExtractorGemaTest extends AndroidTestCase {
public void testGemaError() throws IOException, CrawlingException {
if(testActive) {
try {
new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8",
new YoutubeStreamExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8",
new Downloader());
assertTrue("Gema exception not thrown", false);
} catch(YoutubeVideoExtractor.GemaException ge) {
} catch(YoutubeStreamExtractor.GemaException ge) {
assertTrue(true);
}
}

View File

@ -71,7 +71,7 @@ public class VideoItemDetailActivity extends AppCompatActivity {
if (getIntent().getData() != null) {
videoUrl = getIntent().getData().toString();
StreamingService[] serviceList = ServiceList.getServices();
//VideoExtractor videoExtractor = null;
//StreamExtractor videoExtractor = null;
for (int i = 0; i < serviceList.length; i++) {
if (serviceList[i].getUrlIdHandler().acceptUrl(videoUrl)) {
arguments.putInt(VideoItemDetailFragment.STREAMING_SERVICE, i);

View File

@ -49,11 +49,11 @@ import java.util.Vector;
import org.schabi.newpipe.crawler.MediaFormat;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.ServiceList;
import org.schabi.newpipe.crawler.StreamExtractor;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.services.youtube.YoutubeStreamExtractor;
/**
@ -115,7 +115,7 @@ public class VideoItemDetailFragment extends Fragment {
private class VideoExtractorRunnable implements Runnable {
private final Handler h = new Handler();
private VideoExtractor videoExtractor;
private StreamExtractor streamExtractor;
private final StreamingService service;
private final String videoUrl;
@ -127,8 +127,8 @@ public class VideoItemDetailFragment extends Fragment {
@Override
public void run() {
try {
videoExtractor = service.getExtractorInstance(videoUrl, new Downloader());
VideoInfo videoInfo = VideoInfo.getVideoInfo(videoExtractor, new Downloader());
streamExtractor = service.getExtractorInstance(videoUrl, new Downloader());
VideoInfo videoInfo = VideoInfo.getVideoInfo(streamExtractor, new Downloader());
h.post(new VideoResultReturnedRunnable(videoInfo));
} catch (IOException e) {
@ -136,10 +136,10 @@ public class VideoItemDetailFragment extends Fragment {
e.printStackTrace();
}
// custom service related exceptions
catch (YoutubeVideoExtractor.DecryptException de) {
catch (YoutubeStreamExtractor.DecryptException de) {
postNewErrorToast(h, R.string.youtube_signature_decryption_error);
de.printStackTrace();
} catch (YoutubeVideoExtractor.GemaException ge) {
} catch (YoutubeStreamExtractor.GemaException ge) {
h.post(new Runnable() {
@Override
public void run() {
@ -148,7 +148,7 @@ public class VideoItemDetailFragment extends Fragment {
});
}
// ----------------------------------------
catch(VideoExtractor.ContentNotAvailableException e) {
catch(StreamExtractor.ContentNotAvailableException e) {
h.post(new Runnable() {
@Override
public void run() {

View File

@ -74,6 +74,7 @@ public class DashMpdParser {
break;
case XmlPullParser.TEXT:
// actual stream tag
if(currentTagIsBaseUrl &&
(currentMimeType.contains("audio"))) {
int format = -1;
@ -86,12 +87,14 @@ public class DashMpdParser {
format, currentBandwidth, currentSamplingRate));
}
break;
case XmlPullParser.END_TAG:
if(tagName.equals("AdaptationSet")) {
currentMimeType = "";
} else if(tagName.equals("BaseURL")) {
currentTagIsBaseUrl = false;
}//no break needed here
}
break;
}
}
} catch(Exception e) {

View File

@ -25,10 +25,12 @@ package org.schabi.newpipe.crawler;
/**Static data about various media formats support by Newpipe, eg mime type, extension*/
public enum MediaFormat {
//video and audio combined formats
// id name suffix mime type
MPEG_4 (0x0, "MPEG-4", "mp4", "video/mp4"),
v3GPP (0x1, "3GPP", "3gp", "video/3gpp"),
WEBM (0x2, "WebM", "webm", "video/webm"),
// audio formats
M4A (0x3, "m4a", "m4a", "audio/mp4"),
WEBMA (0x4, "WebM", "webm", "audio/webm");

View File

@ -1,5 +1,9 @@
package org.schabi.newpipe.crawler;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -7,7 +11,7 @@ import java.util.regex.Pattern;
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* RegexHelper.java is part of NewPipe.
* Parser.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -24,7 +28,7 @@ import java.util.regex.Pattern;
*/
/** avoid using regex !!! */
public class RegexHelper {
public class Parser {
public static class RegexException extends ParsingException {
public RegexException(String message) {
@ -44,4 +48,13 @@ public class RegexHelper {
throw new RegexException("failed to find pattern \""+pattern+" inside of "+input+"\"");
}
}
public static Map<String, String> compatParseMap(final String input) throws UnsupportedEncodingException {
Map<String, String> map = new HashMap<>();
for(String arg : input.split("&")) {
String[] split_arg = arg.split("=");
map.put(split_arg[0], URLDecoder.decode(split_arg[1], "UTF-8"));
}
return map;
}
}

View File

@ -4,7 +4,7 @@ package org.schabi.newpipe.crawler;
* Created by Christian Schabesberger on 10.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* VideoExtractor.java is part of NewPipe.
* StreamExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -26,7 +26,7 @@ import java.util.List;
@SuppressWarnings("ALL")
public interface VideoExtractor {
public interface StreamExtractor {
public class ExctractorInitException extends CrawlingException {
public ExctractorInitException() {}
@ -65,6 +65,7 @@ public interface VideoExtractor {
public abstract String getUploaderThumbnailUrl() throws ParsingException;
public abstract List<VideoInfo.AudioStream> getAudioStreams() throws ParsingException;
public abstract List<VideoInfo.VideoStream> getVideoStreams() throws ParsingException;
public abstract List<VideoInfo.VideoStream> getVideoOnlyStreams() throws ParsingException;
public abstract String getDashMpdUrl() throws ParsingException;
public abstract int getAgeLimit() throws ParsingException;
public abstract String getAverageRating() throws ParsingException;

View File

@ -27,7 +27,7 @@ public interface StreamingService {
public String name = "";
}
ServiceInfo getServiceInfo();
VideoExtractor getExtractorInstance(String url, Downloader downloader)
StreamExtractor getExtractorInstance(String url, Downloader downloader)
throws IOException, CrawlingException;
SearchEngine getSearchEngineInstance();

View File

@ -30,7 +30,7 @@ public class VideoInfo extends AbstractVideoInfo {
/**Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses*/
public static VideoInfo getVideoInfo(VideoExtractor extractor, Downloader downloader)
public static VideoInfo getVideoInfo(StreamExtractor extractor, Downloader downloader)
throws CrawlingException, IOException {
VideoInfo videoInfo = new VideoInfo();
@ -46,18 +46,34 @@ public class VideoInfo extends AbstractVideoInfo {
videoInfo.upload_date = extractor.getUploadDate();
videoInfo.thumbnail_url = extractor.getThumbnailUrl();
videoInfo.id = uiconv.getVideoId(extractor.getPageUrl());
//todo: make this quick and dirty solution a real fallback
// The front end should be notified that the dash mpd could not be downloaded
// although not getting the dash mpd is not the end of the world, therfore
// we continue.
try {
videoInfo.dashMpdUrl = extractor.getDashMpdUrl();
} catch(Exception e) {
e.printStackTrace();
}
/** Load and extract audio*/
videoInfo.audio_streams = extractor.getAudioStreams();
if(videoInfo.dashMpdUrl != null && !videoInfo.dashMpdUrl.isEmpty()) {
if(videoInfo.audio_streams == null) {
videoInfo.audio_streams = new Vector<AudioStream>();
}
//todo: make this quick and dirty solution a real fallback
// same as the quick and dirty aboth
try {
videoInfo.audio_streams.addAll(
DashMpdParser.getAudioStreams(videoInfo.dashMpdUrl, downloader));
} catch(Exception e) {
e.printStackTrace();
}
}
/** Extract video stream url*/
videoInfo.video_streams = extractor.getVideoStreams();
/** Extract video only stream url*/
videoInfo.video_only_streams = extractor.getVideoOnlyStreams();
videoInfo.uploader_thumbnail_url = extractor.getUploaderThumbnailUrl();
videoInfo.start_position = extractor.getTimeStamp();
videoInfo.average_rating = extractor.getAverageRating();
@ -78,6 +94,7 @@ public class VideoInfo extends AbstractVideoInfo {
/*todo: make this lists over vectors*/
public List<VideoStream> video_streams = null;
public List<AudioStream> audio_streams = null;
public List<VideoStream> video_only_streams = null;
// video streams provided by the dash mpd do not need to be provided as VideoStream.
// Later on this will also aplly to audio streams. Since dash mpd is standarized,
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
@ -132,6 +149,18 @@ public class VideoInfo extends AbstractVideoInfo {
public VideoStream(String url, int format, String res) {
this.url = url; this.format = format; resolution = res;
}
// reveals wether two streams are the same, but have diferent urls
public boolean equalStats(VideoStream cmp) {
return format == cmp.format
&& resolution == cmp.resolution;
}
// revelas wether two streams are equal
public boolean equals(VideoStream cmp) {
return equalStats(cmp)
&& url == cmp.url;
}
}
@SuppressWarnings("unused")
@ -145,5 +174,18 @@ public class VideoInfo extends AbstractVideoInfo {
this.url = url; this.format = format;
this.bandwidth = bandwidth; this.sampling_rate = samplingRate;
}
// reveals wether two streams are the same, but have diferent urls
public boolean equalStats(AudioStream cmp) {
return format == cmp.format
&& bandwidth == cmp.bandwidth
&& sampling_rate == cmp.sampling_rate;
}
// revelas wether two streams are equal
public boolean equals(AudioStream cmp) {
return equalStats(cmp)
&& url == cmp.url;
}
}
}

View File

@ -6,11 +6,9 @@ import android.util.Log;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

View File

@ -2,9 +2,9 @@ package org.schabi.newpipe.crawler.services.youtube;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.StreamExtractor;
import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoUrlIdHandler;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.SearchEngine;
import java.io.IOException;
@ -38,11 +38,11 @@ public class YoutubeService implements StreamingService {
return serviceInfo;
}
@Override
public VideoExtractor getExtractorInstance(String url, Downloader downloader)
public StreamExtractor getExtractorInstance(String url, Downloader downloader)
throws CrawlingException, IOException {
VideoUrlIdHandler urlIdHandler = new YoutubeVideoUrlIdHandler();
if(urlIdHandler.acceptUrl(url)) {
return new YoutubeVideoExtractor(url, downloader) ;
return new YoutubeStreamExtractor(url, downloader) ;
}
else {
throw new IllegalArgumentException("supplied String is not a valid Youtube URL");

View File

@ -1,5 +1,6 @@
package org.schabi.newpipe.crawler.services.youtube;
import android.provider.MediaStore;
import android.util.Log;
import org.json.JSONException;
@ -7,16 +8,15 @@ import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Parser;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.Parser;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.RegexHelper;
import org.schabi.newpipe.crawler.VideoUrlIdHandler;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.StreamExtractor;
import org.schabi.newpipe.crawler.MediaFormat;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
@ -32,7 +32,7 @@ import java.util.Vector;
* Created by Christian Schabesberger on 06.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeVideoExtractor.java is part of NewPipe.
* YoutubeStreamExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -48,7 +48,98 @@ import java.util.Vector;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeVideoExtractor implements VideoExtractor {
public class YoutubeStreamExtractor implements StreamExtractor {
public enum ItagType {
AUDIO,
VIDEO,
VIDEO_ONLY
}
private static class ItagItem {
public ItagItem(int id, ItagType type, MediaFormat format, String res, int fps) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.resolutionString = res;
this.fps = fps;
}
public ItagItem(int id, ItagType type, MediaFormat format, int samplingRate, int bandWidth) {
this.id = id;
this.itagType = type;
this.mediaFormatId = format.id;
this.samplingRate = samplingRate;
this.bandWidth = bandWidth;
}
public int id;
public ItagType itagType;
public int mediaFormatId;
public String resolutionString = null;
public int fps = -1;
public int samplingRate = -1;
public int bandWidth = -1;
}
private static final ItagItem[] itagList = {
// video streams
// id, ItagType, MediaFormat, Resolution, fps
new ItagItem(17, ItagType.VIDEO, MediaFormat.v3GPP, "144p", 12),
new ItagItem(18, ItagType.VIDEO, MediaFormat.MPEG_4, "360p", 24),
new ItagItem(22, ItagType.VIDEO, MediaFormat.MPEG_4, "720p", 24),
new ItagItem(36, ItagType.VIDEO, MediaFormat.v3GPP, "240p", 24),
new ItagItem(37, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p", 24),
new ItagItem(38, ItagType.VIDEO, MediaFormat.MPEG_4, "1080p", 24),
new ItagItem(43, ItagType.VIDEO, MediaFormat.WEBM, "360p", 24),
new ItagItem(44, ItagType.VIDEO, MediaFormat.WEBM, "480p", 24),
new ItagItem(45, ItagType.VIDEO, MediaFormat.WEBM, "720p", 24),
new ItagItem(46, ItagType.VIDEO, MediaFormat.WEBM, "1080p", 24),
// audio streams
// id, ItagType, MediaFormat, samplingR, bandwidth
new ItagItem(249, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0), // bandwith/samplingR 0 because not known
new ItagItem(250, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0),
new ItagItem(171, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0),
new ItagItem(140, ItagType.AUDIO, MediaFormat.M4A, 0, 0),
new ItagItem(251, ItagType.AUDIO, MediaFormat.WEBMA, 0, 0),
// video only streams
new ItagItem(160, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "144p", 24),
new ItagItem(133, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "240p", 24),
new ItagItem(134, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "360p", 24),
new ItagItem(135, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "480p", 24),
new ItagItem(136, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "720p", 24),
new ItagItem(137, ItagType.VIDEO_ONLY, MediaFormat.MPEG_4, "1080p", 24),
};
/**These lists only contain itag formats that are supported by the common Android Video player.
However if you are looking for a list showing all itag formats, look at
https://github.com/rg3/youtube-dl/issues/1687 */
public static boolean itagIsSupported(int itag) {
for(ItagItem item : itagList) {
if(itag == item.id) {
return true;
}
}
return false;
}
public static ItagItem getItagItem(int itag) throws ParsingException {
for(ItagItem item : itagList) {
if(itag == item.id) {
return item;
}
}
throw new ParsingException("itag=" + Integer.toString(itag) + " not supported");
}
// Sometimes if the html page of youtube is already downloaded, youtube web page will internally
// download the /get_video_info page. Since a certain date dashmpd url is only available over
// this /get_video_info page, so we always need to download this one to.
// %%video_id%% will be replaced by the actual video id
// $$el_type$$ will be replaced by the actual el_type (se the declarations below)
private static final String GET_VIDEO_INFO_URL =
"https://www.youtube.com/get_video_info?video_id=%%video_id%%$$el_type$$&ps=default&eurl=&gl=US&hl=en";
// eltype is nececeary for the url aboth
private static final String EL_INFO = "el=info";
public class DecryptException extends ParsingException {
DecryptException(Throwable cause) {
@ -69,9 +160,10 @@ public class YoutubeVideoExtractor implements VideoExtractor {
// ----------------
private static final String TAG = YoutubeVideoExtractor.class.toString();
private static final String TAG = YoutubeStreamExtractor.class.toString();
private final Document doc;
private JSONObject playerArgs;
private Map<String, String> videoInfoPage;
// static values
private static final String DECRYPTION_FUNC_NAME="decrypt";
@ -84,7 +176,7 @@ public class YoutubeVideoExtractor implements VideoExtractor {
private Downloader downloader;
public YoutubeVideoExtractor(String pageUrl, Downloader dl) throws CrawlingException, IOException {
public YoutubeStreamExtractor(String pageUrl, Downloader dl) throws CrawlingException, IOException {
//most common videoInfo fields are now set in our superclass, for all services
downloader = dl;
this.pageUrl = pageUrl;
@ -96,10 +188,10 @@ public class YoutubeVideoExtractor implements VideoExtractor {
//attempt to load the youtube js player JSON arguments
try {
ytPlayerConfigRaw =
RegexHelper.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
ytPlayerConfig = new JSONObject(ytPlayerConfigRaw);
playerArgs = ytPlayerConfig.getJSONObject("args");
} catch (RegexHelper.RegexException e) {
} catch (Parser.RegexException e) {
String errorReason = findErrorReason(doc);
switch(errorReason) {
case "GEMA":
@ -113,6 +205,17 @@ public class YoutubeVideoExtractor implements VideoExtractor {
throw new ParsingException("Could not parse yt player config");
}
// get videoInfo page
try {
//Parser.unescapeEntities(url_data_str, true).split("&")
String getVideoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%",
urlidhandler.getVideoId(pageUrl)).replace("$$el_type$$", "&" + EL_INFO);
videoInfoPage = Parser.compatParseMap(downloader.download(getVideoInfoUrl));
} catch(Exception e) {
throw new ParsingException("Could not load video info page.", e);
}
//----------------------------------
// load and parse description code, if it isn't already initialised
//----------------------------------
@ -211,7 +314,8 @@ public class YoutubeVideoExtractor implements VideoExtractor {
return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
} catch(Exception e) {
Log.w(TAG, "Could not find high res Thumbnail. Using low res instead");
} try { //fall through to fallback
}
try { //fall through to fallback
return playerArgs.getString("thumbnail_url");
} catch (JSONException je) {
throw new ParsingException(
@ -232,10 +336,11 @@ public class YoutubeVideoExtractor implements VideoExtractor {
@Override
public String getDashMpdUrl() throws ParsingException {
/*
try {
String dashManifest = playerArgs.getString("dashmpd");
if(!dashManifest.contains("/signature/")) {
String encryptedSig = RegexHelper.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest);
String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest);
String decryptedSig;
decryptedSig = decryptSignature(encryptedSig, decryptionCode);
@ -243,47 +348,95 @@ public class YoutubeVideoExtractor implements VideoExtractor {
}
return dashManifest;
} catch(NullPointerException e) {
} catch(JSONException je) {
throw new ParsingException(
"Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", e);
"Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", je);
} catch (Exception e) {
throw new ParsingException(e);
}
*/
try {
String dashManifestUrl = videoInfoPage.get("dashmpd");
if(!dashManifestUrl.contains("/signature/")) {
String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
String decryptedSig;
decryptedSig = decryptSignature(encryptedSig, decryptionCode);
dashManifestUrl = dashManifestUrl.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
}
return dashManifestUrl;
} catch (Exception e) {
throw new ParsingException(
"Could not get \"dashmpd\" maybe VideoInfoPage is broken.", e);
}
}
@Override
public List<VideoInfo.AudioStream> getAudioStreams() throws ParsingException {
/* If we provide a valid dash manifest, we don't need to provide audio streams extra */
return null;
}
@Override
public List<VideoInfo.VideoStream> getVideoStreams() throws ParsingException {
Vector<VideoInfo.VideoStream> videoStreams = new Vector<>();
Vector<VideoInfo.AudioStream> audioStreams = new Vector<>();
try{
String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map");
String encoded_url_map = playerArgs.getString("adaptive_fmts");
for(String url_data_str : encoded_url_map.split(",")) {
try {
Map<String, String> tags = new HashMap<>();
for (String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) {
String[] split_tag = raw_tag.split("=");
tags.put(split_tag[0], split_tag[1]);
}
// This loop iterates through multiple streams, therefor tags
// is related to one and the same stream at a time.
Map<String, String> tags = Parser.compatParseMap(
org.jsoup.parser.Parser.unescapeEntities(url_data_str, true));
int itag = Integer.parseInt(tags.get("itag"));
String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8");
if (itagIsSupported(itag)) {
ItagItem itagItem = getItagItem(itag);
if (itagItem.itagType == ItagType.AUDIO) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
streamUrl = streamUrl + "&signature="
+ decryptSignature(tags.get("s"), decryptionCode);
}
if (resolveFormat(itag) != -1) {
audioStreams.add(new VideoInfo.AudioStream(streamUrl,
itagItem.mediaFormatId,
itagItem.bandWidth,
itagItem.samplingRate));
}
}
}
} catch (Exception e) {
throw new ParsingException("Could not get audiostreams", e);
}
return audioStreams;
}
@Override
public List<VideoInfo.VideoStream> getVideoStreams() throws ParsingException {
Vector<VideoInfo.VideoStream> videoStreams = new Vector<>();
try{
String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map");
for(String url_data_str : encoded_url_map.split(",")) {
try {
// This loop iterates through multiple streams, therefor tags
// is related to one and the same stream at a time.
Map<String, String> tags = Parser.compatParseMap(
org.jsoup.parser.Parser.unescapeEntities(url_data_str, true));
int itag = Integer.parseInt(tags.get("itag"));
if (itagIsSupported(itag)) {
ItagItem itagItem = getItagItem(itag);
if(itagItem.itagType == ItagType.VIDEO) {
String streamUrl = tags.get("url");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
streamUrl = streamUrl + "&signature="
+ decryptSignature(tags.get("s"), decryptionCode);
}
videoStreams.add(new VideoInfo.VideoStream(
streamUrl,
resolveFormat(itag),
resolveResolutionString(itag)));
itagItem.mediaFormatId,
itagItem.resolutionString));
}
}
} catch (Exception e) {
Log.w(TAG, "Could not get Video stream.");
@ -298,19 +451,23 @@ public class YoutubeVideoExtractor implements VideoExtractor {
if(videoStreams.isEmpty()) {
throw new ParsingException("Failed to get any video stream");
}
return videoStreams;
}
@Override
public List<VideoInfo.VideoStream> getVideoOnlyStreams() throws ParsingException {
return null;
}
/**Attempts to parse (and return) the offset to start playing the video from.
* @return the offset (in seconds), or 0 if no timestamp is found.*/
@Override
public int getTimeStamp() throws ParsingException {
//todo: add unit test for timestamp
//todo: use video_info for getting timestamp
String timeStamp;
try {
timeStamp = RegexHelper.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
} catch (RegexHelper.RegexException e) {
timeStamp = Parser.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
} catch (Parser.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp
// -2 because well the testing system will then know its the regex that failed :/
@ -318,22 +475,21 @@ public class YoutubeVideoExtractor implements VideoExtractor {
return -2;
}
//TODO: test this
if(!timeStamp.isEmpty()) {
try {
String secondsString = "";
String minutesString = "";
String hoursString = "";
try {
secondsString = RegexHelper.matchGroup1("(\\d{1,3})s", timeStamp);
minutesString = RegexHelper.matchGroup1("(\\d{1,3})m", timeStamp);
hoursString = RegexHelper.matchGroup1("(\\d{1,3})h", timeStamp);
secondsString = Parser.matchGroup1("(\\d{1,3})s", timeStamp);
minutesString = Parser.matchGroup1("(\\d{1,3})m", timeStamp);
hoursString = Parser.matchGroup1("(\\d{1,3})h", timeStamp);
} catch (Exception e) {
//it could be that time is given in another method
if (secondsString.isEmpty() //if nothing was got,
&& minutesString.isEmpty()//treat as unlabelled seconds
&& hoursString.isEmpty()) {
secondsString = RegexHelper.matchGroup1("t=(\\d{1,3})", timeStamp);
secondsString = Parser.matchGroup1("t=(\\d{1,3})", timeStamp);
}
}
@ -446,7 +602,7 @@ public class YoutubeVideoExtractor implements VideoExtractor {
info.webpage_url = li.select("a.content-link").first()
.attr("abs:href");
info.id = RegexHelper.matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url);
info.id = Parser.matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url);
//todo: check NullPointerException causing
info.title = li.select("span.title").first().text();
@ -498,19 +654,19 @@ public class YoutubeVideoExtractor implements VideoExtractor {
String playerCode = downloader.download(playerUrl);
decryptionFuncName =
RegexHelper.matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode);
Parser.matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode);
String functionPattern = "("
+ decryptionFuncName.replace("$", "\\$")
+ "=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})";
decryptionFunc = "var " + RegexHelper.matchGroup1(functionPattern, playerCode) + ";";
decryptionFunc = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";
helperObjectName = RegexHelper
helperObjectName = Parser
.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc);
String helperPattern = "(var "
+ helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
helperObject = RegexHelper.matchGroup1(helperPattern, playerCode);
helperObject = Parser.matchGroup1(helperPattern, playerCode);
callerFunc = callerFunc.replace("%%", decryptionFuncName);
@ -553,48 +709,4 @@ public class YoutubeVideoExtractor implements VideoExtractor {
}
return "";
}
/**These lists only contain itag formats that are supported by the common Android Video player.
However if you are looking for a list showing all itag formats, look at
https://github.com/rg3/youtube-dl/issues/1687 */
@SuppressWarnings("WeakerAccess")
public static int resolveFormat(int itag) {
switch(itag) {
// !!! lists only supported formats !!!
// video
case 17: return MediaFormat.v3GPP.id;
case 18: return MediaFormat.MPEG_4.id;
case 22: return MediaFormat.MPEG_4.id;
case 36: return MediaFormat.v3GPP.id;
case 37: return MediaFormat.MPEG_4.id;
case 38: return MediaFormat.MPEG_4.id;
case 43: return MediaFormat.WEBM.id;
case 44: return MediaFormat.WEBM.id;
case 45: return MediaFormat.WEBM.id;
case 46: return MediaFormat.WEBM.id;
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return -1;
}
}
@SuppressWarnings("WeakerAccess")
public static String resolveResolutionString(int itag) {
switch(itag) {
case 17: return "144p";
case 18: return "360p";
case 22: return "720p";
case 36: return "240p";
case 37: return "1080p";
case 38: return "1080p";
case 43: return "360p";
case 44: return "480p";
case 45: return "720p";
case 46: return "1080p";
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return null;
}
}
}

View File

@ -1,7 +1,7 @@
package org.schabi.newpipe.crawler.services.youtube;
import org.schabi.newpipe.crawler.Parser;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.RegexHelper;
import org.schabi.newpipe.crawler.VideoUrlIdHandler;
/**
@ -47,7 +47,7 @@ public class YoutubeVideoUrlIdHandler implements VideoUrlIdHandler {
throw new ParsingException("Error no suitable url: " + url);
}
id = RegexHelper.matchGroup1(pat, url);
id = Parser.matchGroup1(pat, url);
if(!id.isEmpty()){
//Log.i(TAG, "string \""+url+"\" matches!");
return id;