From 4d727245e174b88b2121c6ef271eeb5d8e80a0aa Mon Sep 17 00:00:00 2001 From: rrooij Date: Sun, 21 Feb 2016 16:35:32 +0100 Subject: [PATCH 1/5] YTStreamExtractor: add support for age restricted YouTube age restricted videos are now kind of supported. This is not a final solution to this problem and this commit still crashes because of some problem with the thumbnail. --- .../youtube/YoutubeStreamExtractor.java | 119 ++++++++++++------ 1 file changed, 78 insertions(+), 41 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 0e17d1c9a..384df6739 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -24,6 +24,8 @@ import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Vector; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Created by Christian Schabesberger on 06.08.15. @@ -160,7 +162,8 @@ public class YoutubeStreamExtractor implements StreamExtractor { private static final String TAG = YoutubeStreamExtractor.class.toString(); private final Document doc; private JSONObject playerArgs; - //private Map videoInfoPage; + private boolean isAgeRestricted; + private Map videoInfoPage; // static values private static final String DECRYPTION_FUNC_NAME="decrypt"; @@ -180,42 +183,37 @@ public class YoutubeStreamExtractor implements StreamExtractor { String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl)); doc = Jsoup.parse(pageContent, pageUrl); String ytPlayerConfigRaw; - JSONObject ytPlayerConfig; + JSONObject ytPlayerConfig = new JSONObject(); - //attempt to load the youtube js player JSON arguments - try { - ytPlayerConfigRaw = - Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); - ytPlayerConfig = new JSONObject(ytPlayerConfigRaw); - playerArgs = ytPlayerConfig.getJSONObject("args"); - } catch (Parser.RegexException e) { - String errorReason = findErrorReason(doc); - switch(errorReason) { - case "GEMA": - throw new GemaException(errorReason); - case "": - throw new ParsingException("player config empty", e); - default: - throw new ContentNotAvailableException("Content not available", e); - } - } catch (JSONException e) { - throw new ParsingException("Could not parse yt player config"); - } - - - /* not yet nececeary - - - // get videoInfo page - try { - //Parser.unescapeEntities(url_data_str, true).split("&") - String getVideoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%", + // Check if the video is age restricted + if (pageContent.contains(" method je.printStackTrace(); @@ -265,7 +279,11 @@ public class YoutubeStreamExtractor implements StreamExtractor { @Override public String getUploader() throws ParsingException { - try {//json player args method + try { + if (playerArgs == null) { + return videoInfoPage.get("author"); + } + //json player args method return playerArgs.getString("author"); } catch(JSONException je) { je.printStackTrace(); @@ -281,6 +299,9 @@ public class YoutubeStreamExtractor implements StreamExtractor { @Override public int getLength() throws ParsingException { try { + if (playerArgs == null) { + return Integer.valueOf(videoInfoPage.get("length_seconds")); + } return playerArgs.getInt("length_seconds"); } catch (JSONException e) {//todo: find fallback method throw new ParsingException("failed to load video duration from JSON args", e); @@ -321,6 +342,9 @@ public class YoutubeStreamExtractor implements StreamExtractor { } catch (JSONException je) { throw new ParsingException( "failed to extract thumbnail URL from JSON args; trying to extract it from HTML", je); + } catch (NullPointerException ne) { + // Get from the video info page instead + return videoInfoPage.get("thumbnail_url"); } } @@ -361,7 +385,12 @@ public class YoutubeStreamExtractor implements StreamExtractor { public List getAudioStreams() throws ParsingException { Vector audioStreams = new Vector<>(); try{ - String encoded_url_map = playerArgs.getString("adaptive_fmts"); + String encoded_url_map; + if (videoInfoPage == null) { + encoded_url_map = videoInfoPage.get("adaptive_fmts"); + } else { + encoded_url_map = playerArgs.getString("adaptive_fmts"); + } for(String url_data_str : encoded_url_map.split(",")) { // This loop iterates through multiple streams, therefor tags // is related to one and the same stream at a time. @@ -398,7 +427,12 @@ public class YoutubeStreamExtractor implements StreamExtractor { Vector videoStreams = new Vector<>(); try{ - String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map"); + String encoded_url_map; + if (playerArgs == null) { + encoded_url_map = videoInfoPage.get("url_encoded_fmt_stream_map"); + } else { + encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map"); + } for(String url_data_str : encoded_url_map.split(",")) { try { // This loop iterates through multiple streams, therefor tags @@ -504,6 +538,9 @@ public class YoutubeStreamExtractor implements StreamExtractor { @Override public String getAverageRating() throws ParsingException { try { + if (playerArgs == null) { + videoInfoPage.get("avg_rating"); + } return playerArgs.getString("avg_rating"); } catch (JSONException e) { throw new ParsingException("Could not get Average rating", e); From d2b808e5404927d1c913627439e6704a688ecc97 Mon Sep 17 00:00:00 2001 From: rrooij Date: Sun, 21 Feb 2016 18:24:53 +0100 Subject: [PATCH 2/5] YTStreamExtractor: fix audio tracks on restricted Fix audio tracks on restriced videos. --- .../extractor/services/youtube/YoutubeStreamExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 384df6739..1f9e65df3 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -386,7 +386,7 @@ public class YoutubeStreamExtractor implements StreamExtractor { Vector audioStreams = new Vector<>(); try{ String encoded_url_map; - if (videoInfoPage == null) { + if (playerArgs == null) { encoded_url_map = videoInfoPage.get("adaptive_fmts"); } else { encoded_url_map = playerArgs.getString("adaptive_fmts"); From 3f0947fa5b736f26981e16866664b8f42ea960dc Mon Sep 17 00:00:00 2001 From: rrooij Date: Sun, 21 Feb 2016 18:25:18 +0100 Subject: [PATCH 3/5] Add unit test for restricted YouTube videos --- .../YoutubeStreamExtractorRestrictedTest.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorRestrictedTest.java diff --git a/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorRestrictedTest.java b/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorRestrictedTest.java new file mode 100644 index 000000000..ccdf59fcf --- /dev/null +++ b/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -0,0 +1,80 @@ +package org.schabi.newpipe.extractor.youtube; + +import android.test.AndroidTestCase; + +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.extractor.ExctractionException; +import org.schabi.newpipe.extractor.ParsingException; +import org.schabi.newpipe.extractor.VideoInfo; +import org.schabi.newpipe.extractor.services.youtube.YoutubeStreamExtractor; + +import java.io.IOException; + +public class YoutubeStreamExtractorRestrictedTest extends AndroidTestCase { + private YoutubeStreamExtractor extractor; + + public void setUp() throws IOException, ExctractionException { + extractor = new YoutubeStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0", + new Downloader()); + } + + public void testGetInvalidTimeStamp() throws ParsingException { + assertTrue(Integer.toString(extractor.getTimeStamp()), + extractor.getTimeStamp() <= 0); + } + + public void testGetValidTimeStamp() throws ExctractionException, IOException { + YoutubeStreamExtractor extractor = + new YoutubeStreamExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader()); + assertTrue(Integer.toString(extractor.getTimeStamp()), + extractor.getTimeStamp() == 174); + } + + public void testGetTitle() throws ParsingException { + assertTrue(!extractor.getTitle().isEmpty()); + } + + public void testGetDescription() throws ParsingException { + assertTrue(extractor.getDescription() != null); + } + + public void testGetUploader() throws ParsingException { + assertTrue(!extractor.getUploader().isEmpty()); + } + + public void testGetLength() throws ParsingException { + assertTrue(extractor.getLength() > 0); + } + + public void testGetViews() throws ParsingException { + assertTrue(extractor.getLength() > 0); + } + + public void testGetUploadDate() throws ParsingException { + assertTrue(extractor.getUploadDate().length() > 0); + } + + public void testGetThumbnailUrl() throws ParsingException { + assertTrue(extractor.getThumbnailUrl(), + extractor.getThumbnailUrl().contains("https://")); + } + + public void testGetUploaderThumbnailUrl() throws ParsingException { + assertTrue(extractor.getUploaderThumbnailUrl(), + extractor.getUploaderThumbnailUrl().contains("https://")); + } + + public void testGetAudioStreams() throws ParsingException { + assertTrue(!extractor.getAudioStreams().isEmpty()); + } + + public void testGetVideoStreams() throws ParsingException { + for(VideoInfo.VideoStream s : extractor.getVideoStreams()) { + assertTrue(s.url, + s.url.contains("https://")); + assertTrue(s.resolution.length() > 0); + assertTrue(Integer.toString(s.format), + 0 <= s.format && s.format <= 4); + } + } +} From 2edfcb78fef49259d022eecf4c0ae33ec10f362b Mon Sep 17 00:00:00 2001 From: rrooij Date: Sun, 21 Feb 2016 18:45:28 +0100 Subject: [PATCH 4/5] YTStreamExtractor: implement getAgeRestriction --- .../YoutubeStreamExtractorRestrictedTest.java | 4 ++++ .../services/youtube/YoutubeStreamExtractor.java | 14 ++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorRestrictedTest.java b/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorRestrictedTest.java index ccdf59fcf..5e0a83c19 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorRestrictedTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorRestrictedTest.java @@ -30,6 +30,10 @@ public class YoutubeStreamExtractorRestrictedTest extends AndroidTestCase { extractor.getTimeStamp() == 174); } + public void testGetAgeLimit() throws ParsingException { + assertTrue(extractor.getAgeLimit() == 18); + } + public void testGetTitle() throws ParsingException { assertTrue(!extractor.getTitle().isEmpty()); } diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 1f9e65df3..f245fec4d 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -529,10 +529,16 @@ public class YoutubeStreamExtractor implements StreamExtractor { @Override public int getAgeLimit() throws ParsingException { - // Not yet implemented. - // Also you need to be logged in to see age restricted videos on youtube, - // therefore NP is not able to receive such videos. - return 0; + if (!isAgeRestricted) { + return 0; + } + try { + return Integer.valueOf(doc.head() + .getElementsByAttributeValue("property", "og:restrictions:age") + .attr("content").replace("+", "")); + } catch (Exception e) { + throw new ParsingException("Could not get age restriction"); + } } @Override From d12af16f464e36738a108eb9e8f4c27dc44ac717 Mon Sep 17 00:00:00 2001 From: rrooij Date: Sun, 21 Feb 2016 19:07:24 +0100 Subject: [PATCH 5/5] VideoItemDetailFragment: check for next vid Check if the next video is available before loading the thumbnails. --- .../main/java/org/schabi/newpipe/VideoItemDetailFragment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index 89791bc2b..ffef09d44 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -409,7 +409,7 @@ public class VideoItemDetailFragment extends Fragment { imageLoader.displayImage(info.uploader_thumbnail_url, uploaderThumb, displayImageOptions, new ThumbnailLoadingListener()); } - if(info.thumbnail_url != null && !info.thumbnail_url.isEmpty()) { + if(info.thumbnail_url != null && !info.thumbnail_url.isEmpty() && info.next_video != null) { imageLoader.displayImage(info.next_video.thumbnail_url, nextVideoThumb, displayImageOptions, new ThumbnailLoadingListener()); }