From bad576c23d7659cb975a65635cd6373010988755 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 2 Feb 2016 14:06:09 +0100 Subject: [PATCH] got rid of getVideoInfo() in youtube crawler --- .../newpipe/VideoItemDetailFragment.java | 1 + .../schabi/newpipe/VideoItemListFragment.java | 1 + .../newpipe/crawler/VideoExtractor.java | 27 +++++ .../youtube/YoutubeVideoExtractor.java | 109 ++++++++---------- 4 files changed, 80 insertions(+), 58 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index 076a12137..ff3b94933 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -115,6 +115,7 @@ public class VideoItemDetailFragment extends Fragment { VideoInfo videoInfo = videoExtractor.getVideoInfo(); h.post(new VideoResultReturnedRunnable(videoInfo)); h.post(new SetThumbnailRunnable( + //todo: make bitmaps not bypass tor BitmapFactory.decodeStream( new URL(videoInfo.thumbnail_url) .openConnection() diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java index 21781f78c..479a4820d 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java @@ -157,6 +157,7 @@ public class VideoItemListFragment extends ListFragment { if(!downloadedList.get(i)) { Bitmap thumbnail; try { + //todo: make bitmaps not bypass tor thumbnail = BitmapFactory.decodeStream( new URL(thumbnailUrlList.get(i)).openConnection().getInputStream()); h.post(new SetThumbnailRunnable(i, thumbnail, requestId)); diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java index e09b2e01f..ac165c19e 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java @@ -20,6 +20,9 @@ package org.schabi.newpipe.crawler; * along with NewPipe. If not, see . */ +import java.util.List; +import java.util.Vector; + /**Scrapes information from a video streaming service (eg, YouTube).*/ @@ -134,6 +137,25 @@ public abstract class VideoExtractor { videoInfo.dashMpdUrl = getDashMpdUrl(); } + if(videoInfo.average_rating.isEmpty()) { + videoInfo.average_rating = getAverageRating(); + } + + if(videoInfo.like_count == -1) { + videoInfo.like_count = getLikeCount(); + } + + if(videoInfo.dislike_count == -1) { + videoInfo.dislike_count = getDislikeCount(); + } + + if(videoInfo.nextVideo == null) { + videoInfo.nextVideo = getNextVideo(); + } + + if(videoInfo.relatedVideos == null) { + videoInfo.relatedVideos = getRelatedVideos(); + } //Bitmap thumbnail = null; //Bitmap uploader_thumbnail = null; @@ -158,4 +180,9 @@ public abstract class VideoExtractor { public abstract VideoInfo.VideoStream[] getVideoStreams() throws ParsingException; public abstract String getDashMpdUrl() throws ParsingException; public abstract int getAgeLimit() throws ParsingException; + public abstract String getAverageRating() throws ParsingException; + public abstract int getLikeCount() throws ParsingException; + public abstract int getDislikeCount() throws ParsingException; + public abstract VideoPreviewInfo getNextVideo() throws ParsingException; + public abstract Vector getRelatedVideos() throws ParsingException; } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java index 83ebb8ca0..ced24cc95 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.StringReader; import java.net.URLDecoder; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Vector; import java.util.regex.Matcher; @@ -389,77 +390,69 @@ public class YoutubeVideoExtractor extends VideoExtractor { return 0; } + @Override + public String getAverageRating() throws ParsingException { + try { + return playerArgs.getString("avg_rating"); + } catch (JSONException e) { + throw new ParsingException("Could not get Average rating", e); + } + } @Override - public VideoInfo getVideoInfo() throws CrawlingException { - videoInfo = super.getVideoInfo(); - - //todo: replace this with a call to getVideoId, if possible - //videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl); - videoInfo.id = getVideoId(pageUrl); - - if (videoInfo.audioStreams == null - || videoInfo.audioStreams.length == 0) { - Log.e(TAG, "uninitialised audio streams!"); - } - - if (videoInfo.videoStreams == null - || videoInfo.videoStreams.length == 0) { - Log.e(TAG, "uninitialised video streams!"); - } - - videoInfo.age_limit = 0; - - //average rating - try { - videoInfo.average_rating = playerArgs.getString("avg_rating"); - } catch (JSONException e) { - e.printStackTrace(); - } - - //--------------------------------------- - // extracting information from html page - //--------------------------------------- - + public int getLikeCount() throws ParsingException { String likesString = ""; - String dislikesString = ""; try { - // likes likesString = doc.select("button.like-button-renderer-like-button").first() .select("span.yt-uix-button-content").first().text(); - videoInfo.like_count = Integer.parseInt(likesString.replaceAll("[^\\d]", "")); - // dislikes + return Integer.parseInt(likesString.replaceAll("[^\\d]", "")); + } catch (NumberFormatException nfe) { + throw new ParsingException( + "failed to parse likesString \"" + likesString + "\" as integers", nfe); + } catch (Exception e) { + throw new ParsingException("Could not get like count", e); + } + } + + @Override + public int getDislikeCount() throws ParsingException { + String dislikesString = ""; + try { dislikesString = doc.select("button.like-button-renderer-dislike-button").first() .select("span.yt-uix-button-content").first().text(); - - videoInfo.dislike_count = Integer.parseInt(dislikesString.replaceAll("[^\\d]", "")); - } catch (NumberFormatException nfe) { - Log.e(TAG, "failed to parse likesString \"" + likesString + "\" and dislikesString \"" + - dislikesString + "\" as integers"); - } catch (Exception e) { - // if it fails we know that the video does not offer dislikes. - e.printStackTrace(); - videoInfo.like_count = 0; - videoInfo.dislike_count = 0; + return Integer.parseInt(dislikesString.replaceAll("[^\\d]", "")); + } catch(NumberFormatException nfe) { + throw new ParsingException( + "failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe); + } catch(Exception e) { + throw new ParsingException("Could not get dislike count", e); } + } - // next video - videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() - .select("li").first()); + @Override + public VideoPreviewInfo getNextVideo() throws ParsingException { + try { + return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() + .select("li").first()); + } catch(Exception e) { + throw new ParsingException("Could not get next video", e); + } + } - // related videos - Vector relatedVideos = new Vector<>(); - for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) { - // first check if we have a playlist. If so leave them out - if (li.select("a[class*=\"content-link\"]").first() != null) { - relatedVideos.add(extractVideoPreviewInfo(li)); + @Override + public Vector getRelatedVideos() throws ParsingException { + try { + Vector relatedVideos = new Vector<>(); + for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) { + // first check if we have a playlist. If so leave them out + if (li.select("a[class*=\"content-link\"]").first() != null) { + relatedVideos.add(extractVideoPreviewInfo(li)); + } } + return relatedVideos; + } catch(Exception e) { + throw new ParsingException("Could not get related videos", e); } - //todo: replace conversion - videoInfo.relatedVideos = relatedVideos; - //videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]); - - return videoInfo; } private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) throws RegexException, DecryptException {