From 54d318bf043feb8f17158bf967db2bfb6acfac45 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Thu, 28 Jan 2016 21:21:19 +0100 Subject: [PATCH 01/11] detatch android related downloader from crawler --- .../org/schabi/newpipe/DownloadDialog.java | 2 +- .../java/org/schabi/newpipe/Downloader.java | 151 +--------------- .../org/schabi/newpipe/FileDownloader.java | 169 ++++++++++++++++++ .../newpipe/VideoItemDetailFragment.java | 2 +- .../schabi/newpipe/VideoItemListFragment.java | 3 +- .../schabi/newpipe/services/Downloader.java | 37 ++++ .../schabi/newpipe/services/SearchEngine.java | 6 +- .../newpipe/services/StreamingService.java | 2 +- .../newpipe/services/VideoExtractor.java | 2 +- .../services/youtube/YoutubeSearchEngine.java | 12 +- .../services/youtube/YoutubeService.java | 5 +- .../youtube/YoutubeVideoExtractor.java | 17 +- 12 files changed, 243 insertions(+), 165 deletions(-) create mode 100644 app/src/main/java/org/schabi/newpipe/FileDownloader.java create mode 100644 app/src/main/java/org/schabi/newpipe/services/Downloader.java diff --git a/app/src/main/java/org/schabi/newpipe/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/DownloadDialog.java index d7160ccdb..79d24823c 100644 --- a/app/src/main/java/org/schabi/newpipe/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/DownloadDialog.java @@ -107,7 +107,7 @@ public class DownloadDialog extends DialogFragment { long id = 0; if (App.isUsingTor()) { // if using Tor, do not use DownloadManager because the proxy cannot be set - Downloader.downloadFile(getContext(), url, saveFilePath, title); + FileDownloader.downloadFile(getContext(), url, saveFilePath, title); } else { DownloadManager dm = (DownloadManager) context.getSystemService(Context.DOWNLOAD_SERVICE); DownloadManager.Request request = new DownloadManager.Request( diff --git a/app/src/main/java/org/schabi/newpipe/Downloader.java b/app/src/main/java/org/schabi/newpipe/Downloader.java index 6bd982575..7b5950407 100644 --- a/app/src/main/java/org/schabi/newpipe/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/Downloader.java @@ -1,24 +1,8 @@ package org.schabi.newpipe; - -import android.app.NotificationManager; -import android.content.Context; -import android.content.SharedPreferences; -import android.graphics.drawable.BitmapDrawable; -import android.graphics.drawable.Drawable; -import android.os.AsyncTask; -import android.preference.PreferenceManager; -import android.support.v4.app.NotificationCompat; -import android.util.Log; - -import java.io.BufferedInputStream; import java.io.BufferedReader; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; -import java.net.HttpURLConnection; import java.net.URL; import java.net.UnknownHostException; @@ -27,7 +11,7 @@ import javax.net.ssl.HttpsURLConnection; import info.guardianproject.netcipher.NetCipher; /** - * Created by Christian Schabesberger on 14.08.15. + * Created by Christian Schabesberger on 28.01.16. * * Copyright (C) Christian Schabesberger 2015 * Downloader.java is part of NewPipe. @@ -46,38 +30,16 @@ import info.guardianproject.netcipher.NetCipher; * along with NewPipe. If not, see . */ -public class Downloader extends AsyncTask { - public static final String TAG = "Downloader"; +public class Downloader implements org.schabi.newpipe.services.Downloader { + private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"; - private NotificationManager nm; - private NotificationCompat.Builder builder; - private int notifyId = 0x1234; - private int fileSize = 0xffffffff; - - private final Context context; - private final String fileURL; - private final File saveFilePath; - private final String title; - - private final String debugContext; - - public Downloader(Context context, String fileURL, File saveFilePath, String title) { - this.context = context; - this.fileURL = fileURL; - this.saveFilePath = saveFilePath; - this.title = title; - - this.debugContext = "'" + fileURL + - "' => '" + saveFilePath + "'"; - } - /**Download the text file at the supplied URL as in download(String), * but set the HTTP header field "Accept-Language" to the supplied string. * @param siteUrl the URL of the text file to return the contents of * @param language the language (usually a 2-character code) to set as the preferred language * @return the contents of the specified text file*/ - public static String download(String siteUrl, String language) { + public String download(String siteUrl, String language) { String ret = ""; try { URL url = new URL(siteUrl); @@ -118,11 +80,11 @@ public class Downloader extends AsyncTask { return response.toString(); } -/**Download (via HTTP) the text file located at the supplied URL, and return its contents. - * Primarily intended for downloading web pages. - * @param siteUrl the URL of the text file to download - * @return the contents of the specified text file*/ - public static String download(String siteUrl) { + /**Download (via HTTP) the text file located at the supplied URL, and return its contents. + * Primarily intended for downloading web pages. + * @param siteUrl the URL of the text file to download + * @return the contents of the specified text file*/ + public String download(String siteUrl) { String ret = ""; try { @@ -137,99 +99,4 @@ public class Downloader extends AsyncTask { return ret; } - - /** - * Downloads a file from a URL in the background using an {@link AsyncTask}. - * - * @param fileURL HTTP URL of the file to be downloaded - * @param saveFilePath path of the directory to save the file - * @param title - * @throws IOException - */ - public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) { - new Downloader(context, fileURL, saveFilePath, title).execute(); - } - - /** AsyncTask impl: executed in gui thread */ - @Override - protected void onPreExecute() { - super.onPreExecute(); - nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE); - Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher); - builder = new NotificationCompat.Builder(context) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setLargeIcon(((BitmapDrawable) icon).getBitmap()) - .setContentTitle(saveFilePath.getName()) - .setContentText(saveFilePath.getAbsolutePath()) - .setProgress(fileSize, 0, false); - nm.notify(notifyId, builder.build()); - } - - /** AsyncTask impl: executed in background thread does the download */ - @Override - protected Void doInBackground(Void... voids) { - HttpsURLConnection con = null; - InputStream inputStream = null; - FileOutputStream outputStream = null; - try { - con = NetCipher.getHttpsURLConnection(fileURL); - int responseCode = con.getResponseCode(); - - // always check HTTP response code first - if (responseCode == HttpURLConnection.HTTP_OK) { - fileSize = con.getContentLength(); - inputStream = new BufferedInputStream(con.getInputStream()); - outputStream = new FileOutputStream(saveFilePath); - - int bufferSize = 8192; - int downloaded = 0; - - int bytesRead = -1; - byte[] buffer = new byte[bufferSize]; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - downloaded += bytesRead; - if (downloaded % 50000 < bufferSize) { - publishProgress(downloaded); - } - } - - publishProgress(bufferSize); - - } else { - Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode); - } - } catch (IOException e) { - Log.e(TAG, "No file to download. Server replied HTTP code: ", e); - e.printStackTrace(); - } finally { - try { - if (outputStream != null) { - outputStream.close(); - } - if (inputStream != null) { - inputStream.close(); - } - } catch (IOException e) { - e.printStackTrace(); - } - if (con != null) { - con.disconnect(); - } - } - return null; - } - - @Override - protected void onProgressUpdate(Integer... progress) { - builder.setProgress(fileSize, progress[0], false); - nm.notify(notifyId, builder.build()); - } - - @Override - protected void onPostExecute(Void aVoid) { - super.onPostExecute(aVoid); - nm.cancel(notifyId); - } - } diff --git a/app/src/main/java/org/schabi/newpipe/FileDownloader.java b/app/src/main/java/org/schabi/newpipe/FileDownloader.java new file mode 100644 index 000000000..31ce9ecc8 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/FileDownloader.java @@ -0,0 +1,169 @@ +package org.schabi.newpipe; + + +import android.app.NotificationManager; +import android.content.Context; +import android.content.SharedPreferences; +import android.graphics.drawable.BitmapDrawable; +import android.graphics.drawable.Drawable; +import android.os.AsyncTask; +import android.preference.PreferenceManager; +import android.support.v4.app.NotificationCompat; +import android.util.Log; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.UnknownHostException; + +import javax.net.ssl.HttpsURLConnection; + +import info.guardianproject.netcipher.NetCipher; + +/** + * Created by Christian Schabesberger on 14.08.15. + * + * Copyright (C) Christian Schabesberger 2015 + * FileDownloader.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class FileDownloader extends AsyncTask { + public static final String TAG = "FileDownloader"; + + + private NotificationManager nm; + private NotificationCompat.Builder builder; + private int notifyId = 0x1234; + private int fileSize = 0xffffffff; + + private final Context context; + private final String fileURL; + private final File saveFilePath; + private final String title; + + private final String debugContext; + + public FileDownloader(Context context, String fileURL, File saveFilePath, String title) { + this.context = context; + this.fileURL = fileURL; + this.saveFilePath = saveFilePath; + this.title = title; + + this.debugContext = "'" + fileURL + + "' => '" + saveFilePath + "'"; + } + + /** + * Downloads a file from a URL in the background using an {@link AsyncTask}. + * + * @param fileURL HTTP URL of the file to be downloaded + * @param saveFilePath path of the directory to save the file + * @param title + * @throws IOException + */ + public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) { + new FileDownloader(context, fileURL, saveFilePath, title).execute(); + } + + /** AsyncTask impl: executed in gui thread */ + @Override + protected void onPreExecute() { + super.onPreExecute(); + nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE); + Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher); + builder = new NotificationCompat.Builder(context) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setLargeIcon(((BitmapDrawable) icon).getBitmap()) + .setContentTitle(saveFilePath.getName()) + .setContentText(saveFilePath.getAbsolutePath()) + .setProgress(fileSize, 0, false); + nm.notify(notifyId, builder.build()); + } + + /** AsyncTask impl: executed in background thread does the download */ + @Override + protected Void doInBackground(Void... voids) { + HttpsURLConnection con = null; + InputStream inputStream = null; + FileOutputStream outputStream = null; + try { + con = NetCipher.getHttpsURLConnection(fileURL); + int responseCode = con.getResponseCode(); + + // always check HTTP response code first + if (responseCode == HttpURLConnection.HTTP_OK) { + fileSize = con.getContentLength(); + inputStream = new BufferedInputStream(con.getInputStream()); + outputStream = new FileOutputStream(saveFilePath); + + int bufferSize = 8192; + int downloaded = 0; + + int bytesRead = -1; + byte[] buffer = new byte[bufferSize]; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + downloaded += bytesRead; + if (downloaded % 50000 < bufferSize) { + publishProgress(downloaded); + } + } + + publishProgress(bufferSize); + + } else { + Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode); + } + } catch (IOException e) { + Log.e(TAG, "No file to download. Server replied HTTP code: ", e); + e.printStackTrace(); + } finally { + try { + if (outputStream != null) { + outputStream.close(); + } + if (inputStream != null) { + inputStream.close(); + } + } catch (IOException e) { + e.printStackTrace(); + } + if (con != null) { + con.disconnect(); + } + } + return null; + } + + @Override + protected void onProgressUpdate(Integer... progress) { + builder.setProgress(fileSize, progress[0], false); + nm.notify(notifyId, builder.build()); + } + + @Override + protected void onPostExecute(Void aVoid) { + super.onPostExecute(aVoid); + nm.cancel(notifyId); + } + +} diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index e3a924c64..ba34d9960 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -108,7 +108,7 @@ public class VideoItemDetailFragment extends Fragment { @Override public void run() { try { - this.videoExtractor = service.getExtractorInstance(videoUrl); + this.videoExtractor = service.getExtractorInstance(videoUrl, new Downloader()); VideoInfo videoInfo = videoExtractor.getVideoInfo(); h.post(new VideoResultReturnedRunnable(videoInfo)); if (videoInfo.errorCode == VideoInfo.NO_ERROR) { diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java index 16953df1d..8cbd67677 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java @@ -108,7 +108,8 @@ public class VideoItemListFragment extends ListFragment { String searchLanguageKey = getContext().getString(R.string.search_language_key); String searchLanguage = sp.getString(searchLanguageKey, getString(R.string.default_language_value)); - SearchEngine.Result result = engine.search(query, page, searchLanguage); + SearchEngine.Result result = engine.search(query, page, searchLanguage, + new Downloader()); Log.i(TAG, "language code passed:\""+searchLanguage+"\""); if(runs) { diff --git a/app/src/main/java/org/schabi/newpipe/services/Downloader.java b/app/src/main/java/org/schabi/newpipe/services/Downloader.java new file mode 100644 index 000000000..ce83ca5da --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/services/Downloader.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.services; + +/** + * Created by Christian Schabesberger on 28.01.16. + * + * Copyright (C) Christian Schabesberger 2015 + * Downloader.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public interface Downloader { + + /**Download the text file at the supplied URL as in download(String), + * but set the HTTP header field "Accept-Language" to the supplied string. + * @param siteUrl the URL of the text file to return the contents of + * @param language the language (usually a 2-character code) to set as the preferred language + * @return the contents of the specified text file*/ + String download(String siteUrl, String language); + + /**Download (via HTTP) the text file located at the supplied URL, and return its contents. + * Primarily intended for downloading web pages. + * @param siteUrl the URL of the text file to download + * @return the contents of the specified text file*/ + String download(String siteUrl); +} diff --git a/app/src/main/java/org/schabi/newpipe/services/SearchEngine.java b/app/src/main/java/org/schabi/newpipe/services/SearchEngine.java index 98aa42ae5..eebbe0ee2 100644 --- a/app/src/main/java/org/schabi/newpipe/services/SearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/services/SearchEngine.java @@ -27,16 +27,14 @@ import java.util.Vector; @SuppressWarnings("ALL") public interface SearchEngine { - - class Result { public String errorMessage = ""; public String suggestion = ""; public final Vector resultList = new Vector<>(); } - ArrayList suggestionList(String query); + ArrayList suggestionList(String query, Downloader dl); //Result search(String query, int page); - Result search(String query, int page, String contentCountry); + Result search(String query, int page, String contentCountry, Downloader dl); } diff --git a/app/src/main/java/org/schabi/newpipe/services/StreamingService.java b/app/src/main/java/org/schabi/newpipe/services/StreamingService.java index acf887b57..082dc998a 100644 --- a/app/src/main/java/org/schabi/newpipe/services/StreamingService.java +++ b/app/src/main/java/org/schabi/newpipe/services/StreamingService.java @@ -25,7 +25,7 @@ public interface StreamingService { public String name = ""; } ServiceInfo getServiceInfo(); - VideoExtractor getExtractorInstance(String url); + VideoExtractor getExtractorInstance(String url, Downloader downloader); SearchEngine getSearchEngineInstance(); /**When a VIEW_ACTION is caught this function will test if the url delivered within the calling diff --git a/app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java b/app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java index f57ef0894..e4357b4d0 100644 --- a/app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java @@ -28,7 +28,7 @@ public abstract class VideoExtractor { protected VideoInfo videoInfo; @SuppressWarnings("WeakerAccess") - public VideoExtractor(String url) { + public VideoExtractor(String url, Downloader dl) { this.pageUrl = url; } diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java index 30bc92ce7..73b245a48 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java @@ -6,7 +6,7 @@ import android.util.Log; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.services.Downloader; import org.schabi.newpipe.services.SearchEngine; import org.schabi.newpipe.VideoPreviewInfo; import org.w3c.dom.Node; @@ -49,7 +49,7 @@ public class YoutubeSearchEngine implements SearchEngine { private static final String TAG = YoutubeSearchEngine.class.toString(); @Override - public Result search(String query, int page, String languageCode) { + public Result search(String query, int page, String languageCode, Downloader downloader) { //String contentCountry = PreferenceManager.getDefaultSharedPreferences(this).getString(getString(R.string., ""); Uri.Builder builder = new Uri.Builder(); builder.scheme("https") @@ -64,10 +64,10 @@ public class YoutubeSearchEngine implements SearchEngine { //if we've been passed a valid language code, append it to the URL if(!languageCode.isEmpty()) { //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode); - site = Downloader.download(url, languageCode); + site = downloader.download(url, languageCode); } else { - site = Downloader.download(url); + site = downloader.download(url); } @@ -140,7 +140,7 @@ public class YoutubeSearchEngine implements SearchEngine { } @Override - public ArrayList suggestionList(String query) { + public ArrayList suggestionList(String query, Downloader dl) { ArrayList suggestions = new ArrayList<>(); @@ -155,7 +155,7 @@ public class YoutubeSearchEngine implements SearchEngine { .appendQueryParameter("q", query); String url = builder.build().toString(); - String response = Downloader.download(url); + String response = dl.download(url); //TODO: Parse xml data using Jsoup not done DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java index 576d8c065..b90135d1b 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java +++ b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.services.youtube; +import org.schabi.newpipe.services.Downloader; import org.schabi.newpipe.services.StreamingService; import org.schabi.newpipe.services.VideoExtractor; import org.schabi.newpipe.services.SearchEngine; @@ -33,9 +34,9 @@ public class YoutubeService implements StreamingService { return serviceInfo; } @Override - public VideoExtractor getExtractorInstance(String url) { + public VideoExtractor getExtractorInstance(String url, Downloader downloader) { if(acceptUrl(url)) { - return new YoutubeVideoExtractor(url); + return new YoutubeVideoExtractor(url, downloader); } else { throw new IllegalArgumentException("supplied String is not a valid Youtube URL"); diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java index 6477305cf..e1cacef02 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java @@ -12,7 +12,8 @@ import org.jsoup.parser.Parser; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; -import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.FileDownloader; +import org.schabi.newpipe.services.Downloader; import org.schabi.newpipe.services.VideoExtractor; import org.schabi.newpipe.services.MediaFormat; import org.schabi.newpipe.services.VideoInfo; @@ -62,9 +63,13 @@ public class YoutubeVideoExtractor extends VideoExtractor { // cached values private static volatile String decryptionCode = ""; - public YoutubeVideoExtractor(String pageUrl) { - super(pageUrl);//most common videoInfo fields are now set in our superclass, for all services - String pageContent = Downloader.download(cleanUrl(pageUrl)); + private Downloader downloader; + + public YoutubeVideoExtractor(String pageUrl, Downloader dl) { + //most common videoInfo fields are now set in our superclass, for all services + super(pageUrl, dl); + downloader = dl; + String pageContent = downloader.download(cleanUrl(pageUrl)); doc = Jsoup.parse(pageContent, pageUrl); //attempt to load the youtube js player JSON arguments @@ -472,7 +477,7 @@ public class YoutubeVideoExtractor extends VideoExtractor { decryptedSig = decryptSignature(encryptedSig, decryptoinCode); dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig); } - String dashDoc = Downloader.download(dashManifest); + String dashDoc = downloader.download(dashManifest); Vector audioStreams = new Vector<>(); try { XmlPullParser parser = Xml.newPullParser(); @@ -574,7 +579,7 @@ public class YoutubeVideoExtractor extends VideoExtractor { } private String loadDecryptionCode(String playerUrl) { - String playerCode = Downloader.download(playerUrl); + String playerCode = downloader.download(playerUrl); String decryptionFuncName = ""; String decryptionFunc = ""; String helperObjectName; From f8ed96bb2529b4039991380525e3456fa54bc53d Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Thu, 28 Jan 2016 21:34:35 +0100 Subject: [PATCH 02/11] renamed services into crawer --- .../services/youtube/YoutubeSearchEngineTest.java | 2 +- .../java/org/schabi/newpipe/ActionBarHandler.java | 4 ++-- .../main/java/org/schabi/newpipe/Downloader.java | 2 +- .../schabi/newpipe/VideoInfoItemViewCreator.java | 2 ++ .../org/schabi/newpipe/VideoItemDetailActivity.java | 4 ++-- .../org/schabi/newpipe/VideoItemDetailFragment.java | 10 +++++----- .../org/schabi/newpipe/VideoItemListActivity.java | 3 ++- .../org/schabi/newpipe/VideoItemListFragment.java | 5 +++-- .../java/org/schabi/newpipe/VideoListAdapter.java | 2 ++ .../{services => crawler}/AbstractVideoInfo.java | 2 +- .../newpipe/{services => crawler}/Downloader.java | 2 +- .../newpipe/{services => crawler}/MediaFormat.java | 2 +- .../newpipe/{services => crawler}/SearchEngine.java | 4 +--- .../newpipe/{services => crawler}/ServiceList.java | 4 ++-- .../{services => crawler}/StreamingService.java | 2 +- .../{services => crawler}/VideoExtractor.java | 2 +- .../newpipe/{services => crawler}/VideoInfo.java | 11 +++++------ .../newpipe/{ => crawler}/VideoPreviewInfo.java | 4 ++-- .../services/youtube/YoutubeSearchEngine.java | 8 ++++---- .../services/youtube/YoutubeService.java | 10 +++++----- .../services/youtube/YoutubeVideoExtractor.java | 13 ++++++------- 21 files changed, 50 insertions(+), 48 deletions(-) rename app/src/main/java/org/schabi/newpipe/{services => crawler}/AbstractVideoInfo.java (92%) rename app/src/main/java/org/schabi/newpipe/{services => crawler}/Downloader.java (97%) rename app/src/main/java/org/schabi/newpipe/{services => crawler}/MediaFormat.java (98%) rename app/src/main/java/org/schabi/newpipe/{services => crawler}/SearchEngine.java (94%) rename app/src/main/java/org/schabi/newpipe/{services => crawler}/ServiceList.java (94%) rename app/src/main/java/org/schabi/newpipe/{services => crawler}/StreamingService.java (97%) rename app/src/main/java/org/schabi/newpipe/{services => crawler}/VideoExtractor.java (99%) rename app/src/main/java/org/schabi/newpipe/{services => crawler}/VideoInfo.java (93%) rename app/src/main/java/org/schabi/newpipe/{ => crawler}/VideoPreviewInfo.java (96%) rename app/src/main/java/org/schabi/newpipe/{ => crawler}/services/youtube/YoutubeSearchEngine.java (97%) rename app/src/main/java/org/schabi/newpipe/{ => crawler}/services/youtube/YoutubeService.java (86%) rename app/src/main/java/org/schabi/newpipe/{ => crawler}/services/youtube/YoutubeVideoExtractor.java (98%) diff --git a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java index c01474416..b9ded39c2 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java @@ -2,7 +2,7 @@ package org.schabi.newpipe.services.youtube; import android.test.AndroidTestCase; -import org.schabi.newpipe.VideoPreviewInfo; +import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.schabi.newpipe.services.SearchEngine; import java.util.ArrayList; diff --git a/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java b/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java index 490b8d6f4..0cd4f0b1a 100644 --- a/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java +++ b/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java @@ -16,8 +16,8 @@ import android.view.MenuInflater; import android.view.MenuItem; import android.widget.ArrayAdapter; -import org.schabi.newpipe.services.MediaFormat; -import org.schabi.newpipe.services.VideoInfo; +import org.schabi.newpipe.crawler.MediaFormat; +import org.schabi.newpipe.crawler.VideoInfo; /** * Created by Christian Schabesberger on 18.08.15. diff --git a/app/src/main/java/org/schabi/newpipe/Downloader.java b/app/src/main/java/org/schabi/newpipe/Downloader.java index 7b5950407..372857462 100644 --- a/app/src/main/java/org/schabi/newpipe/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/Downloader.java @@ -30,7 +30,7 @@ import info.guardianproject.netcipher.NetCipher; * along with NewPipe. If not, see . */ -public class Downloader implements org.schabi.newpipe.services.Downloader { +public class Downloader implements org.schabi.newpipe.crawler.Downloader { private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"; diff --git a/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java b/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java index c2bbb069e..e254af02d 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java +++ b/app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java @@ -7,6 +7,8 @@ import android.view.ViewGroup; import android.widget.ImageView; import android.widget.TextView; +import org.schabi.newpipe.crawler.VideoPreviewInfo; + /** * Created by Christian Schabesberger on 24.10.15. * diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java index bf28549b3..8514160d4 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java @@ -10,8 +10,8 @@ import android.view.Menu; import android.view.MenuItem; import android.widget.Toast; -import org.schabi.newpipe.services.ServiceList; -import org.schabi.newpipe.services.StreamingService; +import org.schabi.newpipe.crawler.ServiceList; +import org.schabi.newpipe.crawler.StreamingService; /** diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index ba34d9960..ce3dcb317 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -16,7 +16,6 @@ import android.support.v4.app.Fragment; import android.support.v7.app.AppCompatActivity; import android.text.Html; import android.text.method.LinkMovementMethod; -import android.util.DisplayMetrics; import android.util.Log; import android.view.LayoutInflater; import android.view.Menu; @@ -36,10 +35,11 @@ import java.net.URL; import java.util.ArrayList; import java.util.Vector; -import org.schabi.newpipe.services.VideoExtractor; -import org.schabi.newpipe.services.ServiceList; -import org.schabi.newpipe.services.StreamingService; -import org.schabi.newpipe.services.VideoInfo; +import org.schabi.newpipe.crawler.VideoPreviewInfo; +import org.schabi.newpipe.crawler.VideoExtractor; +import org.schabi.newpipe.crawler.ServiceList; +import org.schabi.newpipe.crawler.StreamingService; +import org.schabi.newpipe.crawler.VideoInfo; /** diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java b/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java index 446b916b5..5f8ddde32 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java @@ -16,7 +16,8 @@ import android.view.inputmethod.InputMethodManager; import java.util.ArrayList; -import org.schabi.newpipe.services.ServiceList; +import org.schabi.newpipe.crawler.VideoPreviewInfo; +import org.schabi.newpipe.crawler.ServiceList; /** * Copyright (C) Christian Schabesberger 2015 diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java index 8cbd67677..536b216f4 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java @@ -19,8 +19,9 @@ import java.net.URL; import java.util.List; import java.util.Vector; -import org.schabi.newpipe.services.SearchEngine; -import org.schabi.newpipe.services.StreamingService; +import org.schabi.newpipe.crawler.VideoPreviewInfo; +import org.schabi.newpipe.crawler.SearchEngine; +import org.schabi.newpipe.crawler.StreamingService; /** diff --git a/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java b/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java index de08b2f03..54ff763f0 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java +++ b/app/src/main/java/org/schabi/newpipe/VideoListAdapter.java @@ -9,6 +9,8 @@ import android.view.ViewGroup; import android.widget.BaseAdapter; import android.widget.ListView; +import org.schabi.newpipe.crawler.VideoPreviewInfo; + import java.util.List; import java.util.Vector; diff --git a/app/src/main/java/org/schabi/newpipe/services/AbstractVideoInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java similarity index 92% rename from app/src/main/java/org/schabi/newpipe/services/AbstractVideoInfo.java rename to app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java index 72d43ebde..a338242bb 100644 --- a/app/src/main/java/org/schabi/newpipe/services/AbstractVideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.services; +package org.schabi.newpipe.crawler; import android.graphics.Bitmap; diff --git a/app/src/main/java/org/schabi/newpipe/services/Downloader.java b/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java similarity index 97% rename from app/src/main/java/org/schabi/newpipe/services/Downloader.java rename to app/src/main/java/org/schabi/newpipe/crawler/Downloader.java index ce83ca5da..5fda48f3e 100644 --- a/app/src/main/java/org/schabi/newpipe/services/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.services; +package org.schabi.newpipe.crawler; /** * Created by Christian Schabesberger on 28.01.16. diff --git a/app/src/main/java/org/schabi/newpipe/services/MediaFormat.java b/app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java similarity index 98% rename from app/src/main/java/org/schabi/newpipe/services/MediaFormat.java rename to app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java index 867dc3035..efebf17f3 100644 --- a/app/src/main/java/org/schabi/newpipe/services/MediaFormat.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.services; +package org.schabi.newpipe.crawler; /** * Created by Adam Howard on 08/11/15. diff --git a/app/src/main/java/org/schabi/newpipe/services/SearchEngine.java b/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java similarity index 94% rename from app/src/main/java/org/schabi/newpipe/services/SearchEngine.java rename to app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java index eebbe0ee2..3803e3e38 100644 --- a/app/src/main/java/org/schabi/newpipe/services/SearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java @@ -1,6 +1,4 @@ -package org.schabi.newpipe.services; - -import org.schabi.newpipe.VideoPreviewInfo; +package org.schabi.newpipe.crawler; import java.util.ArrayList; import java.util.Vector; diff --git a/app/src/main/java/org/schabi/newpipe/services/ServiceList.java b/app/src/main/java/org/schabi/newpipe/crawler/ServiceList.java similarity index 94% rename from app/src/main/java/org/schabi/newpipe/services/ServiceList.java rename to app/src/main/java/org/schabi/newpipe/crawler/ServiceList.java index da1b293fb..b1d98a73f 100644 --- a/app/src/main/java/org/schabi/newpipe/services/ServiceList.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/ServiceList.java @@ -1,8 +1,8 @@ -package org.schabi.newpipe.services; +package org.schabi.newpipe.crawler; import android.util.Log; -import org.schabi.newpipe.services.youtube.YoutubeService; +import org.schabi.newpipe.crawler.services.youtube.YoutubeService; /** * Created by Christian Schabesberger on 23.08.15. diff --git a/app/src/main/java/org/schabi/newpipe/services/StreamingService.java b/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java similarity index 97% rename from app/src/main/java/org/schabi/newpipe/services/StreamingService.java rename to app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java index 082dc998a..5069647ee 100644 --- a/app/src/main/java/org/schabi/newpipe/services/StreamingService.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.services; +package org.schabi.newpipe.crawler; /** * Created by Christian Schabesberger on 23.08.15. diff --git a/app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java similarity index 99% rename from app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java rename to app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java index e4357b4d0..ee7977ee0 100644 --- a/app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.services; +package org.schabi.newpipe.crawler; /** * Created by Christian Schabesberger on 10.08.15. diff --git a/app/src/main/java/org/schabi/newpipe/services/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java similarity index 93% rename from app/src/main/java/org/schabi/newpipe/services/VideoInfo.java rename to app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java index dcf871d8c..068b3ce26 100644 --- a/app/src/main/java/org/schabi/newpipe/services/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java @@ -1,7 +1,4 @@ -package org.schabi.newpipe.services; - -import org.schabi.newpipe.VideoPreviewInfo; -import org.schabi.newpipe.services.AbstractVideoInfo; +package org.schabi.newpipe.crawler; import java.util.List; @@ -73,7 +70,8 @@ public class VideoInfo extends AbstractVideoInfo { this.view_count = avi.view_count; //todo: better than this - if(avi instanceof VideoPreviewInfo) {//shitty String to convert code + if(avi instanceof VideoPreviewInfo) { + //shitty String to convert code String dur = ((VideoPreviewInfo)avi).duration; int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":"))); int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length())); @@ -82,7 +80,8 @@ public class VideoInfo extends AbstractVideoInfo { } public static class VideoStream { - public String url = ""; //url of the stream + //url of the stream + public String url = ""; public int format = -1; public String resolution = ""; diff --git a/app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoPreviewInfo.java similarity index 96% rename from app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java rename to app/src/main/java/org/schabi/newpipe/crawler/VideoPreviewInfo.java index 0832114e0..bca13a208 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoPreviewInfo.java @@ -1,10 +1,10 @@ -package org.schabi.newpipe; +package org.schabi.newpipe.crawler; import android.graphics.Bitmap; import android.os.Parcel; import android.os.Parcelable; -import org.schabi.newpipe.services.AbstractVideoInfo; +import org.schabi.newpipe.crawler.AbstractVideoInfo; /** * Created by Christian Schabesberger on 26.08.15. diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java similarity index 97% rename from app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java rename to app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java index 73b245a48..c4bedfe1d 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.services.youtube; +package org.schabi.newpipe.crawler.services.youtube; import android.net.Uri; import android.util.Log; @@ -6,9 +6,9 @@ import android.util.Log; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.schabi.newpipe.services.Downloader; -import org.schabi.newpipe.services.SearchEngine; -import org.schabi.newpipe.VideoPreviewInfo; +import org.schabi.newpipe.crawler.Downloader; +import org.schabi.newpipe.crawler.SearchEngine; +import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java similarity index 86% rename from app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java rename to app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java index b90135d1b..a2f2c3dd1 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java @@ -1,9 +1,9 @@ -package org.schabi.newpipe.services.youtube; +package org.schabi.newpipe.crawler.services.youtube; -import org.schabi.newpipe.services.Downloader; -import org.schabi.newpipe.services.StreamingService; -import org.schabi.newpipe.services.VideoExtractor; -import org.schabi.newpipe.services.SearchEngine; +import org.schabi.newpipe.crawler.Downloader; +import org.schabi.newpipe.crawler.StreamingService; +import org.schabi.newpipe.crawler.VideoExtractor; +import org.schabi.newpipe.crawler.SearchEngine; /** diff --git a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java similarity index 98% rename from app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java rename to app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java index e1cacef02..4292a4e9f 100644 --- a/app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java @@ -1,4 +1,4 @@ -package org.schabi.newpipe.services.youtube; +package org.schabi.newpipe.crawler.services.youtube; import android.util.Log; import android.util.Xml; @@ -12,12 +12,11 @@ import org.jsoup.parser.Parser; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; -import org.schabi.newpipe.FileDownloader; -import org.schabi.newpipe.services.Downloader; -import org.schabi.newpipe.services.VideoExtractor; -import org.schabi.newpipe.services.MediaFormat; -import org.schabi.newpipe.services.VideoInfo; -import org.schabi.newpipe.VideoPreviewInfo; +import org.schabi.newpipe.crawler.Downloader; +import org.schabi.newpipe.crawler.VideoExtractor; +import org.schabi.newpipe.crawler.MediaFormat; +import org.schabi.newpipe.crawler.VideoInfo; +import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.xmlpull.v1.XmlPullParser; import java.io.StringReader; From 9204a8931929d721261b6369b124b9b3e91461e5 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Thu, 28 Jan 2016 23:27:16 +0100 Subject: [PATCH 03/11] fiexed some licence headers --- .../org/schabi/newpipe/ActionBarHandler.java | 2 +- .../java/org/schabi/newpipe/Downloader.java | 2 +- .../newpipe/crawler/AbstractVideoInfo.java | 19 ++++++++++++++++++- .../schabi/newpipe/crawler/Downloader.java | 2 +- .../schabi/newpipe/crawler/MediaFormat.java | 2 +- .../youtube/YoutubeVideoExtractor.java | 19 +++++++++---------- 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java b/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java index 0cd4f0b1a..6e70e07ba 100644 --- a/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java +++ b/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java @@ -22,7 +22,7 @@ import org.schabi.newpipe.crawler.VideoInfo; /** * Created by Christian Schabesberger on 18.08.15. * - * Copyright (C) Christian Schabesberger 2015 + * Copyright (C) Christian Schabesberger 2016 * DetailsMenuHandler.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify diff --git a/app/src/main/java/org/schabi/newpipe/Downloader.java b/app/src/main/java/org/schabi/newpipe/Downloader.java index 372857462..0827f1f99 100644 --- a/app/src/main/java/org/schabi/newpipe/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/Downloader.java @@ -13,7 +13,7 @@ import info.guardianproject.netcipher.NetCipher; /** * Created by Christian Schabesberger on 28.01.16. * - * Copyright (C) Christian Schabesberger 2015 + * Copyright (C) Christian Schabesberger 2016 * Downloader.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify diff --git a/app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java index a338242bb..7a15a8af2 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java @@ -2,12 +2,29 @@ package org.schabi.newpipe.crawler; import android.graphics.Bitmap; +/** + * Copyright (C) Christian Schabesberger 2015 + * AbstractVideoInfo.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + /**Common properties between VideoInfo and VideoPreviewInfo.*/ public abstract class AbstractVideoInfo { public String id = ""; public String title = ""; public String uploader = ""; - //public int duration = -1; public String thumbnail_url = ""; public Bitmap thumbnail = null; public String webpage_url = ""; diff --git a/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java b/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java index 5fda48f3e..4c6511468 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java @@ -3,7 +3,7 @@ package org.schabi.newpipe.crawler; /** * Created by Christian Schabesberger on 28.01.16. * - * Copyright (C) Christian Schabesberger 2015 + * Copyright (C) Christian Schabesberger 2016 * Downloader.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify diff --git a/app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java b/app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java index efebf17f3..63b94fd47 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java @@ -6,7 +6,7 @@ package org.schabi.newpipe.crawler; * Copyright (c) Christian Schabesberger * and Adam Howard 2015 * - * VideoListAdapter.java is part of NewPipe. + * MediaFormat.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java index 4292a4e9f..b0ecd1e4f 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java @@ -240,8 +240,13 @@ public class YoutubeVideoExtractor extends VideoExtractor { //------------------------------------ // extract video stream url //------------------------------------ - String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map"); Vector videoStreams = new Vector<>(); + + // The following line belongs to dash audio stuff. + // We can't use dash audio, unless we have exoplayer as our main player. + //String adaptive_fmts = playerArgs.getString("adaptive_fmts"); + + String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map"); for(String url_data_str : encoded_url_map.split(",")) { Map tags = new HashMap<>(); for(String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) { @@ -264,6 +269,7 @@ public class YoutubeVideoExtractor extends VideoExtractor { resolveResolutionString(itag))); } } + return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]); } catch (Exception e) { @@ -280,6 +286,7 @@ public class YoutubeVideoExtractor extends VideoExtractor { @SuppressWarnings("WeakerAccess") public static int resolveFormat(int itag) { switch(itag) { + // !!! lists only supported formats !!! // video case 17: return MediaFormat.v3GPP.id; case 18: return MediaFormat.MPEG_4.id; @@ -408,15 +415,6 @@ public class YoutubeVideoExtractor extends VideoExtractor { // extracting information from html page //--------------------------------------- - /* Code does not work here anymore. - // Determine what went wrong when the Video is not available - if(videoInfo.errorCode == VideoInfo.ERROR_NO_SPECIFIED_ERROR) { - if(doc.select("h1[id=\"unavailable-message\"]").first().text().contains("GEMA")) { - videoInfo.videoAvailableStatus = VideoInfo.VIDEO_UNAVAILABLE_GEMA; - } - } - */ - String likesString = ""; String dislikesString = ""; try { @@ -530,6 +528,7 @@ public class YoutubeVideoExtractor extends VideoExtractor { } return audioStreams.toArray(new VideoInfo.AudioStream[audioStreams.size()]); } + /**Provides information about links to other videos on the video page, such as related videos. * This is encapsulated in a VideoPreviewInfo object, * which is a subset of the fields in a full VideoInfo.*/ From 46c2db310a635d934027955a324b9bd08a8991ad Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sat, 30 Jan 2016 00:22:16 +0100 Subject: [PATCH 04/11] add suport for dash --- .../org/schabi/newpipe/crawler/VideoExtractor.java | 5 +++++ .../java/org/schabi/newpipe/crawler/VideoInfo.java | 8 +++++++- .../services/youtube/YoutubeVideoExtractor.java | 13 +++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java index ee7977ee0..14a4a15ba 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java @@ -95,6 +95,10 @@ public abstract class VideoExtractor { if (videoInfo.startPosition < 0) { videoInfo.startPosition = getTimeStamp(); } + + if(videoInfo.dashMpdUrl.isEmpty()) { + videoInfo.dashMpdUrl = getDashMpdUrl(); + } } else { videoInfo.errorCode = getErrorCode(); videoInfo.errorMessage = getErrorMessage(); @@ -125,4 +129,5 @@ public abstract class VideoExtractor { public abstract String getUploaderThumbnailUrl(); public abstract VideoInfo.AudioStream[] getAudioStreams(); public abstract VideoInfo.VideoStream[] getVideoStreams(); + public abstract String getDashMpdUrl(); } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java index 068b3ce26..7bf077886 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java @@ -38,6 +38,11 @@ public class VideoInfo extends AbstractVideoInfo { public String description = ""; public VideoStream[] videoStreams = null; public AudioStream[] audioStreams = null; + // video streams provided by the dash mpd do not need to be provided as VideoStream. + // Later on this will also aplly to audio streams. Since dash mpd is standarized, + // crawling such a file is not service dependent. Therefore getting audio only streams by yust + // providing the dash mpd fille will be possible in the future. + public String dashMpdUrl = ""; public int errorCode = NO_ERROR; public String errorMessage = ""; public int duration = -1; @@ -50,7 +55,8 @@ public class VideoInfo extends AbstractVideoInfo { public String average_rating = ""; public VideoPreviewInfo nextVideo = null; public List relatedVideos = null; - public int startPosition = -1;//in seconds. some metadata is not passed using a VideoInfo object! + //in seconds. some metadata is not passed using a VideoInfo object! + public int startPosition = -1; public VideoInfo() {} diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java index b0ecd1e4f..436dbce90 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java @@ -220,10 +220,23 @@ public class YoutubeVideoExtractor extends VideoExtractor { } } + @Override + public String getDashMpdUrl() { + try { + return playerArgs.getString("dashmpd"); + } catch(NullPointerException e) { + Log.e(TAG, "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available)."); + } catch (Exception e) { + e.printStackTrace(); + } + return ""; + } + @Override public VideoInfo.AudioStream[] getAudioStreams() { try { String dashManifest = playerArgs.getString("dashmpd"); + Log.d(TAG, dashManifest); return parseDashManifest(dashManifest, decryptionCode); } catch (NullPointerException e) { From 7f12b58722441cbb43b34861980af87f5bb15108 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 31 Jan 2016 19:57:30 +0100 Subject: [PATCH 05/11] use java error system in the crawler --- .../youtube/YoutubeSearchEngineTest.java | 9 +- .../YoutubeVideoExtractorDefaultTest.java | 53 +- .../YoutubeVideoExtractorGemaTest.java | 40 +- .../java/org/schabi/newpipe/Downloader.java | 54 +- .../newpipe/VideoItemDetailFragment.java | 258 ++++--- .../schabi/newpipe/VideoItemListFragment.java | 30 +- .../newpipe/crawler/CrawlingException.java | 37 + .../schabi/newpipe/crawler/Downloader.java | 12 +- .../newpipe/crawler/ParsingException.java | 35 + .../schabi/newpipe/crawler/SearchEngine.java | 7 +- .../newpipe/crawler/StreamingService.java | 4 +- .../newpipe/crawler/VideoExtractor.java | 178 +++-- .../org/schabi/newpipe/crawler/VideoInfo.java | 11 - .../services/youtube/YoutubeSearchEngine.java | 162 +++-- .../services/youtube/YoutubeService.java | 7 +- .../youtube/YoutubeVideoExtractor.java | 647 ++++++++++-------- app/src/main/res/values/strings.xml | 6 + 17 files changed, 877 insertions(+), 673 deletions(-) create mode 100644 app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java create mode 100644 app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java diff --git a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java index b9ded39c2..d81c4b431 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java @@ -3,7 +3,9 @@ package org.schabi.newpipe.services.youtube; import android.test.AndroidTestCase; import org.schabi.newpipe.crawler.VideoPreviewInfo; -import org.schabi.newpipe.services.SearchEngine; +import org.schabi.newpipe.crawler.SearchEngine; +import org.schabi.newpipe.crawler.services.youtube.YoutubeSearchEngine; +import org.schabi.newpipe.Downloader; import java.util.ArrayList; @@ -35,8 +37,9 @@ public class YoutubeSearchEngineTest extends AndroidTestCase { public void setUp() throws Exception{ super.setUp(); SearchEngine engine = new YoutubeSearchEngine(); - result = engine.search("https://www.youtube.com/results?search_query=bla", 0, "de"); - suggestionReply = engine.suggestionList("hello"); + result = engine.search("https://www.youtube.com/results?search_query=bla", + 0, "de", new Downloader()); + suggestionReply = engine.suggestionList("hello", new Downloader()); } public void testIfNoErrorOccur() { diff --git a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java index 6f6ecaaad..3a067e50d 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java @@ -1,9 +1,14 @@ package org.schabi.newpipe.services.youtube; import android.test.AndroidTestCase; -import android.util.Log; -import org.schabi.newpipe.services.VideoInfo; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.crawler.CrawlingException; +import org.schabi.newpipe.crawler.ParsingException; +import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor; +import org.schabi.newpipe.crawler.VideoInfo; + +import java.io.IOException; /** * Created by the-scrabi on 30.12.15. @@ -28,58 +33,58 @@ import org.schabi.newpipe.services.VideoInfo; public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase { private YoutubeVideoExtractor extractor; - public void setUp() { - extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys"); + public void setUp() throws IOException, CrawlingException { + extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys", + new Downloader()); } - public void testGetErrorCode() { - assertEquals(extractor.getErrorCode(), VideoInfo.NO_ERROR); - } - - public void testGetErrorMessage() { - assertEquals(extractor.getErrorMessage(), ""); - } - - public void testGetTimeStamp() { + public void testGetInvalidTimeStamp() throws ParsingException { assertTrue(Integer.toString(extractor.getTimeStamp()), - extractor.getTimeStamp() >= 0); + extractor.getTimeStamp() <= 0); } - public void testGetTitle() { + public void testGetValidTimeStamp() throws CrawlingException, IOException { + YoutubeVideoExtractor extractor = + new YoutubeVideoExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader()); + assertTrue(Integer.toString(extractor.getTimeStamp()), + extractor.getTimeStamp() == 174); + } + + public void testGetTitle() throws ParsingException { assertTrue(!extractor.getTitle().isEmpty()); } - public void testGetDescription() { + public void testGetDescription() throws ParsingException { assertTrue(extractor.getDescription() != null); } - public void testGetUploader() { + public void testGetUploader() throws ParsingException { assertTrue(!extractor.getUploader().isEmpty()); } - public void testGetLength() { + public void testGetLength() throws ParsingException { assertTrue(extractor.getLength() > 0); } - public void testGetViews() { + public void testGetViews() throws ParsingException { assertTrue(extractor.getLength() > 0); } - public void testGetUploadDate() { + public void testGetUploadDate() throws ParsingException { assertTrue(extractor.getUploadDate().length() > 0); } - public void testGetThumbnailUrl() { + public void testGetThumbnailUrl() throws ParsingException { assertTrue(extractor.getThumbnailUrl(), extractor.getThumbnailUrl().contains("https://")); } - public void testGetUploaderThumbnailUrl() { + public void testGetUploaderThumbnailUrl() throws ParsingException { assertTrue(extractor.getUploaderThumbnailUrl(), extractor.getUploaderThumbnailUrl().contains("https://")); } - public void testGetAudioStreams() { + public void testGetAudioStreams() throws ParsingException { for(VideoInfo.AudioStream s : extractor.getAudioStreams()) { assertTrue(s.url, s.url.contains("https://")); @@ -88,7 +93,7 @@ public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase { } } - public void testGetVideoStreams() { + public void testGetVideoStreams() throws ParsingException { for(VideoInfo.VideoStream s : extractor.getVideoStreams()) { assertTrue(s.url, s.url.contains("https://")); diff --git a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java index 5e5cb40f6..a17b614b7 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java @@ -2,7 +2,13 @@ package org.schabi.newpipe.services.youtube; import android.test.AndroidTestCase; -import org.schabi.newpipe.services.VideoInfo; +import org.schabi.newpipe.Downloader; +import org.schabi.newpipe.crawler.CrawlingException; +import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor; +import org.schabi.newpipe.crawler.VideoInfo; +import org.schabi.newpipe.Downloader; + +import java.io.IOException; /** * Created by the-scrabi on 30.12.15. @@ -29,31 +35,17 @@ import org.schabi.newpipe.services.VideoInfo; public class YoutubeVideoExtractorGemaTest extends AndroidTestCase { // Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail. - private static final boolean testActive = false; + private static final boolean testActive = true; - - private YoutubeVideoExtractor extractor; - - public void setUp() { + public void testGemaError() throws IOException, CrawlingException { if(testActive) { - extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8"); - } - } - - public void testGetErrorCode() { - if(testActive) { - assertEquals(extractor.getErrorCode(), VideoInfo.ERROR_BLOCKED_BY_GEMA); - } else { - assertTrue(true); - } - } - - public void testGetErrorMessage() { - if(testActive) { - assertTrue(extractor.getErrorMessage(), - extractor.getErrorMessage().contains("GEMA")); - } else { - assertTrue(true); + try { + new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8", + new Downloader()); + assertTrue("Gema exception not thrown", false); + } catch(YoutubeVideoExtractor.GemaException ge) { + assertTrue(true); + } } } } diff --git a/app/src/main/java/org/schabi/newpipe/Downloader.java b/app/src/main/java/org/schabi/newpipe/Downloader.java index 0827f1f99..80f1d0dd3 100644 --- a/app/src/main/java/org/schabi/newpipe/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/Downloader.java @@ -39,42 +39,39 @@ public class Downloader implements org.schabi.newpipe.crawler.Downloader { * @param siteUrl the URL of the text file to return the contents of * @param language the language (usually a 2-character code) to set as the preferred language * @return the contents of the specified text file*/ - public String download(String siteUrl, String language) { - String ret = ""; - try { - URL url = new URL(siteUrl); - //HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); - HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); - con.setRequestProperty("Accept-Language", language); - ret = dl(con); - } - catch(Exception e) { - e.printStackTrace(); - } - return ret; + public String download(String siteUrl, String language) throws IOException { + URL url = new URL(siteUrl); + //HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); + HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); + con.setRequestProperty("Accept-Language", language); + return dl(con); } /**Common functionality between download(String url) and download(String url, String language)*/ private static String dl(HttpsURLConnection con) throws IOException { StringBuilder response = new StringBuilder(); + BufferedReader in = null; try { con.setRequestMethod("GET"); con.setRequestProperty("User-Agent", USER_AGENT); - BufferedReader in = new BufferedReader( + in = new BufferedReader( new InputStreamReader(con.getInputStream())); String inputLine; while((inputLine = in.readLine()) != null) { response.append(inputLine); } - in.close(); - - } - catch(UnknownHostException uhe) {//thrown when there's no internet connection - uhe.printStackTrace(); + } catch(UnknownHostException uhe) {//thrown when there's no internet connection + throw new IOException("unknown host or no network", uhe); //Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show(); + } catch(Exception e) { + throw new IOException(e); + } finally { + if(in != null) { + in.close(); + } } return response.toString(); @@ -84,19 +81,10 @@ public class Downloader implements org.schabi.newpipe.crawler.Downloader { * Primarily intended for downloading web pages. * @param siteUrl the URL of the text file to download * @return the contents of the specified text file*/ - public String download(String siteUrl) { - String ret = ""; - - try { - URL url = new URL(siteUrl); - HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); - //HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); - ret = dl(con); - } - catch(Exception e) { - e.printStackTrace(); - } - - return ret; + public String download(String siteUrl) throws IOException { + URL url = new URL(siteUrl); + HttpsURLConnection con = (HttpsURLConnection) url.openConnection(); + //HttpsURLConnection con = NetCipher.getHttpsURLConnection(url); + return dl(con); } } diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index ce3dcb317..076a12137 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -31,15 +31,20 @@ import android.widget.TextView; import android.view.MenuItem; import android.widget.Toast; +import java.io.IOException; import java.net.URL; +import java.nio.charset.MalformedInputException; import java.util.ArrayList; import java.util.Vector; +import org.schabi.newpipe.crawler.CrawlingException; +import org.schabi.newpipe.crawler.ParsingException; import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.schabi.newpipe.crawler.VideoExtractor; import org.schabi.newpipe.crawler.ServiceList; import org.schabi.newpipe.crawler.StreamingService; import org.schabi.newpipe.crawler.VideoInfo; +import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor; /** @@ -68,7 +73,6 @@ public class VideoItemDetailFragment extends Fragment { * The fragment argument representing the item ID that this fragment * represents. */ - //public static final String ARG_ITEM_ID = "item_id"; public static final String VIDEO_URL = "video_url"; public static final String STREAMING_SERVICE = "streaming_service"; public static final String AUTO_PLAY = "auto_play"; @@ -87,7 +91,6 @@ public class VideoItemDetailFragment extends Fragment { private FloatingActionButton playVideoButton; private final Point initialThumbnailPos = new Point(0, 0); - public interface OnInvokeCreateOptionsMenuListener { void createOptionsMenu(); } @@ -108,45 +111,64 @@ public class VideoItemDetailFragment extends Fragment { @Override public void run() { try { - this.videoExtractor = service.getExtractorInstance(videoUrl, new Downloader()); + videoExtractor = service.getExtractorInstance(videoUrl, new Downloader()); VideoInfo videoInfo = videoExtractor.getVideoInfo(); h.post(new VideoResultReturnedRunnable(videoInfo)); - if (videoInfo.errorCode == VideoInfo.NO_ERROR) { + h.post(new SetThumbnailRunnable( + BitmapFactory.decodeStream( + new URL(videoInfo.thumbnail_url) + .openConnection() + .getInputStream()), + SetThumbnailRunnable.VIDEO_THUMBNAIL)); + h.post(new SetThumbnailRunnable( + BitmapFactory.decodeStream( + new URL(videoInfo.uploader_thumbnail_url) + .openConnection() + .getInputStream()), + SetThumbnailRunnable.CHANNEL_THUMBNAIL)); + if (showNextVideoItem) { h.post(new SetThumbnailRunnable( BitmapFactory.decodeStream( - new URL(videoInfo.thumbnail_url) + new URL(videoInfo.nextVideo.thumbnail_url) .openConnection() .getInputStream()), - SetThumbnailRunnable.VIDEO_THUMBNAIL)); - h.post(new SetThumbnailRunnable( - BitmapFactory.decodeStream( - new URL(videoInfo.uploader_thumbnail_url) - .openConnection() - .getInputStream()), - SetThumbnailRunnable.CHANNEL_THUMBNAIL)); - if(showNextVideoItem) { - h.post(new SetThumbnailRunnable( - BitmapFactory.decodeStream( - new URL(videoInfo.nextVideo.thumbnail_url) - .openConnection() - .getInputStream()), - SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL)); - } + SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL)); } - } catch (Exception e) { + } catch (MalformedInputException e) { + postNewErrorToast(h, R.string.could_not_load_thumbnails); + e.printStackTrace(); + } catch (IOException e) { + postNewErrorToast(h, R.string.network_error); + e.printStackTrace(); + } + // custom service related exceptions + catch (YoutubeVideoExtractor.DecryptException de) { + postNewErrorToast(h, R.string.youtube_signature_decryption_error); + de.printStackTrace(); + } catch (YoutubeVideoExtractor.GemaException ge) { h.post(new Runnable() { @Override public void run() { - progressBar.setVisibility(View.GONE); - // This is poor style, but unless we have better error handling in the - // crawler, this may not be better. - Toast.makeText(VideoItemDetailFragment.this.getActivity(), - R.string.network_error, Toast.LENGTH_LONG).show(); + onErrorBlockedByGema(); + } + }); + } + // ---------------------------------------- + catch(VideoExtractor.ContentNotAvailableException e) { + h.post(new Runnable() { + @Override + public void run() { + onNotSpecifiedContentError(); } }); e.printStackTrace(); + } catch (ParsingException e) { + postNewErrorToast(h, e.getMessage()); + e.printStackTrace(); + } catch(Exception e) { + postNewErrorToast(h, R.string.general_error); + e.printStackTrace(); } - } } @@ -213,7 +235,7 @@ public class VideoItemDetailFragment extends Fragment { private void updateInfo(VideoInfo info) { currentVideoInfo = info; - Resources res = activity.getResources(); + try { VideoInfoItemViewCreator videoItemViewCreator = new VideoInfoItemViewCreator(LayoutInflater.from(getActivity())); @@ -226,107 +248,80 @@ public class VideoItemDetailFragment extends Fragment { TextView thumbsDownView = (TextView) activity.findViewById(R.id.detailThumbsDownCountView); TextView uploadDateView = (TextView) activity.findViewById(R.id.detailUploadDateView); TextView descriptionView = (TextView) activity.findViewById(R.id.detailDescriptionView); - ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView); FrameLayout nextVideoFrame = (FrameLayout) activity.findViewById(R.id.detailNextVideoFrame); RelativeLayout nextVideoRootFrame = (RelativeLayout) activity.findViewById(R.id.detailNextVideoRootLayout); - Button backgroundButton = (Button) - activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton); progressBar.setVisibility(View.GONE); - switch (info.errorCode) { - case VideoInfo.NO_ERROR: { - View nextVideoView = videoItemViewCreator - .getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext()); - nextVideoFrame.addView(nextVideoView); + + View nextVideoView = videoItemViewCreator + .getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext()); + nextVideoFrame.addView(nextVideoView); - Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton); - Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton); + Button nextVideoButton = (Button) activity.findViewById(R.id.detailNextVideoButton); + Button similarVideosButton = (Button) activity.findViewById(R.id.detailShowSimilarButton); - textContentLayout.setVisibility(View.VISIBLE); - playVideoButton.setVisibility(View.VISIBLE); - if (!showNextVideoItem) { - nextVideoRootFrame.setVisibility(View.GONE); - similarVideosButton.setVisibility(View.GONE); - } + textContentLayout.setVisibility(View.VISIBLE); + playVideoButton.setVisibility(View.VISIBLE); + if (!showNextVideoItem) { + nextVideoRootFrame.setVisibility(View.GONE); + similarVideosButton.setVisibility(View.GONE); + } - videoTitleView.setText(info.title); - uploaderView.setText(info.uploader); - actionBarHandler.setChannelName(info.uploader); + videoTitleView.setText(info.title); + uploaderView.setText(info.uploader); + actionBarHandler.setChannelName(info.uploader); - String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext()); - viewCountView.setText(localizedViewCount); + String localizedViewCount = Localization.localizeViewCount(info.view_count, getContext()); + viewCountView.setText(localizedViewCount); - String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext()); - thumbsUpView.setText(localizedLikeCount); + String localizedLikeCount = Localization.localizeNumber(info.like_count, getContext()); + thumbsUpView.setText(localizedLikeCount); - String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext()); - thumbsDownView.setText(localizedDislikeCount); + String localizedDislikeCount = Localization.localizeNumber(info.dislike_count, getContext()); + thumbsDownView.setText(localizedDislikeCount); - String localizedDate = Localization.localizeDate(info.upload_date, getContext()); - uploadDateView.setText(localizedDate); + String localizedDate = Localization.localizeDate(info.upload_date, getContext()); + uploadDateView.setText(localizedDate); - descriptionView.setText(Html.fromHtml(info.description)); + descriptionView.setText(Html.fromHtml(info.description)); - descriptionView.setMovementMethod(LinkMovementMethod.getInstance()); + descriptionView.setMovementMethod(LinkMovementMethod.getInstance()); - actionBarHandler.setServiceId(streamingServiceId); - actionBarHandler.setVideoInfo(info.webpage_url, info.title); - actionBarHandler.setStartPosition(info.startPosition); + actionBarHandler.setServiceId(streamingServiceId); + actionBarHandler.setVideoInfo(info.webpage_url, info.title); + actionBarHandler.setStartPosition(info.startPosition); - // parse streams - Vector streamsToUse = new Vector<>(); - for (VideoInfo.VideoStream i : info.videoStreams) { - if (useStream(i, streamsToUse)) { - streamsToUse.add(i); - } - } - VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()]; - for (int i = 0; i < streamList.length; i++) { - streamList[i] = streamsToUse.get(i); - } - actionBarHandler.setStreams(streamList, info.audioStreams); + // parse streams + Vector streamsToUse = new Vector<>(); + for (VideoInfo.VideoStream i : info.videoStreams) { + if (useStream(i, streamsToUse)) { + streamsToUse.add(i); + } + } + VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()]; + for (int i = 0; i < streamList.length; i++) { + streamList[i] = streamsToUse.get(i); + } + actionBarHandler.setStreams(streamList, info.audioStreams); - nextVideoButton.setOnClickListener(new View.OnClickListener() { - @Override - public void onClick(View v) { - Intent detailIntent = - new Intent(getActivity(), VideoItemDetailActivity.class); + nextVideoButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + Intent detailIntent = + new Intent(getActivity(), VideoItemDetailActivity.class); /*detailIntent.putExtra( VideoItemDetailFragment.ARG_ITEM_ID, currentVideoInfo.nextVideo.id); */ - detailIntent.putExtra( - VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url); + detailIntent.putExtra( + VideoItemDetailFragment.VIDEO_URL, currentVideoInfo.nextVideo.webpage_url); - detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId); - startActivity(detailIntent); - } - }); + detailIntent.putExtra(VideoItemDetailFragment.STREAMING_SERVICE, streamingServiceId); + startActivity(detailIntent); } - break; - case VideoInfo.ERROR_BLOCKED_BY_GEMA: - thumbnailView.setImageBitmap(BitmapFactory.decodeResource( - getResources(), R.drawable.gruese_die_gema)); - backgroundButton.setOnClickListener(new View.OnClickListener() { - @Override - public void onClick(View v) { - Intent intent = new Intent(); - intent.setAction(Intent.ACTION_VIEW); - intent.setData(Uri.parse(activity.getString(R.string.c3s_url))); - activity.startActivity(intent); - } - }); - break; - case VideoInfo.ERROR_NO_SPECIFIED_ERROR: - thumbnailView.setImageBitmap(BitmapFactory.decodeResource( - getResources(), R.drawable.not_available_monkey)); - Toast.makeText(activity, info.errorMessage, Toast.LENGTH_LONG) - .show(); - break; - default: - Log.e(TAG, "Video Available Status not known."); - } + }); + if(autoPlayEnabled) { actionBarHandler.playVideo(); @@ -337,6 +332,37 @@ public class VideoItemDetailFragment extends Fragment { } } + private void onErrorBlockedByGema() { + Button backgroundButton = (Button) + activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton); + ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView); + + progressBar.setVisibility(View.GONE); + thumbnailView.setImageBitmap(BitmapFactory.decodeResource( + getResources(), R.drawable.gruese_die_gema)); + backgroundButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + Intent intent = new Intent(); + intent.setAction(Intent.ACTION_VIEW); + intent.setData(Uri.parse(activity.getString(R.string.c3s_url))); + activity.startActivity(intent); + } + }); + + Toast.makeText(VideoItemDetailFragment.this.getActivity(), + R.string.blocked_by_gema, Toast.LENGTH_LONG).show(); + } + + private void onNotSpecifiedContentError() { + ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView); + progressBar.setVisibility(View.GONE); + thumbnailView.setImageBitmap(BitmapFactory.decodeResource( + getResources(), R.drawable.not_available_monkey)); + Toast.makeText(activity, R.string.content_not_available, Toast.LENGTH_LONG) + .show(); + } + private boolean useStream(VideoInfo.VideoStream stream, Vector streams) { for(VideoInfo.VideoStream i : streams) { if(i.resolution.equals(stream.resolution)) { @@ -465,4 +491,24 @@ public class VideoItemDetailFragment extends Fragment { public void setOnInvokeCreateOptionsMenuListener(OnInvokeCreateOptionsMenuListener listener) { this.onInvokeCreateOptionsMenuListener = listener; } + + private void postNewErrorToast(Handler h, final int stringResource) { + h.post(new Runnable() { + @Override + public void run() { + Toast.makeText(VideoItemDetailFragment.this.getActivity(), + stringResource, Toast.LENGTH_LONG).show(); + } + }); + } + + private void postNewErrorToast(Handler h, final String message) { + h.post(new Runnable() { + @Override + public void run() { + Toast.makeText(VideoItemDetailFragment.this.getActivity(), + message, Toast.LENGTH_LONG).show(); + } + }); + } } \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java index 536b216f4..21781f78c 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java @@ -15,10 +15,12 @@ import android.widget.AbsListView; import android.widget.ListView; import android.widget.Toast; +import java.io.IOException; import java.net.URL; import java.util.List; import java.util.Vector; +import org.schabi.newpipe.crawler.CrawlingException; import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.schabi.newpipe.crawler.SearchEngine; import org.schabi.newpipe.crawler.StreamingService; @@ -116,17 +118,15 @@ public class VideoItemListFragment extends ListFragment { if(runs) { h.post(new ResultRunnable(result, requestId)); } - } catch(Exception e) { + } catch(IOException e) { + postNewErrorToast(h, R.string.network_error); + e.printStackTrace(); + } catch(CrawlingException ce) { + postNewErrorToast(h, R.string.parsing_error); + ce.printStackTrace(); + } catch(Exception e) { + postNewErrorToast(h, R.string.general_error); e.printStackTrace(); - - h.post(new Runnable() { - @Override - public void run() { - setListShown(true); - Toast.makeText(getActivity(), getString(R.string.network_error), - Toast.LENGTH_SHORT).show(); - } - }); } } } @@ -386,4 +386,14 @@ public class VideoItemListFragment extends ListFragment { mActivatedPosition = position; } + private void postNewErrorToast(Handler h, final int stringResource) { + h.post(new Runnable() { + @Override + public void run() { + setListShown(true); + Toast.makeText(getActivity(), getString(R.string.network_error), + Toast.LENGTH_SHORT).show(); + } + }); + } } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java b/app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java new file mode 100644 index 000000000..291670953 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.crawler; + +/** + * Created by Christian Schabesberger on 30.01.16. + * + * Copyright (C) Christian Schabesberger 2016 + * CrawlingException.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class CrawlingException extends Exception { + public CrawlingException() {} + + public CrawlingException(String message) { + super(message); + } + + public CrawlingException(Throwable cause) { + super(cause); + } + + public CrawlingException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java b/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java index 4c6511468..8732c0372 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/Downloader.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.crawler; +import java.io.IOException; + /** * Created by Christian Schabesberger on 28.01.16. * @@ -26,12 +28,14 @@ public interface Downloader { * but set the HTTP header field "Accept-Language" to the supplied string. * @param siteUrl the URL of the text file to return the contents of * @param language the language (usually a 2-character code) to set as the preferred language - * @return the contents of the specified text file*/ - String download(String siteUrl, String language); + * @return the contents of the specified text file + * @throws IOException*/ + String download(String siteUrl, String language) throws IOException; /**Download (via HTTP) the text file located at the supplied URL, and return its contents. * Primarily intended for downloading web pages. * @param siteUrl the URL of the text file to download - * @return the contents of the specified text file*/ - String download(String siteUrl); + * @return the contents of the specified text file + * @throws IOException*/ + String download(String siteUrl) throws IOException; } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java b/app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java new file mode 100644 index 000000000..25d46b119 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java @@ -0,0 +1,35 @@ +package org.schabi.newpipe.crawler; + +/** + * Created by Christian Schabesberger on 31.01.16. + * + * Copyright (C) Christian Schabesberger 2016 + * ParsingException.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + + +public class ParsingException extends CrawlingException { + public ParsingException() {} + public ParsingException(String message) { + super(message); + } + public ParsingException(Throwable cause) { + super(cause); + } + public ParsingException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java b/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java index 3803e3e38..686dea912 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.crawler; +import java.io.IOException; import java.util.ArrayList; import java.util.Vector; @@ -31,8 +32,10 @@ public interface SearchEngine { public final Vector resultList = new Vector<>(); } - ArrayList suggestionList(String query, Downloader dl); + ArrayList suggestionList(String query, Downloader dl) + throws CrawlingException, IOException; //Result search(String query, int page); - Result search(String query, int page, String contentCountry, Downloader dl); + Result search(String query, int page, String contentCountry, Downloader dl) + throws CrawlingException, IOException; } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java b/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java index 5069647ee..68c3da265 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.crawler; +import java.io.IOException; + /** * Created by Christian Schabesberger on 23.08.15. * @@ -25,7 +27,7 @@ public interface StreamingService { public String name = ""; } ServiceInfo getServiceInfo(); - VideoExtractor getExtractorInstance(String url, Downloader downloader); + VideoExtractor getExtractorInstance(String url, Downloader downloader) throws IOException, CrawlingException; SearchEngine getSearchEngineInstance(); /**When a VIEW_ACTION is caught this function will test if the url delivered within the calling diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java index 14a4a15ba..e09b2e01f 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java @@ -22,8 +22,43 @@ package org.schabi.newpipe.crawler; /**Scrapes information from a video streaming service (eg, YouTube).*/ + @SuppressWarnings("ALL") public abstract class VideoExtractor { + + public class ExctractorInitException extends CrawlingException { + public ExctractorInitException() {} + public ExctractorInitException(String message) { + super(message); + } + public ExctractorInitException(Throwable cause) { + super(cause); + } + public ExctractorInitException(String message, Throwable cause) { + super(message, cause); + } + } + + public class RegexException extends ParsingException { + public RegexException() {} + public RegexException(String message) { + super(message); + } + } + + public class ContentNotAvailableException extends ParsingException { + public ContentNotAvailableException() {} + public ContentNotAvailableException(String message) { + super(message); + } + public ContentNotAvailableException(Throwable cause) { + super(cause); + } + public ContentNotAvailableException(String message, Throwable cause) { + super(message, cause); + } + } + protected final String pageUrl; protected VideoInfo videoInfo; @@ -34,7 +69,7 @@ public abstract class VideoExtractor { /**Fills out the video info fields which are common to all services. * Probably needs to be overridden by subclasses*/ - public VideoInfo getVideoInfo() + public VideoInfo getVideoInfo() throws CrawlingException { if(videoInfo == null) { videoInfo = new VideoInfo(); @@ -44,90 +79,83 @@ public abstract class VideoExtractor { videoInfo.webpage_url = pageUrl; } - if(getErrorCode() == VideoInfo.NO_ERROR) { - if (videoInfo.title.isEmpty()) { - videoInfo.title = getTitle(); - } - - if (videoInfo.duration < 1) { - videoInfo.duration = getLength(); - } - - - if (videoInfo.uploader.isEmpty()) { - videoInfo.uploader = getUploader(); - } - - if (videoInfo.description.isEmpty()) { - videoInfo.description = getDescription(); - } - - if (videoInfo.view_count == -1) { - videoInfo.view_count = getViews(); - } - - if (videoInfo.upload_date.isEmpty()) { - videoInfo.upload_date = getUploadDate(); - } - - if (videoInfo.thumbnail_url.isEmpty()) { - videoInfo.thumbnail_url = getThumbnailUrl(); - } - - if (videoInfo.id.isEmpty()) { - videoInfo.id = getVideoId(pageUrl); - } - - /** Load and extract audio*/ - if (videoInfo.audioStreams == null) { - videoInfo.audioStreams = getAudioStreams(); - } - /** Extract video stream url*/ - if (videoInfo.videoStreams == null) { - videoInfo.videoStreams = getVideoStreams(); - } - - if (videoInfo.uploader_thumbnail_url.isEmpty()) { - videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl(); - } - - if (videoInfo.startPosition < 0) { - videoInfo.startPosition = getTimeStamp(); - } - - if(videoInfo.dashMpdUrl.isEmpty()) { - videoInfo.dashMpdUrl = getDashMpdUrl(); - } - } else { - videoInfo.errorCode = getErrorCode(); - videoInfo.errorMessage = getErrorMessage(); + if (videoInfo.title.isEmpty()) { + videoInfo.title = getTitle(); } + if (videoInfo.duration < 1) { + videoInfo.duration = getLength(); + } + + + if (videoInfo.uploader.isEmpty()) { + videoInfo.uploader = getUploader(); + } + + if (videoInfo.description.isEmpty()) { + videoInfo.description = getDescription(); + } + + if (videoInfo.view_count == -1) { + videoInfo.view_count = getViews(); + } + + if (videoInfo.upload_date.isEmpty()) { + videoInfo.upload_date = getUploadDate(); + } + + if (videoInfo.thumbnail_url.isEmpty()) { + videoInfo.thumbnail_url = getThumbnailUrl(); + } + + if (videoInfo.id.isEmpty()) { + videoInfo.id = getVideoId(pageUrl); + } + + /** Load and extract audio*/ + if (videoInfo.audioStreams == null) { + videoInfo.audioStreams = getAudioStreams(); + } + /** Extract video stream url*/ + if (videoInfo.videoStreams == null) { + videoInfo.videoStreams = getVideoStreams(); + } + + if (videoInfo.uploader_thumbnail_url.isEmpty()) { + videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl(); + } + + if (videoInfo.startPosition < 0) { + videoInfo.startPosition = getTimeStamp(); + } + + if(videoInfo.dashMpdUrl.isEmpty()) { + videoInfo.dashMpdUrl = getDashMpdUrl(); + } + + //Bitmap thumbnail = null; //Bitmap uploader_thumbnail = null; //int videoAvailableStatus = VIDEO_AVAILABLE; return videoInfo; } - //todo: add licence field - public abstract int getErrorCode(); - public abstract String getErrorMessage(); - //todo: remove these functions, or make them static, otherwise its useles, to have them here public abstract String getVideoUrl(String videoId); - public abstract String getVideoId(String siteUrl); + public abstract String getVideoId(String siteUrl) throws ParsingException; /////////////////////////////////////////////////////////////////////////////////////////// - public abstract int getTimeStamp(); - public abstract String getTitle(); - public abstract String getDescription(); - public abstract String getUploader(); - public abstract int getLength(); - public abstract long getViews(); - public abstract String getUploadDate(); - public abstract String getThumbnailUrl(); - public abstract String getUploaderThumbnailUrl(); - public abstract VideoInfo.AudioStream[] getAudioStreams(); - public abstract VideoInfo.VideoStream[] getVideoStreams(); - public abstract String getDashMpdUrl(); + public abstract int getTimeStamp() throws ParsingException; + public abstract String getTitle() throws ParsingException; + public abstract String getDescription() throws ParsingException; + public abstract String getUploader() throws ParsingException; + public abstract int getLength() throws ParsingException; + public abstract long getViews() throws ParsingException; + public abstract String getUploadDate() throws ParsingException; + public abstract String getThumbnailUrl() throws ParsingException; + public abstract String getUploaderThumbnailUrl() throws ParsingException; + public abstract VideoInfo.AudioStream[] getAudioStreams() throws ParsingException; + public abstract VideoInfo.VideoStream[] getVideoStreams() throws ParsingException; + public abstract String getDashMpdUrl() throws ParsingException; + public abstract int getAgeLimit() throws ParsingException; } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java index 7bf077886..fbea6d0fc 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java @@ -26,14 +26,6 @@ import java.util.List; @SuppressWarnings("ALL") public class VideoInfo extends AbstractVideoInfo { - // If a video could not be parsed, this predefined error codes - // will be returned AND can be parsed by the frontend of the app. - // Error codes: - public final static int NO_ERROR = 0x0; - public final static int ERROR_NO_SPECIFIED_ERROR = 0x1; - // GEMA a german music colecting society. - public final static int ERROR_BLOCKED_BY_GEMA = 0x2; - public String uploader_thumbnail_url = ""; public String description = ""; public VideoStream[] videoStreams = null; @@ -43,8 +35,6 @@ public class VideoInfo extends AbstractVideoInfo { // crawling such a file is not service dependent. Therefore getting audio only streams by yust // providing the dash mpd fille will be possible in the future. public String dashMpdUrl = ""; - public int errorCode = NO_ERROR; - public String errorMessage = ""; public int duration = -1; /*YouTube-specific fields @@ -60,7 +50,6 @@ public class VideoInfo extends AbstractVideoInfo { public VideoInfo() {} - /**Creates a new VideoInfo object from an existing AbstractVideoInfo. * All the shared properties are copied to the new VideoInfo.*/ @SuppressWarnings("WeakerAccess") diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java index c4bedfe1d..9d7ce88ef 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java @@ -6,8 +6,11 @@ import android.util.Log; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.schabi.newpipe.crawler.CrawlingException; import org.schabi.newpipe.crawler.Downloader; +import org.schabi.newpipe.crawler.ParsingException; import org.schabi.newpipe.crawler.SearchEngine; +import org.schabi.newpipe.crawler.VideoExtractor; import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.w3c.dom.Node; import org.w3c.dom.NodeList; @@ -49,8 +52,8 @@ public class YoutubeSearchEngine implements SearchEngine { private static final String TAG = YoutubeSearchEngine.class.toString(); @Override - public Result search(String query, int page, String languageCode, Downloader downloader) { - //String contentCountry = PreferenceManager.getDefaultSharedPreferences(this).getString(getString(R.string., ""); + public Result search(String query, int page, String languageCode, Downloader downloader) throws IOException, ParsingException { + Result result = new Result(); Uri.Builder builder = new Uri.Builder(); builder.scheme("https") .authority("www.youtube.com") @@ -63,22 +66,19 @@ public class YoutubeSearchEngine implements SearchEngine { String url = builder.build().toString(); //if we've been passed a valid language code, append it to the URL if(!languageCode.isEmpty()) { - //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode); - site = downloader.download(url, languageCode); + //assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode); + site = downloader.download(url, languageCode); } else { site = downloader.download(url); } + try { - Document doc = Jsoup.parse(site, url); - Result result = new Result(); - Element list = doc.select("ol[class=\"item-section\"]").first(); + Document doc = Jsoup.parse(site, url); + Element list = doc.select("ol[class=\"item-section\"]").first(); - - int i = 0; - for(Element item : list.children()) { - i++; + for (Element item : list.children()) { /* First we need to determine which kind of item we are working with. Youtube depicts five different kinds of items on its search result page. These are regular videos, playlists, channels, two types of video suggestions, and a "no video @@ -90,57 +90,61 @@ public class YoutubeSearchEngine implements SearchEngine { playlists now. */ - Element el; + Element el; - // both types of spell correction item - if(!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) { - result.suggestion = el.select("a").first().text(); - // search message item - } else if(!((el = item.select("div[class*=\"search-message\"]").first()) == null)) { - result.errorMessage = el.text(); + // both types of spell correction item + if (!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) { + result.suggestion = el.select("a").first().text(); + // search message item + } else if (!((el = item.select("div[class*=\"search-message\"]").first()) == null)) { + result.errorMessage = el.text(); - // video item type - } else if(!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) { - VideoPreviewInfo resultItem = new VideoPreviewInfo(); - Element dl = el.select("h3").first().select("a").first(); - resultItem.webpage_url = dl.attr("abs:href"); - try { - Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)"); - Matcher m = p.matcher(resultItem.webpage_url); - resultItem.id=m.group(1); - } catch (Exception e) { - //e.printStackTrace(); + // video item type + } else if (!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) { + VideoPreviewInfo resultItem = new VideoPreviewInfo(); + Element dl = el.select("h3").first().select("a").first(); + resultItem.webpage_url = dl.attr("abs:href"); + try { + Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)"); + Matcher m = p.matcher(resultItem.webpage_url); + resultItem.id = m.group(1); + } catch (Exception e) { + //e.printStackTrace(); + } + resultItem.title = dl.text(); + + resultItem.duration = item.select("span[class=\"video-time\"]").first().text(); + + resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first() + .select("a").first() + .text(); + resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first() + .select("li").first() + .text(); + Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first() + .select("img").first(); + resultItem.thumbnail_url = te.attr("abs:src"); + // Sometimes youtube sends links to gif files which somehow seem to not exist + // anymore. Items with such gif also offer a secondary image source. So we are going + // to use that if we've caught such an item. + if (resultItem.thumbnail_url.contains(".gif")) { + resultItem.thumbnail_url = te.attr("abs:data-thumb"); + } + result.resultList.add(resultItem); + } else { + //noinspection ConstantConditions + Log.e(TAG, "unexpected element found:\"" + el + "\""); } - resultItem.title = dl.text(); - - resultItem.duration = item.select("span[class=\"video-time\"]").first().text(); - - resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first() - .select("a").first() - .text(); - resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first() - .select("li").first() - .text(); - Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first() - .select("img").first(); - resultItem.thumbnail_url = te.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we've caught such an item. - if(resultItem.thumbnail_url.contains(".gif")) { - resultItem.thumbnail_url = te.attr("abs:data-thumb"); - } - result.resultList.add(resultItem); - } else { - //noinspection ConstantConditions - Log.e(TAG, "unexpected element found:\""+el+"\""); } + } catch(Exception e) { + throw new ParsingException(e); } return result; } @Override - public ArrayList suggestionList(String query, Downloader dl) { + public ArrayList suggestionList(String query, Downloader dl) + throws IOException, ParsingException { ArrayList suggestions = new ArrayList<>(); @@ -155,36 +159,42 @@ public class YoutubeSearchEngine implements SearchEngine { .appendQueryParameter("q", query); String url = builder.build().toString(); + String response = dl.download(url); - //TODO: Parse xml data using Jsoup not done - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder; - org.w3c.dom.Document doc = null; - try { - dBuilder = dbFactory.newDocumentBuilder(); - doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8")))); - doc.getDocumentElement().normalize(); - }catch (ParserConfigurationException | SAXException | IOException e) { - e.printStackTrace(); - } - if(doc!=null){ - NodeList nList = doc.getElementsByTagName("CompleteSuggestion"); - for (int temp = 0; temp < nList.getLength(); temp++) { + //TODO: Parse xml data using Jsoup not done + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder; + org.w3c.dom.Document doc = null; - NodeList nList1 = doc.getElementsByTagName("suggestion"); - Node nNode1 = nList1.item(temp); - if (nNode1.getNodeType() == Node.ELEMENT_NODE) { - org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1; - suggestions.add(eElement.getAttribute("data")); - } + try { + dBuilder = dbFactory.newDocumentBuilder(); + doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8")))); + doc.getDocumentElement().normalize(); + } catch (ParserConfigurationException | SAXException | IOException e) { + e.printStackTrace(); } - }else { - Log.e(TAG, "GREAT FUCKING ERROR"); + + if (doc != null) { + NodeList nList = doc.getElementsByTagName("CompleteSuggestion"); + for (int temp = 0; temp < nList.getLength(); temp++) { + + NodeList nList1 = doc.getElementsByTagName("suggestion"); + Node nNode1 = nList1.item(temp); + if (nNode1.getNodeType() == Node.ELEMENT_NODE) { + org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1; + suggestions.add(eElement.getAttribute("data")); + } + } + } else { + Log.e(TAG, "GREAT FUCKING ERROR"); + } + return suggestions; + } catch(Exception e) { + throw new ParsingException(e); } - return suggestions; } } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java index a2f2c3dd1..1a765ab66 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java @@ -1,10 +1,13 @@ package org.schabi.newpipe.crawler.services.youtube; +import org.schabi.newpipe.crawler.CrawlingException; import org.schabi.newpipe.crawler.Downloader; import org.schabi.newpipe.crawler.StreamingService; import org.schabi.newpipe.crawler.VideoExtractor; import org.schabi.newpipe.crawler.SearchEngine; +import java.io.IOException; + /** * Created by Christian Schabesberger on 23.08.15. @@ -34,9 +37,9 @@ public class YoutubeService implements StreamingService { return serviceInfo; } @Override - public VideoExtractor getExtractorInstance(String url, Downloader downloader) { + public VideoExtractor getExtractorInstance(String url, Downloader downloader) throws CrawlingException, IOException { if(acceptUrl(url)) { - return new YoutubeVideoExtractor(url, downloader); + return new YoutubeVideoExtractor(url, downloader) ; } else { throw new IllegalArgumentException("supplied String is not a valid Youtube URL"); diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java index 436dbce90..83ebb8ca0 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java @@ -12,13 +12,16 @@ import org.jsoup.parser.Parser; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; +import org.schabi.newpipe.crawler.CrawlingException; import org.schabi.newpipe.crawler.Downloader; +import org.schabi.newpipe.crawler.ParsingException; import org.schabi.newpipe.crawler.VideoExtractor; import org.schabi.newpipe.crawler.MediaFormat; import org.schabi.newpipe.crawler.VideoInfo; import org.schabi.newpipe.crawler.VideoPreviewInfo; import org.xmlpull.v1.XmlPullParser; +import java.io.IOException; import java.io.StringReader; import java.net.URLDecoder; import java.util.HashMap; @@ -49,11 +52,28 @@ import java.util.regex.Pattern; public class YoutubeVideoExtractor extends VideoExtractor { + public class DecryptException extends ParsingException { + DecryptException(Throwable cause) { + super(cause); + } + DecryptException(String message, Throwable cause) { + super(message, cause); + } + } + + // special content not available exceptions + + public class GemaException extends ContentNotAvailableException { + GemaException(String message) { + super(message); + } + } + + // ---------------- + private static final String TAG = YoutubeVideoExtractor.class.toString(); private final Document doc; - private JSONObject jsonObj; private JSONObject playerArgs; - private int errorCode = VideoInfo.NO_ERROR; private String errorMessage = ""; // static values @@ -64,29 +84,32 @@ public class YoutubeVideoExtractor extends VideoExtractor { private Downloader downloader; - public YoutubeVideoExtractor(String pageUrl, Downloader dl) { + public YoutubeVideoExtractor(String pageUrl, Downloader dl) throws CrawlingException, IOException { //most common videoInfo fields are now set in our superclass, for all services super(pageUrl, dl); downloader = dl; String pageContent = downloader.download(cleanUrl(pageUrl)); doc = Jsoup.parse(pageContent, pageUrl); + String ytPlayerConfigRaw; + JSONObject ytPlayerConfig; //attempt to load the youtube js player JSON arguments try { - String jsonString = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); - //todo: implement this by try and catch. TESTING THE STRING AGAINST EMPTY IS CONSIDERED POOR STYLE !!! - if(jsonString.isEmpty()) { - errorCode = findErrorReason(doc); - return; + ytPlayerConfigRaw = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); + ytPlayerConfig = new JSONObject(ytPlayerConfigRaw); + playerArgs = ytPlayerConfig.getJSONObject("args"); + } catch (RegexException e) { + String errorReason = findErrorReason(doc); + switch(errorReason) { + case "GEMA": + throw new GemaException(errorReason); + case "": + throw new ParsingException("player config empty", e); + default: + throw new ContentNotAvailableException("Content not available", e); } - - jsonObj = new JSONObject(jsonString); - playerArgs = jsonObj.getJSONObject("args"); - - } catch (Exception e) {//if this fails, the video is most likely not available. - // Determining why is done later. - videoInfo.errorCode = VideoInfo.ERROR_NO_SPECIFIED_ERROR; - Log.e(TAG, "Could not load JSON data for Youtube video \""+pageUrl+"\". This most likely means the video is unavailable"); + } catch (JSONException e) { + throw new ParsingException("Could not parse yt player config"); } //---------------------------------- @@ -94,102 +117,92 @@ public class YoutubeVideoExtractor extends VideoExtractor { //---------------------------------- if (decryptionCode.isEmpty()) { try { - // The Youtube service needs to be initialized by downloading the - // js-Youtube-player. This is done in order to get the algorithm - // for decrypting cryptic signatures inside certain stream urls. - JSONObject ytAssets = jsonObj.getJSONObject("assets"); + // The Youtube service needs to be initialized by downloading the + // js-Youtube-player. This is done in order to get the algorithm + // for decrypting cryptic signatures inside certain stream urls. + JSONObject ytAssets = ytPlayerConfig.getJSONObject("assets"); String playerUrl = ytAssets.getString("js"); if (playerUrl.startsWith("//")) { playerUrl = "https:" + playerUrl; } decryptionCode = loadDecryptionCode(playerUrl); - } catch (Exception e){ - Log.e(TAG, "Could not load decryption code for the Youtube service."); - e.printStackTrace(); + } catch (JSONException e) { + throw new ParsingException( + "Could not load decryption code for the Youtube service.", e); } } } @Override - public String getTitle() { + public String getTitle() throws ParsingException { try {//json player args method return playerArgs.getString("title"); } catch(JSONException je) {//html method je.printStackTrace(); Log.w(TAG, "failed to load title from JSON args; trying to extract it from HTML"); - } try { // fall through to fall-back - return doc.select("meta[name=title]").attr("content"); - } catch (Exception e) { - Log.e(TAG, "failed permanently to load title."); - e.printStackTrace(); - return ""; + try { // fall through to fall-back + return doc.select("meta[name=title]").attr("content"); + } catch (Exception e) { + throw new ParsingException("failed permanently to load title.", e); + } } } @Override - public String getDescription() { + public String getDescription() throws ParsingException { try { return doc.select("p[id=\"eow-description\"]").first().html(); - } catch (Exception e) {//todo: add fallback method - Log.e(TAG, "failed to load description."); - e.printStackTrace(); - return ""; + } catch (Exception e) {//todo: add fallback method <-- there is no ... as long as i know + throw new ParsingException("failed to load description.", e); } } @Override - public String getUploader() { + public String getUploader() throws ParsingException { try {//json player args method return playerArgs.getString("author"); } catch(JSONException je) { je.printStackTrace(); - Log.w(TAG, "failed to load uploader name from JSON args; trying to extract it from HTML"); + Log.w(TAG, + "failed to load uploader name from JSON args; trying to extract it from HTML"); } try {//fall through to fallback HTML method return doc.select("div.yt-user-info").first().text(); } catch (Exception e) { - e.printStackTrace(); - Log.e(TAG, "failed permanently to load uploader name."); - return ""; + throw new ParsingException("failed permanently to load uploader name.", e); } } @Override - public int getLength() { + public int getLength() throws ParsingException { try { return playerArgs.getInt("length_seconds"); - } catch (JSONException je) {//todo: find fallback method - Log.e(TAG, "failed to load video duration from JSON args"); - je.printStackTrace(); - return -1; + } catch (JSONException e) {//todo: find fallback method + throw new ParsingException("failed to load video duration from JSON args", e); } } @Override - public long getViews() { + public long getViews() throws ParsingException { try { String viewCountString = doc.select("meta[itemprop=interactionCount]").attr("content"); return Long.parseLong(viewCountString); } catch (Exception e) {//todo: find fallback method - Log.e(TAG, "failed to number of views"); - e.printStackTrace(); - return -1; + throw new ParsingException("failed to number of views", e); } } @Override - public String getUploadDate() { + public String getUploadDate() throws ParsingException { try { return doc.select("meta[itemprop=datePublished]").attr("content"); } catch (Exception e) {//todo: add fallback method - Log.e(TAG, "failed to get upload date."); - e.printStackTrace(); - return ""; + throw new ParsingException("failed to get upload date.", e); } } @Override - public String getThumbnailUrl() { + public String getThumbnailUrl() throws ParsingException { //first attempt getting a small image version //in the html extracting part we try to get a thumbnail with a higher resolution // Try to get high resolution thumbnail if it fails use low res from the player instead @@ -197,148 +210,98 @@ public class YoutubeVideoExtractor extends VideoExtractor { return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href"); } catch(Exception e) { Log.w(TAG, "Could not find high res Thumbnail. Using low res instead"); - //fall through to fallback - } try { + } try { //fall through to fallback return playerArgs.getString("thumbnail_url"); } catch (JSONException je) { - je.printStackTrace(); - Log.w(TAG, "failed to extract thumbnail URL from JSON args; trying to extract it from HTML"); - return ""; + throw new ParsingException( + "failed to extract thumbnail URL from JSON args; trying to extract it from HTML", je); } } @Override - public String getUploaderThumbnailUrl() { + public String getUploaderThumbnailUrl() throws ParsingException { try { return doc.select("a[class*=\"yt-user-photo\"]").first() .select("img").first() .attr("abs:data-thumb"); } catch (Exception e) {//todo: add fallback method - Log.e(TAG, "failed to get uploader thumbnail URL."); - e.printStackTrace(); - return ""; + throw new ParsingException("failed to get uploader thumbnail URL.", e); } } @Override - public String getDashMpdUrl() { + public String getDashMpdUrl() throws ParsingException { try { return playerArgs.getString("dashmpd"); } catch(NullPointerException e) { - Log.e(TAG, "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available)."); + throw new ParsingException( + "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", e); } catch (Exception e) { - e.printStackTrace(); + throw new ParsingException(e); } - return ""; } @Override - public VideoInfo.AudioStream[] getAudioStreams() { + public VideoInfo.AudioStream[] getAudioStreams() throws ParsingException { try { String dashManifest = playerArgs.getString("dashmpd"); - Log.d(TAG, dashManifest); return parseDashManifest(dashManifest, decryptionCode); - } catch (NullPointerException e) { - Log.e(TAG, "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available)."); + throw new ParsingException( + "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", e); } catch (Exception e) { - e.printStackTrace(); + throw new ParsingException(e); } - return new VideoInfo.AudioStream[0]; } @Override - public VideoInfo.VideoStream[] getVideoStreams() { + public VideoInfo.VideoStream[] getVideoStreams() throws ParsingException { + Vector videoStreams = new Vector<>(); try{ - //------------------------------------ - // extract video stream url - //------------------------------------ - Vector videoStreams = new Vector<>(); - - // The following line belongs to dash audio stuff. - // We can't use dash audio, unless we have exoplayer as our main player. - //String adaptive_fmts = playerArgs.getString("adaptive_fmts"); - String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map"); for(String url_data_str : encoded_url_map.split(",")) { - Map tags = new HashMap<>(); - for(String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) { - String[] split_tag = raw_tag.split("="); - tags.put(split_tag[0], split_tag[1]); - } + try { + Map tags = new HashMap<>(); + for (String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) { + String[] split_tag = raw_tag.split("="); + tags.put(split_tag[0], split_tag[1]); + } - int itag = Integer.parseInt(tags.get("itag")); - String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8"); + int itag = Integer.parseInt(tags.get("itag")); + String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8"); - // if video has a signature: decrypt it and add it to the url - if(tags.get("s") != null) { - streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode); - } + // if video has a signature: decrypt it and add it to the url + if (tags.get("s") != null) { + streamUrl = streamUrl + "&signature=" + + decryptSignature(tags.get("s"), decryptionCode); + } - if(resolveFormat(itag) != -1) { - videoStreams.add(new VideoInfo.VideoStream( - streamUrl, - resolveFormat(itag), - resolveResolutionString(itag))); + if (resolveFormat(itag) != -1) { + videoStreams.add(new VideoInfo.VideoStream( + streamUrl, + resolveFormat(itag), + resolveResolutionString(itag))); + } + } catch (Exception e) { + Log.w(TAG, "Could not get Video stream."); + e.printStackTrace(); } } - return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]); - } catch (Exception e) { - Log.e(TAG, "Failed to get video stream"); - e.printStackTrace(); - return new VideoInfo.VideoStream[0]; + throw new ParsingException("Failed to get video streams", e); } - } - /**These lists only contain itag formats that are supported by the common Android Video player. - However if you are looking for a list showing all itag formats, look at - https://github.com/rg3/youtube-dl/issues/1687 */ - - @SuppressWarnings("WeakerAccess") - public static int resolveFormat(int itag) { - switch(itag) { - // !!! lists only supported formats !!! - // video - case 17: return MediaFormat.v3GPP.id; - case 18: return MediaFormat.MPEG_4.id; - case 22: return MediaFormat.MPEG_4.id; - case 36: return MediaFormat.v3GPP.id; - case 37: return MediaFormat.MPEG_4.id; - case 38: return MediaFormat.MPEG_4.id; - case 43: return MediaFormat.WEBM.id; - case 44: return MediaFormat.WEBM.id; - case 45: return MediaFormat.WEBM.id; - case 46: return MediaFormat.WEBM.id; - default: - //Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported."); - return -1; + if(videoStreams.isEmpty()) { + throw new ParsingException("Failed to get any video stream"); } - } - @SuppressWarnings("WeakerAccess") - public static String resolveResolutionString(int itag) { - switch(itag) { - case 17: return "144p"; - case 18: return "360p"; - case 22: return "720p"; - case 36: return "240p"; - case 37: return "1080p"; - case 38: return "1080p"; - case 43: return "360p"; - case 44: return "480p"; - case 45: return "720p"; - case 46: return "1080p"; - default: - //Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported."); - return null; - } + return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]); } @SuppressWarnings("WeakerAccess") @Override - public String getVideoId(String url) { + public String getVideoId(String url) throws ParsingException { String id; String pat; @@ -349,16 +312,16 @@ public class YoutubeVideoExtractor extends VideoExtractor { pat = "youtu\\.be/([a-zA-Z0-9_-]{11})"; } else { - Log.e(TAG, "Error could not parse url: " + url); - return ""; + throw new ParsingException("Error no suitable url: " + url); } + id = matchGroup1(pat, url); if(!id.isEmpty()){ //Log.i(TAG, "string \""+url+"\" matches!"); return id; + } else { + throw new ParsingException("Error could not parse url: " + url); } - //Log.i(TAG, "string \""+url+"\" does not match."); - return ""; } @SuppressWarnings("WeakerAccess") @@ -370,116 +333,136 @@ public class YoutubeVideoExtractor extends VideoExtractor { /**Attempts to parse (and return) the offset to start playing the video from. * @return the offset (in seconds), or 0 if no timestamp is found.*/ @Override - public int getTimeStamp(){ - String timeStamp = matchGroup1("((#|&)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); + public int getTimeStamp() throws ParsingException { + //todo: add unit test for timestamp + String timeStamp; + try { + timeStamp = matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); + } catch (RegexException e) { + // catch this instantly since an url does not necessarily have to have a time stamp + + // -2 because well the testing system will then know its the regex that failed :/ + // not good i know + return -2; + } //TODO: test this if(!timeStamp.isEmpty()) { - String secondsString = matchGroup1("(\\d{1,3})s", timeStamp); - String minutesString = matchGroup1("(\\d{1,3})m", timeStamp); - String hoursString = matchGroup1("(\\d{1,3})h", timeStamp); + try { + String secondsString = ""; + String minutesString = ""; + String hoursString = ""; + try { + secondsString = matchGroup1("(\\d{1,3})s", timeStamp); + minutesString = matchGroup1("(\\d{1,3})m", timeStamp); + hoursString = matchGroup1("(\\d{1,3})h", timeStamp); + } catch (Exception e) { + //it could be that time is given in another method + if (secondsString.isEmpty() //if nothing was got, + && minutesString.isEmpty()//treat as unlabelled seconds + && hoursString.isEmpty()) { + secondsString = matchGroup1("t=(\\d{1,3})", timeStamp); + } + } - if(secondsString.isEmpty()//if nothing was got, - && minutesString.isEmpty()//treat as unlabelled seconds - && hoursString.isEmpty()) - secondsString = matchGroup1("t=(\\d{1,3})", timeStamp); + int seconds = (secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString)); + int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString)); + int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString)); - int seconds = (secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString)); - int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString)); - int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString)); + int ret = seconds + (60 * minutes) + (3600 * hours);//don't trust BODMAS! + //Log.d(TAG, "derived timestamp value:"+ret); + return ret; + //the ordering varies internationally + } catch (ParsingException e) { + throw new ParsingException("Could not get timestamp.", e); + } + } else { + return -1; + } + } - int ret = seconds + (60*minutes) + (3600*hours);//don't trust BODMAS! - //Log.d(TAG, "derived timestamp value:"+ret); - return ret; - //the ordering varies internationally - }//else, return default 0 + @Override + public int getAgeLimit() throws ParsingException { + // Not yet implemented. + // Also you need to be logged in to see age restricted videos on youtube, + // therefore NP is not able to receive such videos. return 0; } + @Override - public VideoInfo getVideoInfo() { - //todo: @medovax i like your work, but what the fuck: + public VideoInfo getVideoInfo() throws CrawlingException { videoInfo = super.getVideoInfo(); - if(errorCode == VideoInfo.NO_ERROR) { - //todo: replace this with a call to getVideoId, if possible - videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl); + //todo: replace this with a call to getVideoId, if possible + //videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl); + videoInfo.id = getVideoId(pageUrl); - if (videoInfo.audioStreams == null - || videoInfo.audioStreams.length == 0) { - Log.e(TAG, "uninitialised audio streams!"); - } - - if (videoInfo.videoStreams == null - || videoInfo.videoStreams.length == 0) { - Log.e(TAG, "uninitialised video streams!"); - } - - videoInfo.age_limit = 0; - - //average rating - try { - videoInfo.average_rating = playerArgs.getString("avg_rating"); - } catch (JSONException e) { - e.printStackTrace(); - } - - //--------------------------------------- - // extracting information from html page - //--------------------------------------- - - String likesString = ""; - String dislikesString = ""; - try { - // likes - likesString = doc.select("button.like-button-renderer-like-button").first() - .select("span.yt-uix-button-content").first().text(); - videoInfo.like_count = Integer.parseInt(likesString.replaceAll("[^\\d]", "")); - // dislikes - dislikesString = doc.select("button.like-button-renderer-dislike-button").first() - .select("span.yt-uix-button-content").first().text(); - - videoInfo.dislike_count = Integer.parseInt(dislikesString.replaceAll("[^\\d]", "")); - } catch (NumberFormatException nfe) { - Log.e(TAG, "failed to parse likesString \"" + likesString + "\" and dislikesString \"" + - dislikesString + "\" as integers"); - } catch (Exception e) { - // if it fails we know that the video does not offer dislikes. - e.printStackTrace(); - videoInfo.like_count = 0; - videoInfo.dislike_count = 0; - } - - // next video - videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() - .select("li").first()); - - // related videos - Vector relatedVideos = new Vector<>(); - for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) { - // first check if we have a playlist. If so leave them out - if (li.select("a[class*=\"content-link\"]").first() != null) { - relatedVideos.add(extractVideoPreviewInfo(li)); - } - } - //todo: replace conversion - videoInfo.relatedVideos = relatedVideos; - //videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]); + if (videoInfo.audioStreams == null + || videoInfo.audioStreams.length == 0) { + Log.e(TAG, "uninitialised audio streams!"); } + + if (videoInfo.videoStreams == null + || videoInfo.videoStreams.length == 0) { + Log.e(TAG, "uninitialised video streams!"); + } + + videoInfo.age_limit = 0; + + //average rating + try { + videoInfo.average_rating = playerArgs.getString("avg_rating"); + } catch (JSONException e) { + e.printStackTrace(); + } + + //--------------------------------------- + // extracting information from html page + //--------------------------------------- + + String likesString = ""; + String dislikesString = ""; + try { + // likes + likesString = doc.select("button.like-button-renderer-like-button").first() + .select("span.yt-uix-button-content").first().text(); + videoInfo.like_count = Integer.parseInt(likesString.replaceAll("[^\\d]", "")); + // dislikes + dislikesString = doc.select("button.like-button-renderer-dislike-button").first() + .select("span.yt-uix-button-content").first().text(); + + videoInfo.dislike_count = Integer.parseInt(dislikesString.replaceAll("[^\\d]", "")); + } catch (NumberFormatException nfe) { + Log.e(TAG, "failed to parse likesString \"" + likesString + "\" and dislikesString \"" + + dislikesString + "\" as integers"); + } catch (Exception e) { + // if it fails we know that the video does not offer dislikes. + e.printStackTrace(); + videoInfo.like_count = 0; + videoInfo.dislike_count = 0; + } + + // next video + videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() + .select("li").first()); + + // related videos + Vector relatedVideos = new Vector<>(); + for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) { + // first check if we have a playlist. If so leave them out + if (li.select("a[class*=\"content-link\"]").first() != null) { + relatedVideos.add(extractVideoPreviewInfo(li)); + } + } + //todo: replace conversion + videoInfo.relatedVideos = relatedVideos; + //videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]); + return videoInfo; } - @Override - public int getErrorCode() { - return errorCode; - } - - @Override - public String getErrorMessage() { - return errorMessage; - } - - private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) { + private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) throws RegexException, DecryptException { if(!dashManifest.contains("/signature/")) { String encryptedSig = matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest); String decryptedSig; @@ -487,7 +470,12 @@ public class YoutubeVideoExtractor extends VideoExtractor { decryptedSig = decryptSignature(encryptedSig, decryptoinCode); dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig); } - String dashDoc = downloader.download(dashManifest); + String dashDoc; + try { + dashDoc = downloader.download(dashManifest); + } catch(IOException ioe) { + throw new DecryptException("Could not get dash mpd", ioe); + } Vector audioStreams = new Vector<>(); try { XmlPullParser parser = Xml.newPullParser(); @@ -545,63 +533,67 @@ public class YoutubeVideoExtractor extends VideoExtractor { /**Provides information about links to other videos on the video page, such as related videos. * This is encapsulated in a VideoPreviewInfo object, * which is a subset of the fields in a full VideoInfo.*/ - private VideoPreviewInfo extractVideoPreviewInfo(Element li) { + private VideoPreviewInfo extractVideoPreviewInfo(Element li) throws ParsingException { VideoPreviewInfo info = new VideoPreviewInfo(); - info.webpage_url = li.select("a.content-link").first() - .attr("abs:href"); + try { + info.webpage_url = li.select("a.content-link").first() + .attr("abs:href"); + info.id = matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url); + + //todo: check NullPointerException causing + info.title = li.select("span.title").first().text(); + //this page causes the NullPointerException, after finding it by searching for "tjvg": + //https://www.youtube.com/watch?v=Uqg0aEhLFAg + + //this line is unused + //String views = li.select("span.view-count").first().text(); + + //Log.i(TAG, "title:"+info.title); + //Log.i(TAG, "view count:"+views); + + try { + info.view_count = Long.parseLong(li.select("span.view-count") + .first().text().replaceAll("[^\\d]", "")); + } catch (NullPointerException e) {//related videos sometimes have no view count + info.view_count = 0; + } + info.uploader = li.select("span.g-hovercard").first().text(); + + info.duration = li.select("span.video-time").first().text(); + + Element img = li.select("img").first(); + info.thumbnail_url = img.attr("abs:src"); + // Sometimes youtube sends links to gif files which somehow seem to not exist + // anymore. Items with such gif also offer a secondary image source. So we are going + // to use that if we caught such an item. + if (info.thumbnail_url.contains(".gif")) { + info.thumbnail_url = img.attr("data-thumb"); + } + if (info.thumbnail_url.startsWith("//")) { + info.thumbnail_url = "https:" + info.thumbnail_url; + } } catch (Exception e) { - e.printStackTrace(); - } - - //todo: check NullPointerException causing - info.title = li.select("span.title").first().text(); - //this page causes the NullPointerException, after finding it by searching for "tjvg": - //https://www.youtube.com/watch?v=Uqg0aEhLFAg - - //this line is unused - //String views = li.select("span.view-count").first().text(); - - //Log.i(TAG, "title:"+info.title); - //Log.i(TAG, "view count:"+views); - try { - info.view_count = Long.parseLong(li.select("span.view-count") - .first().text().replaceAll("[^\\d]", "")); - } catch (NullPointerException e) {//related videos sometimes have no view count - info.view_count = 0; - } - info.uploader = li.select("span.g-hovercard").first().text(); - - info.duration = li.select("span.video-time").first().text(); - - Element img = li.select("img").first(); - info.thumbnail_url = img.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we caught such an item. - if(info.thumbnail_url.contains(".gif")) { - info.thumbnail_url = img.attr("data-thumb"); - } - if(info.thumbnail_url.startsWith("//")) { - info.thumbnail_url = "https:" + info.thumbnail_url; + throw new ParsingException(e); } return info; } - private String loadDecryptionCode(String playerUrl) { - String playerCode = downloader.download(playerUrl); - String decryptionFuncName = ""; - String decryptionFunc = ""; + private String loadDecryptionCode(String playerUrl) throws DecryptException { + String decryptionFuncName; + String decryptionFunc; String helperObjectName; - String helperObject = ""; + String helperObject; String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}"; String decryptionCode; try { + String playerCode = downloader.download(playerUrl); + decryptionFuncName = matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode); - String functionPattern = "(" + decryptionFuncName.replace("$", "\\$") +"=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})"; + String functionPattern = "(" + decryptionFuncName.replace("$", "\\$") + "=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})"; decryptionFunc = "var " + matchGroup1(functionPattern, playerCode) + ";"; helperObjectName = matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc); @@ -609,17 +601,20 @@ public class YoutubeVideoExtractor extends VideoExtractor { String helperPattern = "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; helperObject = matchGroup1(helperPattern, playerCode); - } catch (Exception e) { - e.printStackTrace(); - } - callerFunc = callerFunc.replace("%%", decryptionFuncName); - decryptionCode = helperObject + decryptionFunc + callerFunc; + callerFunc = callerFunc.replace("%%", decryptionFuncName); + decryptionCode = helperObject + decryptionFunc + callerFunc; + } catch(IOException ioe) { + throw new DecryptException("Could not load decrypt function", ioe); + } catch(Exception e) { + throw new DecryptException("Could not parse decrypt function ", e); + } return decryptionCode; } - private String decryptSignature(String encryptedSig, String decryptionCode) { + private String decryptSignature(String encryptedSig, String decryptionCode) + throws DecryptException{ Context context = Context.enter(); context.setOptimizationLevel(-1); Object result = null; @@ -629,17 +624,18 @@ public class YoutubeVideoExtractor extends VideoExtractor { Function decryptionFunc = (Function) scope.get("decrypt", scope); result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig}); } catch (Exception e) { - e.printStackTrace(); + throw new DecryptException(e); + } finally { + Context.exit(); } - Context.exit(); return (result == null ? "" : result.toString()); } - private String cleanUrl(String complexUrl) { + private String cleanUrl(String complexUrl) throws ParsingException { return getVideoUrl(getVideoId(complexUrl)); } - private String matchGroup1(String pattern, String input) { + private String matchGroup1(String pattern, String input) throws RegexException { Pattern pat = Pattern.compile(pattern); Matcher mat = pat.matcher(input); boolean foundMatch = mat.find(); @@ -647,17 +643,64 @@ public class YoutubeVideoExtractor extends VideoExtractor { return mat.group(1); } else { - Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\""); - new Exception("failed to find pattern \""+pattern+"\"").printStackTrace(); - return ""; + //Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\""); + throw new RegexException("failed to find pattern \""+pattern+" inside of "+input+"\""); } } - private int findErrorReason(Document doc) { + private String findErrorReason(Document doc) { errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text(); if(errorMessage.contains("GEMA")) { - return VideoInfo.ERROR_BLOCKED_BY_GEMA; + // Gema sometimes blocks youtube music content in germany: + // https://www.gema.de/en/ + // Detailed description: + // https://en.wikipedia.org/wiki/GEMA_%28German_organization%29 + return "GEMA"; + } + return ""; + } + + /**These lists only contain itag formats that are supported by the common Android Video player. + However if you are looking for a list showing all itag formats, look at + https://github.com/rg3/youtube-dl/issues/1687 */ + + @SuppressWarnings("WeakerAccess") + public static int resolveFormat(int itag) { + switch(itag) { + // !!! lists only supported formats !!! + // video + case 17: return MediaFormat.v3GPP.id; + case 18: return MediaFormat.MPEG_4.id; + case 22: return MediaFormat.MPEG_4.id; + case 36: return MediaFormat.v3GPP.id; + case 37: return MediaFormat.MPEG_4.id; + case 38: return MediaFormat.MPEG_4.id; + case 43: return MediaFormat.WEBM.id; + case 44: return MediaFormat.WEBM.id; + case 45: return MediaFormat.WEBM.id; + case 46: return MediaFormat.WEBM.id; + default: + //Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported."); + return -1; + } + } + + @SuppressWarnings("WeakerAccess") + public static String resolveResolutionString(int itag) { + switch(itag) { + case 17: return "144p"; + case 18: return "360p"; + case 22: return "720p"; + case 36: return "240p"; + case 37: return "1080p"; + case 38: return "1080p"; + case 43: return "360p"; + case 44: return "480p"; + case 45: return "720p"; + case 46: return "1080p"; + default: + //Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported."); + return null; } - return VideoInfo.ERROR_NO_SPECIFIED_ERROR; } } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 01f742cad..91afd974c 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -65,7 +65,13 @@ Playing in background https://www.c3s.cc/ Play + Error Network error + Could not load Thumbnails + Could not decrypt video url signature. + Could not parse website. + Content not available. + Blocked by GEMA. Video preview thumbnail From fb942912db9f9e26df629d1573731f1a99263d60 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 31 Jan 2016 20:16:21 +0100 Subject: [PATCH 06/11] disable gema test agian --- .../newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java index a17b614b7..8125ef5e3 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java @@ -35,7 +35,7 @@ import java.io.IOException; public class YoutubeVideoExtractorGemaTest extends AndroidTestCase { // Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail. - private static final boolean testActive = true; + private static final boolean testActive = false; public void testGemaError() throws IOException, CrawlingException { if(testActive) { From bad576c23d7659cb975a65635cd6373010988755 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 2 Feb 2016 14:06:09 +0100 Subject: [PATCH 07/11] got rid of getVideoInfo() in youtube crawler --- .../newpipe/VideoItemDetailFragment.java | 1 + .../schabi/newpipe/VideoItemListFragment.java | 1 + .../newpipe/crawler/VideoExtractor.java | 27 +++++ .../youtube/YoutubeVideoExtractor.java | 109 ++++++++---------- 4 files changed, 80 insertions(+), 58 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index 076a12137..ff3b94933 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -115,6 +115,7 @@ public class VideoItemDetailFragment extends Fragment { VideoInfo videoInfo = videoExtractor.getVideoInfo(); h.post(new VideoResultReturnedRunnable(videoInfo)); h.post(new SetThumbnailRunnable( + //todo: make bitmaps not bypass tor BitmapFactory.decodeStream( new URL(videoInfo.thumbnail_url) .openConnection() diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java index 21781f78c..479a4820d 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java @@ -157,6 +157,7 @@ public class VideoItemListFragment extends ListFragment { if(!downloadedList.get(i)) { Bitmap thumbnail; try { + //todo: make bitmaps not bypass tor thumbnail = BitmapFactory.decodeStream( new URL(thumbnailUrlList.get(i)).openConnection().getInputStream()); h.post(new SetThumbnailRunnable(i, thumbnail, requestId)); diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java index e09b2e01f..ac165c19e 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java @@ -20,6 +20,9 @@ package org.schabi.newpipe.crawler; * along with NewPipe. If not, see . */ +import java.util.List; +import java.util.Vector; + /**Scrapes information from a video streaming service (eg, YouTube).*/ @@ -134,6 +137,25 @@ public abstract class VideoExtractor { videoInfo.dashMpdUrl = getDashMpdUrl(); } + if(videoInfo.average_rating.isEmpty()) { + videoInfo.average_rating = getAverageRating(); + } + + if(videoInfo.like_count == -1) { + videoInfo.like_count = getLikeCount(); + } + + if(videoInfo.dislike_count == -1) { + videoInfo.dislike_count = getDislikeCount(); + } + + if(videoInfo.nextVideo == null) { + videoInfo.nextVideo = getNextVideo(); + } + + if(videoInfo.relatedVideos == null) { + videoInfo.relatedVideos = getRelatedVideos(); + } //Bitmap thumbnail = null; //Bitmap uploader_thumbnail = null; @@ -158,4 +180,9 @@ public abstract class VideoExtractor { public abstract VideoInfo.VideoStream[] getVideoStreams() throws ParsingException; public abstract String getDashMpdUrl() throws ParsingException; public abstract int getAgeLimit() throws ParsingException; + public abstract String getAverageRating() throws ParsingException; + public abstract int getLikeCount() throws ParsingException; + public abstract int getDislikeCount() throws ParsingException; + public abstract VideoPreviewInfo getNextVideo() throws ParsingException; + public abstract Vector getRelatedVideos() throws ParsingException; } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java index 83ebb8ca0..ced24cc95 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.StringReader; import java.net.URLDecoder; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Vector; import java.util.regex.Matcher; @@ -389,77 +390,69 @@ public class YoutubeVideoExtractor extends VideoExtractor { return 0; } + @Override + public String getAverageRating() throws ParsingException { + try { + return playerArgs.getString("avg_rating"); + } catch (JSONException e) { + throw new ParsingException("Could not get Average rating", e); + } + } @Override - public VideoInfo getVideoInfo() throws CrawlingException { - videoInfo = super.getVideoInfo(); - - //todo: replace this with a call to getVideoId, if possible - //videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl); - videoInfo.id = getVideoId(pageUrl); - - if (videoInfo.audioStreams == null - || videoInfo.audioStreams.length == 0) { - Log.e(TAG, "uninitialised audio streams!"); - } - - if (videoInfo.videoStreams == null - || videoInfo.videoStreams.length == 0) { - Log.e(TAG, "uninitialised video streams!"); - } - - videoInfo.age_limit = 0; - - //average rating - try { - videoInfo.average_rating = playerArgs.getString("avg_rating"); - } catch (JSONException e) { - e.printStackTrace(); - } - - //--------------------------------------- - // extracting information from html page - //--------------------------------------- - + public int getLikeCount() throws ParsingException { String likesString = ""; - String dislikesString = ""; try { - // likes likesString = doc.select("button.like-button-renderer-like-button").first() .select("span.yt-uix-button-content").first().text(); - videoInfo.like_count = Integer.parseInt(likesString.replaceAll("[^\\d]", "")); - // dislikes + return Integer.parseInt(likesString.replaceAll("[^\\d]", "")); + } catch (NumberFormatException nfe) { + throw new ParsingException( + "failed to parse likesString \"" + likesString + "\" as integers", nfe); + } catch (Exception e) { + throw new ParsingException("Could not get like count", e); + } + } + + @Override + public int getDislikeCount() throws ParsingException { + String dislikesString = ""; + try { dislikesString = doc.select("button.like-button-renderer-dislike-button").first() .select("span.yt-uix-button-content").first().text(); - - videoInfo.dislike_count = Integer.parseInt(dislikesString.replaceAll("[^\\d]", "")); - } catch (NumberFormatException nfe) { - Log.e(TAG, "failed to parse likesString \"" + likesString + "\" and dislikesString \"" + - dislikesString + "\" as integers"); - } catch (Exception e) { - // if it fails we know that the video does not offer dislikes. - e.printStackTrace(); - videoInfo.like_count = 0; - videoInfo.dislike_count = 0; + return Integer.parseInt(dislikesString.replaceAll("[^\\d]", "")); + } catch(NumberFormatException nfe) { + throw new ParsingException( + "failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe); + } catch(Exception e) { + throw new ParsingException("Could not get dislike count", e); } + } - // next video - videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() - .select("li").first()); + @Override + public VideoPreviewInfo getNextVideo() throws ParsingException { + try { + return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first() + .select("li").first()); + } catch(Exception e) { + throw new ParsingException("Could not get next video", e); + } + } - // related videos - Vector relatedVideos = new Vector<>(); - for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) { - // first check if we have a playlist. If so leave them out - if (li.select("a[class*=\"content-link\"]").first() != null) { - relatedVideos.add(extractVideoPreviewInfo(li)); + @Override + public Vector getRelatedVideos() throws ParsingException { + try { + Vector relatedVideos = new Vector<>(); + for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) { + // first check if we have a playlist. If so leave them out + if (li.select("a[class*=\"content-link\"]").first() != null) { + relatedVideos.add(extractVideoPreviewInfo(li)); + } } + return relatedVideos; + } catch(Exception e) { + throw new ParsingException("Could not get related videos", e); } - //todo: replace conversion - videoInfo.relatedVideos = relatedVideos; - //videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]); - - return videoInfo; } private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) throws RegexException, DecryptException { From d097363b24dd9f82421ebe5677fad1c07d1b8cd1 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 2 Feb 2016 18:43:20 +0100 Subject: [PATCH 08/11] restructure parser --- .../newpipe/VideoItemDetailActivity.java | 2 +- .../newpipe/VideoItemDetailFragment.java | 2 +- .../schabi/newpipe/crawler/DashMpdParser.java | 101 +++++++++ .../schabi/newpipe/crawler/RegexHelper.java | 47 +++++ .../newpipe/crawler/StreamingService.java | 10 +- .../schabi/newpipe/crawler/UrlIdHandler.java | 32 +++ .../newpipe/crawler/VideoExtractor.java | 118 +---------- .../org/schabi/newpipe/crawler/VideoInfo.java | 45 ++++ .../services/youtube/YoutubeSearchEngine.java | 6 +- .../services/youtube/YoutubeService.java | 13 +- .../services/youtube/YoutubeUrlIdHandler.java | 68 ++++++ .../youtube/YoutubeVideoExtractor.java | 195 +++++------------- 12 files changed, 365 insertions(+), 274 deletions(-) create mode 100644 app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java create mode 100644 app/src/main/java/org/schabi/newpipe/crawler/RegexHelper.java create mode 100644 app/src/main/java/org/schabi/newpipe/crawler/UrlIdHandler.java create mode 100644 app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeUrlIdHandler.java diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java index 8514160d4..b61095431 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java @@ -72,7 +72,7 @@ public class VideoItemDetailActivity extends AppCompatActivity { StreamingService[] serviceList = ServiceList.getServices(); //VideoExtractor videoExtractor = null; for (int i = 0; i < serviceList.length; i++) { - if (serviceList[i].acceptUrl(videoUrl)) { + if (serviceList[i].getUrlIdHandler().acceptUrl(videoUrl)) { arguments.putInt(VideoItemDetailFragment.STREAMING_SERVICE, i); currentStreamingService = i; //videoExtractor = ServiceList.getService(i).getExtractorInstance(); diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index ff3b94933..f45cff230 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -112,7 +112,7 @@ public class VideoItemDetailFragment extends Fragment { public void run() { try { videoExtractor = service.getExtractorInstance(videoUrl, new Downloader()); - VideoInfo videoInfo = videoExtractor.getVideoInfo(); + VideoInfo videoInfo = VideoInfo.getVideoInfo(videoExtractor, new Downloader()); h.post(new VideoResultReturnedRunnable(videoInfo)); h.post(new SetThumbnailRunnable( //todo: make bitmaps not bypass tor diff --git a/app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java b/app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java new file mode 100644 index 000000000..b893599dd --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java @@ -0,0 +1,101 @@ +package org.schabi.newpipe.crawler; + +import android.util.Xml; + +import org.xmlpull.v1.XmlPullParser; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Vector; + +/** + * Created by Christian Schabesberger on 02.02.16. + * + * Copyright (C) Christian Schabesberger 2016 + * DashMpdParser.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class DashMpdParser { + + static class DashMpdParsingException extends ParsingException { + DashMpdParsingException(String message, Exception e) { + super(message, e); + } + } + + public static VideoInfo.AudioStream[] getAudioStreams(String dashManifestUrl, + Downloader downloader) + throws DashMpdParsingException { + String dashDoc; + try { + dashDoc = downloader.download(dashManifestUrl); + } catch(IOException ioe) { + throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe); + } + Vector audioStreams = new Vector<>(); + try { + XmlPullParser parser = Xml.newPullParser(); + parser.setInput(new StringReader(dashDoc)); + String tagName = ""; + String currentMimeType = ""; + int currentBandwidth = -1; + int currentSamplingRate = -1; + boolean currentTagIsBaseUrl = false; + for(int eventType = parser.getEventType(); + eventType != XmlPullParser.END_DOCUMENT; + eventType = parser.next() ) { + switch(eventType) { + case XmlPullParser.START_TAG: + tagName = parser.getName(); + if(tagName.equals("AdaptationSet")) { + currentMimeType = parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "mimeType"); + } else if(tagName.equals("Representation") && currentMimeType.contains("audio")) { + currentBandwidth = Integer.parseInt( + parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "bandwidth")); + currentSamplingRate = Integer.parseInt( + parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "audioSamplingRate")); + } else if(tagName.equals("BaseURL")) { + currentTagIsBaseUrl = true; + } + break; + + case XmlPullParser.TEXT: + if(currentTagIsBaseUrl && + (currentMimeType.contains("audio"))) { + int format = -1; + if(currentMimeType.equals(MediaFormat.WEBMA.mimeType)) { + format = MediaFormat.WEBMA.id; + } else if(currentMimeType.equals(MediaFormat.M4A.mimeType)) { + format = MediaFormat.M4A.id; + } + audioStreams.add(new VideoInfo.AudioStream(parser.getText(), + format, currentBandwidth, currentSamplingRate)); + } + break; + case XmlPullParser.END_TAG: + if(tagName.equals("AdaptationSet")) { + currentMimeType = ""; + } else if(tagName.equals("BaseURL")) { + currentTagIsBaseUrl = false; + }//no break needed here + } + } + } catch(Exception e) { + throw new DashMpdParsingException("Could not parse Dash mpd", e); + } + return audioStreams.toArray(new VideoInfo.AudioStream[audioStreams.size()]); + } +} diff --git a/app/src/main/java/org/schabi/newpipe/crawler/RegexHelper.java b/app/src/main/java/org/schabi/newpipe/crawler/RegexHelper.java new file mode 100644 index 000000000..a82386182 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/RegexHelper.java @@ -0,0 +1,47 @@ +package org.schabi.newpipe.crawler; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Created by Christian Schabesberger on 02.02.16. + * + * Copyright (C) Christian Schabesberger 2016 + * RegexHelper.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +/** avoid using regex !!! */ +public class RegexHelper { + + public static class RegexException extends ParsingException { + public RegexException(String message) { + super(message); + } + } + + public static String matchGroup1(String pattern, String input) throws RegexException { + Pattern pat = Pattern.compile(pattern); + Matcher mat = pat.matcher(input); + boolean foundMatch = mat.find(); + if (foundMatch) { + return mat.group(1); + } + else { + //Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\""); + throw new RegexException("failed to find pattern \""+pattern+" inside of "+input+"\""); + } + } +} diff --git a/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java b/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java index 68c3da265..f34a40db8 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java @@ -27,11 +27,11 @@ public interface StreamingService { public String name = ""; } ServiceInfo getServiceInfo(); - VideoExtractor getExtractorInstance(String url, Downloader downloader) throws IOException, CrawlingException; + VideoExtractor getExtractorInstance(String url, Downloader downloader) + throws IOException, CrawlingException; SearchEngine getSearchEngineInstance(); - /**When a VIEW_ACTION is caught this function will test if the url delivered within the calling - Intent was meant to be watched with this Service. - Return false if this service shall not allow to be called through ACTIONs.*/ - boolean acceptUrl(String videoUrl); + UrlIdHandler getUrlIdHandler(); + + } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/UrlIdHandler.java b/app/src/main/java/org/schabi/newpipe/crawler/UrlIdHandler.java new file mode 100644 index 000000000..5ff0a2b7d --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/UrlIdHandler.java @@ -0,0 +1,32 @@ +package org.schabi.newpipe.crawler; + +/** + * Created by Christian Schabesberger on 02.02.16. + * + * Copyright (C) Christian Schabesberger 2016 + * UrlIdHandler.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public interface UrlIdHandler { + String getVideoUrl(String videoId); + String getVideoId(String siteUrl) throws ParsingException; + String cleanUrl(String siteUrl) throws ParsingException; + + /**When a VIEW_ACTION is caught this function will test if the url delivered within the calling + Intent was meant to be watched with this Service. + Return false if this service shall not allow to be called through ACTIONs.*/ + boolean acceptUrl(String videoUrl); +} diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java index ac165c19e..41a63ebdb 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java @@ -20,14 +20,14 @@ package org.schabi.newpipe.crawler; * along with NewPipe. If not, see . */ -import java.util.List; +import java.net.URL; import java.util.Vector; /**Scrapes information from a video streaming service (eg, YouTube).*/ @SuppressWarnings("ALL") -public abstract class VideoExtractor { +public interface VideoExtractor { public class ExctractorInitException extends CrawlingException { public ExctractorInitException() {} @@ -42,13 +42,6 @@ public abstract class VideoExtractor { } } - public class RegexException extends ParsingException { - public RegexException() {} - public RegexException(String message) { - super(message); - } - } - public class ContentNotAvailableException extends ParsingException { public ContentNotAvailableException() {} public ContentNotAvailableException(String message) { @@ -62,111 +55,6 @@ public abstract class VideoExtractor { } } - protected final String pageUrl; - protected VideoInfo videoInfo; - - @SuppressWarnings("WeakerAccess") - public VideoExtractor(String url, Downloader dl) { - this.pageUrl = url; - } - - /**Fills out the video info fields which are common to all services. - * Probably needs to be overridden by subclasses*/ - public VideoInfo getVideoInfo() throws CrawlingException - { - if(videoInfo == null) { - videoInfo = new VideoInfo(); - } - - if(videoInfo.webpage_url.isEmpty()) { - videoInfo.webpage_url = pageUrl; - } - - - if (videoInfo.title.isEmpty()) { - videoInfo.title = getTitle(); - } - - if (videoInfo.duration < 1) { - videoInfo.duration = getLength(); - } - - - if (videoInfo.uploader.isEmpty()) { - videoInfo.uploader = getUploader(); - } - - if (videoInfo.description.isEmpty()) { - videoInfo.description = getDescription(); - } - - if (videoInfo.view_count == -1) { - videoInfo.view_count = getViews(); - } - - if (videoInfo.upload_date.isEmpty()) { - videoInfo.upload_date = getUploadDate(); - } - - if (videoInfo.thumbnail_url.isEmpty()) { - videoInfo.thumbnail_url = getThumbnailUrl(); - } - - if (videoInfo.id.isEmpty()) { - videoInfo.id = getVideoId(pageUrl); - } - - /** Load and extract audio*/ - if (videoInfo.audioStreams == null) { - videoInfo.audioStreams = getAudioStreams(); - } - /** Extract video stream url*/ - if (videoInfo.videoStreams == null) { - videoInfo.videoStreams = getVideoStreams(); - } - - if (videoInfo.uploader_thumbnail_url.isEmpty()) { - videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl(); - } - - if (videoInfo.startPosition < 0) { - videoInfo.startPosition = getTimeStamp(); - } - - if(videoInfo.dashMpdUrl.isEmpty()) { - videoInfo.dashMpdUrl = getDashMpdUrl(); - } - - if(videoInfo.average_rating.isEmpty()) { - videoInfo.average_rating = getAverageRating(); - } - - if(videoInfo.like_count == -1) { - videoInfo.like_count = getLikeCount(); - } - - if(videoInfo.dislike_count == -1) { - videoInfo.dislike_count = getDislikeCount(); - } - - if(videoInfo.nextVideo == null) { - videoInfo.nextVideo = getNextVideo(); - } - - if(videoInfo.relatedVideos == null) { - videoInfo.relatedVideos = getRelatedVideos(); - } - - //Bitmap thumbnail = null; - //Bitmap uploader_thumbnail = null; - //int videoAvailableStatus = VIDEO_AVAILABLE; - return videoInfo; - } - - //todo: remove these functions, or make them static, otherwise its useles, to have them here - public abstract String getVideoUrl(String videoId); - public abstract String getVideoId(String siteUrl) throws ParsingException; - /////////////////////////////////////////////////////////////////////////////////////////// public abstract int getTimeStamp() throws ParsingException; public abstract String getTitle() throws ParsingException; public abstract String getDescription() throws ParsingException; @@ -185,4 +73,6 @@ public abstract class VideoExtractor { public abstract int getDislikeCount() throws ParsingException; public abstract VideoPreviewInfo getNextVideo() throws ParsingException; public abstract Vector getRelatedVideos() throws ParsingException; + public abstract UrlIdHandler getUrlIdConverter(); + public abstract String getPageUrl(); } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java index fbea6d0fc..71503b271 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.crawler; +import java.io.IOException; import java.util.List; /** @@ -26,8 +27,52 @@ import java.util.List; @SuppressWarnings("ALL") public class VideoInfo extends AbstractVideoInfo { + /**Fills out the video info fields which are common to all services. + * Probably needs to be overridden by subclasses*/ + public static VideoInfo getVideoInfo(VideoExtractor extractor, Downloader downloader) + throws CrawlingException, IOException { + VideoInfo videoInfo = new VideoInfo(); + + UrlIdHandler uiconv = extractor.getUrlIdConverter(); + + videoInfo.webpage_url = extractor.getPageUrl(); + videoInfo.title = extractor.getTitle(); + videoInfo.duration = extractor.getLength(); + videoInfo.uploader = extractor.getUploader(); + videoInfo.description = extractor.getDescription(); + videoInfo.view_count = extractor.getViews(); + videoInfo.upload_date = extractor.getUploadDate(); + videoInfo.thumbnail_url = extractor.getThumbnailUrl(); + videoInfo.id = uiconv.getVideoId(extractor.getPageUrl()); + videoInfo.dashMpdUrl = extractor.getDashMpdUrl(); + /** Load and extract audio*/ + videoInfo.audioStreams = extractor.getAudioStreams(); + if(videoInfo.dashMpdUrl != null && !videoInfo.dashMpdUrl.isEmpty()) { + if(videoInfo.audioStreams == null || videoInfo.audioStreams.length == 0) { + videoInfo.audioStreams = + DashMpdParser.getAudioStreams(videoInfo.dashMpdUrl, downloader); + } + } + /** Extract video stream url*/ + videoInfo.videoStreams = extractor.getVideoStreams(); + videoInfo.uploader_thumbnail_url = extractor.getUploaderThumbnailUrl(); + videoInfo.startPosition = extractor.getTimeStamp(); + videoInfo.average_rating = extractor.getAverageRating(); + videoInfo.like_count = extractor.getLikeCount(); + videoInfo.dislike_count = extractor.getDislikeCount(); + videoInfo.nextVideo = extractor.getNextVideo(); + videoInfo.relatedVideos = extractor.getRelatedVideos(); + + //Bitmap thumbnail = null; + //Bitmap uploader_thumbnail = null; + //int videoAvailableStatus = VIDEO_AVAILABLE; + return videoInfo; + } + + public String uploader_thumbnail_url = ""; public String description = ""; + /*todo: make this lists over vectors*/ public VideoStream[] videoStreams = null; public AudioStream[] audioStreams = null; // video streams provided by the dash mpd do not need to be provided as VideoStream. diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java index 9d7ce88ef..a6d4857c1 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java @@ -52,7 +52,8 @@ public class YoutubeSearchEngine implements SearchEngine { private static final String TAG = YoutubeSearchEngine.class.toString(); @Override - public Result search(String query, int page, String languageCode, Downloader downloader) throws IOException, ParsingException { + public Result search(String query, int page, String languageCode, Downloader downloader) + throws IOException, ParsingException { Result result = new Result(); Uri.Builder builder = new Uri.Builder(); builder.scheme("https") @@ -171,7 +172,8 @@ public class YoutubeSearchEngine implements SearchEngine { try { dBuilder = dbFactory.newDocumentBuilder(); - doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8")))); + doc = dBuilder.parse(new InputSource( + new ByteArrayInputStream(response.getBytes("utf-8")))); doc.getDocumentElement().normalize(); } catch (ParserConfigurationException | SAXException | IOException e) { e.printStackTrace(); diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java index 1a765ab66..40d7d41f7 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.crawler.services.youtube; import org.schabi.newpipe.crawler.CrawlingException; import org.schabi.newpipe.crawler.Downloader; import org.schabi.newpipe.crawler.StreamingService; +import org.schabi.newpipe.crawler.UrlIdHandler; import org.schabi.newpipe.crawler.VideoExtractor; import org.schabi.newpipe.crawler.SearchEngine; @@ -37,8 +38,10 @@ public class YoutubeService implements StreamingService { return serviceInfo; } @Override - public VideoExtractor getExtractorInstance(String url, Downloader downloader) throws CrawlingException, IOException { - if(acceptUrl(url)) { + public VideoExtractor getExtractorInstance(String url, Downloader downloader) + throws CrawlingException, IOException { + UrlIdHandler urlIdHandler = new YoutubeUrlIdHandler(); + if(urlIdHandler.acceptUrl(url)) { return new YoutubeVideoExtractor(url, downloader) ; } else { @@ -49,9 +52,9 @@ public class YoutubeService implements StreamingService { public SearchEngine getSearchEngineInstance() { return new YoutubeSearchEngine(); } + @Override - public boolean acceptUrl(String videoUrl) { - return videoUrl.contains("youtube") || - videoUrl.contains("youtu.be"); + public UrlIdHandler getUrlIdHandler() { + return new YoutubeUrlIdHandler(); } } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeUrlIdHandler.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeUrlIdHandler.java new file mode 100644 index 000000000..d0ccafafa --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeUrlIdHandler.java @@ -0,0 +1,68 @@ +package org.schabi.newpipe.crawler.services.youtube; + +import org.schabi.newpipe.crawler.ParsingException; +import org.schabi.newpipe.crawler.RegexHelper; +import org.schabi.newpipe.crawler.UrlIdHandler; + +/** + * Created by Christian Schabesberger on 02.02.16. + * + * Copyright (C) Christian Schabesberger 2016 + * YoutubeUrlIdHandler.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class YoutubeUrlIdHandler implements UrlIdHandler { + @SuppressWarnings("WeakerAccess") + @Override + public String getVideoUrl(String videoId) { + return "https://www.youtube.com/watch?v=" + videoId; + } + + @SuppressWarnings("WeakerAccess") + @Override + public String getVideoId(String url) throws ParsingException { + String id; + String pat; + + if(url.contains("youtube")) { + pat = "youtube\\.com/watch\\?v=([\\-a-zA-Z0-9_]{11})"; + } + else if(url.contains("youtu.be")) { + pat = "youtu\\.be/([a-zA-Z0-9_-]{11})"; + } + else { + throw new ParsingException("Error no suitable url: " + url); + } + + id = RegexHelper.matchGroup1(pat, url); + if(!id.isEmpty()){ + //Log.i(TAG, "string \""+url+"\" matches!"); + return id; + } else { + throw new ParsingException("Error could not parse url: " + url); + } + } + + public String cleanUrl(String complexUrl) throws ParsingException { + return getVideoUrl(getVideoId(complexUrl)); + } + + @Override + public boolean acceptUrl(String videoUrl) { + return videoUrl.contains("youtube") || + videoUrl.contains("youtu.be"); + } +} diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java index ced24cc95..2baae72ed 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java @@ -15,6 +15,8 @@ import org.mozilla.javascript.ScriptableObject; import org.schabi.newpipe.crawler.CrawlingException; import org.schabi.newpipe.crawler.Downloader; import org.schabi.newpipe.crawler.ParsingException; +import org.schabi.newpipe.crawler.RegexHelper; +import org.schabi.newpipe.crawler.UrlIdHandler; import org.schabi.newpipe.crawler.VideoExtractor; import org.schabi.newpipe.crawler.MediaFormat; import org.schabi.newpipe.crawler.VideoInfo; @@ -25,11 +27,8 @@ import java.io.IOException; import java.io.StringReader; import java.net.URLDecoder; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Vector; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * Created by Christian Schabesberger on 06.08.15. @@ -51,7 +50,7 @@ import java.util.regex.Pattern; * along with NewPipe. If not, see . */ -public class YoutubeVideoExtractor extends VideoExtractor { +public class YoutubeVideoExtractor implements VideoExtractor { public class DecryptException extends ParsingException { DecryptException(Throwable cause) { @@ -75,7 +74,6 @@ public class YoutubeVideoExtractor extends VideoExtractor { private static final String TAG = YoutubeVideoExtractor.class.toString(); private final Document doc; private JSONObject playerArgs; - private String errorMessage = ""; // static values private static final String DECRYPTION_FUNC_NAME="decrypt"; @@ -83,23 +81,27 @@ public class YoutubeVideoExtractor extends VideoExtractor { // cached values private static volatile String decryptionCode = ""; + UrlIdHandler urlidhandler = new YoutubeUrlIdHandler(); + String pageUrl = ""; + private Downloader downloader; public YoutubeVideoExtractor(String pageUrl, Downloader dl) throws CrawlingException, IOException { //most common videoInfo fields are now set in our superclass, for all services - super(pageUrl, dl); downloader = dl; - String pageContent = downloader.download(cleanUrl(pageUrl)); + this.pageUrl = urlidhandler.cleanUrl(pageUrl); + String pageContent = downloader.download(this.pageUrl); doc = Jsoup.parse(pageContent, pageUrl); String ytPlayerConfigRaw; JSONObject ytPlayerConfig; //attempt to load the youtube js player JSON arguments try { - ytPlayerConfigRaw = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); + ytPlayerConfigRaw = + RegexHelper.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); ytPlayerConfig = new JSONObject(ytPlayerConfigRaw); playerArgs = ytPlayerConfig.getJSONObject("args"); - } catch (RegexException e) { + } catch (RegexHelper.RegexException e) { String errorReason = findErrorReason(doc); switch(errorReason) { case "GEMA": @@ -233,7 +235,16 @@ public class YoutubeVideoExtractor extends VideoExtractor { @Override public String getDashMpdUrl() throws ParsingException { try { - return playerArgs.getString("dashmpd"); + String dashManifest = playerArgs.getString("dashmpd"); + if(!dashManifest.contains("/signature/")) { + String encryptedSig = RegexHelper.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest); + String decryptedSig; + + decryptedSig = decryptSignature(encryptedSig, decryptionCode); + dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig); + } + + return dashManifest; } catch(NullPointerException e) { throw new ParsingException( "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", e); @@ -244,15 +255,8 @@ public class YoutubeVideoExtractor extends VideoExtractor { @Override public VideoInfo.AudioStream[] getAudioStreams() throws ParsingException { - try { - String dashManifest = playerArgs.getString("dashmpd"); - return parseDashManifest(dashManifest, decryptionCode); - } catch (NullPointerException e) { - throw new ParsingException( - "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", e); - } catch (Exception e) { - throw new ParsingException(e); - } + /* If we provide a valid dash manifest, we don't need to provide audio streams extra */ + return null; } @Override @@ -300,37 +304,6 @@ public class YoutubeVideoExtractor extends VideoExtractor { return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]); } - @SuppressWarnings("WeakerAccess") - @Override - public String getVideoId(String url) throws ParsingException { - String id; - String pat; - - if(url.contains("youtube")) { - pat = "youtube\\.com/watch\\?v=([\\-a-zA-Z0-9_]{11})"; - } - else if(url.contains("youtu.be")) { - pat = "youtu\\.be/([a-zA-Z0-9_-]{11})"; - } - else { - throw new ParsingException("Error no suitable url: " + url); - } - - id = matchGroup1(pat, url); - if(!id.isEmpty()){ - //Log.i(TAG, "string \""+url+"\" matches!"); - return id; - } else { - throw new ParsingException("Error could not parse url: " + url); - } - } - - @SuppressWarnings("WeakerAccess") - @Override - public String getVideoUrl(String videoId) { - return "https://www.youtube.com/watch?v=" + videoId; - } - /**Attempts to parse (and return) the offset to start playing the video from. * @return the offset (in seconds), or 0 if no timestamp is found.*/ @Override @@ -338,8 +311,8 @@ public class YoutubeVideoExtractor extends VideoExtractor { //todo: add unit test for timestamp String timeStamp; try { - timeStamp = matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); - } catch (RegexException e) { + timeStamp = RegexHelper.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl); + } catch (RegexHelper.RegexException e) { // catch this instantly since an url does not necessarily have to have a time stamp // -2 because well the testing system will then know its the regex that failed :/ @@ -354,15 +327,15 @@ public class YoutubeVideoExtractor extends VideoExtractor { String minutesString = ""; String hoursString = ""; try { - secondsString = matchGroup1("(\\d{1,3})s", timeStamp); - minutesString = matchGroup1("(\\d{1,3})m", timeStamp); - hoursString = matchGroup1("(\\d{1,3})h", timeStamp); + secondsString = RegexHelper.matchGroup1("(\\d{1,3})s", timeStamp); + minutesString = RegexHelper.matchGroup1("(\\d{1,3})m", timeStamp); + hoursString = RegexHelper.matchGroup1("(\\d{1,3})h", timeStamp); } catch (Exception e) { //it could be that time is given in another method if (secondsString.isEmpty() //if nothing was got, && minutesString.isEmpty()//treat as unlabelled seconds && hoursString.isEmpty()) { - secondsString = matchGroup1("t=(\\d{1,3})", timeStamp); + secondsString = RegexHelper.matchGroup1("t=(\\d{1,3})", timeStamp); } } @@ -455,72 +428,14 @@ public class YoutubeVideoExtractor extends VideoExtractor { } } - private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) throws RegexException, DecryptException { - if(!dashManifest.contains("/signature/")) { - String encryptedSig = matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest); - String decryptedSig; + @Override + public UrlIdHandler getUrlIdConverter() { + return new YoutubeUrlIdHandler(); + } - decryptedSig = decryptSignature(encryptedSig, decryptoinCode); - dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig); - } - String dashDoc; - try { - dashDoc = downloader.download(dashManifest); - } catch(IOException ioe) { - throw new DecryptException("Could not get dash mpd", ioe); - } - Vector audioStreams = new Vector<>(); - try { - XmlPullParser parser = Xml.newPullParser(); - parser.setInput(new StringReader(dashDoc)); - String tagName = ""; - String currentMimeType = ""; - int currentBandwidth = -1; - int currentSamplingRate = -1; - boolean currentTagIsBaseUrl = false; - for(int eventType = parser.getEventType(); - eventType != XmlPullParser.END_DOCUMENT; - eventType = parser.next() ) { - switch(eventType) { - case XmlPullParser.START_TAG: - tagName = parser.getName(); - if(tagName.equals("AdaptationSet")) { - currentMimeType = parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "mimeType"); - } else if(tagName.equals("Representation") && currentMimeType.contains("audio")) { - currentBandwidth = Integer.parseInt( - parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "bandwidth")); - currentSamplingRate = Integer.parseInt( - parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "audioSamplingRate")); - } else if(tagName.equals("BaseURL")) { - currentTagIsBaseUrl = true; - } - break; - - case XmlPullParser.TEXT: - if(currentTagIsBaseUrl && - (currentMimeType.contains("audio"))) { - int format = -1; - if(currentMimeType.equals(MediaFormat.WEBMA.mimeType)) { - format = MediaFormat.WEBMA.id; - } else if(currentMimeType.equals(MediaFormat.M4A.mimeType)) { - format = MediaFormat.M4A.id; - } - audioStreams.add(new VideoInfo.AudioStream(parser.getText(), - format, currentBandwidth, currentSamplingRate)); - } - //missing break here? - case XmlPullParser.END_TAG: - if(tagName.equals("AdaptationSet")) { - currentMimeType = ""; - } else if(tagName.equals("BaseURL")) { - currentTagIsBaseUrl = false; - }//no break needed here - } - } - } catch(Exception e) { - e.printStackTrace(); - } - return audioStreams.toArray(new VideoInfo.AudioStream[audioStreams.size()]); + @Override + public String getPageUrl() { + return pageUrl; } /**Provides information about links to other videos on the video page, such as related videos. @@ -533,7 +448,7 @@ public class YoutubeVideoExtractor extends VideoExtractor { info.webpage_url = li.select("a.content-link").first() .attr("abs:href"); - info.id = matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url); + info.id = RegexHelper.matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url); //todo: check NullPointerException causing info.title = li.select("span.title").first().text(); @@ -584,15 +499,20 @@ public class YoutubeVideoExtractor extends VideoExtractor { try { String playerCode = downloader.download(playerUrl); - decryptionFuncName = matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode); + decryptionFuncName = + RegexHelper.matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode); - String functionPattern = "(" + decryptionFuncName.replace("$", "\\$") + "=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})"; - decryptionFunc = "var " + matchGroup1(functionPattern, playerCode) + ";"; + String functionPattern = "(" + + decryptionFuncName.replace("$", "\\$") + + "=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})"; + decryptionFunc = "var " + RegexHelper.matchGroup1(functionPattern, playerCode) + ";"; - helperObjectName = matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc); + helperObjectName = RegexHelper + .matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc); - String helperPattern = "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; - helperObject = matchGroup1(helperPattern, playerCode); + String helperPattern = "(var " + + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; + helperObject = RegexHelper.matchGroup1(helperPattern, playerCode); callerFunc = callerFunc.replace("%%", decryptionFuncName); @@ -624,25 +544,8 @@ public class YoutubeVideoExtractor extends VideoExtractor { return (result == null ? "" : result.toString()); } - private String cleanUrl(String complexUrl) throws ParsingException { - return getVideoUrl(getVideoId(complexUrl)); - } - - private String matchGroup1(String pattern, String input) throws RegexException { - Pattern pat = Pattern.compile(pattern); - Matcher mat = pat.matcher(input); - boolean foundMatch = mat.find(); - if (foundMatch) { - return mat.group(1); - } - else { - //Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\""); - throw new RegexException("failed to find pattern \""+pattern+" inside of "+input+"\""); - } - } - private String findErrorReason(Document doc) { - errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text(); + String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text(); if(errorMessage.contains("GEMA")) { // Gema sometimes blocks youtube music content in germany: // https://www.gema.de/en/ From 14fb7d8a7ab49fa7fa09468bc208003bc1374a5c Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Tue, 2 Feb 2016 19:01:53 +0100 Subject: [PATCH 09/11] fixed failing tests --- .../services/youtube/YoutubeVideoExtractorDefaultTest.java | 7 +------ .../crawler/services/youtube/YoutubeVideoExtractor.java | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java index 3a067e50d..b63e390c2 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java @@ -85,12 +85,7 @@ public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase { } public void testGetAudioStreams() throws ParsingException { - for(VideoInfo.AudioStream s : extractor.getAudioStreams()) { - assertTrue(s.url, - s.url.contains("https://")); - assertTrue(s.bandwidth > 0); - assertTrue(s.samplingRate > 0); - } + assertTrue(extractor.getAudioStreams() == null); } public void testGetVideoStreams() throws ParsingException { diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java index 2baae72ed..94d36b30b 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java @@ -89,8 +89,8 @@ public class YoutubeVideoExtractor implements VideoExtractor { public YoutubeVideoExtractor(String pageUrl, Downloader dl) throws CrawlingException, IOException { //most common videoInfo fields are now set in our superclass, for all services downloader = dl; - this.pageUrl = urlidhandler.cleanUrl(pageUrl); - String pageContent = downloader.download(this.pageUrl); + this.pageUrl = pageUrl; + String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl)); doc = Jsoup.parse(pageContent, pageUrl); String ytPlayerConfigRaw; JSONObject ytPlayerConfig; From 61471fdd3c5d42a7b5685d0691135780c70ee191 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Fri, 5 Feb 2016 13:34:44 +0100 Subject: [PATCH 10/11] renamed UrlIdHandler into VideoUrlIdHandler --- .../org/schabi/newpipe/crawler/StreamingService.java | 2 +- .../org/schabi/newpipe/crawler/VideoExtractor.java | 3 +-- .../java/org/schabi/newpipe/crawler/VideoInfo.java | 2 +- .../{UrlIdHandler.java => VideoUrlIdHandler.java} | 4 ++-- .../crawler/services/youtube/YoutubeService.java | 8 ++++---- .../services/youtube/YoutubeVideoExtractor.java | 11 ++++------- ...rlIdHandler.java => YoutubeVideoUrlIdHandler.java} | 6 +++--- 7 files changed, 16 insertions(+), 20 deletions(-) rename app/src/main/java/org/schabi/newpipe/crawler/{UrlIdHandler.java => VideoUrlIdHandler.java} (93%) rename app/src/main/java/org/schabi/newpipe/crawler/services/youtube/{YoutubeUrlIdHandler.java => YoutubeVideoUrlIdHandler.java} (92%) diff --git a/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java b/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java index f34a40db8..9b6f4e285 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java @@ -31,7 +31,7 @@ public interface StreamingService { throws IOException, CrawlingException; SearchEngine getSearchEngineInstance(); - UrlIdHandler getUrlIdHandler(); + VideoUrlIdHandler getUrlIdHandler(); } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java index 41a63ebdb..eb336b5ae 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java @@ -20,7 +20,6 @@ package org.schabi.newpipe.crawler; * along with NewPipe. If not, see . */ -import java.net.URL; import java.util.Vector; /**Scrapes information from a video streaming service (eg, YouTube).*/ @@ -73,6 +72,6 @@ public interface VideoExtractor { public abstract int getDislikeCount() throws ParsingException; public abstract VideoPreviewInfo getNextVideo() throws ParsingException; public abstract Vector getRelatedVideos() throws ParsingException; - public abstract UrlIdHandler getUrlIdConverter(); + public abstract VideoUrlIdHandler getUrlIdConverter(); public abstract String getPageUrl(); } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java index 71503b271..d5f10553f 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java @@ -33,7 +33,7 @@ public class VideoInfo extends AbstractVideoInfo { throws CrawlingException, IOException { VideoInfo videoInfo = new VideoInfo(); - UrlIdHandler uiconv = extractor.getUrlIdConverter(); + VideoUrlIdHandler uiconv = extractor.getUrlIdConverter(); videoInfo.webpage_url = extractor.getPageUrl(); videoInfo.title = extractor.getTitle(); diff --git a/app/src/main/java/org/schabi/newpipe/crawler/UrlIdHandler.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoUrlIdHandler.java similarity index 93% rename from app/src/main/java/org/schabi/newpipe/crawler/UrlIdHandler.java rename to app/src/main/java/org/schabi/newpipe/crawler/VideoUrlIdHandler.java index 5ff0a2b7d..66d8c3cd8 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/UrlIdHandler.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoUrlIdHandler.java @@ -4,7 +4,7 @@ package org.schabi.newpipe.crawler; * Created by Christian Schabesberger on 02.02.16. * * Copyright (C) Christian Schabesberger 2016 - * UrlIdHandler.java is part of NewPipe. + * VideoUrlIdHandler.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,7 +20,7 @@ package org.schabi.newpipe.crawler; * along with NewPipe. If not, see . */ -public interface UrlIdHandler { +public interface VideoUrlIdHandler { String getVideoUrl(String videoId); String getVideoId(String siteUrl) throws ParsingException; String cleanUrl(String siteUrl) throws ParsingException; diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java index 40d7d41f7..b49c55b87 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java @@ -3,7 +3,7 @@ package org.schabi.newpipe.crawler.services.youtube; import org.schabi.newpipe.crawler.CrawlingException; import org.schabi.newpipe.crawler.Downloader; import org.schabi.newpipe.crawler.StreamingService; -import org.schabi.newpipe.crawler.UrlIdHandler; +import org.schabi.newpipe.crawler.VideoUrlIdHandler; import org.schabi.newpipe.crawler.VideoExtractor; import org.schabi.newpipe.crawler.SearchEngine; @@ -40,7 +40,7 @@ public class YoutubeService implements StreamingService { @Override public VideoExtractor getExtractorInstance(String url, Downloader downloader) throws CrawlingException, IOException { - UrlIdHandler urlIdHandler = new YoutubeUrlIdHandler(); + VideoUrlIdHandler urlIdHandler = new YoutubeVideoUrlIdHandler(); if(urlIdHandler.acceptUrl(url)) { return new YoutubeVideoExtractor(url, downloader) ; } @@ -54,7 +54,7 @@ public class YoutubeService implements StreamingService { } @Override - public UrlIdHandler getUrlIdHandler() { - return new YoutubeUrlIdHandler(); + public VideoUrlIdHandler getUrlIdHandler() { + return new YoutubeVideoUrlIdHandler(); } } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java index 94d36b30b..744afd48c 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java @@ -1,7 +1,6 @@ package org.schabi.newpipe.crawler.services.youtube; import android.util.Log; -import android.util.Xml; import org.json.JSONException; import org.json.JSONObject; @@ -16,15 +15,13 @@ import org.schabi.newpipe.crawler.CrawlingException; import org.schabi.newpipe.crawler.Downloader; import org.schabi.newpipe.crawler.ParsingException; import org.schabi.newpipe.crawler.RegexHelper; -import org.schabi.newpipe.crawler.UrlIdHandler; +import org.schabi.newpipe.crawler.VideoUrlIdHandler; import org.schabi.newpipe.crawler.VideoExtractor; import org.schabi.newpipe.crawler.MediaFormat; import org.schabi.newpipe.crawler.VideoInfo; import org.schabi.newpipe.crawler.VideoPreviewInfo; -import org.xmlpull.v1.XmlPullParser; import java.io.IOException; -import java.io.StringReader; import java.net.URLDecoder; import java.util.HashMap; import java.util.Map; @@ -81,7 +78,7 @@ public class YoutubeVideoExtractor implements VideoExtractor { // cached values private static volatile String decryptionCode = ""; - UrlIdHandler urlidhandler = new YoutubeUrlIdHandler(); + VideoUrlIdHandler urlidhandler = new YoutubeVideoUrlIdHandler(); String pageUrl = ""; private Downloader downloader; @@ -429,8 +426,8 @@ public class YoutubeVideoExtractor implements VideoExtractor { } @Override - public UrlIdHandler getUrlIdConverter() { - return new YoutubeUrlIdHandler(); + public VideoUrlIdHandler getUrlIdConverter() { + return new YoutubeVideoUrlIdHandler(); } @Override diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeUrlIdHandler.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoUrlIdHandler.java similarity index 92% rename from app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeUrlIdHandler.java rename to app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoUrlIdHandler.java index d0ccafafa..b9d2b4fe8 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeUrlIdHandler.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoUrlIdHandler.java @@ -2,13 +2,13 @@ package org.schabi.newpipe.crawler.services.youtube; import org.schabi.newpipe.crawler.ParsingException; import org.schabi.newpipe.crawler.RegexHelper; -import org.schabi.newpipe.crawler.UrlIdHandler; +import org.schabi.newpipe.crawler.VideoUrlIdHandler; /** * Created by Christian Schabesberger on 02.02.16. * * Copyright (C) Christian Schabesberger 2016 - * YoutubeUrlIdHandler.java is part of NewPipe. + * YoutubeVideoUrlIdHandler.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -24,7 +24,7 @@ import org.schabi.newpipe.crawler.UrlIdHandler; * along with NewPipe. If not, see . */ -public class YoutubeUrlIdHandler implements UrlIdHandler { +public class YoutubeVideoUrlIdHandler implements VideoUrlIdHandler { @SuppressWarnings("WeakerAccess") @Override public String getVideoUrl(String videoId) { From 1bf046a8ba291733aed572e24d513702fb9b0c0f Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Fri, 5 Feb 2016 14:09:04 +0100 Subject: [PATCH 11/11] made all arrays into lists --- .../org/schabi/newpipe/ActionBarHandler.java | 29 ++++++++++++------- .../newpipe/VideoItemDetailFragment.java | 7 ++--- .../schabi/newpipe/crawler/DashMpdParser.java | 5 ++-- .../schabi/newpipe/crawler/SearchEngine.java | 3 +- .../newpipe/crawler/VideoExtractor.java | 8 ++--- .../org/schabi/newpipe/crawler/VideoInfo.java | 12 ++++---- .../youtube/YoutubeVideoExtractor.java | 7 +++-- 7 files changed, 40 insertions(+), 31 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java b/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java index 6e70e07ba..a5e7c8cff 100644 --- a/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java +++ b/app/src/main/java/org/schabi/newpipe/ActionBarHandler.java @@ -19,6 +19,8 @@ import android.widget.ArrayAdapter; import org.schabi.newpipe.crawler.MediaFormat; import org.schabi.newpipe.crawler.VideoInfo; +import java.util.List; + /** * Created by Christian Schabesberger on 18.08.15. * @@ -49,7 +51,7 @@ class ActionBarHandler { private Bitmap videoThumbnail = null; private String channelName = ""; private AppCompatActivity activity; - private VideoInfo.VideoStream[] videoStreams = null; + private List videoStreams = null; private VideoInfo.AudioStream audioStream = null; private int selectedStream = -1; private String videoTitle = ""; @@ -93,19 +95,21 @@ class ActionBarHandler { } @SuppressWarnings("deprecation") - public void setStreams(VideoInfo.VideoStream[] videoStreams, VideoInfo.AudioStream[] audioStreams) { + public void setStreams(List videoStreams, + List audioStreams) { this.videoStreams = videoStreams; selectedStream = 0; defaultPreferences = PreferenceManager.getDefaultSharedPreferences(activity); - String[] itemArray = new String[videoStreams.length]; + String[] itemArray = new String[videoStreams.size()]; String defaultResolution = defaultPreferences .getString(activity.getString(R.string.default_resolution_key), activity.getString(R.string.default_resolution_value)); int defaultResolutionPos = 0; - for(int i = 0; i < videoStreams.length; i++) { - itemArray[i] = MediaFormat.getNameById(videoStreams[i].format) + " " + videoStreams[i].resolution; - if(defaultResolution.equals(videoStreams[i].resolution)) { + for(int i = 0; i < videoStreams.size(); i++) { + VideoInfo.VideoStream item = videoStreams.get(i); + itemArray[i] = MediaFormat.getNameById(item.format) + " " + item.resolution; + if(defaultResolution.equals(item.resolution)) { defaultResolutionPos = i; } } @@ -209,6 +213,8 @@ class ActionBarHandler { public void playVideo() { // ----------- THE MAGIC MOMENT --------------- if(!videoTitle.isEmpty()) { + VideoInfo.VideoStream selectedStreamItem = videoStreams.get(selectedStream); + if (PreferenceManager.getDefaultSharedPreferences(activity) .getBoolean(activity.getString(R.string.use_external_video_player_key), false)) { @@ -217,8 +223,8 @@ class ActionBarHandler { try { intent.setAction(Intent.ACTION_VIEW); - intent.setDataAndType(Uri.parse(videoStreams[selectedStream].url), - MediaFormat.getMimeById(videoStreams[selectedStream].format)); + intent.setDataAndType(Uri.parse(selectedStreamItem.url), + MediaFormat.getMimeById(selectedStreamItem.format)); intent.putExtra(Intent.EXTRA_TITLE, videoTitle); intent.putExtra("title", videoTitle); @@ -248,7 +254,7 @@ class ActionBarHandler { // Internal Player Intent intent = new Intent(activity, PlayVideoActivity.class); intent.putExtra(PlayVideoActivity.VIDEO_TITLE, videoTitle); - intent.putExtra(PlayVideoActivity.STREAM_URL, videoStreams[selectedStream].url); + intent.putExtra(PlayVideoActivity.STREAM_URL, selectedStreamItem.url); intent.putExtra(PlayVideoActivity.VIDEO_URL, websiteUrl); intent.putExtra(PlayVideoActivity.START_POSITION, startPosition); activity.startActivity(intent); //also HERE !!! @@ -264,13 +270,14 @@ class ActionBarHandler { private void downloadVideo() { if(!videoTitle.isEmpty()) { - String videoSuffix = "." + MediaFormat.getSuffixById(videoStreams[selectedStream].format); + VideoInfo.VideoStream selectedStreamItem = videoStreams.get(selectedStream); + String videoSuffix = "." + MediaFormat.getSuffixById(selectedStreamItem.format); String audioSuffix = "." + MediaFormat.getSuffixById(audioStream.format); Bundle args = new Bundle(); args.putString(DownloadDialog.FILE_SUFFIX_VIDEO, videoSuffix); args.putString(DownloadDialog.FILE_SUFFIX_AUDIO, audioSuffix); args.putString(DownloadDialog.TITLE, videoTitle); - args.putString(DownloadDialog.VIDEO_URL, videoStreams[selectedStream].url); + args.putString(DownloadDialog.VIDEO_URL, selectedStreamItem.url); args.putString(DownloadDialog.AUDIO_URL, audioStream.url); DownloadDialog downloadDialog = new DownloadDialog(); downloadDialog.setArguments(args); diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index f45cff230..daa7719c0 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -302,11 +302,8 @@ public class VideoItemDetailFragment extends Fragment { streamsToUse.add(i); } } - VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()]; - for (int i = 0; i < streamList.length; i++) { - streamList[i] = streamsToUse.get(i); - } - actionBarHandler.setStreams(streamList, info.audioStreams); + + actionBarHandler.setStreams(streamsToUse, info.audioStreams); nextVideoButton.setOnClickListener(new View.OnClickListener() { @Override diff --git a/app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java b/app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java index b893599dd..027cc66a5 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java @@ -6,6 +6,7 @@ import org.xmlpull.v1.XmlPullParser; import java.io.IOException; import java.io.StringReader; +import java.util.List; import java.util.Vector; /** @@ -36,7 +37,7 @@ public class DashMpdParser { } } - public static VideoInfo.AudioStream[] getAudioStreams(String dashManifestUrl, + public static List getAudioStreams(String dashManifestUrl, Downloader downloader) throws DashMpdParsingException { String dashDoc; @@ -96,6 +97,6 @@ public class DashMpdParser { } catch(Exception e) { throw new DashMpdParsingException("Could not parse Dash mpd", e); } - return audioStreams.toArray(new VideoInfo.AudioStream[audioStreams.size()]); + return audioStreams; } } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java b/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java index 686dea912..845c78926 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.crawler; import java.io.IOException; import java.util.ArrayList; +import java.util.List; import java.util.Vector; /** @@ -29,7 +30,7 @@ public interface SearchEngine { class Result { public String errorMessage = ""; public String suggestion = ""; - public final Vector resultList = new Vector<>(); + public final List resultList = new Vector<>(); } ArrayList suggestionList(String query, Downloader dl) diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java index eb336b5ae..44b4e743d 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java @@ -20,7 +20,7 @@ package org.schabi.newpipe.crawler; * along with NewPipe. If not, see . */ -import java.util.Vector; +import java.util.List; /**Scrapes information from a video streaming service (eg, YouTube).*/ @@ -63,15 +63,15 @@ public interface VideoExtractor { public abstract String getUploadDate() throws ParsingException; public abstract String getThumbnailUrl() throws ParsingException; public abstract String getUploaderThumbnailUrl() throws ParsingException; - public abstract VideoInfo.AudioStream[] getAudioStreams() throws ParsingException; - public abstract VideoInfo.VideoStream[] getVideoStreams() throws ParsingException; + public abstract List getAudioStreams() throws ParsingException; + public abstract List getVideoStreams() throws ParsingException; public abstract String getDashMpdUrl() throws ParsingException; public abstract int getAgeLimit() throws ParsingException; public abstract String getAverageRating() throws ParsingException; public abstract int getLikeCount() throws ParsingException; public abstract int getDislikeCount() throws ParsingException; public abstract VideoPreviewInfo getNextVideo() throws ParsingException; - public abstract Vector getRelatedVideos() throws ParsingException; + public abstract List getRelatedVideos() throws ParsingException; public abstract VideoUrlIdHandler getUrlIdConverter(); public abstract String getPageUrl(); } diff --git a/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java index d5f10553f..2542c501d 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java @@ -2,6 +2,7 @@ package org.schabi.newpipe.crawler; import java.io.IOException; import java.util.List; +import java.util.Vector; /** * Created by Christian Schabesberger on 26.08.15. @@ -48,10 +49,11 @@ public class VideoInfo extends AbstractVideoInfo { /** Load and extract audio*/ videoInfo.audioStreams = extractor.getAudioStreams(); if(videoInfo.dashMpdUrl != null && !videoInfo.dashMpdUrl.isEmpty()) { - if(videoInfo.audioStreams == null || videoInfo.audioStreams.length == 0) { - videoInfo.audioStreams = - DashMpdParser.getAudioStreams(videoInfo.dashMpdUrl, downloader); + if(videoInfo.audioStreams == null) { + videoInfo.audioStreams = new Vector(); } + videoInfo.audioStreams.addAll( + DashMpdParser.getAudioStreams(videoInfo.dashMpdUrl, downloader)); } /** Extract video stream url*/ videoInfo.videoStreams = extractor.getVideoStreams(); @@ -73,8 +75,8 @@ public class VideoInfo extends AbstractVideoInfo { public String uploader_thumbnail_url = ""; public String description = ""; /*todo: make this lists over vectors*/ - public VideoStream[] videoStreams = null; - public AudioStream[] audioStreams = null; + public List videoStreams = null; + public List audioStreams = null; // video streams provided by the dash mpd do not need to be provided as VideoStream. // Later on this will also aplly to audio streams. Since dash mpd is standarized, // crawling such a file is not service dependent. Therefore getting audio only streams by yust diff --git a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java index 744afd48c..07d2b23a7 100644 --- a/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java @@ -24,6 +24,7 @@ import org.schabi.newpipe.crawler.VideoPreviewInfo; import java.io.IOException; import java.net.URLDecoder; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Vector; @@ -251,13 +252,13 @@ public class YoutubeVideoExtractor implements VideoExtractor { } @Override - public VideoInfo.AudioStream[] getAudioStreams() throws ParsingException { + public List getAudioStreams() throws ParsingException { /* If we provide a valid dash manifest, we don't need to provide audio streams extra */ return null; } @Override - public VideoInfo.VideoStream[] getVideoStreams() throws ParsingException { + public List getVideoStreams() throws ParsingException { Vector videoStreams = new Vector<>(); try{ String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map"); @@ -298,7 +299,7 @@ public class YoutubeVideoExtractor implements VideoExtractor { throw new ParsingException("Failed to get any video stream"); } - return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]); + return videoStreams; } /**Attempts to parse (and return) the offset to start playing the video from.