mirrors/NewPipe
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipe synced 2025-10-31 07:13:00 +00:00
Code Issues Releases Wiki Activity

Merge pull request #167 from theScrabi/crawlerRefactor

Browse Source 
Crawler refactor
This commit is contained in:
Christian Schabesberger
2016-02-05 14:38:48 +01:00
parent a0524fb136 1bf046a8ba
commit 241414f81b
36 changed files with 1842 additions and 1405 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngineTest.java
View File

@@ -2,8 +2,10 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import org.schabi.newpipe.VideoPreviewInfo;
import org.schabi.newpipe.services.SearchEngine;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.services.youtube.YoutubeSearchEngine;
import org.schabi.newpipe.Downloader;
import java.util.ArrayList;
@@ -35,8 +37,9 @@ public class YoutubeSearchEngineTest extends AndroidTestCase {
public void setUp() throws Exception{
super.setUp();
SearchEngine engine = new YoutubeSearchEngine();
result = engine.search("https://www.youtube.com/results?search_query=bla", 0, "de");
suggestionReply = engine.suggestionList("hello");
result = engine.search("https://www.youtube.com/results?search_query=bla",
0, "de", new Downloader());
suggestionReply = engine.suggestionList("hello", new Downloader());
}
public void testIfNoErrorOccur() {

60
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorDefaultTest.java
View File

@@ -1,9 +1,14 @@
package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import android.util.Log;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import java.io.IOException;
/**
* Created by the-scrabi on 30.12.15.
@@ -28,67 +33,62 @@ import org.schabi.newpipe.services.VideoInfo;
public class YoutubeVideoExtractorDefaultTest extends AndroidTestCase {
private YoutubeVideoExtractor extractor;
public void setUp() {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys");
public void setUp() throws IOException, CrawlingException {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=FmG385_uUys",
new Downloader());
}
public void testGetErrorCode() {
assertEquals(extractor.getErrorCode(), VideoInfo.NO_ERROR);
}
public void testGetErrorMessage() {
assertEquals(extractor.getErrorMessage(), "");
}
public void testGetTimeStamp() {
public void testGetInvalidTimeStamp() throws ParsingException {
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() >= 0);
extractor.getTimeStamp() <= 0);
}
public void testGetTitle() {
public void testGetValidTimeStamp() throws CrawlingException, IOException {
YoutubeVideoExtractor extractor =
new YoutubeVideoExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader());
assertTrue(Integer.toString(extractor.getTimeStamp()),
extractor.getTimeStamp() == 174);
}
public void testGetTitle() throws ParsingException {
assertTrue(!extractor.getTitle().isEmpty());
}
public void testGetDescription() {
public void testGetDescription() throws ParsingException {
assertTrue(extractor.getDescription() != null);
}
public void testGetUploader() {
public void testGetUploader() throws ParsingException {
assertTrue(!extractor.getUploader().isEmpty());
}
public void testGetLength() {
public void testGetLength() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
public void testGetViews() {
public void testGetViews() throws ParsingException {
assertTrue(extractor.getLength() > 0);
}
public void testGetUploadDate() {
public void testGetUploadDate() throws ParsingException {
assertTrue(extractor.getUploadDate().length() > 0);
}
public void testGetThumbnailUrl() {
public void testGetThumbnailUrl() throws ParsingException {
assertTrue(extractor.getThumbnailUrl(),
extractor.getThumbnailUrl().contains("https://"));
}
public void testGetUploaderThumbnailUrl() {
public void testGetUploaderThumbnailUrl() throws ParsingException {
assertTrue(extractor.getUploaderThumbnailUrl(),
extractor.getUploaderThumbnailUrl().contains("https://"));
}
public void testGetAudioStreams() {
for(VideoInfo.AudioStream s : extractor.getAudioStreams()) {
assertTrue(s.url,
s.url.contains("https://"));
assertTrue(s.bandwidth > 0);
assertTrue(s.samplingRate > 0);
}
public void testGetAudioStreams() throws ParsingException {
assertTrue(extractor.getAudioStreams() == null);
}
public void testGetVideoStreams() {
public void testGetVideoStreams() throws ParsingException {
for(VideoInfo.VideoStream s : extractor.getVideoStreams()) {
assertTrue(s.url,
s.url.contains("https://"));

34
app/src/androidTest/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractorGemaTest.java
View File

@@ -2,7 +2,13 @@ package org.schabi.newpipe.services.youtube;
import android.test.AndroidTestCase;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.Downloader;
import java.io.IOException;
/**
* Created by the-scrabi on 30.12.15.
@@ -31,29 +37,15 @@ public class YoutubeVideoExtractorGemaTest extends AndroidTestCase {
// Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail.
private static final boolean testActive = false;
private YoutubeVideoExtractor extractor;
public void setUp() {
public void testGemaError() throws IOException, CrawlingException {
if(testActive) {
extractor = new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8");
}
}
public void testGetErrorCode() {
if(testActive) {
assertEquals(extractor.getErrorCode(), VideoInfo.ERROR_BLOCKED_BY_GEMA);
} else {
try {
new YoutubeVideoExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8",
new Downloader());
assertTrue("Gema exception not thrown", false);
} catch(YoutubeVideoExtractor.GemaException ge) {
assertTrue(true);
}
}
public void testGetErrorMessage() {
if(testActive) {
assertTrue(extractor.getErrorMessage(),
extractor.getErrorMessage().contains("GEMA"));
} else {
assertTrue(true);
}
}
}

35
app/src/main/java/org/schabi/newpipe/ActionBarHandler.java
View File

@@ -16,13 +16,15 @@ import android.view.MenuInflater;
import android.view.MenuItem;
import android.widget.ArrayAdapter;
import org.schabi.newpipe.services.MediaFormat;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.crawler.MediaFormat;
import org.schabi.newpipe.crawler.VideoInfo;
import java.util.List;
/**
* Created by Christian Schabesberger on 18.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* DetailsMenuHandler.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
@@ -49,7 +51,7 @@ class ActionBarHandler {
private Bitmap videoThumbnail = null;
private String channelName = "";
private AppCompatActivity activity;
private VideoInfo.VideoStream[] videoStreams = null;
private List<VideoInfo.VideoStream> videoStreams = null;
private VideoInfo.AudioStream audioStream = null;
private int selectedStream = -1;
private String videoTitle = "";
@@ -93,19 +95,21 @@ class ActionBarHandler {
}
@SuppressWarnings("deprecation")
public void setStreams(VideoInfo.VideoStream[] videoStreams, VideoInfo.AudioStream[] audioStreams) {
public void setStreams(List<VideoInfo.VideoStream> videoStreams,
List<VideoInfo.AudioStream> audioStreams) {
this.videoStreams = videoStreams;
selectedStream = 0;
defaultPreferences = PreferenceManager.getDefaultSharedPreferences(activity);
String[] itemArray = new String[videoStreams.length];
String[] itemArray = new String[videoStreams.size()];
String defaultResolution = defaultPreferences
.getString(activity.getString(R.string.default_resolution_key),
activity.getString(R.string.default_resolution_value));
int defaultResolutionPos = 0;
for(int i = 0; i < videoStreams.length; i++) {
itemArray[i] = MediaFormat.getNameById(videoStreams[i].format) + " " + videoStreams[i].resolution;
if(defaultResolution.equals(videoStreams[i].resolution)) {
for(int i = 0; i < videoStreams.size(); i++) {
VideoInfo.VideoStream item = videoStreams.get(i);
itemArray[i] = MediaFormat.getNameById(item.format) + " " + item.resolution;
if(defaultResolution.equals(item.resolution)) {
defaultResolutionPos = i;
}
}
@@ -209,6 +213,8 @@ class ActionBarHandler {
public void playVideo() {
// ----------- THE MAGIC MOMENT ---------------
if(!videoTitle.isEmpty()) {
VideoInfo.VideoStream selectedStreamItem = videoStreams.get(selectedStream);
if (PreferenceManager.getDefaultSharedPreferences(activity)
.getBoolean(activity.getString(R.string.use_external_video_player_key), false)) {
@@ -217,8 +223,8 @@ class ActionBarHandler {
try {
intent.setAction(Intent.ACTION_VIEW);
intent.setDataAndType(Uri.parse(videoStreams[selectedStream].url),
MediaFormat.getMimeById(videoStreams[selectedStream].format));
intent.setDataAndType(Uri.parse(selectedStreamItem.url),
MediaFormat.getMimeById(selectedStreamItem.format));
intent.putExtra(Intent.EXTRA_TITLE, videoTitle);
intent.putExtra("title", videoTitle);
@@ -248,7 +254,7 @@ class ActionBarHandler {
// Internal Player
Intent intent = new Intent(activity, PlayVideoActivity.class);
intent.putExtra(PlayVideoActivity.VIDEO_TITLE, videoTitle);
intent.putExtra(PlayVideoActivity.STREAM_URL, videoStreams[selectedStream].url);
intent.putExtra(PlayVideoActivity.STREAM_URL, selectedStreamItem.url);
intent.putExtra(PlayVideoActivity.VIDEO_URL, websiteUrl);
intent.putExtra(PlayVideoActivity.START_POSITION, startPosition);
activity.startActivity(intent); //also HERE !!!
@@ -264,13 +270,14 @@ class ActionBarHandler {
private void downloadVideo() {
if(!videoTitle.isEmpty()) {
String videoSuffix = "." + MediaFormat.getSuffixById(videoStreams[selectedStream].format);
VideoInfo.VideoStream selectedStreamItem = videoStreams.get(selectedStream);
String videoSuffix = "." + MediaFormat.getSuffixById(selectedStreamItem.format);
String audioSuffix = "." + MediaFormat.getSuffixById(audioStream.format);
Bundle args = new Bundle();
args.putString(DownloadDialog.FILE_SUFFIX_VIDEO, videoSuffix);
args.putString(DownloadDialog.FILE_SUFFIX_AUDIO, audioSuffix);
args.putString(DownloadDialog.TITLE, videoTitle);
args.putString(DownloadDialog.VIDEO_URL, videoStreams[selectedStream].url);
args.putString(DownloadDialog.VIDEO_URL, selectedStreamItem.url);
args.putString(DownloadDialog.AUDIO_URL, audioStream.url);
DownloadDialog downloadDialog = new DownloadDialog();
downloadDialog.setArguments(args);

2
app/src/main/java/org/schabi/newpipe/DownloadDialog.java
View File

@@ -107,7 +107,7 @@ public class DownloadDialog extends DialogFragment {
long id = 0;
if (App.isUsingTor()) {
// if using Tor, do not use DownloadManager because the proxy cannot be set
Downloader.downloadFile(getContext(), url, saveFilePath, title);
FileDownloader.downloadFile(getContext(), url, saveFilePath, title);
} else {
DownloadManager dm = (DownloadManager) context.getSystemService(Context.DOWNLOAD_SERVICE);
DownloadManager.Request request = new DownloadManager.Request(

183
app/src/main/java/org/schabi/newpipe/Downloader.java
View File

@@ -1,24 +1,8 @@
package org.schabi.newpipe;
import android.app.NotificationManager;
import android.content.Context;
import android.content.SharedPreferences;
import android.graphics.drawable.BitmapDrawable;
import android.graphics.drawable.Drawable;
import android.os.AsyncTask;
import android.preference.PreferenceManager;
import android.support.v4.app.NotificationCompat;
import android.util.Log;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.UnknownHostException;
@@ -27,9 +11,9 @@ import javax.net.ssl.HttpsURLConnection;
import info.guardianproject.netcipher.NetCipher;
/**
* Created by Christian Schabesberger on 14.08.15.
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
@@ -46,190 +30,61 @@ import info.guardianproject.netcipher.NetCipher;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class Downloader extends AsyncTask<Void, Integer, Void> {
public static final String TAG = "Downloader";
public class Downloader implements org.schabi.newpipe.crawler.Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private NotificationManager nm;
private NotificationCompat.Builder builder;
private int notifyId = 0x1234;
private int fileSize = 0xffffffff;
private final Context context;
private final String fileURL;
private final File saveFilePath;
private final String title;
private final String debugContext;
public Downloader(Context context, String fileURL, File saveFilePath, String title) {
this.context = context;
this.fileURL = fileURL;
this.saveFilePath = saveFilePath;
this.title = title;
this.debugContext = "'" + fileURL +
"' => '" + saveFilePath + "'";
}
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
public static String download(String siteUrl, String language) {
String ret = "";
try {
public String download(String siteUrl, String language) throws IOException {
URL url = new URL(siteUrl);
//HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
con.setRequestProperty("Accept-Language", language);
ret = dl(con);
}
catch(Exception e) {
e.printStackTrace();
}
return ret;
return dl(con);
}
/**Common functionality between download(String url) and download(String url, String language)*/
private static String dl(HttpsURLConnection con) throws IOException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
try {
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
BufferedReader in = new BufferedReader(
in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
String inputLine;
while((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
}
catch(UnknownHostException uhe) {//thrown when there's no internet connection
uhe.printStackTrace();
} catch(UnknownHostException uhe) {//thrown when there's no internet connection
throw new IOException("unknown host or no network", uhe);
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch(Exception e) {
throw new IOException(e);
} finally {
if(in != null) {
in.close();
}
}
return response.toString();
}
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
public static String download(String siteUrl) {
String ret = "";
try {
public String download(String siteUrl) throws IOException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
ret = dl(con);
return dl(con);
}
catch(Exception e) {
e.printStackTrace();
}
return ret;
}
/**
* Downloads a file from a URL in the background using an {@link AsyncTask}.
*
* @param fileURL HTTP URL of the file to be downloaded
* @param saveFilePath path of the directory to save the file
* @param title
* @throws IOException
*/
public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) {
new Downloader(context, fileURL, saveFilePath, title).execute();
}
/** AsyncTask impl: executed in gui thread */
@Override
protected void onPreExecute() {
super.onPreExecute();
nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE);
Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher);
builder = new NotificationCompat.Builder(context)
.setSmallIcon(android.R.drawable.stat_sys_download)
.setLargeIcon(((BitmapDrawable) icon).getBitmap())
.setContentTitle(saveFilePath.getName())
.setContentText(saveFilePath.getAbsolutePath())
.setProgress(fileSize, 0, false);
nm.notify(notifyId, builder.build());
}
/** AsyncTask impl: executed in background thread does the download */
@Override
protected Void doInBackground(Void... voids) {
HttpsURLConnection con = null;
InputStream inputStream = null;
FileOutputStream outputStream = null;
try {
con = NetCipher.getHttpsURLConnection(fileURL);
int responseCode = con.getResponseCode();
// always check HTTP response code first
if (responseCode == HttpURLConnection.HTTP_OK) {
fileSize = con.getContentLength();
inputStream = new BufferedInputStream(con.getInputStream());
outputStream = new FileOutputStream(saveFilePath);
int bufferSize = 8192;
int downloaded = 0;
int bytesRead = -1;
byte[] buffer = new byte[bufferSize];
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
downloaded += bytesRead;
if (downloaded % 50000 < bufferSize) {
publishProgress(downloaded);
}
}
publishProgress(bufferSize);
} else {
Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode);
}
} catch (IOException e) {
Log.e(TAG, "No file to download. Server replied HTTP code: ", e);
e.printStackTrace();
} finally {
try {
if (outputStream != null) {
outputStream.close();
}
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
if (con != null) {
con.disconnect();
}
}
return null;
}
@Override
protected void onProgressUpdate(Integer... progress) {
builder.setProgress(fileSize, progress[0], false);
nm.notify(notifyId, builder.build());
}
@Override
protected void onPostExecute(Void aVoid) {
super.onPostExecute(aVoid);
nm.cancel(notifyId);
}
}

169
app/src/main/java/org/schabi/newpipe/FileDownloader.java Normal file
View File

@@ -0,0 +1,169 @@
package org.schabi.newpipe;
import android.app.NotificationManager;
import android.content.Context;
import android.content.SharedPreferences;
import android.graphics.drawable.BitmapDrawable;
import android.graphics.drawable.Drawable;
import android.os.AsyncTask;
import android.preference.PreferenceManager;
import android.support.v4.app.NotificationCompat;
import android.util.Log;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.UnknownHostException;
import javax.net.ssl.HttpsURLConnection;
import info.guardianproject.netcipher.NetCipher;
/**
* Created by Christian Schabesberger on 14.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* FileDownloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class FileDownloader extends AsyncTask<Void, Integer, Void> {
public static final String TAG = "FileDownloader";
private NotificationManager nm;
private NotificationCompat.Builder builder;
private int notifyId = 0x1234;
private int fileSize = 0xffffffff;
private final Context context;
private final String fileURL;
private final File saveFilePath;
private final String title;
private final String debugContext;
public FileDownloader(Context context, String fileURL, File saveFilePath, String title) {
this.context = context;
this.fileURL = fileURL;
this.saveFilePath = saveFilePath;
this.title = title;
this.debugContext = "'" + fileURL +
"' => '" + saveFilePath + "'";
}
/**
* Downloads a file from a URL in the background using an {@link AsyncTask}.
*
* @param fileURL HTTP URL of the file to be downloaded
* @param saveFilePath path of the directory to save the file
* @param title
* @throws IOException
*/
public static void downloadFile(final Context context, final String fileURL, final File saveFilePath, String title) {
new FileDownloader(context, fileURL, saveFilePath, title).execute();
}
/** AsyncTask impl: executed in gui thread */
@Override
protected void onPreExecute() {
super.onPreExecute();
nm = (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE);
Drawable icon = context.getResources().getDrawable(R.mipmap.ic_launcher);
builder = new NotificationCompat.Builder(context)
.setSmallIcon(android.R.drawable.stat_sys_download)
.setLargeIcon(((BitmapDrawable) icon).getBitmap())
.setContentTitle(saveFilePath.getName())
.setContentText(saveFilePath.getAbsolutePath())
.setProgress(fileSize, 0, false);
nm.notify(notifyId, builder.build());
}
/** AsyncTask impl: executed in background thread does the download */
@Override
protected Void doInBackground(Void... voids) {
HttpsURLConnection con = null;
InputStream inputStream = null;
FileOutputStream outputStream = null;
try {
con = NetCipher.getHttpsURLConnection(fileURL);
int responseCode = con.getResponseCode();
// always check HTTP response code first
if (responseCode == HttpURLConnection.HTTP_OK) {
fileSize = con.getContentLength();
inputStream = new BufferedInputStream(con.getInputStream());
outputStream = new FileOutputStream(saveFilePath);
int bufferSize = 8192;
int downloaded = 0;
int bytesRead = -1;
byte[] buffer = new byte[bufferSize];
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
downloaded += bytesRead;
if (downloaded % 50000 < bufferSize) {
publishProgress(downloaded);
}
}
publishProgress(bufferSize);
} else {
Log.i(TAG, "No file to download. Server replied HTTP code: " + responseCode);
}
} catch (IOException e) {
Log.e(TAG, "No file to download. Server replied HTTP code: ", e);
e.printStackTrace();
} finally {
try {
if (outputStream != null) {
outputStream.close();
}
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
if (con != null) {
con.disconnect();
}
}
return null;
}
@Override
protected void onProgressUpdate(Integer... progress) {
builder.setProgress(fileSize, progress[0], false);
nm.notify(notifyId, builder.build());
}
@Override
protected void onPostExecute(Void aVoid) {
super.onPostExecute(aVoid);
nm.cancel(notifyId);
}
}

2
app/src/main/java/org/schabi/newpipe/VideoInfoItemViewCreator.java
View File

@@ -7,6 +7,8 @@ import android.view.ViewGroup;
import android.widget.ImageView;
import android.widget.TextView;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
/**
* Created by Christian Schabesberger on 24.10.15.
*

6
app/src/main/java/org/schabi/newpipe/VideoItemDetailActivity.java
View File

@@ -11,8 +11,8 @@ import android.view.Menu;
import android.view.MenuItem;
import android.widget.Toast;
import org.schabi.newpipe.services.ServiceList;
import org.schabi.newpipe.services.StreamingService;
import org.schabi.newpipe.crawler.ServiceList;
import org.schabi.newpipe.crawler.StreamingService;
/**
@@ -73,7 +73,7 @@ public class VideoItemDetailActivity extends AppCompatActivity {
StreamingService[] serviceList = ServiceList.getServices();
//VideoExtractor videoExtractor = null;
for (int i = 0; i < serviceList.length; i++) {
if (serviceList[i].acceptUrl(videoUrl)) {
if (serviceList[i].getUrlIdHandler().acceptUrl(videoUrl)) {
arguments.putInt(VideoItemDetailFragment.STREAMING_SERVICE, i);
currentStreamingService = i;
//videoExtractor = ServiceList.getService(i).getExtractorInstance();

138
app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java
View File

@@ -16,7 +16,6 @@ import android.support.v4.app.Fragment;
import android.support.v7.app.AppCompatActivity;
import android.text.Html;
import android.text.method.LinkMovementMethod;
import android.util.DisplayMetrics;
import android.util.Log;
import android.view.LayoutInflater;
import android.view.Menu;
@@ -32,14 +31,20 @@ import android.widget.TextView;
import android.view.MenuItem;
import android.widget.Toast;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.MalformedInputException;
import java.util.ArrayList;
import java.util.Vector;
import org.schabi.newpipe.services.VideoExtractor;
import org.schabi.newpipe.services.ServiceList;
import org.schabi.newpipe.services.StreamingService;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.ServiceList;
import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.crawler.services.youtube.YoutubeVideoExtractor;
/**
@@ -68,7 +73,6 @@ public class VideoItemDetailFragment extends Fragment {
* The fragment argument representing the item ID that this fragment
* represents.
*/
//public static final String ARG_ITEM_ID = "item_id";
public static final String VIDEO_URL = "video_url";
public static final String STREAMING_SERVICE = "streaming_service";
public static final String AUTO_PLAY = "auto_play";
@@ -87,7 +91,6 @@ public class VideoItemDetailFragment extends Fragment {
private FloatingActionButton playVideoButton;
private final Point initialThumbnailPos = new Point(0, 0);
public interface OnInvokeCreateOptionsMenuListener {
void createOptionsMenu();
}
@@ -108,11 +111,11 @@ public class VideoItemDetailFragment extends Fragment {
@Override
public void run() {
try {
this.videoExtractor = service.getExtractorInstance(videoUrl);
VideoInfo videoInfo = videoExtractor.getVideoInfo();
videoExtractor = service.getExtractorInstance(videoUrl, new Downloader());
VideoInfo videoInfo = VideoInfo.getVideoInfo(videoExtractor, new Downloader());
h.post(new VideoResultReturnedRunnable(videoInfo));
if (videoInfo.errorCode == VideoInfo.NO_ERROR) {
h.post(new SetThumbnailRunnable(
//todo: make bitmaps not bypass tor
BitmapFactory.decodeStream(
new URL(videoInfo.thumbnail_url)
.openConnection()
@@ -124,7 +127,7 @@ public class VideoItemDetailFragment extends Fragment {
.openConnection()
.getInputStream()),
SetThumbnailRunnable.CHANNEL_THUMBNAIL));
if(showNextVideoItem) {
if (showNextVideoItem) {
h.post(new SetThumbnailRunnable(
BitmapFactory.decodeStream(
new URL(videoInfo.nextVideo.thumbnail_url)
@@ -132,21 +135,41 @@ public class VideoItemDetailFragment extends Fragment {
.getInputStream()),
SetThumbnailRunnable.NEXT_VIDEO_THUMBNAIL));
}
} catch (MalformedInputException e) {
postNewErrorToast(h, R.string.could_not_load_thumbnails);
e.printStackTrace();
} catch (IOException e) {
postNewErrorToast(h, R.string.network_error);
e.printStackTrace();
}
} catch (Exception e) {
// custom service related exceptions
catch (YoutubeVideoExtractor.DecryptException de) {
postNewErrorToast(h, R.string.youtube_signature_decryption_error);
de.printStackTrace();
} catch (YoutubeVideoExtractor.GemaException ge) {
h.post(new Runnable() {
@Override
public void run() {
progressBar.setVisibility(View.GONE);
// This is poor style, but unless we have better error handling in the
// crawler, this may not be better.
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
R.string.network_error, Toast.LENGTH_LONG).show();
onErrorBlockedByGema();
}
});
}
// ----------------------------------------
catch(VideoExtractor.ContentNotAvailableException e) {
h.post(new Runnable() {
@Override
public void run() {
onNotSpecifiedContentError();
}
});
e.printStackTrace();
} catch (ParsingException e) {
postNewErrorToast(h, e.getMessage());
e.printStackTrace();
} catch(Exception e) {
postNewErrorToast(h, R.string.general_error);
e.printStackTrace();
}
}
}
@@ -213,7 +236,7 @@ public class VideoItemDetailFragment extends Fragment {
private void updateInfo(VideoInfo info) {
currentVideoInfo = info;
Resources res = activity.getResources();
try {
VideoInfoItemViewCreator videoItemViewCreator =
new VideoInfoItemViewCreator(LayoutInflater.from(getActivity()));
@@ -226,17 +249,13 @@ public class VideoItemDetailFragment extends Fragment {
TextView thumbsDownView = (TextView) activity.findViewById(R.id.detailThumbsDownCountView);
TextView uploadDateView = (TextView) activity.findViewById(R.id.detailUploadDateView);
TextView descriptionView = (TextView) activity.findViewById(R.id.detailDescriptionView);
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
FrameLayout nextVideoFrame = (FrameLayout) activity.findViewById(R.id.detailNextVideoFrame);
RelativeLayout nextVideoRootFrame =
(RelativeLayout) activity.findViewById(R.id.detailNextVideoRootLayout);
Button backgroundButton = (Button)
activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton);
progressBar.setVisibility(View.GONE);
switch (info.errorCode) {
case VideoInfo.NO_ERROR: {
View nextVideoView = videoItemViewCreator
.getViewFromVideoInfoItem(null, nextVideoFrame, info.nextVideo, getContext());
nextVideoFrame.addView(nextVideoView);
@@ -283,11 +302,8 @@ public class VideoItemDetailFragment extends Fragment {
streamsToUse.add(i);
}
}
VideoInfo.VideoStream[] streamList = new VideoInfo.VideoStream[streamsToUse.size()];
for (int i = 0; i < streamList.length; i++) {
streamList[i] = streamsToUse.get(i);
}
actionBarHandler.setStreams(streamList, info.audioStreams);
actionBarHandler.setStreams(streamsToUse, info.audioStreams);
nextVideoButton.setOnClickListener(new View.OnClickListener() {
@Override
@@ -303,9 +319,23 @@ public class VideoItemDetailFragment extends Fragment {
startActivity(detailIntent);
}
});
if(autoPlayEnabled) {
actionBarHandler.playVideo();
}
break;
case VideoInfo.ERROR_BLOCKED_BY_GEMA:
} catch (java.lang.NullPointerException e) {
Log.w(TAG, "updateInfo(): Fragment closed before thread ended work... or else");
e.printStackTrace();
}
}
private void onErrorBlockedByGema() {
Button backgroundButton = (Button)
activity.findViewById(R.id.detailVideoThumbnailWindowBackgroundButton);
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
progressBar.setVisibility(View.GONE);
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.gruese_die_gema));
backgroundButton.setOnClickListener(new View.OnClickListener() {
@@ -317,24 +347,18 @@ public class VideoItemDetailFragment extends Fragment {
activity.startActivity(intent);
}
});
break;
case VideoInfo.ERROR_NO_SPECIFIED_ERROR:
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.not_available_monkey));
Toast.makeText(activity, info.errorMessage, Toast.LENGTH_LONG)
.show();
break;
default:
Log.e(TAG, "Video Available Status not known.");
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
R.string.blocked_by_gema, Toast.LENGTH_LONG).show();
}
if(autoPlayEnabled) {
actionBarHandler.playVideo();
}
} catch (java.lang.NullPointerException e) {
Log.w(TAG, "updateInfo(): Fragment closed before thread ended work... or else");
e.printStackTrace();
}
private void onNotSpecifiedContentError() {
ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView);
progressBar.setVisibility(View.GONE);
thumbnailView.setImageBitmap(BitmapFactory.decodeResource(
getResources(), R.drawable.not_available_monkey));
Toast.makeText(activity, R.string.content_not_available, Toast.LENGTH_LONG)
.show();
}
private boolean useStream(VideoInfo.VideoStream stream, Vector<VideoInfo.VideoStream> streams) {
@@ -465,4 +489,24 @@ public class VideoItemDetailFragment extends Fragment {
public void setOnInvokeCreateOptionsMenuListener(OnInvokeCreateOptionsMenuListener listener) {
this.onInvokeCreateOptionsMenuListener = listener;
}
private void postNewErrorToast(Handler h, final int stringResource) {
h.post(new Runnable() {
@Override
public void run() {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
stringResource, Toast.LENGTH_LONG).show();
}
});
}
private void postNewErrorToast(Handler h, final String message) {
h.post(new Runnable() {
@Override
public void run() {
Toast.makeText(VideoItemDetailFragment.this.getActivity(),
message, Toast.LENGTH_LONG).show();
}
});
}
}

3
app/src/main/java/org/schabi/newpipe/VideoItemListActivity.java
View File

@@ -17,7 +17,8 @@ import android.view.inputmethod.InputMethodManager;
import java.util.ArrayList;
import org.schabi.newpipe.services.ServiceList;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.ServiceList;
/**
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>

39
app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java
View File

@@ -15,12 +15,15 @@ import android.widget.AbsListView;
import android.widget.ListView;
import android.widget.Toast;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import java.util.Vector;
import org.schabi.newpipe.services.SearchEngine;
import org.schabi.newpipe.services.StreamingService;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.StreamingService;
/**
@@ -108,23 +111,22 @@ public class VideoItemListFragment extends ListFragment {
String searchLanguageKey = getContext().getString(R.string.search_language_key);
String searchLanguage = sp.getString(searchLanguageKey,
getString(R.string.default_language_value));
SearchEngine.Result result = engine.search(query, page, searchLanguage);
SearchEngine.Result result = engine.search(query, page, searchLanguage,
new Downloader());
Log.i(TAG, "language code passed:\""+searchLanguage+"\"");
if(runs) {
h.post(new ResultRunnable(result, requestId));
}
} catch(Exception e) {
} catch(IOException e) {
postNewErrorToast(h, R.string.network_error);
e.printStackTrace();
} catch(CrawlingException ce) {
postNewErrorToast(h, R.string.parsing_error);
ce.printStackTrace();
} catch(Exception e) {
postNewErrorToast(h, R.string.general_error);
e.printStackTrace();
h.post(new Runnable() {
@Override
public void run() {
setListShown(true);
Toast.makeText(getActivity(), getString(R.string.network_error),
Toast.LENGTH_SHORT).show();
}
});
}
}
}
@@ -155,6 +157,7 @@ public class VideoItemListFragment extends ListFragment {
if(!downloadedList.get(i)) {
Bitmap thumbnail;
try {
//todo: make bitmaps not bypass tor
thumbnail = BitmapFactory.decodeStream(
new URL(thumbnailUrlList.get(i)).openConnection().getInputStream());
h.post(new SetThumbnailRunnable(i, thumbnail, requestId));
@@ -384,4 +387,14 @@ public class VideoItemListFragment extends ListFragment {
mActivatedPosition = position;
}
private void postNewErrorToast(Handler h, final int stringResource) {
h.post(new Runnable() {
@Override
public void run() {
setListShown(true);
Toast.makeText(getActivity(), getString(R.string.network_error),
Toast.LENGTH_SHORT).show();
}
});
}
}

2
app/src/main/java/org/schabi/newpipe/VideoListAdapter.java
View File

@@ -9,6 +9,8 @@ import android.view.ViewGroup;
import android.widget.BaseAdapter;
import android.widget.ListView;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import java.util.List;
import java.util.Vector;

33
app/src/main/java/org/schabi/newpipe/crawler/AbstractVideoInfo.java Normal file
View File

@@ -0,0 +1,33 @@
package org.schabi.newpipe.crawler;
import android.graphics.Bitmap;
/**
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* AbstractVideoInfo.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**Common properties between VideoInfo and VideoPreviewInfo.*/
public abstract class AbstractVideoInfo {
public String id = "";
public String title = "";
public String uploader = "";
public String thumbnail_url = "";
public Bitmap thumbnail = null;
public String webpage_url = "";
public String upload_date = "";
public long view_count = -1;
}

37
app/src/main/java/org/schabi/newpipe/crawler/CrawlingException.java Normal file
View File

@@ -0,0 +1,37 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 30.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* CrawlingException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class CrawlingException extends Exception {
public CrawlingException() {}
public CrawlingException(String message) {
super(message);
}
public CrawlingException(Throwable cause) {
super(cause);
}
public CrawlingException(String message, Throwable cause) {
super(message, cause);
}
}

102
app/src/main/java/org/schabi/newpipe/crawler/DashMpdParser.java Normal file
View File

@@ -0,0 +1,102 @@
package org.schabi.newpipe.crawler;
import android.util.Xml;
import org.xmlpull.v1.XmlPullParser;
import java.io.IOException;
import java.io.StringReader;
import java.util.List;
import java.util.Vector;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* DashMpdParser.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class DashMpdParser {
static class DashMpdParsingException extends ParsingException {
DashMpdParsingException(String message, Exception e) {
super(message, e);
}
}
public static List<VideoInfo.AudioStream> getAudioStreams(String dashManifestUrl,
Downloader downloader)
throws DashMpdParsingException {
String dashDoc;
try {
dashDoc = downloader.download(dashManifestUrl);
} catch(IOException ioe) {
throw new DashMpdParsingException("Could not get dash mpd: " + dashManifestUrl, ioe);
}
Vector<VideoInfo.AudioStream> audioStreams = new Vector<>();
try {
XmlPullParser parser = Xml.newPullParser();
parser.setInput(new StringReader(dashDoc));
String tagName = "";
String currentMimeType = "";
int currentBandwidth = -1;
int currentSamplingRate = -1;
boolean currentTagIsBaseUrl = false;
for(int eventType = parser.getEventType();
eventType != XmlPullParser.END_DOCUMENT;
eventType = parser.next() ) {
switch(eventType) {
case XmlPullParser.START_TAG:
tagName = parser.getName();
if(tagName.equals("AdaptationSet")) {
currentMimeType = parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "mimeType");
} else if(tagName.equals("Representation") && currentMimeType.contains("audio")) {
currentBandwidth = Integer.parseInt(
parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "bandwidth"));
currentSamplingRate = Integer.parseInt(
parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "audioSamplingRate"));
} else if(tagName.equals("BaseURL")) {
currentTagIsBaseUrl = true;
}
break;
case XmlPullParser.TEXT:
if(currentTagIsBaseUrl &&
(currentMimeType.contains("audio"))) {
int format = -1;
if(currentMimeType.equals(MediaFormat.WEBMA.mimeType)) {
format = MediaFormat.WEBMA.id;
} else if(currentMimeType.equals(MediaFormat.M4A.mimeType)) {
format = MediaFormat.M4A.id;
}
audioStreams.add(new VideoInfo.AudioStream(parser.getText(),
format, currentBandwidth, currentSamplingRate));
}
break;
case XmlPullParser.END_TAG:
if(tagName.equals("AdaptationSet")) {
currentMimeType = "";
} else if(tagName.equals("BaseURL")) {
currentTagIsBaseUrl = false;
}//no break needed here
}
}
} catch(Exception e) {
throw new DashMpdParsingException("Could not parse Dash mpd", e);
}
return audioStreams;
}
}

41
app/src/main/java/org/schabi/newpipe/crawler/Downloader.java Normal file
View File

@@ -0,0 +1,41 @@
package org.schabi.newpipe.crawler;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* Downloader.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface Downloader {
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl, String language) throws IOException;
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
* @throws IOException*/
String download(String siteUrl) throws IOException;
}

4
app/src/main/java/org/schabi/newpipe/services/MediaFormat.java → app/src/main/java/org/schabi/newpipe/crawler/MediaFormat.java
View File

@@ -1,4 +1,4 @@
package org.schabi.newpipe.services;
package org.schabi.newpipe.crawler;
/**
* Created by Adam Howard on 08/11/15.
@@ -6,7 +6,7 @@ package org.schabi.newpipe.services;
* Copyright (c) Christian Schabesberger <chris.schabesberger@mailbox.org>
* and Adam Howard <achdisposable1@gmail.com> 2015
*
* VideoListAdapter.java is part of NewPipe.
* MediaFormat.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by

35
app/src/main/java/org/schabi/newpipe/crawler/ParsingException.java Normal file
View File

@@ -0,0 +1,35 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 31.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* ParsingException.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class ParsingException extends CrawlingException {
public ParsingException() {}
public ParsingException(String message) {
super(message);
}
public ParsingException(Throwable cause) {
super(cause);
}
public ParsingException(String message, Throwable cause) {
super(message, cause);
}
}

47
app/src/main/java/org/schabi/newpipe/crawler/RegexHelper.java Normal file
View File

@@ -0,0 +1,47 @@
package org.schabi.newpipe.crawler;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* RegexHelper.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/** avoid using regex !!! */
public class RegexHelper {
public static class RegexException extends ParsingException {
public RegexException(String message) {
super(message);
}
}
public static String matchGroup1(String pattern, String input) throws RegexException {
Pattern pat = Pattern.compile(pattern);
Matcher mat = pat.matcher(input);
boolean foundMatch = mat.find();
if (foundMatch) {
return mat.group(1);
}
else {
//Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\"");
throw new RegexException("failed to find pattern \""+pattern+" inside of "+input+"\"");
}
}
}

16
app/src/main/java/org/schabi/newpipe/services/SearchEngine.java → app/src/main/java/org/schabi/newpipe/crawler/SearchEngine.java
View File

@@ -1,8 +1,8 @@
package org.schabi.newpipe.services;
import org.schabi.newpipe.VideoPreviewInfo;
package org.schabi.newpipe.crawler;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Vector;
/**
@@ -27,16 +27,16 @@ import java.util.Vector;
@SuppressWarnings("ALL")
public interface SearchEngine {
class Result {
public String errorMessage = "";
public String suggestion = "";
public final Vector<VideoPreviewInfo> resultList = new Vector<>();
public final List<VideoPreviewInfo> resultList = new Vector<>();
}
ArrayList<String> suggestionList(String query);
ArrayList<String> suggestionList(String query, Downloader dl)
throws CrawlingException, IOException;
//Result search(String query, int page);
Result search(String query, int page, String contentCountry);
Result search(String query, int page, String contentCountry, Downloader dl)
throws CrawlingException, IOException;
}

4
app/src/main/java/org/schabi/newpipe/services/ServiceList.java → app/src/main/java/org/schabi/newpipe/crawler/ServiceList.java
View File

@@ -1,8 +1,8 @@
package org.schabi.newpipe.services;
package org.schabi.newpipe.crawler;
import android.util.Log;
import org.schabi.newpipe.services.youtube.YoutubeService;
import org.schabi.newpipe.crawler.services.youtube.YoutubeService;
/**
* Created by Christian Schabesberger on 23.08.15.

14
app/src/main/java/org/schabi/newpipe/services/StreamingService.java → app/src/main/java/org/schabi/newpipe/crawler/StreamingService.java
View File

@@ -1,4 +1,6 @@
package org.schabi.newpipe.services;
package org.schabi.newpipe.crawler;
import java.io.IOException;
/**
* Created by Christian Schabesberger on 23.08.15.
@@ -25,11 +27,11 @@ public interface StreamingService {
public String name = "";
}
ServiceInfo getServiceInfo();
VideoExtractor getExtractorInstance(String url);
VideoExtractor getExtractorInstance(String url, Downloader downloader)
throws IOException, CrawlingException;
SearchEngine getSearchEngineInstance();
/**When a VIEW_ACTION is caught this function will test if the url delivered within the calling
Intent was meant to be watched with this Service.
Return false if this service shall not allow to be called through ACTIONs.*/
boolean acceptUrl(String videoUrl);
VideoUrlIdHandler getUrlIdHandler();
}

77
app/src/main/java/org/schabi/newpipe/crawler/VideoExtractor.java Normal file
View File

@@ -0,0 +1,77 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 10.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* VideoExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
import java.util.List;
/**Scrapes information from a video streaming service (eg, YouTube).*/
@SuppressWarnings("ALL")
public interface VideoExtractor {
public class ExctractorInitException extends CrawlingException {
public ExctractorInitException() {}
public ExctractorInitException(String message) {
super(message);
}
public ExctractorInitException(Throwable cause) {
super(cause);
}
public ExctractorInitException(String message, Throwable cause) {
super(message, cause);
}
}
public class ContentNotAvailableException extends ParsingException {
public ContentNotAvailableException() {}
public ContentNotAvailableException(String message) {
super(message);
}
public ContentNotAvailableException(Throwable cause) {
super(cause);
}
public ContentNotAvailableException(String message, Throwable cause) {
super(message, cause);
}
}
public abstract int getTimeStamp() throws ParsingException;
public abstract String getTitle() throws ParsingException;
public abstract String getDescription() throws ParsingException;
public abstract String getUploader() throws ParsingException;
public abstract int getLength() throws ParsingException;
public abstract long getViews() throws ParsingException;
public abstract String getUploadDate() throws ParsingException;
public abstract String getThumbnailUrl() throws ParsingException;
public abstract String getUploaderThumbnailUrl() throws ParsingException;
public abstract List<VideoInfo.AudioStream> getAudioStreams() throws ParsingException;
public abstract List<VideoInfo.VideoStream> getVideoStreams() throws ParsingException;
public abstract String getDashMpdUrl() throws ParsingException;
public abstract int getAgeLimit() throws ParsingException;
public abstract String getAverageRating() throws ParsingException;
public abstract int getLikeCount() throws ParsingException;
public abstract int getDislikeCount() throws ParsingException;
public abstract VideoPreviewInfo getNextVideo() throws ParsingException;
public abstract List<VideoPreviewInfo> getRelatedVideos() throws ParsingException;
public abstract VideoUrlIdHandler getUrlIdConverter();
public abstract String getPageUrl();
}

79
app/src/main/java/org/schabi/newpipe/services/VideoInfo.java → app/src/main/java/org/schabi/newpipe/crawler/VideoInfo.java
View File

@@ -1,9 +1,8 @@
package org.schabi.newpipe.services;
import org.schabi.newpipe.VideoPreviewInfo;
import org.schabi.newpipe.services.AbstractVideoInfo;
package org.schabi.newpipe.crawler;
import java.io.IOException;
import java.util.List;
import java.util.Vector;
/**
* Created by Christian Schabesberger on 26.08.15.
@@ -29,20 +28,60 @@ import java.util.List;
@SuppressWarnings("ALL")
public class VideoInfo extends AbstractVideoInfo {
// If a video could not be parsed, this predefined error codes
// will be returned AND can be parsed by the frontend of the app.
// Error codes:
public final static int NO_ERROR = 0x0;
public final static int ERROR_NO_SPECIFIED_ERROR = 0x1;
// GEMA a german music colecting society.
public final static int ERROR_BLOCKED_BY_GEMA = 0x2;
/**Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses*/
public static VideoInfo getVideoInfo(VideoExtractor extractor, Downloader downloader)
throws CrawlingException, IOException {
VideoInfo videoInfo = new VideoInfo();
VideoUrlIdHandler uiconv = extractor.getUrlIdConverter();
videoInfo.webpage_url = extractor.getPageUrl();
videoInfo.title = extractor.getTitle();
videoInfo.duration = extractor.getLength();
videoInfo.uploader = extractor.getUploader();
videoInfo.description = extractor.getDescription();
videoInfo.view_count = extractor.getViews();
videoInfo.upload_date = extractor.getUploadDate();
videoInfo.thumbnail_url = extractor.getThumbnailUrl();
videoInfo.id = uiconv.getVideoId(extractor.getPageUrl());
videoInfo.dashMpdUrl = extractor.getDashMpdUrl();
/** Load and extract audio*/
videoInfo.audioStreams = extractor.getAudioStreams();
if(videoInfo.dashMpdUrl != null && !videoInfo.dashMpdUrl.isEmpty()) {
if(videoInfo.audioStreams == null) {
videoInfo.audioStreams = new Vector<AudioStream>();
}
videoInfo.audioStreams.addAll(
DashMpdParser.getAudioStreams(videoInfo.dashMpdUrl, downloader));
}
/** Extract video stream url*/
videoInfo.videoStreams = extractor.getVideoStreams();
videoInfo.uploader_thumbnail_url = extractor.getUploaderThumbnailUrl();
videoInfo.startPosition = extractor.getTimeStamp();
videoInfo.average_rating = extractor.getAverageRating();
videoInfo.like_count = extractor.getLikeCount();
videoInfo.dislike_count = extractor.getDislikeCount();
videoInfo.nextVideo = extractor.getNextVideo();
videoInfo.relatedVideos = extractor.getRelatedVideos();
//Bitmap thumbnail = null;
//Bitmap uploader_thumbnail = null;
//int videoAvailableStatus = VIDEO_AVAILABLE;
return videoInfo;
}
public String uploader_thumbnail_url = "";
public String description = "";
public VideoStream[] videoStreams = null;
public AudioStream[] audioStreams = null;
public int errorCode = NO_ERROR;
public String errorMessage = "";
/*todo: make this lists over vectors*/
public List<VideoStream> videoStreams = null;
public List<AudioStream> audioStreams = null;
// video streams provided by the dash mpd do not need to be provided as VideoStream.
// Later on this will also aplly to audio streams. Since dash mpd is standarized,
// crawling such a file is not service dependent. Therefore getting audio only streams by yust
// providing the dash mpd fille will be possible in the future.
public String dashMpdUrl = "";
public int duration = -1;
/*YouTube-specific fields
@@ -53,11 +92,11 @@ public class VideoInfo extends AbstractVideoInfo {
public String average_rating = "";
public VideoPreviewInfo nextVideo = null;
public List<VideoPreviewInfo> relatedVideos = null;
public int startPosition = -1;//in seconds. some metadata is not passed using a VideoInfo object!
//in seconds. some metadata is not passed using a VideoInfo object!
public int startPosition = -1;
public VideoInfo() {}
/**Creates a new VideoInfo object from an existing AbstractVideoInfo.
* All the shared properties are copied to the new VideoInfo.*/
@SuppressWarnings("WeakerAccess")
@@ -73,7 +112,8 @@ public class VideoInfo extends AbstractVideoInfo {
this.view_count = avi.view_count;
//todo: better than this
if(avi instanceof VideoPreviewInfo) {//shitty String to convert code
if(avi instanceof VideoPreviewInfo) {
//shitty String to convert code
String dur = ((VideoPreviewInfo)avi).duration;
int minutes = Integer.parseInt(dur.substring(0, dur.indexOf(":")));
int seconds = Integer.parseInt(dur.substring(dur.indexOf(":")+1, dur.length()));
@@ -82,7 +122,8 @@ public class VideoInfo extends AbstractVideoInfo {
}
public static class VideoStream {
public String url = ""; //url of the stream
//url of the stream
public String url = "";
public int format = -1;
public String resolution = "";

4
app/src/main/java/org/schabi/newpipe/VideoPreviewInfo.java → app/src/main/java/org/schabi/newpipe/crawler/VideoPreviewInfo.java
View File

@@ -1,10 +1,10 @@
package org.schabi.newpipe;
package org.schabi.newpipe.crawler;
import android.graphics.Bitmap;
import android.os.Parcel;
import android.os.Parcelable;
import org.schabi.newpipe.services.AbstractVideoInfo;
import org.schabi.newpipe.crawler.AbstractVideoInfo;
/**
* Created by Christian Schabesberger on 26.08.15.

32
app/src/main/java/org/schabi/newpipe/crawler/VideoUrlIdHandler.java Normal file
View File

@@ -0,0 +1,32 @@
package org.schabi.newpipe.crawler;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* VideoUrlIdHandler.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public interface VideoUrlIdHandler {
String getVideoUrl(String videoId);
String getVideoId(String siteUrl) throws ParsingException;
String cleanUrl(String siteUrl) throws ParsingException;
/**When a VIEW_ACTION is caught this function will test if the url delivered within the calling
Intent was meant to be watched with this Service.
Return false if this service shall not allow to be called through ACTIONs.*/
boolean acceptUrl(String videoUrl);
}

202
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeSearchEngine.java Normal file
View File

@@ -0,0 +1,202 @@
package org.schabi.newpipe.crawler.services.youtube;
import android.net.Uri;
import android.util.Log;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.SearchEngine;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
/**
* Created by Christian Schabesberger on 09.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeSearchEngine.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeSearchEngine implements SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString();
@Override
public Result search(String query, int page, String languageCode, Downloader downloader)
throws IOException, ParsingException {
Result result = new Result();
Uri.Builder builder = new Uri.Builder();
builder.scheme("https")
.authority("www.youtube.com")
.appendPath("results")
.appendQueryParameter("search_query", query)
.appendQueryParameter("page", Integer.toString(page))
.appendQueryParameter("filters", "video");
String site;
String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL
if(!languageCode.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = downloader.download(url, languageCode);
}
else {
site = downloader.download(url);
}
try {
Document doc = Jsoup.parse(site, url);
Element list = doc.select("ol[class=\"item-section\"]").first();
for (Element item : list.children()) {
/* First we need to determine which kind of item we are working with.
Youtube depicts five different kinds of items on its search result page. These are
regular videos, playlists, channels, two types of video suggestions, and a "no video
found" item. Since we only want videos, we need to filter out all the others.
An example for this can be seen here:
https://www.youtube.com/results?search_query=asdf&page=1
We already applied a filter to the url, so we don't need to care about channels and
playlists now.
*/
Element el;
// both types of spell correction item
if (!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) {
result.suggestion = el.select("a").first().text();
// search message item
} else if (!((el = item.select("div[class*=\"search-message\"]").first()) == null)) {
result.errorMessage = el.text();
// video item type
} else if (!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) {
VideoPreviewInfo resultItem = new VideoPreviewInfo();
Element dl = el.select("h3").first().select("a").first();
resultItem.webpage_url = dl.attr("abs:href");
try {
Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)");
Matcher m = p.matcher(resultItem.webpage_url);
resultItem.id = m.group(1);
} catch (Exception e) {
//e.printStackTrace();
}
resultItem.title = dl.text();
resultItem.duration = item.select("span[class=\"video-time\"]").first().text();
resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
resultItem.thumbnail_url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if (resultItem.thumbnail_url.contains(".gif")) {
resultItem.thumbnail_url = te.attr("abs:data-thumb");
}
result.resultList.add(resultItem);
} else {
//noinspection ConstantConditions
Log.e(TAG, "unexpected element found:\"" + el + "\"");
}
}
} catch(Exception e) {
throw new ParsingException(e);
}
return result;
}
@Override
public ArrayList<String> suggestionList(String query, Downloader dl)
throws IOException, ParsingException {
ArrayList<String> suggestions = new ArrayList<>();
Uri.Builder builder = new Uri.Builder();
builder.scheme("https")
.authority("suggestqueries.google.com")
.appendPath("complete")
.appendPath("search")
.appendQueryParameter("client", "")
.appendQueryParameter("output", "toolbar")
.appendQueryParameter("ds", "yt")
.appendQueryParameter("q", query);
String url = builder.build().toString();
String response = dl.download(url);
try {
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
try {
dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(new InputSource(
new ByteArrayInputStream(response.getBytes("utf-8"))));
doc.getDocumentElement().normalize();
} catch (ParserConfigurationException | SAXException | IOException e) {
e.printStackTrace();
}
if (doc != null) {
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
}
} else {
Log.e(TAG, "GREAT FUCKING ERROR");
}
return suggestions;
} catch(Exception e) {
throw new ParsingException(e);
}
}
}

27
app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeService.java → app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeService.java
View File

@@ -1,8 +1,13 @@
package org.schabi.newpipe.services.youtube;
package org.schabi.newpipe.crawler.services.youtube;
import org.schabi.newpipe.services.StreamingService;
import org.schabi.newpipe.services.VideoExtractor;
import org.schabi.newpipe.services.SearchEngine;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.StreamingService;
import org.schabi.newpipe.crawler.VideoUrlIdHandler;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.SearchEngine;
import java.io.IOException;
/**
@@ -33,9 +38,11 @@ public class YoutubeService implements StreamingService {
return serviceInfo;
}
@Override
public VideoExtractor getExtractorInstance(String url) {
if(acceptUrl(url)) {
return new YoutubeVideoExtractor(url);
public VideoExtractor getExtractorInstance(String url, Downloader downloader)
throws CrawlingException, IOException {
VideoUrlIdHandler urlIdHandler = new YoutubeVideoUrlIdHandler();
if(urlIdHandler.acceptUrl(url)) {
return new YoutubeVideoExtractor(url, downloader) ;
}
else {
throw new IllegalArgumentException("supplied String is not a valid Youtube URL");
@@ -45,9 +52,9 @@ public class YoutubeService implements StreamingService {
public SearchEngine getSearchEngineInstance() {
return new YoutubeSearchEngine();
}
@Override
public boolean acceptUrl(String videoUrl) {
return videoUrl.contains("youtube") ||
videoUrl.contains("youtu.be");
public VideoUrlIdHandler getUrlIdHandler() {
return new YoutubeVideoUrlIdHandler();
}
}

600
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoExtractor.java Normal file
View File

@@ -0,0 +1,600 @@
package org.schabi.newpipe.crawler.services.youtube;
import android.util.Log;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Parser;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.crawler.CrawlingException;
import org.schabi.newpipe.crawler.Downloader;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.RegexHelper;
import org.schabi.newpipe.crawler.VideoUrlIdHandler;
import org.schabi.newpipe.crawler.VideoExtractor;
import org.schabi.newpipe.crawler.MediaFormat;
import org.schabi.newpipe.crawler.VideoInfo;
import org.schabi.newpipe.crawler.VideoPreviewInfo;
import java.io.IOException;
import java.net.URLDecoder;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Vector;
/**
* Created by Christian Schabesberger on 06.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeVideoExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeVideoExtractor implements VideoExtractor {
public class DecryptException extends ParsingException {
DecryptException(Throwable cause) {
super(cause);
}
DecryptException(String message, Throwable cause) {
super(message, cause);
}
}
// special content not available exceptions
public class GemaException extends ContentNotAvailableException {
GemaException(String message) {
super(message);
}
}
// ----------------
private static final String TAG = YoutubeVideoExtractor.class.toString();
private final Document doc;
private JSONObject playerArgs;
// static values
private static final String DECRYPTION_FUNC_NAME="decrypt";
// cached values
private static volatile String decryptionCode = "";
VideoUrlIdHandler urlidhandler = new YoutubeVideoUrlIdHandler();
String pageUrl = "";
private Downloader downloader;
public YoutubeVideoExtractor(String pageUrl, Downloader dl) throws CrawlingException, IOException {
//most common videoInfo fields are now set in our superclass, for all services
downloader = dl;
this.pageUrl = pageUrl;
String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl));
doc = Jsoup.parse(pageContent, pageUrl);
String ytPlayerConfigRaw;
JSONObject ytPlayerConfig;
//attempt to load the youtube js player JSON arguments
try {
ytPlayerConfigRaw =
RegexHelper.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
ytPlayerConfig = new JSONObject(ytPlayerConfigRaw);
playerArgs = ytPlayerConfig.getJSONObject("args");
} catch (RegexHelper.RegexException e) {
String errorReason = findErrorReason(doc);
switch(errorReason) {
case "GEMA":
throw new GemaException(errorReason);
case "":
throw new ParsingException("player config empty", e);
default:
throw new ContentNotAvailableException("Content not available", e);
}
} catch (JSONException e) {
throw new ParsingException("Could not parse yt player config");
}
//----------------------------------
// load and parse description code, if it isn't already initialised
//----------------------------------
if (decryptionCode.isEmpty()) {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
JSONObject ytAssets = ytPlayerConfig.getJSONObject("assets");
String playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = "https:" + playerUrl;
}
decryptionCode = loadDecryptionCode(playerUrl);
} catch (JSONException e) {
throw new ParsingException(
"Could not load decryption code for the Youtube service.", e);
}
}
}
@Override
public String getTitle() throws ParsingException {
try {//json player args method
return playerArgs.getString("title");
} catch(JSONException je) {//html <meta> method
je.printStackTrace();
Log.w(TAG, "failed to load title from JSON args; trying to extract it from HTML");
try { // fall through to fall-back
return doc.select("meta[name=title]").attr("content");
} catch (Exception e) {
throw new ParsingException("failed permanently to load title.", e);
}
}
}
@Override
public String getDescription() throws ParsingException {
try {
return doc.select("p[id=\"eow-description\"]").first().html();
} catch (Exception e) {//todo: add fallback method <-- there is no ... as long as i know
throw new ParsingException("failed to load description.", e);
}
}
@Override
public String getUploader() throws ParsingException {
try {//json player args method
return playerArgs.getString("author");
} catch(JSONException je) {
je.printStackTrace();
Log.w(TAG,
"failed to load uploader name from JSON args; trying to extract it from HTML");
} try {//fall through to fallback HTML method
return doc.select("div.yt-user-info").first().text();
} catch (Exception e) {
throw new ParsingException("failed permanently to load uploader name.", e);
}
}
@Override
public int getLength() throws ParsingException {
try {
return playerArgs.getInt("length_seconds");
} catch (JSONException e) {//todo: find fallback method
throw new ParsingException("failed to load video duration from JSON args", e);
}
}
@Override
public long getViews() throws ParsingException {
try {
String viewCountString = doc.select("meta[itemprop=interactionCount]").attr("content");
return Long.parseLong(viewCountString);
} catch (Exception e) {//todo: find fallback method
throw new ParsingException("failed to number of views", e);
}
}
@Override
public String getUploadDate() throws ParsingException {
try {
return doc.select("meta[itemprop=datePublished]").attr("content");
} catch (Exception e) {//todo: add fallback method
throw new ParsingException("failed to get upload date.", e);
}
}
@Override
public String getThumbnailUrl() throws ParsingException {
//first attempt getting a small image version
//in the html extracting part we try to get a thumbnail with a higher resolution
// Try to get high resolution thumbnail if it fails use low res from the player instead
try {
return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
} catch(Exception e) {
Log.w(TAG, "Could not find high res Thumbnail. Using low res instead");
} try { //fall through to fallback
return playerArgs.getString("thumbnail_url");
} catch (JSONException je) {
throw new ParsingException(
"failed to extract thumbnail URL from JSON args; trying to extract it from HTML", je);
}
}
@Override
public String getUploaderThumbnailUrl() throws ParsingException {
try {
return doc.select("a[class*=\"yt-user-photo\"]").first()
.select("img").first()
.attr("abs:data-thumb");
} catch (Exception e) {//todo: add fallback method
throw new ParsingException("failed to get uploader thumbnail URL.", e);
}
}
@Override
public String getDashMpdUrl() throws ParsingException {
try {
String dashManifest = playerArgs.getString("dashmpd");
if(!dashManifest.contains("/signature/")) {
String encryptedSig = RegexHelper.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest);
String decryptedSig;
decryptedSig = decryptSignature(encryptedSig, decryptionCode);
dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
}
return dashManifest;
} catch(NullPointerException e) {
throw new ParsingException(
"Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).", e);
} catch (Exception e) {
throw new ParsingException(e);
}
}
@Override
public List<VideoInfo.AudioStream> getAudioStreams() throws ParsingException {
/* If we provide a valid dash manifest, we don't need to provide audio streams extra */
return null;
}
@Override
public List<VideoInfo.VideoStream> getVideoStreams() throws ParsingException {
Vector<VideoInfo.VideoStream> videoStreams = new Vector<>();
try{
String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map");
for(String url_data_str : encoded_url_map.split(",")) {
try {
Map<String, String> tags = new HashMap<>();
for (String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) {
String[] split_tag = raw_tag.split("=");
tags.put(split_tag[0], split_tag[1]);
}
int itag = Integer.parseInt(tags.get("itag"));
String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8");
// if video has a signature: decrypt it and add it to the url
if (tags.get("s") != null) {
streamUrl = streamUrl + "&signature="
+ decryptSignature(tags.get("s"), decryptionCode);
}
if (resolveFormat(itag) != -1) {
videoStreams.add(new VideoInfo.VideoStream(
streamUrl,
resolveFormat(itag),
resolveResolutionString(itag)));
}
} catch (Exception e) {
Log.w(TAG, "Could not get Video stream.");
e.printStackTrace();
}
}
} catch (Exception e) {
throw new ParsingException("Failed to get video streams", e);
}
if(videoStreams.isEmpty()) {
throw new ParsingException("Failed to get any video stream");
}
return videoStreams;
}
/**Attempts to parse (and return) the offset to start playing the video from.
* @return the offset (in seconds), or 0 if no timestamp is found.*/
@Override
public int getTimeStamp() throws ParsingException {
//todo: add unit test for timestamp
String timeStamp;
try {
timeStamp = RegexHelper.matchGroup1("((#|&|\\?)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
} catch (RegexHelper.RegexException e) {
// catch this instantly since an url does not necessarily have to have a time stamp
// -2 because well the testing system will then know its the regex that failed :/
// not good i know
return -2;
}
//TODO: test this
if(!timeStamp.isEmpty()) {
try {
String secondsString = "";
String minutesString = "";
String hoursString = "";
try {
secondsString = RegexHelper.matchGroup1("(\\d{1,3})s", timeStamp);
minutesString = RegexHelper.matchGroup1("(\\d{1,3})m", timeStamp);
hoursString = RegexHelper.matchGroup1("(\\d{1,3})h", timeStamp);
} catch (Exception e) {
//it could be that time is given in another method
if (secondsString.isEmpty() //if nothing was got,
&& minutesString.isEmpty()//treat as unlabelled seconds
&& hoursString.isEmpty()) {
secondsString = RegexHelper.matchGroup1("t=(\\d{1,3})", timeStamp);
}
}
int seconds = (secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString));
int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString));
int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString));
int ret = seconds + (60 * minutes) + (3600 * hours);//don't trust BODMAS!
//Log.d(TAG, "derived timestamp value:"+ret);
return ret;
//the ordering varies internationally
} catch (ParsingException e) {
throw new ParsingException("Could not get timestamp.", e);
}
} else {
return -1;
}
}
@Override
public int getAgeLimit() throws ParsingException {
// Not yet implemented.
// Also you need to be logged in to see age restricted videos on youtube,
// therefore NP is not able to receive such videos.
return 0;
}
@Override
public String getAverageRating() throws ParsingException {
try {
return playerArgs.getString("avg_rating");
} catch (JSONException e) {
throw new ParsingException("Could not get Average rating", e);
}
}
@Override
public int getLikeCount() throws ParsingException {
String likesString = "";
try {
likesString = doc.select("button.like-button-renderer-like-button").first()
.select("span.yt-uix-button-content").first().text();
return Integer.parseInt(likesString.replaceAll("[^\\d]", ""));
} catch (NumberFormatException nfe) {
throw new ParsingException(
"failed to parse likesString \"" + likesString + "\" as integers", nfe);
} catch (Exception e) {
throw new ParsingException("Could not get like count", e);
}
}
@Override
public int getDislikeCount() throws ParsingException {
String dislikesString = "";
try {
dislikesString = doc.select("button.like-button-renderer-dislike-button").first()
.select("span.yt-uix-button-content").first().text();
return Integer.parseInt(dislikesString.replaceAll("[^\\d]", ""));
} catch(NumberFormatException nfe) {
throw new ParsingException(
"failed to parse dislikesString \"" + dislikesString + "\" as integers", nfe);
} catch(Exception e) {
throw new ParsingException("Could not get dislike count", e);
}
}
@Override
public VideoPreviewInfo getNextVideo() throws ParsingException {
try {
return extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first());
} catch(Exception e) {
throw new ParsingException("Could not get next video", e);
}
}
@Override
public Vector<VideoPreviewInfo> getRelatedVideos() throws ParsingException {
try {
Vector<VideoPreviewInfo> relatedVideos = new Vector<>();
for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
relatedVideos.add(extractVideoPreviewInfo(li));
}
}
return relatedVideos;
} catch(Exception e) {
throw new ParsingException("Could not get related videos", e);
}
}
@Override
public VideoUrlIdHandler getUrlIdConverter() {
return new YoutubeVideoUrlIdHandler();
}
@Override
public String getPageUrl() {
return pageUrl;
}
/**Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a VideoPreviewInfo object,
* which is a subset of the fields in a full VideoInfo.*/
private VideoPreviewInfo extractVideoPreviewInfo(Element li) throws ParsingException {
VideoPreviewInfo info = new VideoPreviewInfo();
try {
info.webpage_url = li.select("a.content-link").first()
.attr("abs:href");
info.id = RegexHelper.matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url);
//todo: check NullPointerException causing
info.title = li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
info.view_count = Long.parseLong(li.select("span.view-count")
.first().text().replaceAll("[^\\d]", ""));
} catch (NullPointerException e) {//related videos sometimes have no view count
info.view_count = 0;
}
info.uploader = li.select("span.g-hovercard").first().text();
info.duration = li.select("span.video-time").first().text();
Element img = li.select("img").first();
info.thumbnail_url = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if (info.thumbnail_url.contains(".gif")) {
info.thumbnail_url = img.attr("data-thumb");
}
if (info.thumbnail_url.startsWith("//")) {
info.thumbnail_url = "https:" + info.thumbnail_url;
}
} catch (Exception e) {
throw new ParsingException(e);
}
return info;
}
private String loadDecryptionCode(String playerUrl) throws DecryptException {
String decryptionFuncName;
String decryptionFunc;
String helperObjectName;
String helperObject;
String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}";
String decryptionCode;
try {
String playerCode = downloader.download(playerUrl);
decryptionFuncName =
RegexHelper.matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode);
String functionPattern = "("
+ decryptionFuncName.replace("$", "\\$")
+ "=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})";
decryptionFunc = "var " + RegexHelper.matchGroup1(functionPattern, playerCode) + ";";
helperObjectName = RegexHelper
.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc);
String helperPattern = "(var "
+ helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
helperObject = RegexHelper.matchGroup1(helperPattern, playerCode);
callerFunc = callerFunc.replace("%%", decryptionFuncName);
decryptionCode = helperObject + decryptionFunc + callerFunc;
} catch(IOException ioe) {
throw new DecryptException("Could not load decrypt function", ioe);
} catch(Exception e) {
throw new DecryptException("Could not parse decrypt function ", e);
}
return decryptionCode;
}
private String decryptSignature(String encryptedSig, String decryptionCode)
throws DecryptException{
Context context = Context.enter();
context.setOptimizationLevel(-1);
Object result = null;
try {
ScriptableObject scope = context.initStandardObjects();
context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null);
Function decryptionFunc = (Function) scope.get("decrypt", scope);
result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig});
} catch (Exception e) {
throw new DecryptException(e);
} finally {
Context.exit();
}
return (result == null ? "" : result.toString());
}
private String findErrorReason(Document doc) {
String errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
if(errorMessage.contains("GEMA")) {
// Gema sometimes blocks youtube music content in germany:
// https://www.gema.de/en/
// Detailed description:
// https://en.wikipedia.org/wiki/GEMA_%28German_organization%29
return "GEMA";
}
return "";
}
/**These lists only contain itag formats that are supported by the common Android Video player.
However if you are looking for a list showing all itag formats, look at
https://github.com/rg3/youtube-dl/issues/1687 */
@SuppressWarnings("WeakerAccess")
public static int resolveFormat(int itag) {
switch(itag) {
// !!! lists only supported formats !!!
// video
case 17: return MediaFormat.v3GPP.id;
case 18: return MediaFormat.MPEG_4.id;
case 22: return MediaFormat.MPEG_4.id;
case 36: return MediaFormat.v3GPP.id;
case 37: return MediaFormat.MPEG_4.id;
case 38: return MediaFormat.MPEG_4.id;
case 43: return MediaFormat.WEBM.id;
case 44: return MediaFormat.WEBM.id;
case 45: return MediaFormat.WEBM.id;
case 46: return MediaFormat.WEBM.id;
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return -1;
}
}
@SuppressWarnings("WeakerAccess")
public static String resolveResolutionString(int itag) {
switch(itag) {
case 17: return "144p";
case 18: return "360p";
case 22: return "720p";
case 36: return "240p";
case 37: return "1080p";
case 38: return "1080p";
case 43: return "360p";
case 44: return "480p";
case 45: return "720p";
case 46: return "1080p";
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return null;
}
}
}

68
app/src/main/java/org/schabi/newpipe/crawler/services/youtube/YoutubeVideoUrlIdHandler.java Normal file
View File

@@ -0,0 +1,68 @@
package org.schabi.newpipe.crawler.services.youtube;
import org.schabi.newpipe.crawler.ParsingException;
import org.schabi.newpipe.crawler.RegexHelper;
import org.schabi.newpipe.crawler.VideoUrlIdHandler;
/**
* Created by Christian Schabesberger on 02.02.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
* YoutubeVideoUrlIdHandler.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeVideoUrlIdHandler implements VideoUrlIdHandler {
@SuppressWarnings("WeakerAccess")
@Override
public String getVideoUrl(String videoId) {
return "https://www.youtube.com/watch?v=" + videoId;
}
@SuppressWarnings("WeakerAccess")
@Override
public String getVideoId(String url) throws ParsingException {
String id;
String pat;
if(url.contains("youtube")) {
pat = "youtube\\.com/watch\\?v=([\\-a-zA-Z0-9_]{11})";
}
else if(url.contains("youtu.be")) {
pat = "youtu\\.be/([a-zA-Z0-9_-]{11})";
}
else {
throw new ParsingException("Error no suitable url: " + url);
}
id = RegexHelper.matchGroup1(pat, url);
if(!id.isEmpty()){
//Log.i(TAG, "string \""+url+"\" matches!");
return id;
} else {
throw new ParsingException("Error could not parse url: " + url);
}
}
public String cleanUrl(String complexUrl) throws ParsingException {
return getVideoUrl(getVideoId(complexUrl));
}
@Override
public boolean acceptUrl(String videoUrl) {
return videoUrl.contains("youtube") ||
videoUrl.contains("youtu.be");
}
}

16
app/src/main/java/org/schabi/newpipe/services/AbstractVideoInfo.java
View File

@@ -1,16 +0,0 @@
package org.schabi.newpipe.services;
import android.graphics.Bitmap;
/**Common properties between VideoInfo and VideoPreviewInfo.*/
public abstract class AbstractVideoInfo {
public String id = "";
public String title = "";
public String uploader = "";
//public int duration = -1;
public String thumbnail_url = "";
public Bitmap thumbnail = null;
public String webpage_url = "";
public String upload_date = "";
public long view_count = -1;
}

128
app/src/main/java/org/schabi/newpipe/services/VideoExtractor.java
View File

@@ -1,128 +0,0 @@
package org.schabi.newpipe.services;
/**
* Created by Christian Schabesberger on 10.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* VideoExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
/**Scrapes information from a video streaming service (eg, YouTube).*/
@SuppressWarnings("ALL")
public abstract class VideoExtractor {
protected final String pageUrl;
protected VideoInfo videoInfo;
@SuppressWarnings("WeakerAccess")
public VideoExtractor(String url) {
this.pageUrl = url;
}
/**Fills out the video info fields which are common to all services.
* Probably needs to be overridden by subclasses*/
public VideoInfo getVideoInfo()
{
if(videoInfo == null) {
videoInfo = new VideoInfo();
}
if(videoInfo.webpage_url.isEmpty()) {
videoInfo.webpage_url = pageUrl;
}
if(getErrorCode() == VideoInfo.NO_ERROR) {
if (videoInfo.title.isEmpty()) {
videoInfo.title = getTitle();
}
if (videoInfo.duration < 1) {
videoInfo.duration = getLength();
}
if (videoInfo.uploader.isEmpty()) {
videoInfo.uploader = getUploader();
}
if (videoInfo.description.isEmpty()) {
videoInfo.description = getDescription();
}
if (videoInfo.view_count == -1) {
videoInfo.view_count = getViews();
}
if (videoInfo.upload_date.isEmpty()) {
videoInfo.upload_date = getUploadDate();
}
if (videoInfo.thumbnail_url.isEmpty()) {
videoInfo.thumbnail_url = getThumbnailUrl();
}
if (videoInfo.id.isEmpty()) {
videoInfo.id = getVideoId(pageUrl);
}
/** Load and extract audio*/
if (videoInfo.audioStreams == null) {
videoInfo.audioStreams = getAudioStreams();
}
/** Extract video stream url*/
if (videoInfo.videoStreams == null) {
videoInfo.videoStreams = getVideoStreams();
}
if (videoInfo.uploader_thumbnail_url.isEmpty()) {
videoInfo.uploader_thumbnail_url = getUploaderThumbnailUrl();
}
if (videoInfo.startPosition < 0) {
videoInfo.startPosition = getTimeStamp();
}
} else {
videoInfo.errorCode = getErrorCode();
videoInfo.errorMessage = getErrorMessage();
}
//Bitmap thumbnail = null;
//Bitmap uploader_thumbnail = null;
//int videoAvailableStatus = VIDEO_AVAILABLE;
return videoInfo;
}
//todo: add licence field
public abstract int getErrorCode();
public abstract String getErrorMessage();
//todo: remove these functions, or make them static, otherwise its useles, to have them here
public abstract String getVideoUrl(String videoId);
public abstract String getVideoId(String siteUrl);
///////////////////////////////////////////////////////////////////////////////////////////
public abstract int getTimeStamp();
public abstract String getTitle();
public abstract String getDescription();
public abstract String getUploader();
public abstract int getLength();
public abstract long getViews();
public abstract String getUploadDate();
public abstract String getThumbnailUrl();
public abstract String getUploaderThumbnailUrl();
public abstract VideoInfo.AudioStream[] getAudioStreams();
public abstract VideoInfo.VideoStream[] getVideoStreams();
}

190
app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeSearchEngine.java
View File

@@ -1,190 +0,0 @@
package org.schabi.newpipe.services.youtube;
import android.net.Uri;
import android.util.Log;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.services.SearchEngine;
import org.schabi.newpipe.VideoPreviewInfo;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
/**
* Created by Christian Schabesberger on 09.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeSearchEngine.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeSearchEngine implements SearchEngine {
private static final String TAG = YoutubeSearchEngine.class.toString();
@Override
public Result search(String query, int page, String languageCode) {
//String contentCountry = PreferenceManager.getDefaultSharedPreferences(this).getString(getString(R.string., "");
Uri.Builder builder = new Uri.Builder();
builder.scheme("https")
.authority("www.youtube.com")
.appendPath("results")
.appendQueryParameter("search_query", query)
.appendQueryParameter("page", Integer.toString(page))
.appendQueryParameter("filters", "video");
String site;
String url = builder.build().toString();
//if we've been passed a valid language code, append it to the URL
if(!languageCode.isEmpty()) {
//assert Pattern.matches("[a-z]{2}(-([A-Z]{2}|[0-9]{1,3}))?", languageCode);
site = Downloader.download(url, languageCode);
}
else {
site = Downloader.download(url);
}
Document doc = Jsoup.parse(site, url);
Result result = new Result();
Element list = doc.select("ol[class=\"item-section\"]").first();
int i = 0;
for(Element item : list.children()) {
i++;
/* First we need to determine which kind of item we are working with.
Youtube depicts five different kinds of items on its search result page. These are
regular videos, playlists, channels, two types of video suggestions, and a "no video
found" item. Since we only want videos, we need to filter out all the others.
An example for this can be seen here:
https://www.youtube.com/results?search_query=asdf&page=1
We already applied a filter to the url, so we don't need to care about channels and
playlists now.
*/
Element el;
// both types of spell correction item
if(!((el = item.select("div[class*=\"spell-correction\"]").first()) == null)) {
result.suggestion = el.select("a").first().text();
// search message item
} else if(!((el = item.select("div[class*=\"search-message\"]").first()) == null)) {
result.errorMessage = el.text();
// video item type
} else if(!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) {
VideoPreviewInfo resultItem = new VideoPreviewInfo();
Element dl = el.select("h3").first().select("a").first();
resultItem.webpage_url = dl.attr("abs:href");
try {
Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)");
Matcher m = p.matcher(resultItem.webpage_url);
resultItem.id=m.group(1);
} catch (Exception e) {
//e.printStackTrace();
}
resultItem.title = dl.text();
resultItem.duration = item.select("span[class=\"video-time\"]").first().text();
resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first()
.select("a").first()
.text();
resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first()
.select("li").first()
.text();
Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first()
.select("img").first();
resultItem.thumbnail_url = te.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we've caught such an item.
if(resultItem.thumbnail_url.contains(".gif")) {
resultItem.thumbnail_url = te.attr("abs:data-thumb");
}
result.resultList.add(resultItem);
} else {
//noinspection ConstantConditions
Log.e(TAG, "unexpected element found:\""+el+"\"");
}
}
return result;
}
@Override
public ArrayList<String> suggestionList(String query) {
ArrayList<String> suggestions = new ArrayList<>();
Uri.Builder builder = new Uri.Builder();
builder.scheme("https")
.authority("suggestqueries.google.com")
.appendPath("complete")
.appendPath("search")
.appendQueryParameter("client", "")
.appendQueryParameter("output", "toolbar")
.appendQueryParameter("ds", "yt")
.appendQueryParameter("q", query);
String url = builder.build().toString();
String response = Downloader.download(url);
//TODO: Parse xml data using Jsoup not done
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder;
org.w3c.dom.Document doc = null;
try {
dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("utf-8"))));
doc.getDocumentElement().normalize();
}catch (ParserConfigurationException | SAXException | IOException e) {
e.printStackTrace();
}
if(doc!=null){
NodeList nList = doc.getElementsByTagName("CompleteSuggestion");
for (int temp = 0; temp < nList.getLength(); temp++) {
NodeList nList1 = doc.getElementsByTagName("suggestion");
Node nNode1 = nList1.item(temp);
if (nNode1.getNodeType() == Node.ELEMENT_NODE) {
org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1;
suggestions.add(eElement.getAttribute("data"));
}
}
}else {
Log.e(TAG, "GREAT FUCKING ERROR");
}
return suggestions;
}
}

647
app/src/main/java/org/schabi/newpipe/services/youtube/YoutubeVideoExtractor.java
View File

@@ -1,647 +0,0 @@
package org.schabi.newpipe.services.youtube;
import android.util.Log;
import android.util.Xml;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Parser;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.services.VideoExtractor;
import org.schabi.newpipe.services.MediaFormat;
import org.schabi.newpipe.services.VideoInfo;
import org.schabi.newpipe.VideoPreviewInfo;
import org.xmlpull.v1.XmlPullParser;
import java.io.StringReader;
import java.net.URLDecoder;
import java.util.HashMap;
import java.util.Map;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Created by Christian Schabesberger on 06.08.15.
*
* Copyright (C) Christian Schabesberger 2015 <chris.schabesberger@mailbox.org>
* YoutubeVideoExtractor.java is part of NewPipe.
*
* NewPipe is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* NewPipe is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeVideoExtractor extends VideoExtractor {
private static final String TAG = YoutubeVideoExtractor.class.toString();
private final Document doc;
private JSONObject jsonObj;
private JSONObject playerArgs;
private int errorCode = VideoInfo.NO_ERROR;
private String errorMessage = "";
// static values
private static final String DECRYPTION_FUNC_NAME="decrypt";
// cached values
private static volatile String decryptionCode = "";
public YoutubeVideoExtractor(String pageUrl) {
super(pageUrl);//most common videoInfo fields are now set in our superclass, for all services
String pageContent = Downloader.download(cleanUrl(pageUrl));
doc = Jsoup.parse(pageContent, pageUrl);
//attempt to load the youtube js player JSON arguments
try {
String jsonString = matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
//todo: implement this by try and catch. TESTING THE STRING AGAINST EMPTY IS CONSIDERED POOR STYLE !!!
if(jsonString.isEmpty()) {
errorCode = findErrorReason(doc);
return;
}
jsonObj = new JSONObject(jsonString);
playerArgs = jsonObj.getJSONObject("args");
} catch (Exception e) {//if this fails, the video is most likely not available.
// Determining why is done later.
videoInfo.errorCode = VideoInfo.ERROR_NO_SPECIFIED_ERROR;
Log.e(TAG, "Could not load JSON data for Youtube video \""+pageUrl+"\". This most likely means the video is unavailable");
}
//----------------------------------
// load and parse description code, if it isn't already initialised
//----------------------------------
if (decryptionCode.isEmpty()) {
try {
// The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream urls.
JSONObject ytAssets = jsonObj.getJSONObject("assets");
String playerUrl = ytAssets.getString("js");
if (playerUrl.startsWith("//")) {
playerUrl = "https:" + playerUrl;
}
decryptionCode = loadDecryptionCode(playerUrl);
} catch (Exception e){
Log.e(TAG, "Could not load decryption code for the Youtube service.");
e.printStackTrace();
}
}
}
@Override
public String getTitle() {
try {//json player args method
return playerArgs.getString("title");
} catch(JSONException je) {//html <meta> method
je.printStackTrace();
Log.w(TAG, "failed to load title from JSON args; trying to extract it from HTML");
} try { // fall through to fall-back
return doc.select("meta[name=title]").attr("content");
} catch (Exception e) {
Log.e(TAG, "failed permanently to load title.");
e.printStackTrace();
return "";
}
}
@Override
public String getDescription() {
try {
return doc.select("p[id=\"eow-description\"]").first().html();
} catch (Exception e) {//todo: add fallback method
Log.e(TAG, "failed to load description.");
e.printStackTrace();
return "";
}
}
@Override
public String getUploader() {
try {//json player args method
return playerArgs.getString("author");
} catch(JSONException je) {
je.printStackTrace();
Log.w(TAG, "failed to load uploader name from JSON args; trying to extract it from HTML");
} try {//fall through to fallback HTML method
return doc.select("div.yt-user-info").first().text();
} catch (Exception e) {
e.printStackTrace();
Log.e(TAG, "failed permanently to load uploader name.");
return "";
}
}
@Override
public int getLength() {
try {
return playerArgs.getInt("length_seconds");
} catch (JSONException je) {//todo: find fallback method
Log.e(TAG, "failed to load video duration from JSON args");
je.printStackTrace();
return -1;
}
}
@Override
public long getViews() {
try {
String viewCountString = doc.select("meta[itemprop=interactionCount]").attr("content");
return Long.parseLong(viewCountString);
} catch (Exception e) {//todo: find fallback method
Log.e(TAG, "failed to number of views");
e.printStackTrace();
return -1;
}
}
@Override
public String getUploadDate() {
try {
return doc.select("meta[itemprop=datePublished]").attr("content");
} catch (Exception e) {//todo: add fallback method
Log.e(TAG, "failed to get upload date.");
e.printStackTrace();
return "";
}
}
@Override
public String getThumbnailUrl() {
//first attempt getting a small image version
//in the html extracting part we try to get a thumbnail with a higher resolution
// Try to get high resolution thumbnail if it fails use low res from the player instead
try {
return doc.select("link[itemprop=\"thumbnailUrl\"]").first().attr("abs:href");
} catch(Exception e) {
Log.w(TAG, "Could not find high res Thumbnail. Using low res instead");
//fall through to fallback
} try {
return playerArgs.getString("thumbnail_url");
} catch (JSONException je) {
je.printStackTrace();
Log.w(TAG, "failed to extract thumbnail URL from JSON args; trying to extract it from HTML");
return "";
}
}
@Override
public String getUploaderThumbnailUrl() {
try {
return doc.select("a[class*=\"yt-user-photo\"]").first()
.select("img").first()
.attr("abs:data-thumb");
} catch (Exception e) {//todo: add fallback method
Log.e(TAG, "failed to get uploader thumbnail URL.");
e.printStackTrace();
return "";
}
}
@Override
public VideoInfo.AudioStream[] getAudioStreams() {
try {
String dashManifest = playerArgs.getString("dashmpd");
return parseDashManifest(dashManifest, decryptionCode);
} catch (NullPointerException e) {
Log.e(TAG, "Could not find \"dashmpd\" upon the player args (maybe no dash manifest available).");
} catch (Exception e) {
e.printStackTrace();
}
return new VideoInfo.AudioStream[0];
}
@Override
public VideoInfo.VideoStream[] getVideoStreams() {
try{
//------------------------------------
// extract video stream url
//------------------------------------
String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map");
Vector<VideoInfo.VideoStream> videoStreams = new Vector<>();
for(String url_data_str : encoded_url_map.split(",")) {
Map<String, String> tags = new HashMap<>();
for(String raw_tag : Parser.unescapeEntities(url_data_str, true).split("&")) {
String[] split_tag = raw_tag.split("=");
tags.put(split_tag[0], split_tag[1]);
}
int itag = Integer.parseInt(tags.get("itag"));
String streamUrl = URLDecoder.decode(tags.get("url"), "UTF-8");
// if video has a signature: decrypt it and add it to the url
if(tags.get("s") != null) {
streamUrl = streamUrl + "&signature=" + decryptSignature(tags.get("s"), decryptionCode);
}
if(resolveFormat(itag) != -1) {
videoStreams.add(new VideoInfo.VideoStream(
streamUrl,
resolveFormat(itag),
resolveResolutionString(itag)));
}
}
return videoStreams.toArray(new VideoInfo.VideoStream[videoStreams.size()]);
} catch (Exception e) {
Log.e(TAG, "Failed to get video stream");
e.printStackTrace();
return new VideoInfo.VideoStream[0];
}
}
/**These lists only contain itag formats that are supported by the common Android Video player.
However if you are looking for a list showing all itag formats, look at
https://github.com/rg3/youtube-dl/issues/1687 */
@SuppressWarnings("WeakerAccess")
public static int resolveFormat(int itag) {
switch(itag) {
// video
case 17: return MediaFormat.v3GPP.id;
case 18: return MediaFormat.MPEG_4.id;
case 22: return MediaFormat.MPEG_4.id;
case 36: return MediaFormat.v3GPP.id;
case 37: return MediaFormat.MPEG_4.id;
case 38: return MediaFormat.MPEG_4.id;
case 43: return MediaFormat.WEBM.id;
case 44: return MediaFormat.WEBM.id;
case 45: return MediaFormat.WEBM.id;
case 46: return MediaFormat.WEBM.id;
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return -1;
}
}
@SuppressWarnings("WeakerAccess")
public static String resolveResolutionString(int itag) {
switch(itag) {
case 17: return "144p";
case 18: return "360p";
case 22: return "720p";
case 36: return "240p";
case 37: return "1080p";
case 38: return "1080p";
case 43: return "360p";
case 44: return "480p";
case 45: return "720p";
case 46: return "1080p";
default:
//Log.i(TAG, "Itag " + Integer.toString(itag) + " not known or not supported.");
return null;
}
}
@SuppressWarnings("WeakerAccess")
@Override
public String getVideoId(String url) {
String id;
String pat;
if(url.contains("youtube")) {
pat = "youtube\\.com/watch\\?v=([\\-a-zA-Z0-9_]{11})";
}
else if(url.contains("youtu.be")) {
pat = "youtu\\.be/([a-zA-Z0-9_-]{11})";
}
else {
Log.e(TAG, "Error could not parse url: " + url);
return "";
}
id = matchGroup1(pat, url);
if(!id.isEmpty()){
//Log.i(TAG, "string \""+url+"\" matches!");
return id;
}
//Log.i(TAG, "string \""+url+"\" does not match.");
return "";
}
@SuppressWarnings("WeakerAccess")
@Override
public String getVideoUrl(String videoId) {
return "https://www.youtube.com/watch?v=" + videoId;
}
/**Attempts to parse (and return) the offset to start playing the video from.
* @return the offset (in seconds), or 0 if no timestamp is found.*/
@Override
public int getTimeStamp(){
String timeStamp = matchGroup1("((#|&)t=\\d{0,3}h?\\d{0,3}m?\\d{1,3}s?)", pageUrl);
//TODO: test this
if(!timeStamp.isEmpty()) {
String secondsString = matchGroup1("(\\d{1,3})s", timeStamp);
String minutesString = matchGroup1("(\\d{1,3})m", timeStamp);
String hoursString = matchGroup1("(\\d{1,3})h", timeStamp);
if(secondsString.isEmpty()//if nothing was got,
&& minutesString.isEmpty()//treat as unlabelled seconds
&& hoursString.isEmpty())
secondsString = matchGroup1("t=(\\d{1,3})", timeStamp);
int seconds = (secondsString.isEmpty() ? 0 : Integer.parseInt(secondsString));
int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString));
int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString));
int ret = seconds + (60*minutes) + (3600*hours);//don't trust BODMAS!
//Log.d(TAG, "derived timestamp value:"+ret);
return ret;
//the ordering varies internationally
}//else, return default 0
return 0;
}
@Override
public VideoInfo getVideoInfo() {
//todo: @medovax i like your work, but what the fuck:
videoInfo = super.getVideoInfo();
if(errorCode == VideoInfo.NO_ERROR) {
//todo: replace this with a call to getVideoId, if possible
videoInfo.id = matchGroup1("v=([0-9a-zA-Z_-]{11})", pageUrl);
if (videoInfo.audioStreams == null
|| videoInfo.audioStreams.length == 0) {
Log.e(TAG, "uninitialised audio streams!");
}
if (videoInfo.videoStreams == null
|| videoInfo.videoStreams.length == 0) {
Log.e(TAG, "uninitialised video streams!");
}
videoInfo.age_limit = 0;
//average rating
try {
videoInfo.average_rating = playerArgs.getString("avg_rating");
} catch (JSONException e) {
e.printStackTrace();
}
//---------------------------------------
// extracting information from html page
//---------------------------------------
/* Code does not work here anymore.
// Determine what went wrong when the Video is not available
if(videoInfo.errorCode == VideoInfo.ERROR_NO_SPECIFIED_ERROR) {
if(doc.select("h1[id=\"unavailable-message\"]").first().text().contains("GEMA")) {
videoInfo.videoAvailableStatus = VideoInfo.VIDEO_UNAVAILABLE_GEMA;
}
}
*/
String likesString = "";
String dislikesString = "";
try {
// likes
likesString = doc.select("button.like-button-renderer-like-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.like_count = Integer.parseInt(likesString.replaceAll("[^\\d]", ""));
// dislikes
dislikesString = doc.select("button.like-button-renderer-dislike-button").first()
.select("span.yt-uix-button-content").first().text();
videoInfo.dislike_count = Integer.parseInt(dislikesString.replaceAll("[^\\d]", ""));
} catch (NumberFormatException nfe) {
Log.e(TAG, "failed to parse likesString \"" + likesString + "\" and dislikesString \"" +
dislikesString + "\" as integers");
} catch (Exception e) {
// if it fails we know that the video does not offer dislikes.
e.printStackTrace();
videoInfo.like_count = 0;
videoInfo.dislike_count = 0;
}
// next video
videoInfo.nextVideo = extractVideoPreviewInfo(doc.select("div[class=\"watch-sidebar-section\"]").first()
.select("li").first());
// related videos
Vector<VideoPreviewInfo> relatedVideos = new Vector<>();
for (Element li : doc.select("ul[id=\"watch-related\"]").first().children()) {
// first check if we have a playlist. If so leave them out
if (li.select("a[class*=\"content-link\"]").first() != null) {
relatedVideos.add(extractVideoPreviewInfo(li));
}
}
//todo: replace conversion
videoInfo.relatedVideos = relatedVideos;
//videoInfo.relatedVideos = relatedVideos.toArray(new VideoPreviewInfo[relatedVideos.size()]);
}
return videoInfo;
}
@Override
public int getErrorCode() {
return errorCode;
}
@Override
public String getErrorMessage() {
return errorMessage;
}
private VideoInfo.AudioStream[] parseDashManifest(String dashManifest, String decryptoinCode) {
if(!dashManifest.contains("/signature/")) {
String encryptedSig = matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifest);
String decryptedSig;
decryptedSig = decryptSignature(encryptedSig, decryptoinCode);
dashManifest = dashManifest.replace("/s/" + encryptedSig, "/signature/" + decryptedSig);
}
String dashDoc = Downloader.download(dashManifest);
Vector<VideoInfo.AudioStream> audioStreams = new Vector<>();
try {
XmlPullParser parser = Xml.newPullParser();
parser.setInput(new StringReader(dashDoc));
String tagName = "";
String currentMimeType = "";
int currentBandwidth = -1;
int currentSamplingRate = -1;
boolean currentTagIsBaseUrl = false;
for(int eventType = parser.getEventType();
eventType != XmlPullParser.END_DOCUMENT;
eventType = parser.next() ) {
switch(eventType) {
case XmlPullParser.START_TAG:
tagName = parser.getName();
if(tagName.equals("AdaptationSet")) {
currentMimeType = parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "mimeType");
} else if(tagName.equals("Representation") && currentMimeType.contains("audio")) {
currentBandwidth = Integer.parseInt(
parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "bandwidth"));
currentSamplingRate = Integer.parseInt(
parser.getAttributeValue(XmlPullParser.NO_NAMESPACE, "audioSamplingRate"));
} else if(tagName.equals("BaseURL")) {
currentTagIsBaseUrl = true;
}
break;
case XmlPullParser.TEXT:
if(currentTagIsBaseUrl &&
(currentMimeType.contains("audio"))) {
int format = -1;
if(currentMimeType.equals(MediaFormat.WEBMA.mimeType)) {
format = MediaFormat.WEBMA.id;
} else if(currentMimeType.equals(MediaFormat.M4A.mimeType)) {
format = MediaFormat.M4A.id;
}
audioStreams.add(new VideoInfo.AudioStream(parser.getText(),
format, currentBandwidth, currentSamplingRate));
}
//missing break here?
case XmlPullParser.END_TAG:
if(tagName.equals("AdaptationSet")) {
currentMimeType = "";
} else if(tagName.equals("BaseURL")) {
currentTagIsBaseUrl = false;
}//no break needed here
}
}
} catch(Exception e) {
e.printStackTrace();
}
return audioStreams.toArray(new VideoInfo.AudioStream[audioStreams.size()]);
}
/**Provides information about links to other videos on the video page, such as related videos.
* This is encapsulated in a VideoPreviewInfo object,
* which is a subset of the fields in a full VideoInfo.*/
private VideoPreviewInfo extractVideoPreviewInfo(Element li) {
VideoPreviewInfo info = new VideoPreviewInfo();
info.webpage_url = li.select("a.content-link").first()
.attr("abs:href");
try {
info.id = matchGroup1("v=([0-9a-zA-Z-]*)", info.webpage_url);
} catch (Exception e) {
e.printStackTrace();
}
//todo: check NullPointerException causing
info.title = li.select("span.title").first().text();
//this page causes the NullPointerException, after finding it by searching for "tjvg":
//https://www.youtube.com/watch?v=Uqg0aEhLFAg
//this line is unused
//String views = li.select("span.view-count").first().text();
//Log.i(TAG, "title:"+info.title);
//Log.i(TAG, "view count:"+views);
try {
info.view_count = Long.parseLong(li.select("span.view-count")
.first().text().replaceAll("[^\\d]", ""));
} catch (NullPointerException e) {//related videos sometimes have no view count
info.view_count = 0;
}
info.uploader = li.select("span.g-hovercard").first().text();
info.duration = li.select("span.video-time").first().text();
Element img = li.select("img").first();
info.thumbnail_url = img.attr("abs:src");
// Sometimes youtube sends links to gif files which somehow seem to not exist
// anymore. Items with such gif also offer a secondary image source. So we are going
// to use that if we caught such an item.
if(info.thumbnail_url.contains(".gif")) {
info.thumbnail_url = img.attr("data-thumb");
}
if(info.thumbnail_url.startsWith("//")) {
info.thumbnail_url = "https:" + info.thumbnail_url;
}
return info;
}
private String loadDecryptionCode(String playerUrl) {
String playerCode = Downloader.download(playerUrl);
String decryptionFuncName = "";
String decryptionFunc = "";
String helperObjectName;
String helperObject = "";
String callerFunc = "function " + DECRYPTION_FUNC_NAME + "(a){return %%(a);}";
String decryptionCode;
try {
decryptionFuncName = matchGroup1("\\.sig\\|\\|([a-zA-Z0-9$]+)\\(", playerCode);
String functionPattern = "(" + decryptionFuncName.replace("$", "\\$") +"=function\\([a-zA-Z0-9_]*\\)\\{.+?\\})";
decryptionFunc = "var " + matchGroup1(functionPattern, playerCode) + ";";
helperObjectName = matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunc);
String helperPattern = "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
helperObject = matchGroup1(helperPattern, playerCode);
} catch (Exception e) {
e.printStackTrace();
}
callerFunc = callerFunc.replace("%%", decryptionFuncName);
decryptionCode = helperObject + decryptionFunc + callerFunc;
return decryptionCode;
}
private String decryptSignature(String encryptedSig, String decryptionCode) {
Context context = Context.enter();
context.setOptimizationLevel(-1);
Object result = null;
try {
ScriptableObject scope = context.initStandardObjects();
context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null);
Function decryptionFunc = (Function) scope.get("decrypt", scope);
result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig});
} catch (Exception e) {
e.printStackTrace();
}
Context.exit();
return (result == null ? "" : result.toString());
}
private String cleanUrl(String complexUrl) {
return getVideoUrl(getVideoId(complexUrl));
}
private String matchGroup1(String pattern, String input) {
Pattern pat = Pattern.compile(pattern);
Matcher mat = pat.matcher(input);
boolean foundMatch = mat.find();
if (foundMatch) {
return mat.group(1);
}
else {
Log.e(TAG, "failed to find pattern \""+pattern+"\" inside of \""+input+"\"");
new Exception("failed to find pattern \""+pattern+"\"").printStackTrace();
return "";
}
}
private int findErrorReason(Document doc) {
errorMessage = doc.select("h1[id=\"unavailable-message\"]").first().text();
if(errorMessage.contains("GEMA")) {
return VideoInfo.ERROR_BLOCKED_BY_GEMA;
}
return VideoInfo.ERROR_NO_SPECIFIED_ERROR;
}
}

6
app/src/main/res/values/strings.xml
View File

@@ -65,7 +65,13 @@
<string name="background_player_playing_toast">Playing in background</string>
<string name="c3s_url" translatable="false">https://www.c3s.cc/</string>
<string name="play_btn_text">Play</string>
<string name="general_error">Error</string>
<string name="network_error">Network error</string>
<string name="could_not_load_thumbnails">Could not load Thumbnails</string>
<string name="youtube_signature_decryption_error">Could not decrypt video url signature.</string>
<string name="parsing_error">Could not parse website.</string>
<string name="content_not_available">Content not available.</string>
<string name="blocked_by_gema">Blocked by GEMA.</string>
<!-- Content descriptions (for better accessibility) -->
<string name="list_thumbnail_view_description">Video preview thumbnail</string>