From 5eef116aaae1e4a56839374e0b1c55c46efc8c1a Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Wed, 28 Sep 2016 12:53:23 +0200 Subject: [PATCH] put Suggestion extraction into its own class --- .../newpipe/extractor/SearchEngine.java | 4 - .../newpipe/extractor/StreamingService.java | 1 + .../extractor/SuggestionExtractor.java | 41 +++++++ .../services/youtube/YoutubeSearchEngine.java | 49 --------- .../services/youtube/YoutubeService.java | 6 ++ .../youtube/YoutubeSuggestionExtractor.java | 100 ++++++++++++++++++ .../SuggestionSearchRunnable.java | 7 +- 7 files changed, 152 insertions(+), 56 deletions(-) create mode 100644 app/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java create mode 100644 app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java diff --git a/app/src/main/java/org/schabi/newpipe/extractor/SearchEngine.java b/app/src/main/java/org/schabi/newpipe/extractor/SearchEngine.java index f7586a304..d27198c1a 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/SearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/SearchEngine.java @@ -41,10 +41,6 @@ public abstract class SearchEngine { return collector; } - public abstract List suggestionList( - String query,String contentCountry) - throws ExtractionException, IOException; - //Result search(String query, int page); public abstract StreamPreviewInfoSearchCollector search( String query, int page, String contentCountry) diff --git a/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index eea1f1783..ac90e60d0 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -42,6 +42,7 @@ public abstract class StreamingService { public abstract UrlIdHandler getChannelUrlIdHandlerInstance(); public abstract ChannelExtractor getChannelExtractorInstance(String url, int page) throws ExtractionException, IOException; + public abstract SuggestionExtractor getSuggestionExtractorInstance(); public final int getServiceId() { return serviceId; diff --git a/app/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java new file mode 100644 index 000000000..d0e55d5d9 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/extractor/SuggestionExtractor.java @@ -0,0 +1,41 @@ +package org.schabi.newpipe.extractor; + +import java.io.IOException; +import java.util.List; + +/** + * Created by Christian Schabesberger on 28.09.16. + * + * Copyright (C) Christian Schabesberger 2016 + * SuggestionExtractor.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public abstract class SuggestionExtractor { + + private int serviceId; + + public SuggestionExtractor(int serviceId) { + this.serviceId = serviceId; + } + + public abstract List suggestionList( + String query,String contentCountry) + throws ExtractionException, IOException; + + public int getServiceId() { + return serviceId; + } +} diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java index cf79b247a..31d233438 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java @@ -118,55 +118,6 @@ public class YoutubeSearchEngine extends SearchEngine { return collector; } - @Override - public List suggestionList(String query, String contentCountry) - throws IOException, ParsingException { - - List suggestions = new ArrayList<>(); - - Downloader dl = NewPipe.getDownloader(); - - String url = "https://suggestqueries.google.com/complete/search" - + "?client=" + "" - + "&output=" + "toolbar" - + "&ds=" + "yt" - + "&hl=" + URLEncoder.encode(contentCountry, CHARSET_UTF_8) - + "&q=" + URLEncoder.encode(query, CHARSET_UTF_8); - - - String response = dl.download(url); - - //TODO: Parse xml data using Jsoup not done - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder; - org.w3c.dom.Document doc = null; - - try { - dBuilder = dbFactory.newDocumentBuilder(); - doc = dBuilder.parse(new InputSource( - new ByteArrayInputStream(response.getBytes(CHARSET_UTF_8)))); - doc.getDocumentElement().normalize(); - } catch (ParserConfigurationException | SAXException | IOException e) { - throw new ParsingException("Could not parse document."); - } - - try { - NodeList nList = doc.getElementsByTagName("CompleteSuggestion"); - for (int temp = 0; temp < nList.getLength(); temp++) { - - NodeList nList1 = doc.getElementsByTagName("suggestion"); - Node nNode1 = nList1.item(temp); - if (nNode1.getNodeType() == Node.ELEMENT_NODE) { - org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1; - suggestions.add(eElement.getAttribute("data")); - } - } - return suggestions; - } catch(Exception e) { - throw new ParsingException("Could not get suggestions form document.", e); - } - } - private StreamPreviewInfoExtractor extractPreviewInfo(final Element item) { return new YoutubeStreamPreviewInfoExtractor(item); } diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index cd4197474..41ffe4a6d 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -5,6 +5,7 @@ import org.schabi.newpipe.extractor.ExtractionException; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.StreamExtractor; import org.schabi.newpipe.extractor.StreamingService; +import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.extractor.UrlIdHandler; import org.schabi.newpipe.extractor.SearchEngine; @@ -74,4 +75,9 @@ public class YoutubeService extends StreamingService { throws ExtractionException, IOException { return new YoutubeChannelExtractor(getChannelUrlIdHandlerInstance(), url, page, getServiceId()); } + + @Override + public SuggestionExtractor getSuggestionExtractorInstance() { + return new YoutubeSuggestionExtractor(getServiceId()); + } } diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java new file mode 100644 index 000000000..ea5f9ecbb --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSuggestionExtractor.java @@ -0,0 +1,100 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.ExtractionException; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.ParsingException; +import org.schabi.newpipe.extractor.SuggestionExtractor; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; +import java.util.Vector; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +/** + * Created by Christian Schabesberger on 28.09.16. + * + * Copyright (C) Christian Schabesberger 2015 + * YoutubeSuggestionExtractor.java is part of NewPipe. + * + * NewPipe is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe. If not, see . + */ + +public class YoutubeSuggestionExtractor extends SuggestionExtractor { + + public static final String CHARSET_UTF_8 = "UTF-8"; + + public YoutubeSuggestionExtractor(int serviceId) { + super(serviceId); + } + + @Override + public List suggestionList( + String query, String contentCountry) + throws ExtractionException, IOException { + List suggestions = new ArrayList<>(); + + Downloader dl = NewPipe.getDownloader(); + + String url = "https://suggestqueries.google.com/complete/search" + + "?client=" + "" + + "&output=" + "toolbar" + + "&ds=" + "yt" + + "&hl=" + URLEncoder.encode(contentCountry, CHARSET_UTF_8) + + "&q=" + URLEncoder.encode(query, CHARSET_UTF_8); + + + String response = dl.download(url); + + //TODO: Parse xml data using Jsoup not done + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder; + org.w3c.dom.Document doc = null; + + try { + dBuilder = dbFactory.newDocumentBuilder(); + doc = dBuilder.parse(new InputSource( + new ByteArrayInputStream(response.getBytes(CHARSET_UTF_8)))); + doc.getDocumentElement().normalize(); + } catch (ParserConfigurationException | SAXException | IOException e) { + throw new ParsingException("Could not parse document."); + } + + try { + NodeList nList = doc.getElementsByTagName("CompleteSuggestion"); + for (int temp = 0; temp < nList.getLength(); temp++) { + + NodeList nList1 = doc.getElementsByTagName("suggestion"); + Node nNode1 = nList1.item(temp); + if (nNode1.getNodeType() == Node.ELEMENT_NODE) { + org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode1; + suggestions.add(eElement.getAttribute("data")); + } + } + return suggestions; + } catch(Exception e) { + throw new ParsingException("Could not get suggestions form document.", e); + } + } +} diff --git a/app/src/main/java/org/schabi/newpipe/search_fragment/SuggestionSearchRunnable.java b/app/src/main/java/org/schabi/newpipe/search_fragment/SuggestionSearchRunnable.java index bd483f1bf..33fe5de44 100644 --- a/app/src/main/java/org/schabi/newpipe/search_fragment/SuggestionSearchRunnable.java +++ b/app/src/main/java/org/schabi/newpipe/search_fragment/SuggestionSearchRunnable.java @@ -8,6 +8,7 @@ import android.widget.Toast; import org.schabi.newpipe.Downloader; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.SuggestionExtractor; import org.schabi.newpipe.report.ErrorActivity; import org.schabi.newpipe.R; import org.schabi.newpipe.extractor.ExtractionException; @@ -70,13 +71,13 @@ public class SuggestionSearchRunnable implements Runnable{ @Override public void run() { try { - SearchEngine engine = - NewPipe.getService(serviceId).getSearchEngineInstance(); + SuggestionExtractor se = + NewPipe.getService(serviceId).getSuggestionExtractorInstance(); SharedPreferences sp = PreferenceManager.getDefaultSharedPreferences(a); String searchLanguageKey = a.getString(R.string.search_language_key); String searchLanguage = sp.getString(searchLanguageKey, a.getString(R.string.default_language_value)); - List suggestions = engine.suggestionList(query, searchLanguage); + List suggestions = se.suggestionList(query, searchLanguage); h.post(new SuggestionResultRunnable(suggestions, adapter)); } catch (ExtractionException e) { ErrorActivity.reportError(h, a, e, null, a.findViewById(android.R.id.content),