mirrors/NewPipe
1
0
Fork 0
mirror of https://github.com/TeamNewPipe/NewPipe synced 2025-06-27 07:32:54 +00:00
Code Issues Releases Wiki Activity

Merge pull request #11759 from Isira-Seneviratne/Import-export-worker

Browse Source 
Rewrite import and export subscriptions functionality using coroutines
This commit is contained in:
Isira Seneviratne 2025-05-16 01:16:34 +05:30 committed by GitHub
commit f41b34c090
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 568 additions and 1023 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/build.gradle
View File

@ -9,6 +9,7 @@ plugins {
alias libs.plugins.kotlin.compose
alias libs.plugins.kotlin.kapt
alias libs.plugins.kotlin.parcelize
alias libs.plugins.kotlinx.serialization
alias libs.plugins.checkstyle
alias libs.plugins.sonarqube
alias libs.plugins.hilt
@ -16,7 +17,7 @@ plugins {
}
android {
compileSdk 34
compileSdk 35
namespace 'org.schabi.newpipe'
defaultConfig {
@ -226,7 +227,6 @@ dependencies {
implementation libs.androidx.fragment.compose
implementation libs.androidx.lifecycle.livedata
implementation libs.androidx.lifecycle.viewmodel
implementation libs.androidx.localbroadcastmanager
implementation libs.androidx.media
implementation libs.androidx.preference
implementation libs.androidx.recyclerview
@ -319,6 +319,9 @@ dependencies {
// Scroll
implementation libs.lazycolumnscrollbar
// Kotlinx Serialization
implementation libs.kotlinx.serialization.json
/** Debugging **/
// Memory leak detection
debugImplementation libs.leakcanary.object.watcher

15
app/proguard-rules.pro vendored
View File

@ -34,3 +34,18 @@
## For some reason NotificationModeConfigFragment wasn't kept (only referenced in a preference xml)
-keep class org.schabi.newpipe.settings.notifications.** { *; }
## Keep Kotlinx Serialization classes
-keepclassmembers class kotlinx.serialization.json.** {
*** Companion;
}
-keepclasseswithmembers class kotlinx.serialization.json.** {
kotlinx.serialization.KSerializer serializer(...);
}
-keep,includedescriptorclasses class org.schabi.newpipe.**$$serializer { *; }
-keepclassmembers class org.schabi.newpipe.** {
*** Companion;
}
-keepclasseswithmembers class org.schabi.newpipe.** {
kotlinx.serialization.KSerializer serializer(...);
}

7
app/src/main/AndroidManifest.xml
View File

@ -9,6 +9,7 @@
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.SYSTEM_ALERT_WINDOW" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS"/>
<!-- We need to be able to open links in the browser on API 30+ -->
@ -90,8 +91,10 @@
android:exported="false"
android:label="@string/title_activity_about" />
<service android:name=".local.subscription.services.SubscriptionsImportService" />
<service android:name=".local.subscription.services.SubscriptionsExportService" />
<service
android:name="androidx.work.impl.foreground.SystemForegroundService"
android:foregroundServiceType="dataSync"
tools:node="merge" />
<service android:name=".local.feed.service.FeedLoadService" />
<activity

4
app/src/main/java/org/schabi/newpipe/database/subscription/SubscriptionDAO.kt
View File

@ -90,7 +90,7 @@ abstract class SubscriptionDAO : BasicDAO<SubscriptionEntity> {
internal abstract fun silentInsertAllInternal(entities: List<SubscriptionEntity>): List<Long>
@Transaction
open fun upsertAll(entities: List<SubscriptionEntity>): List<SubscriptionEntity> {
open fun upsertAll(entities: List<SubscriptionEntity>) {
val insertUidList = silentInsertAllInternal(entities)
insertUidList.forEachIndexed { index: Int, uidFromInsert: Long ->
@ -106,7 +106,5 @@ abstract class SubscriptionDAO : BasicDAO<SubscriptionEntity> {
update(entity)
}
}
return entities
}
}

55
app/src/main/java/org/schabi/newpipe/local/subscription/ImportConfirmationDialog.java
View File

@ -3,47 +3,64 @@ package org.schabi.newpipe.local.subscription;
import static org.schabi.newpipe.util.Localization.assureCorrectAppLanguage;
import android.app.Dialog;
import android.content.Intent;
import android.os.Bundle;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.appcompat.app.AlertDialog;
import androidx.core.os.BundleCompat;
import androidx.fragment.app.DialogFragment;
import androidx.fragment.app.Fragment;
import androidx.work.Constraints;
import androidx.work.ExistingWorkPolicy;
import androidx.work.NetworkType;
import androidx.work.OneTimeWorkRequest;
import androidx.work.OutOfQuotaPolicy;
import androidx.work.WorkManager;
import com.evernote.android.state.State;
import com.livefront.bridge.Bridge;
import org.schabi.newpipe.R;
import org.schabi.newpipe.local.subscription.workers.SubscriptionImportInput;
import org.schabi.newpipe.local.subscription.workers.SubscriptionImportWorker;
public class ImportConfirmationDialog extends DialogFragment {
@State
protected Intent resultServiceIntent;
private static final String INPUT = "input";
public static void show(@NonNull final Fragment fragment,
@NonNull final Intent resultServiceIntent) {
final ImportConfirmationDialog confirmationDialog = new ImportConfirmationDialog();
confirmationDialog.setResultServiceIntent(resultServiceIntent);
public static void show(@NonNull final Fragment fragment, final SubscriptionImportInput input) {
final var confirmationDialog = new ImportConfirmationDialog();
final var arguments = new Bundle();
arguments.putParcelable(INPUT, input);
confirmationDialog.setArguments(arguments);
confirmationDialog.show(fragment.getParentFragmentManager(), null);
}
public void setResultServiceIntent(final Intent resultServiceIntent) {
this.resultServiceIntent = resultServiceIntent;
}
@NonNull
@Override
public Dialog onCreateDialog(@Nullable final Bundle savedInstanceState) {
assureCorrectAppLanguage(getContext());
return new AlertDialog.Builder(requireContext())
final var context = requireContext();
assureCorrectAppLanguage(context);
return new AlertDialog.Builder(context)
.setMessage(R.string.import_network_expensive_warning)
.setCancelable(true)
.setNegativeButton(R.string.cancel, null)
.setPositiveButton(R.string.ok, (dialogInterface, i) -> {
if (resultServiceIntent != null && getContext() != null) {
getContext().startService(resultServiceIntent);
}
final var constraints = new Constraints.Builder()
.setRequiredNetworkType(NetworkType.CONNECTED)
.build();
final var input = BundleCompat.getParcelable(requireArguments(), INPUT,
SubscriptionImportInput.class);
final var req = new OneTimeWorkRequest.Builder(SubscriptionImportWorker.class)
.setInputData(input.toData())
.setExpedited(OutOfQuotaPolicy.RUN_AS_NON_EXPEDITED_WORK_REQUEST)
.setConstraints(constraints)
.build();
WorkManager.getInstance(context)
.enqueueUniqueWork(SubscriptionImportWorker.WORK_NAME,
ExistingWorkPolicy.APPEND_OR_REPLACE, req);
dismiss();
})
.create();
@ -53,10 +70,6 @@ public class ImportConfirmationDialog extends DialogFragment {
public void onCreate(@Nullable final Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
if (resultServiceIntent == null) {
throw new IllegalStateException("Result intent is null");
}
Bridge.restoreInstanceState(this, savedInstanceState);
}

24
app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionFragment.kt
View File

@ -3,7 +3,6 @@ package org.schabi.newpipe.local.subscription
import android.app.Activity
import android.content.Context
import android.content.DialogInterface
import android.content.Intent
import android.os.Bundle
import android.os.Parcelable
import android.view.LayoutInflater
@ -49,11 +48,8 @@ import org.schabi.newpipe.local.subscription.item.FeedGroupCarouselItem
import org.schabi.newpipe.local.subscription.item.GroupsHeader
import org.schabi.newpipe.local.subscription.item.Header
import org.schabi.newpipe.local.subscription.item.ImportSubscriptionsHintPlaceholderItem
import org.schabi.newpipe.local.subscription.services.SubscriptionsExportService
import org.schabi.newpipe.local.subscription.services.SubscriptionsImportService
import org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.KEY_MODE
import org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.KEY_VALUE
import org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.PREVIOUS_EXPORT_MODE
import org.schabi.newpipe.local.subscription.workers.SubscriptionExportWorker
import org.schabi.newpipe.local.subscription.workers.SubscriptionImportInput
import org.schabi.newpipe.streams.io.NoFileManagerSafeGuard
import org.schabi.newpipe.streams.io.StoredFileHelper
import org.schabi.newpipe.ui.emptystate.setEmptyStateComposable
@ -224,21 +220,17 @@ class SubscriptionFragment : BaseStateFragment<SubscriptionState>() {
}
private fun requestExportResult(result: ActivityResult) {
if (result.data != null && result.resultCode == Activity.RESULT_OK) {
activity.startService(
Intent(activity, SubscriptionsExportService::class.java)
.putExtra(SubscriptionsExportService.KEY_FILE_PATH, result.data?.data)
)
val data = result.data?.data
if (data != null && result.resultCode == Activity.RESULT_OK) {
SubscriptionExportWorker.schedule(activity, data)
}
}
private fun requestImportResult(result: ActivityResult) {
if (result.data != null && result.resultCode == Activity.RESULT_OK) {
val data = result.data?.dataString
if (data != null && result.resultCode == Activity.RESULT_OK) {
ImportConfirmationDialog.show(
this,
Intent(activity, SubscriptionsImportService::class.java)
.putExtra(KEY_MODE, PREVIOUS_EXPORT_MODE)
.putExtra(KEY_VALUE, result.data?.data)
this, SubscriptionImportInput.PreviousExportMode(data)
)
}
}

18
app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionManager.kt
View File

@ -1,7 +1,6 @@
package org.schabi.newpipe.local.subscription
import android.content.Context
import android.util.Pair
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers
import io.reactivex.rxjava3.core.Completable
import io.reactivex.rxjava3.core.Flowable
@ -48,25 +47,18 @@ class SubscriptionManager(context: Context) {
}
}
fun upsertAll(infoList: List<Pair<ChannelInfo, List<ChannelTabInfo>>>): List<SubscriptionEntity> {
val listEntities = subscriptionTable.upsertAll(
infoList.map { SubscriptionEntity.from(it.first) }
)
fun upsertAll(infoList: List<Pair<ChannelInfo, ChannelTabInfo>>) {
val listEntities = infoList.map { SubscriptionEntity.from(it.first) }
subscriptionTable.upsertAll(listEntities)
database.runInTransaction {
infoList.forEachIndexed { index, info ->
info.second.forEach {
feedDatabaseManager.upsertAll(
listEntities[index].uid,
it.relatedItems.filterIsInstance<StreamInfoItem>()
)
val streams = info.second.relatedItems.filterIsInstance<StreamInfoItem>()
feedDatabaseManager.upsertAll(listEntities[index].uid, streams)
}
}
}
return listEntities
}
fun updateChannelInfo(info: ChannelInfo): Completable =
subscriptionTable.getSubscription(info.serviceId, info.url)
.flatMapCompletable {

24
app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionsImportFragment.java
View File

@ -1,10 +1,6 @@
package org.schabi.newpipe.local.subscription;
import static org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.ContentSource.CHANNEL_URL;
import static org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.CHANNEL_URL_MODE;
import static org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.INPUT_STREAM_MODE;
import static org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.KEY_MODE;
import static org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.KEY_VALUE;
import android.app.Activity;
import android.content.Intent;
@ -37,7 +33,7 @@ import org.schabi.newpipe.error.UserAction;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.local.subscription.services.SubscriptionsImportService;
import org.schabi.newpipe.local.subscription.workers.SubscriptionImportInput;
import org.schabi.newpipe.streams.io.NoFileManagerSafeGuard;
import org.schabi.newpipe.streams.io.StoredFileHelper;
import org.schabi.newpipe.util.Constants;
@ -168,10 +164,8 @@ public class SubscriptionsImportFragment extends BaseFragment {
}
public void onImportUrl(final String value) {
ImportConfirmationDialog.show(this, new Intent(activity, SubscriptionsImportService.class)
.putExtra(KEY_MODE, CHANNEL_URL_MODE)
.putExtra(KEY_VALUE, value)
.putExtra(Constants.KEY_SERVICE_ID, currentServiceId));
ImportConfirmationDialog.show(this,
new SubscriptionImportInput.ChannelUrlMode(currentServiceId, value));
}
public void onImportFile() {
@ -186,16 +180,10 @@ public class SubscriptionsImportFragment extends BaseFragment {
}
private void requestImportFileResult(final ActivityResult result) {
if (result.getData() == null) {
return;
}
if (result.getResultCode() == Activity.RESULT_OK && result.getData().getData() != null) {
final String data = result.getData() != null ? result.getData().getDataString() : null;
if (result.getResultCode() == Activity.RESULT_OK && data != null) {
ImportConfirmationDialog.show(this,
new Intent(activity, SubscriptionsImportService.class)
.putExtra(KEY_MODE, INPUT_STREAM_MODE)
.putExtra(KEY_VALUE, result.getData().getData())
.putExtra(Constants.KEY_SERVICE_ID, currentServiceId));
new SubscriptionImportInput.InputStreamMode(currentServiceId, data));
}
}

233
app/src/main/java/org/schabi/newpipe/local/subscription/services/BaseImportExportService.java
View File

@ -1,233 +0,0 @@
/*
* Copyright 2018 Mauricio Colli <mauriciocolli@outlook.com>
* BaseImportExportService.java is part of NewPipe
*
* License: GPL-3.0+
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.schabi.newpipe.local.subscription.services;
import android.app.Service;
import android.content.Intent;
import android.os.Build;
import android.os.IBinder;
import android.text.TextUtils;
import android.widget.Toast;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.StringRes;
import androidx.core.app.NotificationCompat;
import androidx.core.app.NotificationManagerCompat;
import androidx.core.app.ServiceCompat;
import org.reactivestreams.Publisher;
import org.schabi.newpipe.R;
import org.schabi.newpipe.error.ErrorInfo;
import org.schabi.newpipe.error.ErrorUtil;
import org.schabi.newpipe.error.UserAction;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.ktx.ExceptionUtils;
import org.schabi.newpipe.local.subscription.SubscriptionManager;
import java.io.FileNotFoundException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
import io.reactivex.rxjava3.core.Flowable;
import io.reactivex.rxjava3.disposables.CompositeDisposable;
import io.reactivex.rxjava3.functions.Function;
import io.reactivex.rxjava3.processors.PublishProcessor;
public abstract class BaseImportExportService extends Service {
protected final String TAG = this.getClass().getSimpleName();
protected final CompositeDisposable disposables = new CompositeDisposable();
protected final PublishProcessor<String> notificationUpdater = PublishProcessor.create();
protected NotificationManagerCompat notificationManager;
protected NotificationCompat.Builder notificationBuilder;
protected SubscriptionManager subscriptionManager;
private static final int NOTIFICATION_SAMPLING_PERIOD = 2500;
protected final AtomicInteger currentProgress = new AtomicInteger(-1);
protected final AtomicInteger maxProgress = new AtomicInteger(-1);
protected final ImportExportEventListener eventListener = new ImportExportEventListener() {
@Override
public void onSizeReceived(final int size) {
maxProgress.set(size);
currentProgress.set(0);
}
@Override
public void onItemCompleted(final String itemName) {
currentProgress.incrementAndGet();
notificationUpdater.onNext(itemName);
}
};
protected Toast toast;
@Nullable
@Override
public IBinder onBind(final Intent intent) {
return null;
}
@Override
public void onCreate() {
super.onCreate();
subscriptionManager = new SubscriptionManager(this);
setupNotification();
}
@Override
public void onDestroy() {
super.onDestroy();
disposeAll();
}
protected void disposeAll() {
disposables.clear();
}
/*//////////////////////////////////////////////////////////////////////////
// Notification Impl
//////////////////////////////////////////////////////////////////////////*/
protected abstract int getNotificationId();
@StringRes
public abstract int getTitle();
protected void setupNotification() {
notificationManager = NotificationManagerCompat.from(this);
notificationBuilder = createNotification();
startForeground(getNotificationId(), notificationBuilder.build());
final Function<Flowable<String>, Publisher<String>> throttleAfterFirstEmission = flow ->
flow.take(1).concatWith(flow.skip(1)
.throttleLast(NOTIFICATION_SAMPLING_PERIOD, TimeUnit.MILLISECONDS));
disposables.add(notificationUpdater
.filter(s -> !s.isEmpty())
.publish(throttleAfterFirstEmission)
.observeOn(AndroidSchedulers.mainThread())
.subscribe(this::updateNotification));
}
protected void updateNotification(final String text) {
notificationBuilder
.setProgress(maxProgress.get(), currentProgress.get(), maxProgress.get() == -1);
final String progressText = currentProgress + "/" + maxProgress;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
if (!TextUtils.isEmpty(text)) {
notificationBuilder.setContentText(text + " (" + progressText + ")");
}
} else {
notificationBuilder.setContentInfo(progressText);
notificationBuilder.setContentText(text);
}
notificationManager.notify(getNotificationId(), notificationBuilder.build());
}
protected void stopService() {
postErrorResult(null, null);
}
protected void stopAndReportError(final Throwable throwable, final String request) {
stopService();
ErrorUtil.createNotification(this, new ErrorInfo(
throwable, UserAction.SUBSCRIPTION_IMPORT_EXPORT, request));
}
protected void postErrorResult(final String title, final String text) {
disposeAll();
ServiceCompat.stopForeground(this, ServiceCompat.STOP_FOREGROUND_REMOVE);
stopSelf();
if (title == null) {
return;
}
final String textOrEmpty = text == null ? "" : text;
notificationBuilder = new NotificationCompat
.Builder(this, getString(R.string.notification_channel_id))
.setSmallIcon(R.drawable.ic_newpipe_triangle_white)
.setVisibility(NotificationCompat.VISIBILITY_PUBLIC)
.setContentTitle(title)
.setStyle(new NotificationCompat.BigTextStyle().bigText(textOrEmpty))
.setContentText(textOrEmpty);
notificationManager.notify(getNotificationId(), notificationBuilder.build());
}
protected NotificationCompat.Builder createNotification() {
return new NotificationCompat.Builder(this, getString(R.string.notification_channel_id))
.setOngoing(true)
.setProgress(-1, -1, true)
.setSmallIcon(R.drawable.ic_newpipe_triangle_white)
.setVisibility(NotificationCompat.VISIBILITY_PUBLIC)
.setContentTitle(getString(getTitle()));
}
/*//////////////////////////////////////////////////////////////////////////
// Toast
//////////////////////////////////////////////////////////////////////////*/
protected void showToast(@StringRes final int message) {
showToast(getString(message));
}
protected void showToast(final String message) {
if (toast != null) {
toast.cancel();
}
toast = Toast.makeText(this, message, Toast.LENGTH_SHORT);
toast.show();
}
/*//////////////////////////////////////////////////////////////////////////
// Error handling
//////////////////////////////////////////////////////////////////////////*/
protected void handleError(@StringRes final int errorTitle, @NonNull final Throwable error) {
String message = getErrorMessage(error);
if (TextUtils.isEmpty(message)) {
final String errorClassName = error.getClass().getName();
message = getString(R.string.error_occurred_detail, errorClassName);
}
showToast(errorTitle);
postErrorResult(getString(errorTitle), message);
}
protected String getErrorMessage(final Throwable error) {
String message = null;
if (error instanceof SubscriptionExtractor.InvalidSourceException) {
message = getString(R.string.invalid_source);
} else if (error instanceof FileNotFoundException) {
message = getString(R.string.invalid_file);
} else if (ExceptionUtils.isNetworkRelated(error)) {
message = getString(R.string.network_error);
}
return message;
}
}

17
app/src/main/java/org/schabi/newpipe/local/subscription/services/ImportExportEventListener.java
View File

@ -1,17 +0,0 @@
package org.schabi.newpipe.local.subscription.services;
public interface ImportExportEventListener {
/**
* Called when the size has been resolved.
*
* @param size how many items there are to import/export
*/
void onSizeReceived(int size);
/**
* Called every time an item has been parsed/resolved.
*
* @param itemName the name of the subscription item
*/
void onItemCompleted(String itemName);
}

158
app/src/main/java/org/schabi/newpipe/local/subscription/services/ImportExportJsonHelper.java
View File

@ -1,158 +0,0 @@
/*
* Copyright 2018 Mauricio Colli <mauriciocolli@outlook.com>
* ImportExportJsonHelper.java is part of NewPipe
*
* License: GPL-3.0+
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.schabi.newpipe.local.subscription.services;
import androidx.annotation.Nullable;
import com.grack.nanojson.JsonAppendableWriter;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.BuildConfig;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.InvalidSourceException;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
/**
* A JSON implementation capable of importing and exporting subscriptions, it has the advantage
* of being able to transfer subscriptions to any device.
*/
public final class ImportExportJsonHelper {
/*//////////////////////////////////////////////////////////////////////////
// Json implementation
//////////////////////////////////////////////////////////////////////////*/
private static final String JSON_APP_VERSION_KEY = "app_version";
private static final String JSON_APP_VERSION_INT_KEY = "app_version_int";
private static final String JSON_SUBSCRIPTIONS_ARRAY_KEY = "subscriptions";
private static final String JSON_SERVICE_ID_KEY = "service_id";
private static final String JSON_URL_KEY = "url";
private static final String JSON_NAME_KEY = "name";
private ImportExportJsonHelper() { }
/**
* Read a JSON source through the input stream.
*
* @param in the input stream (e.g. a file)
* @param eventListener listener for the events generated
* @return the parsed subscription items
*/
public static List<SubscriptionItem> readFrom(
final InputStream in, @Nullable final ImportExportEventListener eventListener)
throws InvalidSourceException {
if (in == null) {
throw new InvalidSourceException("input is null");
}
final List<SubscriptionItem> channels = new ArrayList<>();
try {
final JsonObject parentObject = JsonParser.object().from(in);
if (!parentObject.has(JSON_SUBSCRIPTIONS_ARRAY_KEY)) {
throw new InvalidSourceException("Channels array is null");
}
final JsonArray channelsArray = parentObject.getArray(JSON_SUBSCRIPTIONS_ARRAY_KEY);
if (eventListener != null) {
eventListener.onSizeReceived(channelsArray.size());
}
for (final Object o : channelsArray) {
if (o instanceof JsonObject) {
final JsonObject itemObject = (JsonObject) o;
final int serviceId = itemObject.getInt(JSON_SERVICE_ID_KEY, 0);
final String url = itemObject.getString(JSON_URL_KEY);
final String name = itemObject.getString(JSON_NAME_KEY);
if (url != null && name != null && !url.isEmpty() && !name.isEmpty()) {
channels.add(new SubscriptionItem(serviceId, url, name));
if (eventListener != null) {
eventListener.onItemCompleted(name);
}
}
}
}
} catch (final Throwable e) {
throw new InvalidSourceException("Couldn't parse json", e);
}
return channels;
}
/**
* Write the subscriptions items list as JSON to the output.
*
* @param items the list of subscriptions items
* @param out the output stream (e.g. a file)
* @param eventListener listener for the events generated
*/
public static void writeTo(final List<SubscriptionItem> items, final OutputStream out,
@Nullable final ImportExportEventListener eventListener) {
final JsonAppendableWriter writer = JsonWriter.on(out);
writeTo(items, writer, eventListener);
writer.done();
}
/**
* @see #writeTo(List, OutputStream, ImportExportEventListener)
* @param items the list of subscriptions items
* @param writer the output {@link JsonAppendableWriter}
* @param eventListener listener for the events generated
*/
public static void writeTo(final List<SubscriptionItem> items,
final JsonAppendableWriter writer,
@Nullable final ImportExportEventListener eventListener) {
if (eventListener != null) {
eventListener.onSizeReceived(items.size());
}
writer.object();
writer.value(JSON_APP_VERSION_KEY, BuildConfig.VERSION_NAME);
writer.value(JSON_APP_VERSION_INT_KEY, BuildConfig.VERSION_CODE);
writer.array(JSON_SUBSCRIPTIONS_ARRAY_KEY);
for (final SubscriptionItem item : items) {
writer.object();
writer.value(JSON_SERVICE_ID_KEY, item.getServiceId());
writer.value(JSON_URL_KEY, item.getUrl());
writer.value(JSON_NAME_KEY, item.getName());
writer.end();
if (eventListener != null) {
eventListener.onItemCompleted(item.getName());
}
}
writer.end();
writer.end();
}
}

171
app/src/main/java/org/schabi/newpipe/local/subscription/services/SubscriptionsExportService.java
View File

@ -1,171 +0,0 @@
/*
* Copyright 2018 Mauricio Colli <mauriciocolli@outlook.com>
* SubscriptionsExportService.java is part of NewPipe
*
* License: GPL-3.0+
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.schabi.newpipe.local.subscription.services;
import static org.schabi.newpipe.MainActivity.DEBUG;
import android.content.Intent;
import android.net.Uri;
import android.util.Log;
import androidx.core.content.IntentCompat;
import androidx.localbroadcastmanager.content.LocalBroadcastManager;
import org.reactivestreams.Subscriber;
import org.reactivestreams.Subscription;
import org.schabi.newpipe.App;
import org.schabi.newpipe.R;
import org.schabi.newpipe.database.subscription.SubscriptionEntity;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import org.schabi.newpipe.streams.io.SharpOutputStream;
import org.schabi.newpipe.streams.io.StoredFileHelper;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
import io.reactivex.rxjava3.functions.Function;
import io.reactivex.rxjava3.schedulers.Schedulers;
public class SubscriptionsExportService extends BaseImportExportService {
public static final String KEY_FILE_PATH = "key_file_path";
/**
* A {@link LocalBroadcastManager local broadcast} will be made with this action
* when the export is successfully completed.
*/
public static final String EXPORT_COMPLETE_ACTION = App.PACKAGE_NAME + ".local.subscription"
+ ".services.SubscriptionsExportService.EXPORT_COMPLETE";
private Subscription subscription;
private StoredFileHelper outFile;
private OutputStream outputStream;
@Override
public int onStartCommand(final Intent intent, final int flags, final int startId) {
if (intent == null || subscription != null) {
return START_NOT_STICKY;
}
final Uri path = IntentCompat.getParcelableExtra(intent, KEY_FILE_PATH, Uri.class);
if (path == null) {
stopAndReportError(new IllegalStateException(
"Exporting to a file, but the path is null"),
"Exporting subscriptions");
return START_NOT_STICKY;
}
try {
outFile = new StoredFileHelper(this, path, "application/json");
// truncate the file before writing to it, otherwise if the new content is smaller than
// the previous file size, the file will retain part of the previous content and be
// corrupted
outputStream = new SharpOutputStream(outFile.openAndTruncateStream());
} catch (final IOException e) {
handleError(e);
return START_NOT_STICKY;
}
startExport();
return START_NOT_STICKY;
}
@Override
protected int getNotificationId() {
return 4567;
}
@Override
public int getTitle() {
return R.string.export_ongoing;
}
@Override
protected void disposeAll() {
super.disposeAll();
if (subscription != null) {
subscription.cancel();
}
}
private void startExport() {
showToast(R.string.export_ongoing);
subscriptionManager.subscriptionTable().getAll().take(1)
.map(subscriptionEntities -> {
final List<SubscriptionItem> result =
new ArrayList<>(subscriptionEntities.size());
for (final SubscriptionEntity entity : subscriptionEntities) {
result.add(new SubscriptionItem(entity.getServiceId(), entity.getUrl(),
entity.getName()));
}
return result;
})
.map(exportToFile())
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe(getSubscriber());
}
private Subscriber<StoredFileHelper> getSubscriber() {
return new Subscriber<StoredFileHelper>() {
@Override
public void onSubscribe(final Subscription s) {
subscription = s;
s.request(1);
}
@Override
public void onNext(final StoredFileHelper file) {
if (DEBUG) {
Log.d(TAG, "startExport() success: file = " + file);
}
}
@Override
public void onError(final Throwable error) {
Log.e(TAG, "onError() called with: error = [" + error + "]", error);
handleError(error);
}
@Override
public void onComplete() {
LocalBroadcastManager.getInstance(SubscriptionsExportService.this)
.sendBroadcast(new Intent(EXPORT_COMPLETE_ACTION));
showToast(R.string.export_complete_toast);
stopService();
}
};
}
private Function<List<SubscriptionItem>, StoredFileHelper> exportToFile() {
return subscriptionItems -> {
ImportExportJsonHelper.writeTo(subscriptionItems, outputStream, eventListener);
return outFile;
};
}
protected void handleError(final Throwable error) {
super.handleError(R.string.subscriptions_export_unsuccessful, error);
}
}

327
app/src/main/java/org/schabi/newpipe/local/subscription/services/SubscriptionsImportService.java
View File

@ -1,327 +0,0 @@
/*
* Copyright 2018 Mauricio Colli <mauriciocolli@outlook.com>
* SubscriptionsImportService.java is part of NewPipe
*
* License: GPL-3.0+
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.schabi.newpipe.local.subscription.services;
import static org.schabi.newpipe.MainActivity.DEBUG;
import static org.schabi.newpipe.streams.io.StoredFileHelper.DEFAULT_MIME;
import android.content.Intent;
import android.net.Uri;
import android.text.TextUtils;
import android.util.Log;
import android.util.Pair;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.core.content.IntentCompat;
import androidx.localbroadcastmanager.content.LocalBroadcastManager;
import org.reactivestreams.Subscriber;
import org.reactivestreams.Subscription;
import org.schabi.newpipe.App;
import org.schabi.newpipe.R;
import org.schabi.newpipe.database.subscription.SubscriptionEntity;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelInfo;
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import org.schabi.newpipe.ktx.ExceptionUtils;
import org.schabi.newpipe.streams.io.SharpInputStream;
import org.schabi.newpipe.streams.io.StoredFileHelper;
import org.schabi.newpipe.util.Constants;
import org.schabi.newpipe.util.ExtractorHelper;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
import io.reactivex.rxjava3.core.Flowable;
import io.reactivex.rxjava3.core.Notification;
import io.reactivex.rxjava3.functions.Consumer;
import io.reactivex.rxjava3.functions.Function;
import io.reactivex.rxjava3.schedulers.Schedulers;
public class SubscriptionsImportService extends BaseImportExportService {
public static final int CHANNEL_URL_MODE = 0;
public static final int INPUT_STREAM_MODE = 1;
public static final int PREVIOUS_EXPORT_MODE = 2;
public static final String KEY_MODE = "key_mode";
public static final String KEY_VALUE = "key_value";
/**
* A {@link LocalBroadcastManager local broadcast} will be made with this action
* when the import is successfully completed.
*/
public static final String IMPORT_COMPLETE_ACTION = App.PACKAGE_NAME + ".local.subscription"
+ ".services.SubscriptionsImportService.IMPORT_COMPLETE";
/**
* How many extractions running in parallel.
*/
public static final int PARALLEL_EXTRACTIONS = 8;
/**
* Number of items to buffer to mass-insert in the subscriptions table,
* this leads to a better performance as we can then use db transactions.
*/
public static final int BUFFER_COUNT_BEFORE_INSERT = 50;
private Subscription subscription;
private int currentMode;
private int currentServiceId;
@Nullable
private String channelUrl;
@Nullable
private InputStream inputStream;
@Nullable
private String inputStreamType;
@Override
public int onStartCommand(final Intent intent, final int flags, final int startId) {
if (intent == null || subscription != null) {
return START_NOT_STICKY;
}
currentMode = intent.getIntExtra(KEY_MODE, -1);
currentServiceId = intent.getIntExtra(Constants.KEY_SERVICE_ID, Constants.NO_SERVICE_ID);
if (currentMode == CHANNEL_URL_MODE) {
channelUrl = intent.getStringExtra(KEY_VALUE);
} else {
final Uri uri = IntentCompat.getParcelableExtra(intent, KEY_VALUE, Uri.class);
if (uri == null) {
stopAndReportError(new IllegalStateException(
"Importing from input stream, but file path is null"),
"Importing subscriptions");
return START_NOT_STICKY;
}
try {
final StoredFileHelper fileHelper = new StoredFileHelper(this, uri, DEFAULT_MIME);
inputStream = new SharpInputStream(fileHelper.getStream());
inputStreamType = fileHelper.getType();
if (inputStreamType == null || inputStreamType.equals(DEFAULT_MIME)) {
// mime type could not be determined, just take file extension
final String name = fileHelper.getName();
final int pointIndex = name.lastIndexOf('.');
if (pointIndex == -1 || pointIndex >= name.length() - 1) {
inputStreamType = DEFAULT_MIME; // no extension, will fail in the extractor
} else {
inputStreamType = name.substring(pointIndex + 1);
}
}
} catch (final IOException e) {
handleError(e);
return START_NOT_STICKY;
}
}
if (currentMode == -1 || currentMode == CHANNEL_URL_MODE && channelUrl == null) {
final String errorDescription = "Some important field is null or in illegal state: "
+ "currentMode=[" + currentMode + "], "
+ "channelUrl=[" + channelUrl + "], "
+ "inputStream=[" + inputStream + "]";
stopAndReportError(new IllegalStateException(errorDescription),
"Importing subscriptions");
return START_NOT_STICKY;
}
startImport();
return START_NOT_STICKY;
}
@Override
protected int getNotificationId() {
return 4568;
}
@Override
public int getTitle() {
return R.string.import_ongoing;
}
@Override
protected void disposeAll() {
super.disposeAll();
if (subscription != null) {
subscription.cancel();
}
}
/*//////////////////////////////////////////////////////////////////////////
// Imports
//////////////////////////////////////////////////////////////////////////*/
private void startImport() {
showToast(R.string.import_ongoing);
Flowable<List<SubscriptionItem>> flowable = null;
switch (currentMode) {
case CHANNEL_URL_MODE:
flowable = importFromChannelUrl();
break;
case INPUT_STREAM_MODE:
flowable = importFromInputStream();
break;
case PREVIOUS_EXPORT_MODE:
flowable = importFromPreviousExport();
break;
}
if (flowable == null) {
final String message = "Flowable given by \"importFrom\" is null "
+ "(current mode: " + currentMode + ")";
stopAndReportError(new IllegalStateException(message), "Importing subscriptions");
return;
}
flowable.doOnNext(subscriptionItems ->
eventListener.onSizeReceived(subscriptionItems.size()))
.flatMap(Flowable::fromIterable)
.parallel(PARALLEL_EXTRACTIONS)
.runOn(Schedulers.io())
.map((Function<SubscriptionItem, Notification<Pair<ChannelInfo,
List<ChannelTabInfo>>>>) subscriptionItem -> {
try {
final ChannelInfo channelInfo = ExtractorHelper
.getChannelInfo(subscriptionItem.getServiceId(),
subscriptionItem.getUrl(), true)
.blockingGet();
return Notification.createOnNext(new Pair<>(channelInfo,
Collections.singletonList(
ExtractorHelper.getChannelTab(
subscriptionItem.getServiceId(),
channelInfo.getTabs().get(0), true).blockingGet()
)));
} catch (final Throwable e) {
return Notification.createOnError(e);
}
})
.sequential()
.observeOn(Schedulers.io())
.doOnNext(getNotificationsConsumer())
.buffer(BUFFER_COUNT_BEFORE_INSERT)
.map(upsertBatch())
.subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread())
.subscribe(getSubscriber());
}
private Subscriber<List<SubscriptionEntity>> getSubscriber() {
return new Subscriber<>() {
@Override
public void onSubscribe(final Subscription s) {
subscription = s;
s.request(Long.MAX_VALUE);
}
@Override
public void onNext(final List<SubscriptionEntity> successfulInserted) {
if (DEBUG) {
Log.d(TAG, "startImport() " + successfulInserted.size()
+ " items successfully inserted into the database");
}
}
@Override
public void onError(final Throwable error) {
Log.e(TAG, "Got an error!", error);
handleError(error);
}
@Override
public void onComplete() {
LocalBroadcastManager.getInstance(SubscriptionsImportService.this)
.sendBroadcast(new Intent(IMPORT_COMPLETE_ACTION));
showToast(R.string.import_complete_toast);
stopService();
}
};
}
private Consumer<Notification<Pair<ChannelInfo,
List<ChannelTabInfo>>>> getNotificationsConsumer() {
return notification -> {
if (notification.isOnNext()) {
final String name = notification.getValue().first.getName();
eventListener.onItemCompleted(!TextUtils.isEmpty(name) ? name : "");
} else if (notification.isOnError()) {
final Throwable error = notification.getError();
final Throwable cause = error.getCause();
if (error instanceof IOException) {
throw error;
} else if (cause instanceof IOException) {
throw cause;
} else if (ExceptionUtils.isNetworkRelated(error)) {
throw new IOException(error);
}
eventListener.onItemCompleted("");
}
};
}
private Function<List<Notification<Pair<ChannelInfo, List<ChannelTabInfo>>>>,
List<SubscriptionEntity>> upsertBatch() {
return notificationList -> {
final List<Pair<ChannelInfo, List<ChannelTabInfo>>> infoList =
new ArrayList<>(notificationList.size());
for (final Notification<Pair<ChannelInfo, List<ChannelTabInfo>>> n : notificationList) {
if (n.isOnNext()) {
infoList.add(n.getValue());
}
}
return subscriptionManager.upsertAll(infoList);
};
}
private Flowable<List<SubscriptionItem>> importFromChannelUrl() {
return Flowable.fromCallable(() -> NewPipe.getService(currentServiceId)
.getSubscriptionExtractor()
.fromChannelUrl(channelUrl));
}
private Flowable<List<SubscriptionItem>> importFromInputStream() {
Objects.requireNonNull(inputStream);
Objects.requireNonNull(inputStreamType);
return Flowable.fromCallable(() -> NewPipe.getService(currentServiceId)
.getSubscriptionExtractor()
.fromInputStream(inputStream, inputStreamType));
}
private Flowable<List<SubscriptionItem>> importFromPreviousExport() {
return Flowable.fromCallable(() -> ImportExportJsonHelper.readFrom(inputStream, null));
}
protected void handleError(@NonNull final Throwable error) {
super.handleError(R.string.subscriptions_import_unsuccessful, error);
}
}

72
app/src/main/java/org/schabi/newpipe/local/subscription/workers/ImportExportJsonHelper.kt Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright 2018 Mauricio Colli <mauriciocolli@outlook.com>
* ImportExportJsonHelper.java is part of NewPipe
*
* License: GPL-3.0+
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.schabi.newpipe.local.subscription.workers
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
import kotlinx.serialization.json.encodeToStream
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.InvalidSourceException
import java.io.InputStream
import java.io.OutputStream
/**
* A JSON implementation capable of importing and exporting subscriptions, it has the advantage
* of being able to transfer subscriptions to any device.
*/
object ImportExportJsonHelper {
private val json = Json { encodeDefaults = true }
/**
* Read a JSON source through the input stream.
*
* @param in the input stream (e.g. a file)
* @return the parsed subscription items
*/
@JvmStatic
@Throws(InvalidSourceException::class)
fun readFrom(`in`: InputStream?): List<SubscriptionItem> {
if (`in` == null) {
throw InvalidSourceException("input is null")
}
try {
@OptIn(ExperimentalSerializationApi::class)
return json.decodeFromStream<SubscriptionData>(`in`).subscriptions
} catch (e: Throwable) {
throw InvalidSourceException("Couldn't parse json", e)
}
}
/**
* Write the subscriptions items list as JSON to the output.
*
* @param items the list of subscriptions items
* @param out the output stream (e.g. a file)
*/
@OptIn(ExperimentalSerializationApi::class)
@JvmStatic
fun writeTo(
items: List<SubscriptionItem>,
out: OutputStream,
) {
json.encodeToStream(SubscriptionData(items), out)
}
}

24
app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionData.kt Normal file
View File

@ -0,0 +1,24 @@
package org.schabi.newpipe.local.subscription.workers
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import org.schabi.newpipe.BuildConfig
@Serializable
class SubscriptionData(
val subscriptions: List<SubscriptionItem>
) {
@SerialName("app_version")
private val appVersion = BuildConfig.VERSION_NAME
@SerialName("app_version_int")
private val appVersionInt = BuildConfig.VERSION_CODE
}
@Serializable
data class SubscriptionItem(
@SerialName("service_id")
val serviceId: Int,
val url: String,
val name: String
)

119
app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionExportWorker.kt Normal file
View File

@ -0,0 +1,119 @@
package org.schabi.newpipe.local.subscription.workers
import android.content.Context
import android.content.pm.ServiceInfo
import android.net.Uri
import android.os.Build
import android.util.Log
import android.widget.Toast
import androidx.core.app.NotificationCompat
import androidx.core.net.toUri
import androidx.work.CoroutineWorker
import androidx.work.ExistingWorkPolicy
import androidx.work.ForegroundInfo
import androidx.work.OneTimeWorkRequestBuilder
import androidx.work.OutOfQuotaPolicy
import androidx.work.WorkManager
import androidx.work.WorkerParameters
import androidx.work.workDataOf
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.reactive.awaitFirst
import kotlinx.coroutines.withContext
import org.schabi.newpipe.BuildConfig
import org.schabi.newpipe.NewPipeDatabase
import org.schabi.newpipe.R
class SubscriptionExportWorker(
appContext: Context,
params: WorkerParameters,
) : CoroutineWorker(appContext, params) {
// This is needed for API levels < 31 (Android S).
override suspend fun getForegroundInfo(): ForegroundInfo {
return createForegroundInfo(applicationContext.getString(R.string.export_ongoing))
}
override suspend fun doWork(): Result {
return try {
val uri = inputData.getString(EXPORT_PATH)!!.toUri()
val table = NewPipeDatabase.getInstance(applicationContext).subscriptionDAO()
val subscriptions =
table.all
.awaitFirst()
.map { SubscriptionItem(it.serviceId, it.url, it.name) }
val qty = subscriptions.size
val title = applicationContext.resources.getQuantityString(R.plurals.export_subscriptions, qty, qty)
setForeground(createForegroundInfo(title))
withContext(Dispatchers.IO) {
// Truncate file if it already exists
applicationContext.contentResolver.openOutputStream(uri, "wt")?.use {
ImportExportJsonHelper.writeTo(subscriptions, it)
}
}
if (BuildConfig.DEBUG) {
Log.i(TAG, "Exported $qty subscriptions")
}
withContext(Dispatchers.Main) {
Toast
.makeText(applicationContext, R.string.export_complete_toast, Toast.LENGTH_SHORT)
.show()
}
Result.success()
} catch (e: Exception) {
if (BuildConfig.DEBUG) {
Log.e(TAG, "Error while exporting subscriptions", e)
}
withContext(Dispatchers.Main) {
Toast
.makeText(applicationContext, R.string.subscriptions_export_unsuccessful, Toast.LENGTH_SHORT)
.show()
}
return Result.failure()
}
}
private fun createForegroundInfo(title: String): ForegroundInfo {
val notification =
NotificationCompat
.Builder(applicationContext, NOTIFICATION_CHANNEL_ID)
.setSmallIcon(R.drawable.ic_newpipe_triangle_white)
.setOngoing(true)
.setProgress(-1, -1, true)
.setVisibility(NotificationCompat.VISIBILITY_PUBLIC)
.setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE)
.setContentTitle(title)
.build()
val serviceType = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC else 0
return ForegroundInfo(NOTIFICATION_ID, notification, serviceType)
}
companion object {
private const val TAG = "SubscriptionExportWork"
private const val NOTIFICATION_ID = 4567
private const val NOTIFICATION_CHANNEL_ID = "newpipe"
private const val WORK_NAME = "exportSubscriptions"
private const val EXPORT_PATH = "exportPath"
fun schedule(
context: Context,
uri: Uri,
) {
val data = workDataOf(EXPORT_PATH to uri.toString())
val workRequest =
OneTimeWorkRequestBuilder<SubscriptionExportWorker>()
.setInputData(data)
.setExpedited(OutOfQuotaPolicy.RUN_AS_NON_EXPEDITED_WORK_REQUEST)
.build()
WorkManager
.getInstance(context)
.enqueueUniqueWork(WORK_NAME, ExistingWorkPolicy.APPEND_OR_REPLACE, workRequest)
}
}
}

237
app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionImportWorker.kt Normal file
View File

@ -0,0 +1,237 @@
package org.schabi.newpipe.local.subscription.workers
import android.content.Context
import android.content.pm.ServiceInfo
import android.os.Build
import android.os.Parcelable
import android.util.Log
import android.webkit.MimeTypeMap
import android.widget.Toast
import androidx.core.app.NotificationCompat
import androidx.core.net.toUri
import androidx.work.CoroutineWorker
import androidx.work.Data
import androidx.work.ForegroundInfo
import androidx.work.WorkManager
import androidx.work.WorkerParameters
import androidx.work.workDataOf
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.rx3.await
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import kotlinx.parcelize.Parcelize
import org.schabi.newpipe.BuildConfig
import org.schabi.newpipe.R
import org.schabi.newpipe.extractor.NewPipe
import org.schabi.newpipe.local.subscription.SubscriptionManager
import org.schabi.newpipe.util.ExtractorHelper
class SubscriptionImportWorker(
appContext: Context,
params: WorkerParameters,
) : CoroutineWorker(appContext, params) {
// This is needed for API levels < 31 (Android S).
override suspend fun getForegroundInfo(): ForegroundInfo {
return createForegroundInfo(applicationContext.getString(R.string.import_ongoing), null, 0, 0)
}
override suspend fun doWork(): Result {
val subscriptions =
try {
loadSubscriptionsFromInput(SubscriptionImportInput.fromData(inputData))
} catch (e: Exception) {
if (BuildConfig.DEBUG) {
Log.e(TAG, "Error while loading subscriptions from path", e)
}
withContext(Dispatchers.Main) {
Toast
.makeText(applicationContext, R.string.subscriptions_import_unsuccessful, Toast.LENGTH_SHORT)
.show()
}
return Result.failure()
}
val mutex = Mutex()
var index = 1
val qty = subscriptions.size
var title =
applicationContext.resources.getQuantityString(R.plurals.load_subscriptions, qty, qty)
val channelInfoList =
try {
withContext(Dispatchers.IO.limitedParallelism(PARALLEL_EXTRACTIONS)) {
subscriptions
.map {
async {
val channelInfo =
ExtractorHelper.getChannelInfo(it.serviceId, it.url, true).await()
val channelTab =
ExtractorHelper.getChannelTab(it.serviceId, channelInfo.tabs[0], true).await()
val currentIndex = mutex.withLock { index++ }
setForeground(createForegroundInfo(title, channelInfo.name, currentIndex, qty))
channelInfo to channelTab
}
}.awaitAll()
}
} catch (e: Exception) {
if (BuildConfig.DEBUG) {
Log.e(TAG, "Error while loading subscription data", e)
}
withContext(Dispatchers.Main) {
Toast.makeText(applicationContext, R.string.subscriptions_import_unsuccessful, Toast.LENGTH_SHORT)
.show()
}
return Result.failure()
}
title = applicationContext.resources.getQuantityString(R.plurals.import_subscriptions, qty, qty)
setForeground(createForegroundInfo(title, null, 0, 0))
index = 0
val subscriptionManager = SubscriptionManager(applicationContext)
for (chunk in channelInfoList.chunked(BUFFER_COUNT_BEFORE_INSERT)) {
withContext(Dispatchers.IO) {
subscriptionManager.upsertAll(chunk)
}
index += chunk.size
setForeground(createForegroundInfo(title, null, index, qty))
}
withContext(Dispatchers.Main) {
Toast.makeText(applicationContext, R.string.import_complete_toast, Toast.LENGTH_SHORT)
.show()
}
return Result.success()
}
private suspend fun loadSubscriptionsFromInput(input: SubscriptionImportInput): List<SubscriptionItem> {
return withContext(Dispatchers.IO) {
when (input) {
is SubscriptionImportInput.ChannelUrlMode ->
NewPipe.getService(input.serviceId).subscriptionExtractor
.fromChannelUrl(input.url)
.map { SubscriptionItem(it.serviceId, it.url, it.name) }
is SubscriptionImportInput.InputStreamMode ->
applicationContext.contentResolver.openInputStream(input.url.toUri())?.use {
val contentType =
MimeTypeMap.getFileExtensionFromUrl(input.url).ifEmpty { DEFAULT_MIME }
NewPipe.getService(input.serviceId).subscriptionExtractor
.fromInputStream(it, contentType)
.map { SubscriptionItem(it.serviceId, it.url, it.name) }
}
is SubscriptionImportInput.PreviousExportMode ->
applicationContext.contentResolver.openInputStream(input.url.toUri())?.use {
ImportExportJsonHelper.readFrom(it)
}
} ?: emptyList()
}
}
private fun createForegroundInfo(
title: String,
text: String?,
currentProgress: Int,
maxProgress: Int,
): ForegroundInfo {
val notification =
NotificationCompat
.Builder(applicationContext, NOTIFICATION_CHANNEL_ID)
.setSmallIcon(R.drawable.ic_newpipe_triangle_white)
.setOngoing(true)
.setProgress(maxProgress, currentProgress, currentProgress == 0)
.setVisibility(NotificationCompat.VISIBILITY_PUBLIC)
.setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE)
.setContentTitle(title)
.setContentText(text)
.addAction(
R.drawable.ic_close,
applicationContext.getString(R.string.cancel),
WorkManager.getInstance(applicationContext).createCancelPendingIntent(id),
).apply {
if (currentProgress > 0 && maxProgress > 0) {
val progressText = "$currentProgress/$maxProgress"
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
setSubText(progressText)
} else {
setContentInfo(progressText)
}
}
}.build()
val serviceType = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC else 0
return ForegroundInfo(NOTIFICATION_ID, notification, serviceType)
}
companion object {
// Log tag length is limited to 23 characters on API levels < 24.
private const val TAG = "SubscriptionImport"
private const val NOTIFICATION_ID = 4568
private const val NOTIFICATION_CHANNEL_ID = "newpipe"
private const val DEFAULT_MIME = "application/octet-stream"
private const val PARALLEL_EXTRACTIONS = 8
private const val BUFFER_COUNT_BEFORE_INSERT = 50
const val WORK_NAME = "SubscriptionImportWorker"
}
}
sealed class SubscriptionImportInput : Parcelable {
@Parcelize
data class ChannelUrlMode(val serviceId: Int, val url: String) : SubscriptionImportInput()
@Parcelize
data class InputStreamMode(val serviceId: Int, val url: String) : SubscriptionImportInput()
@Parcelize
data class PreviousExportMode(val url: String) : SubscriptionImportInput()
fun toData(): Data {
val (mode, serviceId, url) = when (this) {
is ChannelUrlMode -> Triple(CHANNEL_URL_MODE, serviceId, url)
is InputStreamMode -> Triple(INPUT_STREAM_MODE, serviceId, url)
is PreviousExportMode -> Triple(PREVIOUS_EXPORT_MODE, null, url)
}
return workDataOf("mode" to mode, "service_id" to serviceId, "url" to url)
}
companion object {
private const val CHANNEL_URL_MODE = 0
private const val INPUT_STREAM_MODE = 1
private const val PREVIOUS_EXPORT_MODE = 2
fun fromData(data: Data): SubscriptionImportInput {
val mode = data.getInt("mode", PREVIOUS_EXPORT_MODE)
when (mode) {
CHANNEL_URL_MODE -> {
val serviceId = data.getInt("service_id", -1)
if (serviceId == -1) {
throw IllegalArgumentException("No service id provided")
}
val url = data.getString("url")!!
return ChannelUrlMode(serviceId, url)
}
INPUT_STREAM_MODE -> {
val serviceId = data.getInt("service_id", -1)
if (serviceId == -1) {
throw IllegalArgumentException("No service id provided")
}
val url = data.getString("url")!!
return InputStreamMode(serviceId, url)
}
PREVIOUS_EXPORT_MODE -> {
val url = data.getString("url")!!
return PreviousExportMode(url)
}
else -> throw IllegalArgumentException("Unknown mode: $mode")
}
}
}
}

12
app/src/main/res/values/strings.xml
View File

@ -864,4 +864,16 @@
<item quantity="one">%d comment</item>
<item quantity="other">%d comments</item>
</plurals>
<plurals name="export_subscriptions">
<item quantity="one">Exporting %d subscription…</item>
<item quantity="other">Exporting %d subscriptions…</item>
</plurals>
<plurals name="load_subscriptions">
<item quantity="one">Loading %d subscription…</item>
<item quantity="other">Loading %d subscriptions…</item>
</plurals>
<plurals name="import_subscriptions">
<item quantity="one">Importing %d subscription…</item>
<item quantity="other">Importing %d subscriptions…</item>
</plurals>
</resources>

58
app/src/test/java/org/schabi/newpipe/local/subscription/services/ImportExportJsonHelperTest.java
View File

@ -5,11 +5,11 @@ import static org.junit.Assert.fail;
import org.junit.Test;
import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor;
import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import org.schabi.newpipe.local.subscription.workers.ImportExportJsonHelper;
import org.schabi.newpipe.local.subscription.workers.SubscriptionItem;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
@ -23,26 +23,22 @@ public class ImportExportJsonHelperTest {
final String emptySource =
"{\"app_version\":\"0.11.6\",\"app_version_int\": 47,\"subscriptions\":[]}";
final List<SubscriptionItem> items = ImportExportJsonHelper.readFrom(
new ByteArrayInputStream(emptySource.getBytes(StandardCharsets.UTF_8)), null);
final var items = ImportExportJsonHelper.readFrom(
new ByteArrayInputStream(emptySource.getBytes(StandardCharsets.UTF_8)));
assertTrue(items.isEmpty());
}
@Test
public void testInvalidSource() {
final List<String> invalidList = Arrays.asList(
"{}",
"",
null,
"gibberish");
final var invalidList = Arrays.asList("{}", "", null, "gibberish");
for (final String invalidContent : invalidList) {
try {
if (invalidContent != null) {
final byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
ImportExportJsonHelper.readFrom((new ByteArrayInputStream(bytes)), null);
ImportExportJsonHelper.readFrom(new ByteArrayInputStream(bytes));
} else {
ImportExportJsonHelper.readFrom(null, null);
ImportExportJsonHelper.readFrom(null);
}
fail("didn't throw exception");
@ -58,38 +54,24 @@ public class ImportExportJsonHelperTest {
@Test
public void ultimateTest() throws Exception {
// Read from file
final List<SubscriptionItem> itemsFromFile = readFromFile();
final var itemsFromFile = readFromFile();
// Test writing to an output
final String jsonOut = testWriteTo(itemsFromFile);
// Read again
final List<SubscriptionItem> itemsSecondRead = readFromWriteTo(jsonOut);
final var itemsSecondRead = readFromWriteTo(jsonOut);
// Check if both lists have the exact same items
if (itemsFromFile.size() != itemsSecondRead.size()) {
if (!itemsFromFile.equals(itemsSecondRead)) {
fail("The list of items were different from each other");
}
for (int i = 0; i < itemsFromFile.size(); i++) {
final SubscriptionItem item1 = itemsFromFile.get(i);
final SubscriptionItem item2 = itemsSecondRead.get(i);
final boolean equals = item1.getServiceId() == item2.getServiceId()
&& item1.getUrl().equals(item2.getUrl())
&& item1.getName().equals(item2.getName());
if (!equals) {
fail("The list of items were different from each other");
}
}
}
private List<SubscriptionItem> readFromFile() throws Exception {
final InputStream inputStream = getClass().getClassLoader().getResourceAsStream(
"import_export_test.json");
final List<SubscriptionItem> itemsFromFile = ImportExportJsonHelper.readFrom(
inputStream, null);
final var inputStream = getClass().getClassLoader()
.getResourceAsStream("import_export_test.json");
final var itemsFromFile = ImportExportJsonHelper.readFrom(inputStream);
if (itemsFromFile.isEmpty()) {
fail("ImportExportJsonHelper.readFrom(input) returned a null or empty list");
@ -98,10 +80,10 @@ public class ImportExportJsonHelperTest {
return itemsFromFile;
}
private String testWriteTo(final List<SubscriptionItem> itemsFromFile) throws Exception {
final ByteArrayOutputStream out = new ByteArrayOutputStream();
ImportExportJsonHelper.writeTo(itemsFromFile, out, null);
final String jsonOut = out.toString("UTF-8");
private String testWriteTo(final List<SubscriptionItem> itemsFromFile) {
final var out = new ByteArrayOutputStream();
ImportExportJsonHelper.writeTo(itemsFromFile, out);
final String jsonOut = out.toString(StandardCharsets.UTF_8);
if (jsonOut.isEmpty()) {
fail("JSON returned by writeTo was empty");
@ -111,10 +93,8 @@ public class ImportExportJsonHelperTest {
}
private List<SubscriptionItem> readFromWriteTo(final String jsonOut) throws Exception {
final ByteArrayInputStream inputStream = new ByteArrayInputStream(
jsonOut.getBytes(StandardCharsets.UTF_8));
final List<SubscriptionItem> secondReadItems = ImportExportJsonHelper.readFrom(
inputStream, null);
final var inputStream = new ByteArrayInputStream(jsonOut.getBytes(StandardCharsets.UTF_8));
final var secondReadItems = ImportExportJsonHelper.readFrom(inputStream);
if (secondReadItems.isEmpty()) {
fail("second call to readFrom returned an empty list");

1
build.gradle
View File

@ -11,6 +11,7 @@ buildscript {
classpath libs.kotlin.gradle.plugin
classpath libs.hilt.android.gradle.plugin
classpath libs.aboutlibraries.plugin
classpath libs.kotlinx.serialization
// NOTE: Do not place your application dependencies here; they belong
// in the individual module build.gradle files

8
gradle/libs.versions.toml
View File

@ -24,11 +24,11 @@ jsoup = "1.17.2"
junit = "4.13.2"
kotlin = "2.0.21"
kotlinxCoroutinesRx3 = "1.8.1"
kotlinxSerializationJson = "1.7.3"
ktlint = "0.45.2"
lazycolumnscrollbar = "2.2.0"
leakcanary = "2.12"
lifecycle = "2.6.2"
localbroadcastmanager = "1.1.0"
markwon = "4.6.2"
material = "1.11.0"
media = "1.7.0"
@ -60,7 +60,7 @@ teamnewpipe-nanojson = "1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751"
# to cause jitpack to regenerate the artifact.
teamnewpipe-newpipe-extractor = "v0.24.6"
webkit = "1.9.0"
work = "2.8.1"
work = "2.10.0"
[plugins]
aboutlibraries = { id = "com.mikepenz.aboutlibraries.plugin" }
@ -71,6 +71,7 @@ kotlin-android = { id = "kotlin-android" }
kotlin-compose = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" }
kotlin-kapt = { id = "kotlin-kapt" }
kotlin-parcelize = { id = "kotlin-parcelize" }
kotlinx-serialization = { id = "kotlinx-serialization" }
sonarqube = { id = "org.sonarqube", version.ref = "sonarqube" }
[libraries]
@ -98,7 +99,6 @@ androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "a
androidx-lifecycle-livedata = { group = "androidx.lifecycle", name = "lifecycle-livedata-ktx", version.ref = "lifecycle" }
androidx-lifecycle-viewmodel = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-ktx", version.ref = "lifecycle" }
androidx-lifecycle-viewmodel-compose = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-compose" }
androidx-localbroadcastmanager = { group = "androidx.localbroadcastmanager", name = "localbroadcastmanager", version.ref = "localbroadcastmanager" }
androidx-material = { group = "com.google.android.material", name = "material", version.ref = "material" }
androidx-media = { group = "androidx.media", name = "media", version.ref = "media" }
androidx-navigation-compose = { group = "androidx.navigation", name = "navigation-compose", version.ref = "navigationCompose" }
@ -136,6 +136,8 @@ junit = { group = "junit", name = "junit", version.ref = "junit" }
kotlin-gradle-plugin = { group = "org.jetbrains.kotlin", name = "kotlin-gradle-plugin", version.ref = "kotlin" }
kotlin-stdlib = { group = "org.jetbrains.kotlin", name = "kotlin-stdlib", version.ref = "kotlin" }
kotlinx-coroutines-rx3 = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-rx3", version.ref = "kotlinxCoroutinesRx3" }
kotlinx-serialization = { group = "org.jetbrains.kotlin", name = "kotlin-serialization", version.ref = "kotlin" }
kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" }
lazycolumnscrollbar = { group = "com.github.nanihadesuka", name = "LazyColumnScrollbar", version.ref = "lazycolumnscrollbar" }
leakcanary-android-core = { module = "com.squareup.leakcanary:leakcanary-android-core", version.ref = "leakcanary" }
leakcanary-object-watcher = { group = "com.squareup.leakcanary", name = "leakcanary-object-watcher-android", version.ref = "leakcanary" }