diff --git a/app/build.gradle b/app/build.gradle index e67e5c21e..0841086ad 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -9,6 +9,7 @@ plugins { alias libs.plugins.kotlin.compose alias libs.plugins.kotlin.kapt alias libs.plugins.kotlin.parcelize + alias libs.plugins.kotlinx.serialization alias libs.plugins.checkstyle alias libs.plugins.sonarqube alias libs.plugins.hilt @@ -16,7 +17,7 @@ plugins { } android { - compileSdk 34 + compileSdk 35 namespace 'org.schabi.newpipe' defaultConfig { @@ -226,7 +227,6 @@ dependencies { implementation libs.androidx.fragment.compose implementation libs.androidx.lifecycle.livedata implementation libs.androidx.lifecycle.viewmodel - implementation libs.androidx.localbroadcastmanager implementation libs.androidx.media implementation libs.androidx.preference implementation libs.androidx.recyclerview @@ -319,6 +319,9 @@ dependencies { // Scroll implementation libs.lazycolumnscrollbar + // Kotlinx Serialization + implementation libs.kotlinx.serialization.json + /** Debugging **/ // Memory leak detection debugImplementation libs.leakcanary.object.watcher diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro index 0cdffbe2e..54470bdd3 100644 --- a/app/proguard-rules.pro +++ b/app/proguard-rules.pro @@ -34,3 +34,18 @@ ## For some reason NotificationModeConfigFragment wasn't kept (only referenced in a preference xml) -keep class org.schabi.newpipe.settings.notifications.** { *; } + +## Keep Kotlinx Serialization classes +-keepclassmembers class kotlinx.serialization.json.** { + *** Companion; +} +-keepclasseswithmembers class kotlinx.serialization.json.** { + kotlinx.serialization.KSerializer serializer(...); +} +-keep,includedescriptorclasses class org.schabi.newpipe.**$$serializer { *; } +-keepclassmembers class org.schabi.newpipe.** { + *** Companion; +} +-keepclasseswithmembers class org.schabi.newpipe.** { + kotlinx.serialization.KSerializer serializer(...); +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index a66faab84..3faa59f7d 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -9,6 +9,7 @@ + @@ -90,8 +91,10 @@ android:exported="false" android:label="@string/title_activity_about" /> - - + { internal abstract fun silentInsertAllInternal(entities: List): List @Transaction - open fun upsertAll(entities: List): List { + open fun upsertAll(entities: List) { val insertUidList = silentInsertAllInternal(entities) insertUidList.forEachIndexed { index: Int, uidFromInsert: Long -> @@ -106,7 +106,5 @@ abstract class SubscriptionDAO : BasicDAO { update(entity) } } - - return entities } } diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/ImportConfirmationDialog.java b/app/src/main/java/org/schabi/newpipe/local/subscription/ImportConfirmationDialog.java index c3d3843ee..a4b3ea399 100644 --- a/app/src/main/java/org/schabi/newpipe/local/subscription/ImportConfirmationDialog.java +++ b/app/src/main/java/org/schabi/newpipe/local/subscription/ImportConfirmationDialog.java @@ -3,47 +3,64 @@ package org.schabi.newpipe.local.subscription; import static org.schabi.newpipe.util.Localization.assureCorrectAppLanguage; import android.app.Dialog; -import android.content.Intent; import android.os.Bundle; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.appcompat.app.AlertDialog; +import androidx.core.os.BundleCompat; import androidx.fragment.app.DialogFragment; import androidx.fragment.app.Fragment; +import androidx.work.Constraints; +import androidx.work.ExistingWorkPolicy; +import androidx.work.NetworkType; +import androidx.work.OneTimeWorkRequest; +import androidx.work.OutOfQuotaPolicy; +import androidx.work.WorkManager; -import com.evernote.android.state.State; import com.livefront.bridge.Bridge; import org.schabi.newpipe.R; +import org.schabi.newpipe.local.subscription.workers.SubscriptionImportInput; +import org.schabi.newpipe.local.subscription.workers.SubscriptionImportWorker; public class ImportConfirmationDialog extends DialogFragment { - @State - protected Intent resultServiceIntent; + private static final String INPUT = "input"; - public static void show(@NonNull final Fragment fragment, - @NonNull final Intent resultServiceIntent) { - final ImportConfirmationDialog confirmationDialog = new ImportConfirmationDialog(); - confirmationDialog.setResultServiceIntent(resultServiceIntent); + public static void show(@NonNull final Fragment fragment, final SubscriptionImportInput input) { + final var confirmationDialog = new ImportConfirmationDialog(); + final var arguments = new Bundle(); + arguments.putParcelable(INPUT, input); + confirmationDialog.setArguments(arguments); confirmationDialog.show(fragment.getParentFragmentManager(), null); } - public void setResultServiceIntent(final Intent resultServiceIntent) { - this.resultServiceIntent = resultServiceIntent; - } - @NonNull @Override public Dialog onCreateDialog(@Nullable final Bundle savedInstanceState) { - assureCorrectAppLanguage(getContext()); - return new AlertDialog.Builder(requireContext()) + final var context = requireContext(); + assureCorrectAppLanguage(context); + return new AlertDialog.Builder(context) .setMessage(R.string.import_network_expensive_warning) .setCancelable(true) .setNegativeButton(R.string.cancel, null) .setPositiveButton(R.string.ok, (dialogInterface, i) -> { - if (resultServiceIntent != null && getContext() != null) { - getContext().startService(resultServiceIntent); - } + final var constraints = new Constraints.Builder() + .setRequiredNetworkType(NetworkType.CONNECTED) + .build(); + final var input = BundleCompat.getParcelable(requireArguments(), INPUT, + SubscriptionImportInput.class); + + final var req = new OneTimeWorkRequest.Builder(SubscriptionImportWorker.class) + .setInputData(input.toData()) + .setExpedited(OutOfQuotaPolicy.RUN_AS_NON_EXPEDITED_WORK_REQUEST) + .setConstraints(constraints) + .build(); + + WorkManager.getInstance(context) + .enqueueUniqueWork(SubscriptionImportWorker.WORK_NAME, + ExistingWorkPolicy.APPEND_OR_REPLACE, req); + dismiss(); }) .create(); @@ -53,10 +70,6 @@ public class ImportConfirmationDialog extends DialogFragment { public void onCreate(@Nullable final Bundle savedInstanceState) { super.onCreate(savedInstanceState); - if (resultServiceIntent == null) { - throw new IllegalStateException("Result intent is null"); - } - Bridge.restoreInstanceState(this, savedInstanceState); } diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionFragment.kt b/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionFragment.kt index e4a9b79a2..91ff7cd27 100644 --- a/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionFragment.kt +++ b/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionFragment.kt @@ -3,7 +3,6 @@ package org.schabi.newpipe.local.subscription import android.app.Activity import android.content.Context import android.content.DialogInterface -import android.content.Intent import android.os.Bundle import android.os.Parcelable import android.view.LayoutInflater @@ -49,11 +48,8 @@ import org.schabi.newpipe.local.subscription.item.FeedGroupCarouselItem import org.schabi.newpipe.local.subscription.item.GroupsHeader import org.schabi.newpipe.local.subscription.item.Header import org.schabi.newpipe.local.subscription.item.ImportSubscriptionsHintPlaceholderItem -import org.schabi.newpipe.local.subscription.services.SubscriptionsExportService -import org.schabi.newpipe.local.subscription.services.SubscriptionsImportService -import org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.KEY_MODE -import org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.KEY_VALUE -import org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.PREVIOUS_EXPORT_MODE +import org.schabi.newpipe.local.subscription.workers.SubscriptionExportWorker +import org.schabi.newpipe.local.subscription.workers.SubscriptionImportInput import org.schabi.newpipe.streams.io.NoFileManagerSafeGuard import org.schabi.newpipe.streams.io.StoredFileHelper import org.schabi.newpipe.ui.emptystate.setEmptyStateComposable @@ -224,21 +220,17 @@ class SubscriptionFragment : BaseStateFragment() { } private fun requestExportResult(result: ActivityResult) { - if (result.data != null && result.resultCode == Activity.RESULT_OK) { - activity.startService( - Intent(activity, SubscriptionsExportService::class.java) - .putExtra(SubscriptionsExportService.KEY_FILE_PATH, result.data?.data) - ) + val data = result.data?.data + if (data != null && result.resultCode == Activity.RESULT_OK) { + SubscriptionExportWorker.schedule(activity, data) } } private fun requestImportResult(result: ActivityResult) { - if (result.data != null && result.resultCode == Activity.RESULT_OK) { + val data = result.data?.dataString + if (data != null && result.resultCode == Activity.RESULT_OK) { ImportConfirmationDialog.show( - this, - Intent(activity, SubscriptionsImportService::class.java) - .putExtra(KEY_MODE, PREVIOUS_EXPORT_MODE) - .putExtra(KEY_VALUE, result.data?.data) + this, SubscriptionImportInput.PreviousExportMode(data) ) } } diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionManager.kt b/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionManager.kt index 474add4f4..8de2c94db 100644 --- a/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionManager.kt +++ b/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionManager.kt @@ -1,7 +1,6 @@ package org.schabi.newpipe.local.subscription import android.content.Context -import android.util.Pair import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers import io.reactivex.rxjava3.core.Completable import io.reactivex.rxjava3.core.Flowable @@ -48,23 +47,16 @@ class SubscriptionManager(context: Context) { } } - fun upsertAll(infoList: List>>): List { - val listEntities = subscriptionTable.upsertAll( - infoList.map { SubscriptionEntity.from(it.first) } - ) + fun upsertAll(infoList: List>) { + val listEntities = infoList.map { SubscriptionEntity.from(it.first) } + subscriptionTable.upsertAll(listEntities) database.runInTransaction { infoList.forEachIndexed { index, info -> - info.second.forEach { - feedDatabaseManager.upsertAll( - listEntities[index].uid, - it.relatedItems.filterIsInstance() - ) - } + val streams = info.second.relatedItems.filterIsInstance() + feedDatabaseManager.upsertAll(listEntities[index].uid, streams) } } - - return listEntities } fun updateChannelInfo(info: ChannelInfo): Completable = diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionsImportFragment.java b/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionsImportFragment.java index 77a70afa9..aee7c0003 100644 --- a/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionsImportFragment.java +++ b/app/src/main/java/org/schabi/newpipe/local/subscription/SubscriptionsImportFragment.java @@ -1,10 +1,6 @@ package org.schabi.newpipe.local.subscription; import static org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.ContentSource.CHANNEL_URL; -import static org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.CHANNEL_URL_MODE; -import static org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.INPUT_STREAM_MODE; -import static org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.KEY_MODE; -import static org.schabi.newpipe.local.subscription.services.SubscriptionsImportService.KEY_VALUE; import android.app.Activity; import android.content.Intent; @@ -37,7 +33,7 @@ import org.schabi.newpipe.error.UserAction; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; -import org.schabi.newpipe.local.subscription.services.SubscriptionsImportService; +import org.schabi.newpipe.local.subscription.workers.SubscriptionImportInput; import org.schabi.newpipe.streams.io.NoFileManagerSafeGuard; import org.schabi.newpipe.streams.io.StoredFileHelper; import org.schabi.newpipe.util.Constants; @@ -168,10 +164,8 @@ public class SubscriptionsImportFragment extends BaseFragment { } public void onImportUrl(final String value) { - ImportConfirmationDialog.show(this, new Intent(activity, SubscriptionsImportService.class) - .putExtra(KEY_MODE, CHANNEL_URL_MODE) - .putExtra(KEY_VALUE, value) - .putExtra(Constants.KEY_SERVICE_ID, currentServiceId)); + ImportConfirmationDialog.show(this, + new SubscriptionImportInput.ChannelUrlMode(currentServiceId, value)); } public void onImportFile() { @@ -186,16 +180,10 @@ public class SubscriptionsImportFragment extends BaseFragment { } private void requestImportFileResult(final ActivityResult result) { - if (result.getData() == null) { - return; - } - - if (result.getResultCode() == Activity.RESULT_OK && result.getData().getData() != null) { + final String data = result.getData() != null ? result.getData().getDataString() : null; + if (result.getResultCode() == Activity.RESULT_OK && data != null) { ImportConfirmationDialog.show(this, - new Intent(activity, SubscriptionsImportService.class) - .putExtra(KEY_MODE, INPUT_STREAM_MODE) - .putExtra(KEY_VALUE, result.getData().getData()) - .putExtra(Constants.KEY_SERVICE_ID, currentServiceId)); + new SubscriptionImportInput.InputStreamMode(currentServiceId, data)); } } diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/services/BaseImportExportService.java b/app/src/main/java/org/schabi/newpipe/local/subscription/services/BaseImportExportService.java deleted file mode 100644 index b7c11b160..000000000 --- a/app/src/main/java/org/schabi/newpipe/local/subscription/services/BaseImportExportService.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright 2018 Mauricio Colli - * BaseImportExportService.java is part of NewPipe - * - * License: GPL-3.0+ - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.schabi.newpipe.local.subscription.services; - -import android.app.Service; -import android.content.Intent; -import android.os.Build; -import android.os.IBinder; -import android.text.TextUtils; -import android.widget.Toast; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.annotation.StringRes; -import androidx.core.app.NotificationCompat; -import androidx.core.app.NotificationManagerCompat; -import androidx.core.app.ServiceCompat; - -import org.reactivestreams.Publisher; -import org.schabi.newpipe.R; -import org.schabi.newpipe.error.ErrorInfo; -import org.schabi.newpipe.error.ErrorUtil; -import org.schabi.newpipe.error.UserAction; -import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; -import org.schabi.newpipe.ktx.ExceptionUtils; -import org.schabi.newpipe.local.subscription.SubscriptionManager; - -import java.io.FileNotFoundException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers; -import io.reactivex.rxjava3.core.Flowable; -import io.reactivex.rxjava3.disposables.CompositeDisposable; -import io.reactivex.rxjava3.functions.Function; -import io.reactivex.rxjava3.processors.PublishProcessor; - -public abstract class BaseImportExportService extends Service { - protected final String TAG = this.getClass().getSimpleName(); - - protected final CompositeDisposable disposables = new CompositeDisposable(); - protected final PublishProcessor notificationUpdater = PublishProcessor.create(); - - protected NotificationManagerCompat notificationManager; - protected NotificationCompat.Builder notificationBuilder; - protected SubscriptionManager subscriptionManager; - - private static final int NOTIFICATION_SAMPLING_PERIOD = 2500; - - protected final AtomicInteger currentProgress = new AtomicInteger(-1); - protected final AtomicInteger maxProgress = new AtomicInteger(-1); - protected final ImportExportEventListener eventListener = new ImportExportEventListener() { - @Override - public void onSizeReceived(final int size) { - maxProgress.set(size); - currentProgress.set(0); - } - - @Override - public void onItemCompleted(final String itemName) { - currentProgress.incrementAndGet(); - notificationUpdater.onNext(itemName); - } - }; - - protected Toast toast; - - @Nullable - @Override - public IBinder onBind(final Intent intent) { - return null; - } - - @Override - public void onCreate() { - super.onCreate(); - subscriptionManager = new SubscriptionManager(this); - setupNotification(); - } - - @Override - public void onDestroy() { - super.onDestroy(); - disposeAll(); - } - - protected void disposeAll() { - disposables.clear(); - } - - /*////////////////////////////////////////////////////////////////////////// - // Notification Impl - //////////////////////////////////////////////////////////////////////////*/ - - protected abstract int getNotificationId(); - - @StringRes - public abstract int getTitle(); - - protected void setupNotification() { - notificationManager = NotificationManagerCompat.from(this); - notificationBuilder = createNotification(); - startForeground(getNotificationId(), notificationBuilder.build()); - - final Function, Publisher> throttleAfterFirstEmission = flow -> - flow.take(1).concatWith(flow.skip(1) - .throttleLast(NOTIFICATION_SAMPLING_PERIOD, TimeUnit.MILLISECONDS)); - - disposables.add(notificationUpdater - .filter(s -> !s.isEmpty()) - .publish(throttleAfterFirstEmission) - .observeOn(AndroidSchedulers.mainThread()) - .subscribe(this::updateNotification)); - } - - protected void updateNotification(final String text) { - notificationBuilder - .setProgress(maxProgress.get(), currentProgress.get(), maxProgress.get() == -1); - - final String progressText = currentProgress + "/" + maxProgress; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - if (!TextUtils.isEmpty(text)) { - notificationBuilder.setContentText(text + " (" + progressText + ")"); - } - } else { - notificationBuilder.setContentInfo(progressText); - notificationBuilder.setContentText(text); - } - - notificationManager.notify(getNotificationId(), notificationBuilder.build()); - } - - protected void stopService() { - postErrorResult(null, null); - } - - protected void stopAndReportError(final Throwable throwable, final String request) { - stopService(); - ErrorUtil.createNotification(this, new ErrorInfo( - throwable, UserAction.SUBSCRIPTION_IMPORT_EXPORT, request)); - } - - protected void postErrorResult(final String title, final String text) { - disposeAll(); - ServiceCompat.stopForeground(this, ServiceCompat.STOP_FOREGROUND_REMOVE); - stopSelf(); - - if (title == null) { - return; - } - - final String textOrEmpty = text == null ? "" : text; - notificationBuilder = new NotificationCompat - .Builder(this, getString(R.string.notification_channel_id)) - .setSmallIcon(R.drawable.ic_newpipe_triangle_white) - .setVisibility(NotificationCompat.VISIBILITY_PUBLIC) - .setContentTitle(title) - .setStyle(new NotificationCompat.BigTextStyle().bigText(textOrEmpty)) - .setContentText(textOrEmpty); - notificationManager.notify(getNotificationId(), notificationBuilder.build()); - } - - protected NotificationCompat.Builder createNotification() { - return new NotificationCompat.Builder(this, getString(R.string.notification_channel_id)) - .setOngoing(true) - .setProgress(-1, -1, true) - .setSmallIcon(R.drawable.ic_newpipe_triangle_white) - .setVisibility(NotificationCompat.VISIBILITY_PUBLIC) - .setContentTitle(getString(getTitle())); - } - - /*////////////////////////////////////////////////////////////////////////// - // Toast - //////////////////////////////////////////////////////////////////////////*/ - - protected void showToast(@StringRes final int message) { - showToast(getString(message)); - } - - protected void showToast(final String message) { - if (toast != null) { - toast.cancel(); - } - - toast = Toast.makeText(this, message, Toast.LENGTH_SHORT); - toast.show(); - } - - /*////////////////////////////////////////////////////////////////////////// - // Error handling - //////////////////////////////////////////////////////////////////////////*/ - - protected void handleError(@StringRes final int errorTitle, @NonNull final Throwable error) { - String message = getErrorMessage(error); - - if (TextUtils.isEmpty(message)) { - final String errorClassName = error.getClass().getName(); - message = getString(R.string.error_occurred_detail, errorClassName); - } - - showToast(errorTitle); - postErrorResult(getString(errorTitle), message); - } - - protected String getErrorMessage(final Throwable error) { - String message = null; - if (error instanceof SubscriptionExtractor.InvalidSourceException) { - message = getString(R.string.invalid_source); - } else if (error instanceof FileNotFoundException) { - message = getString(R.string.invalid_file); - } else if (ExceptionUtils.isNetworkRelated(error)) { - message = getString(R.string.network_error); - } - return message; - } -} diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/services/ImportExportEventListener.java b/app/src/main/java/org/schabi/newpipe/local/subscription/services/ImportExportEventListener.java deleted file mode 100644 index 7352d1f12..000000000 --- a/app/src/main/java/org/schabi/newpipe/local/subscription/services/ImportExportEventListener.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.schabi.newpipe.local.subscription.services; - -public interface ImportExportEventListener { - /** - * Called when the size has been resolved. - * - * @param size how many items there are to import/export - */ - void onSizeReceived(int size); - - /** - * Called every time an item has been parsed/resolved. - * - * @param itemName the name of the subscription item - */ - void onItemCompleted(String itemName); -} diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/services/ImportExportJsonHelper.java b/app/src/main/java/org/schabi/newpipe/local/subscription/services/ImportExportJsonHelper.java deleted file mode 100644 index 611a1cd30..000000000 --- a/app/src/main/java/org/schabi/newpipe/local/subscription/services/ImportExportJsonHelper.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright 2018 Mauricio Colli - * ImportExportJsonHelper.java is part of NewPipe - * - * License: GPL-3.0+ - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.schabi.newpipe.local.subscription.services; - -import androidx.annotation.Nullable; - -import com.grack.nanojson.JsonAppendableWriter; -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonWriter; - -import org.schabi.newpipe.BuildConfig; -import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.InvalidSourceException; -import org.schabi.newpipe.extractor.subscription.SubscriptionItem; - -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; - -/** - * A JSON implementation capable of importing and exporting subscriptions, it has the advantage - * of being able to transfer subscriptions to any device. - */ -public final class ImportExportJsonHelper { - /*////////////////////////////////////////////////////////////////////////// - // Json implementation - //////////////////////////////////////////////////////////////////////////*/ - - private static final String JSON_APP_VERSION_KEY = "app_version"; - private static final String JSON_APP_VERSION_INT_KEY = "app_version_int"; - - private static final String JSON_SUBSCRIPTIONS_ARRAY_KEY = "subscriptions"; - - private static final String JSON_SERVICE_ID_KEY = "service_id"; - private static final String JSON_URL_KEY = "url"; - private static final String JSON_NAME_KEY = "name"; - - private ImportExportJsonHelper() { } - - /** - * Read a JSON source through the input stream. - * - * @param in the input stream (e.g. a file) - * @param eventListener listener for the events generated - * @return the parsed subscription items - */ - public static List readFrom( - final InputStream in, @Nullable final ImportExportEventListener eventListener) - throws InvalidSourceException { - if (in == null) { - throw new InvalidSourceException("input is null"); - } - - final List channels = new ArrayList<>(); - - try { - final JsonObject parentObject = JsonParser.object().from(in); - - if (!parentObject.has(JSON_SUBSCRIPTIONS_ARRAY_KEY)) { - throw new InvalidSourceException("Channels array is null"); - } - - final JsonArray channelsArray = parentObject.getArray(JSON_SUBSCRIPTIONS_ARRAY_KEY); - - if (eventListener != null) { - eventListener.onSizeReceived(channelsArray.size()); - } - - for (final Object o : channelsArray) { - if (o instanceof JsonObject) { - final JsonObject itemObject = (JsonObject) o; - final int serviceId = itemObject.getInt(JSON_SERVICE_ID_KEY, 0); - final String url = itemObject.getString(JSON_URL_KEY); - final String name = itemObject.getString(JSON_NAME_KEY); - - if (url != null && name != null && !url.isEmpty() && !name.isEmpty()) { - channels.add(new SubscriptionItem(serviceId, url, name)); - if (eventListener != null) { - eventListener.onItemCompleted(name); - } - } - } - } - } catch (final Throwable e) { - throw new InvalidSourceException("Couldn't parse json", e); - } - - return channels; - } - - /** - * Write the subscriptions items list as JSON to the output. - * - * @param items the list of subscriptions items - * @param out the output stream (e.g. a file) - * @param eventListener listener for the events generated - */ - public static void writeTo(final List items, final OutputStream out, - @Nullable final ImportExportEventListener eventListener) { - final JsonAppendableWriter writer = JsonWriter.on(out); - writeTo(items, writer, eventListener); - writer.done(); - } - - /** - * @see #writeTo(List, OutputStream, ImportExportEventListener) - * @param items the list of subscriptions items - * @param writer the output {@link JsonAppendableWriter} - * @param eventListener listener for the events generated - */ - public static void writeTo(final List items, - final JsonAppendableWriter writer, - @Nullable final ImportExportEventListener eventListener) { - if (eventListener != null) { - eventListener.onSizeReceived(items.size()); - } - - writer.object(); - - writer.value(JSON_APP_VERSION_KEY, BuildConfig.VERSION_NAME); - writer.value(JSON_APP_VERSION_INT_KEY, BuildConfig.VERSION_CODE); - - writer.array(JSON_SUBSCRIPTIONS_ARRAY_KEY); - for (final SubscriptionItem item : items) { - writer.object(); - writer.value(JSON_SERVICE_ID_KEY, item.getServiceId()); - writer.value(JSON_URL_KEY, item.getUrl()); - writer.value(JSON_NAME_KEY, item.getName()); - writer.end(); - - if (eventListener != null) { - eventListener.onItemCompleted(item.getName()); - } - } - writer.end(); - - writer.end(); - } -} diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/services/SubscriptionsExportService.java b/app/src/main/java/org/schabi/newpipe/local/subscription/services/SubscriptionsExportService.java deleted file mode 100644 index ab1a5a10c..000000000 --- a/app/src/main/java/org/schabi/newpipe/local/subscription/services/SubscriptionsExportService.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright 2018 Mauricio Colli - * SubscriptionsExportService.java is part of NewPipe - * - * License: GPL-3.0+ - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.schabi.newpipe.local.subscription.services; - -import static org.schabi.newpipe.MainActivity.DEBUG; - -import android.content.Intent; -import android.net.Uri; -import android.util.Log; - -import androidx.core.content.IntentCompat; -import androidx.localbroadcastmanager.content.LocalBroadcastManager; - -import org.reactivestreams.Subscriber; -import org.reactivestreams.Subscription; -import org.schabi.newpipe.App; -import org.schabi.newpipe.R; -import org.schabi.newpipe.database.subscription.SubscriptionEntity; -import org.schabi.newpipe.extractor.subscription.SubscriptionItem; -import org.schabi.newpipe.streams.io.SharpOutputStream; -import org.schabi.newpipe.streams.io.StoredFileHelper; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.List; - -import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers; -import io.reactivex.rxjava3.functions.Function; -import io.reactivex.rxjava3.schedulers.Schedulers; - -public class SubscriptionsExportService extends BaseImportExportService { - public static final String KEY_FILE_PATH = "key_file_path"; - - /** - * A {@link LocalBroadcastManager local broadcast} will be made with this action - * when the export is successfully completed. - */ - public static final String EXPORT_COMPLETE_ACTION = App.PACKAGE_NAME + ".local.subscription" - + ".services.SubscriptionsExportService.EXPORT_COMPLETE"; - - private Subscription subscription; - private StoredFileHelper outFile; - private OutputStream outputStream; - - @Override - public int onStartCommand(final Intent intent, final int flags, final int startId) { - if (intent == null || subscription != null) { - return START_NOT_STICKY; - } - - final Uri path = IntentCompat.getParcelableExtra(intent, KEY_FILE_PATH, Uri.class); - if (path == null) { - stopAndReportError(new IllegalStateException( - "Exporting to a file, but the path is null"), - "Exporting subscriptions"); - return START_NOT_STICKY; - } - - try { - outFile = new StoredFileHelper(this, path, "application/json"); - // truncate the file before writing to it, otherwise if the new content is smaller than - // the previous file size, the file will retain part of the previous content and be - // corrupted - outputStream = new SharpOutputStream(outFile.openAndTruncateStream()); - } catch (final IOException e) { - handleError(e); - return START_NOT_STICKY; - } - - startExport(); - - return START_NOT_STICKY; - } - - @Override - protected int getNotificationId() { - return 4567; - } - - @Override - public int getTitle() { - return R.string.export_ongoing; - } - - @Override - protected void disposeAll() { - super.disposeAll(); - if (subscription != null) { - subscription.cancel(); - } - } - - private void startExport() { - showToast(R.string.export_ongoing); - - subscriptionManager.subscriptionTable().getAll().take(1) - .map(subscriptionEntities -> { - final List result = - new ArrayList<>(subscriptionEntities.size()); - for (final SubscriptionEntity entity : subscriptionEntities) { - result.add(new SubscriptionItem(entity.getServiceId(), entity.getUrl(), - entity.getName())); - } - return result; - }) - .map(exportToFile()) - .subscribeOn(Schedulers.io()) - .observeOn(AndroidSchedulers.mainThread()) - .subscribe(getSubscriber()); - } - - private Subscriber getSubscriber() { - return new Subscriber() { - @Override - public void onSubscribe(final Subscription s) { - subscription = s; - s.request(1); - } - - @Override - public void onNext(final StoredFileHelper file) { - if (DEBUG) { - Log.d(TAG, "startExport() success: file = " + file); - } - } - - @Override - public void onError(final Throwable error) { - Log.e(TAG, "onError() called with: error = [" + error + "]", error); - handleError(error); - } - - @Override - public void onComplete() { - LocalBroadcastManager.getInstance(SubscriptionsExportService.this) - .sendBroadcast(new Intent(EXPORT_COMPLETE_ACTION)); - showToast(R.string.export_complete_toast); - stopService(); - } - }; - } - - private Function, StoredFileHelper> exportToFile() { - return subscriptionItems -> { - ImportExportJsonHelper.writeTo(subscriptionItems, outputStream, eventListener); - return outFile; - }; - } - - protected void handleError(final Throwable error) { - super.handleError(R.string.subscriptions_export_unsuccessful, error); - } -} diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/services/SubscriptionsImportService.java b/app/src/main/java/org/schabi/newpipe/local/subscription/services/SubscriptionsImportService.java deleted file mode 100644 index 442c7fddb..000000000 --- a/app/src/main/java/org/schabi/newpipe/local/subscription/services/SubscriptionsImportService.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Copyright 2018 Mauricio Colli - * SubscriptionsImportService.java is part of NewPipe - * - * License: GPL-3.0+ - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.schabi.newpipe.local.subscription.services; - -import static org.schabi.newpipe.MainActivity.DEBUG; -import static org.schabi.newpipe.streams.io.StoredFileHelper.DEFAULT_MIME; - -import android.content.Intent; -import android.net.Uri; -import android.text.TextUtils; -import android.util.Log; -import android.util.Pair; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.core.content.IntentCompat; -import androidx.localbroadcastmanager.content.LocalBroadcastManager; - -import org.reactivestreams.Subscriber; -import org.reactivestreams.Subscription; -import org.schabi.newpipe.App; -import org.schabi.newpipe.R; -import org.schabi.newpipe.database.subscription.SubscriptionEntity; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.channel.ChannelInfo; -import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo; -import org.schabi.newpipe.extractor.subscription.SubscriptionItem; -import org.schabi.newpipe.ktx.ExceptionUtils; -import org.schabi.newpipe.streams.io.SharpInputStream; -import org.schabi.newpipe.streams.io.StoredFileHelper; -import org.schabi.newpipe.util.Constants; -import org.schabi.newpipe.util.ExtractorHelper; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; - -import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers; -import io.reactivex.rxjava3.core.Flowable; -import io.reactivex.rxjava3.core.Notification; -import io.reactivex.rxjava3.functions.Consumer; -import io.reactivex.rxjava3.functions.Function; -import io.reactivex.rxjava3.schedulers.Schedulers; - -public class SubscriptionsImportService extends BaseImportExportService { - public static final int CHANNEL_URL_MODE = 0; - public static final int INPUT_STREAM_MODE = 1; - public static final int PREVIOUS_EXPORT_MODE = 2; - public static final String KEY_MODE = "key_mode"; - public static final String KEY_VALUE = "key_value"; - - /** - * A {@link LocalBroadcastManager local broadcast} will be made with this action - * when the import is successfully completed. - */ - public static final String IMPORT_COMPLETE_ACTION = App.PACKAGE_NAME + ".local.subscription" - + ".services.SubscriptionsImportService.IMPORT_COMPLETE"; - - /** - * How many extractions running in parallel. - */ - public static final int PARALLEL_EXTRACTIONS = 8; - - /** - * Number of items to buffer to mass-insert in the subscriptions table, - * this leads to a better performance as we can then use db transactions. - */ - public static final int BUFFER_COUNT_BEFORE_INSERT = 50; - - private Subscription subscription; - private int currentMode; - private int currentServiceId; - @Nullable - private String channelUrl; - @Nullable - private InputStream inputStream; - @Nullable - private String inputStreamType; - - @Override - public int onStartCommand(final Intent intent, final int flags, final int startId) { - if (intent == null || subscription != null) { - return START_NOT_STICKY; - } - - currentMode = intent.getIntExtra(KEY_MODE, -1); - currentServiceId = intent.getIntExtra(Constants.KEY_SERVICE_ID, Constants.NO_SERVICE_ID); - - if (currentMode == CHANNEL_URL_MODE) { - channelUrl = intent.getStringExtra(KEY_VALUE); - } else { - final Uri uri = IntentCompat.getParcelableExtra(intent, KEY_VALUE, Uri.class); - if (uri == null) { - stopAndReportError(new IllegalStateException( - "Importing from input stream, but file path is null"), - "Importing subscriptions"); - return START_NOT_STICKY; - } - - try { - final StoredFileHelper fileHelper = new StoredFileHelper(this, uri, DEFAULT_MIME); - inputStream = new SharpInputStream(fileHelper.getStream()); - inputStreamType = fileHelper.getType(); - - if (inputStreamType == null || inputStreamType.equals(DEFAULT_MIME)) { - // mime type could not be determined, just take file extension - final String name = fileHelper.getName(); - final int pointIndex = name.lastIndexOf('.'); - if (pointIndex == -1 || pointIndex >= name.length() - 1) { - inputStreamType = DEFAULT_MIME; // no extension, will fail in the extractor - } else { - inputStreamType = name.substring(pointIndex + 1); - } - } - } catch (final IOException e) { - handleError(e); - return START_NOT_STICKY; - } - } - - if (currentMode == -1 || currentMode == CHANNEL_URL_MODE && channelUrl == null) { - final String errorDescription = "Some important field is null or in illegal state: " - + "currentMode=[" + currentMode + "], " - + "channelUrl=[" + channelUrl + "], " - + "inputStream=[" + inputStream + "]"; - stopAndReportError(new IllegalStateException(errorDescription), - "Importing subscriptions"); - return START_NOT_STICKY; - } - - startImport(); - return START_NOT_STICKY; - } - - @Override - protected int getNotificationId() { - return 4568; - } - - @Override - public int getTitle() { - return R.string.import_ongoing; - } - - @Override - protected void disposeAll() { - super.disposeAll(); - if (subscription != null) { - subscription.cancel(); - } - } - - /*////////////////////////////////////////////////////////////////////////// - // Imports - //////////////////////////////////////////////////////////////////////////*/ - - private void startImport() { - showToast(R.string.import_ongoing); - - Flowable> flowable = null; - switch (currentMode) { - case CHANNEL_URL_MODE: - flowable = importFromChannelUrl(); - break; - case INPUT_STREAM_MODE: - flowable = importFromInputStream(); - break; - case PREVIOUS_EXPORT_MODE: - flowable = importFromPreviousExport(); - break; - } - - if (flowable == null) { - final String message = "Flowable given by \"importFrom\" is null " - + "(current mode: " + currentMode + ")"; - stopAndReportError(new IllegalStateException(message), "Importing subscriptions"); - return; - } - - flowable.doOnNext(subscriptionItems -> - eventListener.onSizeReceived(subscriptionItems.size())) - .flatMap(Flowable::fromIterable) - - .parallel(PARALLEL_EXTRACTIONS) - .runOn(Schedulers.io()) - .map((Function>>>) subscriptionItem -> { - try { - final ChannelInfo channelInfo = ExtractorHelper - .getChannelInfo(subscriptionItem.getServiceId(), - subscriptionItem.getUrl(), true) - .blockingGet(); - return Notification.createOnNext(new Pair<>(channelInfo, - Collections.singletonList( - ExtractorHelper.getChannelTab( - subscriptionItem.getServiceId(), - channelInfo.getTabs().get(0), true).blockingGet() - ))); - } catch (final Throwable e) { - return Notification.createOnError(e); - } - }) - .sequential() - - .observeOn(Schedulers.io()) - .doOnNext(getNotificationsConsumer()) - - .buffer(BUFFER_COUNT_BEFORE_INSERT) - .map(upsertBatch()) - - .subscribeOn(Schedulers.io()) - .observeOn(AndroidSchedulers.mainThread()) - .subscribe(getSubscriber()); - } - - private Subscriber> getSubscriber() { - return new Subscriber<>() { - @Override - public void onSubscribe(final Subscription s) { - subscription = s; - s.request(Long.MAX_VALUE); - } - - @Override - public void onNext(final List successfulInserted) { - if (DEBUG) { - Log.d(TAG, "startImport() " + successfulInserted.size() - + " items successfully inserted into the database"); - } - } - - @Override - public void onError(final Throwable error) { - Log.e(TAG, "Got an error!", error); - handleError(error); - } - - @Override - public void onComplete() { - LocalBroadcastManager.getInstance(SubscriptionsImportService.this) - .sendBroadcast(new Intent(IMPORT_COMPLETE_ACTION)); - showToast(R.string.import_complete_toast); - stopService(); - } - }; - } - - private Consumer>>> getNotificationsConsumer() { - return notification -> { - if (notification.isOnNext()) { - final String name = notification.getValue().first.getName(); - eventListener.onItemCompleted(!TextUtils.isEmpty(name) ? name : ""); - } else if (notification.isOnError()) { - final Throwable error = notification.getError(); - final Throwable cause = error.getCause(); - if (error instanceof IOException) { - throw error; - } else if (cause instanceof IOException) { - throw cause; - } else if (ExceptionUtils.isNetworkRelated(error)) { - throw new IOException(error); - } - - eventListener.onItemCompleted(""); - } - }; - } - - private Function>>>, - List> upsertBatch() { - return notificationList -> { - final List>> infoList = - new ArrayList<>(notificationList.size()); - for (final Notification>> n : notificationList) { - if (n.isOnNext()) { - infoList.add(n.getValue()); - } - } - - return subscriptionManager.upsertAll(infoList); - }; - } - - private Flowable> importFromChannelUrl() { - return Flowable.fromCallable(() -> NewPipe.getService(currentServiceId) - .getSubscriptionExtractor() - .fromChannelUrl(channelUrl)); - } - - private Flowable> importFromInputStream() { - Objects.requireNonNull(inputStream); - Objects.requireNonNull(inputStreamType); - - return Flowable.fromCallable(() -> NewPipe.getService(currentServiceId) - .getSubscriptionExtractor() - .fromInputStream(inputStream, inputStreamType)); - } - - private Flowable> importFromPreviousExport() { - return Flowable.fromCallable(() -> ImportExportJsonHelper.readFrom(inputStream, null)); - } - - protected void handleError(@NonNull final Throwable error) { - super.handleError(R.string.subscriptions_import_unsuccessful, error); - } -} diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/workers/ImportExportJsonHelper.kt b/app/src/main/java/org/schabi/newpipe/local/subscription/workers/ImportExportJsonHelper.kt new file mode 100644 index 000000000..d71f5fa89 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/local/subscription/workers/ImportExportJsonHelper.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2018 Mauricio Colli + * ImportExportJsonHelper.java is part of NewPipe + * + * License: GPL-3.0+ + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package org.schabi.newpipe.local.subscription.workers + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.decodeFromStream +import kotlinx.serialization.json.encodeToStream +import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor.InvalidSourceException +import java.io.InputStream +import java.io.OutputStream + +/** + * A JSON implementation capable of importing and exporting subscriptions, it has the advantage + * of being able to transfer subscriptions to any device. + */ +object ImportExportJsonHelper { + private val json = Json { encodeDefaults = true } + + /** + * Read a JSON source through the input stream. + * + * @param in the input stream (e.g. a file) + * @return the parsed subscription items + */ + @JvmStatic + @Throws(InvalidSourceException::class) + fun readFrom(`in`: InputStream?): List { + if (`in` == null) { + throw InvalidSourceException("input is null") + } + + try { + @OptIn(ExperimentalSerializationApi::class) + return json.decodeFromStream(`in`).subscriptions + } catch (e: Throwable) { + throw InvalidSourceException("Couldn't parse json", e) + } + } + + /** + * Write the subscriptions items list as JSON to the output. + * + * @param items the list of subscriptions items + * @param out the output stream (e.g. a file) + */ + @OptIn(ExperimentalSerializationApi::class) + @JvmStatic + fun writeTo( + items: List, + out: OutputStream, + ) { + json.encodeToStream(SubscriptionData(items), out) + } +} diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionData.kt b/app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionData.kt new file mode 100644 index 000000000..174ae7585 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionData.kt @@ -0,0 +1,24 @@ +package org.schabi.newpipe.local.subscription.workers + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import org.schabi.newpipe.BuildConfig + +@Serializable +class SubscriptionData( + val subscriptions: List +) { + @SerialName("app_version") + private val appVersion = BuildConfig.VERSION_NAME + + @SerialName("app_version_int") + private val appVersionInt = BuildConfig.VERSION_CODE +} + +@Serializable +data class SubscriptionItem( + @SerialName("service_id") + val serviceId: Int, + val url: String, + val name: String +) diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionExportWorker.kt b/app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionExportWorker.kt new file mode 100644 index 000000000..6de91f174 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionExportWorker.kt @@ -0,0 +1,119 @@ +package org.schabi.newpipe.local.subscription.workers + +import android.content.Context +import android.content.pm.ServiceInfo +import android.net.Uri +import android.os.Build +import android.util.Log +import android.widget.Toast +import androidx.core.app.NotificationCompat +import androidx.core.net.toUri +import androidx.work.CoroutineWorker +import androidx.work.ExistingWorkPolicy +import androidx.work.ForegroundInfo +import androidx.work.OneTimeWorkRequestBuilder +import androidx.work.OutOfQuotaPolicy +import androidx.work.WorkManager +import androidx.work.WorkerParameters +import androidx.work.workDataOf +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.reactive.awaitFirst +import kotlinx.coroutines.withContext +import org.schabi.newpipe.BuildConfig +import org.schabi.newpipe.NewPipeDatabase +import org.schabi.newpipe.R + +class SubscriptionExportWorker( + appContext: Context, + params: WorkerParameters, +) : CoroutineWorker(appContext, params) { + // This is needed for API levels < 31 (Android S). + override suspend fun getForegroundInfo(): ForegroundInfo { + return createForegroundInfo(applicationContext.getString(R.string.export_ongoing)) + } + + override suspend fun doWork(): Result { + return try { + val uri = inputData.getString(EXPORT_PATH)!!.toUri() + val table = NewPipeDatabase.getInstance(applicationContext).subscriptionDAO() + val subscriptions = + table.all + .awaitFirst() + .map { SubscriptionItem(it.serviceId, it.url, it.name) } + + val qty = subscriptions.size + val title = applicationContext.resources.getQuantityString(R.plurals.export_subscriptions, qty, qty) + setForeground(createForegroundInfo(title)) + + withContext(Dispatchers.IO) { + // Truncate file if it already exists + applicationContext.contentResolver.openOutputStream(uri, "wt")?.use { + ImportExportJsonHelper.writeTo(subscriptions, it) + } + } + + if (BuildConfig.DEBUG) { + Log.i(TAG, "Exported $qty subscriptions") + } + + withContext(Dispatchers.Main) { + Toast + .makeText(applicationContext, R.string.export_complete_toast, Toast.LENGTH_SHORT) + .show() + } + + Result.success() + } catch (e: Exception) { + if (BuildConfig.DEBUG) { + Log.e(TAG, "Error while exporting subscriptions", e) + } + + withContext(Dispatchers.Main) { + Toast + .makeText(applicationContext, R.string.subscriptions_export_unsuccessful, Toast.LENGTH_SHORT) + .show() + } + + return Result.failure() + } + } + + private fun createForegroundInfo(title: String): ForegroundInfo { + val notification = + NotificationCompat + .Builder(applicationContext, NOTIFICATION_CHANNEL_ID) + .setSmallIcon(R.drawable.ic_newpipe_triangle_white) + .setOngoing(true) + .setProgress(-1, -1, true) + .setVisibility(NotificationCompat.VISIBILITY_PUBLIC) + .setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE) + .setContentTitle(title) + .build() + val serviceType = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC else 0 + return ForegroundInfo(NOTIFICATION_ID, notification, serviceType) + } + + companion object { + private const val TAG = "SubscriptionExportWork" + private const val NOTIFICATION_ID = 4567 + private const val NOTIFICATION_CHANNEL_ID = "newpipe" + private const val WORK_NAME = "exportSubscriptions" + private const val EXPORT_PATH = "exportPath" + + fun schedule( + context: Context, + uri: Uri, + ) { + val data = workDataOf(EXPORT_PATH to uri.toString()) + val workRequest = + OneTimeWorkRequestBuilder() + .setInputData(data) + .setExpedited(OutOfQuotaPolicy.RUN_AS_NON_EXPEDITED_WORK_REQUEST) + .build() + + WorkManager + .getInstance(context) + .enqueueUniqueWork(WORK_NAME, ExistingWorkPolicy.APPEND_OR_REPLACE, workRequest) + } + } +} diff --git a/app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionImportWorker.kt b/app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionImportWorker.kt new file mode 100644 index 000000000..86b9c739a --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/local/subscription/workers/SubscriptionImportWorker.kt @@ -0,0 +1,237 @@ +package org.schabi.newpipe.local.subscription.workers + +import android.content.Context +import android.content.pm.ServiceInfo +import android.os.Build +import android.os.Parcelable +import android.util.Log +import android.webkit.MimeTypeMap +import android.widget.Toast +import androidx.core.app.NotificationCompat +import androidx.core.net.toUri +import androidx.work.CoroutineWorker +import androidx.work.Data +import androidx.work.ForegroundInfo +import androidx.work.WorkManager +import androidx.work.WorkerParameters +import androidx.work.workDataOf +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.rx3.await +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import kotlinx.parcelize.Parcelize +import org.schabi.newpipe.BuildConfig +import org.schabi.newpipe.R +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.local.subscription.SubscriptionManager +import org.schabi.newpipe.util.ExtractorHelper + +class SubscriptionImportWorker( + appContext: Context, + params: WorkerParameters, +) : CoroutineWorker(appContext, params) { + // This is needed for API levels < 31 (Android S). + override suspend fun getForegroundInfo(): ForegroundInfo { + return createForegroundInfo(applicationContext.getString(R.string.import_ongoing), null, 0, 0) + } + + override suspend fun doWork(): Result { + val subscriptions = + try { + loadSubscriptionsFromInput(SubscriptionImportInput.fromData(inputData)) + } catch (e: Exception) { + if (BuildConfig.DEBUG) { + Log.e(TAG, "Error while loading subscriptions from path", e) + } + withContext(Dispatchers.Main) { + Toast + .makeText(applicationContext, R.string.subscriptions_import_unsuccessful, Toast.LENGTH_SHORT) + .show() + } + return Result.failure() + } + + val mutex = Mutex() + var index = 1 + val qty = subscriptions.size + var title = + applicationContext.resources.getQuantityString(R.plurals.load_subscriptions, qty, qty) + + val channelInfoList = + try { + withContext(Dispatchers.IO.limitedParallelism(PARALLEL_EXTRACTIONS)) { + subscriptions + .map { + async { + val channelInfo = + ExtractorHelper.getChannelInfo(it.serviceId, it.url, true).await() + val channelTab = + ExtractorHelper.getChannelTab(it.serviceId, channelInfo.tabs[0], true).await() + + val currentIndex = mutex.withLock { index++ } + setForeground(createForegroundInfo(title, channelInfo.name, currentIndex, qty)) + + channelInfo to channelTab + } + }.awaitAll() + } + } catch (e: Exception) { + if (BuildConfig.DEBUG) { + Log.e(TAG, "Error while loading subscription data", e) + } + withContext(Dispatchers.Main) { + Toast.makeText(applicationContext, R.string.subscriptions_import_unsuccessful, Toast.LENGTH_SHORT) + .show() + } + return Result.failure() + } + + title = applicationContext.resources.getQuantityString(R.plurals.import_subscriptions, qty, qty) + setForeground(createForegroundInfo(title, null, 0, 0)) + index = 0 + + val subscriptionManager = SubscriptionManager(applicationContext) + for (chunk in channelInfoList.chunked(BUFFER_COUNT_BEFORE_INSERT)) { + withContext(Dispatchers.IO) { + subscriptionManager.upsertAll(chunk) + } + index += chunk.size + setForeground(createForegroundInfo(title, null, index, qty)) + } + + withContext(Dispatchers.Main) { + Toast.makeText(applicationContext, R.string.import_complete_toast, Toast.LENGTH_SHORT) + .show() + } + + return Result.success() + } + + private suspend fun loadSubscriptionsFromInput(input: SubscriptionImportInput): List { + return withContext(Dispatchers.IO) { + when (input) { + is SubscriptionImportInput.ChannelUrlMode -> + NewPipe.getService(input.serviceId).subscriptionExtractor + .fromChannelUrl(input.url) + .map { SubscriptionItem(it.serviceId, it.url, it.name) } + + is SubscriptionImportInput.InputStreamMode -> + applicationContext.contentResolver.openInputStream(input.url.toUri())?.use { + val contentType = + MimeTypeMap.getFileExtensionFromUrl(input.url).ifEmpty { DEFAULT_MIME } + NewPipe.getService(input.serviceId).subscriptionExtractor + .fromInputStream(it, contentType) + .map { SubscriptionItem(it.serviceId, it.url, it.name) } + } + + is SubscriptionImportInput.PreviousExportMode -> + applicationContext.contentResolver.openInputStream(input.url.toUri())?.use { + ImportExportJsonHelper.readFrom(it) + } + } ?: emptyList() + } + } + + private fun createForegroundInfo( + title: String, + text: String?, + currentProgress: Int, + maxProgress: Int, + ): ForegroundInfo { + val notification = + NotificationCompat + .Builder(applicationContext, NOTIFICATION_CHANNEL_ID) + .setSmallIcon(R.drawable.ic_newpipe_triangle_white) + .setOngoing(true) + .setProgress(maxProgress, currentProgress, currentProgress == 0) + .setVisibility(NotificationCompat.VISIBILITY_PUBLIC) + .setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE) + .setContentTitle(title) + .setContentText(text) + .addAction( + R.drawable.ic_close, + applicationContext.getString(R.string.cancel), + WorkManager.getInstance(applicationContext).createCancelPendingIntent(id), + ).apply { + if (currentProgress > 0 && maxProgress > 0) { + val progressText = "$currentProgress/$maxProgress" + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + setSubText(progressText) + } else { + setContentInfo(progressText) + } + } + }.build() + val serviceType = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC else 0 + + return ForegroundInfo(NOTIFICATION_ID, notification, serviceType) + } + + companion object { + // Log tag length is limited to 23 characters on API levels < 24. + private const val TAG = "SubscriptionImport" + + private const val NOTIFICATION_ID = 4568 + private const val NOTIFICATION_CHANNEL_ID = "newpipe" + private const val DEFAULT_MIME = "application/octet-stream" + private const val PARALLEL_EXTRACTIONS = 8 + private const val BUFFER_COUNT_BEFORE_INSERT = 50 + + const val WORK_NAME = "SubscriptionImportWorker" + } +} + +sealed class SubscriptionImportInput : Parcelable { + @Parcelize + data class ChannelUrlMode(val serviceId: Int, val url: String) : SubscriptionImportInput() + @Parcelize + data class InputStreamMode(val serviceId: Int, val url: String) : SubscriptionImportInput() + @Parcelize + data class PreviousExportMode(val url: String) : SubscriptionImportInput() + + fun toData(): Data { + val (mode, serviceId, url) = when (this) { + is ChannelUrlMode -> Triple(CHANNEL_URL_MODE, serviceId, url) + is InputStreamMode -> Triple(INPUT_STREAM_MODE, serviceId, url) + is PreviousExportMode -> Triple(PREVIOUS_EXPORT_MODE, null, url) + } + return workDataOf("mode" to mode, "service_id" to serviceId, "url" to url) + } + + companion object { + + private const val CHANNEL_URL_MODE = 0 + private const val INPUT_STREAM_MODE = 1 + private const val PREVIOUS_EXPORT_MODE = 2 + + fun fromData(data: Data): SubscriptionImportInput { + val mode = data.getInt("mode", PREVIOUS_EXPORT_MODE) + when (mode) { + CHANNEL_URL_MODE -> { + val serviceId = data.getInt("service_id", -1) + if (serviceId == -1) { + throw IllegalArgumentException("No service id provided") + } + val url = data.getString("url")!! + return ChannelUrlMode(serviceId, url) + } + INPUT_STREAM_MODE -> { + val serviceId = data.getInt("service_id", -1) + if (serviceId == -1) { + throw IllegalArgumentException("No service id provided") + } + val url = data.getString("url")!! + return InputStreamMode(serviceId, url) + } + PREVIOUS_EXPORT_MODE -> { + val url = data.getString("url")!! + return PreviousExportMode(url) + } + else -> throw IllegalArgumentException("Unknown mode: $mode") + } + } + } +} diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index d56996da9..011c69fcb 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -864,4 +864,16 @@ %d comment %d comments + + Exporting %d subscription… + Exporting %d subscriptions… + + + Loading %d subscription… + Loading %d subscriptions… + + + Importing %d subscription… + Importing %d subscriptions… + diff --git a/app/src/test/java/org/schabi/newpipe/local/subscription/services/ImportExportJsonHelperTest.java b/app/src/test/java/org/schabi/newpipe/local/subscription/services/ImportExportJsonHelperTest.java index 4f0f125ec..96bca9733 100644 --- a/app/src/test/java/org/schabi/newpipe/local/subscription/services/ImportExportJsonHelperTest.java +++ b/app/src/test/java/org/schabi/newpipe/local/subscription/services/ImportExportJsonHelperTest.java @@ -5,11 +5,11 @@ import static org.junit.Assert.fail; import org.junit.Test; import org.schabi.newpipe.extractor.subscription.SubscriptionExtractor; -import org.schabi.newpipe.extractor.subscription.SubscriptionItem; +import org.schabi.newpipe.local.subscription.workers.ImportExportJsonHelper; +import org.schabi.newpipe.local.subscription.workers.SubscriptionItem; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; @@ -23,26 +23,22 @@ public class ImportExportJsonHelperTest { final String emptySource = "{\"app_version\":\"0.11.6\",\"app_version_int\": 47,\"subscriptions\":[]}"; - final List items = ImportExportJsonHelper.readFrom( - new ByteArrayInputStream(emptySource.getBytes(StandardCharsets.UTF_8)), null); + final var items = ImportExportJsonHelper.readFrom( + new ByteArrayInputStream(emptySource.getBytes(StandardCharsets.UTF_8))); assertTrue(items.isEmpty()); } @Test public void testInvalidSource() { - final List invalidList = Arrays.asList( - "{}", - "", - null, - "gibberish"); + final var invalidList = Arrays.asList("{}", "", null, "gibberish"); for (final String invalidContent : invalidList) { try { if (invalidContent != null) { final byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8); - ImportExportJsonHelper.readFrom((new ByteArrayInputStream(bytes)), null); + ImportExportJsonHelper.readFrom(new ByteArrayInputStream(bytes)); } else { - ImportExportJsonHelper.readFrom(null, null); + ImportExportJsonHelper.readFrom(null); } fail("didn't throw exception"); @@ -58,38 +54,24 @@ public class ImportExportJsonHelperTest { @Test public void ultimateTest() throws Exception { // Read from file - final List itemsFromFile = readFromFile(); + final var itemsFromFile = readFromFile(); // Test writing to an output final String jsonOut = testWriteTo(itemsFromFile); // Read again - final List itemsSecondRead = readFromWriteTo(jsonOut); + final var itemsSecondRead = readFromWriteTo(jsonOut); // Check if both lists have the exact same items - if (itemsFromFile.size() != itemsSecondRead.size()) { + if (!itemsFromFile.equals(itemsSecondRead)) { fail("The list of items were different from each other"); } - - for (int i = 0; i < itemsFromFile.size(); i++) { - final SubscriptionItem item1 = itemsFromFile.get(i); - final SubscriptionItem item2 = itemsSecondRead.get(i); - - final boolean equals = item1.getServiceId() == item2.getServiceId() - && item1.getUrl().equals(item2.getUrl()) - && item1.getName().equals(item2.getName()); - - if (!equals) { - fail("The list of items were different from each other"); - } - } } private List readFromFile() throws Exception { - final InputStream inputStream = getClass().getClassLoader().getResourceAsStream( - "import_export_test.json"); - final List itemsFromFile = ImportExportJsonHelper.readFrom( - inputStream, null); + final var inputStream = getClass().getClassLoader() + .getResourceAsStream("import_export_test.json"); + final var itemsFromFile = ImportExportJsonHelper.readFrom(inputStream); if (itemsFromFile.isEmpty()) { fail("ImportExportJsonHelper.readFrom(input) returned a null or empty list"); @@ -98,10 +80,10 @@ public class ImportExportJsonHelperTest { return itemsFromFile; } - private String testWriteTo(final List itemsFromFile) throws Exception { - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - ImportExportJsonHelper.writeTo(itemsFromFile, out, null); - final String jsonOut = out.toString("UTF-8"); + private String testWriteTo(final List itemsFromFile) { + final var out = new ByteArrayOutputStream(); + ImportExportJsonHelper.writeTo(itemsFromFile, out); + final String jsonOut = out.toString(StandardCharsets.UTF_8); if (jsonOut.isEmpty()) { fail("JSON returned by writeTo was empty"); @@ -111,10 +93,8 @@ public class ImportExportJsonHelperTest { } private List readFromWriteTo(final String jsonOut) throws Exception { - final ByteArrayInputStream inputStream = new ByteArrayInputStream( - jsonOut.getBytes(StandardCharsets.UTF_8)); - final List secondReadItems = ImportExportJsonHelper.readFrom( - inputStream, null); + final var inputStream = new ByteArrayInputStream(jsonOut.getBytes(StandardCharsets.UTF_8)); + final var secondReadItems = ImportExportJsonHelper.readFrom(inputStream); if (secondReadItems.isEmpty()) { fail("second call to readFrom returned an empty list"); diff --git a/build.gradle b/build.gradle index f3772ac87..f5e002830 100644 --- a/build.gradle +++ b/build.gradle @@ -11,6 +11,7 @@ buildscript { classpath libs.kotlin.gradle.plugin classpath libs.hilt.android.gradle.plugin classpath libs.aboutlibraries.plugin + classpath libs.kotlinx.serialization // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5c5c607bb..8647d8130 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -24,11 +24,11 @@ jsoup = "1.17.2" junit = "4.13.2" kotlin = "2.0.21" kotlinxCoroutinesRx3 = "1.8.1" +kotlinxSerializationJson = "1.7.3" ktlint = "0.45.2" lazycolumnscrollbar = "2.2.0" leakcanary = "2.12" lifecycle = "2.6.2" -localbroadcastmanager = "1.1.0" markwon = "4.6.2" material = "1.11.0" media = "1.7.0" @@ -60,7 +60,7 @@ teamnewpipe-nanojson = "1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751" # to cause jitpack to regenerate the artifact. teamnewpipe-newpipe-extractor = "v0.24.6" webkit = "1.9.0" -work = "2.8.1" +work = "2.10.0" [plugins] aboutlibraries = { id = "com.mikepenz.aboutlibraries.plugin" } @@ -71,6 +71,7 @@ kotlin-android = { id = "kotlin-android" } kotlin-compose = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } kotlin-kapt = { id = "kotlin-kapt" } kotlin-parcelize = { id = "kotlin-parcelize" } +kotlinx-serialization = { id = "kotlinx-serialization" } sonarqube = { id = "org.sonarqube", version.ref = "sonarqube" } [libraries] @@ -98,7 +99,6 @@ androidx-junit = { group = "androidx.test.ext", name = "junit", version.ref = "a androidx-lifecycle-livedata = { group = "androidx.lifecycle", name = "lifecycle-livedata-ktx", version.ref = "lifecycle" } androidx-lifecycle-viewmodel = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-ktx", version.ref = "lifecycle" } androidx-lifecycle-viewmodel-compose = { group = "androidx.lifecycle", name = "lifecycle-viewmodel-compose" } -androidx-localbroadcastmanager = { group = "androidx.localbroadcastmanager", name = "localbroadcastmanager", version.ref = "localbroadcastmanager" } androidx-material = { group = "com.google.android.material", name = "material", version.ref = "material" } androidx-media = { group = "androidx.media", name = "media", version.ref = "media" } androidx-navigation-compose = { group = "androidx.navigation", name = "navigation-compose", version.ref = "navigationCompose" } @@ -136,6 +136,8 @@ junit = { group = "junit", name = "junit", version.ref = "junit" } kotlin-gradle-plugin = { group = "org.jetbrains.kotlin", name = "kotlin-gradle-plugin", version.ref = "kotlin" } kotlin-stdlib = { group = "org.jetbrains.kotlin", name = "kotlin-stdlib", version.ref = "kotlin" } kotlinx-coroutines-rx3 = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-rx3", version.ref = "kotlinxCoroutinesRx3" } +kotlinx-serialization = { group = "org.jetbrains.kotlin", name = "kotlin-serialization", version.ref = "kotlin" } +kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } lazycolumnscrollbar = { group = "com.github.nanihadesuka", name = "LazyColumnScrollbar", version.ref = "lazycolumnscrollbar" } leakcanary-android-core = { module = "com.squareup.leakcanary:leakcanary-android-core", version.ref = "leakcanary" } leakcanary-object-watcher = { group = "com.squareup.leakcanary", name = "leakcanary-object-watcher-android", version.ref = "leakcanary" }