Load only the selected group and customizable updated status timeout

Now only the subscriptions from the selected group by the user will be
loaded.

Also add an option to decide how much time have to pass since the last
refresh before the subscription is deemed as not up to date. This helps
when a subscription appear in multiple groups, since updating in one
will not require to be fetched again in the others.

app/schemas/org.schabi.newpipe.database.AppDatabase/3.json

@@ -2,7 +2,7 @@
   "formatVersion": 1,
   "database": {
     "version": 3,
-    "identityHash": "ecffbb2ea251aeb38a8f508acf2aa404",
+    "identityHash": "83d5d68663102d5fa28d63caaffb396d",
     "entities": [
       {
         "tableName": "subscriptions",
@@ -119,7 +119,7 @@
       },
       {
         "tableName": "streams",
-        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uid` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `service_id` INTEGER NOT NULL, `url` TEXT NOT NULL, `title` TEXT NOT NULL, `stream_type` TEXT NOT NULL, `duration` INTEGER NOT NULL, `uploader` TEXT NOT NULL, `thumbnail_url` TEXT, `view_count` INTEGER, `textual_upload_date` TEXT, `upload_date` INTEGER)",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`uid` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `service_id` INTEGER NOT NULL, `url` TEXT NOT NULL, `title` TEXT NOT NULL, `stream_type` TEXT NOT NULL, `duration` INTEGER NOT NULL, `uploader` TEXT NOT NULL, `thumbnail_url` TEXT, `view_count` INTEGER, `textual_upload_date` TEXT, `upload_date` INTEGER, `is_upload_date_approximation` INTEGER)",
         "fields": [
           {
             "fieldPath": "uid",
@@ -186,6 +186,12 @@
             "columnName": "upload_date",
             "affinity": "INTEGER",
             "notNull": false
+          },
+          {
+            "fieldPath": "isUploadDateApproximation",
+            "columnName": "is_upload_date_approximation",
+            "affinity": "INTEGER",
+            "notNull": false
           }
         ],
         "primaryKey": {
@@ -637,11 +643,50 @@
             ]
           }
         ]
+      },
+      {
+        "tableName": "feed_last_updated",
+        "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`subscription_id` INTEGER NOT NULL, `last_updated` INTEGER, PRIMARY KEY(`subscription_id`), FOREIGN KEY(`subscription_id`) REFERENCES `subscriptions`(`uid`) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)",
+        "fields": [
+          {
+            "fieldPath": "subscriptionId",
+            "columnName": "subscription_id",
+            "affinity": "INTEGER",
+            "notNull": true
+          },
+          {
+            "fieldPath": "lastUpdated",
+            "columnName": "last_updated",
+            "affinity": "INTEGER",
+            "notNull": false
+          }
+        ],
+        "primaryKey": {
+          "columnNames": [
+            "subscription_id"
+          ],
+          "autoGenerate": false
+        },
+        "indices": [],
+        "foreignKeys": [
+          {
+            "table": "subscriptions",
+            "onDelete": "CASCADE",
+            "onUpdate": "CASCADE",
+            "columns": [
+              "subscription_id"
+            ],
+            "referencedColumns": [
+              "uid"
+            ]
+          }
+        ]
       }
     ],
+    "views": [],
     "setupQueries": [
       "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
-      "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, \"ecffbb2ea251aeb38a8f508acf2aa404\")"
+      "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '83d5d68663102d5fa28d63caaffb396d')"
     ]
   }
 }

app/src/main/java/org/schabi/newpipe/database/AppDatabase.java

@@ -9,6 +9,7 @@ import org.schabi.newpipe.database.feed.dao.FeedGroupDAO;
 import org.schabi.newpipe.database.feed.model.FeedEntity;
 import org.schabi.newpipe.database.feed.model.FeedGroupEntity;
 import org.schabi.newpipe.database.feed.model.FeedGroupSubscriptionEntity;
+import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity;
 import org.schabi.newpipe.database.history.dao.SearchHistoryDAO;
 import org.schabi.newpipe.database.history.dao.StreamHistoryDAO;
 import org.schabi.newpipe.database.history.model.SearchHistoryEntry;
@@ -34,7 +35,8 @@ import static org.schabi.newpipe.database.Migrations.DB_VER_3;
                 SubscriptionEntity.class, SearchHistoryEntry.class,
                 StreamEntity.class, StreamHistoryEntity.class, StreamStateEntity.class,
                 PlaylistEntity.class, PlaylistStreamEntity.class, PlaylistRemoteEntity.class,
-                FeedEntity.class, FeedGroupEntity.class, FeedGroupSubscriptionEntity.class
+                FeedEntity.class, FeedGroupEntity.class, FeedGroupSubscriptionEntity.class,
+                FeedLastUpdatedEntity.class
         },
         version = DB_VER_3
 )

app/src/main/java/org/schabi/newpipe/database/Migrations.java

@@ -78,10 +78,11 @@ public class Migrations {
             // Add NOT NULLs and new fields
             database.execSQL("CREATE TABLE IF NOT EXISTS streams_new " +
                     "(uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, service_id INTEGER NOT NULL, url TEXT NOT NULL, title TEXT NOT NULL, stream_type TEXT NOT NULL," +
-                    " duration INTEGER NOT NULL, uploader TEXT NOT NULL, thumbnail_url TEXT, view_count INTEGER, textual_upload_date TEXT, upload_date INTEGER)");
+                    " duration INTEGER NOT NULL, uploader TEXT NOT NULL, thumbnail_url TEXT, view_count INTEGER, textual_upload_date TEXT, upload_date INTEGER," +
+                    " is_upload_date_approximation INTEGER)");
 
-            database.execSQL("INSERT INTO streams_new (uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, view_count, textual_upload_date, upload_date)"+
+            database.execSQL("INSERT INTO streams_new (uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, view_count, textual_upload_date, upload_date, is_upload_date_approximation)"+
-                    " SELECT uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, NULL, NULL, NULL FROM streams");
+                    " SELECT uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, NULL, NULL, NULL, NULL FROM streams");
 
             database.execSQL("DROP TABLE streams");
             database.execSQL("ALTER TABLE streams_new RENAME TO streams");
@@ -93,6 +94,7 @@ public class Migrations {
             database.execSQL("CREATE TABLE IF NOT EXISTS feed_group (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, icon_id INTEGER NOT NULL)");
             database.execSQL("CREATE TABLE IF NOT EXISTS feed_group_subscription_join (group_id INTEGER NOT NULL, subscription_id INTEGER NOT NULL, PRIMARY KEY(group_id, subscription_id), FOREIGN KEY(group_id) REFERENCES feed_group(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)");
             database.execSQL("CREATE INDEX index_feed_group_subscription_join_subscription_id ON feed_group_subscription_join (subscription_id)");
+            database.execSQL("CREATE TABLE IF NOT EXISTS feed_last_updated (subscription_id INTEGER NOT NULL, last_updated INTEGER, PRIMARY KEY(subscription_id), FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)");
         }
     };
 

app/src/main/java/org/schabi/newpipe/database/feed/dao/FeedDAO.kt

@@ -1,12 +1,11 @@
 package org.schabi.newpipe.database.feed.dao
 
-import androidx.room.Dao
+import androidx.room.*
-import androidx.room.Insert
-import androidx.room.OnConflictStrategy
-import androidx.room.Query
 import io.reactivex.Flowable
 import org.schabi.newpipe.database.feed.model.FeedEntity
+import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity
 import org.schabi.newpipe.database.stream.model.StreamEntity
+import org.schabi.newpipe.database.subscription.SubscriptionEntity
 import java.util.*
 
 @Dao
@@ -80,4 +79,69 @@ abstract class FeedDAO {
 
     @Insert(onConflict = OnConflictStrategy.IGNORE)
     abstract fun insertAll(entities: List<FeedEntity>): List<Long>
+
+    @Insert(onConflict = OnConflictStrategy.IGNORE)
+    internal abstract fun insertLastUpdated(lastUpdatedEntity: FeedLastUpdatedEntity): Long
+
+    @Update(onConflict = OnConflictStrategy.IGNORE)
+    internal abstract fun updateLastUpdated(lastUpdatedEntity: FeedLastUpdatedEntity)
+
+    @Transaction
+    open fun setLastUpdatedForSubscription(lastUpdatedEntity: FeedLastUpdatedEntity) {
+        val id = insertLastUpdated(lastUpdatedEntity)
+
+        if (id == -1L) {
+            updateLastUpdated(lastUpdatedEntity)
+        }
+    }
+
+    @Query("""
+        SELECT MIN(lu.last_updated) FROM feed_last_updated lu
+
+        INNER JOIN feed_group_subscription_join fgs
+        ON fgs.subscription_id = lu.subscription_id AND fgs.group_id = :groupId
+        """)
+    abstract fun oldestSubscriptionUpdate(groupId: Long): Flowable<List<Date>>
+
+    @Query("SELECT MIN(last_updated) FROM feed_last_updated")
+    abstract fun oldestSubscriptionUpdateFromAll(): Flowable<List<Date>>
+
+    @Query("SELECT COUNT(*) FROM feed_last_updated WHERE last_updated IS NULL")
+    abstract fun notLoadedCount(): Flowable<Long>
+
+    @Query("""
+        SELECT COUNT(*) FROM subscriptions s
+        
+        INNER JOIN feed_group_subscription_join fgs
+        ON s.uid = fgs.subscription_id AND fgs.group_id = :groupId
+
+        LEFT JOIN feed_last_updated lu
+        ON s.uid = lu.subscription_id 
+
+        WHERE lu.last_updated IS NULL
+        """)
+    abstract fun notLoadedCountForGroup(groupId: Long): Flowable<Long>
+
+    @Query("""
+        SELECT s.* FROM subscriptions s
+
+        LEFT JOIN feed_last_updated lu
+        ON s.uid = lu.subscription_id 
+
+        WHERE lu.last_updated IS NULL OR lu.last_updated < :outdatedThreshold
+        """)
+    abstract fun getAllOutdated(outdatedThreshold: Date): Flowable<List<SubscriptionEntity>>
+
+    @Query("""
+        SELECT s.* FROM subscriptions s
+
+        INNER JOIN feed_group_subscription_join fgs
+        ON s.uid = fgs.subscription_id AND fgs.group_id = :groupId
+
+        LEFT JOIN feed_last_updated lu
+        ON s.uid = lu.subscription_id
+
+        WHERE lu.last_updated IS NULL OR lu.last_updated < :outdatedThreshold
+        """)
+    abstract fun getAllOutdatedForGroup(groupId: Long, outdatedThreshold: Date): Flowable<List<SubscriptionEntity>>
 }

app/src/main/java/org/schabi/newpipe/database/feed/model/FeedLastUpdatedEntity.kt

@@ -0,0 +1,37 @@
+package org.schabi.newpipe.database.feed.model
+
+import androidx.room.ColumnInfo
+import androidx.room.Entity
+import androidx.room.ForeignKey
+import androidx.room.PrimaryKey
+import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity.Companion.FEED_LAST_UPDATED_TABLE
+import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity.Companion.SUBSCRIPTION_ID
+import org.schabi.newpipe.database.subscription.SubscriptionEntity
+import java.util.*
+
+@Entity(
+        tableName = FEED_LAST_UPDATED_TABLE,
+        foreignKeys = [
+            ForeignKey(
+                    entity = SubscriptionEntity::class,
+                    parentColumns = [SubscriptionEntity.SUBSCRIPTION_UID],
+                    childColumns = [SUBSCRIPTION_ID],
+                    onDelete = ForeignKey.CASCADE, onUpdate = ForeignKey.CASCADE, deferred = true)
+        ]
+)
+data class FeedLastUpdatedEntity(
+        @PrimaryKey
+        @ColumnInfo(name = SUBSCRIPTION_ID)
+        var subscriptionId: Long,
+
+        @ColumnInfo(name = LAST_UPDATED)
+        var lastUpdated: Date? = null
+) {
+
+    companion object {
+        const val FEED_LAST_UPDATED_TABLE = "feed_last_updated"
+
+        const val SUBSCRIPTION_ID = "subscription_id"
+        const val LAST_UPDATED = "last_updated"
+    }
+}

app/src/main/java/org/schabi/newpipe/database/stream/dao/StreamDAO.kt

@@ -6,7 +6,8 @@ import org.schabi.newpipe.database.BasicDAO
 import org.schabi.newpipe.database.stream.model.StreamEntity
 import org.schabi.newpipe.database.stream.model.StreamEntity.Companion.STREAM_ID
 import org.schabi.newpipe.extractor.stream.StreamType
-import org.schabi.newpipe.extractor.stream.StreamType.*
+import org.schabi.newpipe.extractor.stream.StreamType.AUDIO_LIVE_STREAM
+import org.schabi.newpipe.extractor.stream.StreamType.LIVE_STREAM
 import java.util.*
 import kotlin.collections.ArrayList
 
@@ -31,8 +32,8 @@ abstract class StreamDAO : BasicDAO<StreamEntity> {
     internal abstract fun silentInsertAllInternal(streams: List<StreamEntity>): List<Long>
 
     @Query("""
-        SELECT uid, stream_type, textual_upload_date, upload_date FROM streams
+        SELECT uid, stream_type, textual_upload_date, upload_date, is_upload_date_approximation, duration 
-        WHERE url = :url AND service_id = :serviceId
+        FROM streams WHERE url = :url AND service_id = :serviceId
         """)
     internal abstract fun getMinimalStreamForCompare(serviceId: Int, url: String): StreamCompareFeed?
 
@@ -79,8 +80,16 @@ abstract class StreamDAO : BasicDAO<StreamEntity> {
 
         val isNewerStreamLive = newerStream.streamType == AUDIO_LIVE_STREAM || newerStream.streamType == LIVE_STREAM
         if (!isNewerStreamLive) {
-            if (existentMinimalStream.uploadDate != null) newerStream.uploadDate = existentMinimalStream.uploadDate
+            if (existentMinimalStream.uploadDate != null && existentMinimalStream.isUploadDateApproximation != true) {
-            if (existentMinimalStream.textualUploadDate != null) newerStream.textualUploadDate = existentMinimalStream.textualUploadDate
+                newerStream.uploadDate = existentMinimalStream.uploadDate
+                newerStream.textualUploadDate = existentMinimalStream.textualUploadDate
+                newerStream.isUploadDateApproximation = existentMinimalStream.isUploadDateApproximation
+            }
+
+            if (existentMinimalStream.duration > 0 && newerStream.duration < 0) {
+                newerStream.duration = existentMinimalStream.duration
+            }
+
         }
     }
 
@@ -105,12 +114,18 @@ abstract class StreamDAO : BasicDAO<StreamEntity> {
             @ColumnInfo(name = STREAM_ID)
             var uid: Long = 0,
 
-            @field:ColumnInfo(name = StreamEntity.STREAM_TYPE)
+            @ColumnInfo(name = StreamEntity.STREAM_TYPE)
             var streamType: StreamType,
 
-            @field:ColumnInfo(name = StreamEntity.STREAM_TEXTUAL_UPLOAD_DATE)
+            @ColumnInfo(name = StreamEntity.STREAM_TEXTUAL_UPLOAD_DATE)
             var textualUploadDate: String? = null,
 
-            @field:ColumnInfo(name = StreamEntity.STREAM_UPLOAD_DATE)
+            @ColumnInfo(name = StreamEntity.STREAM_UPLOAD_DATE)
-            var uploadDate: Date? = null)
+            var uploadDate: Date? = null,
+
+            @ColumnInfo(name = StreamEntity.STREAM_IS_UPLOAD_DATE_APPROXIMATION)
+            var isUploadDateApproximation: Boolean? = null,
+
+            @ColumnInfo(name = StreamEntity.STREAM_DURATION)
+            var duration: Long)
 }

app/src/main/java/org/schabi/newpipe/database/stream/model/StreamEntity.kt

@@ -50,7 +50,10 @@ data class StreamEntity(
         var textualUploadDate: String? = null,
 
         @ColumnInfo(name = STREAM_UPLOAD_DATE)
-        var uploadDate: Date? = null
+        var uploadDate: Date? = null,
+
+        @ColumnInfo(name = STREAM_IS_UPLOAD_DATE_APPROXIMATION)
+        var isUploadDateApproximation: Boolean? = null
 ) : Serializable {
 
     @Ignore
@@ -58,7 +61,8 @@ data class StreamEntity(
             serviceId = item.serviceId, url = item.url, title = item.name,
             streamType = item.streamType, duration = item.duration, uploader = item.uploaderName,
             thumbnailUrl = item.thumbnailUrl, viewCount = item.viewCount,
-            textualUploadDate = item.textualUploadDate, uploadDate = item.uploadDate?.date()?.time
+            textualUploadDate = item.textualUploadDate, uploadDate = item.uploadDate?.date()?.time,
+            isUploadDateApproximation = item.uploadDate?.isApproximation
     )
 
     @Ignore
@@ -66,7 +70,8 @@ data class StreamEntity(
             serviceId = info.serviceId, url = info.url, title = info.name,
             streamType = info.streamType, duration = info.duration, uploader = info.uploaderName,
             thumbnailUrl = info.thumbnailUrl, viewCount = info.viewCount,
-            textualUploadDate = info.textualUploadDate, uploadDate = info.uploadDate?.date()?.time
+            textualUploadDate = info.textualUploadDate, uploadDate = info.uploadDate?.date()?.time,
+            isUploadDateApproximation = info.uploadDate?.isApproximation
     )
 
     @Ignore
@@ -84,7 +89,9 @@ data class StreamEntity(
 
         if (viewCount != null) item.viewCount = viewCount as Long
         item.textualUploadDate = textualUploadDate
-        item.uploadDate = uploadDate?.let { DateWrapper(Calendar.getInstance().apply { time = it }) }
+        item.uploadDate = uploadDate?.let {
+            DateWrapper(Calendar.getInstance().apply { time = it }, isUploadDateApproximation ?: false)
+        }
 
         return item
     }
@@ -103,5 +110,6 @@ data class StreamEntity(
         const val STREAM_VIEWS = "view_count"
         const val STREAM_TEXTUAL_UPLOAD_DATE = "textual_upload_date"
         const val STREAM_UPLOAD_DATE = "upload_date"
+        const val STREAM_IS_UPLOAD_DATE_APPROXIMATION = "is_upload_date_approximation"
     }
 }

app/src/main/java/org/schabi/newpipe/fragments/detail/VideoDetailFragment.java

@@ -1246,12 +1246,22 @@ public class VideoDetailFragment
         final boolean playbackResumeEnabled =
                 prefs.getBoolean(activity.getString(R.string.enable_watch_history_key), true)
                         && prefs.getBoolean(activity.getString(R.string.enable_playback_resume_key), true);
+
         if (!playbackResumeEnabled || info.getDuration() <= 0) {
             positionView.setVisibility(View.INVISIBLE);
             detailPositionView.setVisibility(View.GONE);
-            return;
+
+            // TODO: Remove this check when separation of concerns is done.
+            //  (live streams weren't getting updated because they are mixed)
+            if (!info.getStreamType().equals(StreamType.LIVE_STREAM) &&
+                    !info.getStreamType().equals(StreamType.AUDIO_LIVE_STREAM)) {
+                return;
+            }
         }
         final HistoryRecordManager recordManager = new HistoryRecordManager(requireContext());
+
+        // TODO: Separate concerns when updating database data.
+        //  (move the updating part to when the loading happens)
         positionSubscriber = recordManager.loadStreamState(info)
                 .subscribeOn(Schedulers.io())
                 .onErrorComplete()

app/src/main/java/org/schabi/newpipe/local/feed/FeedDatabaseManager.kt

@@ -1,7 +1,6 @@
 package org.schabi.newpipe.local.feed
 
 import android.content.Context
-import android.preference.PreferenceManager
 import android.util.Log
 import io.reactivex.Completable
 import io.reactivex.Flowable
@@ -10,9 +9,9 @@ import io.reactivex.android.schedulers.AndroidSchedulers
 import io.reactivex.schedulers.Schedulers
 import org.schabi.newpipe.MainActivity.DEBUG
 import org.schabi.newpipe.NewPipeDatabase
-import org.schabi.newpipe.R
 import org.schabi.newpipe.database.feed.model.FeedEntity
 import org.schabi.newpipe.database.feed.model.FeedGroupEntity
+import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity
 import org.schabi.newpipe.database.stream.model.StreamEntity
 import org.schabi.newpipe.extractor.stream.StreamInfoItem
 import org.schabi.newpipe.extractor.stream.StreamType
@@ -55,6 +54,22 @@ class FeedDatabaseManager(context: Context) {
         }
     }
 
+    fun outdatedSubscriptions(outdatedThreshold: Date) = feedTable.getAllOutdated(outdatedThreshold)
+
+    fun notLoadedCount(groupId: Long = -1): Flowable<Long> {
+        return if (groupId != -1L) {
+            feedTable.notLoadedCountForGroup(groupId)
+        } else {
+            feedTable.notLoadedCount()
+        }
+    }
+
+    fun outdatedSubscriptionsForGroup(groupId: Long = -1, outdatedThreshold: Date) =
+            feedTable.getAllOutdatedForGroup(groupId, outdatedThreshold)
+
+    fun markAsOutdated(subscriptionId: Long) = feedTable
+            .setLastUpdatedForSubscription(FeedLastUpdatedEntity(subscriptionId, null))
+
     fun upsertAll(subscriptionId: Long, items: List<StreamInfoItem>,
                   oldestAllowedDate: Date = FEED_OLDEST_ALLOWED_DATE.time) {
         val itemsToInsert = ArrayList<StreamInfoItem>()
@@ -77,24 +92,8 @@ class FeedDatabaseManager(context: Context) {
 
             feedTable.insertAll(feedEntities)
         }
-    }
 
-    fun getLastUpdated(context: Context): Calendar? {
+        feedTable.setLastUpdatedForSubscription(FeedLastUpdatedEntity(subscriptionId, Calendar.getInstance().time))
-        val lastUpdatedMillis = PreferenceManager.getDefaultSharedPreferences(context)
-                .getLong(context.getString(R.string.feed_last_updated_key), -1)
-
-        val calendar = Calendar.getInstance()
-        if (lastUpdatedMillis > 0) {
-            calendar.timeInMillis = lastUpdatedMillis
-            return calendar
-        }
-
-        return null
-    }
-
-    fun setLastUpdated(context: Context, lastUpdated: Calendar?) {
-        PreferenceManager.getDefaultSharedPreferences(context).edit()
-                .putLong(context.getString(R.string.feed_last_updated_key), lastUpdated?.timeInMillis ?: -1).apply()
     }
 
     fun removeOrphansOrOlderStreams(oldestAllowedDate: Date = FEED_OLDEST_ALLOWED_DATE.time) {
@@ -147,4 +146,13 @@ class FeedDatabaseManager(context: Context) {
                 .subscribeOn(Schedulers.io())
                 .observeOn(AndroidSchedulers.mainThread())
     }
+
+    fun oldestSubscriptionUpdate(groupId: Long): Flowable<List<Date>> {
+        return if (groupId == -1L) {
+            feedTable.oldestSubscriptionUpdateFromAll()
+        } else {
+            feedTable.oldestSubscriptionUpdate(groupId)
+        }
+
+    }
 }

app/src/main/java/org/schabi/newpipe/local/feed/FeedFragment.kt

@@ -35,14 +35,15 @@ import org.schabi.newpipe.local.feed.service.FeedLoadService
 import org.schabi.newpipe.report.UserAction
 import org.schabi.newpipe.util.AnimationUtils.animateView
 import org.schabi.newpipe.util.Localization
+import java.util.*
 
 class FeedFragment : BaseListFragment<FeedState, Unit>() {
     private lateinit var viewModel: FeedViewModel
-    private lateinit var feedDatabaseManager: FeedDatabaseManager
     @State @JvmField var listState: Parcelable? = null
 
     private var groupId = -1L
     private var groupName = ""
+    private var oldestSubscriptionUpdate: Calendar? = null
 
     init {
         setHasOptionsMenu(true)
@@ -54,11 +55,6 @@ class FeedFragment : BaseListFragment<FeedState, Unit>() {
 
         groupId = arguments?.getLong(KEY_GROUP_ID, -1) ?: -1
         groupName = arguments?.getString(KEY_GROUP_NAME) ?: ""
-
-        feedDatabaseManager = FeedDatabaseManager(requireContext())
-        if (feedDatabaseManager.getLastUpdated(requireContext()) == null) {
-            triggerUpdate()
-        }
     }
 
     override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle?): View? {
@@ -193,11 +189,7 @@ class FeedFragment : BaseListFragment<FeedState, Unit>() {
 
         loading_progress_bar.isIndeterminate = isIndeterminate ||
                 (progressState.maxProgress > 0 && progressState.currentProgress == 0)
-        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
+        loading_progress_bar.progress = progressState.currentProgress
-            loading_progress_bar?.setProgress(progressState.currentProgress, true)
-        } else {
-            loading_progress_bar.progress = progressState.currentProgress
-        }
 
         loading_progress_bar.max = progressState.maxProgress
     }
@@ -209,9 +201,18 @@ class FeedFragment : BaseListFragment<FeedState, Unit>() {
             listState = null
         }
 
-        if (!loadedState.itemsErrors.isEmpty()) {
+        oldestSubscriptionUpdate = loadedState.oldestUpdate
+
+        if (loadedState.notLoadedCount > 0) {
+            refresh_subtitle_text.visibility = View.VISIBLE
+            refresh_subtitle_text.text = getString(R.string.feed_subscription_not_loaded_count, loadedState.notLoadedCount)
+        } else {
+            refresh_subtitle_text.visibility = View.GONE
+        }
+
+        if (loadedState.itemsErrors.isNotEmpty()) {
             showSnackBarError(loadedState.itemsErrors, UserAction.REQUESTED_FEED,
-                    "none", "Loading feed", R.string.general_error);
+                    "none", "Loading feed", R.string.general_error)
         }
 
         if (loadedState.items.isEmpty()) {
@@ -237,13 +238,12 @@ class FeedFragment : BaseListFragment<FeedState, Unit>() {
     }
 
     private fun updateRefreshViewState() {
-        val lastUpdated = feedDatabaseManager.getLastUpdated(requireContext())
+        val oldestSubscriptionUpdateText = when {
-        val updatedAt = when {
+            oldestSubscriptionUpdate != null -> Localization.relativeTime(oldestSubscriptionUpdate!!)
-            lastUpdated != null -> Localization.relativeTime(lastUpdated)
             else -> "—"
         }
 
-        refresh_text?.text = getString(R.string.feed_last_updated, updatedAt)
+        refresh_text?.text = getString(R.string.feed_oldest_subscription_update, oldestSubscriptionUpdateText)
     }
 
     ///////////////////////////////////////////////////////////////////////////
@@ -256,7 +256,9 @@ class FeedFragment : BaseListFragment<FeedState, Unit>() {
     override fun hasMoreItems() = false
 
     private fun triggerUpdate() {
-        getActivity()?.startService(Intent(requireContext(), FeedLoadService::class.java))
+        getActivity()?.startService(Intent(requireContext(), FeedLoadService::class.java).apply {
+            putExtra(FeedLoadService.EXTRA_GROUP_ID, groupId)
+        })
         listState = null
     }
 

app/src/main/java/org/schabi/newpipe/local/feed/FeedState.kt

@@ -5,7 +5,20 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem
 import java.util.*
 
 sealed class FeedState {
-    data class ProgressState(val currentProgress: Int = -1, val maxProgress: Int = -1, @StringRes val progressMessage: Int = 0) : FeedState()
+    data class ProgressState(
-    data class LoadedState(val lastUpdated: Calendar? = null, val items: List<StreamInfoItem>, var itemsErrors: List<Throwable> = emptyList()) : FeedState()
+            val currentProgress: Int = -1,
-    data class ErrorState(val error: Throwable? = null) : FeedState()
+            val maxProgress: Int = -1,
+            @StringRes val progressMessage: Int = 0
+    ) : FeedState()
+
+    data class LoadedState(
+            val items: List<StreamInfoItem>,
+            val oldestUpdate: Calendar? = null,
+            val notLoadedCount: Long,
+            val itemsErrors: List<Throwable> = emptyList()
+    ) : FeedState()
+
+    data class ErrorState(
+            val error: Throwable? = null
+    ) : FeedState()
 }

app/src/main/java/org/schabi/newpipe/local/feed/FeedViewModel.kt

@@ -6,12 +6,13 @@ import androidx.lifecycle.ViewModel
 import androidx.lifecycle.ViewModelProvider
 import io.reactivex.Flowable
 import io.reactivex.android.schedulers.AndroidSchedulers
-import io.reactivex.functions.Function3
+import io.reactivex.functions.Function4
 import io.reactivex.schedulers.Schedulers
 import org.schabi.newpipe.extractor.stream.StreamInfoItem
 import org.schabi.newpipe.local.feed.service.FeedEventManager
-import org.schabi.newpipe.local.subscription.SubscriptionManager
+import org.schabi.newpipe.local.feed.service.FeedEventManager.Event.*
 import org.schabi.newpipe.util.DEFAULT_THROTTLE_TIMEOUT
+import java.util.*
 import java.util.concurrent.TimeUnit
 
 class FeedViewModel(applicationContext: Context, val groupId: Long = -1) : ViewModel() {
@@ -23,7 +24,6 @@ class FeedViewModel(applicationContext: Context, val groupId: Long = -1) : ViewM
     }
 
     private var feedDatabaseManager: FeedDatabaseManager = FeedDatabaseManager(applicationContext)
-    private var subscriptionManager: SubscriptionManager = SubscriptionManager(applicationContext)
 
     val stateLiveData = MutableLiveData<FeedState>()
 
@@ -31,30 +31,30 @@ class FeedViewModel(applicationContext: Context, val groupId: Long = -1) : ViewM
             .combineLatest(
                     FeedEventManager.events(),
                     feedDatabaseManager.asStreamItems(groupId),
-                    subscriptionManager.subscriptionTable().rowCount(),
+                    feedDatabaseManager.notLoadedCount(groupId),
+                    feedDatabaseManager.oldestSubscriptionUpdate(groupId),
 
-                    Function3 { t1: FeedEventManager.Event, t2: List<StreamInfoItem>, t3: Long -> return@Function3 Triple(first = t1, second = t2, third = t3) }
+                    Function4 { t1: FeedEventManager.Event, t2: List<StreamInfoItem>, t3: Long, t4: List<Date> ->
+                        return@Function4 CombineResultHolder(t1, t2, t3, t4.firstOrNull())
+                    }
             )
             .throttleLatest(DEFAULT_THROTTLE_TIMEOUT, TimeUnit.MILLISECONDS)
             .subscribeOn(Schedulers.io())
             .observeOn(AndroidSchedulers.mainThread())
             .subscribe {
-                val (event, listFromDB, subsCount) = it
+                val (event, listFromDB, notLoadedCount, oldestUpdate) = it
 
-                var lastUpdated = feedDatabaseManager.getLastUpdated(applicationContext)
+                val oldestUpdateCalendar =
-                if (subsCount == 0L && lastUpdated != null) {
+                        oldestUpdate?.let { Calendar.getInstance().apply { time = it } }
-                    feedDatabaseManager.setLastUpdated(applicationContext, null)
-                    lastUpdated = null
-                }
 
                 stateLiveData.postValue(when (event) {
-                    is FeedEventManager.Event.IdleEvent -> FeedState.LoadedState(lastUpdated, listFromDB)
+                    is IdleEvent -> FeedState.LoadedState(listFromDB, oldestUpdateCalendar, notLoadedCount)
-                    is FeedEventManager.Event.ProgressEvent -> FeedState.ProgressState(event.currentProgress, event.maxProgress, event.progressMessage)
+                    is ProgressEvent -> FeedState.ProgressState(event.currentProgress, event.maxProgress, event.progressMessage)
-                    is FeedEventManager.Event.SuccessResultEvent -> FeedState.LoadedState(lastUpdated, listFromDB, event.itemsErrors)
+                    is SuccessResultEvent -> FeedState.LoadedState(listFromDB, oldestUpdateCalendar, notLoadedCount, event.itemsErrors)
-                    is FeedEventManager.Event.ErrorResultEvent -> throw event.error
+                    is ErrorResultEvent -> FeedState.ErrorState(event.error)
                 })
 
-                if (event is FeedEventManager.Event.ErrorResultEvent || event is FeedEventManager.Event.SuccessResultEvent) {
+                if (event is ErrorResultEvent || event is SuccessResultEvent) {
                     FeedEventManager.reset()
                 }
             }
@@ -63,4 +63,6 @@ class FeedViewModel(applicationContext: Context, val groupId: Long = -1) : ViewM
         super.onCleared()
         combineDisposable.dispose()
     }
+
+    private data class CombineResultHolder(val t1: FeedEventManager.Event, val t2: List<StreamInfoItem>, val t3: Long, val t4: Date?)
 }

app/src/main/java/org/schabi/newpipe/local/feed/service/FeedLoadService.kt

@@ -23,6 +23,7 @@ import android.app.Service
 import android.content.Intent
 import android.os.Build
 import android.os.IBinder
+import android.preference.PreferenceManager
 import android.util.Log
 import androidx.core.app.NotificationCompat
 import androidx.core.app.NotificationManagerCompat
@@ -71,6 +72,8 @@ class FeedLoadService : Service() {
          * Number of items to buffer to mass-insert in the database.
          */
         private const val BUFFER_COUNT_BEFORE_INSERT = 20
+
+        const val EXTRA_GROUP_ID: String = "FeedLoadService.EXTRA_GROUP_ID"
     }
 
     private var loadingSubscription: Subscription? = null
@@ -103,7 +106,15 @@ class FeedLoadService : Service() {
         }
 
         setupNotification()
-        startLoading()
+        val defaultSharedPreferences = PreferenceManager.getDefaultSharedPreferences(this)
+
+        val groupId = intent.getLongExtra(EXTRA_GROUP_ID, -1)
+        val thresholdOutdatedMinutesString = defaultSharedPreferences
+                .getString(getString(R.string.feed_update_threshold_key), getString(R.string.feed_update_threshold_default_value))
+        val thresholdOutdatedMinutes = thresholdOutdatedMinutesString!!.toInt()
+
+        startLoading(groupId, thresholdOutdatedMinutes)
+
         return START_NOT_STICKY
     }
 
@@ -129,23 +140,31 @@ class FeedLoadService : Service() {
     // Loading & Handling
     ///////////////////////////////////////////////////////////////////////////
 
-    private class RequestException(message: String, cause: Throwable) : Exception(message, cause) {
+    private class RequestException(val subscriptionId: Long, message: String, cause: Throwable) : Exception(message, cause) {
         companion object {
-            fun wrapList(info: ChannelInfo): List<Throwable> {
+            fun wrapList(subscriptionId: Long, info: ChannelInfo): List<Throwable> {
                 val toReturn = ArrayList<Throwable>(info.errors.size)
                 for (error in info.errors) {
-                    toReturn.add(RequestException(info.serviceId.toString() + ":" + info.url, error))
+                    toReturn.add(RequestException(subscriptionId, info.serviceId.toString() + ":" + info.url, error))
                 }
                 return toReturn
             }
         }
     }
 
-    private fun startLoading() {
+    private fun startLoading(groupId: Long = -1, thresholdOutdatedMinutes: Int) {
         feedResultsHolder = ResultsHolder()
 
-        subscriptionManager
+        val outdatedThreshold = Calendar.getInstance().apply {
-                .subscriptions()
+            add(Calendar.MINUTE, -thresholdOutdatedMinutes)
+        }.time
+
+        val subscriptions = when (groupId) {
+            -1L -> feedDatabaseManager.outdatedSubscriptions(outdatedThreshold)
+            else -> feedDatabaseManager.outdatedSubscriptionsForGroup(groupId, outdatedThreshold)
+        }
+
+        subscriptions
                 .limit(1)
 
                 .doOnNext {
@@ -174,7 +193,7 @@ class FeedLoadService : Service() {
                         return@map Notification.createOnNext(Pair(subscriptionEntity.uid, channelInfo))
                     } catch (e: Throwable) {
                         val request = "${subscriptionEntity.serviceId}:${subscriptionEntity.url}"
-                        val wrapper = RequestException(request, e)
+                        val wrapper = RequestException(subscriptionEntity.uid, request, e)
                         return@map Notification.createOnError<Pair<Long, ChannelInfo>>(wrapper)
                     }
                 }
@@ -235,7 +254,6 @@ class FeedLoadService : Service() {
 
                             postEvent(ProgressEvent(R.string.feed_processing_message))
                             feedDatabaseManager.removeOrphansOrOlderStreams()
-                            feedDatabaseManager.setLastUpdated(this@FeedLoadService, feedResultsHolder.lastUpdated)
 
                             postEvent(SuccessResultEvent(feedResultsHolder.itemsErrors))
                             true
@@ -266,11 +284,17 @@ class FeedLoadService : Service() {
                         subscriptionManager.updateFromInfo(subscriptionId, info)
 
                         if (info.errors.isNotEmpty()) {
-                            feedResultsHolder.addErrors(RequestException.wrapList(info))
+                            feedResultsHolder.addErrors(RequestException.wrapList(subscriptionId, info))
+                            feedDatabaseManager.markAsOutdated(subscriptionId)
                         }
 
                     } else if (notification.isOnError) {
-                        feedResultsHolder.addError(notification.error!!)
+                        val error = notification.error!!
+                        feedResultsHolder.addError(error)
+
+                        if (error is RequestException) {
+                            feedDatabaseManager.markAsOutdated(error.subscriptionId)
+                        }
                     }
                 }
             }
@@ -371,11 +395,6 @@ class FeedLoadService : Service() {
     ///////////////////////////////////////////////////////////////////////////
 
     class ResultsHolder {
-        /**
-         * The time the items have been loaded.
-         */
-        internal lateinit var lastUpdated: Calendar
-
         /**
          * List of errors that may have happen during loading.
          */
@@ -393,7 +412,6 @@ class FeedLoadService : Service() {
 
         fun ready() {
             itemsErrors = itemsErrorsHolder.toList()
-            lastUpdated = Calendar.getInstance()
         }
     }
 }

app/src/main/res/layout/fragment_feed.xml

@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<RelativeLayout
+<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
-    xmlns:android="http://schemas.android.com/apk/res/android"
     xmlns:app="http://schemas.android.com/apk/res-auto"
     xmlns:tools="http://schemas.android.com/tools"
     android:layout_width="match_parent"
@@ -17,38 +16,58 @@
         android:visibility="gone"
         tools:visibility="visible">
 
-        <androidx.appcompat.widget.AppCompatTextView
+        <LinearLayout
-            android:id="@+id/refresh_text"
+            android:id="@+id/refresh_info_container"
             android:layout_width="match_parent"
             android:layout_height="wrap_content"
             android:layout_marginStart="12dp"
             android:layout_toStartOf="@+id/refreshIcon"
-            android:ellipsize="end"
+            android:gravity="center_vertical"
-            android:gravity="start|center_vertical"
+            android:orientation="vertical">
-            android:maxLines="1"
+
-            android:minHeight="24dp"
+            <TextView
-            android:textAppearance="@style/TextAppearance.AppCompat.Body1"
+                android:id="@+id/refresh_text"
-            android:textSize="14sp"
+                android:layout_width="match_parent"
-            tools:text="@string/feed_last_updated"/>
+                android:layout_height="wrap_content"
+                android:ellipsize="end"
+                android:gravity="start|center_vertical"
+                android:maxLines="2"
+                android:textAppearance="@style/TextAppearance.AppCompat.Body1"
+                android:textSize="14sp"
+                tools:text="@tools:sample/lorem/random" />
+
+            <TextView
+                android:id="@+id/refresh_subtitle_text"
+                android:layout_width="match_parent"
+                android:layout_height="wrap_content"
+                android:ellipsize="end"
+                android:gravity="start|center_vertical"
+                android:maxLines="1"
+                android:textAppearance="@style/TextAppearance.AppCompat.Caption"
+                android:textSize="12sp"
+                tools:text="@tools:sample/lorem/random" />
+        </LinearLayout>
 
         <ImageView
             android:id="@+id/refreshIcon"
             android:layout_width="24dp"
-            android:layout_height="24dp"
+            android:layout_height="wrap_content"
+            android:layout_alignTop="@+id/refresh_info_container"
+            android:layout_alignBottom="@+id/refresh_info_container"
             android:layout_alignParentEnd="true"
-            android:layout_marginEnd="12dp"
             android:layout_marginStart="6dp"
+            android:layout_marginEnd="12dp"
             app:srcCompat="?attr/ic_refresh"
-            tools:ignore="ContentDescription"/>
+            tools:ignore="ContentDescription" />
 
         <View
             android:layout_width="match_parent"
             android:layout_height="1dp"
-            android:layout_below="@+id/refresh_text"
+            android:layout_below="@+id/refresh_info_container"
             android:layout_marginLeft="8dp"
-            android:layout_marginRight="8dp"
             android:layout_marginTop="8dp"
-            android:background="?attr/separator_color"/>
+            android:layout_marginRight="8dp"
+            android:background="?attr/separator_color" />
     </RelativeLayout>
 
     <androidx.recyclerview.widget.RecyclerView
@@ -58,8 +77,8 @@
         android:layout_below="@+id/refresh_root_view"
         android:scrollbars="vertical"
         android:visibility="gone"
-        tools:visibility="visible"
+        tools:listitem="@layout/list_stream_item"
-        tools:listitem="@layout/list_stream_item"/>
+        tools:visibility="visible" />
 
     <LinearLayout
         android:id="@+id/loading_panel_root"
@@ -78,7 +97,7 @@
             android:indeterminate="true"
             android:minWidth="128dp"
             android:visibility="gone"
-            tools:visibility="visible"/>
+            tools:visibility="visible" />
 
         <TextView
             android:id="@+id/loading_progress_text"
@@ -90,7 +109,7 @@
             android:textSize="16sp"
             android:visibility="gone"
             tools:text="1/120"
-            tools:visibility="visible"/>
+            tools:visibility="visible" />
     </LinearLayout>
 
     <!--ERROR PANEL-->
@@ -101,7 +120,7 @@
         android:layout_height="wrap_content"
         android:layout_centerInParent="true"
         android:visibility="gone"
-        tools:visibility="visible"/>
+        tools:visibility="visible" />
 
     <include
         android:id="@+id/empty_state_view"
@@ -111,11 +130,11 @@
         android:layout_centerInParent="true"
         android:layout_marginTop="50dp"
         android:visibility="gone"
-        tools:visibility="visible"/>
+        tools:visibility="visible" />
 
     <View
         android:layout_width="match_parent"
         android:layout_height="4dp"
         android:layout_alignParentTop="true"
-        android:background="?attr/toolbar_shadow_drawable"/>
+        android:background="?attr/toolbar_shadow_drawable" />
 </RelativeLayout>

app/src/main/res/layout/list_stream_item.xml

@@ -75,6 +75,7 @@
         android:layout_alignParentBottom="true"
         android:layout_toRightOf="@+id/itemThumbnailView"
         android:layout_toEndOf="@+id/itemThumbnailView"
+        android:ellipsize="end"
         android:lines="1"
         android:textAppearance="?android:attr/textAppearanceSmall"
         android:textSize="@dimen/video_item_search_upload_date_text_size"

app/src/main/res/layout/list_stream_playlist_item.xml

@@ -78,6 +78,7 @@
         android:layout_toStartOf="@id/itemHandle"
         android:layout_toRightOf="@+id/itemThumbnailView"
         android:layout_toEndOf="@+id/itemThumbnailView"
+        android:ellipsize="end"
         android:lines="1"
         android:textAppearance="?android:attr/textAppearanceSmall"
         android:textSize="@dimen/video_item_search_uploader_text_size"

app/src/main/res/values/settings_keys.xml

@@ -181,6 +181,28 @@
     <string name="app_language_key" translatable="false">app_language_key</string>
     <string name="enable_lock_screen_video_thumbnail_key" translatable="false">enable_lock_screen_video_thumbnail</string>
 
+    <string name="feed_update_threshold_key" translatable="false">feed_update_threshold_key</string>
+    <string name="feed_update_threshold_default_value" translatable="false">5</string>
+
+    <string-array name="feed_update_threshold_options" translatable="false">
+        <item>@string/feed_update_threshold_option_always_update</item>
+        <item>5 minutes</item>
+        <item>15 minutes</item>
+        <item>1 hour</item>
+        <item>6 hours</item>
+        <item>12 hours</item>
+        <item>1 day</item>
+    </string-array>
+    <string-array name="feed_update_threshold_values" translatable="false">
+        <item>0</item>
+        <item>5</item>
+        <item>15</item>
+        <item>60</item>
+        <item>360</item>
+        <item>720</item>
+        <item>1440</item>
+    </string-array>
+
     <string name="import_data" translatable="false">import_data</string>
     <string name="export_data" translatable="false">export_data</string>
 

app/src/main/res/values/strings.xml

@@ -601,7 +601,8 @@
     <!-- Feed -->
     <string name="fragment_feed_title">What\'s New</string>
     <string name="feed_groups_header_title">Feed groups</string>
-    <string name="feed_last_updated">Last updated: %s</string>
+    <string name="feed_oldest_subscription_update">Oldest subscription update: %s</string>
+    <string name="feed_subscription_not_loaded_count">Not loaded: %d</string>
     <string name="feed_notification_loading">Loading feed…</string>
     <string name="feed_processing_message">Processing feed…</string>
     <string name="feed_group_dialog_select_subscriptions">Select subscriptions</string>
@@ -610,4 +611,9 @@
     <string name="feed_group_dialog_empty_name">Empty group name</string>
     <string name="feed_group_dialog_name_input">Name</string>
     <string name="feed_create_new_group_button_title">New</string>
+
+    <string name="settings_category_feed_title">Feed</string>
+    <string name="feed_update_threshold_title">Feed update threshold</string>
+    <string name="feed_update_threshold_summary">Time after last update before a subscription is considered outdated — %s</string>
+    <string name="feed_update_threshold_option_always_update">Always update</string>
 </resources>

app/src/main/res/xml/content_settings.xml

@@ -89,4 +89,18 @@
         android:title="@string/export_data_title"
         android:key="@string/export_data"
         android:summary="@string/export_data_summary"/>
+
+    <PreferenceCategory
+        android:layout="@layout/settings_category_header_layout"
+        android:title="@string/settings_category_feed_title">
+
+        <ListPreference
+            app:iconSpaceReserved="false"
+            android:key="@string/feed_update_threshold_key"
+            android:defaultValue="@string/feed_update_threshold_default_value"
+            android:entries="@array/feed_update_threshold_options"
+            android:entryValues="@array/feed_update_threshold_values"
+            android:title="@string/feed_update_threshold_title"
+            android:summary="@string/feed_update_threshold_summary"/>
+    </PreferenceCategory>
 </PreferenceScreen>