Setup initial database for feed implementation

- Update the database diagram
- Add new migration for the new tables and fields
- Enable schema exports
This commit is contained in:
Mauricio Colli
2020-03-14 00:12:30 -03:00
parent 0e2f062148
commit e8ab5aacc7
29 changed files with 1722 additions and 490 deletions
@@ -1,69 +0,0 @@
package org.schabi.newpipe.database.stream;
import androidx.room.ColumnInfo;
import org.schabi.newpipe.database.LocalItem;
import org.schabi.newpipe.database.history.model.StreamHistoryEntity;
import org.schabi.newpipe.database.stream.model.StreamEntity;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamType;
import java.util.Date;
public class StreamStatisticsEntry implements LocalItem {
final public static String STREAM_LATEST_DATE = "latestAccess";
final public static String STREAM_WATCH_COUNT = "watchCount";
@ColumnInfo(name = StreamEntity.STREAM_ID)
final public long uid;
@ColumnInfo(name = StreamEntity.STREAM_SERVICE_ID)
final public int serviceId;
@ColumnInfo(name = StreamEntity.STREAM_URL)
final public String url;
@ColumnInfo(name = StreamEntity.STREAM_TITLE)
final public String title;
@ColumnInfo(name = StreamEntity.STREAM_TYPE)
final public StreamType streamType;
@ColumnInfo(name = StreamEntity.STREAM_DURATION)
final public long duration;
@ColumnInfo(name = StreamEntity.STREAM_UPLOADER)
final public String uploader;
@ColumnInfo(name = StreamEntity.STREAM_THUMBNAIL_URL)
final public String thumbnailUrl;
@ColumnInfo(name = StreamHistoryEntity.JOIN_STREAM_ID)
final public long streamId;
@ColumnInfo(name = StreamStatisticsEntry.STREAM_LATEST_DATE)
final public Date latestAccessDate;
@ColumnInfo(name = StreamStatisticsEntry.STREAM_WATCH_COUNT)
final public long watchCount;
public StreamStatisticsEntry(long uid, int serviceId, String url, String title,
StreamType streamType, long duration, String uploader,
String thumbnailUrl, long streamId, Date latestAccessDate,
long watchCount) {
this.uid = uid;
this.serviceId = serviceId;
this.url = url;
this.title = title;
this.streamType = streamType;
this.duration = duration;
this.uploader = uploader;
this.thumbnailUrl = thumbnailUrl;
this.streamId = streamId;
this.latestAccessDate = latestAccessDate;
this.watchCount = watchCount;
}
public StreamInfoItem toStreamInfoItem() {
StreamInfoItem item = new StreamInfoItem(serviceId, url, title, streamType);
item.setDuration(duration);
item.setUploaderName(uploader);
item.setThumbnailUrl(thumbnailUrl);
return item;
}
@Override
public LocalItemType getLocalItemType() {
return LocalItemType.STATISTIC_STREAM_ITEM;
}
}
@@ -0,0 +1,42 @@
package org.schabi.newpipe.database.stream
import androidx.room.ColumnInfo
import androidx.room.Embedded
import org.schabi.newpipe.database.LocalItem
import org.schabi.newpipe.database.history.model.StreamHistoryEntity
import org.schabi.newpipe.database.stream.model.StreamEntity
import org.schabi.newpipe.extractor.stream.StreamInfoItem
import java.util.*
class StreamStatisticsEntry(
@Embedded
val streamEntity: StreamEntity,
@ColumnInfo(name = StreamHistoryEntity.JOIN_STREAM_ID)
val streamId: Long,
@ColumnInfo(name = STREAM_LATEST_DATE)
val latestAccessDate: Date,
@ColumnInfo(name = STREAM_WATCH_COUNT)
val watchCount: Long
) : LocalItem {
fun toStreamInfoItem(): StreamInfoItem {
val item = StreamInfoItem(streamEntity.serviceId, streamEntity.url, streamEntity.title, streamEntity.streamType)
item.duration = streamEntity.duration
item.uploaderName = streamEntity.uploader
item.thumbnailUrl = streamEntity.thumbnailUrl
return item
}
override fun getLocalItemType(): LocalItem.LocalItemType {
return LocalItem.LocalItemType.STATISTIC_STREAM_ITEM
}
companion object {
const val STREAM_LATEST_DATE = "latestAccess"
const val STREAM_WATCH_COUNT = "watchCount"
}
}
@@ -1,98 +0,0 @@
package org.schabi.newpipe.database.stream.dao;
import androidx.room.Dao;
import androidx.room.Insert;
import androidx.room.OnConflictStrategy;
import androidx.room.Query;
import androidx.room.Transaction;
import org.schabi.newpipe.database.BasicDAO;
import org.schabi.newpipe.database.playlist.model.PlaylistStreamEntity;
import org.schabi.newpipe.database.stream.model.StreamEntity;
import org.schabi.newpipe.database.history.model.StreamHistoryEntity;
import java.util.ArrayList;
import java.util.List;
import io.reactivex.Flowable;
import static org.schabi.newpipe.database.playlist.model.PlaylistStreamEntity.PLAYLIST_STREAM_JOIN_TABLE;
import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_ID;
import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_SERVICE_ID;
import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_TABLE;
import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_URL;
import static org.schabi.newpipe.database.history.model.StreamHistoryEntity.STREAM_HISTORY_TABLE;
@Dao
public abstract class StreamDAO implements BasicDAO<StreamEntity> {
@Override
@Query("SELECT * FROM " + STREAM_TABLE)
public abstract Flowable<List<StreamEntity>> getAll();
@Override
@Query("DELETE FROM " + STREAM_TABLE)
public abstract int deleteAll();
@Override
@Query("SELECT * FROM " + STREAM_TABLE + " WHERE " + STREAM_SERVICE_ID + " = :serviceId")
public abstract Flowable<List<StreamEntity>> listByService(int serviceId);
@Query("SELECT * FROM " + STREAM_TABLE + " WHERE " +
STREAM_URL + " = :url AND " +
STREAM_SERVICE_ID + " = :serviceId")
public abstract Flowable<List<StreamEntity>> getStream(long serviceId, String url);
@Insert(onConflict = OnConflictStrategy.IGNORE)
abstract void silentInsertAllInternal(final List<StreamEntity> streams);
@Query("SELECT " + STREAM_ID + " FROM " + STREAM_TABLE + " WHERE " +
STREAM_URL + " = :url AND " +
STREAM_SERVICE_ID + " = :serviceId")
abstract Long getStreamIdInternal(long serviceId, String url);
@Transaction
public long upsert(StreamEntity stream) {
final Long streamIdCandidate = getStreamIdInternal(stream.getServiceId(), stream.getUrl());
if (streamIdCandidate == null) {
return insert(stream);
} else {
stream.setUid(streamIdCandidate);
update(stream);
return streamIdCandidate;
}
}
@Transaction
public List<Long> upsertAll(List<StreamEntity> streams) {
silentInsertAllInternal(streams);
final List<Long> streamIds = new ArrayList<>(streams.size());
for (StreamEntity stream : streams) {
final Long streamId = getStreamIdInternal(stream.getServiceId(), stream.getUrl());
if (streamId == null) {
throw new IllegalStateException("StreamID cannot be null just after insertion.");
}
streamIds.add(streamId);
stream.setUid(streamId);
}
update(streams);
return streamIds;
}
@Query("DELETE FROM " + STREAM_TABLE + " WHERE " + STREAM_ID +
" NOT IN " +
"(SELECT DISTINCT " + STREAM_ID + " FROM " + STREAM_TABLE +
" LEFT JOIN " + STREAM_HISTORY_TABLE +
" ON " + STREAM_ID + " = " +
StreamHistoryEntity.STREAM_HISTORY_TABLE + "." + StreamHistoryEntity.JOIN_STREAM_ID +
" LEFT JOIN " + PLAYLIST_STREAM_JOIN_TABLE +
" ON " + STREAM_ID + " = " +
PlaylistStreamEntity.PLAYLIST_STREAM_JOIN_TABLE + "." + PlaylistStreamEntity.JOIN_STREAM_ID +
")")
public abstract int deleteOrphans();
}
@@ -0,0 +1,116 @@
package org.schabi.newpipe.database.stream.dao
import androidx.room.*
import io.reactivex.Flowable
import org.schabi.newpipe.database.BasicDAO
import org.schabi.newpipe.database.stream.model.StreamEntity
import org.schabi.newpipe.database.stream.model.StreamEntity.Companion.STREAM_ID
import org.schabi.newpipe.extractor.stream.StreamType
import org.schabi.newpipe.extractor.stream.StreamType.*
import java.util.*
import kotlin.collections.ArrayList
@Dao
abstract class StreamDAO : BasicDAO<StreamEntity> {
@Query("SELECT * FROM streams")
abstract override fun getAll(): Flowable<List<StreamEntity>>
@Query("DELETE FROM streams")
abstract override fun deleteAll(): Int
@Query("SELECT * FROM streams WHERE service_id = :serviceId")
abstract override fun listByService(serviceId: Int): Flowable<List<StreamEntity>>
@Query("SELECT * FROM streams WHERE url = :url AND service_id = :serviceId")
abstract fun getStream(serviceId: Long, url: String): Flowable<List<StreamEntity>>
@Insert(onConflict = OnConflictStrategy.IGNORE)
internal abstract fun silentInsertInternal(stream: StreamEntity): Long
@Insert(onConflict = OnConflictStrategy.IGNORE)
internal abstract fun silentInsertAllInternal(streams: List<StreamEntity>): List<Long>
@Query("""
SELECT uid, stream_type, textual_upload_date, upload_date FROM streams
WHERE url = :url AND service_id = :serviceId
""")
internal abstract fun getMinimalStreamForCompare(serviceId: Int, url: String): StreamCompareFeed?
@Transaction
open fun upsert(newerStream: StreamEntity): Long {
val uid = silentInsertInternal(newerStream)
if (uid != -1L) {
newerStream.uid = uid
return uid
}
compareAndUpdateStream(newerStream)
update(newerStream)
return newerStream.uid
}
@Transaction
open fun upsertAll(streams: List<StreamEntity>): List<Long> {
val insertUidList = silentInsertAllInternal(streams)
val streamIds = ArrayList<Long>(streams.size)
for ((index, uid) in insertUidList.withIndex()) {
val newerStream = streams[index]
if (uid != -1L) {
streamIds.add(uid)
newerStream.uid = uid
continue
}
compareAndUpdateStream(newerStream)
streamIds.add(newerStream.uid)
}
update(streams)
return streamIds
}
private fun compareAndUpdateStream(newerStream: StreamEntity) {
val existentMinimalStream = getMinimalStreamForCompare(newerStream.serviceId, newerStream.url)
?: throw IllegalStateException("Stream cannot be null just after insertion.")
newerStream.uid = existentMinimalStream.uid
val isNewerStreamLive = newerStream.streamType == AUDIO_LIVE_STREAM || newerStream.streamType == LIVE_STREAM
if (!isNewerStreamLive) {
if (existentMinimalStream.uploadDate != null) newerStream.uploadDate = existentMinimalStream.uploadDate
if (existentMinimalStream.textualUploadDate != null) newerStream.textualUploadDate = existentMinimalStream.textualUploadDate
}
}
@Query("""
DELETE FROM streams WHERE
NOT EXISTS (SELECT 1 FROM stream_history sh
WHERE sh.stream_id = streams.uid)
AND NOT EXISTS (SELECT 1 FROM playlist_stream_join ps
WHERE ps.stream_id = streams.uid)
AND NOT EXISTS (SELECT 1 FROM feed f
WHERE f.stream_id = streams.uid)
""")
abstract fun deleteOrphans(): Int
/**
* Minimal entry class used when comparing/updating an existent stream.
*/
internal data class StreamCompareFeed(
@ColumnInfo(name = STREAM_ID)
var uid: Long = 0,
@field:ColumnInfo(name = StreamEntity.STREAM_TYPE)
var streamType: StreamType,
@field:ColumnInfo(name = StreamEntity.STREAM_TEXTUAL_UPLOAD_DATE)
var textualUploadDate: String? = null,
@field:ColumnInfo(name = StreamEntity.STREAM_UPLOAD_DATE)
var uploadDate: Date? = null)
}
@@ -1,153 +0,0 @@
package org.schabi.newpipe.database.stream.model;
import androidx.room.ColumnInfo;
import androidx.room.Entity;
import androidx.room.Ignore;
import androidx.room.Index;
import androidx.room.PrimaryKey;
import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.player.playqueue.PlayQueueItem;
import org.schabi.newpipe.util.Constants;
import java.io.Serializable;
import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_SERVICE_ID;
import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_TABLE;
import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_URL;
@Entity(tableName = STREAM_TABLE,
indices = {@Index(value = {STREAM_SERVICE_ID, STREAM_URL}, unique = true)})
public class StreamEntity implements Serializable {
final public static String STREAM_TABLE = "streams";
final public static String STREAM_ID = "uid";
final public static String STREAM_SERVICE_ID = "service_id";
final public static String STREAM_URL = "url";
final public static String STREAM_TITLE = "title";
final public static String STREAM_TYPE = "stream_type";
final public static String STREAM_DURATION = "duration";
final public static String STREAM_UPLOADER = "uploader";
final public static String STREAM_THUMBNAIL_URL = "thumbnail_url";
@PrimaryKey(autoGenerate = true)
@ColumnInfo(name = STREAM_ID)
private long uid = 0;
@ColumnInfo(name = STREAM_SERVICE_ID)
private int serviceId = Constants.NO_SERVICE_ID;
@ColumnInfo(name = STREAM_URL)
private String url;
@ColumnInfo(name = STREAM_TITLE)
private String title;
@ColumnInfo(name = STREAM_TYPE)
private StreamType streamType;
@ColumnInfo(name = STREAM_DURATION)
private Long duration;
@ColumnInfo(name = STREAM_UPLOADER)
private String uploader;
@ColumnInfo(name = STREAM_THUMBNAIL_URL)
private String thumbnailUrl;
public StreamEntity(final int serviceId, final String title, final String url,
final StreamType streamType, final String thumbnailUrl, final String uploader,
final long duration) {
this.serviceId = serviceId;
this.title = title;
this.url = url;
this.streamType = streamType;
this.thumbnailUrl = thumbnailUrl;
this.uploader = uploader;
this.duration = duration;
}
@Ignore
public StreamEntity(final StreamInfoItem item) {
this(item.getServiceId(), item.getName(), item.getUrl(), item.getStreamType(), item.getThumbnailUrl(),
item.getUploaderName(), item.getDuration());
}
@Ignore
public StreamEntity(final StreamInfo info) {
this(info.getServiceId(), info.getName(), info.getUrl(), info.getStreamType(), info.getThumbnailUrl(),
info.getUploaderName(), info.getDuration());
}
@Ignore
public StreamEntity(final PlayQueueItem item) {
this(item.getServiceId(), item.getTitle(), item.getUrl(), item.getStreamType(),
item.getThumbnailUrl(), item.getUploader(), item.getDuration());
}
public long getUid() {
return uid;
}
public void setUid(long uid) {
this.uid = uid;
}
public int getServiceId() {
return serviceId;
}
public void setServiceId(int serviceId) {
this.serviceId = serviceId;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public StreamType getStreamType() {
return streamType;
}
public void setStreamType(StreamType type) {
this.streamType = type;
}
public Long getDuration() {
return duration;
}
public void setDuration(Long duration) {
this.duration = duration;
}
public String getUploader() {
return uploader;
}
public void setUploader(String uploader) {
this.uploader = uploader;
}
public String getThumbnailUrl() {
return thumbnailUrl;
}
public void setThumbnailUrl(String thumbnailUrl) {
this.thumbnailUrl = thumbnailUrl;
}
}
@@ -0,0 +1,107 @@
package org.schabi.newpipe.database.stream.model
import androidx.room.*
import org.schabi.newpipe.database.stream.model.StreamEntity.Companion.STREAM_SERVICE_ID
import org.schabi.newpipe.database.stream.model.StreamEntity.Companion.STREAM_TABLE
import org.schabi.newpipe.database.stream.model.StreamEntity.Companion.STREAM_URL
import org.schabi.newpipe.extractor.localization.DateWrapper
import org.schabi.newpipe.extractor.stream.StreamInfo
import org.schabi.newpipe.extractor.stream.StreamInfoItem
import org.schabi.newpipe.extractor.stream.StreamType
import org.schabi.newpipe.player.playqueue.PlayQueueItem
import java.io.Serializable
import java.util.*
@Entity(tableName = STREAM_TABLE,
indices = [
Index(value = [STREAM_SERVICE_ID, STREAM_URL], unique = true)
]
)
data class StreamEntity(
@PrimaryKey(autoGenerate = true)
@ColumnInfo(name = STREAM_ID)
var uid: Long = 0,
@ColumnInfo(name = STREAM_SERVICE_ID)
var serviceId: Int,
@ColumnInfo(name = STREAM_URL)
var url: String,
@ColumnInfo(name = STREAM_TITLE)
var title: String,
@ColumnInfo(name = STREAM_TYPE)
var streamType: StreamType,
@ColumnInfo(name = STREAM_DURATION)
var duration: Long,
@ColumnInfo(name = STREAM_UPLOADER)
var uploader: String,
@ColumnInfo(name = STREAM_THUMBNAIL_URL)
var thumbnailUrl: String? = null,
@ColumnInfo(name = STREAM_VIEWS)
var viewCount: Long? = null,
@ColumnInfo(name = STREAM_TEXTUAL_UPLOAD_DATE)
var textualUploadDate: String? = null,
@ColumnInfo(name = STREAM_UPLOAD_DATE)
var uploadDate: Date? = null
) : Serializable {
@Ignore
constructor(item: StreamInfoItem) : this(
serviceId = item.serviceId, url = item.url, title = item.name,
streamType = item.streamType, duration = item.duration, uploader = item.uploaderName,
thumbnailUrl = item.thumbnailUrl, viewCount = item.viewCount,
textualUploadDate = item.textualUploadDate, uploadDate = item.uploadDate?.date()?.time
)
@Ignore
constructor(info: StreamInfo) : this(
serviceId = info.serviceId, url = info.url, title = info.name,
streamType = info.streamType, duration = info.duration, uploader = info.uploaderName,
thumbnailUrl = info.thumbnailUrl, viewCount = info.viewCount,
textualUploadDate = info.textualUploadDate, uploadDate = info.uploadDate?.date()?.time
)
@Ignore
constructor(item: PlayQueueItem) : this(
serviceId = item.serviceId, url = item.url, title = item.title,
streamType = item.streamType, duration = item.duration, uploader = item.uploader,
thumbnailUrl = item.thumbnailUrl
)
fun toStreamInfoItem(): StreamInfoItem {
val item = StreamInfoItem(serviceId, url, title, streamType)
item.duration = duration
item.uploaderName = uploader
item.thumbnailUrl = thumbnailUrl
if (viewCount != null) item.viewCount = viewCount as Long
item.textualUploadDate = textualUploadDate
item.uploadDate = uploadDate?.let { DateWrapper(Calendar.getInstance().apply { time = it }) }
return item
}
companion object {
const val STREAM_TABLE = "streams"
const val STREAM_ID = "uid"
const val STREAM_SERVICE_ID = "service_id"
const val STREAM_URL = "url"
const val STREAM_TITLE = "title"
const val STREAM_TYPE = "stream_type"
const val STREAM_DURATION = "duration"
const val STREAM_UPLOADER = "uploader"
const val STREAM_THUMBNAIL_URL = "thumbnail_url"
const val STREAM_VIEWS = "view_count"
const val STREAM_TEXTUAL_UPLOAD_DATE = "textual_upload_date"
const val STREAM_UPLOAD_DATE = "upload_date"
}
}