1
0
mirror of https://github.com/TeamNewPipe/NewPipe synced 2025-02-09 15:40:08 +00:00

-Improved bulk stream upsert into playlist performance by 5x.

-Added custom info item type for plain stream entity.
This commit is contained in:
John Zhen Mo 2018-01-22 14:13:11 -08:00
parent 776dbc34f7
commit a74c4168f3
7 changed files with 64 additions and 31 deletions

View File

@ -1,6 +1,8 @@
package org.schabi.newpipe.database.stream.dao; package org.schabi.newpipe.database.stream.dao;
import android.arch.persistence.room.Dao; import android.arch.persistence.room.Dao;
import android.arch.persistence.room.Insert;
import android.arch.persistence.room.OnConflictStrategy;
import android.arch.persistence.room.Query; import android.arch.persistence.room.Query;
import android.arch.persistence.room.Transaction; import android.arch.persistence.room.Transaction;
@ -12,6 +14,7 @@ import java.util.List;
import io.reactivex.Flowable; import io.reactivex.Flowable;
import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_ID;
import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_SERVICE_ID; import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_SERVICE_ID;
import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_TABLE; import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_TABLE;
import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_URL; import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_URL;
@ -31,27 +34,47 @@ public abstract class StreamDAO implements BasicDAO<StreamEntity> {
public abstract Flowable<List<StreamEntity>> listByService(int serviceId); public abstract Flowable<List<StreamEntity>> listByService(int serviceId);
@Query("SELECT * FROM " + STREAM_TABLE + " WHERE " + @Query("SELECT * FROM " + STREAM_TABLE + " WHERE " +
STREAM_URL + " LIKE :url AND " + STREAM_URL + " = :url AND " +
STREAM_SERVICE_ID + " = :serviceId") STREAM_SERVICE_ID + " = :serviceId")
public abstract Flowable<List<StreamEntity>> getStream(long serviceId, String url); public abstract Flowable<List<StreamEntity>> getStream(long serviceId, String url);
@Query("SELECT * FROM " + STREAM_TABLE + " WHERE " + @Insert(onConflict = OnConflictStrategy.IGNORE)
STREAM_URL + " LIKE :url AND " + abstract void silentInsertAllInternal(final List<StreamEntity> streams);
@Query("SELECT " + STREAM_ID + " FROM " + STREAM_TABLE + " WHERE " +
STREAM_URL + " = :url AND " +
STREAM_SERVICE_ID + " = :serviceId") STREAM_SERVICE_ID + " = :serviceId")
abstract List<StreamEntity> getStreamInternal(long serviceId, String url); abstract Long getStreamIdInternal(long serviceId, String url);
@Transaction @Transaction
public long upsert(StreamEntity stream) { public long upsert(StreamEntity stream) {
final List<StreamEntity> streams = getStreamInternal(stream.getServiceId(), stream.getUrl()); final Long streamIdCandidate = getStreamIdInternal(stream.getServiceId(), stream.getUrl());
final long uid; if (streamIdCandidate == null) {
if (streams.isEmpty()) { return insert(stream);
uid = insert(stream);
} else { } else {
uid = streams.get(0).getUid(); stream.setUid(streamIdCandidate);
stream.setUid(uid);
update(stream); update(stream);
return streamIdCandidate;
} }
return uid; }
@Transaction
public List<Long> upsertAll(List<StreamEntity> streams) {
silentInsertAllInternal(streams);
final List<Long> streamIds = new ArrayList<>(streams.size());
for (StreamEntity stream : streams) {
final Long streamId = getStreamIdInternal(stream.getServiceId(), stream.getUrl());
if (streamId == null) {
throw new IllegalStateException("StreamID cannot be null just after insertion.");
}
streamIds.add(streamId);
stream.setUid(streamId);
}
update(streams);
return streamIds;
} }
} }

View File

@ -9,6 +9,7 @@ import android.arch.persistence.room.PrimaryKey;
import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.StreamInfo;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.info_list.stored.StreamEntityInfoItem;
import org.schabi.newpipe.playlist.PlayQueueItem; import org.schabi.newpipe.playlist.PlayQueueItem;
import org.schabi.newpipe.util.Constants; import org.schabi.newpipe.util.Constants;
@ -88,9 +89,9 @@ public class StreamEntity implements Serializable {
} }
@Ignore @Ignore
public StreamInfoItem toStreamInfoItem() throws IllegalArgumentException { public StreamEntityInfoItem toStreamEntityInfoItem() throws IllegalArgumentException {
StreamInfoItem item = new StreamInfoItem( StreamEntityInfoItem item = new StreamEntityInfoItem(getUid(), getServiceId(),
getServiceId(), getUrl(), getTitle(), getStreamType()); getUrl(), getTitle(), getStreamType());
item.setThumbnailUrl(getThumbnailUrl()); item.setThumbnailUrl(getThumbnailUrl());
item.setUploaderName(getUploader()); item.setUploaderName(getUploader());
item.setDuration(getDuration()); item.setDuration(getDuration());

View File

@ -288,7 +288,7 @@ public class LocalPlaylistFragment extends BaseListFragment<List<StreamEntity>,
private List<InfoItem> getStreamItems(final List<StreamEntity> streams) { private List<InfoItem> getStreamItems(final List<StreamEntity> streams) {
List<InfoItem> items = new ArrayList<>(streams.size()); List<InfoItem> items = new ArrayList<>(streams.size());
for (final StreamEntity stream : streams) { for (final StreamEntity stream : streams) {
items.add(stream.toStreamInfoItem()); items.add(stream.toStreamEntityInfoItem());
} }
return items; return items;
} }

View File

@ -58,10 +58,9 @@ public class LocalPlaylistManager {
final int indexOffset) { final int indexOffset) {
List<PlaylistStreamEntity> joinEntities = new ArrayList<>(streams.size()); List<PlaylistStreamEntity> joinEntities = new ArrayList<>(streams.size());
for (int index = 0; index < streams.size(); index++) { final List<Long> streamIds = streamTable.upsertAll(streams);
// Upsert streams and get their ids for (int index = 0; index < streamIds.size(); index++) {
final long streamId = streamTable.upsert(streams.get(index)); joinEntities.add(new PlaylistStreamEntity(playlistId, streamIds.get(index),
joinEntities.add(new PlaylistStreamEntity(playlistId, streamId,
index + indexOffset)); index + indexOffset));
} }
return playlistStreamTable.insertAll(joinEntities); return playlistStreamTable.insertAll(joinEntities);
@ -76,7 +75,7 @@ public class LocalPlaylistManager {
return Completable.fromRunnable(() -> database.runInTransaction(() -> { return Completable.fromRunnable(() -> database.runInTransaction(() -> {
playlistStreamTable.deleteBatch(playlistId); playlistStreamTable.deleteBatch(playlistId);
playlistStreamTable.insertAll(joinEntities); playlistStreamTable.insertAll(joinEntities);
})); })).subscribeOn(Schedulers.io());
} }
public Flowable<List<PlaylistMetadataEntry>> getPlaylists() { public Flowable<List<PlaylistMetadataEntry>> getPlaylists() {

View File

@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem;
import static org.schabi.newpipe.util.Constants.NO_SERVICE_ID; import static org.schabi.newpipe.util.Constants.NO_SERVICE_ID;
import static org.schabi.newpipe.util.Constants.NO_URL; import static org.schabi.newpipe.util.Constants.NO_URL;
public class LocalPlaylistInfoItem extends PlaylistInfoItem { public final class LocalPlaylistInfoItem extends PlaylistInfoItem {
private final long playlistId; private final long playlistId;
public LocalPlaylistInfoItem(final long playlistId, final String name) { public LocalPlaylistInfoItem(final long playlistId, final String name) {

View File

@ -0,0 +1,18 @@
package org.schabi.newpipe.info_list.stored;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamType;
public class StreamEntityInfoItem extends StreamInfoItem {
protected final long streamId;
public StreamEntityInfoItem(final long streamId, final int serviceId,
final String url, final String name, final StreamType type) {
super(serviceId, url, name, type);
this.streamId = streamId;
}
public long getStreamId() {
return streamId;
}
}

View File

@ -1,24 +1,16 @@
package org.schabi.newpipe.info_list.stored; package org.schabi.newpipe.info_list.stored;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.StreamType;
import java.util.Date; import java.util.Date;
public class StreamStatisticsInfoItem extends StreamInfoItem { public final class StreamStatisticsInfoItem extends StreamEntityInfoItem {
private final long streamId;
private Date latestAccessDate; private Date latestAccessDate;
private long watchCount; private long watchCount;
public StreamStatisticsInfoItem(final long streamId, final int serviceId, public StreamStatisticsInfoItem(final long streamId, final int serviceId,
final String url, final String name, final StreamType type) { final String url, final String name, final StreamType type) {
super(serviceId, url, name, type); super(streamId, serviceId, url, name, type);
this.streamId = streamId;
}
public long getStreamId() {
return streamId;
} }
public Date getLatestAccessDate() { public Date getLatestAccessDate() {