From a74c4168f301da7bf692047c9cfb1f78035475b7 Mon Sep 17 00:00:00 2001 From: John Zhen Mo Date: Mon, 22 Jan 2018 14:13:11 -0800 Subject: [PATCH] -Improved bulk stream upsert into playlist performance by 5x. -Added custom info item type for plain stream entity. --- .../database/stream/dao/StreamDAO.java | 45 ++++++++++++++----- .../database/stream/model/StreamEntity.java | 7 +-- .../local/LocalPlaylistFragment.java | 2 +- .../fragments/local/LocalPlaylistManager.java | 9 ++-- .../stored/LocalPlaylistInfoItem.java | 2 +- .../stored/StreamEntityInfoItem.java | 18 ++++++++ .../stored/StreamStatisticsInfoItem.java | 12 +---- 7 files changed, 64 insertions(+), 31 deletions(-) create mode 100644 app/src/main/java/org/schabi/newpipe/info_list/stored/StreamEntityInfoItem.java diff --git a/app/src/main/java/org/schabi/newpipe/database/stream/dao/StreamDAO.java b/app/src/main/java/org/schabi/newpipe/database/stream/dao/StreamDAO.java index f7807ef42..ee246db1a 100644 --- a/app/src/main/java/org/schabi/newpipe/database/stream/dao/StreamDAO.java +++ b/app/src/main/java/org/schabi/newpipe/database/stream/dao/StreamDAO.java @@ -1,6 +1,8 @@ package org.schabi.newpipe.database.stream.dao; import android.arch.persistence.room.Dao; +import android.arch.persistence.room.Insert; +import android.arch.persistence.room.OnConflictStrategy; import android.arch.persistence.room.Query; import android.arch.persistence.room.Transaction; @@ -12,6 +14,7 @@ import java.util.List; import io.reactivex.Flowable; +import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_ID; import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_SERVICE_ID; import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_TABLE; import static org.schabi.newpipe.database.stream.model.StreamEntity.STREAM_URL; @@ -31,27 +34,47 @@ public abstract class StreamDAO implements BasicDAO { public abstract Flowable> listByService(int serviceId); @Query("SELECT * FROM " + STREAM_TABLE + " WHERE " + - STREAM_URL + " LIKE :url AND " + + STREAM_URL + " = :url AND " + STREAM_SERVICE_ID + " = :serviceId") public abstract Flowable> getStream(long serviceId, String url); - @Query("SELECT * FROM " + STREAM_TABLE + " WHERE " + - STREAM_URL + " LIKE :url AND " + + @Insert(onConflict = OnConflictStrategy.IGNORE) + abstract void silentInsertAllInternal(final List streams); + + @Query("SELECT " + STREAM_ID + " FROM " + STREAM_TABLE + " WHERE " + + STREAM_URL + " = :url AND " + STREAM_SERVICE_ID + " = :serviceId") - abstract List getStreamInternal(long serviceId, String url); + abstract Long getStreamIdInternal(long serviceId, String url); @Transaction public long upsert(StreamEntity stream) { - final List streams = getStreamInternal(stream.getServiceId(), stream.getUrl()); + final Long streamIdCandidate = getStreamIdInternal(stream.getServiceId(), stream.getUrl()); - final long uid; - if (streams.isEmpty()) { - uid = insert(stream); + if (streamIdCandidate == null) { + return insert(stream); } else { - uid = streams.get(0).getUid(); - stream.setUid(uid); + stream.setUid(streamIdCandidate); update(stream); + return streamIdCandidate; } - return uid; + } + + @Transaction + public List upsertAll(List streams) { + silentInsertAllInternal(streams); + + final List streamIds = new ArrayList<>(streams.size()); + for (StreamEntity stream : streams) { + final Long streamId = getStreamIdInternal(stream.getServiceId(), stream.getUrl()); + if (streamId == null) { + throw new IllegalStateException("StreamID cannot be null just after insertion."); + } + + streamIds.add(streamId); + stream.setUid(streamId); + } + + update(streams); + return streamIds; } } diff --git a/app/src/main/java/org/schabi/newpipe/database/stream/model/StreamEntity.java b/app/src/main/java/org/schabi/newpipe/database/stream/model/StreamEntity.java index 0b73e81e9..eb078a03c 100644 --- a/app/src/main/java/org/schabi/newpipe/database/stream/model/StreamEntity.java +++ b/app/src/main/java/org/schabi/newpipe/database/stream/model/StreamEntity.java @@ -9,6 +9,7 @@ import android.arch.persistence.room.PrimaryKey; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamType; +import org.schabi.newpipe.info_list.stored.StreamEntityInfoItem; import org.schabi.newpipe.playlist.PlayQueueItem; import org.schabi.newpipe.util.Constants; @@ -88,9 +89,9 @@ public class StreamEntity implements Serializable { } @Ignore - public StreamInfoItem toStreamInfoItem() throws IllegalArgumentException { - StreamInfoItem item = new StreamInfoItem( - getServiceId(), getUrl(), getTitle(), getStreamType()); + public StreamEntityInfoItem toStreamEntityInfoItem() throws IllegalArgumentException { + StreamEntityInfoItem item = new StreamEntityInfoItem(getUid(), getServiceId(), + getUrl(), getTitle(), getStreamType()); item.setThumbnailUrl(getThumbnailUrl()); item.setUploaderName(getUploader()); item.setDuration(getDuration()); diff --git a/app/src/main/java/org/schabi/newpipe/fragments/local/LocalPlaylistFragment.java b/app/src/main/java/org/schabi/newpipe/fragments/local/LocalPlaylistFragment.java index 44ecfb924..802532272 100644 --- a/app/src/main/java/org/schabi/newpipe/fragments/local/LocalPlaylistFragment.java +++ b/app/src/main/java/org/schabi/newpipe/fragments/local/LocalPlaylistFragment.java @@ -288,7 +288,7 @@ public class LocalPlaylistFragment extends BaseListFragment, private List getStreamItems(final List streams) { List items = new ArrayList<>(streams.size()); for (final StreamEntity stream : streams) { - items.add(stream.toStreamInfoItem()); + items.add(stream.toStreamEntityInfoItem()); } return items; } diff --git a/app/src/main/java/org/schabi/newpipe/fragments/local/LocalPlaylistManager.java b/app/src/main/java/org/schabi/newpipe/fragments/local/LocalPlaylistManager.java index 89d69d4b4..5633e104d 100644 --- a/app/src/main/java/org/schabi/newpipe/fragments/local/LocalPlaylistManager.java +++ b/app/src/main/java/org/schabi/newpipe/fragments/local/LocalPlaylistManager.java @@ -58,10 +58,9 @@ public class LocalPlaylistManager { final int indexOffset) { List joinEntities = new ArrayList<>(streams.size()); - for (int index = 0; index < streams.size(); index++) { - // Upsert streams and get their ids - final long streamId = streamTable.upsert(streams.get(index)); - joinEntities.add(new PlaylistStreamEntity(playlistId, streamId, + final List streamIds = streamTable.upsertAll(streams); + for (int index = 0; index < streamIds.size(); index++) { + joinEntities.add(new PlaylistStreamEntity(playlistId, streamIds.get(index), index + indexOffset)); } return playlistStreamTable.insertAll(joinEntities); @@ -76,7 +75,7 @@ public class LocalPlaylistManager { return Completable.fromRunnable(() -> database.runInTransaction(() -> { playlistStreamTable.deleteBatch(playlistId); playlistStreamTable.insertAll(joinEntities); - })); + })).subscribeOn(Schedulers.io()); } public Flowable> getPlaylists() { diff --git a/app/src/main/java/org/schabi/newpipe/info_list/stored/LocalPlaylistInfoItem.java b/app/src/main/java/org/schabi/newpipe/info_list/stored/LocalPlaylistInfoItem.java index 3ac5fabb7..b0afe1948 100644 --- a/app/src/main/java/org/schabi/newpipe/info_list/stored/LocalPlaylistInfoItem.java +++ b/app/src/main/java/org/schabi/newpipe/info_list/stored/LocalPlaylistInfoItem.java @@ -5,7 +5,7 @@ import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import static org.schabi.newpipe.util.Constants.NO_SERVICE_ID; import static org.schabi.newpipe.util.Constants.NO_URL; -public class LocalPlaylistInfoItem extends PlaylistInfoItem { +public final class LocalPlaylistInfoItem extends PlaylistInfoItem { private final long playlistId; public LocalPlaylistInfoItem(final long playlistId, final String name) { diff --git a/app/src/main/java/org/schabi/newpipe/info_list/stored/StreamEntityInfoItem.java b/app/src/main/java/org/schabi/newpipe/info_list/stored/StreamEntityInfoItem.java new file mode 100644 index 000000000..a54135211 --- /dev/null +++ b/app/src/main/java/org/schabi/newpipe/info_list/stored/StreamEntityInfoItem.java @@ -0,0 +1,18 @@ +package org.schabi.newpipe.info_list.stored; + +import org.schabi.newpipe.extractor.stream.StreamInfoItem; +import org.schabi.newpipe.extractor.stream.StreamType; + +public class StreamEntityInfoItem extends StreamInfoItem { + protected final long streamId; + + public StreamEntityInfoItem(final long streamId, final int serviceId, + final String url, final String name, final StreamType type) { + super(serviceId, url, name, type); + this.streamId = streamId; + } + + public long getStreamId() { + return streamId; + } +} diff --git a/app/src/main/java/org/schabi/newpipe/info_list/stored/StreamStatisticsInfoItem.java b/app/src/main/java/org/schabi/newpipe/info_list/stored/StreamStatisticsInfoItem.java index 76984d363..6659b551a 100644 --- a/app/src/main/java/org/schabi/newpipe/info_list/stored/StreamStatisticsInfoItem.java +++ b/app/src/main/java/org/schabi/newpipe/info_list/stored/StreamStatisticsInfoItem.java @@ -1,24 +1,16 @@ package org.schabi.newpipe.info_list.stored; -import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamType; import java.util.Date; -public class StreamStatisticsInfoItem extends StreamInfoItem { - private final long streamId; - +public final class StreamStatisticsInfoItem extends StreamEntityInfoItem { private Date latestAccessDate; private long watchCount; public StreamStatisticsInfoItem(final long streamId, final int serviceId, final String url, final String name, final StreamType type) { - super(serviceId, url, name, type); - this.streamId = streamId; - } - - public long getStreamId() { - return streamId; + super(streamId, serviceId, url, name, type); } public Date getLatestAccessDate() {