1
0
mirror of https://github.com/TeamNewPipe/NewPipe synced 2024-12-25 09:30:32 +00:00

Space reserving tweaks for huge video resolutions

* improve space reserving, allows write better 4K/8K video data
* do not use cache dirs in the muxers, Android can force close NewPipe if the device is running out of storage. Is a aggressive cache cleaning >:/
* (for devs) webm & mkv are the same thing
* calculate the final file size inside of the mission, instead getting from the UI
* simplify ps algorithms constructors
* [missing old commit message] simplify the loading of pending downloads
This commit is contained in:
kapodamy 2019-04-25 00:34:29 -03:00
parent 34b2b96158
commit 7b948f83c3
11 changed files with 608 additions and 550 deletions

View File

@ -767,7 +767,8 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck
psArgs = null; psArgs = null;
long videoSize = wrappedVideoStreams.getSizeInBytes((VideoStream) selectedStream); long videoSize = wrappedVideoStreams.getSizeInBytes((VideoStream) selectedStream);
// set nearLength, only, if both sizes are fetched or known. this probably does not work on slow networks // set nearLength, only, if both sizes are fetched or known. This probably
// does not work on slow networks but is later updated in the downloader
if (secondaryStream.getSizeInBytes() > 0 && videoSize > 0) { if (secondaryStream.getSizeInBytes() > 0 && videoSize > 0) {
nearLength = secondaryStream.getSizeInBytes() + videoSize; nearLength = secondaryStream.getSizeInBytes() + videoSize;
} }
@ -793,7 +794,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck
if (secondaryStreamUrl == null) { if (secondaryStreamUrl == null) {
urls = new String[]{selectedStream.getUrl()}; urls = new String[]{selectedStream.getUrl()};
} else { } else {
urls = new String[]{secondaryStreamUrl, selectedStream.getUrl()}; urls = new String[]{selectedStream.getUrl(), secondaryStreamUrl};
} }
DownloadManagerService.startMission(context, urls, storage, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength); DownloadManagerService.startMission(context, urls, storage, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength);

View File

@ -17,6 +17,8 @@ import static org.schabi.newpipe.BuildConfig.DEBUG;
public class DownloadInitializer extends Thread { public class DownloadInitializer extends Thread {
private final static String TAG = "DownloadInitializer"; private final static String TAG = "DownloadInitializer";
final static int mId = 0; final static int mId = 0;
private final static int RESERVE_SPACE_DEFAULT = 5 * 1024 * 1024;// 5 MiB
private final static int RESERVE_SPACE_MAXIMUM = 150 * 1024 * 1024;// 150 MiB
private DownloadMission mMission; private DownloadMission mMission;
private HttpURLConnection mConn; private HttpURLConnection mConn;
@ -35,15 +37,46 @@ public class DownloadInitializer extends Thread {
try { try {
mMission.currentThreadCount = mMission.threadCount; mMission.currentThreadCount = mMission.threadCount;
if (mMission.blocks < 0 && mMission.current == 0) {
// calculate the whole size of the mission
long finalLength = 0;
long lowestSize = Long.MAX_VALUE;
for (int i = 0; i < mMission.urls.length && mMission.running; i++) {
mConn = mMission.openConnection(mMission.urls[i], mId, -1, -1);
mMission.establishConnection(mId, mConn);
if (Thread.interrupted()) return;
long length = Utility.getContentLength(mConn);
if (i == 0) mMission.length = length;
if (length > 0) finalLength += length;
if (length < lowestSize) lowestSize = length;
}
mMission.nearLength = finalLength;
// reserve space at the start of the file
if (mMission.psAlgorithm != null && mMission.psAlgorithm.reserveSpace) {
if (lowestSize < 1) {
// the length is unknown use the default size
mMission.offsets[0] = RESERVE_SPACE_DEFAULT;
} else {
// use the smallest resource size to download, otherwise, use the maximum
mMission.offsets[0] = lowestSize < RESERVE_SPACE_MAXIMUM ? lowestSize : RESERVE_SPACE_MAXIMUM;
}
}
} else {
// ask for the current resource length
mConn = mMission.openConnection(mId, -1, -1); mConn = mMission.openConnection(mId, -1, -1);
mMission.establishConnection(mId, mConn); mMission.establishConnection(mId, mConn);
if (!mMission.running || Thread.interrupted()) return; if (!mMission.running || Thread.interrupted()) return;
mMission.length = Utility.getContentLength(mConn); mMission.length = Utility.getContentLength(mConn);
}
if (mMission.length == 0 || mConn.getResponseCode() == 204) {
if (mMission.length == 0) {
mMission.notifyError(DownloadMission.ERROR_HTTP_NO_CONTENT, null); mMission.notifyError(DownloadMission.ERROR_HTTP_NO_CONTENT, null);
return; return;
} }

View File

@ -147,16 +147,12 @@ public class DownloadMission extends Mission {
this.enqueued = true; this.enqueued = true;
this.maxRetry = 3; this.maxRetry = 3;
this.storage = storage; this.storage = storage;
if (psInstance != null) {
this.psAlgorithm = psInstance; this.psAlgorithm = psInstance;
this.offsets[0] = psInstance.recommendedReserve;
} else { if (DEBUG && psInstance == null && urls.length > 1) {
if (DEBUG && urls.length > 1) {
Log.w(TAG, "mission created with multiple urls ¿missing post-processing algorithm?"); Log.w(TAG, "mission created with multiple urls ¿missing post-processing algorithm?");
} }
} }
}
private void checkBlock(long block) { private void checkBlock(long block) {
if (block < 0 || block >= blocks) { if (block < 0 || block >= blocks) {
@ -233,10 +229,14 @@ public class DownloadMission extends Mission {
* @throws IOException if an I/O exception occurs. * @throws IOException if an I/O exception occurs.
*/ */
HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException { HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException {
URL url = new URL(urls[current]); return openConnection(urls[current], threadId, rangeStart, rangeEnd);
HttpURLConnection conn = (HttpURLConnection) url.openConnection(); }
HttpURLConnection openConnection(String url, int threadId, long rangeStart, long rangeEnd) throws IOException {
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
conn.setInstanceFollowRedirects(true); conn.setInstanceFollowRedirects(true);
conn.setRequestProperty("User-Agent", Downloader.USER_AGENT); conn.setRequestProperty("User-Agent", Downloader.USER_AGENT);
conn.setRequestProperty("Accept", "*/*");
// BUG workaround: switching between networks can freeze the download forever // BUG workaround: switching between networks can freeze the download forever
conn.setConnectTimeout(30000); conn.setConnectTimeout(30000);
@ -536,8 +536,11 @@ public class DownloadMission extends Mission {
@Override @Override
public boolean delete() { public boolean delete() {
deleted = true; deleted = true;
if (psAlgorithm != null) psAlgorithm.cleanupTemporalDir();
boolean res = deleteThisFromFile(); boolean res = deleteThisFromFile();
if (!super.delete()) res = false;
if (!super.delete()) return false;
return res; return res;
} }
@ -626,6 +629,11 @@ public class DownloadMission extends Mission {
return blocks >= 0; // DownloadMissionInitializer was executed return blocks >= 0; // DownloadMissionInitializer was executed
} }
/**
* Gets the approximated final length of the file
*
* @return the length in bytes
*/
public long getLength() { public long getLength() {
long calculated; long calculated;
if (psState == 1 || psState == 3) { if (psState == 1 || psState == 3) {
@ -681,6 +689,8 @@ public class DownloadMission extends Mission {
private boolean doPostprocessing() { private boolean doPostprocessing() {
if (psAlgorithm == null || psState == 2) return true; if (psAlgorithm == null || psState == 2) return true;
errObject = null;
notifyPostProcessing(1); notifyPostProcessing(1);
notifyProgress(0); notifyProgress(0);

View File

@ -6,10 +6,10 @@ import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException; import java.io.IOException;
public class M4aNoDash extends Postprocessing { class M4aNoDash extends Postprocessing {
M4aNoDash() { M4aNoDash() {
super(0, true); super(false, true, ALGORITHM_M4A_NO_DASH);
} }
@Override @Override

View File

@ -11,7 +11,7 @@ import java.io.IOException;
class Mp4FromDashMuxer extends Postprocessing { class Mp4FromDashMuxer extends Postprocessing {
Mp4FromDashMuxer() { Mp4FromDashMuxer() {
super(3 * 1024 * 1024/* 3 MiB */, true); super(true, true, ALGORITHM_MP4_FROM_DASH_MUXER);
} }
@Override @Override

View File

@ -29,7 +29,7 @@ public abstract class Postprocessing implements Serializable {
public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4"; public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4";
public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a"; public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a";
public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args, @NonNull File cacheDir) { public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args) {
Postprocessing instance; Postprocessing instance;
switch (algorithmName) { switch (algorithmName) {
@ -48,13 +48,10 @@ public abstract class Postprocessing implements Serializable {
/*case "example-algorithm": /*case "example-algorithm":
instance = new ExampleAlgorithm();*/ instance = new ExampleAlgorithm();*/
default: default:
throw new RuntimeException("Unimplemented post-processing algorithm: " + algorithmName); throw new UnsupportedOperationException("Unimplemented post-processing algorithm: " + algorithmName);
} }
instance.args = args; instance.args = args;
instance.name = algorithmName;// for debug only, maybe remove this field in the future
instance.cacheDir = cacheDir;
return instance; return instance;
} }
@ -62,34 +59,47 @@ public abstract class Postprocessing implements Serializable {
* Get a boolean value that indicate if the given algorithm work on the same * Get a boolean value that indicate if the given algorithm work on the same
* file * file
*/ */
public boolean worksOnSameFile; public final boolean worksOnSameFile;
/** /**
* Get the recommended space to reserve for the given algorithm. The amount * Indicates whether the selected algorithm needs space reserved at the beginning of the file
* is in bytes
*/ */
public int recommendedReserve; public final boolean reserveSpace;
/** /**
* the download to post-process * Gets the given algorithm short name
*/ */
protected transient DownloadMission mission; private final String name;
public transient File cacheDir;
private String[] args; private String[] args;
private String name; protected transient DownloadMission mission;
Postprocessing(int recommendedReserve, boolean worksOnSameFile) { private File tempFile;
this.recommendedReserve = recommendedReserve;
Postprocessing(boolean reserveSpace, boolean worksOnSameFile, String algorithmName) {
this.reserveSpace = reserveSpace;
this.worksOnSameFile = worksOnSameFile; this.worksOnSameFile = worksOnSameFile;
this.name = algorithmName;// for debugging only
} }
public void setTemporalDir(@NonNull File directory) {
long rnd = (int) (Math.random() * 100000f);
tempFile = new File(directory, rnd + "_" + System.nanoTime() + ".tmp");
}
public void cleanupTemporalDir() {
if (tempFile != null && tempFile.exists()) {
//noinspection ResultOfMethodCallIgnored
tempFile.delete();
}
}
public void run(DownloadMission target) throws IOException { public void run(DownloadMission target) throws IOException {
this.mission = target; this.mission = target;
File temp = null;
CircularFileWriter out = null; CircularFileWriter out = null;
int result; int result;
long finalLength = -1; long finalLength = -1;
@ -125,9 +135,7 @@ public abstract class Postprocessing implements Serializable {
return -1; return -1;
}; };
temp = new File(cacheDir, mission.storage.getName() + ".tmp"); out = new CircularFileWriter(mission.storage.getStream(), tempFile, checker);
out = new CircularFileWriter(mission.storage.getStream(), temp, checker);
out.onProgress = this::progressReport; out.onProgress = this::progressReport;
out.onWriteError = (err) -> { out.onWriteError = (err) -> {
@ -163,9 +171,10 @@ public abstract class Postprocessing implements Serializable {
if (out != null) { if (out != null) {
out.close(); out.close();
} }
if (temp != null) { if (tempFile != null) {
//noinspection ResultOfMethodCallIgnored //noinspection ResultOfMethodCallIgnored
temp.delete(); tempFile.delete();
tempFile = null;
} }
} }
} else { } else {

View File

@ -20,7 +20,7 @@ class TtmlConverter extends Postprocessing {
TtmlConverter() { TtmlConverter() {
// due how XmlPullParser works, the xml is fully loaded on the ram // due how XmlPullParser works, the xml is fully loaded on the ram
super(0, true); super(false, true, ALGORITHM_TTML_CONVERTER);
} }
@Override @Override

View File

@ -13,7 +13,7 @@ import java.io.IOException;
class WebMMuxer extends Postprocessing { class WebMMuxer extends Postprocessing {
WebMMuxer() { WebMMuxer() {
super(5 * 1024 * 1024/* 5 MiB */, true); super(true, true, ALGORITHM_WEBM_MUXER);
} }
@Override @Override

View File

@ -154,7 +154,9 @@ public class DownloadManager {
if (mis.psAlgorithm.worksOnSameFile) { if (mis.psAlgorithm.worksOnSameFile) {
// Incomplete post-processing results in a corrupted download file // Incomplete post-processing results in a corrupted download file
// because the selected algorithm works on the same file to save space. // because the selected algorithm works on the same file to save space.
if (exists && !mis.storage.delete()) // the file will be deleted if the storage API
// is Java IO (avoid showing the "Save as..." dialog)
if (exists && mis.storage.isDirect() && !mis.storage.delete())
Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath()); Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath());
exists = true; exists = true;
@ -162,7 +164,6 @@ public class DownloadManager {
mis.psState = 0; mis.psState = 0;
mis.errCode = DownloadMission.ERROR_POSTPROCESSING_STOPPED; mis.errCode = DownloadMission.ERROR_POSTPROCESSING_STOPPED;
mis.errObject = null;
} else if (!exists) { } else if (!exists) {
tryRecover(mis); tryRecover(mis);
@ -171,8 +172,10 @@ public class DownloadManager {
mis.resetState(true, true, DownloadMission.ERROR_PROGRESS_LOST); mis.resetState(true, true, DownloadMission.ERROR_PROGRESS_LOST);
} }
if (mis.psAlgorithm != null) if (mis.psAlgorithm != null) {
mis.psAlgorithm.cacheDir = pickAvailableCacheDir(ctx); mis.psAlgorithm.cleanupTemporalDir();
mis.psAlgorithm.setTemporalDir(pickAvailableTemporalDir(ctx));
}
mis.recovered = exists; mis.recovered = exists;
mis.metadata = sub; mis.metadata = sub;
@ -532,14 +535,14 @@ public class DownloadManager {
} }
private static boolean isDirectoryAvailable(File directory) { private static boolean isDirectoryAvailable(File directory) {
return directory != null && directory.canWrite(); return directory != null && directory.canWrite() && directory.exists();
} }
static File pickAvailableCacheDir(@NonNull Context ctx) { static File pickAvailableTemporalDir(@NonNull Context ctx) {
if (isDirectoryAvailable(ctx.getExternalCacheDir())) if (isDirectoryAvailable(ctx.getExternalFilesDir(null)))
return ctx.getExternalCacheDir(); return ctx.getExternalFilesDir(null);
else if (isDirectoryAvailable(ctx.getCacheDir())) else if (isDirectoryAvailable(ctx.getFilesDir()))
return ctx.getCacheDir(); return ctx.getFilesDir();
// this never should happen // this never should happen
return ctx.getDir("tmp", Context.MODE_PRIVATE); return ctx.getDir("tmp", Context.MODE_PRIVATE);
@ -550,7 +553,7 @@ public class DownloadManager {
if (tag.equals(TAG_AUDIO)) return mMainStorageAudio; if (tag.equals(TAG_AUDIO)) return mMainStorageAudio;
if (tag.equals(TAG_VIDEO)) return mMainStorageVideo; if (tag.equals(TAG_VIDEO)) return mMainStorageVideo;
Log.w(TAG, "Unknown download category, not [audio video]: " + String.valueOf(tag)); Log.w(TAG, "Unknown download category, not [audio video]: " + tag);
return null;// this never should happen return null;// this never should happen
} }

View File

@ -450,13 +450,16 @@ public class DownloadManagerService extends Service {
if (psName == null) if (psName == null)
ps = null; ps = null;
else else
ps = Postprocessing.getAlgorithm(psName, psArgs, DownloadManager.pickAvailableCacheDir(this)); ps = Postprocessing.getAlgorithm(psName, psArgs);
final DownloadMission mission = new DownloadMission(urls, storage, kind, ps); final DownloadMission mission = new DownloadMission(urls, storage, kind, ps);
mission.threadCount = threads; mission.threadCount = threads;
mission.source = source; mission.source = source;
mission.nearLength = nearLength; mission.nearLength = nearLength;
if (ps != null)
ps.setTemporalDir(DownloadManager.pickAvailableTemporalDir(this));
handleConnectivityState(true);// first check the actual network status handleConnectivityState(true);// first check the actual network status
mManager.startMission(mission); mManager.startMission(mission);

View File

@ -267,8 +267,7 @@ public class Utility {
} }
try { try {
long length = Long.parseLong(connection.getHeaderField("Content-Length")); return Long.parseLong(connection.getHeaderField("Content-Length"));
if (length >= 0) return length;
} catch (Exception err) { } catch (Exception err) {
// nothing to do // nothing to do
} }