mirror of
https://github.com/TeamNewPipe/NewPipe
synced 2025-11-04 01:03:00 +00:00
misc changes
* OggFromWebMWriter: rewrite (again), reduce iterations over the input. Works as-is (video streams are not supported) * WebMReader: use int for SimpleBlock.dataSize instead of long * Download Recovery: allow recovering uninitialized downloads * check range-requests using HEAD method instead of GET * DownloadRunnableFallback: add workaround for 32kB/s issue, unknown issue origin, wont fix * reporting downloads errors now include the source url with the selected quality and format
This commit is contained in:
kapodamy
@@ -12,8 +12,6 @@ import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Random;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
@@ -23,15 +21,13 @@ import javax.annotation.Nullable;
|
||||
public class OggFromWebMWriter implements Closeable {
|
||||
|
||||
private static final byte FLAG_UNSET = 0x00;
|
||||
private static final byte FLAG_CONTINUED = 0x01;
|
||||
//private static final byte FLAG_CONTINUED = 0x01;
|
||||
private static final byte FLAG_FIRST = 0x02;
|
||||
private static final byte FLAG_LAST = 0x04;
|
||||
|
||||
private final static byte HEADER_CHECKSUM_OFFSET = 22;
|
||||
private final static byte HEADER_SIZE = 27;
|
||||
|
||||
private final static short BUFFER_SIZE = 8 * 1024;// 8KiB
|
||||
|
||||
private final static int TIME_SCALE_NS = 1000000000;
|
||||
|
||||
private boolean done = false;
|
||||
@@ -43,7 +39,6 @@ public class OggFromWebMWriter implements Closeable {
|
||||
private int sequence_count = 0;
|
||||
private final int STREAM_ID;
|
||||
private byte packet_flag = FLAG_FIRST;
|
||||
private int track_index = 0;
|
||||
|
||||
private WebMReader webm = null;
|
||||
private WebMTrack webm_track = null;
|
||||
@@ -71,7 +66,7 @@ public class OggFromWebMWriter implements Closeable {
|
||||
this.source = source;
|
||||
this.output = target;
|
||||
|
||||
this.STREAM_ID = (new Random(System.currentTimeMillis())).nextInt();
|
||||
this.STREAM_ID = (int) System.currentTimeMillis();
|
||||
|
||||
populate_crc32_table();
|
||||
}
|
||||
@@ -130,7 +125,6 @@ public class OggFromWebMWriter implements Closeable {
|
||||
|
||||
try {
|
||||
webm_track = webm.selectTrack(trackIndex);
|
||||
track_index = trackIndex;
|
||||
} finally {
|
||||
parsed = true;
|
||||
}
|
||||
@@ -154,8 +148,11 @@ public class OggFromWebMWriter implements Closeable {
|
||||
|
||||
public void build() throws IOException {
|
||||
float resolution;
|
||||
int read;
|
||||
byte[] buffer;
|
||||
SimpleBlock bloq;
|
||||
ByteBuffer header = ByteBuffer.allocate(27 + (255 * 255));
|
||||
ByteBuffer page = ByteBuffer.allocate(64 * 1024);
|
||||
|
||||
header.order(ByteOrder.LITTLE_ENDIAN);
|
||||
|
||||
/* step 1: get the amount of frames per seconds */
|
||||
switch (webm_track.kind) {
|
||||
@@ -176,57 +173,32 @@ public class OggFromWebMWriter implements Closeable {
|
||||
throw new RuntimeException("not implemented");
|
||||
}
|
||||
|
||||
/* step 2a: create packet with code init data */
|
||||
ArrayList<byte[]> data_extra = new ArrayList<>(4);
|
||||
|
||||
/* step 2: create packet with code init data */
|
||||
if (webm_track.codecPrivate != null) {
|
||||
addPacketSegment(webm_track.codecPrivate.length);
|
||||
ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + webm_track.codecPrivate.length);
|
||||
|
||||
make_packetHeader(0x00, buff, webm_track.codecPrivate);
|
||||
data_extra.add(buff.array());
|
||||
make_packetHeader(0x00, header, webm_track.codecPrivate);
|
||||
write(header);
|
||||
output.write(webm_track.codecPrivate);
|
||||
}
|
||||
|
||||
/* step 2b: create packet with metadata */
|
||||
buffer = make_metadata();
|
||||
/* step 3: create packet with metadata */
|
||||
byte[] buffer = make_metadata();
|
||||
if (buffer != null) {
|
||||
addPacketSegment(buffer.length);
|
||||
ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + buffer.length);
|
||||
|
||||
make_packetHeader(0x00, buff, buffer);
|
||||
data_extra.add(buff.array());
|
||||
make_packetHeader(0x00, header, buffer);
|
||||
write(header);
|
||||
output.write(buffer);
|
||||
}
|
||||
|
||||
|
||||
/* step 3: calculate amount of packets */
|
||||
SimpleBlock bloq;
|
||||
int reserve_header = 0;
|
||||
int headers_amount = 0;
|
||||
|
||||
/* step 4: calculate amount of packets */
|
||||
while (webm_segment != null) {
|
||||
bloq = getNextBlock();
|
||||
|
||||
if (addPacketSegment(bloq)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
reserve_header += HEADER_SIZE + segment_table_size;// header size
|
||||
clearSegmentTable();
|
||||
webm_block = bloq;
|
||||
headers_amount++;
|
||||
}
|
||||
|
||||
/* step 4: create packet headers */
|
||||
rewind_source();
|
||||
|
||||
ByteBuffer headers = byte_buffer(reserve_header);
|
||||
short[] headers_size = new short[headers_amount];
|
||||
int header_index = 0;
|
||||
|
||||
while (webm_segment != null) {
|
||||
bloq = getNextBlock();
|
||||
|
||||
if (addPacketSegment(bloq)) {
|
||||
if (bloq != null && addPacketSegment(bloq)) {
|
||||
int pos = page.position();
|
||||
//noinspection ResultOfMethodCallIgnored
|
||||
bloq.data.read(page.array(), pos, bloq.dataSize);
|
||||
page.position(pos + bloq.dataSize);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -251,75 +223,21 @@ public class OggFromWebMWriter implements Closeable {
|
||||
elapsed_ns = elapsed_ns / TIME_SCALE_NS;
|
||||
elapsed_ns = Math.ceil(elapsed_ns * resolution);
|
||||
|
||||
// create header
|
||||
headers_size[header_index++] = make_packetHeader((long) elapsed_ns, headers, null);
|
||||
// create header and calculate page checksum
|
||||
int checksum = make_packetHeader((long) elapsed_ns, header, null);
|
||||
checksum = calc_crc32(checksum, page.array(), page.position());
|
||||
|
||||
header.putInt(HEADER_CHECKSUM_OFFSET, checksum);
|
||||
|
||||
// dump data
|
||||
write(header);
|
||||
write(page);
|
||||
|
||||
webm_block = bloq;
|
||||
}
|
||||
|
||||
|
||||
/* step 5: calculate checksums */
|
||||
rewind_source();
|
||||
|
||||
int offset = 0;
|
||||
buffer = new byte[BUFFER_SIZE];
|
||||
|
||||
for (header_index = 0; header_index < headers_size.length; header_index++) {
|
||||
int checksum_offset = offset + HEADER_CHECKSUM_OFFSET;
|
||||
int checksum = headers.getInt(checksum_offset);
|
||||
|
||||
while (webm_segment != null) {
|
||||
bloq = getNextBlock();
|
||||
|
||||
if (!addPacketSegment(bloq)) {
|
||||
clearSegmentTable();
|
||||
webm_block = bloq;
|
||||
break;
|
||||
}
|
||||
|
||||
// calculate page checksum
|
||||
while ((read = bloq.data.read(buffer)) > 0) {
|
||||
checksum = calc_crc32(checksum, buffer, 0, read);
|
||||
}
|
||||
}
|
||||
|
||||
headers.putInt(checksum_offset, checksum);
|
||||
offset += headers_size[header_index];
|
||||
}
|
||||
|
||||
/* step 6: write extra headers */
|
||||
rewind_source();
|
||||
|
||||
for (byte[] buff : data_extra) {
|
||||
output.write(buff);
|
||||
}
|
||||
|
||||
/* step 7: write stream packets */
|
||||
byte[] headers_buffers = headers.array();
|
||||
offset = 0;
|
||||
buffer = new byte[BUFFER_SIZE];
|
||||
|
||||
for (header_index = 0; header_index < headers_size.length; header_index++) {
|
||||
output.write(headers_buffers, offset, headers_size[header_index]);
|
||||
offset += headers_size[header_index];
|
||||
|
||||
while (webm_segment != null) {
|
||||
bloq = getNextBlock();
|
||||
|
||||
if (addPacketSegment(bloq)) {
|
||||
while ((read = bloq.data.read(buffer)) > 0) {
|
||||
output.write(buffer, 0, read);
|
||||
}
|
||||
} else {
|
||||
clearSegmentTable();
|
||||
webm_block = bloq;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private short make_packetHeader(long gran_pos, ByteBuffer buffer, byte[] immediate_page) {
|
||||
int offset = buffer.position();
|
||||
private int make_packetHeader(long gran_pos, @NonNull ByteBuffer buffer, byte[] immediate_page) {
|
||||
short length = HEADER_SIZE;
|
||||
|
||||
buffer.putInt(0x5367674f);// "OggS" binary string in little-endian
|
||||
@@ -340,17 +258,15 @@ public class OggFromWebMWriter implements Closeable {
|
||||
|
||||
clearSegmentTable();// clear segment table for next header
|
||||
|
||||
int checksum_crc32 = calc_crc32(0x00, buffer.array(), offset, length);
|
||||
int checksum_crc32 = calc_crc32(0x00, buffer.array(), length);
|
||||
|
||||
if (immediate_page != null) {
|
||||
checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, 0, immediate_page.length);
|
||||
System.arraycopy(immediate_page, 0, buffer.array(), length, immediate_page.length);
|
||||
checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, immediate_page.length);
|
||||
buffer.putInt(HEADER_CHECKSUM_OFFSET, checksum_crc32);
|
||||
segment_table_next_timestamp -= TIME_SCALE_NS;
|
||||
}
|
||||
|
||||
buffer.putInt(offset + HEADER_CHECKSUM_OFFSET, checksum_crc32);
|
||||
|
||||
return length;
|
||||
return checksum_crc32;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@@ -358,7 +274,7 @@ public class OggFromWebMWriter implements Closeable {
|
||||
if ("A_OPUS".equals(webm_track.codecId)) {
|
||||
return new byte[]{
|
||||
0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73,// "OpusTags" binary string
|
||||
0x07, 0x00, 0x00, 0x00,// writing application string size
|
||||
0x07, 0x00, 0x00, 0x00,// writting application string size
|
||||
0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string
|
||||
0x00, 0x00, 0x00, 0x00// additional tags count (zero means no tags)
|
||||
};
|
||||
@@ -366,7 +282,7 @@ public class OggFromWebMWriter implements Closeable {
|
||||
return new byte[]{
|
||||
0x03,// ????????
|
||||
0x76, 0x6f, 0x72, 0x62, 0x69, 0x73,// "vorbis" binary string
|
||||
0x07, 0x00, 0x00, 0x00,// writing application string size
|
||||
0x07, 0x00, 0x00, 0x00,// writting application string size
|
||||
0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string
|
||||
0x01, 0x00, 0x00, 0x00,// additional tags count (zero means no tags)
|
||||
|
||||
@@ -387,22 +303,9 @@ public class OggFromWebMWriter implements Closeable {
|
||||
return null;
|
||||
}
|
||||
|
||||
private void rewind_source() throws IOException {
|
||||
source.rewind();
|
||||
|
||||
webm = new WebMReader(source);
|
||||
webm.parse();
|
||||
webm_track = webm.selectTrack(track_index);
|
||||
webm_segment = webm.getNextSegment();
|
||||
webm_cluster = null;
|
||||
webm_block = null;
|
||||
webm_block_last_timecode = 0L;
|
||||
|
||||
segment_table_next_timestamp = TIME_SCALE_NS;
|
||||
}
|
||||
|
||||
private ByteBuffer byte_buffer(int size) {
|
||||
return ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN);
|
||||
private void write(ByteBuffer buffer) throws IOException {
|
||||
output.write(buffer.array(), 0, buffer.position());
|
||||
buffer.position(0);
|
||||
}
|
||||
|
||||
//<editor-fold defaultstate="collapsed" desc="WebM track handling">
|
||||
@@ -460,41 +363,32 @@ public class OggFromWebMWriter implements Closeable {
|
||||
|
||||
//<editor-fold defaultstate="collapsed" desc="Segment table writing">
|
||||
private void clearSegmentTable() {
|
||||
if (packet_flag != FLAG_CONTINUED) {
|
||||
segment_table_next_timestamp += TIME_SCALE_NS;
|
||||
packet_flag = FLAG_UNSET;
|
||||
}
|
||||
segment_table_size = 0;
|
||||
}
|
||||
|
||||
private boolean addPacketSegment(SimpleBlock block) {
|
||||
if (block == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
long timestamp = block.absoluteTimeCodeNs + webm_track.codecDelay;
|
||||
|
||||
if (timestamp >= segment_table_next_timestamp) {
|
||||
return false;
|
||||
}
|
||||
|
||||
boolean result = addPacketSegment((int) block.dataSize);
|
||||
|
||||
if (!result && segment_table_next_timestamp < timestamp) {
|
||||
// WARNING: ¡¡¡¡ not implemented (lack of documentation) !!!!
|
||||
packet_flag = FLAG_CONTINUED;
|
||||
}
|
||||
|
||||
return result;
|
||||
return addPacketSegment(block.dataSize);
|
||||
}
|
||||
|
||||
private boolean addPacketSegment(int size) {
|
||||
if (size > 65025) {
|
||||
throw new UnsupportedOperationException("page size cannot be larger than 65025");
|
||||
}
|
||||
|
||||
int available = (segment_table.length - segment_table_size) * 255;
|
||||
boolean extra = size == 255;
|
||||
boolean extra = (size % 255) == 0;
|
||||
|
||||
if (extra) {
|
||||
// add a zero byte entry in the table
|
||||
// required to indicate the sample size is exactly 255
|
||||
// required to indicate the sample size is multiple of 255
|
||||
available -= 255;
|
||||
}
|
||||
|
||||
@@ -528,12 +422,10 @@ public class OggFromWebMWriter implements Closeable {
|
||||
}
|
||||
}
|
||||
|
||||
private int calc_crc32(int initial_crc, byte[] buffer, int offset, int size) {
|
||||
size += offset;
|
||||
|
||||
for (; offset < size; offset++) {
|
||||
private int calc_crc32(int initial_crc, byte[] buffer, int size) {
|
||||
for (int i = 0; i < size; i++) {
|
||||
int reg = (initial_crc >>> 24) & 0xff;
|
||||
initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[offset] & 0xff)];
|
||||
initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[i] & 0xff)];
|
||||
}
|
||||
|
||||
return initial_crc;
|
||||
|
||||
@@ -368,7 +368,7 @@ public class WebMReader {
|
||||
obj.trackNumber = readEncodedNumber();
|
||||
obj.relativeTimeCode = stream.readShort();
|
||||
obj.flags = (byte) stream.read();
|
||||
obj.dataSize = (ref.offset + ref.size) - stream.position();
|
||||
obj.dataSize = (int) ((ref.offset + ref.size) - stream.position());
|
||||
obj.createdFromBlock = ref.type == ID_Block;
|
||||
|
||||
// NOTE: lacing is not implemented, and will be mixed with the stream data
|
||||
@@ -465,7 +465,7 @@ public class WebMReader {
|
||||
public short relativeTimeCode;
|
||||
public long absoluteTimeCodeNs;
|
||||
public byte flags;
|
||||
public long dataSize;
|
||||
public int dataSize;
|
||||
private final Element ref;
|
||||
|
||||
public boolean isKeyframe() {
|
||||
|
||||
@@ -14,6 +14,7 @@ import java.nio.channels.ClosedByInterruptException;
|
||||
import us.shandian.giga.util.Utility;
|
||||
|
||||
import static org.schabi.newpipe.BuildConfig.DEBUG;
|
||||
import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_FORBIDDEN;
|
||||
|
||||
public class DownloadInitializer extends Thread {
|
||||
private final static String TAG = "DownloadInitializer";
|
||||
@@ -29,9 +30,9 @@ public class DownloadInitializer extends Thread {
|
||||
mConn = null;
|
||||
}
|
||||
|
||||
private static void safeClose(HttpURLConnection con) {
|
||||
private void dispose() {
|
||||
try {
|
||||
con.getInputStream().close();
|
||||
mConn.getInputStream().close();
|
||||
} catch (Exception e) {
|
||||
// nothing to do
|
||||
}
|
||||
@@ -52,9 +53,9 @@ public class DownloadInitializer extends Thread {
|
||||
long lowestSize = Long.MAX_VALUE;
|
||||
|
||||
for (int i = 0; i < mMission.urls.length && mMission.running; i++) {
|
||||
mConn = mMission.openConnection(mMission.urls[i], mId, -1, -1);
|
||||
mConn = mMission.openConnection(mMission.urls[i], true, -1, -1);
|
||||
mMission.establishConnection(mId, mConn);
|
||||
safeClose(mConn);
|
||||
dispose();
|
||||
|
||||
if (Thread.interrupted()) return;
|
||||
long length = Utility.getContentLength(mConn);
|
||||
@@ -82,9 +83,9 @@ public class DownloadInitializer extends Thread {
|
||||
}
|
||||
} else {
|
||||
// ask for the current resource length
|
||||
mConn = mMission.openConnection(mId, -1, -1);
|
||||
mConn = mMission.openConnection(true, -1, -1);
|
||||
mMission.establishConnection(mId, mConn);
|
||||
safeClose(mConn);
|
||||
dispose();
|
||||
|
||||
if (!mMission.running || Thread.interrupted()) return;
|
||||
|
||||
@@ -108,9 +109,9 @@ public class DownloadInitializer extends Thread {
|
||||
}
|
||||
} else {
|
||||
// Open again
|
||||
mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length);
|
||||
mConn = mMission.openConnection(true, mMission.length - 10, mMission.length);
|
||||
mMission.establishConnection(mId, mConn);
|
||||
safeClose(mConn);
|
||||
dispose();
|
||||
|
||||
if (!mMission.running || Thread.interrupted()) return;
|
||||
|
||||
@@ -171,7 +172,14 @@ public class DownloadInitializer extends Thread {
|
||||
} catch (InterruptedIOException | ClosedByInterruptException e) {
|
||||
return;
|
||||
} catch (Exception e) {
|
||||
if (!mMission.running) return;
|
||||
if (!mMission.running || super.isInterrupted()) return;
|
||||
|
||||
if (e instanceof DownloadMission.HttpError && ((DownloadMission.HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) {
|
||||
// for youtube streams. The url has expired
|
||||
interrupt();
|
||||
mMission.doRecover(e);
|
||||
return;
|
||||
}
|
||||
|
||||
if (e instanceof IOException && e.getMessage().contains("Permission denied")) {
|
||||
mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e);
|
||||
@@ -194,13 +202,6 @@ public class DownloadInitializer extends Thread {
|
||||
@Override
|
||||
public void interrupt() {
|
||||
super.interrupt();
|
||||
|
||||
if (mConn != null) {
|
||||
try {
|
||||
mConn.disconnect();
|
||||
} catch (Exception e) {
|
||||
// nothing to do
|
||||
}
|
||||
}
|
||||
if (mConn != null) dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -204,22 +204,24 @@ public class DownloadMission extends Mission {
|
||||
/**
|
||||
* Opens a connection
|
||||
*
|
||||
* @param threadId id of the calling thread, used only for debugging
|
||||
* @param headRequest {@code true} for use {@code HEAD} request method, otherwise, {@code GET} is used
|
||||
* @param rangeStart range start
|
||||
* @param rangeEnd range end
|
||||
* @return a {@link java.net.URLConnection URLConnection} linking to the URL.
|
||||
* @throws IOException if an I/O exception occurs.
|
||||
*/
|
||||
HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException {
|
||||
return openConnection(urls[current], threadId, rangeStart, rangeEnd);
|
||||
HttpURLConnection openConnection(boolean headRequest, long rangeStart, long rangeEnd) throws IOException {
|
||||
return openConnection(urls[current], headRequest, rangeStart, rangeEnd);
|
||||
}
|
||||
|
||||
HttpURLConnection openConnection(String url, int threadId, long rangeStart, long rangeEnd) throws IOException {
|
||||
HttpURLConnection openConnection(String url, boolean headRequest, long rangeStart, long rangeEnd) throws IOException {
|
||||
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
|
||||
conn.setInstanceFollowRedirects(true);
|
||||
conn.setRequestProperty("User-Agent", DownloaderImpl.USER_AGENT);
|
||||
conn.setRequestProperty("Accept", "*/*");
|
||||
|
||||
if (headRequest) conn.setRequestMethod("HEAD");
|
||||
|
||||
// BUG workaround: switching between networks can freeze the download forever
|
||||
conn.setConnectTimeout(30000);
|
||||
conn.setReadTimeout(10000);
|
||||
@@ -229,10 +231,6 @@ public class DownloadMission extends Mission {
|
||||
if (rangeEnd > 0) req += rangeEnd;
|
||||
|
||||
conn.setRequestProperty("Range", req);
|
||||
|
||||
if (DEBUG) {
|
||||
Log.d(TAG, threadId + ":" + conn.getRequestProperty("Range"));
|
||||
}
|
||||
}
|
||||
|
||||
return conn;
|
||||
@@ -245,13 +243,14 @@ public class DownloadMission extends Mission {
|
||||
* @throws HttpError if the HTTP Status-Code is not satisfiable
|
||||
*/
|
||||
void establishConnection(int threadId, HttpURLConnection conn) throws IOException, HttpError {
|
||||
conn.connect();
|
||||
int statusCode = conn.getResponseCode();
|
||||
|
||||
if (DEBUG) {
|
||||
Log.d(TAG, threadId + ":Range=" + conn.getRequestProperty("Range"));
|
||||
Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + statusCode);
|
||||
}
|
||||
|
||||
|
||||
switch (statusCode) {
|
||||
case 204:
|
||||
case 205:
|
||||
@@ -676,6 +675,15 @@ public class DownloadMission extends Mission {
|
||||
return (isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) || isFinished();
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicates if mission urls has expired and there an attempt to renovate them
|
||||
*
|
||||
* @return {@code true} if the mission is running a recovery procedure, otherwise, {@code false}
|
||||
*/
|
||||
public boolean isRecovering() {
|
||||
return threads != null && threads.length > 0 && threads[0] instanceof DownloadRunnable && threads[0].isAlive();
|
||||
}
|
||||
|
||||
private boolean doPostprocessing() {
|
||||
if (psAlgorithm == null || psState == 2) return true;
|
||||
|
||||
@@ -742,10 +750,8 @@ public class DownloadMission extends Mission {
|
||||
}
|
||||
}
|
||||
|
||||
// set the current download url to null in case if the recovery
|
||||
// process is canceled. Next time start() method is called the
|
||||
// recovery will be executed, saving time
|
||||
urls[current] = null;
|
||||
errCode = ERROR_NOTHING;
|
||||
errObject = null;
|
||||
|
||||
if (recoveryInfo[current].attempts >= maxRetry) {
|
||||
recoveryInfo[current].attempts = 0;
|
||||
|
||||
@@ -10,10 +10,12 @@ import org.schabi.newpipe.extractor.stream.SubtitlesStream;
|
||||
import org.schabi.newpipe.extractor.stream.VideoStream;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InterruptedIOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.nio.channels.ClosedByInterruptException;
|
||||
import java.util.List;
|
||||
|
||||
import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING;
|
||||
import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE;
|
||||
|
||||
public class DownloadMissionRecover extends Thread {
|
||||
@@ -21,14 +23,17 @@ public class DownloadMissionRecover extends Thread {
|
||||
static final int mID = -3;
|
||||
|
||||
private final DownloadMission mMission;
|
||||
private final MissionRecoveryInfo mRecovery;
|
||||
private final Exception mFromError;
|
||||
private final boolean notInitialized;
|
||||
|
||||
private HttpURLConnection mConn;
|
||||
private MissionRecoveryInfo mRecovery;
|
||||
private StreamExtractor mExtractor;
|
||||
|
||||
DownloadMissionRecover(DownloadMission mission, Exception originError) {
|
||||
mMission = mission;
|
||||
mFromError = originError;
|
||||
mRecovery = mission.recoveryInfo[mission.current];
|
||||
notInitialized = mission.blocks == null && mission.current == 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -38,28 +43,78 @@ public class DownloadMissionRecover extends Thread {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
/*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) {
|
||||
resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length()));
|
||||
return;
|
||||
}*/
|
||||
|
||||
try {
|
||||
StreamingService svr = NewPipe.getServiceByUrl(mMission.source);
|
||||
|
||||
if (svr == null) {
|
||||
throw new RuntimeException("Unknown source service");
|
||||
mExtractor = svr.getStreamExtractor(mMission.source);
|
||||
mExtractor.fetchPage();
|
||||
} catch (InterruptedIOException | ClosedByInterruptException e) {
|
||||
return;
|
||||
} catch (Exception e) {
|
||||
if (!mMission.running || super.isInterrupted()) return;
|
||||
mMission.notifyError(e);
|
||||
return;
|
||||
}
|
||||
|
||||
StreamExtractor extractor = svr.getStreamExtractor(mMission.source);
|
||||
extractor.fetchPage();
|
||||
// maybe the following check is redundant
|
||||
if (!mMission.running || super.isInterrupted()) return;
|
||||
|
||||
if (!notInitialized) {
|
||||
// set the current download url to null in case if the recovery
|
||||
// process is canceled. Next time start() method is called the
|
||||
// recovery will be executed, saving time
|
||||
mMission.urls[mMission.current] = null;
|
||||
|
||||
mRecovery = mMission.recoveryInfo[mMission.current];
|
||||
resolveStream();
|
||||
return;
|
||||
}
|
||||
|
||||
Log.w(TAG, "mission is not fully initialized, this will take a while");
|
||||
|
||||
try {
|
||||
for (; mMission.current < mMission.urls.length; mMission.current++) {
|
||||
mRecovery = mMission.recoveryInfo[mMission.current];
|
||||
|
||||
if (test()) continue;
|
||||
if (!mMission.running) return;
|
||||
|
||||
resolveStream();
|
||||
if (!mMission.running) return;
|
||||
|
||||
// before continue, check if the current stream was resolved
|
||||
if (mMission.urls[mMission.current] == null || mMission.errCode != ERROR_NOTHING) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
mMission.current = 0;
|
||||
}
|
||||
|
||||
mMission.writeThisToFile();
|
||||
|
||||
if (!mMission.running || super.isInterrupted()) return;
|
||||
|
||||
mMission.running = false;
|
||||
mMission.start();
|
||||
}
|
||||
|
||||
private void resolveStream() {
|
||||
if (mExtractor.getErrorMessage() != null) {
|
||||
mMission.notifyError(mFromError);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
String url = null;
|
||||
|
||||
switch (mMission.kind) {
|
||||
switch (mRecovery.kind) {
|
||||
case 'a':
|
||||
for (AudioStream audio : extractor.getAudioStreams()) {
|
||||
for (AudioStream audio : mExtractor.getAudioStreams()) {
|
||||
if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) {
|
||||
url = audio.getUrl();
|
||||
break;
|
||||
@@ -69,9 +124,9 @@ public class DownloadMissionRecover extends Thread {
|
||||
case 'v':
|
||||
List<VideoStream> videoStreams;
|
||||
if (mRecovery.desired2)
|
||||
videoStreams = extractor.getVideoOnlyStreams();
|
||||
videoStreams = mExtractor.getVideoOnlyStreams();
|
||||
else
|
||||
videoStreams = extractor.getVideoStreams();
|
||||
videoStreams = mExtractor.getVideoStreams();
|
||||
for (VideoStream video : videoStreams) {
|
||||
if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) {
|
||||
url = video.getUrl();
|
||||
@@ -80,7 +135,7 @@ public class DownloadMissionRecover extends Thread {
|
||||
}
|
||||
break;
|
||||
case 's':
|
||||
for (SubtitlesStream subtitles : extractor.getSubtitles(mRecovery.format)) {
|
||||
for (SubtitlesStream subtitles : mExtractor.getSubtitles(mRecovery.format)) {
|
||||
String tag = subtitles.getLanguageTag();
|
||||
if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) {
|
||||
url = subtitles.getURL();
|
||||
@@ -114,7 +169,7 @@ public class DownloadMissionRecover extends Thread {
|
||||
////// Validate the http resource doing a range request
|
||||
/////////////////////
|
||||
try {
|
||||
mConn = mMission.openConnection(url, mID, mMission.length - 10, mMission.length);
|
||||
mConn = mMission.openConnection(url, true, mMission.length - 10, mMission.length);
|
||||
mConn.setRequestProperty("If-Range", mRecovery.validateCondition);
|
||||
mMission.establishConnection(mID, mConn);
|
||||
|
||||
@@ -140,22 +195,24 @@ public class DownloadMissionRecover extends Thread {
|
||||
if (!mMission.running || e instanceof ClosedByInterruptException) return;
|
||||
throw e;
|
||||
} finally {
|
||||
this.interrupt();
|
||||
disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
private void recover(String url, boolean stale) {
|
||||
Log.i(TAG,
|
||||
String.format("download recovered name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url)
|
||||
String.format("recover() name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url)
|
||||
);
|
||||
|
||||
mMission.urls[mMission.current] = url;
|
||||
mRecovery.attempts = 0;
|
||||
|
||||
if (url == null) {
|
||||
mMission.notifyError(ERROR_RESOURCE_GONE, null);
|
||||
return;
|
||||
}
|
||||
|
||||
mMission.urls[mMission.current] = url;
|
||||
mRecovery.attempts = 0;
|
||||
if (notInitialized) return;
|
||||
|
||||
if (stale) {
|
||||
mMission.resetState(false, false, DownloadMission.ERROR_NOTHING);
|
||||
@@ -208,15 +265,40 @@ public class DownloadMissionRecover extends Thread {
|
||||
return range;
|
||||
}
|
||||
|
||||
private boolean test() {
|
||||
if (mMission.urls[mMission.current] == null) return false;
|
||||
|
||||
try {
|
||||
mConn = mMission.openConnection(mMission.urls[mMission.current], true, -1, -1);
|
||||
mMission.establishConnection(mID, mConn);
|
||||
|
||||
if (mConn.getResponseCode() == 200) return true;
|
||||
} catch (Exception e) {
|
||||
// nothing to do
|
||||
} finally {
|
||||
disconnect();
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private void disconnect() {
|
||||
try {
|
||||
try {
|
||||
mConn.getInputStream().close();
|
||||
} finally {
|
||||
mConn.disconnect();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// nothing to do
|
||||
} finally {
|
||||
mConn = null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void interrupt() {
|
||||
super.interrupt();
|
||||
if (mConn != null) {
|
||||
try {
|
||||
mConn.disconnect();
|
||||
} catch (Exception e) {
|
||||
// nothing to do
|
||||
}
|
||||
}
|
||||
if (mConn != null) disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,7 +80,7 @@ public class DownloadRunnable extends Thread {
|
||||
}
|
||||
|
||||
try {
|
||||
mConn = mMission.openConnection(mId, start, end);
|
||||
mConn = mMission.openConnection(false, start, end);
|
||||
mMission.establishConnection(mId, mConn);
|
||||
|
||||
// check if the download can be resumed
|
||||
|
||||
@@ -34,8 +34,12 @@ public class DownloadRunnableFallback extends Thread {
|
||||
}
|
||||
|
||||
private void dispose() {
|
||||
try {
|
||||
try {
|
||||
if (mIs != null) mIs.close();
|
||||
} finally {
|
||||
mConn.disconnect();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// nothing to do
|
||||
}
|
||||
@@ -68,7 +72,13 @@ public class DownloadRunnableFallback extends Thread {
|
||||
long rangeStart = (mMission.unknownLength || start < 1) ? -1 : start;
|
||||
|
||||
int mId = 1;
|
||||
mConn = mMission.openConnection(mId, rangeStart, -1);
|
||||
mConn = mMission.openConnection(false, rangeStart, -1);
|
||||
|
||||
if (mRetryCount == 0 && rangeStart == -1) {
|
||||
// workaround: bypass android connection pool
|
||||
mConn.setRequestProperty("Range", "bytes=0-");
|
||||
}
|
||||
|
||||
mMission.establishConnection(mId, mConn);
|
||||
|
||||
// check if the download can be resumed
|
||||
@@ -96,6 +106,8 @@ public class DownloadRunnableFallback extends Thread {
|
||||
mMission.notifyProgress(len);
|
||||
}
|
||||
|
||||
dispose();
|
||||
|
||||
// if thread goes interrupted check if the last part is written. This avoid re-download the whole file
|
||||
done = len == -1;
|
||||
} catch (Exception e) {
|
||||
@@ -107,8 +119,8 @@ public class DownloadRunnableFallback extends Thread {
|
||||
|
||||
if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) {
|
||||
// for youtube streams. The url has expired, recover
|
||||
mMission.doRecover(e);
|
||||
dispose();
|
||||
mMission.doRecover(e);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -125,8 +137,6 @@ public class DownloadRunnableFallback extends Thread {
|
||||
return;
|
||||
}
|
||||
|
||||
dispose();
|
||||
|
||||
if (done) {
|
||||
mMission.notifyFinished();
|
||||
} else {
|
||||
|
||||
@@ -16,25 +16,28 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
|
||||
private static final long serialVersionUID = 0L;
|
||||
//public static final String DIRECT_SOURCE = "direct-source://";
|
||||
|
||||
public MediaFormat format;
|
||||
MediaFormat format;
|
||||
String desired;
|
||||
boolean desired2;
|
||||
int desiredBitrate;
|
||||
byte kind;
|
||||
String validateCondition = null;
|
||||
|
||||
transient int attempts = 0;
|
||||
|
||||
String validateCondition = null;
|
||||
|
||||
public MissionRecoveryInfo(@NonNull Stream stream) {
|
||||
if (stream instanceof AudioStream) {
|
||||
desiredBitrate = ((AudioStream) stream).average_bitrate;
|
||||
desired2 = false;
|
||||
kind = 'a';
|
||||
} else if (stream instanceof VideoStream) {
|
||||
desired = ((VideoStream) stream).getResolution();
|
||||
desired2 = ((VideoStream) stream).isVideoOnly();
|
||||
kind = 'v';
|
||||
} else if (stream instanceof SubtitlesStream) {
|
||||
desired = ((SubtitlesStream) stream).getLanguageTag();
|
||||
desired2 = ((SubtitlesStream) stream).isAutoGenerated();
|
||||
kind = 's';
|
||||
} else {
|
||||
throw new RuntimeException("Unknown stream kind");
|
||||
}
|
||||
@@ -43,6 +46,38 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
|
||||
if (format == null) throw new NullPointerException("Stream format cannot be null");
|
||||
}
|
||||
|
||||
@NonNull
|
||||
@Override
|
||||
public String toString() {
|
||||
String info;
|
||||
StringBuilder str = new StringBuilder();
|
||||
str.append("type=");
|
||||
switch (kind) {
|
||||
case 'a':
|
||||
str.append("audio");
|
||||
info = "bitrate=" + desiredBitrate;
|
||||
break;
|
||||
case 'v':
|
||||
str.append("video");
|
||||
info = "quality=" + desired + " videoOnly=" + desired2;
|
||||
break;
|
||||
case 's':
|
||||
str.append("subtitles");
|
||||
info = "language=" + desired + " autoGenerated=" + desired2;
|
||||
break;
|
||||
default:
|
||||
info = "";
|
||||
str.append("other");
|
||||
}
|
||||
|
||||
str.append(" format=")
|
||||
.append(format.getName())
|
||||
.append(' ')
|
||||
.append(info);
|
||||
|
||||
return str.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int describeContents() {
|
||||
return 0;
|
||||
@@ -54,6 +89,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
|
||||
parcel.writeString(this.desired);
|
||||
parcel.writeInt(this.desired2 ? 0x01 : 0x00);
|
||||
parcel.writeInt(this.desiredBitrate);
|
||||
parcel.writeByte(this.kind);
|
||||
parcel.writeString(this.validateCondition);
|
||||
}
|
||||
|
||||
@@ -62,6 +98,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
|
||||
this.desired = parcel.readString();
|
||||
this.desired2 = parcel.readInt() != 0x00;
|
||||
this.desiredBitrate = parcel.readInt();
|
||||
this.kind = parcel.readByte();
|
||||
this.validateCondition = parcel.readString();
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ import android.widget.Toast;
|
||||
|
||||
import org.schabi.newpipe.BuildConfig;
|
||||
import org.schabi.newpipe.R;
|
||||
import org.schabi.newpipe.extractor.NewPipe;
|
||||
import org.schabi.newpipe.report.ErrorActivity;
|
||||
import org.schabi.newpipe.report.UserAction;
|
||||
import org.schabi.newpipe.util.NavigationHelper;
|
||||
@@ -44,11 +45,11 @@ import java.io.File;
|
||||
import java.lang.ref.WeakReference;
|
||||
import java.net.URI;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
|
||||
import us.shandian.giga.get.DownloadMission;
|
||||
import us.shandian.giga.get.FinishedMission;
|
||||
import us.shandian.giga.get.Mission;
|
||||
import us.shandian.giga.get.MissionRecoveryInfo;
|
||||
import us.shandian.giga.io.StoredFileHelper;
|
||||
import us.shandian.giga.service.DownloadManager;
|
||||
import us.shandian.giga.service.DownloadManagerService;
|
||||
@@ -234,7 +235,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
|
||||
// hide on error
|
||||
// show if current resource length is not fetched
|
||||
// show if length is unknown
|
||||
h.progress.setMarquee(!hasError && (!mission.isInitialized() || mission.unknownLength));
|
||||
h.progress.setMarquee(mission.isRecovering() || !hasError && (!mission.isInitialized() || mission.unknownLength));
|
||||
|
||||
float progress;
|
||||
if (mission.unknownLength) {
|
||||
@@ -463,13 +464,13 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
|
||||
break;
|
||||
case ERROR_POSTPROCESSING:
|
||||
case ERROR_POSTPROCESSING_HOLD:
|
||||
showError(mission.errObject, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed);
|
||||
showError(mission, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed);
|
||||
return;
|
||||
case ERROR_INSUFFICIENT_STORAGE:
|
||||
msg = R.string.error_insufficient_storage;
|
||||
break;
|
||||
case ERROR_UNKNOWN_EXCEPTION:
|
||||
showError(mission.errObject, UserAction.DOWNLOAD_FAILED, R.string.general_error);
|
||||
showError(mission, UserAction.DOWNLOAD_FAILED, R.string.general_error);
|
||||
return;
|
||||
case ERROR_PROGRESS_LOST:
|
||||
msg = R.string.error_progress_lost;
|
||||
@@ -486,7 +487,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
|
||||
} else if (mission.errObject == null) {
|
||||
msgEx = "(not_decelerated_error_code)";
|
||||
} else {
|
||||
showError(mission.errObject, UserAction.DOWNLOAD_FAILED, msg);
|
||||
showError(mission, UserAction.DOWNLOAD_FAILED, msg);
|
||||
return;
|
||||
}
|
||||
break;
|
||||
@@ -503,7 +504,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
|
||||
if (mission.errObject != null && (mission.errCode < 100 || mission.errCode >= 600)) {
|
||||
@StringRes final int mMsg = msg;
|
||||
builder.setPositiveButton(R.string.error_report_title, (dialog, which) ->
|
||||
showError(mission.errObject, UserAction.DOWNLOAD_FAILED, mMsg)
|
||||
showError(mission, UserAction.DOWNLOAD_FAILED, mMsg)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -513,13 +514,30 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
|
||||
.show();
|
||||
}
|
||||
|
||||
private void showError(Exception exception, UserAction action, @StringRes int reason) {
|
||||
private void showError(DownloadMission mission, UserAction action, @StringRes int reason) {
|
||||
StringBuilder request = new StringBuilder(256);
|
||||
request.append(mission.source);
|
||||
|
||||
request.append(" [");
|
||||
if (mission.recoveryInfo != null) {
|
||||
for (MissionRecoveryInfo recovery : mission.recoveryInfo)
|
||||
request.append(" {").append(recovery.toString()).append("} ");
|
||||
}
|
||||
request.append("]");
|
||||
|
||||
String service;
|
||||
try {
|
||||
service = NewPipe.getServiceByUrl(mission.source).getServiceInfo().getName();
|
||||
} catch (Exception e) {
|
||||
service = "-";
|
||||
}
|
||||
|
||||
ErrorActivity.reportError(
|
||||
mContext,
|
||||
Collections.singletonList(exception),
|
||||
mission.errObject,
|
||||
null,
|
||||
null,
|
||||
ErrorActivity.ErrorInfo.make(action, "-", "-", reason)
|
||||
ErrorActivity.ErrorInfo.make(action, service, request.toString(), reason)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -453,7 +453,7 @@
|
||||
<string name="error_insufficient_storage">No hay suficiente espacio disponible en el dispositivo</string>
|
||||
<string name="error_progress_lost">Se perdió el progreso porque el archivo fue eliminado</string>
|
||||
<string name="error_timeout">Tiempo de espera excedido</string>
|
||||
<string name="error_download_resource_gone">El recurso solicitado ya no esta disponible</string>
|
||||
<string name="error_download_resource_gone">No se puede recuperar esta descarga</string>
|
||||
<string name="downloads_storage_ask_title">Preguntar dónde descargar</string>
|
||||
<string name="downloads_storage_ask_summary">Se preguntará dónde guardar cada descarga</string>
|
||||
<string name="downloads_storage_ask_summary_kitkat">Se le preguntará dónde guardar cada descarga.
|
||||
|
||||
@@ -557,7 +557,7 @@
|
||||
<string name="error_insufficient_storage">No space left on device</string>
|
||||
<string name="error_progress_lost">Progress lost, because the file was deleted</string>
|
||||
<string name="error_timeout">Connection timeout</string>
|
||||
<string name="error_download_resource_gone">The solicited resource is not available anymore</string>
|
||||
<string name="error_download_resource_gone">Cannot recover this download</string>
|
||||
<string name="clear_finished_download">Clear finished downloads</string>
|
||||
<string name="confirm_prompt">Are you sure?</string>
|
||||
<string name="msg_pending_downloads">Continue your %s pending transfers from Downloads</string>
|
||||
|
||||
Reference in New Issue
Block a user