misc changes

* OggFromWebMWriter: rewrite (again), reduce iterations over the input. Works as-is (video streams are not supported)
* WebMReader: use int for SimpleBlock.dataSize instead of long
* Download Recovery: allow recovering uninitialized downloads
* check range-requests using HEAD method instead of GET
* DownloadRunnableFallback: add workaround for 32kB/s issue, unknown issue origin, wont fix
* reporting downloads errors now include the source url with the selected quality and format

app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java

@@ -12,8 +12,6 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
-import java.util.ArrayList;
-import java.util.Random;
 
 import javax.annotation.Nullable;
 
@@ -23,15 +21,13 @@ import javax.annotation.Nullable;
 public class OggFromWebMWriter implements Closeable {
 
     private static final byte FLAG_UNSET = 0x00;
-    private static final byte FLAG_CONTINUED = 0x01;
+    //private static final byte FLAG_CONTINUED = 0x01;
     private static final byte FLAG_FIRST = 0x02;
     private static final byte FLAG_LAST = 0x04;
 
     private final static byte HEADER_CHECKSUM_OFFSET = 22;
     private final static byte HEADER_SIZE = 27;
 
-    private final static short BUFFER_SIZE = 8 * 1024;// 8KiB
-
     private final static int TIME_SCALE_NS = 1000000000;
 
     private boolean done = false;
@@ -43,7 +39,6 @@ public class OggFromWebMWriter implements Closeable {
     private int sequence_count = 0;
     private final int STREAM_ID;
     private byte packet_flag = FLAG_FIRST;
-    private int track_index = 0;
 
     private WebMReader webm = null;
     private WebMTrack webm_track = null;
@@ -71,7 +66,7 @@ public class OggFromWebMWriter implements Closeable {
         this.source = source;
         this.output = target;
 
-        this.STREAM_ID = (new Random(System.currentTimeMillis())).nextInt();
+        this.STREAM_ID = (int) System.currentTimeMillis();
 
         populate_crc32_table();
     }
@@ -130,7 +125,6 @@ public class OggFromWebMWriter implements Closeable {
 
         try {
             webm_track = webm.selectTrack(trackIndex);
-            track_index = trackIndex;
         } finally {
             parsed = true;
         }
@@ -154,8 +148,11 @@ public class OggFromWebMWriter implements Closeable {
 
     public void build() throws IOException {
         float resolution;
-        int read;
-        byte[] buffer;
+        SimpleBlock bloq;
+        ByteBuffer header = ByteBuffer.allocate(27 + (255 * 255));
+        ByteBuffer page = ByteBuffer.allocate(64 * 1024);
+
+        header.order(ByteOrder.LITTLE_ENDIAN);
 
         /* step 1: get the amount of frames per seconds */
         switch (webm_track.kind) {
@@ -176,57 +173,32 @@ public class OggFromWebMWriter implements Closeable {
                 throw new RuntimeException("not implemented");
         }
 
-        /* step 2a: create packet with code init data */
-        ArrayList<byte[]> data_extra = new ArrayList<>(4);
-
+        /* step 2: create packet with code init data */
         if (webm_track.codecPrivate != null) {
             addPacketSegment(webm_track.codecPrivate.length);
-            ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + webm_track.codecPrivate.length);
-
-            make_packetHeader(0x00, buff, webm_track.codecPrivate);
-            data_extra.add(buff.array());
+            make_packetHeader(0x00, header, webm_track.codecPrivate);
+            write(header);
+            output.write(webm_track.codecPrivate);
         }
 
-        /* step 2b: create packet with metadata */
-        buffer = make_metadata();
+        /* step 3: create packet with metadata */
+        byte[] buffer = make_metadata();
         if (buffer != null) {
             addPacketSegment(buffer.length);
-            ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + buffer.length);
-
-            make_packetHeader(0x00, buff, buffer);
-            data_extra.add(buff.array());
+            make_packetHeader(0x00, header, buffer);
+            write(header);
+            output.write(buffer);
         }
 
-
-        /* step 3: calculate amount of packets */
-        SimpleBlock bloq;
-        int reserve_header = 0;
-        int headers_amount = 0;
-
+        /* step 4: calculate amount of packets */
         while (webm_segment != null) {
             bloq = getNextBlock();
 
-            if (addPacketSegment(bloq)) {
-                continue;
-            }
-
-            reserve_header += HEADER_SIZE + segment_table_size;// header size
-            clearSegmentTable();
-            webm_block = bloq;
-            headers_amount++;
-        }
-
-        /* step 4: create packet headers */
-        rewind_source();
-
-        ByteBuffer headers = byte_buffer(reserve_header);
-        short[] headers_size = new short[headers_amount];
-        int header_index = 0;
-
-        while (webm_segment != null) {
-            bloq = getNextBlock();
-
-            if (addPacketSegment(bloq)) {
+            if (bloq != null && addPacketSegment(bloq)) {
+                int pos = page.position();
+                //noinspection ResultOfMethodCallIgnored
+                bloq.data.read(page.array(), pos, bloq.dataSize);
+                page.position(pos + bloq.dataSize);
                 continue;
             }
 
@@ -251,75 +223,21 @@ public class OggFromWebMWriter implements Closeable {
             elapsed_ns = elapsed_ns / TIME_SCALE_NS;
             elapsed_ns = Math.ceil(elapsed_ns * resolution);
 
-            // create header
-            headers_size[header_index++] = make_packetHeader((long) elapsed_ns, headers, null);
+            // create header and calculate page checksum
+            int checksum = make_packetHeader((long) elapsed_ns, header, null);
+            checksum = calc_crc32(checksum, page.array(), page.position());
+
+            header.putInt(HEADER_CHECKSUM_OFFSET, checksum);
+
+            // dump data
+            write(header);
+            write(page);
+
             webm_block = bloq;
         }
-
-
-        /* step 5: calculate checksums */
-        rewind_source();
-
-        int offset = 0;
-        buffer = new byte[BUFFER_SIZE];
-
-        for (header_index = 0; header_index < headers_size.length; header_index++) {
-            int checksum_offset = offset + HEADER_CHECKSUM_OFFSET;
-            int checksum = headers.getInt(checksum_offset);
-
-            while (webm_segment != null) {
-                bloq = getNextBlock();
-
-                if (!addPacketSegment(bloq)) {
-                    clearSegmentTable();
-                    webm_block = bloq;
-                    break;
-                }
-
-                // calculate page checksum
-                while ((read = bloq.data.read(buffer)) > 0) {
-                    checksum = calc_crc32(checksum, buffer, 0, read);
-                }
-            }
-
-            headers.putInt(checksum_offset, checksum);
-            offset += headers_size[header_index];
-        }
-
-        /* step 6: write extra headers */
-        rewind_source();
-
-        for (byte[] buff : data_extra) {
-            output.write(buff);
-        }
-
-        /* step 7: write stream packets */
-        byte[] headers_buffers = headers.array();
-        offset = 0;
-        buffer = new byte[BUFFER_SIZE];
-
-        for (header_index = 0; header_index < headers_size.length; header_index++) {
-            output.write(headers_buffers, offset, headers_size[header_index]);
-            offset += headers_size[header_index];
-
-            while (webm_segment != null) {
-                bloq = getNextBlock();
-
-                if (addPacketSegment(bloq)) {
-                    while ((read = bloq.data.read(buffer)) > 0) {
-                        output.write(buffer, 0, read);
-                    }
-                } else {
-                    clearSegmentTable();
-                    webm_block = bloq;
-                    break;
-                }
-            }
-        }
     }
 
-    private short make_packetHeader(long gran_pos, ByteBuffer buffer, byte[] immediate_page) {
-        int offset = buffer.position();
+    private int make_packetHeader(long gran_pos, @NonNull ByteBuffer buffer, byte[] immediate_page) {
         short length = HEADER_SIZE;
 
         buffer.putInt(0x5367674f);// "OggS" binary string in little-endian
@@ -340,17 +258,15 @@ public class OggFromWebMWriter implements Closeable {
 
         clearSegmentTable();// clear segment table for next header
 
-        int checksum_crc32 = calc_crc32(0x00, buffer.array(), offset, length);
+        int checksum_crc32 = calc_crc32(0x00, buffer.array(), length);
 
         if (immediate_page != null) {
-            checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, 0, immediate_page.length);
-            System.arraycopy(immediate_page, 0, buffer.array(), length, immediate_page.length);
+            checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, immediate_page.length);
+            buffer.putInt(HEADER_CHECKSUM_OFFSET, checksum_crc32);
             segment_table_next_timestamp -= TIME_SCALE_NS;
         }
 
-        buffer.putInt(offset + HEADER_CHECKSUM_OFFSET, checksum_crc32);
-
-        return length;
+        return checksum_crc32;
     }
 
     @Nullable
@@ -358,7 +274,7 @@ public class OggFromWebMWriter implements Closeable {
         if ("A_OPUS".equals(webm_track.codecId)) {
             return new byte[]{
                     0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73,// "OpusTags" binary string
-                    0x07, 0x00, 0x00, 0x00,// writing application string size
+                    0x07, 0x00, 0x00, 0x00,// writting application string size
                     0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string
                     0x00, 0x00, 0x00, 0x00// additional tags count (zero means no tags)
             };
@@ -366,7 +282,7 @@ public class OggFromWebMWriter implements Closeable {
             return new byte[]{
                     0x03,// ????????
                     0x76, 0x6f, 0x72, 0x62, 0x69, 0x73,// "vorbis" binary string
-                    0x07, 0x00, 0x00, 0x00,// writing application string size
+                    0x07, 0x00, 0x00, 0x00,// writting application string size
                     0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string
                     0x01, 0x00, 0x00, 0x00,// additional tags count (zero means no tags)
 
@@ -387,22 +303,9 @@ public class OggFromWebMWriter implements Closeable {
         return null;
     }
 
-    private void rewind_source() throws IOException {
-        source.rewind();
-
-        webm = new WebMReader(source);
-        webm.parse();
-        webm_track = webm.selectTrack(track_index);
-        webm_segment = webm.getNextSegment();
-        webm_cluster = null;
-        webm_block = null;
-        webm_block_last_timecode = 0L;
-
-        segment_table_next_timestamp = TIME_SCALE_NS;
-    }
-
-    private ByteBuffer byte_buffer(int size) {
-        return ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN);
+    private void write(ByteBuffer buffer) throws IOException {
+        output.write(buffer.array(), 0, buffer.position());
+        buffer.position(0);
     }
 
     //<editor-fold defaultstate="collapsed" desc="WebM track handling">
@@ -460,41 +363,32 @@ public class OggFromWebMWriter implements Closeable {
 
     //<editor-fold defaultstate="collapsed" desc="Segment table writing">
     private void clearSegmentTable() {
-        if (packet_flag != FLAG_CONTINUED) {
-            segment_table_next_timestamp += TIME_SCALE_NS;
-            packet_flag = FLAG_UNSET;
-        }
+        segment_table_next_timestamp += TIME_SCALE_NS;
+        packet_flag = FLAG_UNSET;
         segment_table_size = 0;
     }
 
     private boolean addPacketSegment(SimpleBlock block) {
-        if (block == null) {
-            return false;
-        }
-
         long timestamp = block.absoluteTimeCodeNs + webm_track.codecDelay;
 
         if (timestamp >= segment_table_next_timestamp) {
             return false;
         }
 
-        boolean result = addPacketSegment((int) block.dataSize);
-
-        if (!result && segment_table_next_timestamp < timestamp) {
-            // WARNING: ¡¡¡¡ not implemented (lack of documentation) !!!!
-            packet_flag = FLAG_CONTINUED;
-        }
-
-        return result;
+        return addPacketSegment(block.dataSize);
     }
 
     private boolean addPacketSegment(int size) {
+        if (size > 65025) {
+            throw new UnsupportedOperationException("page size cannot be larger than 65025");
+        }
+
         int available = (segment_table.length - segment_table_size) * 255;
-        boolean extra = size == 255;
+        boolean extra = (size % 255) == 0;
 
         if (extra) {
             // add a zero byte entry in the table
-            // required to indicate the sample size is exactly 255
+            // required to indicate the sample size is multiple of 255
             available -= 255;
         }
 
@@ -528,12 +422,10 @@ public class OggFromWebMWriter implements Closeable {
         }
     }
 
-    private int calc_crc32(int initial_crc, byte[] buffer, int offset, int size) {
-        size += offset;
-
-        for (; offset < size; offset++) {
+    private int calc_crc32(int initial_crc, byte[] buffer, int size) {
+        for (int i = 0; i < size; i++) {
             int reg = (initial_crc >>> 24) & 0xff;
-            initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[offset] & 0xff)];
+            initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[i] & 0xff)];
         }
 
         return initial_crc;

app/src/main/java/org/schabi/newpipe/streams/WebMReader.java

@@ -368,7 +368,7 @@ public class WebMReader {
         obj.trackNumber = readEncodedNumber();
         obj.relativeTimeCode = stream.readShort();
         obj.flags = (byte) stream.read();
-        obj.dataSize = (ref.offset + ref.size) - stream.position();
+        obj.dataSize = (int) ((ref.offset + ref.size) - stream.position());
         obj.createdFromBlock = ref.type == ID_Block;
 
         // NOTE: lacing is not implemented, and will be mixed with the stream data
@@ -465,7 +465,7 @@ public class WebMReader {
         public short relativeTimeCode;
         public long absoluteTimeCodeNs;
         public byte flags;
-        public long dataSize;
+        public int dataSize;
         private final Element ref;
 
         public boolean isKeyframe() {

app/src/main/java/us/shandian/giga/get/DownloadInitializer.java

@@ -14,6 +14,7 @@ import java.nio.channels.ClosedByInterruptException;
 import us.shandian.giga.util.Utility;
 
 import static org.schabi.newpipe.BuildConfig.DEBUG;
+import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_FORBIDDEN;
 
 public class DownloadInitializer extends Thread {
     private final static String TAG = "DownloadInitializer";
@@ -29,9 +30,9 @@ public class DownloadInitializer extends Thread {
         mConn = null;
     }
 
-    private static void safeClose(HttpURLConnection con) {
+    private void dispose() {
         try {
-            con.getInputStream().close();
+            mConn.getInputStream().close();
         } catch (Exception e) {
             // nothing to do
         }
@@ -52,9 +53,9 @@ public class DownloadInitializer extends Thread {
                     long lowestSize = Long.MAX_VALUE;
 
                     for (int i = 0; i < mMission.urls.length && mMission.running; i++) {
-                        mConn = mMission.openConnection(mMission.urls[i], mId, -1, -1);
+                        mConn = mMission.openConnection(mMission.urls[i], true, -1, -1);
                         mMission.establishConnection(mId, mConn);
-                        safeClose(mConn);
+                        dispose();
 
                         if (Thread.interrupted()) return;
                         long length = Utility.getContentLength(mConn);
@@ -82,9 +83,9 @@ public class DownloadInitializer extends Thread {
                     }
                 } else {
                     // ask for the current resource length
-                    mConn = mMission.openConnection(mId, -1, -1);
+                    mConn = mMission.openConnection(true, -1, -1);
                     mMission.establishConnection(mId, mConn);
-                    safeClose(mConn);
+                    dispose();
 
                     if (!mMission.running || Thread.interrupted()) return;
 
@@ -108,9 +109,9 @@ public class DownloadInitializer extends Thread {
                     }
                 } else {
                     // Open again
-                    mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length);
+                    mConn = mMission.openConnection(true, mMission.length - 10, mMission.length);
                     mMission.establishConnection(mId, mConn);
-                    safeClose(mConn);
+                    dispose();
 
                     if (!mMission.running || Thread.interrupted()) return;
 
@@ -171,7 +172,14 @@ public class DownloadInitializer extends Thread {
             } catch (InterruptedIOException | ClosedByInterruptException e) {
                 return;
             } catch (Exception e) {
-                if (!mMission.running) return;
+                if (!mMission.running || super.isInterrupted()) return;
+
+                if (e instanceof DownloadMission.HttpError && ((DownloadMission.HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) {
+                    // for youtube streams. The url has expired
+                    interrupt();
+                    mMission.doRecover(e);
+                    return;
+                }
 
                 if (e instanceof IOException && e.getMessage().contains("Permission denied")) {
                     mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e);
@@ -194,13 +202,6 @@ public class DownloadInitializer extends Thread {
     @Override
     public void interrupt() {
         super.interrupt();
-
-        if (mConn != null) {
-            try {
-                mConn.disconnect();
-            } catch (Exception e) {
-                // nothing to do
-            }
-        }
+        if (mConn != null) dispose();
     }
 }

app/src/main/java/us/shandian/giga/get/DownloadMission.java

@@ -204,22 +204,24 @@ public class DownloadMission extends Mission {
     /**
      * Opens a connection
      *
-     * @param threadId   id of the calling thread, used only for debugging
-     * @param rangeStart range start
-     * @param rangeEnd   range end
+     * @param headRequest {@code true} for use {@code HEAD} request method, otherwise, {@code GET} is used
+     * @param rangeStart  range start
+     * @param rangeEnd    range end
      * @return a {@link java.net.URLConnection URLConnection} linking to the URL.
      * @throws IOException if an I/O exception occurs.
      */
-    HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException {
-        return openConnection(urls[current], threadId, rangeStart, rangeEnd);
+    HttpURLConnection openConnection(boolean headRequest, long rangeStart, long rangeEnd) throws IOException {
+        return openConnection(urls[current], headRequest, rangeStart, rangeEnd);
     }
 
-    HttpURLConnection openConnection(String url, int threadId, long rangeStart, long rangeEnd) throws IOException {
+    HttpURLConnection openConnection(String url, boolean headRequest, long rangeStart, long rangeEnd) throws IOException {
         HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
         conn.setInstanceFollowRedirects(true);
         conn.setRequestProperty("User-Agent", DownloaderImpl.USER_AGENT);
         conn.setRequestProperty("Accept", "*/*");
 
+        if (headRequest) conn.setRequestMethod("HEAD");
+
         // BUG workaround: switching between networks can freeze the download forever
         conn.setConnectTimeout(30000);
         conn.setReadTimeout(10000);
@@ -229,10 +231,6 @@ public class DownloadMission extends Mission {
             if (rangeEnd > 0) req += rangeEnd;
 
             conn.setRequestProperty("Range", req);
-
-            if (DEBUG) {
-                Log.d(TAG, threadId + ":" + conn.getRequestProperty("Range"));
-            }
         }
 
         return conn;
@@ -245,13 +243,14 @@ public class DownloadMission extends Mission {
      * @throws HttpError   if the HTTP Status-Code is not satisfiable
      */
     void establishConnection(int threadId, HttpURLConnection conn) throws IOException, HttpError {
-        conn.connect();
         int statusCode = conn.getResponseCode();
 
         if (DEBUG) {
+            Log.d(TAG, threadId + ":Range=" + conn.getRequestProperty("Range"));
             Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + statusCode);
         }
 
+
         switch (statusCode) {
             case 204:
             case 205:
@@ -676,6 +675,15 @@ public class DownloadMission extends Mission {
         return (isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) || isFinished();
     }
 
+    /**
+     * Indicates if mission urls has expired and there an attempt to renovate them
+     *
+     * @return {@code true} if the mission is running a recovery procedure, otherwise, {@code false}
+     */
+    public boolean isRecovering() {
+        return threads != null && threads.length > 0 && threads[0] instanceof DownloadRunnable && threads[0].isAlive();
+    }
+
     private boolean doPostprocessing() {
         if (psAlgorithm == null || psState == 2) return true;
 
@@ -742,10 +750,8 @@ public class DownloadMission extends Mission {
             }
         }
 
-        // set the current download url to null in case if the recovery
-        // process is canceled. Next time start() method is called the
-        // recovery will be executed, saving time
-        urls[current] = null;
+        errCode = ERROR_NOTHING;
+        errObject = null;
 
         if (recoveryInfo[current].attempts >= maxRetry) {
             recoveryInfo[current].attempts = 0;

app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java

@@ -10,10 +10,12 @@ import org.schabi.newpipe.extractor.stream.SubtitlesStream;
 import org.schabi.newpipe.extractor.stream.VideoStream;
 
 import java.io.IOException;
+import java.io.InterruptedIOException;
 import java.net.HttpURLConnection;
 import java.nio.channels.ClosedByInterruptException;
 import java.util.List;
 
+import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING;
 import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE;
 
 public class DownloadMissionRecover extends Thread {
@@ -21,14 +23,17 @@ public class DownloadMissionRecover extends Thread {
     static final int mID = -3;
 
     private final DownloadMission mMission;
-    private final MissionRecoveryInfo mRecovery;
     private final Exception mFromError;
+    private final boolean notInitialized;
+
     private HttpURLConnection mConn;
+    private MissionRecoveryInfo mRecovery;
+    private StreamExtractor mExtractor;
 
     DownloadMissionRecover(DownloadMission mission, Exception originError) {
         mMission = mission;
         mFromError = originError;
-        mRecovery = mission.recoveryInfo[mission.current];
+        notInitialized = mission.blocks == null && mission.current == 0;
     }
 
     @Override
@@ -38,28 +43,78 @@ public class DownloadMissionRecover extends Thread {
             return;
         }
 
+        /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) {
+            resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length()));
+            return;
+        }*/
+
         try {
-            /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) {
-                resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length()));
-                return;
-            }*/
-
             StreamingService svr = NewPipe.getServiceByUrl(mMission.source);
-
-            if (svr == null) {
-                throw new RuntimeException("Unknown source service");
-            }
-
-            StreamExtractor extractor = svr.getStreamExtractor(mMission.source);
-            extractor.fetchPage();
-
+            mExtractor = svr.getStreamExtractor(mMission.source);
+            mExtractor.fetchPage();
+        } catch (InterruptedIOException | ClosedByInterruptException e) {
+            return;
+        } catch (Exception e) {
             if (!mMission.running || super.isInterrupted()) return;
+            mMission.notifyError(e);
+            return;
+        }
 
+        // maybe the following check is redundant
+        if (!mMission.running || super.isInterrupted()) return;
+
+        if (!notInitialized) {
+            // set the current download url to null in case if the recovery
+            // process is canceled. Next time start() method is called the
+            // recovery will be executed, saving time
+            mMission.urls[mMission.current] = null;
+
+            mRecovery = mMission.recoveryInfo[mMission.current];
+            resolveStream();
+            return;
+        }
+
+        Log.w(TAG, "mission is not fully initialized, this will take a while");
+
+        try {
+            for (; mMission.current < mMission.urls.length; mMission.current++) {
+                mRecovery = mMission.recoveryInfo[mMission.current];
+
+                if (test()) continue;
+                if (!mMission.running) return;
+
+                resolveStream();
+                if (!mMission.running) return;
+
+                // before continue, check if the current stream was resolved
+                if (mMission.urls[mMission.current] == null || mMission.errCode != ERROR_NOTHING) {
+                    break;
+                }
+            }
+        } finally {
+            mMission.current = 0;
+        }
+
+        mMission.writeThisToFile();
+
+        if (!mMission.running || super.isInterrupted()) return;
+
+        mMission.running = false;
+        mMission.start();
+    }
+
+    private void resolveStream() {
+        if (mExtractor.getErrorMessage() != null) {
+            mMission.notifyError(mFromError);
+            return;
+        }
+
+        try {
             String url = null;
 
-            switch (mMission.kind) {
+            switch (mRecovery.kind) {
                 case 'a':
-                    for (AudioStream audio : extractor.getAudioStreams()) {
+                    for (AudioStream audio : mExtractor.getAudioStreams()) {
                         if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) {
                             url = audio.getUrl();
                             break;
@@ -69,9 +124,9 @@ public class DownloadMissionRecover extends Thread {
                 case 'v':
                     List<VideoStream> videoStreams;
                     if (mRecovery.desired2)
-                        videoStreams = extractor.getVideoOnlyStreams();
+                        videoStreams = mExtractor.getVideoOnlyStreams();
                     else
-                        videoStreams = extractor.getVideoStreams();
+                        videoStreams = mExtractor.getVideoStreams();
                     for (VideoStream video : videoStreams) {
                         if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) {
                             url = video.getUrl();
@@ -80,7 +135,7 @@ public class DownloadMissionRecover extends Thread {
                     }
                     break;
                 case 's':
-                    for (SubtitlesStream subtitles : extractor.getSubtitles(mRecovery.format)) {
+                    for (SubtitlesStream subtitles : mExtractor.getSubtitles(mRecovery.format)) {
                         String tag = subtitles.getLanguageTag();
                         if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) {
                             url = subtitles.getURL();
@@ -114,7 +169,7 @@ public class DownloadMissionRecover extends Thread {
         ////// Validate the http resource doing a range request
         /////////////////////
         try {
-            mConn = mMission.openConnection(url, mID, mMission.length - 10, mMission.length);
+            mConn = mMission.openConnection(url, true, mMission.length - 10, mMission.length);
             mConn.setRequestProperty("If-Range", mRecovery.validateCondition);
             mMission.establishConnection(mID, mConn);
 
@@ -140,22 +195,24 @@ public class DownloadMissionRecover extends Thread {
             if (!mMission.running || e instanceof ClosedByInterruptException) return;
             throw e;
         } finally {
-            this.interrupt();
+            disconnect();
         }
     }
 
     private void recover(String url, boolean stale) {
         Log.i(TAG,
-                String.format("download recovered  name=%s  isStale=%s  url=%s", mMission.storage.getName(), stale, url)
+                String.format("recover()  name=%s  isStale=%s  url=%s", mMission.storage.getName(), stale, url)
         );
 
+        mMission.urls[mMission.current] = url;
+        mRecovery.attempts = 0;
+
         if (url == null) {
             mMission.notifyError(ERROR_RESOURCE_GONE, null);
             return;
         }
 
-        mMission.urls[mMission.current] = url;
-        mRecovery.attempts = 0;
+        if (notInitialized) return;
 
         if (stale) {
             mMission.resetState(false, false, DownloadMission.ERROR_NOTHING);
@@ -208,15 +265,40 @@ public class DownloadMissionRecover extends Thread {
         return range;
     }
 
+    private boolean test() {
+        if (mMission.urls[mMission.current] == null) return false;
+
+        try {
+            mConn = mMission.openConnection(mMission.urls[mMission.current], true, -1, -1);
+            mMission.establishConnection(mID, mConn);
+
+            if (mConn.getResponseCode() == 200) return true;
+        } catch (Exception e) {
+            // nothing to do
+        } finally {
+            disconnect();
+        }
+
+        return false;
+    }
+
+    private void disconnect() {
+        try {
+            try {
+                mConn.getInputStream().close();
+            } finally {
+                mConn.disconnect();
+            }
+        } catch (Exception e) {
+            // nothing to do
+        } finally {
+            mConn = null;
+        }
+    }
+
     @Override
     public void interrupt() {
         super.interrupt();
-        if (mConn != null) {
-            try {
-                mConn.disconnect();
-            } catch (Exception e) {
-                // nothing to do
-            }
-        }
+        if (mConn != null) disconnect();
     }
 }

app/src/main/java/us/shandian/giga/get/DownloadRunnable.java

@@ -80,7 +80,7 @@ public class DownloadRunnable extends Thread {
             }
 
             try {
-                mConn = mMission.openConnection(mId, start, end);
+                mConn = mMission.openConnection(false, start, end);
                 mMission.establishConnection(mId, mConn);
 
                 // check if the download can be resumed

app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java

@@ -35,7 +35,11 @@ public class DownloadRunnableFallback extends Thread {
 
     private void dispose() {
         try {
-            if (mIs != null) mIs.close();
+            try {
+                if (mIs != null) mIs.close();
+            } finally {
+                mConn.disconnect();
+            }
         } catch (IOException e) {
             // nothing to do
         }
@@ -68,7 +72,13 @@ public class DownloadRunnableFallback extends Thread {
             long rangeStart = (mMission.unknownLength || start < 1) ? -1 : start;
 
             int mId = 1;
-            mConn = mMission.openConnection(mId, rangeStart, -1);
+            mConn = mMission.openConnection(false, rangeStart, -1);
+
+            if (mRetryCount == 0 && rangeStart == -1) {
+                // workaround: bypass android connection pool
+                mConn.setRequestProperty("Range", "bytes=0-");
+            }
+
             mMission.establishConnection(mId, mConn);
 
             // check if the download can be resumed
@@ -96,6 +106,8 @@ public class DownloadRunnableFallback extends Thread {
                 mMission.notifyProgress(len);
             }
 
+            dispose();
+
             // if thread goes interrupted check if the last part is written. This avoid re-download the whole file
             done = len == -1;
         } catch (Exception e) {
@@ -107,8 +119,8 @@ public class DownloadRunnableFallback extends Thread {
 
             if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) {
                 // for youtube streams. The url has expired, recover
-                mMission.doRecover(e);
                 dispose();
+                mMission.doRecover(e);
                 return;
             }
 
@@ -125,8 +137,6 @@ public class DownloadRunnableFallback extends Thread {
             return;
         }
 
-        dispose();
-
         if (done) {
             mMission.notifyFinished();
         } else {

app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java

@@ -16,25 +16,28 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
     private static final long serialVersionUID = 0L;
     //public static final String DIRECT_SOURCE = "direct-source://";
 
-    public MediaFormat format;
+    MediaFormat format;
     String desired;
     boolean desired2;
     int desiredBitrate;
+    byte kind;
+    String validateCondition = null;
 
     transient int attempts = 0;
 
-    String validateCondition = null;
-
     public MissionRecoveryInfo(@NonNull Stream stream) {
         if (stream instanceof AudioStream) {
             desiredBitrate = ((AudioStream) stream).average_bitrate;
             desired2 = false;
+            kind = 'a';
         } else if (stream instanceof VideoStream) {
             desired = ((VideoStream) stream).getResolution();
             desired2 = ((VideoStream) stream).isVideoOnly();
+            kind = 'v';
         } else if (stream instanceof SubtitlesStream) {
             desired = ((SubtitlesStream) stream).getLanguageTag();
             desired2 = ((SubtitlesStream) stream).isAutoGenerated();
+            kind = 's';
         } else {
             throw new RuntimeException("Unknown stream kind");
         }
@@ -43,6 +46,38 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
         if (format == null) throw new NullPointerException("Stream format cannot be null");
     }
 
+    @NonNull
+    @Override
+    public String toString() {
+        String info;
+        StringBuilder str = new StringBuilder();
+        str.append("type=");
+        switch (kind) {
+            case 'a':
+                str.append("audio");
+                info = "bitrate=" + desiredBitrate;
+                break;
+            case 'v':
+                str.append("video");
+                info = "quality=" + desired + " videoOnly=" + desired2;
+                break;
+            case 's':
+                str.append("subtitles");
+                info = "language=" + desired + " autoGenerated=" + desired2;
+                break;
+            default:
+                info = "";
+                str.append("other");
+        }
+
+        str.append(" format=")
+                .append(format.getName())
+                .append(' ')
+                .append(info);
+
+        return str.toString();
+    }
+
     @Override
     public int describeContents() {
         return 0;
@@ -54,6 +89,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
         parcel.writeString(this.desired);
         parcel.writeInt(this.desired2 ? 0x01 : 0x00);
         parcel.writeInt(this.desiredBitrate);
+        parcel.writeByte(this.kind);
         parcel.writeString(this.validateCondition);
     }
 
@@ -62,6 +98,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
         this.desired = parcel.readString();
         this.desired2 = parcel.readInt() != 0x00;
         this.desiredBitrate = parcel.readInt();
+        this.kind = parcel.readByte();
         this.validateCondition = parcel.readString();
     }
 

app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java

@@ -36,6 +36,7 @@ import android.widget.Toast;
 
 import org.schabi.newpipe.BuildConfig;
 import org.schabi.newpipe.R;
+import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.report.ErrorActivity;
 import org.schabi.newpipe.report.UserAction;
 import org.schabi.newpipe.util.NavigationHelper;
@@ -44,11 +45,11 @@ import java.io.File;
 import java.lang.ref.WeakReference;
 import java.net.URI;
 import java.util.ArrayList;
-import java.util.Collections;
 
 import us.shandian.giga.get.DownloadMission;
 import us.shandian.giga.get.FinishedMission;
 import us.shandian.giga.get.Mission;
+import us.shandian.giga.get.MissionRecoveryInfo;
 import us.shandian.giga.io.StoredFileHelper;
 import us.shandian.giga.service.DownloadManager;
 import us.shandian.giga.service.DownloadManagerService;
@@ -234,7 +235,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
         // hide on error
         // show if current resource length is not fetched
         // show if length is unknown
-        h.progress.setMarquee(!hasError && (!mission.isInitialized() || mission.unknownLength));
+        h.progress.setMarquee(mission.isRecovering() || !hasError && (!mission.isInitialized() || mission.unknownLength));
 
         float progress;
         if (mission.unknownLength) {
@@ -463,13 +464,13 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
                 break;
             case ERROR_POSTPROCESSING:
             case ERROR_POSTPROCESSING_HOLD:
-                showError(mission.errObject, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed);
+                showError(mission, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed);
                 return;
             case ERROR_INSUFFICIENT_STORAGE:
                 msg = R.string.error_insufficient_storage;
                 break;
             case ERROR_UNKNOWN_EXCEPTION:
-                showError(mission.errObject, UserAction.DOWNLOAD_FAILED, R.string.general_error);
+                showError(mission, UserAction.DOWNLOAD_FAILED, R.string.general_error);
                 return;
             case ERROR_PROGRESS_LOST:
                 msg = R.string.error_progress_lost;
@@ -486,7 +487,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
                 } else if (mission.errObject == null) {
                     msgEx = "(not_decelerated_error_code)";
                 } else {
-                    showError(mission.errObject, UserAction.DOWNLOAD_FAILED, msg);
+                    showError(mission, UserAction.DOWNLOAD_FAILED, msg);
                     return;
                 }
                 break;
@@ -503,7 +504,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
         if (mission.errObject != null && (mission.errCode < 100 || mission.errCode >= 600)) {
             @StringRes final int mMsg = msg;
             builder.setPositiveButton(R.string.error_report_title, (dialog, which) ->
-                    showError(mission.errObject, UserAction.DOWNLOAD_FAILED, mMsg)
+                    showError(mission, UserAction.DOWNLOAD_FAILED, mMsg)
             );
         }
 
@@ -513,13 +514,30 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
                 .show();
     }
 
-    private void showError(Exception exception, UserAction action, @StringRes int reason) {
+    private void showError(DownloadMission mission, UserAction action, @StringRes int reason) {
+        StringBuilder request = new StringBuilder(256);
+        request.append(mission.source);
+
+        request.append(" [");
+        if (mission.recoveryInfo != null) {
+            for (MissionRecoveryInfo recovery : mission.recoveryInfo)
+                request.append(" {").append(recovery.toString()).append("} ");
+        }
+        request.append("]");
+
+        String service;
+        try {
+            service = NewPipe.getServiceByUrl(mission.source).getServiceInfo().getName();
+        } catch (Exception e) {
+            service = "-";
+        }
+
         ErrorActivity.reportError(
                 mContext,
-                Collections.singletonList(exception),
+                mission.errObject,
                 null,
                 null,
-                ErrorActivity.ErrorInfo.make(action, "-", "-", reason)
+                ErrorActivity.ErrorInfo.make(action, service, request.toString(), reason)
         );
     }
 

app/src/main/res/values-es/strings.xml

@@ -453,7 +453,7 @@
     <string name="error_insufficient_storage">No hay suficiente espacio disponible en el dispositivo</string>
     <string name="error_progress_lost">Se perdió el progreso porque el archivo fue eliminado</string>
     <string name="error_timeout">Tiempo de espera excedido</string>
-    <string name="error_download_resource_gone">El recurso solicitado ya no esta disponible</string>
+    <string name="error_download_resource_gone">No se puede recuperar esta descarga</string>
     <string name="downloads_storage_ask_title">Preguntar dónde descargar</string>
     <string name="downloads_storage_ask_summary">Se preguntará dónde guardar cada descarga</string>
     <string name="downloads_storage_ask_summary_kitkat">Se le preguntará dónde guardar cada descarga.

app/src/main/res/values/strings.xml

@@ -557,7 +557,7 @@
     <string name="error_insufficient_storage">No space left on device</string>
     <string name="error_progress_lost">Progress lost, because the file was deleted</string>
     <string name="error_timeout">Connection timeout</string>
-    <string name="error_download_resource_gone">The solicited resource is not available anymore</string>
+    <string name="error_download_resource_gone">Cannot recover this download</string>
     <string name="clear_finished_download">Clear finished downloads</string>
     <string name="confirm_prompt">Are you sure?</string>
     <string name="msg_pending_downloads">Continue your %s pending transfers from Downloads</string>