Retrieve MediaFormat for streams that could not be extracted by the extractor

app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java

@@ -766,7 +766,7 @@ public class DownloadDialog extends DialogFragment
     }
 
     private void showFailedDialog(@StringRes final int msg) {
-        assureCorrectAppLanguage(getContext());
+        assureCorrectAppLanguage(requireContext());
         new AlertDialog.Builder(context)
                 .setTitle(R.string.general_error)
                 .setMessage(msg)
@@ -799,7 +799,7 @@ public class DownloadDialog extends DialogFragment
                     filenameTmp += "opus";
                 } else if (format != null) {
                     mimeTmp = format.mimeType;
-                    filenameTmp += format.suffix;
+                    filenameTmp += format.getSuffix();
                 }
                 break;
             case R.id.video_button:
@@ -808,7 +808,7 @@ public class DownloadDialog extends DialogFragment
                 format = videoStreamsAdapter.getItem(selectedVideoIndex).getFormat();
                 if (format != null) {
                     mimeTmp = format.mimeType;
-                    filenameTmp += format.suffix;
+                    filenameTmp += format.getSuffix();
                 }
                 break;
             case R.id.subtitle_button:
@@ -820,9 +820,9 @@ public class DownloadDialog extends DialogFragment
                 }
 
                 if (format == MediaFormat.TTML) {
-                    filenameTmp += MediaFormat.SRT.suffix;
+                    filenameTmp += MediaFormat.SRT.getSuffix();
                 } else if (format != null) {
-                    filenameTmp += format.suffix;
+                    filenameTmp += format.getSuffix();
                 }
                 break;
             default:

app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java

@@ -1,5 +1,7 @@
 package org.schabi.newpipe.util;
 
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
 import android.content.Context;
 import android.view.LayoutInflater;
 import android.view.View;
@@ -16,16 +18,20 @@ import androidx.collection.SparseArrayCompat;
 import org.schabi.newpipe.DownloaderImpl;
 import org.schabi.newpipe.R;
 import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.stream.AudioStream;
 import org.schabi.newpipe.extractor.stream.Stream;
 import org.schabi.newpipe.extractor.stream.SubtitlesStream;
 import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
 
 import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
 import io.reactivex.rxjava3.core.Single;
@@ -228,6 +234,7 @@ public class StreamItemAdapter<T extends Stream, U extends Stream> extends BaseA
 
         private final List<T> streamsList;
         private final long[] streamSizes;
+        private final MediaFormat[] streamFormats;
         private final String unknownSize;
 
         public StreamInfoWrapper(@NonNull final List<T> streamList,
@@ -236,31 +243,42 @@ public class StreamItemAdapter<T extends Stream, U extends Stream> extends BaseA
             this.streamSizes = new long[streamsList.size()];
             this.unknownSize = context == null
                     ? "--.-" : context.getString(R.string.unknown_content);
-
+            this.streamFormats = new MediaFormat[streamsList.size()];
-            resetSizes();
+            resetInfo();
         }
 
         /**
-         * Helper method to fetch the sizes of all the streams in a wrapper.
+         * Helper method to fetch the sizes and missing media formats
+         * of all the streams in a wrapper.
          *
          * @param <X> the stream type's class extending {@link Stream}
          * @param streamsWrapper the wrapper
          * @return a {@link Single} that returns a boolean indicating if any elements were changed
          */
         @NonNull
-        public static <X extends Stream> Single<Boolean> fetchSizeForWrapper(
+        public static <X extends Stream> Single<Boolean> fetchMoreInfoForWrapper(
                 final StreamInfoWrapper<X> streamsWrapper) {
             final Callable<Boolean> fetchAndSet = () -> {
                 boolean hasChanged = false;
                 for (final X stream : streamsWrapper.getStreamsList()) {
-                    if (streamsWrapper.getSizeInBytes(stream) > SIZE_UNSET) {
+                    final boolean changeSize = streamsWrapper.getSizeInBytes(stream) <= SIZE_UNSET;
+                    final boolean changeFormat = stream.getFormat() == null;
+                    if (!changeSize && !changeFormat) {
                         continue;
                     }
-
+                    final Response response = DownloaderImpl.getInstance()
-                    final long contentLength = DownloaderImpl.getInstance().getContentLength(
+                            .head(stream.getContent());
-                            stream.getContent());
+                    if (changeSize) {
-                    streamsWrapper.setSize(stream, contentLength);
+                        final String contentLength = response.getHeader("Content-Length");
-                    hasChanged = true;
+                        if (!isNullOrEmpty(contentLength)) {
+                            streamsWrapper.setSize(stream, Long.parseLong(contentLength));
+                            hasChanged = true;
+                        }
+                    }
+                    if (changeFormat) {
+                        hasChanged = retrieveMediaFormat(stream, streamsWrapper, response)
+                                || hasChanged;
+                    }
                 }
                 return hasChanged;
             };
@@ -271,8 +289,135 @@ public class StreamItemAdapter<T extends Stream, U extends Stream> extends BaseA
                     .onErrorReturnItem(true);
         }
 
-        public void resetSizes() {
+        /**
+         * Try to retrieve the {@link MediaFormat} for a stream from the request headers.
+         *
+         * @param <X>            the stream type to get the {@link MediaFormat} for
+         * @param stream         the stream to find the {@link MediaFormat} for
+         * @param streamsWrapper the wrapper to store the found {@link MediaFormat} in
+         * @param response       the response of the head request for the given stream
+         * @return {@code true} if the media format could be retrieved; {@code false} otherwise
+         */
+        private static <X extends Stream> boolean retrieveMediaFormat(
+                @NonNull final X stream,
+                @NonNull final StreamInfoWrapper<X> streamsWrapper,
+                @NonNull final Response response) {
+            return retrieveMediaFormatFromFileTypeHeaders(stream, streamsWrapper, response)
+                    || retrieveMediaFormatFromContentDispositionHeader(
+                            stream, streamsWrapper, response)
+                    || retrieveMediaFormatFromContentTypeHeader(stream, streamsWrapper, response);
+        }
+
+        private static <X extends Stream> boolean retrieveMediaFormatFromFileTypeHeaders(
+                @NonNull final X stream,
+                @NonNull final StreamInfoWrapper<X> streamsWrapper,
+                @NonNull final Response response) {
+            // try to use additional headers from CDNs or servers,
+            // e.g. x-amz-meta-file-type (e.g. for SoundCloud)
+            final List<String> keys = response.responseHeaders().keySet().stream()
+                    .filter(k -> k.endsWith("file-type")).collect(Collectors.toList());
+            if (!keys.isEmpty()) {
+                for (final String key : keys) {
+                    final String suffix = response.getHeader(key);
+                    final MediaFormat format = MediaFormat.getFromSuffix(suffix);
+                    if (format != null) {
+                        streamsWrapper.setFormat(stream, format);
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
+
+        /**
+         * <p>Retrieve a {@link MediaFormat} from a HTTP Content-Disposition header
+         * for a stream and store the info in a wrapper.</p>
+         * @see
+         * <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition">
+         *     mdn Web Docs for the HTTP Content-Disposition Header</a>
+         * @param stream the stream to get the {@link MediaFormat} for
+         * @param streamsWrapper the wrapper to store the {@link MediaFormat} in
+         * @param response the response to get the Content-Disposition header from
+         * @return {@code true} if the {@link MediaFormat} could be retrieved from the response;
+         * otherwise {@code false}
+         * @param <X>
+         */
+        public static <X extends Stream> boolean retrieveMediaFormatFromContentDispositionHeader(
+                @NonNull final X stream,
+                @NonNull final StreamInfoWrapper<X> streamsWrapper,
+                @NonNull final Response response) {
+            // parse the Content-Disposition header,
+            // see
+            // there can be two filename directives
+            String contentDisposition = response.getHeader("Content-Disposition");
+            if (contentDisposition == null) {
+                return false;
+            }
+            try {
+                contentDisposition = Utils.decodeUrlUtf8(contentDisposition);
+                final String[] parts = contentDisposition.split(";");
+                for (String part : parts) {
+                    final String fileName;
+                    part = part.trim();
+
+                    // extract the filename
+                    if (part.startsWith("filename=")) {
+                        // remove directive and decode
+                        fileName = Utils.decodeUrlUtf8(part.substring(9));
+                    } else if (part.startsWith("filename*=")) {
+                        fileName = Utils.decodeUrlUtf8(part.substring(10));
+                    } else {
+                        continue;
+                    }
+
+                    // extract the file extension / suffix
+                    final String[] p = fileName.split("\\.");
+                    String suffix = p[p.length - 1];
+                    if (suffix.endsWith("\"") || suffix.endsWith("'")) {
+                        // remove trailing quotes if present, end index is exclusive
+                        suffix = suffix.substring(0, suffix.length() - 1);
+                    }
+
+                    // get the corresponding media format
+                    final MediaFormat format = MediaFormat.getFromSuffix(suffix);
+                    if (format != null) {
+                        streamsWrapper.setFormat(stream, format);
+                        return true;
+                    }
+                }
+            } catch (final Exception ignored) {
+                // fail silently
+            }
+            return false;
+        }
+
+        private static <X extends Stream> boolean retrieveMediaFormatFromContentTypeHeader(
+                @NonNull final X stream,
+                @NonNull final StreamInfoWrapper<X> streamsWrapper,
+                @NonNull final Response response) {
+            // try to get the format by content type
+            // some mime types are not unique for every format, those are omitted
+            final List<MediaFormat> formats = MediaFormat.getAllFromMimeType(
+                    response.getHeader("Content-Type"));
+            final List<MediaFormat> uniqueFormats = new ArrayList<>(formats.size());
+            for (int i = 0; i < formats.size(); i++) {
+                final MediaFormat format = formats.get(i);
+                if (uniqueFormats.stream().filter(f -> f.id == format.id).count() == 0) {
+                    uniqueFormats.add(format);
+                }
+            }
+            if (uniqueFormats.size() == 1) {
+                streamsWrapper.setFormat(stream, uniqueFormats.get(0));
+                return true;
+            }
+            return false;
+        }
+
+        public void resetInfo() {
             Arrays.fill(streamSizes, SIZE_UNSET);
+            for (int i = 0; i < streamsList.size(); i++) {
+                streamFormats[i] = streamsList.get(i).getFormat();
+            }
         }
 
         public static <X extends Stream> StreamInfoWrapper<X> empty() {
@@ -306,5 +451,13 @@ public class StreamItemAdapter<T extends Stream, U extends Stream> extends BaseA
         public void setSize(final T stream, final long sizeInBytes) {
             streamSizes[streamsList.indexOf(stream)] = sizeInBytes;
         }
+
+        public MediaFormat getFormat(final int streamIndex) {
+            return streamFormats[streamIndex];
+        }
+
+        public void setFormat(final T stream, final MediaFormat format) {
+            streamFormats[streamsList.indexOf(stream)] = format;
+        }
     }
 }