diff --git a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java index 4525c5988..8f4b569cd 100644 --- a/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java +++ b/app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java @@ -15,12 +15,13 @@ import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.support.annotation.StringRes; import android.support.v4.app.DialogFragment; +import android.support.v4.provider.DocumentFile; import android.support.v7.app.AlertDialog; +import android.support.v7.view.menu.ActionMenuItemView; import android.support.v7.widget.Toolbar; import android.util.Log; import android.util.SparseArray; import android.view.LayoutInflater; -import android.view.MenuItem; import android.view.View; import android.view.ViewGroup; import android.widget.AdapterView; @@ -177,9 +178,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck return; } - final Context context = getContext(); - if (context == null) - throw new RuntimeException("Context was null"); + context = getContext(); setStyle(STYLE_NO_TITLE, ThemeHelper.getDialogTheme(context)); Icepick.restoreInstanceState(this, savedInstanceState); @@ -321,11 +320,15 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck showFailedDialog(R.string.general_error); return; } - try { - continueSelectedDownload(new StoredFileHelper(getContext(), data.getData(), "")); - } catch (IOException e) { - showErrorActivity(e); + + DocumentFile docFile = DocumentFile.fromSingleUri(context, data.getData()); + if (docFile == null) { + showFailedDialog(R.string.general_error); + return; } + + // check if the selected file was previously used + checkSelectedDownload(null, data.getData(), docFile.getName(), docFile.getType()); } } @@ -337,14 +340,14 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck if (DEBUG) Log.d(TAG, "initToolbar() called with: toolbar = [" + toolbar + "]"); boolean isLight = ThemeHelper.isLightThemeSelected(getActivity()); - okButton = toolbar.findViewById(R.id.okay); - okButton.setEnabled(false);// disable until the download service connection is done toolbar.setTitle(R.string.download_dialog_title); toolbar.setNavigationIcon(isLight ? R.drawable.ic_arrow_back_black_24dp : R.drawable.ic_arrow_back_white_24dp); toolbar.inflateMenu(R.menu.dialog_url); toolbar.setNavigationOnClickListener(v -> getDialog().dismiss()); + okButton = toolbar.findViewById(R.id.okay); + okButton.setEnabled(false);// disable until the download service connection is done toolbar.setOnMenuItemClickListener(item -> { if (item.getItemId() == R.id.okay) { @@ -504,15 +507,17 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck StoredDirectoryHelper mainStorageAudio = null; StoredDirectoryHelper mainStorageVideo = null; DownloadManager downloadManager = null; - - MenuItem okButton = null; + ActionMenuItemView okButton = null; + Context context; private String getNameEditText() { - return nameEditText.getText().toString().trim(); + String str = nameEditText.getText().toString().trim(); + + return FilenameUtils.createFilename(context, str.isEmpty() ? currentInfo.getName() : str); } private void showFailedDialog(@StringRes int msg) { - new AlertDialog.Builder(getContext()) + new AlertDialog.Builder(context) .setMessage(msg) .setNegativeButton(android.R.string.ok, null) .create() @@ -521,7 +526,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck private void showErrorActivity(Exception e) { ErrorActivity.reportError( - getContext(), + context, Collections.singletonList(e), null, null, @@ -530,18 +535,14 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck } private void prepareSelectedDownload() { - final Context context = getContext(); StoredDirectoryHelper mainStorage; MediaFormat format; String mime; // first, build the filename and get the output folder (if possible) + // later, run a very very very large file checking logic - String filename = getNameEditText() + "."; - if (filename.isEmpty()) { - filename = FilenameUtils.createFilename(context, currentInfo.getName()); - } - filename += "."; + String filename = getNameEditText().concat("."); switch (radioStreamsGroup.getCheckedRadioButtonId()) { case R.id.audio_button: @@ -567,34 +568,33 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck } if (mainStorage == null) { - // this part is called if... - // older android version running with SAF preferred - // save path not defined (via download settings) + // This part is called if with SAF preferred: + // * older android version running + // * save path not defined (via download settings) StoredFileHelper.requestSafWithFileCreation(this, REQUEST_DOWNLOAD_PATH_SAF, filename, mime); return; } // check for existing file with the same name - Uri result = mainStorage.findFile(filename); + checkSelectedDownload(mainStorage, mainStorage.findFile(filename), filename, mime); + } - if (result == null) { - // the file does not exists, create - StoredFileHelper storage = mainStorage.createFile(filename, mime); - if (storage == null || !storage.canWrite()) { - showFailedDialog(R.string.error_file_creation); - return; - } - - continueSelectedDownload(storage); - return; - } - - // the target filename is already use, try load + private void checkSelectedDownload(StoredDirectoryHelper mainStorage, Uri targetFile, String filename, String mime) { StoredFileHelper storage; + try { - storage = new StoredFileHelper(context, result, mime); - } catch (IOException e) { + if (mainStorage == null) { + // using SAF on older android version + storage = new StoredFileHelper(context, null, targetFile, ""); + } else if (targetFile == null) { + // the file does not exist, but it is probably used in a pending download + storage = new StoredFileHelper(mainStorage.getUri(), filename, mime, mainStorage.getTag()); + } else { + // the target filename is already use, attempt to use it + storage = new StoredFileHelper(context, mainStorage.getUri(), targetFile, mainStorage.getTag()); + } + } catch (Exception e) { showErrorActivity(e); return; } @@ -618,6 +618,25 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck msgBody = R.string.download_already_running; break; case None: + if (mainStorage == null) { + // This part is called if: + // * using SAF on older android version + // * save path not defined + continueSelectedDownload(storage); + return; + } else if (targetFile == null) { + // This part is called if: + // * the filename is not used in a pending/finished download + // * the file does not exists, create + storage = mainStorage.createFile(filename, mime); + if (storage == null || !storage.canWrite()) { + showFailedDialog(R.string.error_file_creation); + return; + } + + continueSelectedDownload(storage); + return; + } msgBtn = R.string.overwrite; msgBody = R.string.overwrite_unrelated_warning; break; @@ -625,49 +644,73 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck return; } - // handle user answer (overwrite or create another file with different name) - final String finalFilename = filename; - AlertDialog.Builder builder = new AlertDialog.Builder(context); - builder.setTitle(R.string.download_dialog_title) - .setMessage(msgBody) - .setPositiveButton(msgBtn, (dialog, which) -> { - dialog.dismiss(); - StoredFileHelper storageNew; - switch (state) { - case Finished: - case Pending: - downloadManager.forgetMission(storage); - case None: - // try take (or steal) the file permissions - try { - storageNew = new StoredFileHelper(context, result, mainStorage.getTag()); - if (storageNew.canWrite()) - continueSelectedDownload(storageNew); - else - showFailedDialog(R.string.error_file_creation); - } catch (IOException e) { - showErrorActivity(e); - } - break; - case PendingRunning: - // FIXME: createUniqueFile() is not tested properly - storageNew = mainStorage.createUniqueFile(finalFilename, mime); - if (storageNew == null) - showFailedDialog(R.string.error_file_creation); - else - continueSelectedDownload(storageNew); - break; + AlertDialog.Builder askDialog = new AlertDialog.Builder(context) + .setTitle(R.string.download_dialog_title) + .setMessage(msgBody) + .setNegativeButton(android.R.string.cancel, null); + final StoredFileHelper finalStorage = storage; + + + if (mainStorage == null) { + // This part is called if: + // * using SAF on older android version + // * save path not defined + switch (state) { + case Pending: + case Finished: + askDialog.setPositiveButton(msgBtn, (dialog, which) -> { + dialog.dismiss(); + downloadManager.forgetMission(finalStorage); + continueSelectedDownload(finalStorage); + }); + break; + } + + askDialog.create().show(); + return; + } + + askDialog.setPositiveButton(msgBtn, (dialog, which) -> { + dialog.dismiss(); + + StoredFileHelper storageNew; + switch (state) { + case Finished: + case Pending: + downloadManager.forgetMission(finalStorage); + case None: + if (targetFile == null) { + storageNew = mainStorage.createFile(filename, mime); + } else { + try { + // try take (or steal) the file + storageNew = new StoredFileHelper(context, mainStorage.getUri(), targetFile, mainStorage.getTag()); + } catch (IOException e) { + Log.e(TAG, "Failed to take (or steal) the file in " + targetFile.toString()); + storageNew = null; + } } - }) - .setNegativeButton(android.R.string.cancel, null) - .create() - .show(); + + if (storageNew != null && storageNew.canWrite()) + continueSelectedDownload(storageNew); + else + showFailedDialog(R.string.error_file_creation); + break; + case PendingRunning: + storageNew = mainStorage.createUniqueFile(filename, mime); + if (storageNew == null) + showFailedDialog(R.string.error_file_creation); + else + continueSelectedDownload(storageNew); + break; + } + }); + + askDialog.create().show(); } private void continueSelectedDownload(@NonNull StoredFileHelper storage) { - final Context context = getContext(); - if (!storage.canWrite()) { showFailedDialog(R.string.permission_denied); return; @@ -678,7 +721,6 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck if (storage.length() > 0) storage.truncate(); } catch (IOException e) { Log.e(TAG, "failed to overwrite the file: " + storage.getUri().toString(), e); - //showErrorActivity(e); showFailedDialog(R.string.overwrite_failed); return; } @@ -748,7 +790,7 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck if (secondaryStreamUrl == null) { urls = new String[]{selectedStream.getUrl()}; } else { - urls = new String[]{selectedStream.getUrl(), secondaryStreamUrl}; + urls = new String[]{secondaryStreamUrl, selectedStream.getUrl()}; } DownloadManagerService.startMission(context, urls, storage, kind, threads, currentInfo.getUrl(), psName, psArgs, nearLength); diff --git a/app/src/main/java/org/schabi/newpipe/settings/DownloadSettingsFragment.java b/app/src/main/java/org/schabi/newpipe/settings/DownloadSettingsFragment.java index 3737d1c17..5d4ccf3f8 100644 --- a/app/src/main/java/org/schabi/newpipe/settings/DownloadSettingsFragment.java +++ b/app/src/main/java/org/schabi/newpipe/settings/DownloadSettingsFragment.java @@ -14,18 +14,23 @@ import android.support.v7.preference.Preference; import android.util.Log; import android.widget.Toast; +import com.nononsenseapps.filepicker.Utils; + import org.schabi.newpipe.R; import org.schabi.newpipe.util.FilePickerActivityHelper; import java.io.File; import java.io.IOException; -import java.net.URI; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import us.shandian.giga.io.StoredDirectoryHelper; public class DownloadSettingsFragment extends BasePreferenceFragment { private static final int REQUEST_DOWNLOAD_VIDEO_PATH = 0x1235; private static final int REQUEST_DOWNLOAD_AUDIO_PATH = 0x1236; + public static final boolean IGNORE_RELEASE_OLD_PATH = true; private String DOWNLOAD_PATH_VIDEO_PREFERENCE; private String DOWNLOAD_PATH_AUDIO_PREFERENCE; @@ -35,41 +40,46 @@ public class DownloadSettingsFragment extends BasePreferenceFragment { private Preference prefPathVideo; private Preference prefPathAudio; - + private Context ctx; + private boolean lastAPIJavaIO; + @Override public void onCreate(@Nullable Bundle savedInstanceState) { super.onCreate(savedInstanceState); - initKeys(); - updatePreferencesSummary(); - } - - @Override - public void onCreatePreferences(Bundle savedInstanceState, String rootKey) { - addPreferencesFromResource(R.xml.download_settings); + DOWNLOAD_PATH_VIDEO_PREFERENCE = getString(R.string.download_path_video_key); + DOWNLOAD_PATH_AUDIO_PREFERENCE = getString(R.string.download_path_audio_key); + DOWNLOAD_STORAGE_API = getString(R.string.downloads_storage_api); + DOWNLOAD_STORAGE_API_DEFAULT = getString(R.string.downloads_storage_api_default); prefPathVideo = findPreference(DOWNLOAD_PATH_VIDEO_PREFERENCE); prefPathAudio = findPreference(DOWNLOAD_PATH_AUDIO_PREFERENCE); - updatePathPickers(usingJavaIO()); + lastAPIJavaIO = usingJavaIO(); + + updatePreferencesSummary(); + updatePathPickers(lastAPIJavaIO); findPreference(DOWNLOAD_STORAGE_API).setOnPreferenceChangeListener((preference, value) -> { boolean javaIO = DOWNLOAD_STORAGE_API_DEFAULT.equals(value); - if (!javaIO && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + if (javaIO == lastAPIJavaIO) return true; + lastAPIJavaIO = javaIO; + + boolean res; + + if (javaIO && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + // forget save paths (if necessary) + res = forgetPath(DOWNLOAD_PATH_VIDEO_PREFERENCE); + res |= forgetPath(DOWNLOAD_PATH_AUDIO_PREFERENCE); + } else { + res = hasInvalidPath(DOWNLOAD_PATH_VIDEO_PREFERENCE) || hasInvalidPath(DOWNLOAD_PATH_AUDIO_PREFERENCE); + } + + if (res) { Toast.makeText(ctx, R.string.download_pick_path, Toast.LENGTH_LONG).show(); - - // forget save paths - forgetSAFTree(DOWNLOAD_PATH_VIDEO_PREFERENCE); - forgetSAFTree(DOWNLOAD_PATH_AUDIO_PREFERENCE); - - defaultPreferences.edit() - .putString(DOWNLOAD_PATH_VIDEO_PREFERENCE, "") - .putString(DOWNLOAD_PATH_AUDIO_PREFERENCE, "") - .apply(); - updatePreferencesSummary(); } @@ -78,6 +88,30 @@ public class DownloadSettingsFragment extends BasePreferenceFragment { }); } + @RequiresApi(api = Build.VERSION_CODES.LOLLIPOP) + private boolean forgetPath(String prefKey) { + String path = defaultPreferences.getString(prefKey, ""); + if (path == null || path.isEmpty()) return true; + + if (path.startsWith("file://")) return false; + + // forget SAF path (file:// is compatible with the SAF wrapper) + forgetSAFTree(getContext(), prefKey); + defaultPreferences.edit().putString(prefKey, "").apply(); + + return true; + } + + private boolean hasInvalidPath(String prefKey) { + String value = defaultPreferences.getString(prefKey, null); + return value == null || value.isEmpty(); + } + + @Override + public void onCreatePreferences(Bundle savedInstanceState, String rootKey) { + addPreferencesFromResource(R.xml.download_settings); + } + @Override public void onAttach(Context context) { super.onAttach(context); @@ -91,20 +125,25 @@ public class DownloadSettingsFragment extends BasePreferenceFragment { findPreference(DOWNLOAD_STORAGE_API).setOnPreferenceChangeListener(null); } - private void initKeys() { - DOWNLOAD_PATH_VIDEO_PREFERENCE = getString(R.string.download_path_video_key); - DOWNLOAD_PATH_AUDIO_PREFERENCE = getString(R.string.download_path_audio_key); - DOWNLOAD_STORAGE_API = getString(R.string.downloads_storage_api); - DOWNLOAD_STORAGE_API_DEFAULT = getString(R.string.downloads_storage_api_default); + private void updatePreferencesSummary() { + showPathInSummary(DOWNLOAD_PATH_VIDEO_PREFERENCE, R.string.download_path_summary, prefPathVideo); + showPathInSummary(DOWNLOAD_PATH_AUDIO_PREFERENCE, R.string.download_path_audio_summary, prefPathAudio); } - private void updatePreferencesSummary() { - prefPathVideo.setSummary( - defaultPreferences.getString(DOWNLOAD_PATH_VIDEO_PREFERENCE, getString(R.string.download_path_summary)) - ); - prefPathAudio.setSummary( - defaultPreferences.getString(DOWNLOAD_PATH_AUDIO_PREFERENCE, getString(R.string.download_path_audio_summary)) - ); + private void showPathInSummary(String prefKey, @StringRes int defaultString, Preference target) { + String rawUri = defaultPreferences.getString(prefKey, null); + if (rawUri == null || rawUri.isEmpty()) { + target.setSummary(getString(defaultString)); + return; + } + + try { + rawUri = URLDecoder.decode(rawUri, StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + // nothing to do + } + + target.setSummary(rawUri); } private void updatePathPickers(boolean useJavaIO) { @@ -119,20 +158,25 @@ public class DownloadSettingsFragment extends BasePreferenceFragment { ); } + // FIXME: after releasing the old path, all downloads created on the folder becomes inaccessible @RequiresApi(Build.VERSION_CODES.LOLLIPOP) - private void forgetSAFTree(String prefKey) { + private void forgetSAFTree(Context ctx, String prefKey) { + if (IGNORE_RELEASE_OLD_PATH) { + return; + } String oldPath = defaultPreferences.getString(prefKey, ""); - if (oldPath != null && !oldPath.isEmpty() && oldPath.charAt(0) != File.separatorChar) { + if (oldPath != null && !oldPath.isEmpty() && oldPath.charAt(0) != File.separatorChar && !oldPath.startsWith("file://")) { try { - StoredDirectoryHelper mainStorage = new StoredDirectoryHelper(ctx, Uri.parse(oldPath), null); - if (!mainStorage.isDirect()) { - mainStorage.revokePermissions(); - Log.i(TAG, "revokePermissions() [uri=" + oldPath + "] ¡success!"); - } - } catch (IOException err) { - Log.e(TAG, "Error revoking Tree uri permissions [uri=" + oldPath + "]", err); + Uri uri = Uri.parse(oldPath); + + ctx.getContentResolver().releasePersistableUriPermission(uri, StoredDirectoryHelper.PERMISSION_FLAGS); + ctx.revokeUriPermission(uri, StoredDirectoryHelper.PERMISSION_FLAGS); + + Log.i(TAG, "Revoke old path permissions success on " + oldPath); + } catch (Exception err) { + Log.e(TAG, "Error revoking old path permissions on " + oldPath, err); } } } @@ -167,7 +211,7 @@ public class DownloadSettingsFragment extends BasePreferenceFragment { if (safPick && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { i = new Intent(Intent.ACTION_OPEN_DOCUMENT_TREE) .putExtra("android.content.extra.SHOW_ADVANCED", true) - .addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION | StoredDirectoryHelper.PERMISSION_FLAGS); + .addFlags(Intent.FLAG_GRANT_PERSISTABLE_URI_PERMISSION | StoredDirectoryHelper.PERMISSION_FLAGS); } else { i = new Intent(getActivity(), FilePickerActivityHelper.class) .putExtra(FilePickerActivityHelper.EXTRA_ALLOW_MULTIPLE, false) @@ -208,27 +252,37 @@ public class DownloadSettingsFragment extends BasePreferenceFragment { if (!usingJavaIO() && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { // steps: - // 1. acquire permissions on the new save path - // 2. save the new path, if step(1) was successful + // 1. revoke permissions on the old save path + // 2. acquire permissions on the new save path + // 3. save the new path, if step(2) was successful + final Context ctx = getContext(); + if (ctx == null) throw new NullPointerException("getContext()"); + + forgetSAFTree(ctx, key); try { + ctx.grantUriPermission(ctx.getPackageName(), uri, StoredDirectoryHelper.PERMISSION_FLAGS); + StoredDirectoryHelper mainStorage = new StoredDirectoryHelper(ctx, uri, null); - mainStorage.acquirePermissions(); - Log.i(TAG, "acquirePermissions() [uri=" + uri.toString() + "] ¡success!"); + Log.i(TAG, "Acquiring tree success from " + uri.toString()); + + if (!mainStorage.canWrite()) + throw new IOException("No write permissions on " + uri.toString()); } catch (IOException err) { - Log.e(TAG, "Error acquiring permissions on " + uri.toString()); + Log.e(TAG, "Error acquiring tree from " + uri.toString(), err); showMessageDialog(R.string.general_error, R.string.no_available_dir); return; } - - defaultPreferences.edit().putString(key, uri.toString()).apply(); } else { - defaultPreferences.edit().putString(key, uri.toString()).apply(); - updatePreferencesSummary(); - - File target = new File(URI.create(uri.toString())); - if (!target.canWrite()) + File target = Utils.getFileForUri(data.getData()); + if (!target.canWrite()) { showMessageDialog(R.string.download_to_sdcard_error_title, R.string.download_to_sdcard_error_message); + return; + } + uri = Uri.fromFile(target); } + + defaultPreferences.edit().putString(key, uri.toString()).apply(); + updatePreferencesSummary(); } } diff --git a/app/src/main/java/org/schabi/newpipe/streams/DataReader.java b/app/src/main/java/org/schabi/newpipe/streams/DataReader.java index 567fa5229..0e62810c5 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/DataReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/DataReader.java @@ -16,6 +16,8 @@ public class DataReader { public final static int INTEGER_SIZE = 4; public final static int FLOAT_SIZE = 4; + private final static int BUFFER_SIZE = 128 * 1024;// 128 KiB + private long position = 0; private final SharpStream stream; @@ -229,7 +231,7 @@ public class DataReader { } } - private final byte[] readBuffer = new byte[8 * 1024]; + private final byte[] readBuffer = new byte[BUFFER_SIZE]; private int readOffset; private int readCount; diff --git a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java index 61f793e5d..03aab447c 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/Mp4FromDashWriter.java @@ -12,7 +12,6 @@ import java.io.IOException; import java.nio.ByteBuffer; /** - * * @author kapodamy */ public class Mp4FromDashWriter { @@ -262,12 +261,12 @@ public class Mp4FromDashWriter { final int ftyp_size = make_ftyp(); // reserve moov space in the output stream - if (outStream.canSetLength()) { + /*if (outStream.canSetLength()) { long length = writeOffset + auxSize; outStream.setLength(length); outSeek(length); - } else { - // hard way + } else {*/ + if (auxSize > 0) { int length = auxSize; byte[] buffer = new byte[8 * 1024];// 8 KiB while (length > 0) { @@ -276,6 +275,7 @@ public class Mp4FromDashWriter { length -= count; } } + if (auxBuffer == null) { outSeek(ftyp_size); } diff --git a/app/src/main/java/org/schabi/newpipe/util/FilenameUtils.java b/app/src/main/java/org/schabi/newpipe/util/FilenameUtils.java index b874a9eca..37d94cd16 100644 --- a/app/src/main/java/org/schabi/newpipe/util/FilenameUtils.java +++ b/app/src/main/java/org/schabi/newpipe/util/FilenameUtils.java @@ -10,6 +10,9 @@ import java.util.regex.Pattern; public class FilenameUtils { + private static final String CHARSET_MOST_SPECIAL = "[\\n\\r|?*<\":\\\\>/']+"; + private static final String CHARSET_ONLY_LETTERS_AND_DIGITS = "[^\\w\\d]+"; + /** * #143 #44 #42 #22: make sure that the filename does not contain illegal chars. * @param context the context to retrieve strings and preferences from @@ -18,11 +21,28 @@ public class FilenameUtils { */ public static String createFilename(Context context, String title) { SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); - final String key = context.getString(R.string.settings_file_charset_key); - final String value = sharedPreferences.getString(key, context.getString(R.string.default_file_charset_value)); - Pattern pattern = Pattern.compile(value); + + final String charset_ld = context.getString(R.string.charset_letters_and_digits_value); + final String charset_ms = context.getString(R.string.charset_most_special_value); + final String defaultCharset = context.getString(R.string.default_file_charset_value); final String replacementChar = sharedPreferences.getString(context.getString(R.string.settings_file_replacement_character_key), "_"); + String selectedCharset = sharedPreferences.getString(context.getString(R.string.settings_file_charset_key), null); + + final String charset; + + if (selectedCharset == null || selectedCharset.isEmpty()) selectedCharset = defaultCharset; + + if (selectedCharset.equals(charset_ld)) { + charset = CHARSET_ONLY_LETTERS_AND_DIGITS; + } else if (selectedCharset.equals(charset_ms)) { + charset = CHARSET_MOST_SPECIAL; + } else { + charset = selectedCharset;// ¿is the user using a custom charset? + } + + Pattern pattern = Pattern.compile(charset); + return createFilename(title, pattern, replacementChar); } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java index 1e05983d8..f6b6b459a 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java @@ -28,7 +28,7 @@ public class DownloadInitializer extends Thread { @Override public void run() { - if (mMission.current > 0) mMission.resetState(); + if (mMission.current > 0) mMission.resetState(false,true, DownloadMission.ERROR_NOTHING); int retryCount = 0; while (true) { diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 9ec3418b0..838acc162 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -2,7 +2,6 @@ package us.shandian.giga.get; import android.os.Handler; import android.os.Message; -import android.support.annotation.NonNull; import android.util.Log; import java.io.File; @@ -86,7 +85,7 @@ public class DownloadMission extends Mission { /** * the post-processing algorithm instance */ - public transient Postprocessing psAlgorithm; + public Postprocessing psAlgorithm; /** * The current resource to download, see {@code urls[current]} and {@code offsets[current]} @@ -483,7 +482,7 @@ public class DownloadMission extends Mission { if (init != null && Thread.currentThread() != init && init.isAlive()) { init.interrupt(); synchronized (blockState) { - resetState(); + resetState(false, true, ERROR_NOTHING); } return; } @@ -525,10 +524,18 @@ public class DownloadMission extends Mission { return res; } - void resetState() { + + /** + * Resets the mission state + * + * @param rollback {@code true} true to forget all progress, otherwise, {@code false} + * @param persistChanges {@code true} to commit changes to the metadata file, otherwise, {@code false} + */ + public void resetState(boolean rollback, boolean persistChanges, int errorCode) { done = 0; blocks = -1; - errCode = ERROR_NOTHING; + errCode = errorCode; + errObject = null; fallback = false; unknownLength = false; finishCount = 0; @@ -537,7 +544,10 @@ public class DownloadMission extends Mission { blockState.clear(); threads = new Thread[0]; - Utility.writeToFile(metadata, DownloadMission.this); + if (rollback) current = 0; + + if (persistChanges) + Utility.writeToFile(metadata, DownloadMission.this); } private void initializer() { @@ -633,33 +643,22 @@ public class DownloadMission extends Mission { threads[0].interrupt(); } - /** - * changes the StoredFileHelper for another and saves the changes to the metadata file - * - * @param newStorage the new StoredFileHelper instance to use - */ - public void changeStorage(@NonNull StoredFileHelper newStorage) { - storage = newStorage; - // commit changes on the metadata file - runAsync(-2, this::writeThisToFile); - } - /** * Indicates whatever the backed storage is invalid * * @return {@code true}, if storage is invalid and cannot be used */ public boolean hasInvalidStorage() { - return errCode == ERROR_PROGRESS_LOST || storage == null || storage.isInvalid(); + return errCode == ERROR_PROGRESS_LOST || storage == null || storage.isInvalid() || !storage.existsAsFile(); } /** * Indicates whatever is possible to start the mission * - * @return {@code true} is this mission is "sane", otherwise, {@code false} + * @return {@code true} is this mission its "healthy", otherwise, {@code false} */ - public boolean canDownload() { - return !(isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) && !isFinished() && !hasInvalidStorage(); + public boolean isCorrupt() { + return (isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) || isFinished() || hasInvalidStorage(); } private boolean doPostprocessing() { diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java index ced579b20..7a68cd778 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java @@ -137,6 +137,10 @@ public class DownloadRunnable extends Thread { mMission.setThreadBytePosition(mId, total);// download paused, save progress for this block } catch (Exception e) { + if (DEBUG) { + Log.d(TAG, mId + ": position=" + blockPosition + " total=" + total + " stopped due exception", e); + } + mMission.setThreadBytePosition(mId, total); if (!mMission.running || e instanceof ClosedByInterruptException) break; @@ -146,10 +150,6 @@ public class DownloadRunnable extends Thread { break; } - if (DEBUG) { - Log.d(TAG, mId + ":position " + blockPosition + " retrying due exception", e); - } - retry = true; } } diff --git a/app/src/main/java/us/shandian/giga/get/FinishedMission.java b/app/src/main/java/us/shandian/giga/get/FinishedMission.java index 5540b44a1..2a01896fe 100644 --- a/app/src/main/java/us/shandian/giga/get/FinishedMission.java +++ b/app/src/main/java/us/shandian/giga/get/FinishedMission.java @@ -12,5 +12,7 @@ public class FinishedMission extends Mission { length = mission.length;// ¿or mission.done? timestamp = mission.timestamp; kind = mission.kind; + storage = mission.storage; + } } diff --git a/app/src/main/java/us/shandian/giga/get/Mission.java b/app/src/main/java/us/shandian/giga/get/Mission.java index ce201d960..a9ed08fc2 100644 --- a/app/src/main/java/us/shandian/giga/get/Mission.java +++ b/app/src/main/java/us/shandian/giga/get/Mission.java @@ -1,6 +1,5 @@ package us.shandian.giga.get; -import android.net.Uri; import android.support.annotation.NonNull; import java.io.Serializable; @@ -36,15 +35,6 @@ public abstract class Mission implements Serializable { */ public StoredFileHelper storage; - /** - * get the target file on the storage - * - * @return File object - */ - public Uri getDownloadedFileUri() { - return storage.getUri(); - } - /** * Delete the downloaded file * @@ -52,7 +42,7 @@ public abstract class Mission implements Serializable { */ public boolean delete() { if (storage != null) return storage.delete(); - return true; + return true; } /** @@ -65,6 +55,6 @@ public abstract class Mission implements Serializable { public String toString() { Calendar calendar = Calendar.getInstance(); calendar.setTimeInMillis(timestamp); - return "[" + calendar.getTime().toString() + "] " + getDownloadedFileUri().getPath(); + return "[" + calendar.getTime().toString() + "] " + (storage.isInvalid() ? storage.getName() : storage.getUri()); } } diff --git a/app/src/main/java/us/shandian/giga/get/sqlite/FinishedMissionStore.java b/app/src/main/java/us/shandian/giga/get/sqlite/FinishedMissionStore.java index 6d63b9ff7..4650f75d0 100644 --- a/app/src/main/java/us/shandian/giga/get/sqlite/FinishedMissionStore.java +++ b/app/src/main/java/us/shandian/giga/get/sqlite/FinishedMissionStore.java @@ -35,7 +35,7 @@ public class FinishedMissionStore extends SQLiteOpenHelper { /** * The table name of download missions */ - private static final String FINISHED_MISSIONS_TABLE_NAME = "finished_missions"; + private static final String FINISHED_TABLE_NAME = "finished_missions"; /** * The key to the urls of a mission @@ -58,7 +58,7 @@ public class FinishedMissionStore extends SQLiteOpenHelper { * The statement to create the table */ private static final String MISSIONS_CREATE_TABLE = - "CREATE TABLE " + FINISHED_MISSIONS_TABLE_NAME + " (" + + "CREATE TABLE " + FINISHED_TABLE_NAME + " (" + KEY_PATH + " TEXT NOT NULL, " + KEY_SOURCE + " TEXT NOT NULL, " + KEY_DONE + " INTEGER NOT NULL, " + @@ -111,7 +111,7 @@ public class FinishedMissionStore extends SQLiteOpenHelper { ) ).toString()); - db.insert(FINISHED_MISSIONS_TABLE_NAME, null, values); + db.insert(FINISHED_TABLE_NAME, null, values); } db.setTransactionSuccessful(); db.endTransaction(); @@ -154,10 +154,10 @@ public class FinishedMissionStore extends SQLiteOpenHelper { mission.kind = kind.charAt(0); try { - mission.storage = new StoredFileHelper(context, Uri.parse(path), ""); + mission.storage = new StoredFileHelper(context,null, Uri.parse(path), ""); } catch (Exception e) { Log.e("FinishedMissionStore", "failed to load the storage path of: " + path, e); - mission.storage = new StoredFileHelper(path, "", ""); + mission.storage = new StoredFileHelper(null, path, "", ""); } return mission; @@ -170,7 +170,7 @@ public class FinishedMissionStore extends SQLiteOpenHelper { public ArrayList loadFinishedMissions() { SQLiteDatabase database = getReadableDatabase(); - Cursor cursor = database.query(FINISHED_MISSIONS_TABLE_NAME, null, null, + Cursor cursor = database.query(FINISHED_TABLE_NAME, null, null, null, null, null, KEY_TIMESTAMP + " DESC"); int count = cursor.getCount(); @@ -188,33 +188,47 @@ public class FinishedMissionStore extends SQLiteOpenHelper { if (downloadMission == null) throw new NullPointerException("downloadMission is null"); SQLiteDatabase database = getWritableDatabase(); ContentValues values = getValuesOfMission(downloadMission); - database.insert(FINISHED_MISSIONS_TABLE_NAME, null, values); + database.insert(FINISHED_TABLE_NAME, null, values); } public void deleteMission(Mission mission) { if (mission == null) throw new NullPointerException("mission is null"); - String path = mission.getDownloadedFileUri().toString(); + String ts = String.valueOf(mission.timestamp); SQLiteDatabase database = getWritableDatabase(); - if (mission instanceof FinishedMission) - database.delete(FINISHED_MISSIONS_TABLE_NAME, KEY_TIMESTAMP + " = ?, " + KEY_PATH + " = ?", new String[]{path}); - else + if (mission instanceof FinishedMission) { + if (mission.storage.isInvalid()) { + database.delete(FINISHED_TABLE_NAME, KEY_TIMESTAMP + " = ?", new String[]{ts}); + } else { + database.delete(FINISHED_TABLE_NAME, KEY_TIMESTAMP + " = ? AND " + KEY_PATH + " = ?", new String[]{ + ts, mission.storage.getUri().toString() + }); + } + } else { throw new UnsupportedOperationException("DownloadMission"); + } } public void updateMission(Mission mission) { if (mission == null) throw new NullPointerException("mission is null"); SQLiteDatabase database = getWritableDatabase(); ContentValues values = getValuesOfMission(mission); - String path = mission.getDownloadedFileUri().toString(); + String ts = String.valueOf(mission.timestamp); int rowsAffected; - if (mission instanceof FinishedMission) - rowsAffected = database.update(FINISHED_MISSIONS_TABLE_NAME, values, KEY_PATH + " = ?", new String[]{path}); - else + if (mission instanceof FinishedMission) { + if (mission.storage.isInvalid()) { + rowsAffected = database.update(FINISHED_TABLE_NAME, values, KEY_TIMESTAMP + " = ?", new String[]{ts}); + } else { + rowsAffected = database.update(FINISHED_TABLE_NAME, values, KEY_PATH + " = ?", new String[]{ + mission.storage.getUri().toString() + }); + } + } else { throw new UnsupportedOperationException("DownloadMission"); + } if (rowsAffected != 1) { Log.e("FinishedMissionStore", "Expected 1 row to be affected by update but got " + rowsAffected); diff --git a/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java b/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java index 650725a76..327b9149e 100644 --- a/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java +++ b/app/src/main/java/us/shandian/giga/io/CircularFileWriter.java @@ -12,7 +12,7 @@ public class CircularFileWriter extends SharpStream { private final static int QUEUE_BUFFER_SIZE = 8 * 1024;// 8 KiB private final static int NOTIFY_BYTES_INTERVAL = 64 * 1024;// 64 KiB - private final static int THRESHOLD_AUX_LENGTH = 3 * 1024 * 1024;// 3 MiB + private final static int THRESHOLD_AUX_LENGTH = 15 * 1024 * 1024;// 15 MiB private OffsetChecker callback; @@ -44,41 +44,84 @@ public class CircularFileWriter extends SharpStream { reportPosition = NOTIFY_BYTES_INTERVAL; } - private void flushAuxiliar() throws IOException { + private void flushAuxiliar(long amount) throws IOException { if (aux.length < 1) { return; } - boolean underflow = out.getOffset() >= out.length; - out.flush(); aux.flush(); + boolean underflow = aux.offset < aux.length || out.offset < out.length; + aux.target.seek(0); out.target.seek(out.length); - long length = aux.length; - out.length += aux.length; - + long length = amount; while (length > 0) { int read = (int) Math.min(length, Integer.MAX_VALUE); read = aux.target.read(aux.queue, 0, Math.min(read, aux.queue.length)); + if (read < 1) { + amount -= length; + break; + } + out.writeProof(aux.queue, read); length -= read; } if (underflow) { - out.offset += aux.offset; - out.target.seek(out.offset); + if (out.offset >= out.length) { + // calculate the aux underflow pointer + if (aux.offset < amount) { + out.offset += aux.offset; + aux.offset = 0; + out.target.seek(out.offset); + } else { + aux.offset -= amount; + out.offset = out.length + amount; + } + } else { + aux.offset = 0; + } } else { - out.offset = out.length; + out.offset += amount; + aux.offset -= amount; } + out.length += amount; + if (out.length > maxLengthKnown) { maxLengthKnown = out.length; } + if (amount < aux.length) { + // move the excess data to the beginning of the file + long readOffset = amount; + long writeOffset = 0; + byte[] buffer = new byte[128 * 1024]; // 128 KiB + + aux.length -= amount; + length = aux.length; + while (length > 0) { + int read = (int) Math.min(length, Integer.MAX_VALUE); + read = aux.target.read(buffer, 0, Math.min(read, buffer.length)); + + aux.target.seek(writeOffset); + aux.writeProof(buffer, read); + + writeOffset += read; + readOffset += read; + length -= read; + + aux.target.seek(readOffset); + } + + aux.target.setLength(aux.length); + return; + } + if (aux.length > THRESHOLD_AUX_LENGTH) { aux.target.setLength(THRESHOLD_AUX_LENGTH);// or setLength(0); } @@ -94,7 +137,7 @@ public class CircularFileWriter extends SharpStream { * @throws IOException if an I/O error occurs */ public long finalizeFile() throws IOException { - flushAuxiliar(); + flushAuxiliar(aux.length); out.flush(); @@ -148,7 +191,7 @@ public class CircularFileWriter extends SharpStream { if (end == -1) { available = Integer.MAX_VALUE; } else if (end < offsetOut) { - throw new IOException("The reported offset is invalid: " + String.valueOf(offsetOut)); + throw new IOException("The reported offset is invalid: " + end + "<" + offsetOut); } else { available = end - offsetOut; } @@ -167,16 +210,10 @@ public class CircularFileWriter extends SharpStream { length = aux.length + len; } - if (length > available || length < THRESHOLD_AUX_LENGTH) { - aux.write(b, off, len); - } else { - if (underflow) { - aux.write(b, off, len); - flushAuxiliar(); - } else { - flushAuxiliar(); - out.write(b, off, len);// write directly on the output - } + aux.write(b, off, len); + + if (length >= THRESHOLD_AUX_LENGTH && length <= available) { + flushAuxiliar(available); } } else { if (underflow) { @@ -234,8 +271,13 @@ public class CircularFileWriter extends SharpStream { @Override public void seek(long offset) throws IOException { long total = out.length + aux.length; + if (offset == total) { - return;// nothing to do + // do not ignore the seek offset if a underflow exists + long relativeOffset = out.getOffset() + aux.getOffset(); + if (relativeOffset == total) { + return; + } } // flush everything, avoid any underflow @@ -409,6 +451,9 @@ public class CircularFileWriter extends SharpStream { } protected void seek(long absoluteOffset) throws IOException { + if (absoluteOffset == offset) { + return;// nothing to do + } offset = absoluteOffset; target.seek(absoluteOffset); } diff --git a/app/src/main/java/us/shandian/giga/io/FileStreamSAF.java b/app/src/main/java/us/shandian/giga/io/FileStreamSAF.java index cb4786280..ec6629268 100644 --- a/app/src/main/java/us/shandian/giga/io/FileStreamSAF.java +++ b/app/src/main/java/us/shandian/giga/io/FileStreamSAF.java @@ -137,4 +137,9 @@ public class FileStreamSAF extends SharpStream { public void seek(long offset) throws IOException { channel.position(offset); } + + @Override + public long length() throws IOException { + return channel.size(); + } } diff --git a/app/src/main/java/us/shandian/giga/io/StoredDirectoryHelper.java b/app/src/main/java/us/shandian/giga/io/StoredDirectoryHelper.java index f5c2fd3f5..eb3c9b817 100644 --- a/app/src/main/java/us/shandian/giga/io/StoredDirectoryHelper.java +++ b/app/src/main/java/us/shandian/giga/io/StoredDirectoryHelper.java @@ -4,8 +4,10 @@ import android.annotation.TargetApi; import android.content.ContentResolver; import android.content.Context; import android.content.Intent; +import android.database.Cursor; import android.net.Uri; import android.os.Build; +import android.provider.DocumentsContract; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.support.annotation.RequiresApi; @@ -13,8 +15,13 @@ import android.support.v4.provider.DocumentFile; import java.io.File; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; + +import static android.provider.DocumentsContract.Document.COLUMN_DISPLAY_NAME; +import static android.provider.DocumentsContract.Root.COLUMN_DOCUMENT_ID; + public class StoredDirectoryHelper { public final static int PERMISSION_FLAGS = Intent.FLAG_GRANT_READ_URI_PERMISSION | Intent.FLAG_GRANT_WRITE_URI_PERMISSION; @@ -22,14 +29,27 @@ public class StoredDirectoryHelper { private File ioTree; private DocumentFile docTree; - private ContentResolver contentResolver; + private Context context; private String tag; @RequiresApi(Build.VERSION_CODES.LOLLIPOP) public StoredDirectoryHelper(@NonNull Context context, @NonNull Uri path, String tag) throws IOException { - this.contentResolver = context.getContentResolver(); this.tag = tag; + + if (ContentResolver.SCHEME_FILE.equalsIgnoreCase(path.getScheme())) { + this.ioTree = new File(URI.create(path.toString())); + return; + } + + this.context = context; + + try { + this.context.getContentResolver().takePersistableUriPermission(path, PERMISSION_FLAGS); + } catch (Exception e) { + throw new IOException(e); + } + this.docTree = DocumentFile.fromTreeUri(context, path); if (this.docTree == null) @@ -37,23 +57,75 @@ public class StoredDirectoryHelper { } @TargetApi(Build.VERSION_CODES.KITKAT) - public StoredDirectoryHelper(@NonNull String location, String tag) { + public StoredDirectoryHelper(@NonNull URI location, String tag) { ioTree = new File(location); this.tag = tag; } - @Nullable public StoredFileHelper createFile(String filename, String mime) { + return createFile(filename, mime, false); + } + + public StoredFileHelper createUniqueFile(String name, String mime) { + ArrayList matches = new ArrayList<>(); + String[] filename = splitFilename(name); + String lcFilename = filename[0].toLowerCase(); + + if (docTree == null || Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { + for (File file : ioTree.listFiles()) + addIfStartWith(matches, lcFilename, file.getName()); + } else { + // warning: SAF file listing is very slow + Uri docTreeChildren = DocumentsContract.buildChildDocumentsUriUsingTree( + docTree.getUri(), DocumentsContract.getDocumentId(docTree.getUri()) + ); + + String[] projection = new String[]{COLUMN_DISPLAY_NAME}; + String selection = "(LOWER(" + COLUMN_DISPLAY_NAME + ") LIKE ?%"; + ContentResolver cr = context.getContentResolver(); + + try (Cursor cursor = cr.query(docTreeChildren, projection, selection, new String[]{lcFilename}, null)) { + if (cursor != null) { + while (cursor.moveToNext()) + addIfStartWith(matches, lcFilename, cursor.getString(0)); + } + } + } + + if (matches.size() < 1) { + return createFile(name, mime, true); + } else { + // check if the filename is in use + String lcName = name.toLowerCase(); + for (String testName : matches) { + if (testName.equals(lcName)) { + lcName = null; + break; + } + } + + // check if not in use + if (lcName != null) return createFile(name, mime, true); + } + + Collections.sort(matches, String::compareTo); + + for (int i = 1; i < 1000; i++) { + if (Collections.binarySearch(matches, makeFileName(lcFilename, i, filename[1])) < 0) + return createFile(makeFileName(filename[0], i, filename[1]), mime, true); + } + + return createFile(String.valueOf(System.currentTimeMillis()).concat(filename[1]), mime, false); + } + + private StoredFileHelper createFile(String filename, String mime, boolean safe) { StoredFileHelper storage; try { - if (docTree == null) { - storage = new StoredFileHelper(ioTree, filename, tag); - storage.sourceTree = Uri.fromFile(ioTree).toString(); - } else { - storage = new StoredFileHelper(docTree, contentResolver, filename, mime, tag); - storage.sourceTree = docTree.getUri().toString(); - } + if (docTree == null) + storage = new StoredFileHelper(ioTree, filename, mime); + else + storage = new StoredFileHelper(context, docTree, filename, mime, safe); } catch (IOException e) { return null; } @@ -63,67 +135,6 @@ public class StoredDirectoryHelper { return storage; } - public StoredFileHelper createUniqueFile(String filename, String mime) { - ArrayList existingNames = new ArrayList<>(50); - - String ext; - - int dotIndex = filename.lastIndexOf('.'); - if (dotIndex < 0 || (dotIndex == filename.length() - 1)) { - ext = ""; - } else { - ext = filename.substring(dotIndex); - filename = filename.substring(0, dotIndex - 1); - } - - String name; - if (docTree == null) { - for (File file : ioTree.listFiles()) { - name = file.getName().toLowerCase(); - if (name.startsWith(filename)) existingNames.add(name); - } - } else { - for (DocumentFile file : docTree.listFiles()) { - name = file.getName(); - if (name == null) continue; - name = name.toLowerCase(); - if (name.startsWith(filename)) existingNames.add(name); - } - } - - boolean free = true; - String lwFilename = filename.toLowerCase(); - for (String testName : existingNames) { - if (testName.equals(lwFilename)) { - free = false; - break; - } - } - - if (free) return createFile(filename, mime); - - String[] sortedNames = existingNames.toArray(new String[0]); - Arrays.sort(sortedNames); - - String newName; - int downloadIndex = 0; - do { - newName = filename + " (" + downloadIndex + ")" + ext; - ++downloadIndex; - if (downloadIndex == 1000) { // Probably an error on our side - newName = System.currentTimeMillis() + ext; - break; - } - } while (Arrays.binarySearch(sortedNames, newName) >= 0); - - - return createFile(newName, mime); - } - - public boolean isDirect() { - return docTree == null; - } - public Uri getUri() { return docTree == null ? Uri.fromFile(ioTree) : docTree.getUri(); } @@ -136,34 +147,18 @@ public class StoredDirectoryHelper { return tag; } - public void acquirePermissions() throws IOException { - if (docTree == null) return; - - try { - contentResolver.takePersistableUriPermission(docTree.getUri(), PERMISSION_FLAGS); - } catch (Throwable e) { - throw new IOException(e); - } - } - - public void revokePermissions() throws IOException { - if (docTree == null) return; - - try { - contentResolver.releasePersistableUriPermission(docTree.getUri(), PERMISSION_FLAGS); - } catch (Throwable e) { - throw new IOException(e); - } - } - public Uri findFile(String filename) { - if (docTree == null) - return Uri.fromFile(new File(ioTree, filename)); + if (docTree == null) { + File res = new File(ioTree, filename); + return res.exists() ? Uri.fromFile(res) : null; + } - // findFile() method is very slow - DocumentFile file = docTree.findFile(filename); + DocumentFile res = findFileSAFHelper(context, docTree, filename); + return res == null ? null : res.getUri(); + } - return file == null ? null : file.getUri(); + public boolean canWrite() { + return docTree == null ? ioTree.canWrite() : docTree.canWrite(); } @NonNull @@ -172,4 +167,76 @@ public class StoredDirectoryHelper { return docTree == null ? Uri.fromFile(ioTree).toString() : docTree.getUri().toString(); } + + //////////////////// + // Utils + /////////////////// + + private static void addIfStartWith(ArrayList list, @NonNull String base, String str) { + if (str == null || str.isEmpty()) return; + str = str.toLowerCase(); + if (str.startsWith(base)) list.add(str); + } + + private static String[] splitFilename(@NonNull String filename) { + int dotIndex = filename.lastIndexOf('.'); + + if (dotIndex < 0 || (dotIndex == filename.length() - 1)) + return new String[]{filename, ""}; + + return new String[]{filename.substring(0, dotIndex), filename.substring(dotIndex)}; + } + + private static String makeFileName(String name, int idx, String ext) { + return name.concat(" (").concat(String.valueOf(idx)).concat(")").concat(ext); + } + + /** + * Fast (but not enough) file/directory finder under the storage access framework + * + * @param context The context + * @param tree Directory where search + * @param filename Target filename + * @return A {@link android.support.v4.provider.DocumentFile} contain the reference, otherwise, null + */ + static DocumentFile findFileSAFHelper(@Nullable Context context, DocumentFile tree, String filename) { + if (context == null || Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { + return tree.findFile(filename);// warning: this is very slow + } + + if (!tree.canRead()) return null;// missing read permission + + final int name = 0; + final int documentId = 1; + + // LOWER() SQL function is not supported + String selection = COLUMN_DISPLAY_NAME + " = ?"; + //String selection = COLUMN_DISPLAY_NAME + " LIKE ?%"; + + Uri childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree( + tree.getUri(), DocumentsContract.getDocumentId(tree.getUri()) + ); + String[] projection = {COLUMN_DISPLAY_NAME, COLUMN_DOCUMENT_ID}; + ContentResolver contentResolver = context.getContentResolver(); + + filename = filename.toLowerCase(); + + try (Cursor cursor = contentResolver.query(childrenUri, projection, selection, new String[]{filename}, null)) { + if (cursor == null) return null; + + while (cursor.moveToNext()) { + if (cursor.isNull(name) || !cursor.getString(name).toLowerCase().startsWith(filename)) + continue; + + return DocumentFile.fromSingleUri( + context, DocumentsContract.buildDocumentUriUsingTree( + tree.getUri(), cursor.getString(documentId) + ) + ); + } + } + + return null; + } + } diff --git a/app/src/main/java/us/shandian/giga/io/StoredFileHelper.java b/app/src/main/java/us/shandian/giga/io/StoredFileHelper.java index 0db442f1c..f90a756a9 100644 --- a/app/src/main/java/us/shandian/giga/io/StoredFileHelper.java +++ b/app/src/main/java/us/shandian/giga/io/StoredFileHelper.java @@ -8,6 +8,7 @@ import android.net.Uri; import android.os.Build; import android.provider.DocumentsContract; import android.support.annotation.NonNull; +import android.support.annotation.Nullable; import android.support.v4.app.Fragment; import android.support.v4.provider.DocumentFile; @@ -25,79 +26,52 @@ public class StoredFileHelper implements Serializable { private transient DocumentFile docFile; private transient DocumentFile docTree; private transient File ioFile; - private transient ContentResolver contentResolver; + private transient Context context; protected String source; - String sourceTree; + private String sourceTree; protected String tag; private String srcName; private String srcType; - public StoredFileHelper(String filename, String mime, String tag) { + public StoredFileHelper(@Nullable Uri parent, String filename, String mime, String tag) { this.source = null;// this instance will be "invalid" see invalidate()/isInvalid() methods this.srcName = filename; this.srcType = mime == null ? DEFAULT_MIME : mime; + if (parent != null) this.sourceTree = parent.toString(); this.tag = tag; } @TargetApi(Build.VERSION_CODES.LOLLIPOP) - StoredFileHelper(DocumentFile tree, ContentResolver contentResolver, String filename, String mime, String tag) throws IOException { + StoredFileHelper(@Nullable Context context, DocumentFile tree, String filename, String mime, boolean safe) throws IOException { this.docTree = tree; - this.contentResolver = contentResolver; + this.context = context; - // this is very slow, because SAF does not allow overwrite - DocumentFile res = this.docTree.findFile(filename); + DocumentFile res; - if (res != null && res.exists() && res.isDirectory()) { - if (!res.delete()) - throw new IOException("Directory with the same name found but cannot delete"); - res = null; - } - - if (res == null) { - res = this.docTree.createFile(mime == null ? DEFAULT_MIME : mime, filename); + if (safe) { + // no conflicts (the filename is not in use) + res = this.docTree.createFile(mime, filename); if (res == null) throw new IOException("Cannot create the file"); + } else { + res = createSAF(context, mime, filename); } this.docFile = res; - this.source = res.getUri().toString(); - this.srcName = getName(); - this.srcType = getType(); + + this.source = docFile.getUri().toString(); + this.sourceTree = docTree.getUri().toString(); + + this.srcName = this.docFile.getName(); + this.srcType = this.docFile.getType(); } - @TargetApi(Build.VERSION_CODES.KITKAT) - public StoredFileHelper(Context context, @NonNull Uri path, String tag) throws IOException { - this.source = path.toString(); - this.tag = tag; - - if (path.getScheme() == null || path.getScheme().equalsIgnoreCase("file")) { - this.ioFile = new File(URI.create(this.source)); - } else { - DocumentFile file = DocumentFile.fromSingleUri(context, path); - if (file == null) - throw new UnsupportedOperationException("Cannot get the file via SAF"); - - this.contentResolver = context.getContentResolver(); - this.docFile = file; - - try { - this.contentResolver.takePersistableUriPermission(docFile.getUri(), StoredDirectoryHelper.PERMISSION_FLAGS); - } catch (Exception e) { - throw new IOException(e); - } - } - - this.srcName = getName(); - this.srcType = getType(); - } - - public StoredFileHelper(File location, String filename, String tag) throws IOException { + StoredFileHelper(File location, String filename, String mime) throws IOException { this.ioFile = new File(location, filename); - this.tag = tag; if (this.ioFile.exists()) { if (!this.ioFile.isFile() && !this.ioFile.delete()) @@ -108,22 +82,58 @@ public class StoredFileHelper implements Serializable { } this.source = Uri.fromFile(this.ioFile).toString(); + this.sourceTree = Uri.fromFile(location).toString(); + + this.srcName = ioFile.getName(); + this.srcType = mime; + } + + @TargetApi(Build.VERSION_CODES.KITKAT) + public StoredFileHelper(Context context, @Nullable Uri parent, @NonNull Uri path, String tag) throws IOException { + this.tag = tag; + this.source = path.toString(); + + if (path.getScheme() == null || path.getScheme().equalsIgnoreCase(ContentResolver.SCHEME_FILE)) { + this.ioFile = new File(URI.create(this.source)); + } else { + DocumentFile file = DocumentFile.fromSingleUri(context, path); + + if (file == null) throw new RuntimeException("SAF not available"); + + this.context = context; + + if (file.getName() == null) { + this.source = null; + return; + } else { + this.docFile = file; + takePermissionSAF(); + } + } + + if (parent != null) { + if (!ContentResolver.SCHEME_FILE.equals(parent.getScheme())) + this.docTree = DocumentFile.fromTreeUri(context, parent); + + this.sourceTree = parent.toString(); + } + this.srcName = getName(); this.srcType = getType(); } + public static StoredFileHelper deserialize(@NonNull StoredFileHelper storage, Context context) throws IOException { + Uri treeUri = storage.sourceTree == null ? null : Uri.parse(storage.sourceTree); + if (storage.isInvalid()) - return new StoredFileHelper(storage.srcName, storage.srcType, storage.tag); + return new StoredFileHelper(treeUri, storage.srcName, storage.srcType, storage.tag); - StoredFileHelper instance = new StoredFileHelper(context, Uri.parse(storage.source), storage.tag); + StoredFileHelper instance = new StoredFileHelper(context, treeUri, Uri.parse(storage.source), storage.tag); - if (storage.sourceTree != null) { - instance.docTree = DocumentFile.fromTreeUri(context, Uri.parse(instance.sourceTree)); - - if (instance.docTree == null) - throw new IOException("Cannot deserialize the tree, ¿revoked permissions?"); - } + // under SAF, if the target document is deleted, conserve the filename and mime + if (instance.srcName == null) instance.srcName = storage.srcName; + if (instance.srcType == null) instance.srcType = storage.srcType; return instance; } @@ -143,13 +153,14 @@ public class StoredFileHelper implements Serializable { who.startActivityForResult(intent, requestCode); } + public SharpStream getStream() throws IOException { invalid(); if (docFile == null) return new FileStream(ioFile); else - return new FileStreamSAF(contentResolver, docFile.getUri()); + return new FileStreamSAF(context.getContentResolver(), docFile.getUri()); } /** @@ -173,6 +184,12 @@ public class StoredFileHelper implements Serializable { return docFile == null ? Uri.fromFile(ioFile) : docFile.getUri(); } + public Uri getParentUri() { + invalid(); + + return sourceTree == null ? null : Uri.parse(sourceTree); + } + public void truncate() throws IOException { invalid(); @@ -182,17 +199,17 @@ public class StoredFileHelper implements Serializable { } public boolean delete() { - invalid(); - + if (source == null) return true; if (docFile == null) return ioFile.delete(); + boolean res = docFile.delete(); try { int flags = Intent.FLAG_GRANT_READ_URI_PERMISSION | Intent.FLAG_GRANT_WRITE_URI_PERMISSION; - contentResolver.releasePersistableUriPermission(docFile.getUri(), flags); + context.getContentResolver().releasePersistableUriPermission(docFile.getUri(), flags); } catch (Exception ex) { - // ¿what happen? + // nothing to do } return res; @@ -209,18 +226,22 @@ public class StoredFileHelper implements Serializable { return docFile == null ? ioFile.canWrite() : docFile.canWrite(); } - public File getIOFile() { - return ioFile; - } - public String getName() { - if (source == null) return srcName; - return docFile == null ? ioFile.getName() : docFile.getName(); + if (source == null) + return srcName; + else if (docFile == null) + return ioFile.getName(); + + String name = docFile.getName(); + return name == null ? srcName : name; } public String getType() { - if (source == null) return srcType; - return docFile == null ? DEFAULT_MIME : docFile.getType();// not obligatory for Java IO + if (source == null || docFile == null) + return srcType; + + String type = docFile.getType(); + return type == null ? srcType : type; } public String getTag() { @@ -231,29 +252,41 @@ public class StoredFileHelper implements Serializable { if (source == null) return false; boolean exists = docFile == null ? ioFile.exists() : docFile.exists(); - boolean asFile = docFile == null ? ioFile.isFile() : docFile.isFile();// ¿docFile.isVirtual() means is no-physical? + boolean isFile = docFile == null ? ioFile.isFile() : docFile.isFile();// ¿docFile.isVirtual() means is no-physical? - return exists && asFile; + return exists && isFile; } public boolean create() { invalid(); + boolean result; if (docFile == null) { try { - return ioFile.createNewFile(); + result = ioFile.createNewFile(); + } catch (IOException e) { + return false; + } + } else if (docTree == null) { + result = false; + } else { + if (!docTree.canRead() || !docTree.canWrite()) return false; + try { + docFile = createSAF(context, srcType, srcName); + if (docFile == null || docFile.getName() == null) return false; + result = true; } catch (IOException e) { return false; } } - if (docTree == null || docFile.getName() == null) return false; + if (result) { + source = (docFile == null ? Uri.fromFile(ioFile) : docFile.getUri()).toString(); + srcName = getName(); + srcType = getType(); + } - DocumentFile res = docTree.createFile(docFile.getName(), docFile.getType() == null ? DEFAULT_MIME : docFile.getType()); - if (res == null) return false; - - docFile = res; - return true; + return result; } public void invalidate() { @@ -264,20 +297,25 @@ public class StoredFileHelper implements Serializable { source = null; - sourceTree = null; docTree = null; docFile = null; ioFile = null; - contentResolver = null; - } - - private void invalid() { - if (source == null) - throw new IllegalStateException("In invalid state"); + context = null; } public boolean equals(StoredFileHelper storage) { - if (this.isInvalid() != storage.isInvalid()) return false; + if (this == storage) return true; + + // note: do not compare tags, files can have the same parent folder + //if (stringMismatch(this.tag, storage.tag)) return false; + + if (stringMismatch(getLowerCase(this.sourceTree), getLowerCase(this.sourceTree))) + return false; + + if (this.isInvalid() || storage.isInvalid()) { + return this.srcName.equalsIgnoreCase(storage.srcName) && this.srcType.equalsIgnoreCase(storage.srcType); + } + if (this.isDirect() != storage.isDirect()) return false; if (this.isDirect()) @@ -298,4 +336,46 @@ public class StoredFileHelper implements Serializable { else return "sourceFile=" + source + " treeSource=" + (sourceTree == null ? "" : sourceTree) + " tag=" + tag; } + + + private void invalid() { + if (source == null) + throw new IllegalStateException("In invalid state"); + } + + private void takePermissionSAF() throws IOException { + try { + context.getContentResolver().takePersistableUriPermission(docFile.getUri(), StoredDirectoryHelper.PERMISSION_FLAGS); + } catch (Exception e) { + if (docFile.getName() == null) throw new IOException(e); + } + } + + private DocumentFile createSAF(@Nullable Context context, String mime, String filename) throws IOException { + DocumentFile res = StoredDirectoryHelper.findFileSAFHelper(context, docTree, filename); + + if (res != null && res.exists() && res.isDirectory()) { + if (!res.delete()) + throw new IOException("Directory with the same name found but cannot delete"); + res = null; + } + + if (res == null) { + res = this.docTree.createFile(srcType == null ? DEFAULT_MIME : mime, filename); + if (res == null) throw new IOException("Cannot create the file"); + } + + return res; + } + + private String getLowerCase(String str) { + return str == null ? null : str.toLowerCase(); + } + + private boolean stringMismatch(String str1, String str2) { + if (str1 == null && str2 == null) return false; + if ((str1 == null) != (str2 == null)) return true; + + return !str1.equals(str2); + } } diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java index 98ab29dbb..f12c1c2d2 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/Mp4FromDashMuxer.java @@ -11,7 +11,7 @@ import java.io.IOException; class Mp4FromDashMuxer extends Postprocessing { Mp4FromDashMuxer() { - super(2 * 1024 * 1024/* 2 MiB */, true); + super(3 * 1024 * 1024/* 3 MiB */, true); } @Override diff --git a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java index 7bc32ea05..3d10628e7 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java @@ -8,6 +8,7 @@ import org.schabi.newpipe.streams.io.SharpStream; import java.io.File; import java.io.IOException; +import java.io.Serializable; import us.shandian.giga.get.DownloadMission; import us.shandian.giga.io.ChunkFileInputStream; @@ -19,7 +20,7 @@ import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD; import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION; -public abstract class Postprocessing { +public abstract class Postprocessing implements Serializable { static transient final byte OK_RESULT = ERROR_NOTHING; @@ -28,12 +29,10 @@ public abstract class Postprocessing { public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4"; public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a"; - public static Postprocessing getAlgorithm(String algorithmName, String[] args) { + public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args, @NonNull File cacheDir) { Postprocessing instance; - if (null == algorithmName) { - throw new NullPointerException("algorithmName"); - } else switch (algorithmName) { + switch (algorithmName) { case ALGORITHM_TTML_CONVERTER: instance = new TtmlConverter(); break; @@ -47,13 +46,14 @@ public abstract class Postprocessing { instance = new M4aNoDash(); break; /*case "example-algorithm": - instance = new ExampleAlgorithm(mission);*/ + instance = new ExampleAlgorithm();*/ default: throw new RuntimeException("Unimplemented post-processing algorithm: " + algorithmName); } instance.args = args; - instance.name = algorithmName; + instance.name = algorithmName;// for debug only, maybe remove this field in the future + instance.cacheDir = cacheDir; return instance; } @@ -125,7 +125,6 @@ public abstract class Postprocessing { return -1; }; - // TODO: use Context.getCache() for this operation temp = new File(cacheDir, mission.storage.getName() + ".tmp"); out = new CircularFileWriter(mission.storage.getStream(), temp, checker); diff --git a/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java b/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java index 37295f2e3..3d5ecb3cd 100644 --- a/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java +++ b/app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java @@ -13,7 +13,7 @@ import java.io.IOException; class WebMMuxer extends Postprocessing { WebMMuxer() { - super(2048 * 1024/* 2 MiB */, true); + super(5 * 1024 * 1024/* 5 MiB */, true); } @Override diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManager.java b/app/src/main/java/us/shandian/giga/service/DownloadManager.java index 3624fb6c2..479c4b92f 100644 --- a/app/src/main/java/us/shandian/giga/service/DownloadManager.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManager.java @@ -62,13 +62,15 @@ public class DownloadManager { * @param context Context for the data source for finished downloads * @param handler Thread required for Messaging */ - DownloadManager(@NonNull Context context, Handler handler) { + DownloadManager(@NonNull Context context, Handler handler, StoredDirectoryHelper storageVideo, StoredDirectoryHelper storageAudio) { if (DEBUG) { Log.d(TAG, "new DownloadManager instance. 0x" + Integer.toHexString(this.hashCode())); } mFinishedMissionStore = new FinishedMissionStore(context); mHandler = handler; + mMainStorageAudio = storageAudio; + mMainStorageVideo = storageVideo; mMissionsFinished = loadFinishedMissions(); mPendingMissionsDir = getPendingDir(context); @@ -129,91 +131,59 @@ public class DownloadManager { } for (File sub : subs) { - if (sub.isFile()) { - DownloadMission mis = Utility.readFromFile(sub); + if (!sub.isFile()) continue; - if (mis == null) { - //noinspection ResultOfMethodCallIgnored - sub.delete(); - } else { - if (mis.isFinished()) { - //noinspection ResultOfMethodCallIgnored - sub.delete(); - continue; - } - - boolean exists; - try { - mis.storage = StoredFileHelper.deserialize(mis.storage, ctx); - exists = !mis.storage.isInvalid() && mis.storage.existsAsFile(); - - } catch (Exception ex) { - Log.e(TAG, "Failed to load the file source of " + mis.storage.toString()); - mis.storage.invalidate(); - exists = false; - } - - if (mis.isPsRunning()) { - if (mis.psAlgorithm.worksOnSameFile) { - // Incomplete post-processing results in a corrupted download file - // because the selected algorithm works on the same file to save space. - if (exists && !mis.storage.delete()) - Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath()); - - exists = true; - } - - mis.psState = 0; - mis.errCode = DownloadMission.ERROR_POSTPROCESSING_STOPPED; - mis.errObject = null; - } else if (!exists) { - - StoredDirectoryHelper mainStorage = getMainStorage(mis.storage.getTag()); - - if (!mis.storage.isInvalid() && !mis.storage.create()) { - // using javaIO cannot recreate the file - // using SAF in older devices (no tree available) - // - // force the user to pick again the save path - mis.storage.invalidate(); - } else if (mainStorage != null) { - // if the user has changed the save path before this download, the original save path will be lost - StoredFileHelper newStorage = mainStorage.createFile(mis.storage.getName(), mis.storage.getType()); - if (newStorage == null) - mis.storage.invalidate(); - else - mis.storage = newStorage; - } - - if (mis.isInitialized()) { - // the progress is lost, reset mission state - DownloadMission m = new DownloadMission(mis.urls, mis.storage, mis.kind, mis.psAlgorithm); - m.timestamp = mis.timestamp; - m.threadCount = mis.threadCount; - m.source = mis.source; - m.nearLength = mis.nearLength; - m.enqueued = mis.enqueued; - m.errCode = DownloadMission.ERROR_PROGRESS_LOST; - mis = m; - } - } - - if (mis.psAlgorithm != null) mis.psAlgorithm.cacheDir = ctx.getCacheDir(); - - mis.running = false; - mis.recovered = exists; - mis.metadata = sub; - mis.maxRetry = mPrefMaxRetry; - mis.mHandler = mHandler; - - mMissionsPending.add(mis); - } + DownloadMission mis = Utility.readFromFile(sub); + if (mis == null || mis.isFinished()) { + //noinspection ResultOfMethodCallIgnored + sub.delete(); + continue; } + + boolean exists; + try { + mis.storage = StoredFileHelper.deserialize(mis.storage, ctx); + exists = !mis.storage.isInvalid() && mis.storage.existsAsFile(); + } catch (Exception ex) { + Log.e(TAG, "Failed to load the file source of " + mis.storage.toString(), ex); + mis.storage.invalidate(); + exists = false; + } + + if (mis.isPsRunning()) { + if (mis.psAlgorithm.worksOnSameFile) { + // Incomplete post-processing results in a corrupted download file + // because the selected algorithm works on the same file to save space. + if (exists && !mis.storage.delete()) + Log.w(TAG, "Unable to delete incomplete download file: " + sub.getPath()); + + exists = true; + } + + mis.psState = 0; + mis.errCode = DownloadMission.ERROR_POSTPROCESSING_STOPPED; + mis.errObject = null; + } else if (!exists) { + tryRecover(mis); + + // the progress is lost, reset mission state + if (mis.isInitialized()) + mis.resetState(true, true, DownloadMission.ERROR_PROGRESS_LOST); + } + + if (mis.psAlgorithm != null) + mis.psAlgorithm.cacheDir = pickAvailableCacheDir(ctx); + + mis.recovered = exists; + mis.metadata = sub; + mis.maxRetry = mPrefMaxRetry; + mis.mHandler = mHandler; + + mMissionsPending.add(mis); } - if (mMissionsPending.size() > 1) { + if (mMissionsPending.size() > 1) Collections.sort(mMissionsPending, (mission1, mission2) -> Long.compare(mission1.timestamp, mission2.timestamp)); - } } /** @@ -313,6 +283,25 @@ public class DownloadManager { } } + public void tryRecover(DownloadMission mission) { + StoredDirectoryHelper mainStorage = getMainStorage(mission.storage.getTag()); + + if (!mission.storage.isInvalid() && mission.storage.create()) return; + + // using javaIO cannot recreate the file + // using SAF in older devices (no tree available) + // + // force the user to pick again the save path + mission.storage.invalidate(); + + if (mainStorage == null) return; + + // if the user has changed the save path before this download, the original save path will be lost + StoredFileHelper newStorage = mainStorage.createFile(mission.storage.getName(), mission.storage.getType()); + + if (newStorage != null) mission.storage = newStorage; + } + /** * Get a pending mission by its path @@ -392,7 +381,7 @@ public class DownloadManager { synchronized (this) { for (DownloadMission mission : mMissionsPending) { - if (mission.running || !mission.canDownload()) continue; + if (mission.running || mission.isCorrupt()) continue; flag = true; mission.start(); @@ -482,7 +471,7 @@ public class DownloadManager { int paused = 0; synchronized (this) { for (DownloadMission mission : mMissionsPending) { - if (!mission.canDownload() || mission.isPsRunning()) continue; + if (mission.isCorrupt() || mission.isPsRunning()) continue; if (mission.running && isMetered) { paused++; @@ -542,6 +531,20 @@ public class DownloadManager { return MissionState.None; } + private static boolean isDirectoryAvailable(File directory) { + return directory != null && directory.canWrite(); + } + + static File pickAvailableCacheDir(@NonNull Context ctx) { + if (isDirectoryAvailable(ctx.getExternalCacheDir())) + return ctx.getExternalCacheDir(); + else if (isDirectoryAvailable(ctx.getCacheDir())) + return ctx.getCacheDir(); + + // this never should happen + return ctx.getDir("tmp", Context.MODE_PRIVATE); + } + @Nullable private StoredDirectoryHelper getMainStorage(@NonNull String tag) { if (tag.equals(TAG_AUDIO)) return mMainStorageAudio; @@ -656,7 +659,7 @@ public class DownloadManager { synchronized (DownloadManager.this) { for (DownloadMission mission : mMissionsPending) { - if (hidden.contains(mission) || mission.canDownload()) + if (hidden.contains(mission) || mission.isCorrupt()) continue; if (mission.running) diff --git a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java index deed9e8e3..da63cb545 100755 --- a/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java +++ b/app/src/main/java/us/shandian/giga/service/DownloadManagerService.java @@ -6,6 +6,7 @@ import android.app.NotificationManager; import android.app.PendingIntent; import android.app.Service; import android.content.BroadcastReceiver; +import android.content.ContentResolver; import android.content.Context; import android.content.Intent; import android.content.IntentFilter; @@ -40,6 +41,7 @@ import org.schabi.newpipe.player.helper.LockManager; import java.io.File; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; import us.shandian.giga.get.DownloadMission; @@ -65,14 +67,15 @@ public class DownloadManagerService extends Service { private static final int DOWNLOADS_NOTIFICATION_ID = 1001; private static final String EXTRA_URLS = "DownloadManagerService.extra.urls"; - private static final String EXTRA_PATH = "DownloadManagerService.extra.path"; private static final String EXTRA_KIND = "DownloadManagerService.extra.kind"; private static final String EXTRA_THREADS = "DownloadManagerService.extra.threads"; private static final String EXTRA_POSTPROCESSING_NAME = "DownloadManagerService.extra.postprocessingName"; private static final String EXTRA_POSTPROCESSING_ARGS = "DownloadManagerService.extra.postprocessingArgs"; private static final String EXTRA_SOURCE = "DownloadManagerService.extra.source"; private static final String EXTRA_NEAR_LENGTH = "DownloadManagerService.extra.nearLength"; - private static final String EXTRA_MAIN_STORAGE_TAG = "DownloadManagerService.extra.tag"; + private static final String EXTRA_PATH = "DownloadManagerService.extra.storagePath"; + private static final String EXTRA_PARENT_PATH = "DownloadManagerService.extra.storageParentPath"; + private static final String EXTRA_STORAGE_TAG = "DownloadManagerService.extra.storageTag"; private static final String ACTION_RESET_DOWNLOAD_FINISHED = APPLICATION_ID + ".reset_download_finished"; private static final String ACTION_OPEN_DOWNLOADS_FINISHED = APPLICATION_ID + ".open_downloads_finished"; @@ -136,7 +139,9 @@ public class DownloadManagerService extends Service { } }; - mManager = new DownloadManager(this, mHandler); + mPrefs = PreferenceManager.getDefaultSharedPreferences(this); + + mManager = new DownloadManager(this, mHandler, getVideoStorage(), getAudioStorage()); Intent openDownloadListIntent = new Intent(this, DownloadActivity.class) .setAction(Intent.ACTION_MAIN); @@ -182,7 +187,6 @@ public class DownloadManagerService extends Service { registerReceiver(mNetworkStateListener, new IntentFilter(ConnectivityManager.CONNECTIVITY_ACTION)); } - mPrefs = PreferenceManager.getDefaultSharedPreferences(this); mPrefs.registerOnSharedPreferenceChangeListener(mPrefChangeListener); handlePreferenceChange(mPrefs, getString(R.string.downloads_cross_network)); @@ -190,8 +194,6 @@ public class DownloadManagerService extends Service { handlePreferenceChange(mPrefs, getString(R.string.downloads_queue_limit)); mLock = new LockManager(this); - - setupStorageAPI(true); } @Override @@ -347,11 +349,12 @@ public class DownloadManagerService extends Service { } else if (key.equals(getString(R.string.downloads_queue_limit))) { mManager.mPrefQueueLimit = prefs.getBoolean(key, true); } else if (key.equals(getString(R.string.downloads_storage_api))) { - setupStorageAPI(false); + mManager.mMainStorageVideo = loadMainStorage(getString(R.string.download_path_video_key), DownloadManager.TAG_VIDEO); + mManager.mMainStorageAudio = loadMainStorage(getString(R.string.download_path_audio_key), DownloadManager.TAG_AUDIO); } else if (key.equals(getString(R.string.download_path_video_key))) { - loadMainStorage(key, DownloadManager.TAG_VIDEO, false); + mManager.mMainStorageVideo = loadMainStorage(key, DownloadManager.TAG_VIDEO); } else if (key.equals(getString(R.string.download_path_audio_key))) { - loadMainStorage(key, DownloadManager.TAG_AUDIO, false); + mManager.mMainStorageAudio = loadMainStorage(key, DownloadManager.TAG_AUDIO); } } @@ -387,36 +390,46 @@ public class DownloadManagerService extends Service { Intent intent = new Intent(context, DownloadManagerService.class); intent.setAction(Intent.ACTION_RUN); intent.putExtra(EXTRA_URLS, urls); - intent.putExtra(EXTRA_PATH, storage.getUri()); intent.putExtra(EXTRA_KIND, kind); intent.putExtra(EXTRA_THREADS, threads); intent.putExtra(EXTRA_SOURCE, source); intent.putExtra(EXTRA_POSTPROCESSING_NAME, psName); intent.putExtra(EXTRA_POSTPROCESSING_ARGS, psArgs); intent.putExtra(EXTRA_NEAR_LENGTH, nearLength); - intent.putExtra(EXTRA_MAIN_STORAGE_TAG, storage.getTag()); + + intent.putExtra(EXTRA_PARENT_PATH, storage.getParentUri()); + intent.putExtra(EXTRA_PATH, storage.getUri()); + intent.putExtra(EXTRA_STORAGE_TAG, storage.getTag()); + context.startService(intent); } - public void startMission(Intent intent) { + private void startMission(Intent intent) { String[] urls = intent.getStringArrayExtra(EXTRA_URLS); Uri path = intent.getParcelableExtra(EXTRA_PATH); + Uri parentPath = intent.getParcelableExtra(EXTRA_PARENT_PATH); int threads = intent.getIntExtra(EXTRA_THREADS, 1); char kind = intent.getCharExtra(EXTRA_KIND, '?'); String psName = intent.getStringExtra(EXTRA_POSTPROCESSING_NAME); String[] psArgs = intent.getStringArrayExtra(EXTRA_POSTPROCESSING_ARGS); String source = intent.getStringExtra(EXTRA_SOURCE); long nearLength = intent.getLongExtra(EXTRA_NEAR_LENGTH, 0); - String tag = intent.getStringExtra(EXTRA_MAIN_STORAGE_TAG); + String tag = intent.getStringExtra(EXTRA_STORAGE_TAG); StoredFileHelper storage; try { - storage = new StoredFileHelper(this, path, tag); + storage = new StoredFileHelper(this, parentPath, path, tag); } catch (IOException e) { throw new RuntimeException(e);// this never should happen } - final DownloadMission mission = new DownloadMission(urls, storage, kind, Postprocessing.getAlgorithm(psName, psArgs)); + Postprocessing ps; + if (psName == null) + ps = null; + else + ps = Postprocessing.getAlgorithm(psName, psArgs, DownloadManager.pickAvailableCacheDir(this)); + + final DownloadMission mission = new DownloadMission(urls, storage, kind, ps); mission.threadCount = threads; mission.source = source; mission.nearLength = nearLength; @@ -525,60 +538,63 @@ public class DownloadManagerService extends Service { mLockAcquired = acquire; } - private void setupStorageAPI(boolean acquire) { - loadMainStorage(getString(R.string.download_path_audio_key), DownloadManager.TAG_VIDEO, acquire); - loadMainStorage(getString(R.string.download_path_video_key), DownloadManager.TAG_AUDIO, acquire); + private StoredDirectoryHelper getVideoStorage() { + return loadMainStorage(getString(R.string.download_path_video_key), DownloadManager.TAG_VIDEO); } - void loadMainStorage(String prefKey, String tag, boolean acquire) { + private StoredDirectoryHelper getAudioStorage() { + return loadMainStorage(getString(R.string.download_path_audio_key), DownloadManager.TAG_AUDIO); + } + + + private StoredDirectoryHelper loadMainStorage(String prefKey, String tag) { String path = mPrefs.getString(prefKey, null); final String JAVA_IO = getString(R.string.downloads_storage_api_default); boolean useJavaIO = JAVA_IO.equals(mPrefs.getString(getString(R.string.downloads_storage_api), JAVA_IO)); final String defaultPath; - if (tag.equals(DownloadManager.TAG_VIDEO)) - defaultPath = Environment.DIRECTORY_MOVIES; - else// if (tag.equals(DownloadManager.TAG_AUDIO)) - defaultPath = Environment.DIRECTORY_MUSIC; - - StoredDirectoryHelper mainStorage; - if (path == null || path.isEmpty()) { - mainStorage = useJavaIO ? new StoredDirectoryHelper(defaultPath, tag) : null; - } else { - - if (path.charAt(0) == File.separatorChar) { - Log.i(TAG, "Migrating old save path: " + path); - - useJavaIO = true; - path = Uri.fromFile(new File(path)).toString(); - - mPrefs.edit().putString(prefKey, path).apply(); - } - - if (useJavaIO) { - mainStorage = new StoredDirectoryHelper(path, tag); - } else { - - // tree api is not available in older versions - if (android.os.Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { - mainStorage = null; - } else { - try { - mainStorage = new StoredDirectoryHelper(this, Uri.parse(path), tag); - if (acquire) mainStorage.acquirePermissions(); - } catch (IOException e) { - Log.e(TAG, "Failed to load the storage of " + tag + " from path: " + path, e); - mainStorage = null; - } - } - } + switch (tag) { + case DownloadManager.TAG_VIDEO: + defaultPath = Environment.DIRECTORY_MOVIES; + break; + case DownloadManager.TAG_AUDIO: + defaultPath = Environment.DIRECTORY_MUSIC; + break; + default: + return null; } - if (tag.equals(DownloadManager.TAG_VIDEO)) - mManager.mMainStorageVideo = mainStorage; - else// if (tag.equals(DownloadManager.TAG_AUDIO)) - mManager.mMainStorageAudio = mainStorage; + if (path == null || path.isEmpty()) { + return useJavaIO ? new StoredDirectoryHelper(new File(defaultPath).toURI(), tag) : null; + } + + if (path.charAt(0) == File.separatorChar) { + Log.i(TAG, "Migrating old save path: " + path); + + useJavaIO = true; + path = Uri.fromFile(new File(path)).toString(); + + mPrefs.edit().putString(prefKey, path).apply(); + } + + boolean override = path.startsWith(ContentResolver.SCHEME_FILE) && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP; + if (useJavaIO || override) { + return new StoredDirectoryHelper(URI.create(path), tag); + } + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) { + return null;// SAF Directory API is not available in older versions + } + + try { + return new StoredDirectoryHelper(this, Uri.parse(path), tag); + } catch (Exception e) { + Log.e(TAG, "Failed to load the storage of " + tag + " from " + path, e); + Toast.makeText(this, R.string.no_available_dir, Toast.LENGTH_LONG).show(); + } + + return null; } //////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index 4d80588e0..1892f4437 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -41,7 +41,9 @@ import org.schabi.newpipe.report.ErrorActivity; import org.schabi.newpipe.report.UserAction; import org.schabi.newpipe.util.NavigationHelper; +import java.io.File; import java.lang.ref.WeakReference; +import java.net.URI; import java.util.ArrayList; import java.util.Collections; @@ -346,7 +348,7 @@ public class MissionAdapter extends Adapter { uri = FileProvider.getUriForFile( mContext, BuildConfig.APPLICATION_ID + ".provider", - mission.storage.getIOFile() + new File(URI.create(mission.storage.getUri().toString())) ); } else { uri = mission.storage.getUri(); @@ -384,10 +386,18 @@ public class MissionAdapter extends Adapter { } private static String resolveMimeType(@NonNull Mission mission) { + String mimeType; + + if (!mission.storage.isInvalid()) { + mimeType = mission.storage.getType(); + if (mimeType != null && mimeType.length() > 0 && !mimeType.equals(StoredFileHelper.DEFAULT_MIME)) + return mimeType; + } + String ext = Utility.getFileExt(mission.storage.getName()); if (ext == null) return DEFAULT_MIME_TYPE; - String mimeType = MimeTypeMap.getSingleton().getMimeTypeFromExtension(ext.substring(1)); + mimeType = MimeTypeMap.getSingleton().getMimeTypeFromExtension(ext.substring(1)); return mimeType == null ? DEFAULT_MIME_TYPE : mimeType; } @@ -476,6 +486,7 @@ public class MissionAdapter extends Adapter { return; case ERROR_PROGRESS_LOST: msg = R.string.error_progress_lost; + break; default: if (mission.errCode >= 100 && mission.errCode < 600) { msgEx = "HTTP " + mission.errCode; @@ -554,7 +565,12 @@ public class MissionAdapter extends Adapter { return true; case R.id.retry: if (mission.hasInvalidStorage()) { - mRecover.tryRecover(mission); + mDownloadManager.tryRecover(mission); + if (mission.storage.isInvalid()) + mRecover.tryRecover(mission); + else + recoverMission(mission); + return true; } mission.psContinue(true); @@ -672,13 +688,12 @@ public class MissionAdapter extends Adapter { if (mDeleter != null) mDeleter.resume(); } - public void recoverMission(DownloadMission mission, StoredFileHelper newStorage) { + public void recoverMission(DownloadMission mission) { for (ViewHolderItem h : mPendingDownloadsItems) { if (mission != h.item.mission) continue; - mission.changeStorage(newStorage); - mission.errCode = DownloadMission.ERROR_NOTHING; mission.errObject = null; + mission.resetState(true, false, DownloadMission.ERROR_NOTHING); h.status.setText(UNDEFINED_PROGRESS); h.state = -1; @@ -822,9 +837,9 @@ public class MissionAdapter extends Adapter { if (mission != null) { if (mission.hasInvalidStorage()) { - retry.setEnabled(true); - delete.setEnabled(true); - showError.setEnabled(true); + retry.setVisible(true); + delete.setVisible(true); + showError.setVisible(true); } else if (mission.isPsRunning()) { switch (mission.errCode) { case ERROR_INSUFFICIENT_STORAGE: diff --git a/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java b/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java index 82ab777b0..bd5ce9215 100644 --- a/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java +++ b/app/src/main/java/us/shandian/giga/ui/fragment/MissionsFragment.java @@ -9,6 +9,7 @@ import android.content.SharedPreferences; import android.os.Bundle; import android.os.IBinder; import android.preference.PreferenceManager; +import android.support.annotation.NonNull; import android.support.v4.app.Fragment; import android.support.v7.widget.GridLayoutManager; import android.support.v7.widget.LinearLayoutManager; @@ -66,14 +67,7 @@ public class MissionsFragment extends Fragment { mAdapter = new MissionAdapter(mContext, mBinder.getDownloadManager(), mEmpty); mAdapter.deleterLoad(getView()); - mAdapter.setRecover(mission -> - StoredFileHelper.requestSafWithFileCreation( - MissionsFragment.this, - REQUEST_DOWNLOAD_PATH_SAF, - mission.storage.getName(), - mission.storage.getType() - ) - ); + mAdapter.setRecover(MissionsFragment.this::recoverMission); setAdapterButtons(); @@ -92,7 +86,7 @@ public class MissionsFragment extends Fragment { }; @Override - public View onCreateView(LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { + public View onCreateView(@NonNull LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { View v = inflater.inflate(R.layout.missions, container, false); mPrefs = PreferenceManager.getDefaultSharedPreferences(getActivity()); @@ -239,8 +233,18 @@ public class MissionsFragment extends Fragment { mAdapter.setMasterButtons(mStart, mPause); } + private void recoverMission(@NonNull DownloadMission mission) { + unsafeMissionTarget = mission; + StoredFileHelper.requestSafWithFileCreation( + MissionsFragment.this, + REQUEST_DOWNLOAD_PATH_SAF, + mission.storage.getName(), + mission.storage.getType() + ); + } + @Override - public void onSaveInstanceState(Bundle outState) { + public void onSaveInstanceState(@NonNull Bundle outState) { super.onSaveInstanceState(outState); if (mAdapter != null) { @@ -285,8 +289,9 @@ public class MissionsFragment extends Fragment { } try { - StoredFileHelper storage = new StoredFileHelper(mContext, data.getData(), unsafeMissionTarget.storage.getTag()); - mAdapter.recoverMission(unsafeMissionTarget, storage); + String tag = unsafeMissionTarget.storage.getTag(); + unsafeMissionTarget.storage = new StoredFileHelper(mContext, null, data.getData(), tag); + mAdapter.recoverMission(unsafeMissionTarget); } catch (IOException e) { Toast.makeText(mContext, R.string.general_error, Toast.LENGTH_LONG).show(); } diff --git a/app/src/main/java/us/shandian/giga/util/Utility.java b/app/src/main/java/us/shandian/giga/util/Utility.java index d77e598d8..793cbea18 100644 --- a/app/src/main/java/us/shandian/giga/util/Utility.java +++ b/app/src/main/java/us/shandian/giga/util/Utility.java @@ -9,6 +9,7 @@ import android.support.annotation.DrawableRes; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.support.v4.content.ContextCompat; +import android.util.Log; import android.widget.Toast; import org.schabi.newpipe.R; @@ -81,6 +82,7 @@ public class Utility { objectInputStream = new ObjectInputStream(new FileInputStream(file)); object = (T) objectInputStream.readObject(); } catch (Exception e) { + Log.e("Utility", "Failed to deserialize the object", e); object = null; } diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index eee110474..4cc394357 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -442,8 +442,8 @@ abrir en modo popup Mostrar error Codigo - No se puede crear la carpeta de destino - No se puede crear el archivo + No se puede crear el archivo + No se puede crear la carpeta de destino Permiso denegado por el sistema Fallo la conexión segura No se pudo encontrar el servidor diff --git a/app/src/main/res/values/settings_keys.xml b/app/src/main/res/values/settings_keys.xml index b2be3135b..42df857c1 100644 --- a/app/src/main/res/values/settings_keys.xml +++ b/app/src/main/res/values/settings_keys.xml @@ -176,13 +176,17 @@ - file_rename + file_rename_charset file_replacement_character _ + + CHARSET_LETTERS_AND_DIGITS + CHARSET_MOST_SPECIAL + @string/charset_letters_and_digits_value - @string/charset_most_special_characters_value + @string/charset_most_special_value @@ -190,8 +194,8 @@ @string/charset_most_special_characters - @string/charset_most_special_characters_value - + @string/charset_most_special_value + downloads_max_retry 3 diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 7907d2974..10b36c1c7 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -305,8 +305,7 @@ Allowed characters in filenames Invalid characters are replaced with this value Replacement character - [^\\w\\d]+ - [\\n\\r|\\?*<":>/']+ + Letters and digits Most special characters No app installed to play this file diff --git a/app/src/main/res/xml/download_settings.xml b/app/src/main/res/xml/download_settings.xml index bbb91acac..2f62aa89e 100644 --- a/app/src/main/res/xml/download_settings.xml +++ b/app/src/main/res/xml/download_settings.xml @@ -5,12 +5,6 @@ android:title="@string/settings_category_downloads_title"> - -