From cbb8c91e744e7a1dc0eff493ffc0ad57d4342596 Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR <16546989+deldesir@users.noreply.github.com> Date: Fri, 15 Sep 2023 11:04:27 -0400 Subject: [PATCH 1/5] Add modal macro YouTube download modal used in layout.html visible when clicking on "Add YouTube videos" button. --- cps/templates/modal_dialogs.html | 81 ++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/cps/templates/modal_dialogs.html b/cps/templates/modal_dialogs.html index 3e0b5bf1..0353e89d 100644 --- a/cps/templates/modal_dialogs.html +++ b/cps/templates/modal_dialogs.html @@ -139,3 +139,84 @@ {% endmacro %} + +{% macro youtube_download_modal() %} + +
  • + +
    • {% endif %} {% if not current_user.is_anonymous and not simple%}
    • @@ -205,7 +212,9 @@ - {% block modal %}{% endblock %} + {% block modal %} + {{ youtube_download_modal() }} + {% endblock %} From 03c5eebc2c826b7e8db4ddbcadd7d8fd80297f15 Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR <16546989+deldesir@users.noreply.github.com> Date: Fri, 15 Sep 2023 11:53:00 -0400 Subject: [PATCH 3/5] Add YouTube button AJAX YouTube download button AJAX call which uses a custom route that downloads videos from a playlist or channel --- cps/static/js/main.js | 127 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/cps/static/js/main.js b/cps/static/js/main.js index 8d7354ef..5e7b1dc6 100644 --- a/cps/static/js/main.js +++ b/cps/static/js/main.js @@ -149,6 +149,133 @@ $(document).ready(function() { inp.val('').blur().focus().val(val) } } + + // Function to toggle advanced options visibility + function toggleAdvancedOptions() { + var advancedOptions = $("#advancedOptions"); + if (advancedOptions.is(":visible")) { + advancedOptions.hide(); + $("#advancedOptionsToggle").text("Show advanced options") + } else { + advancedOptions.show(); + $("#advancedOptionsToggle").text("Hide advanced options") + } + } + + // Handle click event for the advanced options toggle + $("#advancedOptionsToggle").click(function(event) { + event.preventDefault(); + toggleAdvancedOptions(); + }); + + // Function to initiate the YouTube download AJAX request + function initiateYoutubeDownload() { + var url = $("#youtubeURL").val(); + var videoQuality = $("input[name='videoQuality']:checked").val(); + var maxVideos = $("#maxVideos").val(); + var maxVideosSize = $("#maxVideosSize").val(); + var addToBookshelf = $("#addToBookshelf").is(":checked"); + + // Set empty number values to zero + maxVideos = maxVideos === "" ? 0 : parseInt(maxVideos); + maxVideosSize = maxVideosSize === "" ? 0 : parseInt(maxVideosSize); + + // Check if the input URL is a valid YouTube URL + if (!isValidYoutubeURL(url)) { + alert("Invalid YouTube URL"); + return; + } + + $.ajax({ + url: "/books/youtube", + method: "POST", + data: { + csrf_token: $("#youtubeDownloadForm input[name=csrf_token]").val(), + youtubeURL: url, + videoQuality: videoQuality, + maxVideos: maxVideos, + maxVideosSize: maxVideosSize, + addToBookshelf: addToBookshelf + }, + success: function(response) { + // Handle success response here + if (response && response.location) { + // Redirect to the specified location + window.location.href = response.location; + } else { + // Handle any specific success behavior + console.log("YouTube download request successful."); + } + }, + error: function(xhr, status, error) { + // Handle error here + console.log("YouTube download request failed:", error); + $("#youtubeDownloadForm .error-message").text("YouTube download request failed."); + } + }); + } + + // Handle Enter key press event in the input field + $(document).on('keydown', function(event) { + // Check if the pressed key is Enter (key code 13) + if (event.which === 13 && $("#youtubeDownloadModal").is(":visible")) { + initiateYoutubeDownload(); + } + }); + + // Handle the "Start" button click event + $("#btn-download-youtube-submit").click(function() { + initiateYoutubeDownload(); + }); + + // Handle change event for the video quality radio buttons + $("input[name='videoQuality']").change(function() { + // Handle change event + }); + + // Handle input event for the max videos input + $("#maxVideos").on('input', function() { + var inputValue = $(this).val(); + if (!/^\d*$/.test(inputValue)) { + alert("Please enter a valid number."); + $(this).val(""); + } + + // If maxVideos is changed, disable and clear maxVideosSize + if (inputValue) { + $("#maxVideosSize").prop("disabled", true).val(""); + } else { + $("#maxVideosSize").prop("disabled", false); + } + }); + + // Handle input event for the max size input + $("#maxVideosSize").on('input', function() { + var inputValue = $(this).val(); + if (!/^\d*$/.test(inputValue)) { + alert("Please enter a valid number."); + $(this).val(""); + } + + // If maxVideosSize is changed, disable and clear maxVideos + if (inputValue) { + $("#maxVideos").prop("disabled", true).val(""); + } else { + $("#maxVideos").prop("disabled", false); + } + }); + + // Handle change event for the add to bookshelf checkbox + $("#addToBookshelf").change(function() { + // Handle change event + }); + + // Function to validate YouTube URL (updated to handle https://youtube.com/@handle) + function isValidYoutubeURL(url) { + var youtubeURLPattern = /^(https?:\/\/)?(www\.)?(youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/playlist\?list=|youtube\.com\/channel\/|youtube\.com\/@)([a-zA-Z0-9_-]{11}|[a-zA-Z0-9_-]{34})/; + return youtubeURLPattern.test(url); + } + }); $(".session").click(function() { From e000c82e18f5928ce42b8cd822a68a669b7e97b8 Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR <16546989+deldesir@users.noreply.github.com> Date: Fri, 15 Sep 2023 12:06:02 -0400 Subject: [PATCH 4/5] Add YouTube route Server side logic that handles videos download from YouTube collections using external tools --- cps/editbooks.py | 217 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) diff --git a/cps/editbooks.py b/cps/editbooks.py index f52f08aa..e8e9eacd 100755 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -309,6 +309,223 @@ def upload(): return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') +@editbook.route("/youtube", methods=["POST"]) +@login_required_if_no_ano +@upload_required +def youtube(): + if not config.config_uploading: + abort(404) + + def get_yb_executable(): + yb_executable = os.getenv("YB_EXECUTABLE", "yb") + return yb_executable + + def run_subprocess(command_args): + try: + completed_process = subprocess.run( + command_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + text=True + ) + return (True, completed_process.stdout, completed_process.stderr) + + except subprocess.CalledProcessError as e: + error_message = f"Subprocess error (return code {e.returncode}): {e.stderr}, {e.stdout}" + log.error(error_message) + return False, error_message, e.stdout, e.stderr + except Exception as e: + error_message = f"An error occurred while running the subprocess: {e}" + log.error(error_message) + return False, error_message + + def process_youtube_download(youtube_url, video_quality): + yb_executable = get_yb_executable() + + if youtube_url: + youtube_id = extract_youtube_url(youtube_url) + + download_args = [ + yb_executable, + youtube_id, + "/output", + video_quality, + ] + subprocess_result = run_subprocess(download_args) + + log.info("Subprocess result: {}".format(subprocess_result)) + + if subprocess_result[0]: + log.info("Renaming files in /output/{}/videos".format(youtube_id)) + # make a list of requested files with names of directories found in /output//videos + requested_files = os.listdir("/output/{}/videos".format(youtube_id)) + # remove "youtube-nsig" from the list of requested files + requested_files.remove("youtube-nsig") + log.info("Requested files: {}".format(requested_files)) + renamed_files = rename_files(requested_files, youtube_id) + if renamed_files: + for requested_file in renamed_files: + requested_file = open(requested_file, "rb") + requested_file.filename = os.path.basename(requested_file.name) + requested_file.save = lambda path: copyfile(requested_file.name, path) + + log.info("Processing file: {}".format(requested_file)) + try: + modify_date = False + calibre_db.update_title_sort(config) + calibre_db.session.connection().connection.connection.create_function( + "uuid4", 0, lambda: str(uuid4()) + ) + + meta, error = file_handling_on_upload(requested_file) + if error: + return error + + ( + db_book, + input_authors, + title_dir, + renamed_authors, + ) = create_book_on_upload(modify_date, meta) + + # Comments need book id therefore only possible after flush + modify_date |= edit_book_comments( + Markup(meta.description).unescape(), db_book + ) + + book_id = db_book.id + title = db_book.title + + error = helper.update_dir_structure( + book_id, + config.config_calibre_dir, + input_authors[0], + meta.file_path, + title_dir + meta.extension.lower(), + renamed_author=renamed_authors, + ) + + move_coverfile(meta, db_book) + + if modify_date: + calibre_db.set_metadata_dirty(book_id) + # save data to database, reread data + calibre_db.session.commit() + + if error: + flash(error, category="error") + link = '{}'.format( + url_for("web.show_book", book_id=book_id), escape(title) + ) + upload_text = N_("File %(file)s uploaded", file=link) + WorkerThread.add( + current_user.name, TaskUpload(upload_text, escape(title)) + ) + helper.add_book_to_thumbnail_cache(book_id) + + if len(renamed_files) < 2: + if current_user.role_edit() or current_user.role_admin(): + resp = { + "location": url_for( + "edit-book.show_edit_book", book_id=book_id + ) + } + return Response(json.dumps(resp), mimetype="application/json") + else: + resp = {"location": url_for("web.show_book", book_id=book_id)} + return Response(json.dumps(resp), mimetype="application/json") + + except (OperationalError, IntegrityError, StaleDataError) as e: + calibre_db.session.rollback() + log.error_or_exception("Database error: {}".format(e)) + flash( + _( + "Oops! Database Error: %(error)s.", + error=e.orig if hasattr(e, "orig") else e, + ), + category="error", + ) + else: + flash("Error: 'poetry' executable not found in PATH", category="error") + return False + + if request.method == "POST" and "youtubeURL" in request.form: + youtube_url = request.form["youtubeURL"] + video_quality = request.form.get("videoQuality", "720") + + if process_youtube_download(youtube_url, video_quality): + response = { + "success": "Downloaded YouTube media successfully", + } + return jsonify(response) + else: + response = { + "error": "Failed to download YouTube media", + } + return jsonify(response), 500 + +def extract_youtube_url(url): + try: + if "youtube.com" in url: + if "watch?v=" in url: + return url.split("watch?v=")[1] + elif "playlist?list=" in url: + return url.split("playlist?list=")[1] + elif "channel/" in url: + return url.split("channel/")[1] + elif "user/" in url: + return url.split("user/")[1] + elif "@" in url: + return extract_channel_id_from_handle(url) + elif "youtu.be" in url: + return url.split("youtu.be/")[1] + + flash("Error: Invalid YouTube URL", category="error") + return None + except Exception as e: + flash("An error occurred while processing the YouTube URL: {}".format(e), category="error") + return None + +def extract_channel_id_from_handle(url): + handle = url.split("@")[1] + operational_api_url = "https://yt.lemnoslife.com/channels?handle=" + handle + response = requests.get(operational_api_url) + + if response.status_code == 200: + return response.json()["items"][0]["id"] + else: + flash("Error: Failed to retrieve YouTube channel ID from API", category="error") + return None + +def rename_files(requested_files, youtube_id): + # cache_directory_path = "/output/{}/cache".format(youtube_id) + video_dir_path = "/output/{}/videos".format(youtube_id) + renamed_files = [] + if not os.path.exists("/tmp/calibre_web"): + os.makedirs("/tmp/calibre_web") + + for video_id in requested_files: + # video_json_path = os.path.join(cache_directory_path, "videos.json") + video_webm_path = os.path.join(video_dir_path, video_id, "video.webm") + thumbnail_path = os.path.join(video_dir_path, video_id, "video.webp") + + try: + thumbnail_path_new = os.path.join("/tmp", "calibre_web", "{}.webp".format(video_id + youtube_id)) + move(thumbnail_path, thumbnail_path_new) + video_webm_path_new = os.path.join("/tmp", "calibre_web", "{}.webm".format(video_id + youtube_id)) + move(video_webm_path, video_webm_path_new) + renamed_files.append(video_webm_path_new) + + if not os.listdir(video_dir_path): + os.rmdir(video_dir_path) + + except Exception as e: + flash("An error occurred while renaming the YouTube video file: {}".format(e), category="error") + return None + + return renamed_files + @editbook.route("/admin/book/convert/", methods=['POST']) @login_required_if_no_ano @edit_required From cdb75a3cfcc13df27ea228ce9a1064eb87d75b80 Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR <16546989+deldesir@users.noreply.github.com> Date: Fri, 15 Sep 2023 12:27:01 -0400 Subject: [PATCH 5/5] Add metadata handling Metadata handling for YouTube videos --- cps/uploader.py | 97 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/cps/uploader.py b/cps/uploader.py index 23dfc4a6..88a94a19 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -16,8 +16,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import json import os import hashlib +import shutil +from subprocess import run from tempfile import gettempdir from flask_babel import gettext as _ @@ -84,6 +87,12 @@ def process(tmp_file_path, original_file_name, original_file_extension, rar_exec original_file_name, original_file_extension, rar_executable) + elif extension_upper in ['.MP4', '.WEBM', '.AVI', '.MKV', '.M4V', '.MPG', '.MPEG','.OGV']: + meta = video_metadata(tmp_file_path, original_file_name, original_file_extension) + + elif extension_upper in ['.JPG', '.JPEG', '.PNG', '.GIF', '.SVG', '.WEBP']: + meta = image_metadata(tmp_file_path, original_file_name, original_file_extension) + except Exception as ex: log.warning('cannot parse metadata, using default: %s', ex) @@ -239,6 +248,94 @@ def pdf_preview(tmp_file_path, tmp_dir): return None +def video_metadata(tmp_file_path, original_file_name, original_file_extension): + video_id = os.path.splitext(original_file_name)[0][:11] + youtube_id = os.path.splitext(original_file_name)[0][11:] + json_file_path = os.path.join("/output", youtube_id, "cache", "videos.json") + coverfile_path = os.path.splitext(original_file_name)[0] + '.webp' + if os.path.isfile(coverfile_path): + coverfile_path = os.path.splitext(tmp_file_path)[0] + '.cover.webp' + os.rename(os.path.splitext(original_file_name)[0] + '.webp', coverfile_path) + os.remove(os.path.splitext(original_file_name)[0] + '.webm') + return coverfile_path + if os.path.isfile(json_file_path): + with open(json_file_path) as json_file: + data = json.load(json_file) + title = data[video_id]['snippet']['title'] + author = data[video_id]['snippet']['videoOwnerChannelTitle'] + description = data[video_id]['snippet']['description'] + publisher = 'YouTube' + pubdate = data[video_id]['contentDetails']['videoPublishedAt'][0:10] + + meta = BookMeta( + file_path=tmp_file_path, + extension=original_file_extension, + title=title, + author=author, + cover=coverfile_path, + description=description, + tags='', + series="", + series_id="", + languages="", + publisher=publisher, + pubdate=pubdate, + identifiers=[]) + return meta + else: + ffmpeg_executable = os.getenv('FFMPEG_PATH', 'ffmpeg') + ffmpeg_output_file = os.path.splitext(tmp_file_path)[0] + '.cover.jpg' + ffmpeg_args = [ + ffmpeg_executable, + '-i', tmp_file_path, + '-vframes', '1', + '-y', ffmpeg_output_file + ] + + try: + ffmpeg_result = run(ffmpeg_args, capture_output=True, check=True) + log.debug(f"ffmpeg output: {ffmpeg_result.stdout}") + + except Exception as e: + log.warning(f"ffmpeg failed: {e}") + return None + + meta = BookMeta( + file_path=tmp_file_path, + extension=original_file_extension, + title=original_file_name, + author='Unknown', + cover=os.path.splitext(tmp_file_path)[0] + '.cover.jpg', + description='', + tags='', + series="", + series_id="", + languages="", + publisher="", + pubdate="", + identifiers=[]) + return meta + + +def image_metadata(tmp_file_path, original_file_name, original_file_extension): + shutil.copyfile(tmp_file_path, os.path.splitext(tmp_file_path)[0] + '.cover.jpg') + meta = BookMeta( + file_path=tmp_file_path, + extension=original_file_extension, + title=original_file_name, + author='Unknown', + cover=os.path.splitext(tmp_file_path)[0] + '.cover.jpg', + description='', + tags='', + series="", + series_id="", + languages="", + publisher="", + pubdate="", + identifiers=[]) + return meta + + def get_magick_version(): ret = dict() if not use_generic_pdf_cover: