diff --git a/cps/about.py b/cps/about.py index 7b6cc71a..1d081fe2 100644 --- a/cps/about.py +++ b/cps/about.py @@ -49,9 +49,9 @@ sorted_modules = OrderedDict((sorted(modules.items(), key=lambda x: x[0].casefol def collect_stats(): if constants.NIGHTLY_VERSION[0] == "$Format:%H$": - calibre_web_version = constants.STABLE_VERSION['version'] + calibre_web_version = constants.STABLE_VERSION['version'].replace("b", " Beta") else: - calibre_web_version = (constants.STABLE_VERSION['version'] + ' - ' + calibre_web_version = (constants.STABLE_VERSION['version'].replace("b", " Beta") + ' - ' + constants.NIGHTLY_VERSION[0].replace('%', '%%') + ' - ' + constants.NIGHTLY_VERSION[1].replace('%', '%%')) diff --git a/cps/admin.py b/cps/admin.py index 022acc8e..fa29759e 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -47,7 +47,7 @@ from . import constants, logger, helper, services, cli_param from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \ kobo_sync_status, schedule from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ - valid_email, check_username + valid_email, check_username, get_calibre_binarypath from .gdriveutils import is_gdrive_ready, gdrive_support from .render_template import render_title_template, get_sidebar_config from .services.worker import WorkerThread @@ -217,7 +217,7 @@ def admin(): form_date += timedelta(hours=int(commit[20:22]), minutes=int(commit[23:])) commit = format_datetime(form_date - tz, format='short') else: - commit = version['version'] + commit = version['version'].replace("b", " Beta") all_user = ub.session.query(ub.User).all() # email_settings = mail_config.get_mail_settings() @@ -1751,6 +1751,7 @@ def _configuration_update_helper(): _config_checkbox_int(to_save, "config_uploading") _config_checkbox_int(to_save, "config_unicode_filename") + _config_checkbox_int(to_save, "config_embed_metadata") # Reboot on config_anonbrowse with enabled ldap, as decoraters are changed in this case reboot_required |= (_config_checkbox_int(to_save, "config_anonbrowse") and config.config_login_type == constants.LOGIN_LDAP) @@ -1767,8 +1768,14 @@ def _configuration_update_helper(): constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',') _config_string(to_save, "config_calibre") - _config_string(to_save, "config_converterpath") + _config_string(to_save, "config_binariesdir") _config_string(to_save, "config_kepubifypath") + if "config_binariesdir" in to_save: + calibre_status = helper.check_calibre(config.config_binariesdir) + if calibre_status: + return _configuration_result(calibre_status) + to_save["config_converterpath"] = get_calibre_binarypath("ebook-convert") + _config_string(to_save, "config_converterpath") reboot_required |= _config_int(to_save, "config_login_type") diff --git a/cps/cli.py b/cps/cli.py index e9b97b9d..855ad899 100644 --- a/cps/cli.py +++ b/cps/cli.py @@ -29,8 +29,8 @@ from .constants import DEFAULT_SETTINGS_FILE, DEFAULT_GDRIVE_FILE def version_info(): if _NIGHTLY_VERSION[1].startswith('$Format'): - return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version'] - return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'], _NIGHTLY_VERSION[1]) + return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version'].replace("b", " Beta") + return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'].replace("b", " Beta"), _NIGHTLY_VERSION[1]) class CliParameter(object): diff --git a/cps/config_sql.py b/cps/config_sql.py index f6c0991c..c4f94b4e 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -34,6 +34,7 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from . import constants, logger +from .subproc_wrapper import process_wait log = logger.create() @@ -140,10 +141,12 @@ class _Settings(_Base): config_kepubifypath = Column(String, default=None) config_converterpath = Column(String, default=None) + config_binariesdir = Column(String, default=None) config_calibre = Column(String) config_rarfile_location = Column(String, default=None) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) config_unicode_filename = Column(Boolean, default=False) + config_embed_metadata = Column(Boolean, default=True) config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) @@ -186,9 +189,11 @@ class ConfigSQL(object): self.load() change = False - if self.config_converterpath == None: # pylint: disable=access-member-before-definition + + if self.config_binariesdir == None: # pylint: disable=access-member-before-definition change = True - self.config_converterpath = autodetect_calibre_binary() + self.config_binariesdir = autodetect_calibre_binaries() + self.config_converterpath = autodetect_converter_binary(self.config_binariesdir) if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition change = True @@ -474,17 +479,32 @@ def _migrate_table(session, orm_class, secret_key=None): session.rollback() -def autodetect_calibre_binary(): +def autodetect_calibre_binaries(): if sys.platform == "win32": - calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre2\\ebook-convert.exe", - "C:\\program files\\calibre2\\ebook-convert.exe"] + calibre_path = ["C:\\program files\\calibre\\", + "C:\\program files(x86)\\calibre\\", + "C:\\program files(x86)\\calibre2\\", + "C:\\program files\\calibre2\\"] else: - calibre_path = ["/opt/calibre/ebook-convert"] + calibre_path = ["/opt/calibre/"] for element in calibre_path: - if os.path.isfile(element) and os.access(element, os.X_OK): - return element + supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()] + if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + return element + return "" + + +def autodetect_converter_binary(calibre_path): + if sys.platform == "win32": + converter_path = os.path.join(calibre_path, "ebook-convert.exe") + else: + converter_path = os.path.join(calibre_path, "ebook-convert") + if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): + return converter_path return "" @@ -526,6 +546,7 @@ def load_configuration(session, secret_key): session.commit() + def get_flask_session_key(_session): flask_settings = _session.query(_Flask_Settings).one_or_none() if flask_settings == None: diff --git a/cps/constants.py b/cps/constants.py index 87ce6f59..ef207e02 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -156,6 +156,11 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' 'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac', 'm4a', 'm4b'} +_extension = "" +if sys.platform == "win32": + _extension = ".exe" +SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]} + def has_flag(value, bit_flag): return bit_flag == (bit_flag & (value or 0)) @@ -169,13 +174,11 @@ BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, d 'series_id, languages, publisher, pubdate, identifiers') # python build process likes to have x.y.zbw -> b for beta and w a counting number -STABLE_VERSION = {'version': '0.6.22 Beta'} +STABLE_VERSION = {'version': '0.6.22b'} NIGHTLY_VERSION = dict() NIGHTLY_VERSION[0] = '$Format:%H$' NIGHTLY_VERSION[1] = '$Format:%cI$' -# NIGHTLY_VERSION[0] = 'bb7d2c6273ae4560e83950d36d64533343623a57' -# NIGHTLY_VERSION[1] = '2018-09-09T10:13:08+02:00' # CACHE CACHE_TYPE_THUMBNAILS = 'thumbnails' diff --git a/cps/epub.py b/cps/epub.py index 2103cb3f..e78e5b88 100644 --- a/cps/epub.py +++ b/cps/epub.py @@ -23,10 +23,12 @@ from lxml import etree from . import isoLanguages, cover from . import config, logger from .helper import split_authors +from .epub_helper import get_content_opf, default_ns from .constants import BookMeta log = logger.create() + def _extract_cover(zip_file, cover_file, cover_path, tmp_file_name): if cover_file is None: return None @@ -44,24 +46,14 @@ def _extract_cover(zip_file, cover_file, cover_path, tmp_file_name): return cover.cover_processing(tmp_file_name, cf, extension) def get_epub_layout(book, book_data): - ns = { - 'n': 'urn:oasis:names:tc:opendocument:xmlns:container', - 'pkg': 'http://www.idpf.org/2007/opf', - } file_path = os.path.normpath(os.path.join(config.get_book_path(), book.path, book_data.name + "." + book_data.format.lower())) try: - epubZip = zipfile.ZipFile(file_path) - txt = epubZip.read('META-INF/container.xml') - tree = etree.fromstring(txt) - cfname = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0] - cf = epubZip.read(cfname) + tree, __ = get_content_opf(file_path, default_ns) + p = tree.xpath('/pkg:package/pkg:metadata', namespaces=default_ns)[0] - tree = etree.fromstring(cf) - p = tree.xpath('/pkg:package/pkg:metadata', namespaces=ns)[0] - - layout = p.xpath('pkg:meta[@property="rendition:layout"]/text()', namespaces=ns) + layout = p.xpath('pkg:meta[@property="rendition:layout"]/text()', namespaces=default_ns) except (etree.XMLSyntaxError, KeyError, IndexError) as e: log.error("Could not parse epub metadata of book {} during kobo sync: {}".format(book.id, e)) layout = [] @@ -80,12 +72,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): } epub_zip = zipfile.ZipFile(tmp_file_path) - - txt = epub_zip.read('META-INF/container.xml') - tree = etree.fromstring(txt) - cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0] - cf = epub_zip.read(cf_name) - tree = etree.fromstring(cf) + tree, cf_name = get_content_opf(epub_zip, ns) cover_path = os.path.dirname(cf_name) diff --git a/cps/epub_helper.py b/cps/epub_helper.py new file mode 100644 index 00000000..603ccc3d --- /dev/null +++ b/cps/epub_helper.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2018 lemmsh, Kennyl, Kyosfonica, matthazinski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import zipfile +from lxml import etree + +from . import isoLanguages + +default_ns = { + 'n': 'urn:oasis:names:tc:opendocument:xmlns:container', + 'pkg': 'http://www.idpf.org/2007/opf', +} + +OPF_NAMESPACE = "http://www.idpf.org/2007/opf" +PURL_NAMESPACE = "http://purl.org/dc/elements/1.1/" + +OPF = "{%s}" % OPF_NAMESPACE +PURL = "{%s}" % PURL_NAMESPACE + +etree.register_namespace("opf", OPF_NAMESPACE) +etree.register_namespace("dc", PURL_NAMESPACE) + +OPF_NS = {None: OPF_NAMESPACE} # the default namespace (no prefix) +NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE} + + +def updateEpub(src, dest, filename, data, ): + # create a temp copy of the archive without filename + with zipfile.ZipFile(src, 'r') as zin: + with zipfile.ZipFile(dest, 'w') as zout: + zout.comment = zin.comment # preserve the comment + for item in zin.infolist(): + if item.filename != filename: + zout.writestr(item, zin.read(item.filename)) + + # now add filename with its new data + with zipfile.ZipFile(dest, mode='a', compression=zipfile.ZIP_DEFLATED) as zf: + zf.writestr(filename, data) + + +def get_content_opf(file_path, ns=default_ns): + epubZip = zipfile.ZipFile(file_path) + txt = epubZip.read('META-INF/container.xml') + tree = etree.fromstring(txt) + cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0] + cf = epubZip.read(cf_name) + + return etree.fromstring(cf), cf_name + + +def create_new_metadata_backup(book, custom_columns, export_language, translated_cover_name, lang_type=3): + # generate root package element + package = etree.Element(OPF + "package", nsmap=OPF_NS) + package.set("unique-identifier", "uuid_id") + package.set("version", "2.0") + + # generate metadata element and all sub elements of it + metadata = etree.SubElement(package, "metadata", nsmap=NSMAP) + identifier = etree.SubElement(metadata, PURL + "identifier", id="calibre_id", nsmap=NSMAP) + identifier.set(OPF + "scheme", "calibre") + identifier.text = str(book.id) + identifier2 = etree.SubElement(metadata, PURL + "identifier", id="uuid_id", nsmap=NSMAP) + identifier2.set(OPF + "scheme", "uuid") + identifier2.text = book.uuid + title = etree.SubElement(metadata, PURL + "title", nsmap=NSMAP) + title.text = book.title + for author in book.authors: + creator = etree.SubElement(metadata, PURL + "creator", nsmap=NSMAP) + creator.text = str(author.name) + creator.set(OPF + "file-as", book.author_sort) # ToDo Check + creator.set(OPF + "role", "aut") + contributor = etree.SubElement(metadata, PURL + "contributor", nsmap=NSMAP) + contributor.text = "calibre (5.7.2) [https://calibre-ebook.com]" + contributor.set(OPF + "file-as", "calibre") # ToDo Check + contributor.set(OPF + "role", "bkp") + + date = etree.SubElement(metadata, PURL + "date", nsmap=NSMAP) + date.text = '{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(d=book.pubdate) + if book.comments and book.comments[0].text: + for b in book.comments: + description = etree.SubElement(metadata, PURL + "description", nsmap=NSMAP) + description.text = b.text + for b in book.publishers: + publisher = etree.SubElement(metadata, PURL + "publisher", nsmap=NSMAP) + publisher.text = str(b.name) + if not book.languages: + language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP) + language.text = export_language + else: + for b in book.languages: + language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP) + language.text = str(b.lang_code) if lang_type == 3 else isoLanguages.get(part3=b.lang_code).part1 + for b in book.tags: + subject = etree.SubElement(metadata, PURL + "subject", nsmap=NSMAP) + subject.text = str(b.name) + etree.SubElement(metadata, "meta", name="calibre:author_link_map", + content="{" + ", ".join(['"' + str(a.name) + '": ""' for a in book.authors]) + "}", + nsmap=NSMAP) + for b in book.series: + etree.SubElement(metadata, "meta", name="calibre:series", + content=str(str(b.name)), + nsmap=NSMAP) + if book.series: + etree.SubElement(metadata, "meta", name="calibre:series_index", + content=str(book.series_index), + nsmap=NSMAP) + if len(book.ratings) and book.ratings[0].rating > 0: + etree.SubElement(metadata, "meta", name="calibre:rating", + content=str(book.ratings[0].rating), + nsmap=NSMAP) + etree.SubElement(metadata, "meta", name="calibre:timestamp", + content='{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format( + d=book.timestamp), + nsmap=NSMAP) + etree.SubElement(metadata, "meta", name="calibre:title_sort", + content=book.sort, + nsmap=NSMAP) + sequence = 0 + for cc in custom_columns: + value = None + extra = None + cc_entry = getattr(book, "custom_column_" + str(cc.id)) + if cc_entry.__len__(): + value = [c.value for c in cc_entry] if cc.is_multiple else cc_entry[0].value + extra = cc_entry[0].extra if hasattr(cc_entry[0], "extra") else None + etree.SubElement(metadata, "meta", name="calibre:user_metadata:#{}".format(cc.label), + content=cc.to_json(value, extra, sequence), + nsmap=NSMAP) + sequence += 1 + + # generate guide element and all sub elements of it + # Title is translated from default export language + guide = etree.SubElement(package, "guide") + etree.SubElement(guide, "reference", type="cover", title=translated_cover_name, href="cover.jpg") + + return package + +def replace_metadata(tree, package): + rep_element = tree.xpath('/pkg:package/pkg:metadata', namespaces=default_ns)[0] + new_element = package.xpath('//metadata', namespaces=default_ns)[0] + tree.replace(rep_element, new_element) + return etree.tostring(tree, + xml_declaration=True, + encoding='utf-8', + pretty_print=True).decode('utf-8') + + diff --git a/cps/file_helper.py b/cps/file_helper.py new file mode 100644 index 00000000..7c3e5291 --- /dev/null +++ b/cps/file_helper.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2023 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from tempfile import gettempdir +import os +import shutil + +def get_temp_dir(): + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + return tmp_dir + + +def del_temp_dir(): + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + shutil.rmtree(tmp_dir) diff --git a/cps/gdrive.py b/cps/gdrive.py index 832350e1..4d110f83 100644 --- a/cps/gdrive.py +++ b/cps/gdrive.py @@ -23,7 +23,6 @@ import os import hashlib import json -import tempfile from uuid import uuid4 from time import time from shutil import move, copyfile @@ -34,6 +33,7 @@ from flask_login import login_required from . import logger, gdriveutils, config, ub, calibre_db, csrf from .admin import admin_required +from .file_helper import get_temp_dir gdrive = Blueprint('gdrive', __name__, url_prefix='/gdrive') log = logger.create() @@ -139,9 +139,7 @@ try: dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode() if not response['deleted'] and response['file']['title'] == 'metadata.db' \ and response['file']['md5Checksum'] != hashlib.md5(dbpath): # nosec - tmp_dir = os.path.join(tempfile.gettempdir(), 'calibre_web') - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() log.info('Database file updated') copyfile(dbpath, os.path.join(tmp_dir, "metadata.db_" + str(current_milli_time()))) diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 08ead47d..b1d30596 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -34,7 +34,6 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.exc import OperationalError, InvalidRequestError, IntegrityError from sqlalchemy.orm.exc import StaleDataError -from sqlalchemy.sql.expression import text try: from httplib2 import __version__ as httplib2_version diff --git a/cps/helper.py b/cps/helper.py index 0e0c4de3..ad89e2bf 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -25,9 +25,10 @@ import re import shutil import socket from datetime import datetime, timedelta -from tempfile import gettempdir import requests import unidecode +from uuid import uuid4 +from lxml import etree from flask import send_from_directory, make_response, redirect, abort, url_for from flask_babel import gettext as _ @@ -54,12 +55,14 @@ from . import calibre_db, cli_param from .tasks.convert import TaskConvert from . import logger, config, db, ub, fs from . import gdriveutils as gd -from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES -from .subproc_wrapper import process_wait +from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES +from .subproc_wrapper import process_wait, process_open from .services.worker import WorkerThread from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails from .tasks.metadata_backup import TaskBackupMetadata +from .file_helper import get_temp_dir +from .epub_helper import get_content_opf, create_new_metadata_backup, updateEpub, replace_metadata log = logger.create() @@ -921,10 +924,7 @@ def save_cover(img, book_path): return False, _("Only jpg/jpeg files are supported as coverfile") if config.config_use_google_drive: - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() ret, message = save_cover_from_filestorage(tmp_dir, "uploaded_cover.jpg", img) if ret is True: gd.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg').replace("\\", "/"), @@ -938,29 +938,87 @@ def save_cover(img, book_path): def do_download_file(book, book_format, client, data, headers): + book_name = data.name if config.config_use_google_drive: # startTime = time.time() - df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) + df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: - return gd.do_gdrive_download(df, headers) + if config.config_embed_metadata and ( + (book_format == "kepub" and config.config_kepubifypath ) or + (book_format != "kepub" and config.config_binariesdir)): + output_path = os.path.join(config.config_calibre_dir, book.path) + if not os.path.exists(output_path): + os.makedirs(output_path) + output = os.path.join(config.config_calibre_dir, book.path, book_name + "." + book_format) + gd.downloadFile(book.path, book_name + "." + book_format, output) + if book_format == "kepub" and config.config_kepubifypath: + filename, download_name = do_kepubify_metadata_replace(book, output) + elif book_format != "kepub" and config.config_binariesdir: + filename, download_name = do_calibre_export(book.id, book_format) + else: + return gd.do_gdrive_download(df, headers) else: abort(404) else: filename = os.path.join(config.get_book_path(), book.path) - if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)): + if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)): # ToDo: improve error handling - log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format)) + log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format)) if client == "kobo" and book_format == "kepub": headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") - response = make_response(send_from_directory(filename, data.name + "." + book_format)) - # ToDo Check headers parameter - for element in headers: - response.headers[element[0]] = element[1] - log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format))) - return response + if book_format == "kepub" and config.config_kepubifypath and config.config_embed_metadata: + filename, download_name = do_kepubify_metadata_replace(book, os.path.join(filename, + book_name + "." + book_format)) + elif book_format != "kepub" and config.config_binariesdir and config.config_embed_metadata: + filename, download_name = do_calibre_export(book.id, book_format) + else: + download_name = book_name + + response = make_response(send_from_directory(filename, download_name + "." + book_format)) + # ToDo Check headers parameter + for element in headers: + response.headers[element[0]] = element[1] + log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format))) + return response + + +def do_kepubify_metadata_replace(book, file_path): + custom_columns = (calibre_db.session.query(db.CustomColumns) + .filter(db.CustomColumns.mark_for_delete == 0) + .filter(db.CustomColumns.datatype.notin_(db.cc_exceptions)) + .order_by(db.CustomColumns.label).all()) + + tree, cf_name = get_content_opf(file_path) + package = create_new_metadata_backup(book, custom_columns, current_user.locale, _("Cover"), lang_type=2) + content = replace_metadata(tree, package) + tmp_dir = get_temp_dir() + temp_file_name = str(uuid4()) + # open zipfile and replace metadata block in content.opf + updateEpub(file_path, os.path.join(tmp_dir, temp_file_name + ".kepub"), cf_name, content) + return tmp_dir, temp_file_name + + +def do_calibre_export(book_id, book_format, ): + try: + quotes = [3, 5, 7, 9] + tmp_dir = get_temp_dir() + calibredb_binarypath = get_calibre_binarypath("calibredb") + temp_file_name = str(uuid4()) + opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', '--with-library', config.config_calibre_dir, + '--to-dir', tmp_dir, '--formats', book_format, "--template", "{}".format(temp_file_name), + str(book_id)] + p = process_open(opf_command, quotes) + _, err = p.communicate() + if err: + log.error('Metadata embedder encountered an error: %s', err) + return tmp_dir, temp_file_name + except OSError as ex: + # ToDo real error handling + log.error_or_exception(ex) + ################################## @@ -984,6 +1042,47 @@ def check_unrar(unrar_location): return _('Error executing UnRar') +def check_calibre(calibre_location): + if not calibre_location: + return + + if not os.path.exists(calibre_location): + return _('Could not find the specified directory') + + if not os.path.isdir(calibre_location): + return _('Please specify a directory, not a file') + + try: + supported_binary_paths = [os.path.join(calibre_location, binary) + for binary in SUPPORTED_CALIBRE_BINARIES.values()] + binaries_available = [os.path.isfile(binary_path) for binary_path in supported_binary_paths] + binaries_executable = [os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] + if all(binaries_available) and all(binaries_executable): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') + for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + else: + return _('Calibre binaries not viable') + else: + ret_val = [] + missing_binaries=[path for path, available in + zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] + + missing_perms=[path for path, available in + zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_executable) if not available] + if missing_binaries: + ret_val.append(_('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries))) + if missing_perms: + ret_val.append(_('Missing executable permissions: %(missing)s', missing=", ".join(missing_perms))) + return ", ".join(ret_val) + + except (OSError, UnicodeDecodeError) as err: + log.error_or_exception(err) + return _('Error excecuting Calibre') + + def json_serial(obj): """JSON serializer for objects not serializable by default json code""" @@ -1008,7 +1107,7 @@ def tags_filters(): # checks if domain is in database (including wildcards) -# example SELECT * FROM @TABLE WHERE 'abcdefg' LIKE Name; +# example SELECT * FROM @TABLE WHERE 'abcdefg' LIKE Name; # from https://code.luasoftware.com/tutorials/flask/execute-raw-sql-in-flask-sqlalchemy/ # in all calls the email address is checked for validity def check_valid_domain(domain_text): @@ -1042,6 +1141,17 @@ def get_download_link(book_id, book_format, client): abort(404) +def get_calibre_binarypath(binary): + binariesdir = config.config_binariesdir + if binariesdir: + try: + return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary]) + except KeyError as ex: + log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary]) + pass + return "" + + def clear_cover_thumbnail_cache(book_id): if config.schedule_generate_book_covers: WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True) diff --git a/cps/schedule.py b/cps/schedule.py index 05367e99..bf622b36 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -21,6 +21,7 @@ import datetime from . import config, constants from .services.background_scheduler import BackgroundScheduler, CronTrigger, use_APScheduler from .tasks.database import TaskReconnectDatabase +from .tasks.tempFolder import TaskDeleteTempFolder from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails, TaskClearCoverThumbnailCache from .services.worker import WorkerThread from .tasks.metadata_backup import TaskBackupMetadata @@ -31,6 +32,9 @@ def get_scheduled_tasks(reconnect=True): if reconnect: tasks.append([lambda: TaskReconnectDatabase(), 'reconnect', False]) + # Delete temp folder + tasks.append([lambda: TaskDeleteTempFolder(), 'delete temp', True]) + # Generate metadata.opf file for each changed book if config.schedule_metadata_backup: tasks.append([lambda: TaskBackupMetadata("en"), 'backup metadata', False]) @@ -86,6 +90,8 @@ def register_startup_tasks(): # Ignore tasks that should currently be running, as these will be added when registering scheduled tasks if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, duration): scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False)) + else: + scheduler.schedule_tasks_immediately(tasks=[[lambda: TaskDeleteTempFolder(), 'delete temp', True]]) def should_task_be_running(start, duration): diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 7b2c8718..3b0ee2ea 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -19,8 +19,10 @@ import os import re from glob import glob -from shutil import copyfile +from shutil import copyfile, copyfileobj from markupsafe import escape +from time import time +from uuid import uuid4 from sqlalchemy.exc import SQLAlchemyError from flask_babel import lazy_gettext as N_ @@ -32,13 +34,15 @@ from cps.subproc_wrapper import process_open from flask_babel import gettext as _ from cps.kobo_sync_status import remove_synced_book from cps.ub import init_db_thread +from cps.file_helper import get_temp_dir from cps.tasks.mail import TaskEmail -from cps import gdriveutils - +from cps import gdriveutils, helper +from cps.constants import SUPPORTED_CALIBRE_BINARIES log = logger.create() +current_milli_time = lambda: int(round(time() * 1000)) class TaskConvert(CalibreTask): def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None): @@ -61,24 +65,33 @@ class TaskConvert(CalibreTask): data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) df = gdriveutils.getFileFromEbooksFolder(cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) + df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") if df: datafile = os.path.join(config.get_book_path(), cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) + if df_cover: + datafile_cover = os.path.join(config.get_book_path(), + cur_book.path, "cover.jpg") if not os.path.exists(os.path.join(config.get_book_path(), cur_book.path)): os.makedirs(os.path.join(config.get_book_path(), cur_book.path)) df.GetContentFile(datafile) + if df_cover: + df_cover.GetContentFile(datafile_cover) worker_db.session.close() else: + # ToDo Include cover in error handling error_message = _("%(format)s not found on Google Drive: %(fn)s", format=self.settings['old_book_format'], fn=data.name + "." + self.settings['old_book_format'].lower()) worker_db.session.close() - return error_message + return self._handleError(self, error_message) filename = self._convert_ebook_format() if config.config_use_google_drive: os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) + if df_cover: + os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) if filename: if config.config_use_google_drive: @@ -112,7 +125,7 @@ class TaskConvert(CalibreTask): # check to see if destination format already exists - or if book is in database # if it does - mark the conversion task as complete and return a success - # this will allow send to E-Reader workflow to continue to work + # this will allow to send to E-Reader workflow to continue to work if os.path.isfile(file_path + format_new_ext) or\ local_db.get_book_format(self.book_id, self.settings['new_book_format']): log.info("Book id %d already converted to %s", book_id, format_new_ext) @@ -152,7 +165,8 @@ class TaskConvert(CalibreTask): if not os.path.exists(config.config_converterpath): self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath)) return - check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext) + has_cover = local_db.get_book(book_id).has_cover + check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover) if check == 0: cur_book = local_db.get_book(book_id) @@ -194,8 +208,15 @@ class TaskConvert(CalibreTask): return def _convert_kepubify(self, file_path, format_old_ext, format_new_ext): + if config.config_embed_metadata: + tmp_dir, temp_file_name = helper.do_calibre_export(self.book_id, format_old_ext[1:]) + filename = os.path.join(tmp_dir, temp_file_name + format_old_ext) + temp_file_path = tmp_dir + else: + filename = file_path + format_old_ext + temp_file_path = os.path.dirname(file_path) quotes = [1, 3] - command = [config.config_kepubifypath, (file_path + format_old_ext), '-o', os.path.dirname(file_path)] + command = [config.config_kepubifypath, filename, '-o', temp_file_path, '-i'] try: p = process_open(command, quotes) except OSError as e: @@ -209,13 +230,12 @@ class TaskConvert(CalibreTask): if p.poll() is not None: break - # ToD Handle # process returncode check = p.returncode # move file if check == 0: - converted_file = glob(os.path.join(os.path.dirname(file_path), "*.kepub.epub")) + converted_file = glob(os.path.splitext(filename)[0] + "*.kepub.epub") if len(converted_file) == 1: copyfile(converted_file[0], (file_path + format_new_ext)) os.unlink(converted_file[0]) @@ -224,16 +244,28 @@ class TaskConvert(CalibreTask): folder=os.path.dirname(file_path)) return check, None - def _convert_calibre(self, file_path, format_old_ext, format_new_ext): + def _convert_calibre(self, file_path, format_old_ext, format_new_ext, has_cover): try: - # Linux py2.7 encode as list without quotes no empty element for parameters - # linux py3.x no encode and as list without quotes no empty element for parameters - # windows py2.7 encode as string with quotes empty element for parameters is okay - # windows py 3.x no encode and as string with quotes empty element for parameters is okay - # separate handling for windows and linux - quotes = [1, 2] + # path_tmp_opf = self._embed_metadata() + if config.config_embed_metadata: + quotes = [3, 5] + tmp_dir = get_temp_dir() + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(self.book_id), + '--with-library', config.config_calibre_dir] + p = process_open(opf_command, quotes) + p.wait() + path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(uuid4()) + ".opf") + with open(path_tmp_opf, 'w') as fd: + copyfileobj(p.stdout, fd) + + quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), (file_path + format_new_ext)] + if config.config_embed_metadata: + command.extend(['--from-opf', path_tmp_opf]) + if has_cover: + command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]) quotes_index = 3 if config.config_calibre: parameters = config.config_calibre.split(" ") diff --git a/cps/tasks/metadata_backup.py b/cps/tasks/metadata_backup.py index 45015ccf..9ca6b830 100644 --- a/cps/tasks/metadata_backup.py +++ b/cps/tasks/metadata_backup.py @@ -17,26 +17,13 @@ # along with this program. If not, see . import os -from urllib.request import urlopen from lxml import etree - from cps import config, db, gdriveutils, logger from cps.services.worker import CalibreTask from flask_babel import lazy_gettext as N_ -OPF_NAMESPACE = "http://www.idpf.org/2007/opf" -PURL_NAMESPACE = "http://purl.org/dc/elements/1.1/" - -OPF = "{%s}" % OPF_NAMESPACE -PURL = "{%s}" % PURL_NAMESPACE - -etree.register_namespace("opf", OPF_NAMESPACE) -etree.register_namespace("dc", PURL_NAMESPACE) - -OPF_NS = {None: OPF_NAMESPACE} # the default namespace (no prefix) -NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE} - +from ..epub_helper import create_new_metadata_backup class TaskBackupMetadata(CalibreTask): @@ -101,7 +88,8 @@ class TaskBackupMetadata(CalibreTask): self.calibre_db.session.close() def open_metadata(self, book, custom_columns): - package = self.create_new_metadata_backup(book, custom_columns) + # package = self.create_new_metadata_backup(book, custom_columns) + package = create_new_metadata_backup(book, custom_columns, self.export_language) if config.config_use_google_drive: if not gdriveutils.is_gdrive_ready(): raise Exception('Google Drive is configured but not ready') @@ -123,7 +111,7 @@ class TaskBackupMetadata(CalibreTask): except Exception as ex: raise Exception('Writing Metadata failed with error: {} '.format(ex)) - def create_new_metadata_backup(self, book, custom_columns): + '''def create_new_metadata_backup(self, book, custom_columns): # generate root package element package = etree.Element(OPF + "package", nsmap=OPF_NS) package.set("unique-identifier", "uuid_id") @@ -208,7 +196,7 @@ class TaskBackupMetadata(CalibreTask): guide = etree.SubElement(package, "guide") etree.SubElement(guide, "reference", type="cover", title=self.translated_title, href="cover.jpg") - return package + return package''' @property def name(self): diff --git a/cps/tasks/tempFolder.py b/cps/tasks/tempFolder.py new file mode 100644 index 00000000..e740cd1e --- /dev/null +++ b/cps/tasks/tempFolder.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2023 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from urllib.request import urlopen + +from flask_babel import lazy_gettext as N_ + +from cps import logger, file_helper +from cps.services.worker import CalibreTask + + +class TaskDeleteTempFolder(CalibreTask): + def __init__(self, task_message=N_('Delete temp folder contents')): + super(TaskDeleteTempFolder, self).__init__(task_message) + self.log = logger.create() + + def run(self, worker_thread): + try: + file_helper.del_temp_dir() + except FileNotFoundError: + pass + except (PermissionError, OSError) as e: + self.log.error("Error deleting temp folder: {}".format(e)) + self._handleSuccess() + + @property + def name(self): + return "Delete Temp Folder" + + @property + def is_cancellable(self): + return False diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index d101f960..d83831db 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -103,6 +103,10 @@ +
+ + +
@@ -323,12 +327,12 @@
- +
- - - - + + + +
diff --git a/cps/updater.py b/cps/updater.py index 6d6e408f..67b3653f 100644 --- a/cps/updater.py +++ b/cps/updater.py @@ -25,13 +25,13 @@ import threading import time import zipfile from io import BytesIO -from tempfile import gettempdir - import requests + from flask_babel import format_datetime from flask_babel import gettext as _ from . import constants, logger # config, web_server +from .file_helper import get_temp_dir log = logger.create() @@ -85,7 +85,7 @@ class Updater(threading.Thread): z = zipfile.ZipFile(BytesIO(r.content)) self.status = 3 log.debug('Extracting zipfile') - tmp_dir = gettempdir() + tmp_dir = get_temp_dir() z.extractall(tmp_dir) folder_name = os.path.join(tmp_dir, z.namelist()[0])[:-1] if not os.path.isdir(folder_name): @@ -566,7 +566,7 @@ class Updater(threading.Thread): try: current_version[2] = int(current_version[2]) except ValueError: - current_version[2] = int(current_version[2].split(' ')[0])-1 + current_version[2] = int(current_version[2].replace("b", "").split(' ')[0])-1 # Check if major versions are identical search for newest non-equal commit and update to this one if major_version_update == current_version[0]: diff --git a/cps/uploader.py b/cps/uploader.py index 23dfc4a6..8f20762f 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -18,12 +18,12 @@ import os import hashlib -from tempfile import gettempdir from flask_babel import gettext as _ from . import logger, comic, isoLanguages from .constants import BookMeta from .helper import split_authors +from .file_helper import get_temp_dir log = logger.create() @@ -249,10 +249,7 @@ def get_magick_version(): def upload(uploadfile, rar_excecutable): - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() filename = uploadfile.filename filename_root, file_extension = os.path.splitext(filename) diff --git a/setup.cfg b/setup.cfg index e73c4663..eb73462d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,7 +66,7 @@ include = cps/services* [options.extras_require] gdrive = - google-api-python-client>=1.7.11,<2.98.0 + google-api-python-client>=1.7.11,<2.108.0 gevent>20.6.0,<24.0.0 greenlet>=0.4.17,<2.1.0 httplib2>=0.9.2,<0.23.0 @@ -79,7 +79,7 @@ gdrive = rsa>=3.4.2,<4.10.0 gmail = google-auth-oauthlib>=0.4.3,<1.1.0 - google-api-python-client>=1.7.11,<2.98.0 + google-api-python-client>=1.7.11,<2.108.0 goodreads = goodreads>=0.3.2,<0.4.0 python-Levenshtein>=0.12.0,<0.22.0 diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 57e3a094..879f65f4 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2023-11-08 21:04:56

+

Start Time: 2023-12-16 20:44:16

-

Stop Time: 2023-11-09 03:52:12

+

Stop Time: 2023-12-17 03:29:30

-

Duration: 5h 45 min

+

Duration: 5h 43 min

@@ -236,13 +236,13 @@ TestBackupMetadata - 22 - 22 + 21 + 21 0 0 0 - Detail + Detail @@ -429,15 +429,6 @@ - -
TestBackupMetadata - test_gdrive
- - PASS - - - - -
TestBackupMetadata - test_upload_book
@@ -1932,13 +1923,13 @@ - + TestLoadMetadata 1 - 0 - 0 1 0 + 0 + 0 Detail @@ -1946,31 +1937,11 @@ - +
TestLoadMetadata - test_load_metadata
- -
- ERROR -
- - - - + PASS @@ -2297,6 +2268,48 @@ IndexError: list index out of range + + TestEmbedMetadata + 3 + 3 + 0 + 0 + 0 + + Detail + + + + + + + +
TestEmbedMetadata - test_convert_file_embed_metadata
+ + PASS + + + + + + +
TestEmbedMetadata - test_download_check_metadata
+ + PASS + + + + + + +
TestEmbedMetadata - test_download_permissions_missing_file
+ + PASS + + + + + TestBookDatabase 1 @@ -2305,13 +2318,13 @@ IndexError: list index out of range 0 0 - Detail + Detail - +
TestBookDatabase - test_invalid_book_path
@@ -2329,13 +2342,13 @@ IndexError: list index out of range 0 0 - Detail + Detail - +
TestErrorReadColumn - test_invalid_custom_column
@@ -2344,7 +2357,7 @@ IndexError: list index out of range - +
TestErrorReadColumn - test_invalid_custom_read_column
@@ -2362,13 +2375,13 @@ IndexError: list index out of range 0 1 - Detail + Detail - +
TestFilePicker - test_filepicker_limited_file
@@ -2377,19 +2390,19 @@ IndexError: list index out of range - +
TestFilePicker - test_filepicker_new_file
- SKIP + SKIP
-