diff --git a/cps/admin.py b/cps/admin.py index 045a9523..51fddbee 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -47,7 +47,7 @@ from . import constants, logger, helper, services, cli_param from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \ kobo_sync_status, schedule from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ - valid_email, check_username + valid_email, check_username, get_calibre_binarypath from .gdriveutils import is_gdrive_ready, gdrive_support from .render_template import render_title_template, get_sidebar_config from .services.worker import WorkerThread @@ -1761,8 +1761,14 @@ def _configuration_update_helper(): constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',') _config_string(to_save, "config_calibre") - _config_string(to_save, "config_converterpath") + _config_string(to_save, "config_binariesdir") _config_string(to_save, "config_kepubifypath") + if "config_binariesdir" in to_save: + calibre_status = helper.check_calibre(config.config_binariesdir) + if calibre_status: + return _configuration_result(calibre_status) + to_save["config_converterpath"] = get_calibre_binarypath("ebook-convert") + _config_string(to_save, "config_converterpath") reboot_required |= _config_int(to_save, "config_login_type") diff --git a/cps/config_sql.py b/cps/config_sql.py index 21644ccd..485c3fc2 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -34,6 +34,7 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from . import constants, logger +from .subproc_wrapper import process_wait log = logger.create() @@ -138,6 +139,7 @@ class _Settings(_Base): config_kepubifypath = Column(String, default=None) config_converterpath = Column(String, default=None) + config_binariesdir = Column(String, default=None) config_calibre = Column(String) config_rarfile_location = Column(String, default=None) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) @@ -184,9 +186,11 @@ class ConfigSQL(object): self.load() change = False - if self.config_converterpath == None: # pylint: disable=access-member-before-definition + + if self.config_binariesdir == None: # pylint: disable=access-member-before-definition change = True - self.config_converterpath = autodetect_calibre_binary() + self.config_binariesdir = autodetect_calibre_binaries() + self.config_converterpath = autodetect_converter_binary(self.config_binariesdir) if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition change = True @@ -469,17 +473,32 @@ def _migrate_table(session, orm_class, secret_key=None): session.rollback() -def autodetect_calibre_binary(): +def autodetect_calibre_binaries(): if sys.platform == "win32": - calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre2\\ebook-convert.exe", - "C:\\program files\\calibre2\\ebook-convert.exe"] + calibre_path = ["C:\\program files\\calibre\\", + "C:\\program files(x86)\\calibre\\", + "C:\\program files(x86)\\calibre2\\", + "C:\\program files\\calibre2\\"] else: - calibre_path = ["/opt/calibre/ebook-convert"] + calibre_path = ["/opt/calibre/"] for element in calibre_path: - if os.path.isfile(element) and os.access(element, os.X_OK): - return element + supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()] + if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + return element + return "" + + +def autodetect_converter_binary(calibre_path): + if sys.platform == "win32": + converter_path = os.path.join(calibre_path, "ebook-convert.exe") + else: + converter_path = os.path.join(calibre_path, "ebook-convert") + if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): + return converter_path return "" @@ -521,6 +540,7 @@ def load_configuration(session, secret_key): session.commit() + def get_flask_session_key(_session): flask_settings = _session.query(_Flask_Settings).one_or_none() if flask_settings == None: diff --git a/cps/constants.py b/cps/constants.py index 18c4f1b1..d8842e78 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -156,6 +156,11 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' 'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac', 'm4a', 'm4b'} +_extension = "" +if sys.platform == "win32": + _extension = ".exe" +SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]} + def has_flag(value, bit_flag): return bit_flag == (bit_flag & (value or 0)) diff --git a/cps/helper.py b/cps/helper.py index 0c526d01..cabc0363 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -54,8 +54,8 @@ from . import calibre_db, cli_param from .tasks.convert import TaskConvert from . import logger, config, db, ub, fs from . import gdriveutils as gd -from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES -from .subproc_wrapper import process_wait +from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES +from .subproc_wrapper import process_wait, process_open from .services.worker import WorkerThread from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails @@ -938,9 +938,10 @@ def save_cover(img, book_path): def do_download_file(book, book_format, client, data, headers): + book_name = data.name if config.config_use_google_drive: # startTime = time.time() - df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) + df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: return gd.do_gdrive_download(df, headers) @@ -948,20 +949,47 @@ def do_download_file(book, book_format, client, data, headers): abort(404) else: filename = os.path.join(config.config_calibre_dir, book.path) - if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)): + if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)): # ToDo: improve error handling - log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format)) + log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format)) if client == "kobo" and book_format == "kepub": headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") - response = make_response(send_from_directory(filename, data.name + "." + book_format)) + if config.config_binariesdir: + filename, book_name = do_calibre_export(book, book_format) + + response = make_response(send_from_directory(filename, book_name + "." + book_format)) # ToDo Check headers parameter for element in headers: response.headers[element[0]] = element[1] - log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format))) + log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format))) return response + +def do_calibre_export(book, book_format): + try: + quotes = [3, 5, 7, 9] + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + calibredb_binarypath = get_calibre_binarypath("calibredb") + opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), + '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, + '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] + file_name = book.title + if len(book.authors) > 0: + file_name = file_name + ' - ' + book.authors[0].name + p = process_open(opf_command, quotes) + _, err = p.communicate() + if err: + log.error('Metadata embedder encountered an error: %s', err) + return tmp_dir, file_name + except OSError as ex: + # ToDo real error handling + log.error_or_exception(ex) + + ################################## @@ -984,6 +1012,35 @@ def check_unrar(unrar_location): return _('Error executing UnRar') +def check_calibre(calibre_location): + if not calibre_location: + return + + if not os.path.exists(calibre_location): + return _('Could not find the specified directory') + + if not os.path.isdir(calibre_location): + return _('Please specify a directory, not a file') + + try: + supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES.values()] + binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] + if all(binaries_available): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + else: + return _('Calibre binaries not viable') + else: + missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] + return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)) + + except (OSError, UnicodeDecodeError) as err: + log.error_or_exception(err) + return _('Error excecuting Calibre') + + def json_serial(obj): """JSON serializer for objects not serializable by default json code""" @@ -1047,6 +1104,17 @@ def get_download_link(book_id, book_format, client): abort(404) +def get_calibre_binarypath(binary): + binariesdir = config.config_binariesdir + if binariesdir: + try: + return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary]) + except KeyError as ex: + log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary]) + pass + return "" + + def clear_cover_thumbnail_cache(book_id): if config.schedule_generate_book_covers: WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index df6ae104..2bef9a20 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -19,8 +19,10 @@ import os import re from glob import glob -from shutil import copyfile +from shutil import copyfile, copyfileobj from markupsafe import escape +from tempfile import gettempdir +from time import time from sqlalchemy.exc import SQLAlchemyError from flask_babel import lazy_gettext as N_ @@ -35,10 +37,11 @@ from cps.ub import init_db_thread from cps.tasks.mail import TaskEmail from cps import gdriveutils - +from cps.constants import SUPPORTED_CALIBRE_BINARIES log = logger.create() +current_milli_time = lambda: int(round(time() * 1000)) class TaskConvert(CalibreTask): def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None): @@ -61,15 +64,20 @@ class TaskConvert(CalibreTask): data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) df = gdriveutils.getFileFromEbooksFolder(cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) - if df: + df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") + if df and df_cover: datafile = os.path.join(config.config_calibre_dir, cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) + datafile_cover = os.path.join(config.config_calibre_dir, + cur_book.path, "cover.jpg") if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)): os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path)) df.GetContentFile(datafile) + df_cover.GetContentFile(datafile_cover) worker_db.session.close() else: + # ToDo Include cover in error handling error_message = _("%(format)s not found on Google Drive: %(fn)s", format=self.settings['old_book_format'], fn=data.name + "." + self.settings['old_book_format'].lower()) @@ -79,6 +87,7 @@ class TaskConvert(CalibreTask): filename = self._convert_ebook_format() if config.config_use_google_drive: os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) + os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) if filename: if config.config_use_google_drive: @@ -225,15 +234,30 @@ class TaskConvert(CalibreTask): return check, None def _convert_calibre(self, file_path, format_old_ext, format_new_ext): + book_id = self.book_id try: # Linux py2.7 encode as list without quotes no empty element for parameters # linux py3.x no encode and as list without quotes no empty element for parameters # windows py2.7 encode as string with quotes empty element for parameters is okay # windows py 3.x no encode and as string with quotes empty element for parameters is okay # separate handling for windows and linux - quotes = [1, 2] + + quotes = [3, 5] + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] + p = process_open(opf_command, quotes) + p.wait() + path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") + with open(path_tmp_opf, 'w') as fd: + copyfileobj(p.stdout, fd) + + quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext)] + (file_path + format_new_ext), '--from-opf', path_tmp_opf, + '--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')] quotes_index = 3 if config.config_calibre: parameters = config.config_calibre.split(" ") diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index d101f960..2ec0575c 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -323,12 +323,12 @@
- +
- - - - + + + +