Add metadata change code

cps/admin.py

@@ -47,7 +47,7 @@ from . import constants, logger, helper, services, cli_param
 from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \
     kobo_sync_status, schedule
 from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \
-    valid_email, check_username
+    valid_email, check_username, get_calibre_binarypath
 from .gdriveutils import is_gdrive_ready, gdrive_support
 from .render_template import render_title_template, get_sidebar_config
 from .services.worker import WorkerThread
@@ -1761,8 +1761,14 @@ def _configuration_update_helper():
             constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',')
 
         _config_string(to_save, "config_calibre")
-        _config_string(to_save, "config_converterpath")
+        _config_string(to_save, "config_binariesdir")
         _config_string(to_save, "config_kepubifypath")
+        if "config_binariesdir" in to_save:
+            calibre_status = helper.check_calibre(config.config_binariesdir)
+            if calibre_status:
+                return _configuration_result(calibre_status)
+            to_save["config_converterpath"] = get_calibre_binarypath("ebook-convert")
+            _config_string(to_save, "config_converterpath")
 
         reboot_required |= _config_int(to_save, "config_login_type")
 

cps/config_sql.py

@@ -34,6 +34,7 @@ except ImportError:
     from sqlalchemy.ext.declarative import declarative_base
 
 from . import constants, logger
+from .subproc_wrapper import process_wait
 
 
 log = logger.create()
@@ -138,6 +139,7 @@ class _Settings(_Base):
 
     config_kepubifypath = Column(String, default=None)
     config_converterpath = Column(String, default=None)
+    config_binariesdir = Column(String, default=None)
     config_calibre = Column(String)
     config_rarfile_location = Column(String, default=None)
     config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD))
@@ -184,9 +186,11 @@ class ConfigSQL(object):
         self.load()
 
         change = False
-        if self.config_converterpath == None:  # pylint: disable=access-member-before-definition
+
+        if self.config_binariesdir == None: # pylint: disable=access-member-before-definition
             change = True
-            self.config_converterpath = autodetect_calibre_binary()
+            self.config_binariesdir = autodetect_calibre_binaries()
+            self.config_converterpath = autodetect_converter_binary(self.config_binariesdir)
 
         if self.config_kepubifypath == None:  # pylint: disable=access-member-before-definition
             change = True
@@ -469,20 +473,35 @@ def _migrate_table(session, orm_class, secret_key=None):
             session.rollback()
 
 
-def autodetect_calibre_binary():
+def autodetect_calibre_binaries():
     if sys.platform == "win32":
-        calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe",
+        calibre_path = ["C:\\program files\\calibre\\",
-                        "C:\\program files(x86)\\calibre\\ebook-convert.exe",
+                        "C:\\program files(x86)\\calibre\\",
-                        "C:\\program files(x86)\\calibre2\\ebook-convert.exe",
+                        "C:\\program files(x86)\\calibre2\\",
-                        "C:\\program files\\calibre2\\ebook-convert.exe"]
+                        "C:\\program files\\calibre2\\"]
     else:
-        calibre_path = ["/opt/calibre/ebook-convert"]
+        calibre_path = ["/opt/calibre/"]
     for element in calibre_path:
-        if os.path.isfile(element) and os.access(element, os.X_OK):
+        supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()]
+        if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths):
+            values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths]
+            if all(values):
+                version = values[0].group(1)
+                log.debug("calibre version %s", version)
                 return element 
     return ""
 
 
+def autodetect_converter_binary(calibre_path):
+    if sys.platform == "win32":
+        converter_path = os.path.join(calibre_path, "ebook-convert.exe")
+    else:
+        converter_path = os.path.join(calibre_path, "ebook-convert")
+    if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK):
+        return converter_path
+    return ""
+
+
 def autodetect_unrar_binary():
     if sys.platform == "win32":
         calibre_path = ["C:\\program files\\WinRar\\unRAR.exe",
@@ -521,6 +540,7 @@ def load_configuration(session, secret_key):
         session.commit()
 
 
+
 def get_flask_session_key(_session):
     flask_settings = _session.query(_Flask_Settings).one_or_none()
     if flask_settings == None:

cps/constants.py

@@ -156,6 +156,11 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr'
                      'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg',
                      'opus', 'wav', 'flac', 'm4a', 'm4b'}
 
+_extension = ""
+if sys.platform == "win32":
+    _extension = ".exe"
+SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]}
+
 
 def has_flag(value, bit_flag):
     return bit_flag == (bit_flag & (value or 0))

cps/helper.py

@@ -54,8 +54,8 @@ from . import calibre_db, cli_param
 from .tasks.convert import TaskConvert
 from . import logger, config, db, ub, fs
 from . import gdriveutils as gd
-from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES
+from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES
-from .subproc_wrapper import process_wait
+from .subproc_wrapper import process_wait, process_open
 from .services.worker import WorkerThread
 from .tasks.mail import TaskEmail
 from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails
@@ -938,9 +938,10 @@ def save_cover(img, book_path):
 
 
 def do_download_file(book, book_format, client, data, headers):
+    book_name = data.name
     if config.config_use_google_drive:
         # startTime = time.time()
-        df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format)
+        df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format)
         # log.debug('%s', time.time() - startTime)
         if df:
             return gd.do_gdrive_download(df, headers)
@@ -948,20 +949,47 @@ def do_download_file(book, book_format, client, data, headers):
             abort(404)
     else:
         filename = os.path.join(config.config_calibre_dir, book.path)
-        if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)):
+        if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)):
             # ToDo: improve error handling
-            log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format))
+            log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format))
 
         if client == "kobo" and book_format == "kepub":
             headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub")
 
-        response = make_response(send_from_directory(filename, data.name + "." + book_format))
+        if config.config_binariesdir:
+            filename, book_name = do_calibre_export(book, book_format)
+
+        response = make_response(send_from_directory(filename, book_name + "." + book_format))
         # ToDo Check headers parameter
         for element in headers:
             response.headers[element[0]] = element[1]
-        log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format)))
+        log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format)))
         return response
 
+
+def do_calibre_export(book, book_format):
+    try:
+        quotes = [3, 5, 7, 9]
+        tmp_dir = os.path.join(gettempdir(), 'calibre_web')
+        if not os.path.isdir(tmp_dir):
+            os.mkdir(tmp_dir)
+        calibredb_binarypath = get_calibre_binarypath("calibredb")
+        opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), 
+                    '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir,
+                    '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)]
+        file_name = book.title
+        if len(book.authors) > 0:
+            file_name = file_name + ' - ' + book.authors[0].name
+        p = process_open(opf_command, quotes)
+        _, err = p.communicate()
+        if err:
+            log.error('Metadata embedder encountered an error: %s', err)
+        return tmp_dir, file_name
+    except OSError as ex:
+        # ToDo real error handling
+        log.error_or_exception(ex)
+
+
 ##################################
 
 
@@ -984,6 +1012,35 @@ def check_unrar(unrar_location):
         return _('Error executing UnRar')
 
 
+def check_calibre(calibre_location):
+    if not calibre_location:
+        return
+
+    if not os.path.exists(calibre_location):
+        return _('Could not find the specified directory')
+
+    if not os.path.isdir(calibre_location):
+        return _('Please specify a directory, not a file')
+
+    try:
+        supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES.values()]
+        binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths]
+        if all(binaries_available):
+            values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths]
+            if all(values):
+                version = values[0].group(1)
+                log.debug("calibre version %s", version)
+            else:
+                return _('Calibre binaries not viable')
+        else:
+            missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available]
+            return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries))
+
+    except (OSError, UnicodeDecodeError) as err:
+        log.error_or_exception(err)
+        return _('Error excecuting Calibre')
+
+
 def json_serial(obj):
     """JSON serializer for objects not serializable by default json code"""
 
@@ -1047,6 +1104,17 @@ def get_download_link(book_id, book_format, client):
         abort(404)
 
 
+def get_calibre_binarypath(binary):
+        binariesdir = config.config_binariesdir
+        if binariesdir:
+            try:
+                return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary])
+            except KeyError as ex:
+                log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary])
+                pass
+        return ""
+
+
 def clear_cover_thumbnail_cache(book_id):
     if config.schedule_generate_book_covers:
         WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True)

cps/tasks/convert.py

@@ -19,8 +19,10 @@
 import os
 import re
 from glob import glob
-from shutil import copyfile
+from shutil import copyfile, copyfileobj
 from markupsafe import escape
+from tempfile import gettempdir
+from time import time
 
 from sqlalchemy.exc import SQLAlchemyError
 from flask_babel import lazy_gettext as N_
@@ -35,10 +37,11 @@ from cps.ub import init_db_thread
 
 from cps.tasks.mail import TaskEmail
 from cps import gdriveutils
-
+from cps.constants import SUPPORTED_CALIBRE_BINARIES
 
 log = logger.create()
 
+current_milli_time = lambda: int(round(time() * 1000))
 
 class TaskConvert(CalibreTask):
     def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None):
@@ -61,15 +64,20 @@ class TaskConvert(CalibreTask):
             data = worker_db.get_book_format(self.book_id, self.settings['old_book_format'])
             df = gdriveutils.getFileFromEbooksFolder(cur_book.path,
                                                      data.name + "." + self.settings['old_book_format'].lower())
-            if df:
+            df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg")
+            if df and df_cover:
                 datafile = os.path.join(config.config_calibre_dir,
                                         cur_book.path,
                                         data.name + "." + self.settings['old_book_format'].lower())
+                datafile_cover = os.path.join(config.config_calibre_dir,
+                                        cur_book.path, "cover.jpg")
                 if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)):
                     os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path))
                 df.GetContentFile(datafile)
+                df_cover.GetContentFile(datafile_cover)
                 worker_db.session.close()
             else:
+            	# ToDo Include cover in error handling 
                 error_message = _("%(format)s not found on Google Drive: %(fn)s",
                                   format=self.settings['old_book_format'],
                                   fn=data.name + "." + self.settings['old_book_format'].lower())
@@ -79,6 +87,7 @@ class TaskConvert(CalibreTask):
         filename = self._convert_ebook_format()
         if config.config_use_google_drive:
             os.remove(self.file_path + '.' + self.settings['old_book_format'].lower())
+            os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg"))
 
         if filename:
             if config.config_use_google_drive:
@@ -225,15 +234,30 @@ class TaskConvert(CalibreTask):
         return check, None
 
     def _convert_calibre(self, file_path, format_old_ext, format_new_ext):
+        book_id = self.book_id
         try:
             # Linux py2.7 encode as list without quotes no empty element for parameters
             # linux py3.x no encode and as list without quotes no empty element for parameters
             # windows py2.7 encode as string with quotes empty element for parameters is okay
             # windows py 3.x no encode and as string with quotes empty element for parameters is okay
             # separate handling for windows and linux
-            quotes = [1, 2]
+
+            quotes = [3, 5]
+            tmp_dir = os.path.join(gettempdir(), 'calibre_web')
+            if not os.path.isdir(tmp_dir):
+                os.mkdir(tmp_dir)
+            calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"])
+            opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir]
+            p = process_open(opf_command, quotes)
+            p.wait()
+            path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf")
+            with open(path_tmp_opf, 'w') as fd:
+                copyfileobj(p.stdout, fd)
+
+            quotes = [1, 2, 4, 6]
             command = [config.config_converterpath, (file_path + format_old_ext),
-                       (file_path + format_new_ext)]
+                       (file_path + format_new_ext), '--from-opf', path_tmp_opf,
+                       '--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]
             quotes_index = 3
             if config.config_calibre:
                 parameters = config.config_calibre.split(" ")

cps/templates/config_edit.html

@@ -323,11 +323,11 @@
     </div>
     <div id="collapsefive" class="panel-collapse collapse">
       <div class="panel-body">
-           <label for="config_converterpath">{{_('Path to Calibre E-Book Converter')}}</label>
+           <label for="config_binariesdir">{{_('Path to Calibre Binaries')}}</label>
            <div class="form-group input-group">
-            <input type="text" class="form-control" id="config_converterpath" name="config_converterpath" value="{% if config.config_converterpath != None %}{{ config.config_converterpath }}{% endif %}" autocomplete="off">
+           <input type="text" class="form-control" id="config_binariesdir" name="config_binariesdir" value="{% if config.config_binariesdir != None %}{{ config.config_binariesdir }}{% endif %}" autocomplete="off">
            <span class="input-group-btn">
-              <button type="button" data-toggle="modal" id="converter_modal_path" data-link="config_converterpath" data-target="#fileModal" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
+             <button type="button" data-toggle="modal" id="binaries_modal_path" data-link="config_binariesdir" data-target="#fileModal" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
            </span>
            </div>
            <div class="form-group">