Add metadata change code

This commit is contained in:
Ozzie Isaacs 2023-11-02 17:05:02 +01:00
parent b2e4907165
commit b7aaa0f24d
6 changed files with 152 additions and 29 deletions

View File

@ -47,7 +47,7 @@ from . import constants, logger, helper, services, cli_param
from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \ from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \
kobo_sync_status, schedule kobo_sync_status, schedule
from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \
valid_email, check_username valid_email, check_username, get_calibre_binarypath
from .gdriveutils import is_gdrive_ready, gdrive_support from .gdriveutils import is_gdrive_ready, gdrive_support
from .render_template import render_title_template, get_sidebar_config from .render_template import render_title_template, get_sidebar_config
from .services.worker import WorkerThread from .services.worker import WorkerThread
@ -1761,8 +1761,14 @@ def _configuration_update_helper():
constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',') constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',')
_config_string(to_save, "config_calibre") _config_string(to_save, "config_calibre")
_config_string(to_save, "config_converterpath") _config_string(to_save, "config_binariesdir")
_config_string(to_save, "config_kepubifypath") _config_string(to_save, "config_kepubifypath")
if "config_binariesdir" in to_save:
calibre_status = helper.check_calibre(config.config_binariesdir)
if calibre_status:
return _configuration_result(calibre_status)
to_save["config_converterpath"] = get_calibre_binarypath("ebook-convert")
_config_string(to_save, "config_converterpath")
reboot_required |= _config_int(to_save, "config_login_type") reboot_required |= _config_int(to_save, "config_login_type")

View File

@ -34,6 +34,7 @@ except ImportError:
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from . import constants, logger from . import constants, logger
from .subproc_wrapper import process_wait
log = logger.create() log = logger.create()
@ -138,6 +139,7 @@ class _Settings(_Base):
config_kepubifypath = Column(String, default=None) config_kepubifypath = Column(String, default=None)
config_converterpath = Column(String, default=None) config_converterpath = Column(String, default=None)
config_binariesdir = Column(String, default=None)
config_calibre = Column(String) config_calibre = Column(String)
config_rarfile_location = Column(String, default=None) config_rarfile_location = Column(String, default=None)
config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD))
@ -184,9 +186,11 @@ class ConfigSQL(object):
self.load() self.load()
change = False change = False
if self.config_converterpath == None: # pylint: disable=access-member-before-definition
if self.config_binariesdir == None: # pylint: disable=access-member-before-definition
change = True change = True
self.config_converterpath = autodetect_calibre_binary() self.config_binariesdir = autodetect_calibre_binaries()
self.config_converterpath = autodetect_converter_binary(self.config_binariesdir)
if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition
change = True change = True
@ -469,17 +473,32 @@ def _migrate_table(session, orm_class, secret_key=None):
session.rollback() session.rollback()
def autodetect_calibre_binary(): def autodetect_calibre_binaries():
if sys.platform == "win32": if sys.platform == "win32":
calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe", calibre_path = ["C:\\program files\\calibre\\",
"C:\\program files(x86)\\calibre\\ebook-convert.exe", "C:\\program files(x86)\\calibre\\",
"C:\\program files(x86)\\calibre2\\ebook-convert.exe", "C:\\program files(x86)\\calibre2\\",
"C:\\program files\\calibre2\\ebook-convert.exe"] "C:\\program files\\calibre2\\"]
else: else:
calibre_path = ["/opt/calibre/ebook-convert"] calibre_path = ["/opt/calibre/"]
for element in calibre_path: for element in calibre_path:
if os.path.isfile(element) and os.access(element, os.X_OK): supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()]
return element if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths):
values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths]
if all(values):
version = values[0].group(1)
log.debug("calibre version %s", version)
return element
return ""
def autodetect_converter_binary(calibre_path):
if sys.platform == "win32":
converter_path = os.path.join(calibre_path, "ebook-convert.exe")
else:
converter_path = os.path.join(calibre_path, "ebook-convert")
if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK):
return converter_path
return "" return ""
@ -521,6 +540,7 @@ def load_configuration(session, secret_key):
session.commit() session.commit()
def get_flask_session_key(_session): def get_flask_session_key(_session):
flask_settings = _session.query(_Flask_Settings).one_or_none() flask_settings = _session.query(_Flask_Settings).one_or_none()
if flask_settings == None: if flask_settings == None:

View File

@ -156,6 +156,11 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr'
'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg', 'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg',
'opus', 'wav', 'flac', 'm4a', 'm4b'} 'opus', 'wav', 'flac', 'm4a', 'm4b'}
_extension = ""
if sys.platform == "win32":
_extension = ".exe"
SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]}
def has_flag(value, bit_flag): def has_flag(value, bit_flag):
return bit_flag == (bit_flag & (value or 0)) return bit_flag == (bit_flag & (value or 0))

View File

@ -54,8 +54,8 @@ from . import calibre_db, cli_param
from .tasks.convert import TaskConvert from .tasks.convert import TaskConvert
from . import logger, config, db, ub, fs from . import logger, config, db, ub, fs
from . import gdriveutils as gd from . import gdriveutils as gd
from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES
from .subproc_wrapper import process_wait from .subproc_wrapper import process_wait, process_open
from .services.worker import WorkerThread from .services.worker import WorkerThread
from .tasks.mail import TaskEmail from .tasks.mail import TaskEmail
from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails
@ -938,9 +938,10 @@ def save_cover(img, book_path):
def do_download_file(book, book_format, client, data, headers): def do_download_file(book, book_format, client, data, headers):
book_name = data.name
if config.config_use_google_drive: if config.config_use_google_drive:
# startTime = time.time() # startTime = time.time()
df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format)
# log.debug('%s', time.time() - startTime) # log.debug('%s', time.time() - startTime)
if df: if df:
return gd.do_gdrive_download(df, headers) return gd.do_gdrive_download(df, headers)
@ -948,20 +949,47 @@ def do_download_file(book, book_format, client, data, headers):
abort(404) abort(404)
else: else:
filename = os.path.join(config.config_calibre_dir, book.path) filename = os.path.join(config.config_calibre_dir, book.path)
if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)): if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)):
# ToDo: improve error handling # ToDo: improve error handling
log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format)) log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format))
if client == "kobo" and book_format == "kepub": if client == "kobo" and book_format == "kepub":
headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub")
response = make_response(send_from_directory(filename, data.name + "." + book_format)) if config.config_binariesdir:
filename, book_name = do_calibre_export(book, book_format)
response = make_response(send_from_directory(filename, book_name + "." + book_format))
# ToDo Check headers parameter # ToDo Check headers parameter
for element in headers: for element in headers:
response.headers[element[0]] = element[1] response.headers[element[0]] = element[1]
log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format))) log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format)))
return response return response
def do_calibre_export(book, book_format):
try:
quotes = [3, 5, 7, 9]
tmp_dir = os.path.join(gettempdir(), 'calibre_web')
if not os.path.isdir(tmp_dir):
os.mkdir(tmp_dir)
calibredb_binarypath = get_calibre_binarypath("calibredb")
opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id),
'--with-library', config.config_calibre_dir, '--to-dir', tmp_dir,
'--formats', book_format, "--template", "{} - {{authors}}".format(book.title)]
file_name = book.title
if len(book.authors) > 0:
file_name = file_name + ' - ' + book.authors[0].name
p = process_open(opf_command, quotes)
_, err = p.communicate()
if err:
log.error('Metadata embedder encountered an error: %s', err)
return tmp_dir, file_name
except OSError as ex:
# ToDo real error handling
log.error_or_exception(ex)
################################## ##################################
@ -984,6 +1012,35 @@ def check_unrar(unrar_location):
return _('Error executing UnRar') return _('Error executing UnRar')
def check_calibre(calibre_location):
if not calibre_location:
return
if not os.path.exists(calibre_location):
return _('Could not find the specified directory')
if not os.path.isdir(calibre_location):
return _('Please specify a directory, not a file')
try:
supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES.values()]
binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths]
if all(binaries_available):
values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths]
if all(values):
version = values[0].group(1)
log.debug("calibre version %s", version)
else:
return _('Calibre binaries not viable')
else:
missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available]
return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries))
except (OSError, UnicodeDecodeError) as err:
log.error_or_exception(err)
return _('Error excecuting Calibre')
def json_serial(obj): def json_serial(obj):
"""JSON serializer for objects not serializable by default json code""" """JSON serializer for objects not serializable by default json code"""
@ -1047,6 +1104,17 @@ def get_download_link(book_id, book_format, client):
abort(404) abort(404)
def get_calibre_binarypath(binary):
binariesdir = config.config_binariesdir
if binariesdir:
try:
return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary])
except KeyError as ex:
log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary])
pass
return ""
def clear_cover_thumbnail_cache(book_id): def clear_cover_thumbnail_cache(book_id):
if config.schedule_generate_book_covers: if config.schedule_generate_book_covers:
WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True) WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True)

View File

@ -19,8 +19,10 @@
import os import os
import re import re
from glob import glob from glob import glob
from shutil import copyfile from shutil import copyfile, copyfileobj
from markupsafe import escape from markupsafe import escape
from tempfile import gettempdir
from time import time
from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.exc import SQLAlchemyError
from flask_babel import lazy_gettext as N_ from flask_babel import lazy_gettext as N_
@ -35,10 +37,11 @@ from cps.ub import init_db_thread
from cps.tasks.mail import TaskEmail from cps.tasks.mail import TaskEmail
from cps import gdriveutils from cps import gdriveutils
from cps.constants import SUPPORTED_CALIBRE_BINARIES
log = logger.create() log = logger.create()
current_milli_time = lambda: int(round(time() * 1000))
class TaskConvert(CalibreTask): class TaskConvert(CalibreTask):
def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None): def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None):
@ -61,15 +64,20 @@ class TaskConvert(CalibreTask):
data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) data = worker_db.get_book_format(self.book_id, self.settings['old_book_format'])
df = gdriveutils.getFileFromEbooksFolder(cur_book.path, df = gdriveutils.getFileFromEbooksFolder(cur_book.path,
data.name + "." + self.settings['old_book_format'].lower()) data.name + "." + self.settings['old_book_format'].lower())
if df: df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg")
if df and df_cover:
datafile = os.path.join(config.config_calibre_dir, datafile = os.path.join(config.config_calibre_dir,
cur_book.path, cur_book.path,
data.name + "." + self.settings['old_book_format'].lower()) data.name + "." + self.settings['old_book_format'].lower())
datafile_cover = os.path.join(config.config_calibre_dir,
cur_book.path, "cover.jpg")
if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)): if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)):
os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path)) os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path))
df.GetContentFile(datafile) df.GetContentFile(datafile)
df_cover.GetContentFile(datafile_cover)
worker_db.session.close() worker_db.session.close()
else: else:
# ToDo Include cover in error handling
error_message = _("%(format)s not found on Google Drive: %(fn)s", error_message = _("%(format)s not found on Google Drive: %(fn)s",
format=self.settings['old_book_format'], format=self.settings['old_book_format'],
fn=data.name + "." + self.settings['old_book_format'].lower()) fn=data.name + "." + self.settings['old_book_format'].lower())
@ -79,6 +87,7 @@ class TaskConvert(CalibreTask):
filename = self._convert_ebook_format() filename = self._convert_ebook_format()
if config.config_use_google_drive: if config.config_use_google_drive:
os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) os.remove(self.file_path + '.' + self.settings['old_book_format'].lower())
os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg"))
if filename: if filename:
if config.config_use_google_drive: if config.config_use_google_drive:
@ -225,15 +234,30 @@ class TaskConvert(CalibreTask):
return check, None return check, None
def _convert_calibre(self, file_path, format_old_ext, format_new_ext): def _convert_calibre(self, file_path, format_old_ext, format_new_ext):
book_id = self.book_id
try: try:
# Linux py2.7 encode as list without quotes no empty element for parameters # Linux py2.7 encode as list without quotes no empty element for parameters
# linux py3.x no encode and as list without quotes no empty element for parameters # linux py3.x no encode and as list without quotes no empty element for parameters
# windows py2.7 encode as string with quotes empty element for parameters is okay # windows py2.7 encode as string with quotes empty element for parameters is okay
# windows py 3.x no encode and as string with quotes empty element for parameters is okay # windows py 3.x no encode and as string with quotes empty element for parameters is okay
# separate handling for windows and linux # separate handling for windows and linux
quotes = [1, 2]
quotes = [3, 5]
tmp_dir = os.path.join(gettempdir(), 'calibre_web')
if not os.path.isdir(tmp_dir):
os.mkdir(tmp_dir)
calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"])
opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir]
p = process_open(opf_command, quotes)
p.wait()
path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf")
with open(path_tmp_opf, 'w') as fd:
copyfileobj(p.stdout, fd)
quotes = [1, 2, 4, 6]
command = [config.config_converterpath, (file_path + format_old_ext), command = [config.config_converterpath, (file_path + format_old_ext),
(file_path + format_new_ext)] (file_path + format_new_ext), '--from-opf', path_tmp_opf,
'--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]
quotes_index = 3 quotes_index = 3
if config.config_calibre: if config.config_calibre:
parameters = config.config_calibre.split(" ") parameters = config.config_calibre.split(" ")

View File

@ -323,12 +323,12 @@
</div> </div>
<div id="collapsefive" class="panel-collapse collapse"> <div id="collapsefive" class="panel-collapse collapse">
<div class="panel-body"> <div class="panel-body">
<label for="config_converterpath">{{_('Path to Calibre E-Book Converter')}}</label> <label for="config_binariesdir">{{_('Path to Calibre Binaries')}}</label>
<div class="form-group input-group"> <div class="form-group input-group">
<input type="text" class="form-control" id="config_converterpath" name="config_converterpath" value="{% if config.config_converterpath != None %}{{ config.config_converterpath }}{% endif %}" autocomplete="off"> <input type="text" class="form-control" id="config_binariesdir" name="config_binariesdir" value="{% if config.config_binariesdir != None %}{{ config.config_binariesdir }}{% endif %}" autocomplete="off">
<span class="input-group-btn"> <span class="input-group-btn">
<button type="button" data-toggle="modal" id="converter_modal_path" data-link="config_converterpath" data-target="#fileModal" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button> <button type="button" data-toggle="modal" id="binaries_modal_path" data-link="config_binariesdir" data-target="#fileModal" class="btn btn-default"><span class="glyphicon glyphicon-folder-open"></span></button>
</span> </span>
</div> </div>
<div class="form-group"> <div class="form-group">
<label for="config_calibre">{{_('Calibre E-Book Converter Settings')}}</label> <label for="config_calibre">{{_('Calibre E-Book Converter Settings')}}</label>