diff --git a/cps/admin.py b/cps/admin.py index 1a1b1389c..7bddc1bd1 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -23,6 +23,7 @@ import os import re import json +import ipaddress import operator import time import sys @@ -50,7 +51,9 @@ from .helper import check_valid_domain, send_test_mail, reset_password, generate valid_email, check_username from .embed_helper import get_calibre_binarypath from .gdriveutils import is_gdrive_ready, gdrive_support +from .binary_helper import resolve_binary_path, SUPPORTED_KEPUBIFY_BINARIES, SUPPORTED_UNRAR_BINARIES from .render_template import render_title_template, get_sidebar_config +from .reverse_proxy_auth import is_valid_header_name from .services.worker import WorkerThread from .usermanagement import user_login_required from .cw_babel import get_available_translations, get_available_locale, get_user_locale_language @@ -1121,6 +1124,31 @@ def _config_string(to_save, x): return config.set_from_dictionary(to_save, x, lambda y: strip_whitespaces(y) if y else y) +def _validate_reverse_proxy_trusted_ips(value): + normalized_values = [] + for entry in value.split(","): + entry = strip_whitespaces(entry) + if not entry: + continue + try: + network = ipaddress.ip_network(entry, strict=False) + except ValueError: + return None, _configuration_result( + _('Invalid reverse proxy trusted IP/CIDR entry: %(entry)s', entry=entry) + ) + if "/" in entry: + normalized_values.append(str(network)) + else: + normalized_values.append(str(network.network_address)) + return ",".join(normalized_values), None + + +def _validate_reverse_proxy_header_name(value, label): + if value and not is_valid_header_name(value): + return _configuration_result(_('Invalid %(label)s: %(value)s', label=label, value=value)) + return None + + def _configuration_gdrive_helper(to_save): gdrive_error = None if to_save.get("config_use_google_drive"): @@ -1811,6 +1839,13 @@ def _configuration_update_helper(): _config_string(to_save, "config_calibre") _config_string(to_save, "config_binariesdir") _config_string(to_save, "config_kepubifypath") + if "config_kepubifypath" in to_save: + if config.config_kepubifypath: + kepubify_binary = resolve_binary_path(config.config_kepubifypath, SUPPORTED_KEPUBIFY_BINARIES) + if not kepubify_binary: + return _configuration_result(_('Kepubify binary not found')) + config.config_kepubifypath = os.path.dirname(kepubify_binary) + if "config_binariesdir" in to_save: calibre_status = helper.check_calibre(config.config_binariesdir) if calibre_status: @@ -1846,7 +1881,46 @@ def _configuration_update_helper(): # Reverse proxy login configuration _config_checkbox(to_save, "config_allow_reverse_proxy_header_login") + _config_checkbox(to_save, "config_reverse_proxy_use_shared_secret") _config_string(to_save, "config_reverse_proxy_login_header_name") + _config_string(to_save, "config_reverse_proxy_login_secret_header_name") + trusted_proxy_ips, error = _validate_reverse_proxy_trusted_ips( + to_save.get("config_reverse_proxy_trusted_ips", "") + ) + if error: + return error + error = _validate_reverse_proxy_header_name( + config.config_reverse_proxy_login_header_name, + _('reverse proxy login header name') + ) + if error: + return error + if to_save.get("config_reverse_proxy_login_header_secret_e", ""): + _config_string(to_save, "config_reverse_proxy_login_header_secret_e") + + if config.config_reverse_proxy_use_shared_secret: + error = _validate_reverse_proxy_header_name( + config.config_reverse_proxy_login_secret_header_name, + _('reverse proxy shared secret header name') + ) + if error: + return error + + configured_secret = bool(config.config_reverse_proxy_login_header_secret_e) + configured_secret_header = bool(config.config_reverse_proxy_login_secret_header_name) + if configured_secret ^ configured_secret_header: + return _configuration_result( + _('Please configure both reverse proxy shared secret header name and secret') + ) + if not configured_secret: + return _configuration_result( + _('Please configure reverse proxy shared secret header name and secret or disable the feature') + ) + + to_save["config_reverse_proxy_trusted_ips"] = trusted_proxy_ips + if config.config_allow_reverse_proxy_header_login and not trusted_proxy_ips: + return _configuration_result(_('Please configure at least one trusted reverse proxy IP or CIDR')) + _config_string(to_save, "config_reverse_proxy_trusted_ips") # OAuth configuration if config.config_login_type == constants.LOGIN_OAUTH: @@ -1878,6 +1952,10 @@ def _configuration_update_helper(): # Rarfile Content configuration _config_string(to_save, "config_rarfile_location") if "config_rarfile_location" in to_save: + unrar_binary = resolve_binary_path(config.config_rarfile_location, SUPPORTED_UNRAR_BINARIES) + if not unrar_binary: + return _configuration_result(_('Please specify a valid UnRar directory')) + config.config_rarfile_location = os.path.dirname(unrar_binary) unrar_status = helper.check_unrar(config.config_rarfile_location) if unrar_status: return _configuration_result(unrar_status) diff --git a/cps/binary_helper.py b/cps/binary_helper.py new file mode 100644 index 000000000..dd8d54220 --- /dev/null +++ b/cps/binary_helper.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +import os +import sys + +if sys.platform == "win32": + SUPPORTED_UNRAR_BINARIES = ("unRAR.exe", "unrar.exe") + SUPPORTED_KEPUBIFY_BINARIES = ("kepubify-windows-64Bit.exe",) +elif sys.platform.startswith("freebsd"): + SUPPORTED_UNRAR_BINARIES = ("unrar", "unrar-free", "unrar-nonfree") + SUPPORTED_KEPUBIFY_BINARIES = ("kepubify",) +else: + SUPPORTED_UNRAR_BINARIES = ("unrar","unrar-free", "unrar-nonfree") + SUPPORTED_KEPUBIFY_BINARIES = ("kepubify-linux-64bit", "kepubify-linux-32bit") + + +def resolve_binary_path(configured_path, binary_names): + if not configured_path: + return "" + + allowed_names = {binary_name.lower() for binary_name in binary_names} + if os.path.isfile(configured_path) and os.access(configured_path, os.X_OK): + if os.path.basename(os.path.realpath(configured_path)).lower() in allowed_names: + return configured_path + return "" + + if os.path.isdir(configured_path): + for binary_name in binary_names: + binary_path = os.path.join(configured_path, binary_name) + if os.path.isfile(binary_path) and os.access(binary_path, os.X_OK): + return binary_path + return "" diff --git a/cps/clean_html.py b/cps/clean_html.py index 06c3ee8a6..1faa73408 100644 --- a/cps/clean_html.py +++ b/cps/clean_html.py @@ -18,32 +18,45 @@ from . import logger from lxml.etree import ParserError +import re log = logger.create() +CONTROL_CHAR_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") + try: - # at least bleach 6.0 is needed -> incomplatible change from list arguments to set arguments - from bleach import clean as clean_html - from bleach.sanitizer import ALLOWED_TAGS - bleach = True -except ImportError: + import nh3 from nh3 import clean as clean_html bleach = False + BASE_ALLOWED_TAGS = nh3.ALLOWED_TAGS + BASE_ALLOWED_ATTRIBUTES = nh3.ALLOWED_ATTRIBUTES +except ImportError: + # at least bleach 6.0 is needed -> incomplatible change from list arguments to set arguments + from bleach import clean as clean_html + from bleach.sanitizer import ALLOWED_ATTRIBUTES as BASE_ALLOWED_ATTRIBUTES + from bleach.sanitizer import ALLOWED_TAGS as BASE_ALLOWED_TAGS + bleach = True + +ALLOWED_TAGS = set(BASE_ALLOWED_TAGS) | {"p", "span", "div", "pre", "br", "h1", "h2", "h3", "h4", "h5", "h6", "img"} +ALLOWED_ATTRIBUTES = { + key: set(value) for key, value in BASE_ALLOWED_ATTRIBUTES.items() +} +ALLOWED_ATTRIBUTES["*"] = set(ALLOWED_ATTRIBUTES.get("*", set())) | {"class", "style"} +ALLOWED_ATTRIBUTES["a"] = set(ALLOWED_ATTRIBUTES.get("a", set())) | {"href", "title", "rel"} +ALLOWED_ATTRIBUTES["img"] = set(ALLOWED_ATTRIBUTES.get("img", set())) | {"src", "alt", "title", "width", "height"} + + +def _normalize_html_input(unsafe_text): + return CONTROL_CHAR_RE.sub("", unsafe_text) def clean_string(unsafe_text, book_id=0): try: + unsafe_text = _normalize_html_input(unsafe_text) if bleach: - allowed_tags = list(ALLOWED_TAGS) - allowed_tags.extend(["p", "span", "div", "pre", "br", "h1", "h2", "h3", "h4", "h5", "h6", "img"]) - allowed_attributes = { - "*": ["class", "style"], - "a": ["href", "title", "rel"], - "img": ["src", "alt", "title", "width", "height"], - } - safe_text = clean_html(unsafe_text, tags=set(allowed_tags), attributes=allowed_attributes) + safe_text = clean_html(unsafe_text, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES) else: - safe_text = clean_html(unsafe_text) + safe_text = clean_html(unsafe_text, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None) except ParserError as e: log.error("Comments of book {} are corrupted: {}".format(book_id, e)) safe_text = "" diff --git a/cps/comic.py b/cps/comic.py index f0a81b4a0..1fa4e63dc 100644 --- a/cps/comic.py +++ b/cps/comic.py @@ -29,6 +29,13 @@ except (ImportError, RuntimeError) as e: log = logger.create() +try: + from natsort import natsorted as sort +except ImportError: + log.debug("Natural sorting unavailable: using standard " + "sorted() method instead.") + sort = sorted + try: from comicapi.comicarchive import ComicArchive, MetaDataStyle use_comic_meta = True @@ -65,7 +72,7 @@ def _extract_cover_from_archive(original_file_extension, tmp_file_name, rar_exec cover_data = extension = None if original_file_extension.upper() == '.CBZ': cf = zipfile.ZipFile(tmp_file_name) - for name in sorted(cf.namelist()): + for name in sort(cf.namelist()): ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() @@ -74,7 +81,7 @@ def _extract_cover_from_archive(original_file_extension, tmp_file_name, rar_exec break elif original_file_extension.upper() == '.CBT': cf = tarfile.TarFile(tmp_file_name) - for name in sorted(cf.getnames()): + for name in sort(cf.getnames()): ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() @@ -85,7 +92,7 @@ def _extract_cover_from_archive(original_file_extension, tmp_file_name, rar_exec try: rarfile.UNRAR_TOOL = rar_executable cf = rarfile.RarFile(tmp_file_name) - for name in sorted(cf.namelist()): + for name in sort(cf.namelist()): ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() @@ -96,7 +103,7 @@ def _extract_cover_from_archive(original_file_extension, tmp_file_name, rar_exec log.error('Rarfile failed with error: {}'.format(ex)) elif original_file_extension.upper() == '.CB7' and use_7zip: cf = py7zr.SevenZipFile(tmp_file_name) - for name in sorted(cf.getnames()): + for name in sort(cf.getnames()): ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() diff --git a/cps/config_sql.py b/cps/config_sql.py index 7638afcce..ed59829d6 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -34,6 +34,7 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from . import constants, logger +from .binary_helper import resolve_binary_path, SUPPORTED_KEPUBIFY_BINARIES, SUPPORTED_UNRAR_BINARIES from .subproc_wrapper import process_wait from .string_helper import strip_whitespaces @@ -153,6 +154,10 @@ class _Settings(_Base): config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) config_reverse_proxy_login_header_name = Column(String) + config_reverse_proxy_use_shared_secret = Column(Boolean, default=False) + config_reverse_proxy_login_secret_header_name = Column(String, default="") + config_reverse_proxy_login_header_secret_e = Column(String, default="") + config_reverse_proxy_trusted_ips = Column(String, default='127.0.0.1,::1') config_allow_reverse_proxy_header_login = Column(Boolean, default=False) schedule_start_time = Column(Integer, default=4) @@ -204,10 +209,18 @@ class ConfigSQL(object): if self.config_kepubifypath is None: change = True self.config_kepubifypath = autodetect_kepubify_binary() + elif self.config_kepubifypath and os.path.isfile(self.config_kepubifypath): + if resolve_binary_path(self.config_kepubifypath, SUPPORTED_KEPUBIFY_BINARIES): + change = True + self.config_kepubifypath = os.path.dirname(self.config_kepubifypath) if self.config_rarfile_location is None: change = True self.config_rarfile_location = autodetect_unrar_binary() + elif self.config_rarfile_location and os.path.isfile(self.config_rarfile_location): + if resolve_binary_path(self.config_rarfile_location, SUPPORTED_UNRAR_BINARIES): + change = True + self.config_rarfile_location = os.path.dirname(self.config_rarfile_location) if change: self.save() @@ -526,7 +539,7 @@ def autodetect_unrar_binary(): calibre_path = ["/usr/bin/unrar"] for element in calibre_path: if os.path.isfile(element) and os.access(element, os.X_OK): - return element + return os.path.dirname(element) return "" @@ -540,7 +553,7 @@ def autodetect_kepubify_binary(): calibre_path = ["/opt/kepubify/kepubify-linux-64bit", "/opt/kepubify/kepubify-linux-32bit"] for element in calibre_path: if os.path.isfile(element) and os.access(element, os.X_OK): - return element + return os.path.dirname(element) return "" diff --git a/cps/constants.py b/cps/constants.py index 1ddef87cf..dd233bca5 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -175,7 +175,7 @@ BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, d 'series_id, languages, publisher, pubdate, identifiers') # python build process likes to have x.y.zbw -> b for beta and w a counting number -STABLE_VERSION = '0.6.27b' +STABLE_VERSION = '0.6.28b' NIGHTLY_VERSION = dict() NIGHTLY_VERSION[0] = '$Format:%H$' diff --git a/cps/converter.py b/cps/converter.py index 981b39097..4d5154f01 100644 --- a/cps/converter.py +++ b/cps/converter.py @@ -22,6 +22,7 @@ import re from flask_babel import lazy_gettext as N_ from . import config, logger +from .binary_helper import resolve_binary_path, SUPPORTED_KEPUBIFY_BINARIES, SUPPORTED_UNRAR_BINARIES from .subproc_wrapper import process_wait @@ -33,7 +34,7 @@ _EXECUTION_ERROR = N_('Execution permissions missing') def _get_command_version(path, pattern, argument=None): - if os.path.exists(path): + if path and os.path.isfile(path) and os.access(path, os.X_OK): command = [path] if argument: command.append(argument) @@ -52,11 +53,16 @@ def get_calibre_version(): def get_unrar_version(): - unrar_version = _get_command_version(config.config_rarfile_location, r'UNRAR.*\d') + unrar_binary = resolve_binary_path(config.config_rarfile_location, SUPPORTED_UNRAR_BINARIES) + if not unrar_binary: + return _NOT_INSTALLED + + unrar_version = _get_command_version(unrar_binary, r'UNRAR.*\d') if unrar_version == "not installed": - unrar_version = _get_command_version(config.config_rarfile_location, r'unrar.*\d', '-V') + unrar_version = _get_command_version(unrar_binary, r'unrar.*\d', '-V') return unrar_version def get_kepubify_version(): - return _get_command_version(config.config_kepubifypath, r'kepubify\s', '--version') + kepubify_binary = resolve_binary_path(config.config_kepubifypath, SUPPORTED_KEPUBIFY_BINARIES) + return _get_command_version(kepubify_binary, r'kepubify\s', '--version') diff --git a/cps/editbooks.py b/cps/editbooks.py index 218f249d6..0ea23c29d 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -43,6 +43,7 @@ from . import config, ub, db, calibre_db from .services.worker import WorkerThread from .tasks.upload import TaskUpload from .render_template import render_title_template +from .binary_helper import resolve_binary_path, SUPPORTED_UNRAR_BINARIES from .kobo_sync_status import change_archived_books from .redirect import get_redirect_location from .file_helper import validate_mime_type @@ -184,6 +185,22 @@ def convert_bookformat(book_id): flash(_("Source or destination format for conversion missing"), category="error") return redirect(url_for('edit-book.show_edit_book', book_id=book_id)) + if book_format_from.lower() not in constants.EXTENSIONS_CONVERT_FROM: + flash(_("Source format for conversion not supported"), category="error") + return redirect(url_for('edit-book.show_edit_book', book_id=book_id)) + + allowed_destination_formats = set(constants.EXTENSIONS_CONVERT_TO) + if config.config_kepubifypath: + allowed_destination_formats.add('kepub') + + if book_format_to.lower() == 'kepub' and book_format_from.lower() != 'epub': + flash(_("Source format for conversion not supported"), category="error") + return redirect(url_for('edit-book.show_edit_book', book_id=book_id)) + + if book_format_to.lower() not in allowed_destination_formats: + flash(_("Destination format for conversion not supported"), category="error") + return redirect(url_for('edit-book.show_edit_book', book_id=book_id)) + log.info('converting: book id: %s from: %s to: %s', book_id, book_format_from, book_format_to) rtn = helper.convert_book_format(book_id, config.get_book_path(), book_format_from.upper(), book_format_to.upper(), current_user.name) @@ -809,7 +826,13 @@ def prepare_authors(authr, calibre_path, gdrive=False): for in_aut in input_authors: renamed_author = calibre_db.session.query(db.Authors).filter(func.lower(db.Authors.name).ilike(in_aut)).first() - if renamed_author and in_aut != renamed_author.name: + # The custom lower() SQL function transliterates via unidecode (see cps.db.lcase), so the + # case-insensitive lookup above also matches accent- or homophone-twins with a genuinely + # different name (e.g. "José Luis Corral"/"Jose Luis Corral", "姚尧"/"妖妖"). Only treat the + # match as a case-rename when the names are equal ignoring case; otherwise the rename below + # would overwrite a distinct author and hit the authors.name UNIQUE constraint (#3403, #3170). + if renamed_author and in_aut != renamed_author.name \ + and in_aut.casefold() == renamed_author.name.casefold(): old_author_name = renamed_author.name # rename author in Database create_objects_for_addition(renamed_author, in_aut,"author") @@ -962,7 +985,8 @@ def file_handling_on_upload(requested_file): # extract metadata from file try: - meta = uploader.upload(requested_file, config.config_rarfile_location) + meta = uploader.upload(requested_file, + resolve_binary_path(config.config_rarfile_location, SUPPORTED_UNRAR_BINARIES)) except (IOError, OSError): log.error("File %s could not saved to temp dir", requested_file.filename) flash(_("File %(filename)s could not saved to temp dir", @@ -1212,8 +1236,17 @@ def edit_book_tags(tags, book): if tags is not None: input_tags = tags.split(',') input_tags = list(map(lambda it: strip_whitespaces(it), input_tags)) - # Remove duplicates input_tags = helper.uniq(input_tags) + # Tag names are unique with NOCASE collation in the database. Remove + # case-insensitive duplicates before creating book-tag associations. + unique_tags = [] + seen_tags = set() + for tag in input_tags: + normalized_tag = tag.casefold() + if normalized_tag not in seen_tags: + seen_tags.add(normalized_tag) + unique_tags.append(tag) + input_tags = unique_tags return modify_database_object(input_tags, book.tags, db.Tags, calibre_db.session, 'tags') return False @@ -1397,8 +1430,19 @@ def edit_cc_data(book_id, book, to_save, cc): calibre_db.session.delete(del_cc) changed = True else: + # Tag names are unique with NOCASE collation in the database. Remove + # case-insensitive duplicates before creating book-tag associations. input_tags = to_save[cc_string].split(',') input_tags = list(map(lambda it: strip_whitespaces(it), input_tags)) + input_tags = helper.uniq(input_tags) + unique_tags = [] + seen_tags = set() + for tag in input_tags: + normalized_tag = tag.casefold() + if normalized_tag not in seen_tags: + seen_tags.add(normalized_tag) + unique_tags.append(tag) + input_tags = unique_tags changed |= modify_database_object(input_tags, getattr(book, cc_string), db.cc_classes[c.id], @@ -1483,7 +1527,7 @@ def upload_book_formats(requested_files, book, book_id, no_cover=True): meta = uploader.process( saved_filename, *os.path.splitext(current_filename), - rar_executable=config.config_rarfile_location, + rar_executable=resolve_binary_path(config.config_rarfile_location, SUPPORTED_UNRAR_BINARIES), no_cover=no_cover) merge_metadata(book, meta, to_save) #if to_save.get('languages'): diff --git a/cps/file_helper.py b/cps/file_helper.py index 361709f84..485903596 100644 --- a/cps/file_helper.py +++ b/cps/file_helper.py @@ -21,6 +21,7 @@ import os import shutil import zipfile import mimetypes +import hashlib from io import BytesIO from . import logger @@ -44,14 +45,18 @@ def get_mimetype(ext): def get_temp_dir(): - tmp_dir = os.path.join(gettempdir(), 'calibre_web') + base_tmp_dir = os.path.join(gettempdir(), 'calibre_web') + from . import config + instance_value = "{}|{}".format(config.config_calibre_dir or "", config.config_port or "") + instance_key = hashlib.md5(instance_value.encode('utf-8')).hexdigest() # nosec + tmp_dir = os.path.join(base_tmp_dir, "instance_{}".format(instance_key[:12])) if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + os.makedirs(tmp_dir) return tmp_dir def del_temp_dir(): - tmp_dir = os.path.join(gettempdir(), 'calibre_web') + tmp_dir = get_temp_dir() shutil.rmtree(tmp_dir) diff --git a/cps/helper.py b/cps/helper.py index 9f59e8313..d813c1e34 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -59,6 +59,7 @@ from . import logger, config, db, ub, fs from . import gdriveutils as gd from .constants import (STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES) +from .binary_helper import resolve_binary_path, SUPPORTED_UNRAR_BINARIES from .subproc_wrapper import process_wait from .services.worker import WorkerThread from .tasks.mail import TaskEmail @@ -69,7 +70,6 @@ from .epub_helper import get_content_opf, create_new_metadata_backup, updateEpub from .embed_helper import do_calibre_export log = logger.create() - try: from wand.image import Image from wand.exceptions import MissingDelegateError, BlobError @@ -959,7 +959,7 @@ def do_download_file(book, book_format, client, data, headers): else: download_name = book_name - # Clean up staged copies in /tmp/calibre_web after the response is sent + # Clean up staged copies in the instance temp directory after the response is sent # (kepubify / calibre-export branches) so the temp dir does not grow unbounded. if filename == get_temp_dir(): _tmp_path = os.path.join(filename, download_name + "." + book_format) @@ -1002,12 +1002,12 @@ def check_unrar(unrar_location): if not unrar_location: return - if not os.path.exists(unrar_location): - return _('UnRar binary file not found') + unrar_binary = resolve_binary_path(unrar_location, SUPPORTED_UNRAR_BINARIES) + if not unrar_binary: + return _('Please specify a valid UnRar directory') try: - unrar_location = [unrar_location] - value = process_wait(unrar_location, pattern='UNRAR (.*) freeware') + value = process_wait([unrar_binary], pattern='UNRAR (.*) freeware') if value: version = value.group(1) log.debug("UnRar version %s", version) diff --git a/cps/metadata_provider/amazon.py b/cps/metadata_provider/amazon.py index 5b07819cb..26a8910d1 100644 --- a/cps/metadata_provider/amazon.py +++ b/cps/metadata_provider/amazon.py @@ -64,7 +64,7 @@ class Amazon(Metadata): log.warning(ex) return [] long_soup = BS(r.text, "lxml") #~4sec :/ - soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-ppd_csm_instrumentation_wrapper"}) + soup2 = long_soup.find("div", attrs={"id": "dp-container"}) if soup2 is None: return [] try: @@ -97,8 +97,8 @@ class Amazon(Metadata): try: match.authors = [next( filter(lambda i: i != " " and i != "\n" and not i.startswith("{"), - x.findAll(string=True))).strip() - for x in soup2.findAll("span", attrs={"class": "author"})] + x.find_all(string=True))).strip() + for x in soup2.find_all("span", attrs={"class": "author"})] except (AttributeError, TypeError, StopIteration): match.authors = "" try: @@ -131,8 +131,8 @@ class Amazon(Metadata): log.warning(e) return [] soup = BS(results.text, 'html.parser') - links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in - soup.findAll("div", attrs={"data-component-type": "s-search-result"})] + links_list = [next(filter(lambda i: "digital-text" in i["href"], x.find_all("a")))["href"] for x in + soup.find_all("div", attrs={"data-component-type": "s-search-result"})] with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:3])} val = list(map(lambda x : x.result(), concurrent.futures.as_completed(fut))) diff --git a/cps/mobi.py b/cps/mobi.py new file mode 100644 index 000000000..8a99dc803 --- /dev/null +++ b/cps/mobi.py @@ -0,0 +1,326 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2026 Martin Grzeslowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import re +import struct + +from . import cover, isoLanguages, logger +from .constants import BookMeta +from .string_helper import strip_whitespaces + +log = logger.create() + +PDB_HEADER_LEN = 78 +PDB_RECORD_INFO_LEN = 8 +PALMDOC_HEADER_LEN = 16 +MOBI_HEADER_START = PALMDOC_HEADER_LEN +MOBI_FULL_NAME_OFFSET = 0x44 +MOBI_FULL_NAME_LENGTH = 0x48 +MOBI_FIRST_IMAGE_INDEX = 0x5C +UNKNOWN_INDEX = 0xFFFFFFFF + +EXTH_AUTHOR = 100 +EXTH_PUBLISHER = 101 +EXTH_DESCRIPTION = 103 +EXTH_ISBN = 104 +EXTH_SUBJECT = 105 +EXTH_PUBLISH_DATE = 106 +EXTH_COVER_OFFSET = 201 +EXTH_THUMBNAIL_OFFSET = 202 +EXTH_TITLE = 503 +EXTH_LANGUAGE = 524 + + +def get_mobi_info(tmp_file_path, original_file_name, original_file_extension, no_cover_processing): + data = _read_file(tmp_file_path) + record_offsets = _read_record_offsets(data) + record0 = _record(data, record_offsets, 0) + _ensure_mobi_record(record0) + + encoding = _text_encoding(record0) + exth_records = _read_exth_records(record0) + + title = _book_title(record0, encoding) or _first_text(exth_records, EXTH_TITLE, encoding) or original_file_name + author = _author_string(_texts(exth_records, EXTH_AUTHOR, encoding)) + publisher = _first_text(exth_records, EXTH_PUBLISHER, encoding) + description = _first_text(exth_records, EXTH_DESCRIPTION, encoding) + tags = ', '.join(_unique(_texts(exth_records, EXTH_SUBJECT, encoding))) + pubdate = _first_text(exth_records, EXTH_PUBLISH_DATE, encoding)[:10] + languages = _language_code(_first_text(exth_records, EXTH_LANGUAGE, encoding)) + + identifiers = [] + isbn = _first_text(exth_records, EXTH_ISBN, encoding) + if isbn: + identifiers.append(['isbn', isbn]) + + cover_file = None + if not no_cover_processing: + cover_file = _extract_cover(data, record_offsets, record0, exth_records, tmp_file_path) + + return BookMeta( + file_path=tmp_file_path, + extension=original_file_extension, + title=title, + author=author or 'Unknown', + cover=cover_file, + description=description, + tags=tags, + series="", + series_id="", + languages=languages, + publisher=publisher, + pubdate=pubdate, + identifiers=identifiers) + + +def _read_file(file_path): + with open(file_path, 'rb') as mobi_file: + return mobi_file.read() + + +def _u16(data, offset): + return struct.unpack('>H', data[offset:offset + 2])[0] + + +def _u32(data, offset): + return struct.unpack('>I', data[offset:offset + 4])[0] + + +def _read_record_offsets(data): + if len(data) < PDB_HEADER_LEN: + raise ValueError('MOBI file is shorter than the Palm database header') + + record_count = _u16(data, 76) + table_end = PDB_HEADER_LEN + record_count * PDB_RECORD_INFO_LEN + if record_count < 1 or table_end > len(data): + raise ValueError('Invalid Palm database record table') + + offsets = [_u32(data, PDB_HEADER_LEN + index * PDB_RECORD_INFO_LEN) for index in range(record_count)] + offsets.append(len(data)) + + previous = table_end + for offset in offsets[:-1]: + if offset < previous or offset > len(data): + raise ValueError('Invalid Palm database record offset') + previous = offset + return offsets + + +def _record(data, record_offsets, index): + if index < 0 or index >= len(record_offsets) - 1: + return b'' + return data[record_offsets[index]:record_offsets[index + 1]] + + +def _ensure_mobi_record(record0): + if len(record0) < MOBI_HEADER_START + 8 or record0[MOBI_HEADER_START:MOBI_HEADER_START + 4] != b'MOBI': + raise ValueError('File does not contain a MOBI header') + + +def _mobi_header_len(record0): + return _u32(record0, MOBI_HEADER_START + 4) + + +def _text_encoding(record0): + encoding = _u32(record0, MOBI_HEADER_START + 12) + if encoding == 1252: + return 'cp1252' + if encoding == 65001: + return 'utf-8' + return 'utf-8' + + +def _read_exth_records(record0): + header_len = _mobi_header_len(record0) + exth_start = MOBI_HEADER_START + header_len + + if record0[exth_start:exth_start + 4] != b'EXTH': + exth_start = record0.find(b'EXTH', MOBI_HEADER_START) + if exth_start < 0: + return {} + + exth_len = _u32(record0, exth_start + 4) + record_count = _u32(record0, exth_start + 8) + exth_end = min(len(record0), exth_start + exth_len) + position = exth_start + 12 + records = {} + + for __ in range(record_count): + if position + 8 > exth_end: + break + record_type = _u32(record0, position) + record_len = _u32(record0, position + 4) + if record_len < 8 or position + record_len > exth_end: + break + records.setdefault(record_type, []).append(record0[position + 8:position + record_len]) + position += record_len + + return records + + +def _decode_text(value, encoding): + value = value.rstrip(b'\0') + for candidate in (encoding, 'utf-8', 'cp1252', 'latin-1'): + try: + return strip_whitespaces(value.decode(candidate)) + except UnicodeDecodeError: + pass + return '' + + +def _texts(exth_records, record_type, encoding): + return [text for text in (_decode_text(value, encoding) for value in exth_records.get(record_type, [])) if text] + + +def _first_text(exth_records, record_type, encoding): + values = _texts(exth_records, record_type, encoding) + return values[0] if values else '' + + +def _book_title(record0, encoding): + header_len = _mobi_header_len(record0) + if header_len < MOBI_FULL_NAME_LENGTH + 4: + return '' + + name_offset = _u32(record0, MOBI_HEADER_START + MOBI_FULL_NAME_OFFSET) + name_length = _u32(record0, MOBI_HEADER_START + MOBI_FULL_NAME_LENGTH) + if name_length and name_offset + name_length <= len(record0): + return _decode_text(record0[name_offset:name_offset + name_length], encoding) + + alternate_offset = MOBI_HEADER_START + name_offset + if name_length and alternate_offset + name_length <= len(record0): + return _decode_text(record0[alternate_offset:alternate_offset + name_length], encoding) + return '' + + +def _author_string(authors): + normalized = [] + for value in authors: + normalized.extend(_split_author_value(value)) + + return ' & '.join(_unique(normalized)) + + +def _split_author_value(value): + authors = [] + for part in re.split('[&;]', value): + part = strip_whitespaces(part) + if not part: + continue + + if part.count(',') == 1: + left, right = [strip_whitespaces(item) for item in part.split(',', 1)] + if ' ' in left and ' ' in right: + authors.extend([left, right]) + else: + authors.append(strip_whitespaces('%s %s' % (right, left))) + elif part.count(',') > 1: + authors.extend([strip_whitespaces(item) for item in part.split(',') if strip_whitespaces(item)]) + else: + authors.append(part) + return authors + + +def _unique(values): + seen = set() + result = [] + for value in values: + if value not in seen: + seen.add(value) + result.append(value) + return result + + +def _language_code(language): + if not language: + return '' + language = language.split('-', 1)[0].lower() + return isoLanguages.get_lang3(language) + + +def _first_image_index(record0): + header_len = _mobi_header_len(record0) + if header_len < MOBI_FIRST_IMAGE_INDEX + 4: + return None + index = _u32(record0, MOBI_HEADER_START + MOBI_FIRST_IMAGE_INDEX) + return None if index == UNKNOWN_INDEX else index + + +def _exth_int(exth_records, record_type): + values = exth_records.get(record_type, []) + if not values or len(values[0]) < 4: + return None + return _u32(values[0], 0) + + +def _extract_cover(data, record_offsets, record0, exth_records, tmp_file_path): + first_image_index = _first_image_index(record0) + candidates = [] + if first_image_index is not None: + for record_type in (EXTH_COVER_OFFSET, EXTH_THUMBNAIL_OFFSET): + image_offset = _exth_int(exth_records, record_type) + if image_offset is not None: + candidates.append(first_image_index + image_offset) + + for index in candidates: + cover_file = _cover_from_record(data, record_offsets, index, tmp_file_path) + if cover_file: + return cover_file + + return _largest_image_cover(data, record_offsets, first_image_index, tmp_file_path) + + +def _cover_from_record(data, record_offsets, index, tmp_file_path): + blob = _record(data, record_offsets, index) + extension = _image_extension(blob) + if not extension: + return None + try: + return cover.cover_processing(tmp_file_path, blob, extension) + except Exception as ex: + log.warning('Cannot extract MOBI cover image, using default: %s', ex) + return None + + +def _largest_image_cover(data, record_offsets, first_image_index, tmp_file_path): + start = first_image_index if first_image_index is not None else 1 + images = [] + for index in range(start, len(record_offsets) - 1): + blob = _record(data, record_offsets, index) + extension = _image_extension(blob) + if extension: + images.append((len(blob), index)) + + for __, index in sorted(images, reverse=True): + cover_file = _cover_from_record(data, record_offsets, index, tmp_file_path) + if cover_file: + return cover_file + return None + + +def _image_extension(blob): + if blob.startswith(b'\xff\xd8\xff'): + return '.jpg' + if blob.startswith(b'\x89PNG\r\n\x1a\n'): + return '.png' + if blob.startswith(b'RIFF') and blob[8:12] == b'WEBP': + return '.webp' + if blob.startswith(b'BM'): + return '.bmp' + return None diff --git a/cps/opds.py b/cps/opds.py index 329a6a0e5..0bd6756ff 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -440,7 +440,7 @@ def feed_shelf(book_id): def opds_download_link(book_id, book_format): if not auth.current_user().role_download(): return abort(401) - client = "kobo" if "Kobo" in request.headers.get('User-Agent') else "" + client = request.headers.get('User-Agent', "").lower() return get_download_link(book_id, book_format.lower(), client) diff --git a/cps/remotelogin.py b/cps/remotelogin.py index 73b712e22..a5d69b45a 100644 --- a/cps/remotelogin.py +++ b/cps/remotelogin.py @@ -38,6 +38,27 @@ remotelogin = Blueprint('remotelogin', __name__) log = logger.create() +def _load_remote_auth_token(token): + auth_token = ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.auth_token == token).first() + + # Token not found + if auth_token is None: + flash(_("Token not found"), category="error") + log.error("Remote Login token not found") + return None + + # Token expired + if datetime.now() > auth_token.expiration: + ub.session.delete(auth_token) + ub.session_commit() + + flash(_("Token has expired"), category="error") + log.error("Remote Login token expired") + return None + + return auth_token + + def remote_login_required(f): @wraps(f) def inner(*args, **kwargs): @@ -56,6 +77,8 @@ def remote_login_required(f): @remote_login_required def remote_login(): auth_token = ub.RemoteAuthToken() + auth_token.request_ip = request.remote_addr + auth_token.request_user_agent = request.headers.get('User-Agent', "") ub.session.add(auth_token) ub.session_commit() verify_url = url_for('remotelogin.verify_token', token=auth_token.auth_token, _external=true) @@ -64,35 +87,31 @@ def remote_login(): verify_url=verify_url, page="remotelogin") -@remotelogin.route('/verify/') +@remotelogin.route('/verify/', methods=['GET', 'POST']) @remote_login_required @user_login_required def verify_token(token): - auth_token = ub.session.query(ub.RemoteAuthToken).filter(ub.RemoteAuthToken.auth_token == token).first() - - # Token not found + auth_token = _load_remote_auth_token(token) if auth_token is None: - flash(_("Token not found"), category="error") - log.error("Remote Login token not found") return redirect(url_for('web.index')) - # Token expired - elif datetime.now() > auth_token.expiration: - ub.session.delete(auth_token) + if request.method == 'POST': + if auth_token.verified: + flash(_("Token already approved"), category="error") + log.error("Remote Login token already approved") + return redirect(url_for('web.index')) + + auth_token.user_id = current_user.id + auth_token.verified = True ub.session_commit() - flash(_("Token has expired"), category="error") - log.error("Remote Login token expired") + flash(_("Success! Please return to your device"), category="success") + log.debug("Remote Login token for userid %s verified", auth_token.user_id) return redirect(url_for('web.index')) - # Update token with user information - auth_token.user_id = current_user.id - auth_token.verified = True - ub.session_commit() - - flash(_("Success! Please return to your device"), category="success") - log.debug("Remote Login token for userid %s verified", auth_token.user_id) - return redirect(url_for('web.index')) + return render_title_template('remote_login_verify.html', title=_("Login"), token=auth_token.auth_token, + request_ip=auth_token.request_ip, + request_user_agent=auth_token.request_user_agent, page="remotelogin") @remotelogin.route('/ajax/verify_token', methods=['POST']) diff --git a/cps/reverse_proxy_auth.py b/cps/reverse_proxy_auth.py new file mode 100644 index 000000000..94f7dee4c --- /dev/null +++ b/cps/reverse_proxy_auth.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +import hmac +import ipaddress +import re +from functools import lru_cache + +from . import logger + +log = logger.create() + +HEADER_NAME_PATTERN = re.compile(r"^[A-Za-z0-9-]+$") + + +def to_wsgi_header_key(header_name): + return "HTTP_{}".format(header_name.strip().upper().replace("-", "_")) + + +def is_valid_header_name(header_name): + return bool(header_name and HEADER_NAME_PATTERN.fullmatch(header_name.strip())) + + +@lru_cache(maxsize=32) +def parse_trusted_proxy_networks(trusted_proxy_config): + networks = [] + for entry in trusted_proxy_config.split(","): + value = entry.strip() + if not value: + continue + try: + networks.append(ipaddress.ip_network(value, strict=False)) + except ValueError: + log.warning("Ignoring invalid reverse proxy trusted network entry in config: %s", value) + return tuple(networks) + + +def is_trusted_proxy_source(remote_addr, trusted_proxy_config): + if not remote_addr or not trusted_proxy_config: + return False + + try: + remote_ip = ipaddress.ip_address(remote_addr) + except ValueError: + log.warning("Unable to parse remote address '%s' for reverse proxy trust check", remote_addr) + return False + + for network in parse_trusted_proxy_networks(trusted_proxy_config): + if remote_ip in network: + return True + return False + + +def is_shared_secret_valid(provided_secret, expected_secret): + if provided_secret is None or expected_secret is None: + return False + return hmac.compare_digest(str(provided_secret), str(expected_secret)) diff --git a/cps/reverseproxy.py b/cps/reverseproxy.py index 1c98c5d1a..e53a76b38 100644 --- a/cps/reverseproxy.py +++ b/cps/reverseproxy.py @@ -36,6 +36,11 @@ # # Inspired by http://flask.pocoo.org/snippets/35/ +from . import logger +from .reverse_proxy_auth import is_trusted_proxy_source, to_wsgi_header_key + +log = logger.create() + class ReverseProxied(object): """Wrap the application in this middleware and configure the @@ -60,6 +65,21 @@ class ReverseProxied(object): self.proxied = False def __call__(self, environ, start_response): + from . import config + + rp_header_name = config.config_reverse_proxy_login_header_name or "" + rp_secret_header_name = config.config_reverse_proxy_login_secret_header_name or "" + header_names = [header for header in (rp_header_name, rp_secret_header_name) if header] + header_keys = [(header, to_wsgi_header_key(header)) for header in header_names] + trusted_proxy_config = config.config_reverse_proxy_trusted_ips or "" + if config.config_allow_reverse_proxy_header_login and any(key in environ for _, key in header_keys): + if not is_trusted_proxy_source(environ.get("REMOTE_ADDR"), trusted_proxy_config): + for header_name, header_key in header_keys: + if header_key in environ: + environ.pop(header_key, None) + log.warning("Discarded reverse proxy auth header '%s' from untrusted source IP-address: %s", + header_name, environ.get("REMOTE_ADDR")) + self.proxied = False self.script_name = "/" script_name = environ.get('HTTP_X_SCRIPT_NAME', '') diff --git a/cps/search_metadata.py b/cps/search_metadata.py index da7ac8324..d8a12983d 100644 --- a/cps/search_metadata.py +++ b/cps/search_metadata.py @@ -22,6 +22,7 @@ import inspect import json import os import sys +from urllib.parse import urlsplit from flask import Blueprint, request, url_for, make_response, jsonify from .cw_login import current_user @@ -38,6 +39,37 @@ meta = Blueprint("metadata", __name__) log = logger.create() + +def _safe_metadata_url(value): + if not isinstance(value, str): + return "" + value = value.strip() + if not value: + return "" + parsed = urlsplit(value) + if parsed.scheme in ("http", "https"): + return value + if not parsed.scheme and value.startswith("/"): + return value + return "" + + +def _sanitize_metadata_record(record): + record["url"] = _safe_metadata_url(record.get("url")) + record["cover"] = _safe_metadata_url(record.get("cover")) + source = record.get("source") + if isinstance(source, dict): + source["link"] = _safe_metadata_url(source.get("link")) + return record + + +def _serialize_metadata_records(records): + return [ + _sanitize_metadata_record(asdict(record)) + for record in (records or []) + if record + ] + try: from dataclasses import asdict except ImportError: @@ -114,7 +146,7 @@ def metadata_change_active_provider(prov_name): provider = next((c for c in cl if c.__id__ == prov_name), None) if provider is not None: data = provider.search(new_state.get("query", "")) - return make_response(jsonify([asdict(x) for x in data])) + return make_response(jsonify(_serialize_metadata_records(data))) return "" @@ -135,5 +167,5 @@ def metadata_search(): if active.get(c.__id__, True) } for future in concurrent.futures.as_completed(meta): - data.extend([asdict(x) for x in (future.result() or []) if x]) - return make_response(jsonify(data)) + data.extend(_serialize_metadata_records(future.result())) + return make_response(jsonify(data)) diff --git a/cps/static/css/caliBlur.css b/cps/static/css/caliBlur.css index cc22ae2c7..79a1e0301 100644 --- a/cps/static/css/caliBlur.css +++ b/cps/static/css/caliBlur.css @@ -477,7 +477,7 @@ body.shelforder > div.container-fluid > div.row-fluid > div.col-sm-10:before { color: #fff !important } -#have_read_form span, div[aria-label="Download, send to Kindle, reading"] > .btn-group:first-child > p:first-child, div[aria-label="Download, send to Kindle, reading"] > .btn-group:first-child > p:last-child { +#have_read_form span, div[aria-label^="Download, send"] > .btn-group:first-child > p:first-child, div[aria-label^="Download, send"] > .btn-group:first-child > p:last-child { display: none } @@ -538,7 +538,7 @@ body.shelforder > div.container-fluid > div.row-fluid > div.col-sm-10:before { color: #fff !important } -#archived_form span, div[aria-label="Download, send to Kindle, reading"] > .btn-group:first-child > p:first-child, div[aria-label="Download, send to Kindle, reading"] > .btn-group:first-child > p:last-child { +#archived_form span, div[aria-label^="Download, send"] > .btn-group:first-child > p:first-child, div[aria-label^="Download, send"] > .btn-group:first-child > p:last-child { display: none } @@ -3160,14 +3160,14 @@ body.book .author { margin-left: 0 } -div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] { +div.btn-group[role=group][aria-label^="Download, send"] { white-space: nowrap; display: -webkit-box; display: -ms-flexbox; display: flex } -div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] > div.btn-group[role=group] > p { +div.btn-group[role=group][aria-label^="Download, send"] > div.btn-group[role=group] > p { display: none } @@ -3314,7 +3314,7 @@ ul.dropdown-menu.offscreen { margin: 2px -160px 0 } -div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] .dropdown-menu.offscreen { +div.btn-group[role=group][aria-label^="Download, send"] .dropdown-menu.offscreen { position: fixed; top: 120px; right: 5px @@ -6927,7 +6927,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. width: 50px } - div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] { + div.btn-group[role=group][aria-label^="Download, send"] { display: block } @@ -6935,7 +6935,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. display: none } - div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] > .downloadBtn { + div.btn-group[role=group][aria-label^="Download, send"] > .downloadBtn { border-left: 0 } diff --git a/cps/static/css/style.css b/cps/static/css/style.css index 9d5a8f6a2..0d3248008 100644 --- a/cps/static/css/style.css +++ b/cps/static/css/style.css @@ -422,6 +422,21 @@ input.pill:not(:checked) + label .glyphicon { display: none; } overflow-y: auto; } +#fileModal .modal-content { + max-height: calc(100vh - 60px); + display: flex; + flex-direction: column; +} + +#fileModal .modal-body { + overflow: hidden; +} + +#file_table_wrapper { + max-height: calc(100vh - 220px); + overflow-y: auto; +} + div.log { font-family: Courier New, serif; font-size: 12px; diff --git a/cps/static/js/caliBlur.js b/cps/static/js/caliBlur.js index 48bb26e6c..deb0b683e 100644 --- a/cps/static/js/caliBlur.js +++ b/cps/static/js/caliBlur.js @@ -154,7 +154,7 @@ if ($("body.book").length > 0) { // Move dropdown lists higher in dom, replace bootstrap toggle with own toggle. $('ul[aria-labelledby="read-in-browser"]').insertBefore(".blur-wrapper").addClass("readinbrowser-drop"); $('ul[aria-labelledby="listen-in-browser"]').insertBefore(".blur-wrapper").addClass("readinbrowser-drop"); - $('ul[aria-labelledby="send-to-kereader"]').insertBefore(".blur-wrapper").addClass("sendtoereader-drop"); + $('ul[aria-labelledby="send-to-ereader"]').insertBefore(".blur-wrapper").addClass("sendtoereader-drop"); $(".leramslist").insertBefore(".blur-wrapper"); $('ul[aria-labelledby="btnGroupDrop1"]').insertBefore(".blur-wrapper").addClass("leramslist"); $("#add-to-shelves").insertBefore(".blur-wrapper"); diff --git a/cps/static/js/get_meta.js b/cps/static/js/get_meta.js index ce1fbfd20..918ed83d9 100644 --- a/cps/static/js/get_meta.js +++ b/cps/static/js/get_meta.js @@ -135,9 +135,10 @@ $(function () { $(document).on("change", ".pill", function () { var element = $(this); var id = element.data("control"); - var initial = element.data("initial"); + var initial = String(element.data("initial")) === "true"; var val = element.prop('checked'); var params = {id : id, value: val}; + $('[data-related="' + id + '"]').toggle(val); if (!initial) { params['initial'] = initial; params['query'] = keyword; diff --git a/cps/static/js/main.js b/cps/static/js/main.js index 9138d80e6..bba191fc0 100644 --- a/cps/static/js/main.js +++ b/cps/static/js/main.js @@ -414,7 +414,7 @@ $(function() { $(".col-sm-10").bind("scroll", function () { if ( $(this).scrollTop() + $(this).innerHeight() >= - $(this)[0].scrollHeight + $(this)[0].scrollHeight - 5 ) { $loadMore.infiniteScroll("loadNextPage"); window.history.replaceState({}, null, $loadMore.infiniteScroll("getAbsolutePath")); diff --git a/cps/static/js/remote_login.js b/cps/static/js/remote_login.js index f0d364dcb..7a5d10bea 100644 --- a/cps/static/js/remote_login.js +++ b/cps/static/js/remote_login.js @@ -17,8 +17,9 @@ (function () { // Poll the server to check if the user has authenticated + var csrf_token = $("input[name='csrf_token']").val(); var t = setInterval(function () { - $.post(getPath() + "/ajax/verify_token", { token: $("#verify_url").data("token") }) + $.post(getPath() + "/ajax/verify_token", { token: $("#verify_url").data("token"), csrf_token: csrf_token }) .done(function(response) { if (response.status === 'success') { // Wait a tick so cookies are updated diff --git a/cps/string_helper.py b/cps/string_helper.py index d263dd2f2..29ac2a3e5 100644 --- a/cps/string_helper.py +++ b/cps/string_helper.py @@ -15,9 +15,23 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import re +_SPECIAL_WHITESPACE_CHARS = {"\x00", "\u200b", "\u200c", "\u200d", "\ufeff"} def strip_whitespaces(text): - return re.sub(r"(^[\s\u200B-\u200D\ufeff]+)|([\s\u200B-\u200D\ufeff]+$)","", text) + if text is None: + return "" + if not isinstance(text, str): + return text + + start = 0 + end = len(text) + + while start < end and (text[start].isspace() or text[start] in _SPECIAL_WHITESPACE_CHARS): + start += 1 + + while end > start and (text[end - 1].isspace() or text[end - 1] in _SPECIAL_WHITESPACE_CHARS): + end -= 1 + + return text[start:end] diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 9e9fbb4a6..18a590001 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -1,357 +1,360 @@ -# -*- coding: utf-8 -*- - -# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) -# Copyright (C) 2020 pwr -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import re -import glob -from shutil import copyfile, copyfileobj -from markupsafe import escape -from time import time -from uuid import uuid4 - -from sqlalchemy.exc import SQLAlchemyError -from flask_babel import lazy_gettext as N_ - -from cps.services.worker import CalibreTask -from cps import db, app -from cps import logger, config -from cps.subproc_wrapper import process_open -from flask_babel import gettext as _ -from cps.kobo_sync_status import remove_synced_book -from cps.ub import init_db_thread -from cps.file_helper import get_temp_dir - -from cps.tasks.mail import TaskEmail -from cps import gdriveutils, helper -from cps.constants import SUPPORTED_CALIBRE_BINARIES -from cps.string_helper import strip_whitespaces - -log = logger.create() - -current_milli_time = lambda: int(round(time() * 1000)) - - -class TaskConvert(CalibreTask): - def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None): - super(TaskConvert, self).__init__(task_message) - self.worker_thread = None - self.file_path = file_path - self.book_id = book_id - self.title = "" - self.settings = settings - self.ereader_mail = ereader_mail - self.user = user - - self.results = dict() - - def run(self, worker_thread): - df_cover = None - cur_book = None - self.worker_thread = worker_thread - if config.config_use_google_drive: - with app.app_context(): - worker_db = db.CalibreDB(app) - cur_book = worker_db.get_book(self.book_id) - self.title = cur_book.title - data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) - df = gdriveutils.getFileFromEbooksFolder(cur_book.path, - data.name + "." + self.settings['old_book_format'].lower()) - df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") - if df: - datafile_cover = None - datafile = os.path.join(config.get_book_path(), - cur_book.path, - data.name + "." + self.settings['old_book_format'].lower()) - if df_cover: - datafile_cover = os.path.join(config.get_book_path(), - cur_book.path, "cover.jpg") - if not os.path.exists(os.path.join(config.get_book_path(), cur_book.path)): - os.makedirs(os.path.join(config.get_book_path(), cur_book.path)) - df.GetContentFile(datafile) - if df_cover: - df_cover.GetContentFile(datafile_cover) - # worker_db.session.close() - else: - # ToDo Include cover in error handling - error_message = _("%(format)s not found on Google Drive: %(fn)s", - format=self.settings['old_book_format'], - fn=data.name + "." + self.settings['old_book_format'].lower()) - # worker_db.session.close() - return self._handleError(error_message) - - filename = self._convert_ebook_format() - if config.config_use_google_drive: - os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) - if df_cover: - os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) - - if filename: - if config.config_use_google_drive: - # Upload files to gdrive - gdriveutils.updateGdriveCalibreFromLocal() - self._handleSuccess() - if self.ereader_mail: - # if we're sending to E-Reader after converting, create a one-off task and run it immediately - # todo: figure out how to incorporate this into the progress - try: - EmailText = N_(u"%(book)s send to E-Reader", book=escape(self.title)) - for email in self.ereader_mail.split(','): - email = strip_whitespaces(email) - worker_thread.add(self.user, TaskEmail(self.settings['subject'], - self.results["path"], - filename, - self.settings, - email, - EmailText, - self.settings['body'], - id=self.book_id, - internal=True) - ) - except Exception as ex: - return self._handleError(str(ex)) - - def _convert_ebook_format(self): - error_message = None - with app.app_context(): - local_db = db.CalibreDB(app) - file_path = self.file_path - book_id = self.book_id - format_old_ext = '.' + self.settings['old_book_format'].lower() - format_new_ext = '.' + self.settings['new_book_format'].lower() - - # check to see if destination format already exists - or if book is in database - # if it does - mark the conversion task as complete and return a success - # this will allow to send to E-Reader workflow to continue to work - if os.path.isfile(file_path + format_new_ext) or\ - local_db.get_book_format(self.book_id, self.settings['new_book_format']): - log.info("Book id %d already converted to %s", book_id, format_new_ext) - cur_book = local_db.get_book(book_id) - self.title = cur_book.title - self.results['path'] = cur_book.path - self.results['title'] = self.title - new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id)\ - .filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none() - if not new_format: - new_format = db.Data(name=os.path.basename(file_path), - book_format=self.settings['new_book_format'].upper(), - book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) - try: - local_db.session.merge(new_format) - local_db.session.commit() - except SQLAlchemyError as e: - local_db.session.rollback() - log.error("Database error: %s", e) - local_db.session.close() - self._handleError(N_("Oops! Database Error: %(error)s.", error=e)) - return - self._handleSuccess() - local_db.session.close() - return os.path.basename(file_path + format_new_ext) - else: - log.info("Book id %d - target format of %s does not exist. Moving forward with convert.", - book_id, - format_new_ext) - - if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub': - check, error_message = self._convert_kepubify(file_path, - format_old_ext, - format_new_ext) - else: - # check if calibre converter-executable is existing - if not os.path.exists(config.config_converterpath): - self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath)) - return - has_cover = local_db.get_book(book_id).has_cover - check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover) - - if check == 0: - cur_book = local_db.get_book(book_id) - if os.path.isfile(file_path + format_new_ext): - new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id) \ - .filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none() - if not new_format: - new_format = db.Data(name=cur_book.data[0].name, - book_format=self.settings['new_book_format'].upper(), - book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) - try: - local_db.session.merge(new_format) - local_db.session.commit() - if self.settings['new_book_format'].upper() in ['KEPUB', 'EPUB', 'EPUB3']: - ub_session = init_db_thread() - remove_synced_book(book_id, True, ub_session) - ub_session.close() - except SQLAlchemyError as e: - local_db.session.rollback() - log.error("Database error: %s", e) - local_db.session.close() - self._handleError(error_message) - return - self.results['path'] = cur_book.path - self.title = cur_book.title - self.results['title'] = self.title - if not config.config_use_google_drive: - self._handleSuccess() - return os.path.basename(file_path + format_new_ext) - else: - error_message = N_('%(format)s format not found on disk', format=format_new_ext.upper()) - local_db.session.close() - log.info("ebook converter failed with error while converting book") - if not error_message: - error_message = N_('Ebook converter failed with unknown error') - else: - log.error(error_message) - self._handleError(error_message) - return - - def _convert_kepubify(self, file_path, format_old_ext, format_new_ext): - if config.config_embed_metadata and config.config_binariesdir: - tmp_dir, temp_file_name = helper.do_calibre_export(self.book_id, format_old_ext[1:]) - filename = os.path.join(tmp_dir, temp_file_name + format_old_ext) - temp_file_path = tmp_dir - else: - filename = file_path + format_old_ext - temp_file_path = os.path.dirname(file_path) - quotes = [1, 3] - command = [config.config_kepubifypath, filename, '-o', temp_file_path, '-i'] - try: - p = process_open(command, quotes) - except OSError as e: - return 1, N_("Kepubify-converter failed: %(error)s", error=e) - self.progress = 0.01 - while True: - nextline = p.stdout.readlines() - nextline = [x.strip('\n') for x in nextline if x != '\n'] - for line in nextline: - log.debug(line) - if p.poll() is not None: - break - - # process returncode - check = p.returncode - - # move file - if check == 0: - converted_file = glob.glob(glob.escape(os.path.splitext(filename)[0]) + "*.kepub.epub") - if len(converted_file) == 1: - copyfile(converted_file[0], (file_path + format_new_ext)) - os.unlink(converted_file[0]) - else: - return 1, N_("Converted file not found or more than one file in folder %(folder)s", - folder=os.path.dirname(file_path)) - return check, None - - def _convert_calibre(self, file_path, format_old_ext, format_new_ext, has_cover): - path_tmp_opf = None - try: - # path_tmp_opf = self._embed_metadata() - if config.config_embed_metadata: - quotes = [5] - tmp_dir = get_temp_dir() - calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) - my_env = os.environ.copy() - if config.config_calibre_split: - my_env['CALIBRE_OVERRIDE_DATABASE_PATH'] = os.path.join(config.config_calibre_dir, "metadata.db") - library_path = config.config_calibre_split_dir - else: - library_path = config.config_calibre_dir - - opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(self.book_id), - '--with-library', library_path] - p = process_open(opf_command, quotes, my_env, newlines=False) - lines = list() - while p.poll() is None: - lines.append(p.stdout.readline()) - check = p.returncode - calibre_traceback = p.stderr.readlines() - if check == 0: - path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(uuid4()) + ".opf") - with open(path_tmp_opf, 'wb') as fd: - fd.write(b''.join(lines)) - else: - error_message = "" - for ele in calibre_traceback: - ele = ele.decode('utf-8', errors="ignore").strip('\n') - log.debug(ele) - if not ele.startswith('Traceback') and not ele.startswith(' File'): - error_message = N_("Calibre failed with error: %(error)s", error=ele) - return check, error_message - quotes = [1, 2] - quotes_index = 3 - command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext)] - if config.config_embed_metadata: - quotes.append(4) - quotes_index = 5 - command.extend(['--from-opf', path_tmp_opf]) - if has_cover: - quotes.append(6) - command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]) - quotes_index = 7 - if config.config_calibre: - parameters = re.findall(r"(--[\w-]+)(?:(\s(?:(\".+\")|(?:.+?)))(?:\s|$))?", - config.config_calibre, re.IGNORECASE | re.UNICODE) - if parameters: - for param in parameters: - command.append(strip_whitespaces(param[0])) - quotes_index += 1 - if param[1] != "": - parsed = strip_whitespaces(param[1]).strip("\"") - command.append(parsed) - quotes.append(quotes_index) - quotes_index += 1 - p = process_open(command, quotes, newlines=False) - except OSError as e: - return 1, N_("Ebook-converter failed: %(error)s", error=e) - - while p.poll() is None: - nextline = p.stdout.readline() - if isinstance(nextline, bytes): - nextline = nextline.decode('utf-8', errors="ignore").strip('\r\n') - if nextline: - log.debug(nextline) - # parse progress string from calibre-converter - progress = re.search(r"(\d+)%\s.*", nextline) - if progress: - self.progress = int(progress.group(1)) / 100 - if config.config_use_google_drive: - self.progress *= 0.9 - - # process returncode - check = p.returncode - calibre_traceback = p.stderr.readlines() - error_message = "" - for ele in calibre_traceback: - ele = ele.decode('utf-8', errors="ignore").strip('\n') - log.debug(ele) - if not ele.startswith('Traceback') and not ele.startswith(' File'): - error_message = N_("Calibre failed with error: %(error)s", error=ele) - return check, error_message - - @property - def name(self): - return N_("Convert") - - def __str__(self): - if self.ereader_mail: - return "Convert Book {} and mail it to {}".format(self.book_id, self.ereader_mail) - else: - return "Convert Book {}".format(self.book_id) - - @property - def is_cancellable(self): - return False +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 pwr +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import re +import glob +from shutil import copyfile, copyfileobj +from markupsafe import escape +from time import time +from uuid import uuid4 + +from sqlalchemy.exc import SQLAlchemyError +from flask_babel import lazy_gettext as N_ + +from cps.services.worker import CalibreTask +from cps import db, app +from cps import logger, config +from cps.subproc_wrapper import process_open +from flask_babel import gettext as _ +from cps.kobo_sync_status import remove_synced_book +from cps.ub import init_db_thread +from cps.file_helper import get_temp_dir +from cps.binary_helper import resolve_binary_path, SUPPORTED_KEPUBIFY_BINARIES + +from cps.tasks.mail import TaskEmail +from cps import gdriveutils, helper +from cps.constants import SUPPORTED_CALIBRE_BINARIES +from cps.string_helper import strip_whitespaces + +log = logger.create() + +current_milli_time = lambda: int(round(time() * 1000)) + + +class TaskConvert(CalibreTask): + def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None): + super(TaskConvert, self).__init__(task_message) + self.worker_thread = None + self.file_path = file_path + self.book_id = book_id + self.title = "" + self.settings = settings + self.ereader_mail = ereader_mail + self.user = user + + self.results = dict() + + def run(self, worker_thread): + df_cover = None + cur_book = None + self.worker_thread = worker_thread + if config.config_use_google_drive: + with app.app_context(): + worker_db = db.CalibreDB(app) + cur_book = worker_db.get_book(self.book_id) + self.title = cur_book.title + data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) + df = gdriveutils.getFileFromEbooksFolder(cur_book.path, + data.name + "." + self.settings['old_book_format'].lower()) + df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") + if df: + datafile_cover = None + datafile = os.path.join(config.get_book_path(), + cur_book.path, + data.name + "." + self.settings['old_book_format'].lower()) + if df_cover: + datafile_cover = os.path.join(config.get_book_path(), + cur_book.path, "cover.jpg") + if not os.path.exists(os.path.join(config.get_book_path(), cur_book.path)): + os.makedirs(os.path.join(config.get_book_path(), cur_book.path)) + df.GetContentFile(datafile) + if df_cover: + df_cover.GetContentFile(datafile_cover) + # worker_db.session.close() + else: + # ToDo Include cover in error handling + error_message = _("%(format)s not found on Google Drive: %(fn)s", + format=self.settings['old_book_format'], + fn=data.name + "." + self.settings['old_book_format'].lower()) + # worker_db.session.close() + return self._handleError(error_message) + + filename = self._convert_ebook_format() + if config.config_use_google_drive: + os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) + if df_cover: + os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) + + if filename: + if config.config_use_google_drive: + # Upload files to gdrive + gdriveutils.updateGdriveCalibreFromLocal() + self._handleSuccess() + if self.ereader_mail: + # if we're sending to E-Reader after converting, create a one-off task and run it immediately + # todo: figure out how to incorporate this into the progress + try: + EmailText = N_(u"%(book)s send to E-Reader", book=escape(self.title)) + for email in self.ereader_mail.split(','): + email = strip_whitespaces(email) + worker_thread.add(self.user, TaskEmail(self.settings['subject'], + self.results["path"], + filename, + self.settings, + email, + EmailText, + self.settings['body'], + id=self.book_id, + internal=True) + ) + except Exception as ex: + return self._handleError(str(ex)) + + def _convert_ebook_format(self): + error_message = None + with app.app_context(): + local_db = db.CalibreDB(app) + file_path = self.file_path + book_id = self.book_id + format_old_ext = '.' + self.settings['old_book_format'].lower() + format_new_ext = '.' + self.settings['new_book_format'].lower() + + # check to see if destination format already exists - or if book is in database + # if it does - mark the conversion task as complete and return a success + # this will allow to send to E-Reader workflow to continue to work + if os.path.isfile(file_path + format_new_ext) or\ + local_db.get_book_format(self.book_id, self.settings['new_book_format']): + log.info("Book id %d already converted to %s", book_id, format_new_ext) + cur_book = local_db.get_book(book_id) + self.title = cur_book.title + self.results['path'] = cur_book.path + self.results['title'] = self.title + new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id)\ + .filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none() + if not new_format: + new_format = db.Data(name=os.path.basename(file_path), + book_format=self.settings['new_book_format'].upper(), + book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) + try: + local_db.session.merge(new_format) + local_db.session.commit() + except SQLAlchemyError as e: + local_db.session.rollback() + log.error("Database error: %s", e) + local_db.session.close() + self._handleError(N_("Oops! Database Error: %(error)s.", error=e)) + return + self._handleSuccess() + local_db.session.close() + return os.path.basename(file_path + format_new_ext) + else: + log.info("Book id %d - target format of %s does not exist. Moving forward with convert.", + book_id, + format_new_ext) + + kepubify_binary = resolve_binary_path(config.config_kepubifypath, SUPPORTED_KEPUBIFY_BINARIES) + if kepubify_binary and format_old_ext == '.epub' and format_new_ext == '.kepub': + check, error_message = self._convert_kepubify(file_path, + format_old_ext, + format_new_ext, + kepubify_binary) + else: + # check if calibre converter-executable is existing + if not os.path.exists(config.config_converterpath): + self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath)) + return + has_cover = local_db.get_book(book_id).has_cover + check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover) + + if check == 0: + cur_book = local_db.get_book(book_id) + if os.path.isfile(file_path + format_new_ext): + new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id) \ + .filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none() + if not new_format: + new_format = db.Data(name=cur_book.data[0].name, + book_format=self.settings['new_book_format'].upper(), + book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) + try: + local_db.session.merge(new_format) + local_db.session.commit() + if self.settings['new_book_format'].upper() in ['KEPUB', 'EPUB', 'EPUB3']: + ub_session = init_db_thread() + remove_synced_book(book_id, True, ub_session) + ub_session.close() + except SQLAlchemyError as e: + local_db.session.rollback() + log.error("Database error: %s", e) + local_db.session.close() + self._handleError(error_message) + return + self.results['path'] = cur_book.path + self.title = cur_book.title + self.results['title'] = self.title + if not config.config_use_google_drive: + self._handleSuccess() + return os.path.basename(file_path + format_new_ext) + else: + error_message = N_('%(format)s format not found on disk', format=format_new_ext.upper()) + local_db.session.close() + log.info("ebook converter failed with error while converting book") + if not error_message: + error_message = N_('Ebook converter failed with unknown error') + else: + log.error(error_message) + self._handleError(error_message) + return + + def _convert_kepubify(self, file_path, format_old_ext, format_new_ext, kepubify_binary): + if config.config_embed_metadata and config.config_binariesdir: + tmp_dir, temp_file_name = helper.do_calibre_export(self.book_id, format_old_ext[1:]) + filename = os.path.join(tmp_dir, temp_file_name + format_old_ext) + temp_file_path = tmp_dir + else: + filename = file_path + format_old_ext + temp_file_path = os.path.dirname(file_path) + quotes = [1, 3] + command = [kepubify_binary, filename, '-o', temp_file_path, '-i'] + try: + p = process_open(command, quotes) + except OSError as e: + return 1, N_("Kepubify-converter failed: %(error)s", error=e) + self.progress = 0.01 + while True: + nextline = p.stdout.readlines() + nextline = [x.strip('\n') for x in nextline if x != '\n'] + for line in nextline: + log.debug(line) + if p.poll() is not None: + break + + # process returncode + check = p.returncode + + # move file + if check == 0: + converted_file = glob.glob(glob.escape(os.path.splitext(filename)[0]) + "*.kepub.epub") + if len(converted_file) == 1: + copyfile(converted_file[0], (file_path + format_new_ext)) + os.unlink(converted_file[0]) + else: + return 1, N_("Converted file not found or more than one file in folder %(folder)s", + folder=os.path.dirname(file_path)) + return check, None + + def _convert_calibre(self, file_path, format_old_ext, format_new_ext, has_cover): + path_tmp_opf = None + try: + # path_tmp_opf = self._embed_metadata() + if config.config_embed_metadata: + quotes = [5] + tmp_dir = get_temp_dir() + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) + my_env = os.environ.copy() + if config.config_calibre_split: + my_env['CALIBRE_OVERRIDE_DATABASE_PATH'] = os.path.join(config.config_calibre_dir, "metadata.db") + library_path = config.config_calibre_split_dir + else: + library_path = config.config_calibre_dir + + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(self.book_id), + '--with-library', library_path] + p = process_open(opf_command, quotes, my_env, newlines=False) + lines = list() + while p.poll() is None: + lines.append(p.stdout.readline()) + check = p.returncode + calibre_traceback = p.stderr.readlines() + if check == 0: + path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(uuid4()) + ".opf") + with open(path_tmp_opf, 'wb') as fd: + fd.write(b''.join(lines)) + else: + error_message = "" + for ele in calibre_traceback: + ele = ele.decode('utf-8', errors="ignore").strip('\n') + log.debug(ele) + if not ele.startswith('Traceback') and not ele.startswith(' File'): + error_message = N_("Calibre failed with error: %(error)s", error=ele) + return check, error_message + quotes = [1, 2] + quotes_index = 3 + command = [config.config_converterpath, (file_path + format_old_ext), + (file_path + format_new_ext)] + if config.config_embed_metadata: + quotes.append(4) + quotes_index = 5 + command.extend(['--from-opf', path_tmp_opf]) + if has_cover: + quotes.append(6) + command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]) + quotes_index = 7 + if config.config_calibre: + parameters = re.findall(r"(--[\w-]+)(?:(\s(?:(\".+\")|(?:.+?)))(?:\s|$))?", + config.config_calibre, re.IGNORECASE | re.UNICODE) + if parameters: + for param in parameters: + command.append(strip_whitespaces(param[0])) + quotes_index += 1 + if param[1] != "": + parsed = strip_whitespaces(param[1]).strip("\"") + command.append(parsed) + quotes.append(quotes_index) + quotes_index += 1 + p = process_open(command, quotes, newlines=False) + except OSError as e: + return 1, N_("Ebook-converter failed: %(error)s", error=e) + + while p.poll() is None: + nextline = p.stdout.readline() + if isinstance(nextline, bytes): + nextline = nextline.decode('utf-8', errors="ignore").strip('\r\n') + if nextline: + log.debug(nextline) + # parse progress string from calibre-converter + progress = re.search(r"(\d+)%\s.*", nextline) + if progress: + self.progress = int(progress.group(1)) / 100 + if config.config_use_google_drive: + self.progress *= 0.9 + + # process returncode + check = p.returncode + calibre_traceback = p.stderr.readlines() + error_message = "" + for ele in calibre_traceback: + ele = ele.decode('utf-8', errors="ignore").strip('\n') + log.debug(ele) + if not ele.startswith('Traceback') and not ele.startswith(' File'): + error_message = N_("Calibre failed with error: %(error)s", error=ele) + return check, error_message + + @property + def name(self): + return N_("Convert") + + def __str__(self): + if self.ereader_mail: + return "Convert Book {} and mail it to {}".format(self.book_id, self.ereader_mail) + else: + return "Convert Book {}".format(self.book_id) + + @property + def is_cancellable(self): + return False diff --git a/cps/templates/admin.html b/cps/templates/admin.html index ac124fe84..4e068955c 100644 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -154,6 +154,22 @@
{{_('Reverse Proxy Header Name')}}
{{ config.config_reverse_proxy_login_header_name }}
+
+
{{_('Reverse Proxy Trusted IPs/CIDRs')}}
+
{{ config.config_reverse_proxy_trusted_ips }}
+
+
+
{{_('Reverse Proxy Shared Secret Header Name')}}
+
{{ config.config_reverse_proxy_login_secret_header_name }}
+
+
+
{{_('Reverse Proxy Shared Secret Header')}}
+
{{ display_bool_setting(config.config_reverse_proxy_use_shared_secret) }}
+
+
+
{{_('Reverse Proxy Shared Secret Configured')}}
+
{{ display_bool_setting(config.config_reverse_proxy_login_header_secret_e) }}
+
{% endif %} {{_('Edit Calibre Database Configuration')}} diff --git a/cps/templates/author.html b/cps/templates/author.html index a01bb40bf..fe326e238 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -36,7 +36,7 @@
- {{ image.book_cover(entry.Books, alt=author.name|safe) }} + {{ image.book_cover(entry.Books, alt=author.name) }} {% if entry[2] == True %}{% endif %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 615b40c2e..23d548e06 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -263,26 +263,26 @@ {% block js %} - - \ No newline at end of file