From ee3541d74ed372272a2008a50db39c386d40736c Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sat, 4 Jul 2020 13:35:08 +0200 Subject: [PATCH 01/27] Fix kobo links for reverse proxies with subdomains (and docker?) #1470 --- cps/kobo.py | 4 +--- cps/reverseproxy.py | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cps/kobo.py b/cps/kobo.py index 97d55db0..69708d0c 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -19,8 +19,6 @@ import base64 import datetime -import itertools -import json import sys import os import uuid @@ -267,7 +265,7 @@ def HandleMetadataRequest(book_uuid): def get_download_url_for_book(book, book_format): if not current_app.wsgi_app.is_proxied: - if ':' in request.host and not request.host.endswith(']') : + if ':' in request.host and not request.host.endswith(']'): host = "".join(request.host.split(':')[:-1]) else: host = request.host diff --git a/cps/reverseproxy.py b/cps/reverseproxy.py index 42b64050..3bcbd3b7 100644 --- a/cps/reverseproxy.py +++ b/cps/reverseproxy.py @@ -77,6 +77,7 @@ class ReverseProxied(object): servr = environ.get('HTTP_X_FORWARDED_HOST', '') if servr: environ['HTTP_HOST'] = servr + self.proxied = True return self.app(environ, start_response) @property From ee6f1405d4ef2f0af2330c19173b4b5a5a736cdf Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sat, 4 Jul 2020 21:22:19 +0200 Subject: [PATCH 02/27] Fix #1456 (Added route to robots.txt) --- cps/web.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cps/web.py b/cps/web.py index 4d59e61c..9af9858d 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1256,6 +1256,9 @@ def render_archived_books(page, order): def get_cover(book_id): return get_book_cover(book_id) +@web.route("/robots.txt") +def get_robots(): + return send_from_directory(constants.STATIC_DIR, "robots.txt") @web.route("/show//", defaults={'anyname': 'None'}) @web.route("/show///") From 20c6f79a4472c0986bed0409c841bfae2b66d1dc Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 5 Jul 2020 10:44:49 +0200 Subject: [PATCH 03/27] Changed behavior delete books with subfolders (additional warning message, but book is deleted) --- cps/helper.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 48efc69b..20f3d4ad 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -295,15 +295,16 @@ def delete_book_file(book, calibrepath, book_format=None): return True, None else: if os.path.isdir(path): - if len(next(os.walk(path))[1]): - log.error("Deleting book %s failed, path has subfolders: %s", book.id, book.path) - return False , _("Deleting book %(id)s failed, path has subfolders: %(path)s", - id=book.id, - path=book.path) try: - for root, __, files in os.walk(path): + for root, folders, files in os.walk(path): for f in files: os.unlink(os.path.join(root, f)) + if len(folders): + log.warning("Deleting book {} failed, path {} has subfolders: {}".format(book.id, + book.path, folders)) + return True, _("Deleting bookfolder for book %(id)s failed, path has subfolders: %(path)s", + id=book.id, + path=book.path) shutil.rmtree(path) except (IOError, OSError) as e: log.error("Deleting book %s failed: %s", book.id, e) From c1f5252b3ff7d2f7acf0b57028c346c2f7175896 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 5 Jul 2020 13:40:33 +0200 Subject: [PATCH 04/27] Fix #1509 (OSError thrown during sync on Windows if one of the timestamps is outside range 1970 to 2038) --- cps/services/SyncToken.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cps/services/SyncToken.py b/cps/services/SyncToken.py index 1dd4f084..9790150c 100644 --- a/cps/services/SyncToken.py +++ b/cps/services/SyncToken.py @@ -45,7 +45,8 @@ def to_epoch_timestamp(datetime_object): def get_datetime_from_json(json_object, field_name): try: return datetime.utcfromtimestamp(json_object[field_name]) - except KeyError: + except (KeyError, OSError): + # OSError is thrown on Windows if timestamp is <1970 or >2038 return datetime.min From 76f914cbc2819e60bd42df1655fde8a17215c3d0 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 5 Jul 2020 13:52:29 +0200 Subject: [PATCH 05/27] Fixed logging in SyncToken --- cps/services/SyncToken.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cps/services/SyncToken.py b/cps/services/SyncToken.py index 9790150c..2fe7fe95 100644 --- a/cps/services/SyncToken.py +++ b/cps/services/SyncToken.py @@ -27,7 +27,10 @@ except ImportError: from urllib.parse import unquote from flask import json -from .. import logger as log +from .. import logger + + +log = logger.create() def b64encode_json(json_data): From 12263ff02f931a0d33bf4073e6cbae50940346b1 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 5 Jul 2020 13:55:59 +0200 Subject: [PATCH 06/27] Fix #1423 (OverflowError thrown during sync on some Linux Systems if one of the timestamps is outside range 1970 to 2038) --- cps/services/SyncToken.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/services/SyncToken.py b/cps/services/SyncToken.py index 2fe7fe95..f6db960b 100644 --- a/cps/services/SyncToken.py +++ b/cps/services/SyncToken.py @@ -48,7 +48,7 @@ def to_epoch_timestamp(datetime_object): def get_datetime_from_json(json_object, field_name): try: return datetime.utcfromtimestamp(json_object[field_name]) - except (KeyError, OSError): + except (KeyError, OSError, OverflowError): # OSError is thrown on Windows if timestamp is <1970 or >2038 return datetime.min From d1889a5e06ee690bb356a33e0fc8152d51b22b73 Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 5 Jul 2020 14:00:40 +0200 Subject: [PATCH 07/27] Fix #1502 (program info only visible in about section if user is admin) --- cps/templates/stats.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cps/templates/stats.html b/cps/templates/stats.html index 69712cc4..966abf2a 100644 --- a/cps/templates/stats.html +++ b/cps/templates/stats.html @@ -25,6 +25,7 @@ +{% if g.user.role_admin() %}

{{_('Linked Libraries')}}

@@ -44,4 +45,5 @@ {% endfor %}
+{% endif %} {% endblock %} From f80c67828bbd70ee482ce4178bfaffffb1f415ef Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 5 Jul 2020 14:35:57 +0200 Subject: [PATCH 08/27] Fix #1500 (Custom ratings of increment 0.5 are allowed) --- cps/templates/book_edit.html | 4 ++-- cps/templates/detail.html | 2 +- cps/templates/search_form.html | 2 +- cps/web.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 14bc590a..8bbe2460 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -159,9 +159,9 @@ {% endif %} {% if c.datatype == 'rating' %} - 0 %} - value="{{ '%d' % (book['custom_column_' ~ c.id][0].value / 2) }}" + value="{{ '%.1f' % (book['custom_column_' ~ c.id][0].value / 2) }}" {% endif %}> {% endif %} diff --git a/cps/templates/detail.html b/cps/templates/detail.html index a87695eb..d4b4252c 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -174,7 +174,7 @@ {{ c.name }}: {% for column in entry['custom_column_' ~ c.id] %} {% if c.datatype == 'rating' %} - {{ '%d' % (column.value / 2) }} + {{ '%d' % (column.value / 2) }}{% if ((column.value /2) % 1) != 0 %}{{ '.%d' % (((column.value /2) % 1)*10) }} {% endif %} {% else %} {% if c.datatype == 'bool' %} {% if column.value == true %} diff --git a/cps/templates/search_form.html b/cps/templates/search_form.html index 60cffa1d..fdc2990b 100644 --- a/cps/templates/search_form.html +++ b/cps/templates/search_form.html @@ -165,7 +165,7 @@ {% endif %} {% if c.datatype == 'rating' %} - + {% endif %} {% endfor %} diff --git a/cps/web.py b/cps/web.py index 9af9858d..7e39054d 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1142,7 +1142,7 @@ def advanced_search(): db.cc_classes[c.id].value == custom_query)) elif c.datatype == 'rating': q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any( - db.cc_classes[c.id].value == int(custom_query) * 2)) + db.cc_classes[c.id].value == int(float(custom_query) * 2))) else: q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any( func.lower(db.cc_classes[c.id].value).ilike("%" + custom_query + "%"))) From 450411a732c86e392523cb020303a624f2e61e6b Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Sun, 5 Jul 2020 20:54:36 +0200 Subject: [PATCH 09/27] #1344 (Support Multiple authors, but not showing up on Kobo reader) Fix for #1439 (reading progress was not stored, as user login was wrong) --- cps/kobo.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/cps/kobo.py b/cps/kobo.py index 69708d0c..70c4b45a 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -315,8 +315,15 @@ def get_description(book): # TODO handle multiple authors def get_author(book): if not book.authors: - return None - return book.authors[0].name + return {"Contributors": None} + if len(book.authors) > 1: + author_list = [] + autor_roles = [] + for author in book.authors: + autor_roles.append({"Name":author.name, "Role":"Author"}) + author_list.append(author.name) + return {"ContributorRoles": autor_roles, "Contributors":author_list} + return {"ContributorRoles": [{"Name":book.authors[0].name, "Role":"Author"}], "Contributors": book.authors[0].name} def get_publisher(book): @@ -355,7 +362,7 @@ def get_metadata(book): book_uuid = book.uuid metadata = { "Categories": ["00000000-0000-0000-0000-000000000001",], - "Contributors": get_author(book), + # "Contributors": get_author(book), "CoverImageId": book_uuid, "CrossRevisionId": book_uuid, "CurrentDisplayPrice": {"CurrencyCode": "USD", "TotalAmount": 0}, @@ -379,6 +386,7 @@ def get_metadata(book): "Title": book.title, "WorkId": book_uuid, } + metadata.update(get_author(book)) if get_series(book): if sys.version_info < (3, 0): @@ -397,7 +405,7 @@ def get_metadata(book): @kobo.route("/v1/library/tags", methods=["POST", "DELETE"]) -@login_required +@requires_kobo_auth # Creates a Shelf with the given items, and returns the shelf's uuid. def HandleTagCreate(): # catch delete requests, otherwise the are handeld in the book delete handler @@ -432,6 +440,7 @@ def HandleTagCreate(): @kobo.route("/v1/library/tags/", methods=["DELETE", "PUT"]) +@requires_kobo_auth def HandleTagUpdate(tag_id): shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.uuid == tag_id, ub.Shelf.user_id == current_user.id).one_or_none() @@ -486,7 +495,7 @@ def add_items_to_shelf(items, shelf): @kobo.route("/v1/library/tags//items", methods=["POST"]) -@login_required +@requires_kobo_auth def HandleTagAddItem(tag_id): items = None try: @@ -516,7 +525,7 @@ def HandleTagAddItem(tag_id): @kobo.route("/v1/library/tags//items/delete", methods=["POST"]) -@login_required +@requires_kobo_auth def HandleTagRemoveItem(tag_id): items = None try: @@ -625,7 +634,7 @@ def create_kobo_tag(shelf): @kobo.route("/v1/library//state", methods=["GET", "PUT"]) -@login_required +@requires_kobo_auth def HandleStateRequest(book_uuid): book = calibre_db.get_book_by_uuid(book_uuid) if not book or not book.data: @@ -799,7 +808,7 @@ def TopLevelEndpoint(): @kobo.route("/v1/library/", methods=["DELETE"]) -@login_required +@requires_kobo_auth def HandleBookDeletionRequest(book_uuid): log.info("Kobo book deletion request received for book %s" % book_uuid) book = calibre_db.get_book_by_uuid(book_uuid) From 93a0217d5f0d88f61ea3411430d954ced677b320 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Mon, 20 Jul 2020 20:28:10 +0200 Subject: [PATCH 10/27] Fix for #1530 (Handle improper migrated database with config_mature_content_tags NULL instead of "") --- cps/config_sql.py | 10 +++++++--- cps/ub.py | 14 ++++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/cps/config_sql.py b/cps/config_sql.py index 1135516d..b965eace 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -102,7 +102,6 @@ class _Settings(_Base): config_kobo_proxy = Column(Boolean, default=False) - config_ldap_provider_url = Column(String, default='example.org') config_ldap_port = Column(SmallInteger, default=389) config_ldap_authentication = Column(SmallInteger, default=constants.LDAP_AUTH_SIMPLE) @@ -215,8 +214,13 @@ class _ConfigSQL(object): return self.show_element_new_user(constants.DETAIL_RANDOM) def list_denied_tags(self): - mct = self.config_denied_tags.split(",") - return [t.strip() for t in mct] + try: + mct = self.config_denied_tags.split(",") + return [t.strip() for t in mct] + except AttributeError: + # Fix for improper migrated database with config_mature_content_tags NULL instead of "" + self.config_denied_tags = "" + return [''] def list_allowed_tags(self): mct = self.config_allowed_tags.split(",") diff --git a/cps/ub.py b/cps/ub.py index 1a2c260a..8fcec1af 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -176,8 +176,14 @@ class UserBase: return self.check_visibility(constants.DETAIL_RANDOM) def list_denied_tags(self): - mct = self.denied_tags.split(",") - return [t.strip() for t in mct] + try: + mct = self.denied_tags.split(",") + return [t.strip() for t in mct] + except AttributeError: + # Fix for improper migrated database with config_mature_content_tags NULL instead of "" + self.denied_tags="" + return [''] + def list_allowed_tags(self): mct = self.allowed_tags.split(",") @@ -563,8 +569,8 @@ def migrate_Database(session): conn = engine.connect() conn.execute("ALTER TABLE user ADD column `denied_tags` String DEFAULT ''") conn.execute("ALTER TABLE user ADD column `allowed_tags` String DEFAULT ''") - conn.execute("ALTER TABLE user ADD column `denied_column_value` DEFAULT ''") - conn.execute("ALTER TABLE user ADD column `allowed_column_value` DEFAULT ''") + conn.execute("ALTER TABLE user ADD column `denied_column_value` String DEFAULT ''") + conn.execute("ALTER TABLE user ADD column `allowed_column_value` String DEFAULT ''") session.commit() try: session.query(exists().where(User.series_view)).scalar() From 66acd1821d79da64ea14d13df48d4284fc31bb10 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Tue, 21 Jul 2020 18:01:38 +0200 Subject: [PATCH 11/27] Update Gdrive integration (Error Handling on Callback and wording) --- cps/gdrive.py | 17 ++++++++--------- cps/gdriveutils.py | 12 ++++++++++-- cps/templates/config_edit.html | 2 +- cps/translations/cs/LC_MESSAGES/messages.po | 2 +- cps/translations/de/LC_MESSAGES/messages.po | 2 +- cps/translations/es/LC_MESSAGES/messages.po | 2 +- cps/translations/fi/LC_MESSAGES/messages.po | 2 +- cps/translations/fr/LC_MESSAGES/messages.po | 2 +- cps/translations/hu/LC_MESSAGES/messages.po | 2 +- cps/translations/it/LC_MESSAGES/messages.po | 2 +- cps/translations/ja/LC_MESSAGES/messages.po | 2 +- cps/translations/km/LC_MESSAGES/messages.po | 2 +- cps/translations/nl/LC_MESSAGES/messages.po | 2 +- cps/translations/pl/LC_MESSAGES/messages.po | 2 +- cps/translations/ru/LC_MESSAGES/messages.po | 2 +- cps/translations/sv/LC_MESSAGES/messages.po | 2 +- cps/translations/tr/LC_MESSAGES/messages.po | 2 +- cps/translations/uk/LC_MESSAGES/messages.po | 2 +- .../zh_Hans_CN/LC_MESSAGES/messages.po | 2 +- messages.pot | 2 +- 20 files changed, 36 insertions(+), 29 deletions(-) diff --git a/cps/gdrive.py b/cps/gdrive.py index aa3743d2..466097c9 100644 --- a/cps/gdrive.py +++ b/cps/gdrive.py @@ -34,18 +34,17 @@ from flask import Blueprint, flash, request, redirect, url_for, abort from flask_babel import gettext as _ from flask_login import login_required -try: - from googleapiclient.errors import HttpError -except ImportError: - pass - from . import logger, gdriveutils, config, ub, calibre_db from .web import admin_required - gdrive = Blueprint('gdrive', __name__) log = logger.create() +try: + from googleapiclient.errors import HttpError +except ImportError as err: + log.debug(("Cannot import googleapiclient, using gdrive will not work: %s", err)) + current_milli_time = lambda: int(round(time() * 1000)) gdrive_watch_callback_token = 'target=calibreweb-watch_files' @@ -73,7 +72,7 @@ def google_drive_callback(): credentials = gdriveutils.Gauth.Instance().auth.flow.step2_exchange(auth_code) with open(gdriveutils.CREDENTIALS, 'w') as f: f.write(credentials.to_json()) - except ValueError as error: + except (ValueError, AttributeError) as error: log.error(error) return redirect(url_for('admin.configuration')) @@ -94,7 +93,7 @@ def watch_gdrive(): try: result = gdriveutils.watchChange(gdriveutils.Gdrive.Instance().drive, notification_id, 'web_hook', address, gdrive_watch_callback_token, current_milli_time() + 604800*1000) - config.config_google_drive_watch_changes_response = json.dumps(result) + # config.config_google_drive_watch_changes_response = json.dumps(result) # after save(), config_google_drive_watch_changes_response will be a json object, not string config.save() except HttpError as e: @@ -118,7 +117,7 @@ def revoke_watch_gdrive(): last_watch_response['resourceId']) except HttpError: pass - config.config_google_drive_watch_changes_response = None + config.config_google_drive_watch_changes_response = {} config.save() return redirect(url_for('admin.configuration')) diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 9ea0479d..683605ff 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -36,7 +36,9 @@ try: from apiclient import errors from httplib2 import ServerNotFoundError gdrive_support = True -except ImportError: + importError = None +except ImportError as err: + importError = err gdrive_support = False from . import logger, cli, config @@ -52,6 +54,8 @@ if gdrive_support: logger.get('googleapiclient.discovery_cache').setLevel(logger.logging.ERROR) if not logger.is_debug_enabled(): logger.get('googleapiclient.discovery').setLevel(logger.logging.ERROR) +else: + log.debug("Cannot import pydrive,httplib2, using gdrive will not work: %s", importError) class Singleton: @@ -99,7 +103,11 @@ class Singleton: @Singleton class Gauth: def __init__(self): - self.auth = GoogleAuth(settings_file=SETTINGS_YAML) + try: + self.auth = GoogleAuth(settings_file=SETTINGS_YAML) + except NameError as error: + log.error(error) + self.auth = None @Singleton diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 77a60c1b..6cb95764 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -41,7 +41,7 @@ {% else %} {% if show_authenticate_google_drive and g.user.is_authenticated and not config.config_use_google_drive %} -
{{_('Please hit submit to continue with setup')}}
+
{{_('Please hit save to continue with setup')}}
{% endif %} {% if not g.user.is_authenticated %}
{{_('Please finish Google Drive setup after login')}}
diff --git a/cps/translations/cs/LC_MESSAGES/messages.po b/cps/translations/cs/LC_MESSAGES/messages.po index 6ef881ed..3dca9fe8 100644 --- a/cps/translations/cs/LC_MESSAGES/messages.po +++ b/cps/translations/cs/LC_MESSAGES/messages.po @@ -1556,7 +1556,7 @@ msgid "Authenticate Google Drive" msgstr "Ověřit Google Drive" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "Klikněte na odeslat pro pokračování v nastavení" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/de/LC_MESSAGES/messages.po b/cps/translations/de/LC_MESSAGES/messages.po index a1372adb..4d9725e5 100644 --- a/cps/translations/de/LC_MESSAGES/messages.po +++ b/cps/translations/de/LC_MESSAGES/messages.po @@ -1557,7 +1557,7 @@ msgid "Authenticate Google Drive" msgstr "Google Drive authentifizieren" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "Bitte auf Abschicken drücken, um mit dem Setup fortzufahren" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/es/LC_MESSAGES/messages.po b/cps/translations/es/LC_MESSAGES/messages.po index 99bac86e..4bd142ca 100644 --- a/cps/translations/es/LC_MESSAGES/messages.po +++ b/cps/translations/es/LC_MESSAGES/messages.po @@ -1560,7 +1560,7 @@ msgid "Authenticate Google Drive" msgstr "Autentificar Google Drive" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "Por favor, pulsa enviar para continuar con la configuración" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/fi/LC_MESSAGES/messages.po b/cps/translations/fi/LC_MESSAGES/messages.po index 09b133f1..26d644fb 100644 --- a/cps/translations/fi/LC_MESSAGES/messages.po +++ b/cps/translations/fi/LC_MESSAGES/messages.po @@ -1557,7 +1557,7 @@ msgid "Authenticate Google Drive" msgstr "Autentikoi Google Drive" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "Ole hyvä ja paina lähetä jatkaaksesi asennusta" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/fr/LC_MESSAGES/messages.po b/cps/translations/fr/LC_MESSAGES/messages.po index 6860cc52..5ba89d85 100644 --- a/cps/translations/fr/LC_MESSAGES/messages.po +++ b/cps/translations/fr/LC_MESSAGES/messages.po @@ -1571,7 +1571,7 @@ msgid "Authenticate Google Drive" msgstr "Authentification Google Drive" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "Veuillez cliquer sur soumettre pour continuer l’initialisation" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/hu/LC_MESSAGES/messages.po b/cps/translations/hu/LC_MESSAGES/messages.po index 0b0f71b4..4550161a 100644 --- a/cps/translations/hu/LC_MESSAGES/messages.po +++ b/cps/translations/hu/LC_MESSAGES/messages.po @@ -1557,7 +1557,7 @@ msgid "Authenticate Google Drive" msgstr "Google Drive hitelesítés" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "A beállítás folytatásához kattints a Küldés gombra" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/it/LC_MESSAGES/messages.po b/cps/translations/it/LC_MESSAGES/messages.po index 8c544937..1e069ae6 100644 --- a/cps/translations/it/LC_MESSAGES/messages.po +++ b/cps/translations/it/LC_MESSAGES/messages.po @@ -1556,7 +1556,7 @@ msgid "Authenticate Google Drive" msgstr "Autenticazione Google Drive" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "Per favore premi invio per proseguire con la configurazione" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/ja/LC_MESSAGES/messages.po b/cps/translations/ja/LC_MESSAGES/messages.po index 4ae537dc..01b8e37f 100644 --- a/cps/translations/ja/LC_MESSAGES/messages.po +++ b/cps/translations/ja/LC_MESSAGES/messages.po @@ -1557,7 +1557,7 @@ msgid "Authenticate Google Drive" msgstr "Googleドライブを認証" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "決定を押して設定を続けてください" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/km/LC_MESSAGES/messages.po b/cps/translations/km/LC_MESSAGES/messages.po index 10d2d6af..e8def12b 100644 --- a/cps/translations/km/LC_MESSAGES/messages.po +++ b/cps/translations/km/LC_MESSAGES/messages.po @@ -1558,7 +1558,7 @@ msgid "Authenticate Google Drive" msgstr "វាយបញ្ចូលគណនី Google Drive" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/nl/LC_MESSAGES/messages.po b/cps/translations/nl/LC_MESSAGES/messages.po index 4ce11eb2..eda57375 100644 --- a/cps/translations/nl/LC_MESSAGES/messages.po +++ b/cps/translations/nl/LC_MESSAGES/messages.po @@ -1558,7 +1558,7 @@ msgid "Authenticate Google Drive" msgstr "Google Drive goedkeuren" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "Druk op 'Opslaan' om door te gaan met instellen" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/pl/LC_MESSAGES/messages.po b/cps/translations/pl/LC_MESSAGES/messages.po index e6dc6b45..8718b9b6 100644 --- a/cps/translations/pl/LC_MESSAGES/messages.po +++ b/cps/translations/pl/LC_MESSAGES/messages.po @@ -1570,7 +1570,7 @@ msgid "Authenticate Google Drive" msgstr "Uwierzytelnij Dysk Google" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "Kliknij przycisk, aby kontynuować instalację" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/ru/LC_MESSAGES/messages.po b/cps/translations/ru/LC_MESSAGES/messages.po index 8b265dab..403e0c94 100644 --- a/cps/translations/ru/LC_MESSAGES/messages.po +++ b/cps/translations/ru/LC_MESSAGES/messages.po @@ -1558,7 +1558,7 @@ msgid "Authenticate Google Drive" msgstr "Аутентификация Google Drive" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "Пожалуйста, нажмите «Отправить», чтобы продолжить настройку" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/sv/LC_MESSAGES/messages.po b/cps/translations/sv/LC_MESSAGES/messages.po index b41fd929..ceb839d4 100644 --- a/cps/translations/sv/LC_MESSAGES/messages.po +++ b/cps/translations/sv/LC_MESSAGES/messages.po @@ -1557,7 +1557,7 @@ msgid "Authenticate Google Drive" msgstr "Autentisera Google Drive" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "Klicka på skicka för att fortsätta med installationen" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/tr/LC_MESSAGES/messages.po b/cps/translations/tr/LC_MESSAGES/messages.po index b6ee8906..4abd7bc2 100644 --- a/cps/translations/tr/LC_MESSAGES/messages.po +++ b/cps/translations/tr/LC_MESSAGES/messages.po @@ -1557,7 +1557,7 @@ msgid "Authenticate Google Drive" msgstr "Google Drive Doğrula" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "Kuruluma devam etmek için Gönder'e tıklayın" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/uk/LC_MESSAGES/messages.po b/cps/translations/uk/LC_MESSAGES/messages.po index 41c448b2..d7dc9af3 100644 --- a/cps/translations/uk/LC_MESSAGES/messages.po +++ b/cps/translations/uk/LC_MESSAGES/messages.po @@ -1556,7 +1556,7 @@ msgid "Authenticate Google Drive" msgstr "Автентифікація Google Drive" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "" #: cps/templates/config_edit.html:47 diff --git a/cps/translations/zh_Hans_CN/LC_MESSAGES/messages.po b/cps/translations/zh_Hans_CN/LC_MESSAGES/messages.po index 76b99287..759263c8 100644 --- a/cps/translations/zh_Hans_CN/LC_MESSAGES/messages.po +++ b/cps/translations/zh_Hans_CN/LC_MESSAGES/messages.po @@ -1557,7 +1557,7 @@ msgid "Authenticate Google Drive" msgstr "认证 Google Drive" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "请点击提交以继续设置" #: cps/templates/config_edit.html:47 diff --git a/messages.pot b/messages.pot index 8b630b6c..078f1669 100644 --- a/messages.pot +++ b/messages.pot @@ -1556,7 +1556,7 @@ msgid "Authenticate Google Drive" msgstr "" #: cps/templates/config_edit.html:44 -msgid "Please hit submit to continue with setup" +msgid "Please hit save to continue with setup" msgstr "" #: cps/templates/config_edit.html:47 From 0ccc3f72523aff28f1c330665126a2e927090ffc Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Tue, 21 Jul 2020 20:14:08 +0200 Subject: [PATCH 12/27] Fixes for windows (moving files not allowed -> Close pdf after metadata extracting, add os.path.normcase to path while renaming folders, as thi caused also trouble Added hint for missing ghostcript on cover extraction --- cps/helper.py | 13 +++++++------ cps/uploader.py | 5 +++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 20f3d4ad..681719a9 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -340,13 +340,13 @@ def update_dir_structure_file(book_id, calibrepath, first_author): new_title_path = os.path.join(os.path.dirname(path), new_titledir) try: if not os.path.exists(new_title_path): - os.renames(path, new_title_path) + os.renames(os.path.normcase(path), os.path.normcase(new_title_path)) else: log.info("Copying title: %s into existing: %s", path, new_title_path) for dir_name, __, file_list in os.walk(path): for file in file_list: - os.renames(os.path.join(dir_name, file), - os.path.join(new_title_path + dir_name[len(path):], file)) + os.renames(os.path.normcase(os.path.join(dir_name, file)), + os.path.normcase(os.path.join(new_title_path + dir_name[len(path):], file))) path = new_title_path localbook.path = localbook.path.split('/')[0] + '/' + new_titledir except OSError as ex: @@ -357,7 +357,7 @@ def update_dir_structure_file(book_id, calibrepath, first_author): if authordir != new_authordir: new_author_path = os.path.join(calibrepath, new_authordir, os.path.basename(path)) try: - os.renames(path, new_author_path) + os.renames(os.path.normcase(path), os.path.normcase(new_author_path)) localbook.path = new_authordir + '/' + localbook.path.split('/')[1] except OSError as ex: log.error("Rename author from: %s to %s: %s", path, new_author_path, ex) @@ -370,8 +370,9 @@ def update_dir_structure_file(book_id, calibrepath, first_author): new_name = get_valid_filename(localbook.title) + ' - ' + get_valid_filename(new_authordir) path_name = os.path.join(calibrepath, new_authordir, os.path.basename(path)) for file_format in localbook.data: - os.renames(os.path.join(path_name, file_format.name + '.' + file_format.format.lower()), - os.path.join(path_name, new_name + '.' + file_format.format.lower())) + os.renames(os.path.normcase( + os.path.join(path_name, file_format.name + '.' + file_format.format.lower())), + os.path.normcase(os.path.join(path_name, new_name + '.' + file_format.format.lower()))) file_format.name = new_name except OSError as ex: log.error("Rename file in path %s to %s: %s", path, new_name, ex) diff --git a/cps/uploader.py b/cps/uploader.py index 1323e3d0..74c94d83 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -116,8 +116,8 @@ def default_meta(tmp_file_path, original_file_name, original_file_extension): def pdf_meta(tmp_file_path, original_file_name, original_file_extension): doc_info = None if use_pdf_meta: - doc_info = PdfFileReader(open(tmp_file_path, 'rb')).getDocumentInfo() - + with open(tmp_file_path, 'rb') as f: + doc_info = PdfFileReader(f).getDocumentInfo() if doc_info: author = doc_info.author if doc_info.author else u'Unknown' title = doc_info.title if doc_info.title else original_file_name @@ -156,6 +156,7 @@ def pdf_preview(tmp_file_path, tmp_dir): return None except Exception as ex: log.warning('Cannot extract cover image, using default: %s', ex) + log.warning('On Windows this error could be caused by missing ghostscript') return None From 25fc6f193746f8924f3fca06fbdda260b282afdf Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Wed, 22 Jul 2020 18:44:03 +0200 Subject: [PATCH 13/27] Further fixes for #1530 (Handle improper migrated database with config_mature_content_tags, allowed_tags, denied_tags, denied_column_value, allowed_column_value NULL instead of "") --- cps/config_sql.py | 21 ++++++++------------- cps/ub.py | 22 ++++++++-------------- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/cps/config_sql.py b/cps/config_sql.py index b965eace..f0cd78af 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -214,25 +214,20 @@ class _ConfigSQL(object): return self.show_element_new_user(constants.DETAIL_RANDOM) def list_denied_tags(self): - try: - mct = self.config_denied_tags.split(",") - return [t.strip() for t in mct] - except AttributeError: - # Fix for improper migrated database with config_mature_content_tags NULL instead of "" - self.config_denied_tags = "" - return [''] + mct = self.config_denied_tags or "" + return [t.strip() for t in mct.split(",")] def list_allowed_tags(self): - mct = self.config_allowed_tags.split(",") - return [t.strip() for t in mct] + mct = self.config_allowed_tags or "" + return [t.strip() for t in mct.split(",")] def list_denied_column_values(self): - mct = self.config_denied_column_value.split(",") - return [t.strip() for t in mct] + mct = self.config_denied_column_value or "" + return [t.strip() for t in mct.split(",")] def list_allowed_column_values(self): - mct = self.config_allowed_column_value.split(",") - return [t.strip() for t in mct] + mct = self.config_allowed_column_value or "" + return [t.strip() for t in mct.split(",")] def get_log_level(self): return logger.get_level_name(self.config_log_level) diff --git a/cps/ub.py b/cps/ub.py index 8fcec1af..2b732cc2 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -176,26 +176,20 @@ class UserBase: return self.check_visibility(constants.DETAIL_RANDOM) def list_denied_tags(self): - try: - mct = self.denied_tags.split(",") - return [t.strip() for t in mct] - except AttributeError: - # Fix for improper migrated database with config_mature_content_tags NULL instead of "" - self.denied_tags="" - return [''] - + mct = self.denied_tags or "" + return [t.strip() for t in mct.split(",")] def list_allowed_tags(self): - mct = self.allowed_tags.split(",") - return [t.strip() for t in mct] + mct = self.allowed_tags or "" + return [t.strip() for t in mct.split(",")] def list_denied_column_values(self): - mct = self.denied_column_value.split(",") - return [t.strip() for t in mct] + mct = self.denied_column_value or "" + return [t.strip() for t in mct.split(",")] def list_allowed_column_values(self): - mct = self.allowed_column_value.split(",") - return [t.strip() for t in mct] + mct = self.allowed_column_value or "" + return [t.strip() for t in mct.split(",")] def __repr__(self): return '' % self.nickname From e27b08203d3e8679308931c14e5e4810631c5ea6 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 23 Jul 2020 18:41:38 +0200 Subject: [PATCH 14/27] Fix #1531 (white background on transparent cover instead of black one) --- cps/uploader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cps/uploader.py b/cps/uploader.py index 74c94d83..e693d8e8 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -35,7 +35,7 @@ except ImportError: lxmlversion = None try: - from wand.image import Image + from wand.image import Image, Color from wand import version as ImageVersion from wand.exceptions import PolicyError use_generic_pdf_cover = False @@ -149,6 +149,9 @@ def pdf_preview(tmp_file_path, tmp_dir): img.options["pdf:use-cropbox"] = "true" img.read(filename=tmp_file_path + '[0]', resolution=150) img.compression_quality = 88 + if img.alpha_channel: + img.alpha_channel = 'remove' + img.background_color = Color('white') img.save(filename=os.path.join(tmp_dir, cover_file_name)) return cover_file_name except PolicyError as ex: From 42a0639bb502e3586f65b735ead3288e590f5fdb Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 23 Jul 2020 19:23:57 +0200 Subject: [PATCH 15/27] Fix #1533 (Invalid LDAP Cert Path no longer crashes app) --- cps/services/simpleldap.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cps/services/simpleldap.py b/cps/services/simpleldap.py index 0933a933..336b0f2c 100644 --- a/cps/services/simpleldap.py +++ b/cps/services/simpleldap.py @@ -65,7 +65,10 @@ def init_app(app, config): app.config['LDAP_GROUP_OBJECT_FILTER'] = config.config_ldap_group_object_filter app.config['LDAP_GROUP_MEMBERS_FIELD'] = config.config_ldap_group_members_field - _ldap.init_app(app) + try: + _ldap.init_app(app) + except RuntimeError as e: + log.error(e) def get_object_details(user=None, group=None, query_filter=None, dn_only=False): From 1a1d105faee147738071bc44cdb577f72d2e4d6c Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 23 Jul 2020 19:31:48 +0200 Subject: [PATCH 16/27] Fix #1538 (Floating point numbers showing 2 decimals on details page --- cps/templates/detail.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cps/templates/detail.html b/cps/templates/detail.html index d4b4252c..0dfa1730 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -182,9 +182,13 @@ {% else %} {% endif %} + {% else %} + {% if c.datatype == 'float' %} + {{ '%d' % (column.value) }}{% if (column.value % 1) != 0 %}{{ '.%d' % ((column.value % 1)*100) }} {% endif %} {% else %} {{ column.value }} {% endif %} + {% endif %} {% endif %} {% endfor %} {% endif %} From a7e723d8d4ea72b2d530112456af79569d1b03b9 Mon Sep 17 00:00:00 2001 From: Efreak Date: Sat, 8 Aug 2020 01:06:19 +0000 Subject: [PATCH 17/27] Spelling/grammar in CONTRIBUTING.md --- CONTRIBUTING.md | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c06689a2..ce2bd780 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,49 +1,46 @@ ## How to contribute to Calibre-Web -First of all, we would like to thank you for reading this text. we are happy you are willing to contribute to Calibre-Web +First of all, we would like to thank you for reading this text. We are happy you are willing to contribute to Calibre-Web. ### **General** -**Communication language** is english. Google translated texts are not as bad as you might think, they are usually understandable, so don't worry if you generate your post that way. +**Communication language** is English. Google translated texts are not as bad as you might think, they are usually understandable, so don't worry if you generate your post that way. **Calibre-Web** is not **Calibre**. If you are having a question regarding Calibre please post this at their [repository](https://github.com/kovidgoyal/calibre). -**Docker-Containers** of Calibre-Web are maintained by different persons than the people who drive Calibre-Web. If we come to the conclusion during our analysis that the problem is related to Docker, we might ask you to open a new issue at the reprository of the Docker Container. +**Docker-Containers** of Calibre-Web are maintained by different persons than the people who drive Calibre-Web. If we come to the conclusion during our analysis that the problem is related to Docker, we might ask you to open a new issue at the repository of the Docker Container. -If you are having **Basic Installation Problems** with python or it's dependencies, please consider using your favorite search engine to find a solution. In case you can't find a solution, we are happy to help you. +If you are having **Basic Installation Problems** with python or its dependencies, please consider using your favorite search engine to find a solution. In case you can't find a solution, we are happy to help you. -We can offer only very limited support regarding configuration of **Reverse-Proxy Installations**, **OPDS-Reader** or other programs in combination with Calibre-Web. +We can offer only very limited support regarding configuration of **Reverse-Proxy Installations**, **OPDS-Reader** or other programs in combination with Calibre-Web. ### **Translation** -Some of the user languages in Calibre-Web having missing translations. We are happy to add the missing texts if you translate them. Create a Pull Request, create an issue with the .po file attached, or write an email to "ozzie.fernandez.isaacs@googlemail.com" with attached translation file. To display all book languages in your native language an additional file is used (iso_language_names.py). The content of this file is autogenerated with the corresponding translations of Calibre, please do not edit this file on your own. +Some of the user languages in Calibre-Web having missing translations. We are happy to add the missing texts if you translate them. Create a Pull Request, create an issue with the .po file attached, or write an email to "ozzie.fernandez.isaacs@googlemail.com" with attached translation file. To display all book languages in your native language an additional file is used (iso_language_names.py). The content of this file is auto-generated with the corresponding translations of Calibre, please do not edit this file on your own. ### **Documentation** -The Calibre-Web documentation is hosted in the Github [Wiki](https://github.com/janeczku/calibre-web/wiki). The Wiki is open to everybody, if you find a problem, feel free to correct it. If information is missing, you are welcome to add it. The content will be reviewed time by time. Please try to be consitent with the form with the other Wiki pages (e.g. the project name is Calibre-Web with 2 capital letters and a dash in between). +The Calibre-Web documentation is hosted in the Github [Wiki](https://github.com/janeczku/calibre-web/wiki). The Wiki is open to everybody, if you find a problem, feel free to correct it. If information is missing, you are welcome to add it. The content will be reviewed time by time. Please try to be consistent with the form with the other Wiki pages (e.g. the project name is Calibre-Web with 2 capital letters and a dash in between). ### **Reporting a bug** -Do not open up a GitHub issue if the bug is a **security vulnerability** in Calibre-Web. Please write intead an email to "ozzie.fernandez.isaacs@googlemail.com". +Do not open up a GitHub issue if the bug is a **security vulnerability** in Calibre-Web. Instead, please write an email to "ozzie.fernandez.isaacs@googlemail.com". Ensure the ***bug was not already reported** by searching on GitHub under [Issues](https://github.com/janeczku/calibre-web/issues). Please also check if a solution for your problem can be found in the [wiki](https://github.com/janeczku/calibre-web/wiki). -If you're unable to find an **open issue** addressing the problem, open a [new one](https://github.com/janeczku/calibre-web/issues/new?assignees=&labels=&template=bug_report.md&title=). Be sure to include a **title** and **clear description**, as much relevant information as possible, the **issue form** helps you providing the right information. Deleting the form and just pasting the stack trace doesn't speed up fixing the problem. If your issue could be resolved, consider closing the issue. +If you're unable to find an **open issue** addressing the problem, open a [new one](https://github.com/janeczku/calibre-web/issues/new?assignees=&labels=&template=bug_report.md&title=). Be sure to include a **title** and **clear description**, as much relevant information as possible, the **issue form** helps you providing the right information. Deleting the form and just pasting the stack trace doesn't speed up fixing the problem. If your issue could be resolved, consider closing the issue. ### **Feature Request** -If there is a feature missing in Calibre-Web and you can't find a feature request in the [Issues](https://github.com/janeczku/calibre-web/issues) section, you could create a [feature request](https://github.com/janeczku/calibre-web/issues/new?assignees=&labels=&template=feature_request.md&title=). -We will not extend Calibre-Web with any more login abilitys or add further files storages, or file syncing ability. Furthermore Calibre-Web is made for home usage for company inhouse usage, so requests regarding any sorts of social interaction capability, payment routines, search engine or web site analytics integration will not be implemeted. +If there is a feature missing in Calibre-Web and you can't find a feature request in the [Issues](https://github.com/janeczku/calibre-web/issues) section, you could create a [feature request](https://github.com/janeczku/calibre-web/issues/new?assignees=&labels=&template=feature_request.md&title=). +We will not extend Calibre-Web with any more login abilities or add further files storages, or file syncing ability. Furthermore Calibre-Web is made for home usage for company in-house usage, so requests regarding any sorts of social interaction capability, payment routines, search engine or web site analytics integration will not be implemented. ### **Contributing code to Calibre-Web** Open a new GitHub pull request with the patch. Ensure the PR description clearly describes the problem and solution. Include the relevant issue number if applicable. -In case your code enhances features of Calibre-Web: Create your pull request for the development branch if your enhancement consits of more than some lines of code in a local section of Calibre-Webs code. This makes it easier to test it and check all implication before it's made public. - -Please check if your code runs on Python 2.7 (still necessary in 2020) and mainly on python 3. If possible and the feature is related to operating system functions, try to check it on Windows and Linux. -Calibre-Web is automatically tested on Linux in combination with python 3.7. The code for testing is in a [seperate repo](https://github.com/OzzieIsaacs/calibre-web-test) on Github. It uses unittest and performs real system tests with selenium, would be great if you could consider also writing some tests. -A static code analysis is done by Codacy, but it's partly broken and doesn't run automatically. You could check your code with ESLint before contributing, a configuration file can be found in the projects root folder. - - +In case your code enhances features of Calibre-Web: Create your pull request for the development branch if your enhancement consists of more than some lines of code in a local section of Calibre-Webs code. This makes it easier to test it and check all implication before it's made public. +Please check if your code runs on Python 2.7 (still necessary in 2020) and mainly on python 3. If possible and the feature is related to operating system functions, try to check it on Windows and Linux. +Calibre-Web is automatically tested on Linux in combination with python 3.7. The code for testing is in a [separate repo](https://github.com/OzzieIsaacs/calibre-web-test) on Github. It uses unit tests and performs real system tests with selenium; it would be great if you could consider also writing some tests. +A static code analysis is done by Codacy, but it's partly broken and doesn't run automatically. You could check your code with ESLint before contributing, a configuration file can be found in the projects root folder. From 704dcb3e587eed371ede66f54434ea92c3270f86 Mon Sep 17 00:00:00 2001 From: Ryan Holmes Date: Sun, 9 Aug 2020 00:55:56 -0400 Subject: [PATCH 18/27] Move assignment of `new_coverpath` With the assignment originally being within the try, if the try failed, the exception wouldn't have access to the value --- cps/editbooks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index f14da1bf..ee260e66 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -223,7 +223,6 @@ def delete_book(book_id, book_format): getattr(book, cc_string).remove(del_cc) log.debug('remove ' + str(c.id)) calibre_db.session.delete(del_cc) - calibre_db.session.commit() else: modify_database_object([u''], getattr(book, cc_string), db.cc_classes[c.id], calibre_db.session, 'custom') @@ -833,8 +832,8 @@ def upload(): # move cover to final directory, including book id if has_cover: + new_coverpath = os.path.join(config.config_calibre_dir, db_book.path, "cover.jpg") try: - new_coverpath = os.path.join(config.config_calibre_dir, db_book.path, "cover.jpg") copyfile(meta.cover, new_coverpath) os.unlink(meta.cover) except OSError as e: From 28bfb06c67a6c5c1a3230f8e73be336d1dfc9111 Mon Sep 17 00:00:00 2001 From: Ryan Holmes Date: Sun, 9 Aug 2020 00:57:30 -0400 Subject: [PATCH 19/27] Revert the deletion of another line This was done by mistake, whoops! --- cps/editbooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cps/editbooks.py b/cps/editbooks.py index ee260e66..7330572b 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -223,6 +223,7 @@ def delete_book(book_id, book_format): getattr(book, cc_string).remove(del_cc) log.debug('remove ' + str(c.id)) calibre_db.session.delete(del_cc) + calibre_db.session.commit() else: modify_database_object([u''], getattr(book, cc_string), db.cc_classes[c.id], calibre_db.session, 'custom') From b309c1fc91409b9d01467438b6c96b928f77f8e9 Mon Sep 17 00:00:00 2001 From: OzzieIsaacs Date: Sun, 9 Aug 2020 15:11:54 +0200 Subject: [PATCH 20/27] Fix filenames (Escape "\") for searching calibre excecutable on windows --- cps/config_sql.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cps/config_sql.py b/cps/config_sql.py index f0cd78af..172ec88d 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -360,10 +360,10 @@ def _migrate_table(session, orm_class): def autodetect_calibre_binary(): if sys.platform == "win32": - calibre_path = ["C:\\program files\calibre\ebook-convert.exe", - "C:\\program files(x86)\calibre\ebook-convert.exe", - "C:\\program files(x86)\calibre2\ebook-convert.exe", - "C:\\program files\calibre2\ebook-convert.exe"] + calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe", + "C:\\program files(x86)\\calibre\\ebook-convert.exe", + "C:\\program files(x86)\\calibre2\\ebook-convert.exe", + "C:\\program files\\calibre2\\ebook-convert.exe"] else: calibre_path = ["/opt/calibre/ebook-convert"] for element in calibre_path: From 76b0505bd914df6b59f2cf1665002d28582809c1 Mon Sep 17 00:00:00 2001 From: Marvel Renju Date: Mon, 10 Aug 2020 03:42:36 +0100 Subject: [PATCH 21/27] Change all "conn = engine.connect()" statements to "with engine.connect() as conn:" in ub.py Changed all the statements in ub.py to use python context managers so that conn.close() is called automatically. (https://docs.sqlalchemy.org/en/13/core/connections.html) --- cps/ub.py | 90 +++++++++++++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/cps/ub.py b/cps/ub.py index 2b732cc2..d5f51d51 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -474,34 +474,34 @@ def migrate_Database(session): ArchivedBook.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "registration"): ReadBook.__table__.create(bind=engine) - conn = engine.connect() - conn.execute("insert into registration (domain, allow) values('%.%',1)") + with engine.connect() as conn: + conn.execute("insert into registration (domain, allow) values('%.%',1)") session.commit() try: session.query(exists().where(Registration.allow)).scalar() session.commit() except exc.OperationalError: # Database is not compatible, some columns are missing - conn = engine.connect() - conn.execute("ALTER TABLE registration ADD column 'allow' INTEGER") - conn.execute("update registration set 'allow' = 1") + with engine.connect() as conn: + conn.execute("ALTER TABLE registration ADD column 'allow' INTEGER") + conn.execute("update registration set 'allow' = 1") session.commit() try: session.query(exists().where(RemoteAuthToken.token_type)).scalar() session.commit() except exc.OperationalError: # Database is not compatible, some columns are missing - conn = engine.connect() - conn.execute("ALTER TABLE remote_auth_token ADD column 'token_type' INTEGER DEFAULT 0") - conn.execute("update remote_auth_token set 'token_type' = 0") + with engine.connect() as conn: + conn.execute("ALTER TABLE remote_auth_token ADD column 'token_type' INTEGER DEFAULT 0") + conn.execute("update remote_auth_token set 'token_type' = 0") session.commit() try: session.query(exists().where(ReadBook.read_status)).scalar() except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE book_read_link ADD column 'read_status' INTEGER DEFAULT 0") - conn.execute("UPDATE book_read_link SET 'read_status' = 1 WHERE is_read") - conn.execute("ALTER TABLE book_read_link ADD column 'last_modified' DATETIME") - conn.execute("ALTER TABLE book_read_link ADD column 'last_time_started_reading' DATETIME") - conn.execute("ALTER TABLE book_read_link ADD column 'times_started_reading' INTEGER DEFAULT 0") + with engine.connect() as conn: + conn.execute("ALTER TABLE book_read_link ADD column 'read_status' INTEGER DEFAULT 0") + conn.execute("UPDATE book_read_link SET 'read_status' = 1 WHERE is_read") + conn.execute("ALTER TABLE book_read_link ADD column 'last_modified' DATETIME") + conn.execute("ALTER TABLE book_read_link ADD column 'last_time_started_reading' DATETIME") + conn.execute("ALTER TABLE book_read_link ADD column 'times_started_reading' INTEGER DEFAULT 0") session.commit() test = session.query(ReadBook).filter(ReadBook.last_modified == None).all() for book in test: @@ -510,11 +510,11 @@ def migrate_Database(session): try: session.query(exists().where(Shelf.uuid)).scalar() except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE shelf ADD column 'uuid' STRING") - conn.execute("ALTER TABLE shelf ADD column 'created' DATETIME") - conn.execute("ALTER TABLE shelf ADD column 'last_modified' DATETIME") - conn.execute("ALTER TABLE book_shelf_link ADD column 'date_added' DATETIME") + with engine.connect() as conn: + conn.execute("ALTER TABLE shelf ADD column 'uuid' STRING") + conn.execute("ALTER TABLE shelf ADD column 'created' DATETIME") + conn.execute("ALTER TABLE shelf ADD column 'last_modified' DATETIME") + conn.execute("ALTER TABLE book_shelf_link ADD column 'date_added' DATETIME") for shelf in session.query(Shelf).all(): shelf.uuid = str(uuid.uuid4()) shelf.created = datetime.datetime.now() @@ -525,31 +525,31 @@ def migrate_Database(session): # Handle table exists, but no content cnt = session.query(Registration).count() if not cnt: - conn = engine.connect() - conn.execute("insert into registration (domain, allow) values('%.%',1)") + with engine.connect() as conn: + conn.execute("insert into registration (domain, allow) values('%.%',1)") session.commit() try: session.query(exists().where(BookShelf.order)).scalar() except exc.OperationalError: # Database is not compatible, some columns are missing - conn = engine.connect() - conn.execute("ALTER TABLE book_shelf_link ADD column 'order' INTEGER DEFAULT 1") + with engine.connect() as conn: + conn.execute("ALTER TABLE book_shelf_link ADD column 'order' INTEGER DEFAULT 1") session.commit() try: create = False session.query(exists().where(User.sidebar_view)).scalar() except exc.OperationalError: # Database is not compatible, some columns are missing - conn = engine.connect() - conn.execute("ALTER TABLE user ADD column `sidebar_view` Integer DEFAULT 1") + with engine.connect() as conn: + conn.execute("ALTER TABLE user ADD column `sidebar_view` Integer DEFAULT 1") session.commit() create = True try: if create: - conn = engine.connect() - conn.execute("SELECT language_books FROM user") + with engine.connect() as conn: + conn.execute("SELECT language_books FROM user") session.commit() except exc.OperationalError: - conn = engine.connect() - conn.execute("UPDATE user SET 'sidebar_view' = (random_books* :side_random + language_books * :side_lang " + with engine.connect() as conn: + conn.execute("UPDATE user SET 'sidebar_view' = (random_books* :side_random + language_books * :side_lang " "+ series_books * :side_series + category_books * :side_category + hot_books * " ":side_hot + :side_autor + :detail_random)", {'side_random': constants.SIDEBAR_RANDOM, 'side_lang': constants.SIDEBAR_LANGUAGE, @@ -560,29 +560,29 @@ def migrate_Database(session): try: session.query(exists().where(User.denied_tags)).scalar() except exc.OperationalError: # Database is not compatible, some columns are missing - conn = engine.connect() - conn.execute("ALTER TABLE user ADD column `denied_tags` String DEFAULT ''") - conn.execute("ALTER TABLE user ADD column `allowed_tags` String DEFAULT ''") - conn.execute("ALTER TABLE user ADD column `denied_column_value` String DEFAULT ''") - conn.execute("ALTER TABLE user ADD column `allowed_column_value` String DEFAULT ''") + with engine.connect() as conn: + conn.execute("ALTER TABLE user ADD column `denied_tags` String DEFAULT ''") + conn.execute("ALTER TABLE user ADD column `allowed_tags` String DEFAULT ''") + conn.execute("ALTER TABLE user ADD column `denied_column_value` String DEFAULT ''") + conn.execute("ALTER TABLE user ADD column `allowed_column_value` String DEFAULT ''") session.commit() try: session.query(exists().where(User.series_view)).scalar() except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE user ADD column `series_view` VARCHAR(10) DEFAULT 'list'") + with engine.connect() as conn: + conn.execute("ALTER TABLE user ADD column `series_view` VARCHAR(10) DEFAULT 'list'") if session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS).first() \ is None: create_anonymous_user(session) try: # check if one table with autoincrement is existing (should be user table) - conn = engine.connect() - conn.execute("SELECT COUNT(*) FROM sqlite_sequence WHERE name='user'") + with engine.connect() as conn: + conn.execute("SELECT COUNT(*) FROM sqlite_sequence WHERE name='user'") except exc.OperationalError: # Create new table user_id and copy contents of table user into it - conn = engine.connect() - conn.execute("CREATE TABLE user_id (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT," + with engine.connect() as conn: + conn.execute("CREATE TABLE user_id (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT," "nickname VARCHAR(64)," "email VARCHAR(120)," "role SMALLINT," @@ -594,19 +594,19 @@ def migrate_Database(session): "series_view VARCHAR(10)," "UNIQUE (nickname)," "UNIQUE (email))") - conn.execute("INSERT INTO user_id(id, nickname, email, role, password, kindle_mail,locale," + conn.execute("INSERT INTO user_id(id, nickname, email, role, password, kindle_mail,locale," "sidebar_view, default_language, series_view) " "SELECT id, nickname, email, role, password, kindle_mail, locale," "sidebar_view, default_language FROM user") # delete old user table and rename new user_id table to user: - conn.execute("DROP TABLE user") - conn.execute("ALTER TABLE user_id RENAME TO user") + conn.execute("DROP TABLE user") + conn.execute("ALTER TABLE user_id RENAME TO user") session.commit() # Remove login capability of user Guest try: - conn = engine.connect() - conn.execute("UPDATE user SET password='' where nickname = 'Guest' and password !=''") + with engine.connect() as conn: + conn.execute("UPDATE user SET password='' where nickname = 'Guest' and password !=''") session.commit() except exc.OperationalError: print('Settings database is not writeable. Exiting...') From 0e1ec5034e2b6b199c5829e749f0889ebc55ff3a Mon Sep 17 00:00:00 2001 From: blitzmann Date: Tue, 11 Aug 2020 12:44:55 -0400 Subject: [PATCH 22/27] Fix for #1564 - using memory database in multiple threads. See also: https://docs.sqlalchemy.org/en/13/dialects/sqlite.html#using-a-memory-database-in-multiple-threads --- cps/db.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cps/db.py b/cps/db.py index 1296c84a..02a0aa60 100644 --- a/cps/db.py +++ b/cps/db.py @@ -32,6 +32,7 @@ from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float from sqlalchemy.orm import relationship, sessionmaker, scoped_session from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.exc import OperationalError +from sqlalchemy.pool import StaticPool from flask_login import current_user from sqlalchemy.sql.expression import and_, true, false, text, func, or_ from babel import Locale as LC @@ -386,7 +387,8 @@ class CalibreDB(threading.Thread): self.engine = create_engine('sqlite://', echo=False, isolation_level="SERIALIZABLE", - connect_args={'check_same_thread': False}) + connect_args={'check_same_thread': False}, + poolclass=StaticPool) self.engine.execute("attach database '{}' as calibre;".format(dbpath)) self.engine.execute("attach database '{}' as app_settings;".format(app_db_path)) From 94ad93ebd789a50c9f32c78e2eeaa861a069eb1a Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 25 Jul 2020 19:39:19 +0200 Subject: [PATCH 23/27] Added series like custom columns #1501 --- cps/db.py | 76 ++++++++++++++++++++++-------------- cps/jinjia.py | 7 ++++ cps/templates/book_edit.html | 17 ++++---- cps/templates/detail.html | 8 +++- 4 files changed, 69 insertions(+), 39 deletions(-) diff --git a/cps/db.py b/cps/db.py index 1296c84a..014a615d 100644 --- a/cps/db.py +++ b/cps/db.py @@ -34,6 +34,7 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.exc import OperationalError from flask_login import current_user from sqlalchemy.sql.expression import and_, true, false, text, func, or_ +from sqlalchemy.ext.associationproxy import association_proxy from babel import Locale as LC from babel.core import UnknownLocaleError from flask_babel import gettext as _ @@ -48,7 +49,7 @@ except ImportError: use_unidecode = False -cc_exceptions = ['datetime', 'comments', 'composite', 'series'] +cc_exceptions = ['datetime', 'comments', 'composite'] cc_classes = {} Base = declarative_base() @@ -280,7 +281,7 @@ class Books(Base): flags = Column(Integer, nullable=False, default=1) authors = relationship('Authors', secondary=books_authors_link, backref='books') - tags = relationship('Tags', secondary=books_tags_link, backref='books',order_by="Tags.name") + tags = relationship('Tags', secondary=books_tags_link, backref='books', order_by="Tags.name") comments = relationship('Comments', backref='books') data = relationship('Data', backref='books') series = relationship('Series', secondary=books_series_link, backref='books') @@ -406,33 +407,45 @@ class CalibreDB(threading.Thread): books_custom_column_links = {} for row in cc: if row.datatype not in cc_exceptions: - books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), - primary_key=True), - Column('value', Integer, - ForeignKey('custom_column_' + str(row.id) + '.id'), - primary_key=True) - ) - cc_ids.append([row.id, row.datatype]) - if row.datatype == 'bool': - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'book': Column(Integer, ForeignKey('books.id')), - 'value': Column(Boolean)} - elif row.datatype == 'int': - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'book': Column(Integer, ForeignKey('books.id')), - 'value': Column(Integer)} - elif row.datatype == 'float': - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'book': Column(Integer, ForeignKey('books.id')), - 'value': Column(Float)} + if row.datatype == 'series': + dicttable = {'__tablename__': 'books_custom_column_' + str(row.id) + '_link', + 'id': Column(Integer, primary_key=True), + 'book': Column(Integer, ForeignKey('books.id'), + primary_key=True), + 'map_value': Column('value', Integer, + ForeignKey('custom_column_' + + str(row.id) + '.id'), + primary_key=True), + 'extra': Column(Float), + 'asoc' : relationship('Custom_Column_' + str(row.id), uselist=False), + 'value' : association_proxy('asoc', 'value') + } + books_custom_column_links[row.id] = type(str('Books_Custom_Column_' + str(row.id) + '_link'), + (Base,), dicttable) else: - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'value': Column(String)} + books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', + Base.metadata, + Column('book', Integer, ForeignKey('books.id'), + primary_key=True), + Column('value', Integer, + ForeignKey('custom_column_' + + str(row.id) + '.id'), + primary_key=True) + ) + cc_ids.append([row.id, row.datatype]) + + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True)} + if row.datatype == 'float': + ccdict['value'] = Column(Float) + elif row.datatype == 'int': + ccdict['value'] = Column(Integer) + elif row.datatype == 'bool': + ccdict['value'] = Column(Boolean) + else: + ccdict['value'] = Column(String) + if row.datatype in ['float', 'int', 'bool']: + ccdict['book'] = Column(Integer, ForeignKey('books.id')) cc_classes[row.id] = type(str('Custom_Column_' + str(row.id)), (Base,), ccdict) for cc_id in cc_ids: @@ -440,9 +453,14 @@ class CalibreDB(threading.Thread): setattr(Books, 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]], - primaryjoin=( + primaryjoin=( # ToDo: Check Remove Books.id == cc_classes[cc_id[0]].book), backref='books')) + elif (cc_id[1] == 'series'): + setattr(Books, + 'custom_column_' + str(cc_id[0]), + relationship(books_custom_column_links[cc_id[0]], + backref='books')) else: setattr(Books, 'custom_column_' + str(cc_id[0]), diff --git a/cps/jinjia.py b/cps/jinjia.py index 28c2621a..c91534eb 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -111,3 +111,10 @@ def timestamptodate(date, fmt=None): @jinjia.app_template_filter('yesno') def yesno(value, yes, no): return yes if value else no + +@jinjia.app_template_filter('formatfloat') +def formatfloat(value, decimals=1): + formatedstring = '%d' % value + if (value % 1) != 0: + formatedstring = ('%s.%d' % (formatedstring, (value % 1) * 10**decimals)).rstrip('0') + return formatedstring diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 8bbe2460..96620a35 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -132,19 +132,20 @@ {% endif %} - {% if c.datatype in ['text', 'series'] and not c.is_multiple %} - 0 %} - value="{{ book['custom_column_' ~ c.id][0].value }}" - {% endif %}> - {% endif %} - - {% if c.datatype in ['text', 'series'] and c.is_multiple %} + {% if c.datatype == 'text' %} 0 %} value="{% for column in book['custom_column_' ~ c.id] %}{{ column.value.strip() }}{% if not loop.last %}, {% endif %}{% endfor %}"{% endif %}> {% endif %} + {% if c.datatype == 'series' %} + 0 %} + value="{% for column in book['custom_column_' ~ c.id] %} {{ '%s [%s]' % (book['custom_column_' ~ c.id][0].value, book['custom_column_' ~ c.id][0].extra|formatfloat(2)) }}{% if not loop.last %}, {% endif %}{% endfor %}" + {% endif %}> + {% endif %} + + {% if c.datatype == 'enumeration' %}