From 7f6e88ce5ea36ea26954efeea7a80d334aa3ed91 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Wed, 1 Jun 2022 22:06:28 +0200 Subject: [PATCH 1/7] fixes bug 2419 --- cps/admin.py | 7 ++++--- cps/db.py | 9 ++++----- cps/static/js/main.js | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 3c889643..20b533c7 100755 --- a/cps/admin.py +++ b/cps/admin.py @@ -1668,9 +1668,10 @@ def _db_configuration_update_helper(): if db_change or not db_valid or not config.db_configured \ or config.config_calibre_dir != to_save["config_calibre_dir"]: - if not calibre_db.setup_db(to_save['config_calibre_dir'], ub.app_DB_path): - return _db_configuration_result(_('DB Location is not Valid, Please Enter Correct Path'), - gdrive_error) + if not os.path.exists(metadata_db) or not to_save['config_calibre_dir']: + return _db_configuration_result(_('DB Location is not Valid, Please Enter Correct Path'), gdrive_error) + else: + calibre_db.setup_db(to_save['config_calibre_dir'], ub.app_DB_path) config.store_calibre_uuid(calibre_db, db.Library_Id) # if db changed -> delete shelfs, delete download books, delete read books, kobo sync... if db_change: diff --git a/cps/db.py b/cps/db.py index c4f7ed38..0b98f683 100644 --- a/cps/db.py +++ b/cps/db.py @@ -567,12 +567,12 @@ class CalibreDB: if not config_calibre_dir: cls.config.invalidate() - return False + return None dbpath = os.path.join(config_calibre_dir, "metadata.db") if not os.path.exists(dbpath): cls.config.invalidate() - return False + return None try: cls.engine = create_engine('sqlite://', @@ -588,7 +588,7 @@ class CalibreDB: # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302 except Exception as ex: cls.config.invalidate(ex) - return False + return None cls.config.db_configured = True @@ -598,7 +598,7 @@ class CalibreDB: cls.setup_db_cc_classes(cc) except OperationalError as e: log.error_or_exception(e) - return False + return None cls.session_factory = scoped_session(sessionmaker(autocommit=False, autoflush=True, @@ -607,7 +607,6 @@ class CalibreDB: inst.init_session() cls._init = True - return True def get_book(self, book_id): return self.session.query(Books).filter(Books.id == book_id).first() diff --git a/cps/static/js/main.js b/cps/static/js/main.js index 04d47d6b..cfb0b2f0 100644 --- a/cps/static/js/main.js +++ b/cps/static/js/main.js @@ -677,8 +677,8 @@ $(function() { if ( data.change ) { if ( data.valid ) { confirmDialog( - "db_submit", - "GeneralChangeModal", + "db_submit", + "GeneralChangeModal", 0, changeDbSettings ); From 25f2af3f03ae84f4c0df685058b4d9bf9e455fd9 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 5 Jun 2022 10:06:43 +0200 Subject: [PATCH 2/7] Whitespaces are striped from email address (fixes #2432) --- cps/admin.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 45d69dbc..ae9da540 100755 --- a/cps/admin.py +++ b/cps/admin.py @@ -1264,13 +1264,13 @@ def update_mailsettings(): return edit_mailsettings() else: - _config_string(to_save, "mail_server") _config_int(to_save, "mail_port") _config_int(to_save, "mail_use_ssl") - _config_string(to_save, "mail_login") _config_string(to_save, "mail_password") - _config_string(to_save, "mail_from") _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024) + config.mail_server = to_save.get('mail_server', "").strip() + config.mail_from = to_save.get('mail_login', "").strip() + config.email = to_save.get('mail_login', "").strip() try: config.save() except (OperationalError, InvalidRequestError) as e: @@ -1278,6 +1278,9 @@ def update_mailsettings(): log.error_or_exception("Settings Database error: {}".format(e)) flash(_(u"Database error: %(error)s.", error=e.orig), category="error") return edit_mailsettings() + except Exception as e: + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") + return edit_mailsettings() if to_save.get("test"): if current_user.email: From 91df265d40ac56bef1d5d388dfeb6a622168582a Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Wed, 8 Jun 2022 17:17:07 +0200 Subject: [PATCH 3/7] Fix for #2437 (advanced search for read status crashes calibre-web) --- cps/db.py | 8 +++--- cps/helper.py | 4 +-- cps/render_template.py | 13 ---------- cps/search.py | 59 ++++++++++++------------------------------ cps/web.py | 4 +-- 5 files changed, 25 insertions(+), 63 deletions(-) diff --git a/cps/db.py b/cps/db.py index 0b98f683..d1c5ec71 100644 --- a/cps/db.py +++ b/cps/db.py @@ -627,7 +627,7 @@ class CalibreDB: .join(read_column, read_column.book == book_id, isouter=True)) except (KeyError, AttributeError, IndexError): - log.error("Custom Column No.{} is not existing in calibre database".format(read_column)) + log.error("Custom Column No.{} does not exist in calibre database".format(read_column)) # Skip linking read column and return None instead of read status bd = self.session.query(Books, None, ub.ArchivedBook.is_archived) return (bd.filter(Books.id == book_id) @@ -674,9 +674,9 @@ class CalibreDB: except (KeyError, AttributeError, IndexError): pos_content_cc_filter = false() neg_content_cc_filter = true() - log.error("Custom Column No.{} is not existing in calibre database".format( + log.error("Custom Column No.{} does not exist in calibre database".format( self.config.config_restricted_column)) - flash(_("Custom Column No.%(column)d is not existing in calibre database", + flash(_("Custom Column No.%(column)d does not exist in calibre database", column=self.config.config_restricted_column), category="error") @@ -699,7 +699,7 @@ class CalibreDB: .select_from(Books) .outerjoin(read_column, read_column.book == Books.id)) except (KeyError, AttributeError, IndexError): - log.error("Custom Column No.{} is not existing in calibre database".format(config_read_column)) + log.error("Custom Column No.{} does not exist in calibre database".format(config_read_column)) # Skip linking read column and return None instead of read status query = self.session.query(database, None, ub.ArchivedBook.is_archived) return query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id, diff --git a/cps/helper.py b/cps/helper.py index 6db991b0..f936ed39 100755 --- a/cps/helper.py +++ b/cps/helper.py @@ -336,8 +336,8 @@ def edit_book_read_status(book_id, read_status=None): calibre_db.session.commit() except (KeyError, AttributeError, IndexError): log.error( - "Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) - return "Custom Column No.{} is not existing in calibre database".format(config.config_read_column) + "Custom Column No.{} does not exist in calibre database".format(config.config_read_column)) + return "Custom Column No.{} does not exist in calibre database".format(config.config_read_column) except (OperationalError, InvalidRequestError) as ex: calibre_db.session.rollback() log.error(u"Read status could not set: {}".format(ex)) diff --git a/cps/render_template.py b/cps/render_template.py index 0750a9c4..04a50bbb 100644 --- a/cps/render_template.py +++ b/cps/render_template.py @@ -101,19 +101,6 @@ def get_sidebar_config(kwargs=None): "show_text": _('Show Books List'), "config_show": content}) return sidebar, simple -'''def get_readbooks_ids(): - if not config.config_read_column: - readBooks = ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id))\ - .filter(ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED).all() - return frozenset([x.book_id for x in readBooks]) - else: - try: - readBooks = calibre_db.session.query(db.cc_classes[config.config_read_column])\ - .filter(db.cc_classes[config.config_read_column].value == True).all() - return frozenset([x.book for x in readBooks]) - except (KeyError, AttributeError, IndexError): - log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) - return []''' # Returns the template for rendering and includes the instance name def render_title_template(*args, **kwargs): diff --git a/cps/search.py b/cps/search.py index 602881bf..88d790fc 100644 --- a/cps/search.py +++ b/cps/search.py @@ -22,7 +22,7 @@ from flask import session as flask_session from flask_login import current_user from flask_babel import format_date from flask_babel import gettext as _ -from sqlalchemy.sql.expression import func, not_, and_, or_, text +from sqlalchemy.sql.expression import func, not_, and_, or_, text, true from sqlalchemy.sql.functions import coalesce from . import logger, db, calibre_db, config, ub @@ -119,32 +119,23 @@ def adv_search_ratings(q, rating_high, rating_low): return q -def adv_search_read_status(q, read_status): - if read_status: - if config.config_read_column: - try: - if read_status == "True": - q = q.join(db.cc_classes[config.config_read_column], isouter=True) \ - .filter(db.cc_classes[config.config_read_column].value == True) - else: - q = q.join(db.cc_classes[config.config_read_column], isouter=True) \ - .filter(coalesce(db.cc_classes[config.config_read_column].value, False) != True) - except (KeyError, AttributeError): - log.error(u"Custom Column No.%d is not existing in calibre database", config.config_read_column) - flash(_("Custom Column No.%(column)d is not existing in calibre database", - column=config.config_read_column), - category="error") - return q +def adv_search_read_status(read_status): + if not config.config_read_column: + if read_status == "True": + db_filter = and_(ub.ReadBook.user_id == int(current_user.id), + ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED) else: + db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED + else: + try: if read_status == "True": - q = q.join(ub.ReadBook, db.Books.id == ub.ReadBook.book_id, isouter=True) \ - .filter(ub.ReadBook.user_id == int(current_user.id), - ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED) + db_filter = db.cc_classes[config.config_read_column].value == True else: - q = q.join(ub.ReadBook, db.Books.id == ub.ReadBook.book_id, isouter=True) \ - .filter(ub.ReadBook.user_id == int(current_user.id), - coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED) - return q + db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True + except (KeyError, AttributeError, IndexError): + log.error("Custom Column No.{} does not exist in calibre database".format(config.config_read_column)) + return true() + return db_filter def adv_search_extension(q, include_extension_inputs, exclude_extension_inputs): @@ -238,23 +229,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): cc = calibre_db.get_cc_columns(config, filter_config_custom_read=True) calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase) - if not config.config_read_column: - query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, ub.ReadBook).select_from(db.Books) - .outerjoin(ub.ReadBook, and_(db.Books.id == ub.ReadBook.book_id, - int(current_user.id) == ub.ReadBook.user_id))) - else: - try: - read_column = cc[config.config_read_column] - query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, read_column.value) - .select_from(db.Books) - .outerjoin(read_column, read_column.book == db.Books.id)) - except (KeyError, AttributeError): - log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column) - # Skip linking read column - query = calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, None) - query = query.outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id)) - + query = calibre_db.generate_linked_query(config.config_read_column, db.Books) q = query.outerjoin(db.books_series_link, db.Books.id == db.books_series_link.c.book)\ .outerjoin(db.Series)\ .filter(calibre_db.common_filters(True)) @@ -324,7 +299,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): q = q.filter(func.datetime(db.Books.pubdate) > func.datetime(pub_start)) if pub_end: q = q.filter(func.datetime(db.Books.pubdate) < func.datetime(pub_end)) - q = adv_search_read_status(q, read_status) + q = q.filter(adv_search_read_status(read_status)) if publisher: q = q.filter(db.Books.publishers.any(func.lower(db.Publishers.name).ilike("%" + publisher + "%"))) q = adv_search_tag(q, tags['include_tag'], tags['exclude_tag']) diff --git a/cps/web.py b/cps/web.py index 4a2a796b..dcccbc32 100755 --- a/cps/web.py +++ b/cps/web.py @@ -715,9 +715,9 @@ def render_read_books(page, are_read, as_xml=False, order=None): else: db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True except (KeyError, AttributeError, IndexError): - log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) + log.error("Custom Column No.{} does not exist in calibre database".format(config.config_read_column)) if not as_xml: - flash(_("Custom Column No.%(column)d is not existing in calibre database", + flash(_("Custom Column No.%(column)d does not exist in calibre database", column=config.config_read_column), category="error") return redirect(url_for("web.index")) From 78b45f716abede09ba8c671372b8a4fcfe11ae52 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Wed, 8 Jun 2022 20:25:35 +0200 Subject: [PATCH 4/7] Bugfix for #2433 (LazyString is not JSON serializable if one of kepubify/ebook-convert/unrar is not installed) --- cps/debug_info.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cps/debug_info.py b/cps/debug_info.py index 64fda557..6cb30edb 100644 --- a/cps/debug_info.py +++ b/cps/debug_info.py @@ -22,6 +22,7 @@ import glob import zipfile import json from io import BytesIO +from flask_babel.speaklater import LazyString import os @@ -32,6 +33,12 @@ from .about import collect_stats log = logger.create() +class lazyEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, LazyString): + return str(obj) + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) def assemble_logfiles(file_name): log_list = sorted(glob.glob(file_name + '*'), reverse=True) @@ -58,8 +65,8 @@ def send_debug(): file_list.remove(element) memory_zip = BytesIO() with zipfile.ZipFile(memory_zip, 'w', compression=zipfile.ZIP_DEFLATED) as zf: - zf.writestr('settings.txt', json.dumps(config.toDict())) - zf.writestr('libs.txt', json.dumps(collect_stats())) + zf.writestr('settings.txt', json.dumps(config.toDict(), sort_keys=True, indent=2)) + zf.writestr('libs.txt', json.dumps(collect_stats(), sort_keys=True, indent=2, cls=lazyEncoder)) for fp in file_list: zf.write(fp, os.path.basename(fp)) memory_zip.seek(0) From 89d522e389b2dad36722152029f4b5863e115c68 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 11 Jun 2022 11:35:02 +0200 Subject: [PATCH 5/7] Fix for #2445 (book read status can't be set if custom column is linked and read status was set before) --- cps/helper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index f936ed39..d40ffc33 100755 --- a/cps/helper.py +++ b/cps/helper.py @@ -322,12 +322,12 @@ def edit_book_read_status(book_id, read_status=None): try: calibre_db.update_title_sort(config) book = calibre_db.get_filtered_book(book_id) - read_status = getattr(book, 'custom_column_' + str(config.config_read_column)) - if len(read_status): + book_read_status = getattr(book, 'custom_column_' + str(config.config_read_column)) + if len(book_read_status): if read_status is None: - read_status[0].value = not read_status[0].value + book_read_status[0].value = not book_read_status[0].value else: - read_status[0].value = read_status is True + book_read_status[0].value = read_status is True calibre_db.session.commit() else: cc_class = db.cc_classes[config.config_read_column] From f85b587d0a6d2e35a648337d7952b980d28caf39 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Mon, 13 Jun 2022 17:04:35 +0200 Subject: [PATCH 6/7] Prevent converting of kepub on every new user (#2446) Added error logging message if convert fails If convert only task, only convert book message is logged --- cps/kobo_auth.py | 2 +- cps/tasks/convert.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cps/kobo_auth.py b/cps/kobo_auth.py index 9865b993..ea9b71b1 100644 --- a/cps/kobo_auth.py +++ b/cps/kobo_auth.py @@ -107,7 +107,7 @@ def generate_auth_token(user_id): for book in books: formats = [data.format for data in book.data] - if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats: + if 'KEPUB' not in formats and config.config_kepubifypath and 'EPUB' in formats: helper.convert_book_format(book.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name) return render_title_template( diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index a9f0f055..48fd528e 100755 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -188,6 +188,8 @@ class TaskConvert(CalibreTask): log.info("ebook converter failed with error while converting book") if not error_message: error_message = N_('Ebook converter failed with unknown error') + else: + log.error(error_message) self._handleError(error_message) return @@ -273,7 +275,10 @@ class TaskConvert(CalibreTask): return N_("Convert") def __str__(self): - return "Convert {} {}".format(self.book_id, self.ereader_mail) + if self.ereader_mail: + return "Convert {} {}".format(self.book_id, self.ereader_mail) + else: + return "Convert {}".format(self.book_id) @property def is_cancellable(self): From 02e1be09df667c41ff8f9f775047c921b55d0589 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Mon, 13 Jun 2022 17:54:35 +0200 Subject: [PATCH 7/7] Catch StaleDataError Update requirements jsonschema --- cps/editbooks.py | 11 ++++++----- cps/gdriveutils.py | 3 ++- optional-requirements.txt | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index d3615050..0e7588ea 100755 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -39,6 +39,7 @@ from flask_babel import lazy_gettext as N_ from flask_babel import get_locale from flask_login import current_user, login_required from sqlalchemy.exc import OperationalError, IntegrityError +from sqlalchemy.orm.exc import StaleDataError from . import constants, logger, isoLanguages, gdriveutils, uploader, helper, kobo_sync_status from . import config, ub, db, calibre_db @@ -221,7 +222,7 @@ def edit_book(book_id): calibre_db.session.rollback() flash(str(e), category="error") return redirect(url_for('web.show_book', book_id=book.id)) - except (OperationalError, IntegrityError) as e: + except (OperationalError, IntegrityError, StaleDataError) as e: log.error_or_exception("Database error: {}".format(e)) calibre_db.session.rollback() flash(_(u"Database error: %(error)s.", error=e.orig), category="error") @@ -295,7 +296,7 @@ def upload(): else: resp = {"location": url_for('web.show_book', book_id=book_id)} return Response(json.dumps(resp), mimetype='application/json') - except (OperationalError, IntegrityError) as e: + except (OperationalError, IntegrityError, StaleDataError) as e: calibre_db.session.rollback() log.error_or_exception("Database error: {}".format(e)) flash(_(u"Database error: %(error)s.", error=e.orig), category="error") @@ -443,7 +444,7 @@ def edit_list_book(param): if param == 'title' and vals.get('checkT') == "false": book.sort = sort_param calibre_db.session.commit() - except (OperationalError, IntegrityError) as e: + except (OperationalError, IntegrityError, StaleDataError) as e: calibre_db.session.rollback() log.error_or_exception("Database error: {}".format(e)) ret = Response(json.dumps({'success': False, @@ -556,7 +557,7 @@ def table_xchange_author_title(): book.last_modified = datetime.utcnow() try: calibre_db.session.commit() - except (OperationalError, IntegrityError) as e: + except (OperationalError, IntegrityError, StaleDataError) as e: calibre_db.session.rollback() log.error_or_exception("Database error: %s", e) return json.dumps({'success': False}) @@ -1190,7 +1191,7 @@ def upload_single_file(file_request, book, book_id): calibre_db.session.add(db_format) calibre_db.session.commit() calibre_db.update_title_sort(config) - except (OperationalError, IntegrityError) as e: + except (OperationalError, IntegrityError, StaleDataError) as e: calibre_db.session.rollback() log.error_or_exception("Database error: {}".format(e)) flash(_(u"Database error: %(error)s.", error=e.orig), category="error") diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index e2e0a536..866defaa 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -33,6 +33,7 @@ try: except ImportError: from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.exc import OperationalError, InvalidRequestError, IntegrityError +from sqlalchemy.orm.exc import StaleDataError from sqlalchemy.sql.expression import text try: @@ -318,7 +319,7 @@ def getFolderId(path, drive): session.commit() else: currentFolderId = storedPathName.gdrive_id - except (OperationalError, IntegrityError) as ex: + except (OperationalError, IntegrityError, StaleDataError) as ex: log.error_or_exception('Database error: {}'.format(ex)) session.rollback() except ApiRequestError as ex: diff --git a/optional-requirements.txt b/optional-requirements.txt index a0384bcd..97f355f2 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -41,4 +41,4 @@ natsort>=2.2.0,<8.2.0 comicapi>=2.2.0,<2.3.0 # Kobo integration -jsonschema>=3.2.0,<4.6.0 +jsonschema>=3.2.0,<4.7.0