Merge branch 'master' into Develop

cps/admin.py

@@ -1264,13 +1264,13 @@ def update_mailsettings():
             return edit_mailsettings()
 
     else:
-        _config_string(to_save, "mail_server")
         _config_int(to_save, "mail_port")
         _config_int(to_save, "mail_use_ssl")
-        _config_string(to_save, "mail_login")
         _config_string(to_save, "mail_password")
-        _config_string(to_save, "mail_from")
         _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024)
+        config.mail_server = to_save.get('mail_server', "").strip()
+        config.mail_from = to_save.get('mail_from', "").strip()
+        config.mail_login = to_save.get('mail_login', "").strip()
     try:
         config.save()
     except (OperationalError, InvalidRequestError) as e:
@@ -1278,6 +1278,9 @@ def update_mailsettings():
         log.error_or_exception("Settings Database error: {}".format(e))
         flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
         return edit_mailsettings()
+    except Exception as e:
+        flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
+        return edit_mailsettings()
 
     if to_save.get("test"):
         if current_user.email:
@@ -1668,9 +1671,10 @@ def _db_configuration_update_helper():
 
     if db_change or not db_valid or not config.db_configured \
             or config.config_calibre_dir != to_save["config_calibre_dir"]:
-        if not calibre_db.setup_db(to_save['config_calibre_dir'], ub.app_DB_path):
+        if not os.path.exists(metadata_db) or not to_save['config_calibre_dir']:
-            return _db_configuration_result(_('DB Location is not Valid, Please Enter Correct Path'),
+            return _db_configuration_result(_('DB Location is not Valid, Please Enter Correct Path'), gdrive_error)
-                                            gdrive_error)
+        else:
+            calibre_db.setup_db(to_save['config_calibre_dir'], ub.app_DB_path)
         config.store_calibre_uuid(calibre_db, db.Library_Id)
         # if db changed -> delete shelfs, delete download books, delete read books, kobo sync...
         if db_change:

cps/db.py

@@ -567,12 +567,12 @@ class CalibreDB:
 
         if not config_calibre_dir:
             cls.config.invalidate()
-            return False
+            return None
 
         dbpath = os.path.join(config_calibre_dir, "metadata.db")
         if not os.path.exists(dbpath):
             cls.config.invalidate()
-            return False
+            return None
 
         try:
             cls.engine = create_engine('sqlite://',
@@ -588,7 +588,7 @@ class CalibreDB:
             # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
         except Exception as ex:
             cls.config.invalidate(ex)
-            return False
+            return None
 
         cls.config.db_configured = True
 
@@ -598,7 +598,7 @@ class CalibreDB:
                 cls.setup_db_cc_classes(cc)
             except OperationalError as e:
                 log.error_or_exception(e)
-                return False
+                return None
 
         cls.session_factory = scoped_session(sessionmaker(autocommit=False,
                                                           autoflush=True,
@@ -607,7 +607,6 @@ class CalibreDB:
             inst.init_session()
 
         cls._init = True
-        return True
 
     def get_book(self, book_id):
         return self.session.query(Books).filter(Books.id == book_id).first()
@@ -628,7 +627,7 @@ class CalibreDB:
                       .join(read_column, read_column.book == book_id,
                       isouter=True))
             except (KeyError, AttributeError, IndexError):
-                log.error("Custom Column No.{} is not existing in calibre database".format(read_column))
+                log.error("Custom Column No.{} does not exist in calibre database".format(read_column))
                 # Skip linking read column and return None instead of read status
                 bd = self.session.query(Books, None, ub.ArchivedBook.is_archived)
         return (bd.filter(Books.id == book_id)
@@ -675,9 +674,9 @@ class CalibreDB:
             except (KeyError, AttributeError, IndexError):
                 pos_content_cc_filter = false()
                 neg_content_cc_filter = true()
-                log.error("Custom Column No.{} is not existing in calibre database".format(
+                log.error("Custom Column No.{} does not exist in calibre database".format(
                     self.config.config_restricted_column))
-                flash(_("Custom Column No.%(column)d is not existing in calibre database",
+                flash(_("Custom Column No.%(column)d does not exist in calibre database",
                         column=self.config.config_restricted_column),
                       category="error")
 
@@ -700,7 +699,7 @@ class CalibreDB:
                          .select_from(Books)
                          .outerjoin(read_column, read_column.book == Books.id))
             except (KeyError, AttributeError, IndexError):
-                log.error("Custom Column No.{} is not existing in calibre database".format(config_read_column))
+                log.error("Custom Column No.{} does not exist in calibre database".format(config_read_column))
                 # Skip linking read column and return None instead of read status
                 query = self.session.query(database, None, ub.ArchivedBook.is_archived)
         return query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id,

cps/debug_info.py

@@ -22,6 +22,7 @@ import glob
 import zipfile
 import json
 from io import BytesIO
+from flask_babel.speaklater import LazyString
 
 import os
 
@@ -32,6 +33,12 @@ from .about import collect_stats
 
 log = logger.create()
 
+class lazyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, LazyString):
+            return str(obj)
+        # Let the base class default method raise the TypeError
+        return json.JSONEncoder.default(self, obj)
 
 def assemble_logfiles(file_name):
     log_list = sorted(glob.glob(file_name + '*'), reverse=True)
@@ -58,8 +65,8 @@ def send_debug():
             file_list.remove(element)
     memory_zip = BytesIO()
     with zipfile.ZipFile(memory_zip, 'w', compression=zipfile.ZIP_DEFLATED) as zf:
-        zf.writestr('settings.txt', json.dumps(config.toDict()))
+        zf.writestr('settings.txt', json.dumps(config.toDict(), sort_keys=True, indent=2))
-        zf.writestr('libs.txt', json.dumps(collect_stats()))
+        zf.writestr('libs.txt', json.dumps(collect_stats(), sort_keys=True, indent=2, cls=lazyEncoder))
         for fp in file_list:
             zf.write(fp, os.path.basename(fp))
     memory_zip.seek(0)

cps/editbooks.py

@@ -39,6 +39,7 @@ from flask_babel import lazy_gettext as N_
 from flask_babel import get_locale
 from flask_login import current_user, login_required
 from sqlalchemy.exc import OperationalError, IntegrityError
+from sqlalchemy.orm.exc import StaleDataError
 
 from . import constants, logger, isoLanguages, gdriveutils, uploader, helper, kobo_sync_status
 from . import config, ub, db, calibre_db
@@ -221,7 +222,7 @@ def edit_book(book_id):
         calibre_db.session.rollback()
         flash(str(e), category="error")
         return redirect(url_for('web.show_book', book_id=book.id))
-    except (OperationalError, IntegrityError) as e:
+    except (OperationalError, IntegrityError, StaleDataError) as e:
         log.error_or_exception("Database error: {}".format(e))
         calibre_db.session.rollback()
         flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
@@ -295,7 +296,7 @@ def upload():
                     else:
                         resp = {"location": url_for('web.show_book', book_id=book_id)}
                         return Response(json.dumps(resp), mimetype='application/json')
-            except (OperationalError, IntegrityError) as e:
+            except (OperationalError, IntegrityError, StaleDataError) as e:
                 calibre_db.session.rollback()
                 log.error_or_exception("Database error: {}".format(e))
                 flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
@@ -443,7 +444,7 @@ def edit_list_book(param):
         if param == 'title' and vals.get('checkT') == "false":
             book.sort = sort_param
             calibre_db.session.commit()
-    except (OperationalError, IntegrityError) as e:
+    except (OperationalError, IntegrityError, StaleDataError) as e:
         calibre_db.session.rollback()
         log.error_or_exception("Database error: {}".format(e))
         ret = Response(json.dumps({'success': False,
@@ -556,7 +557,7 @@ def table_xchange_author_title():
                 book.last_modified = datetime.utcnow()
             try:
                 calibre_db.session.commit()
-            except (OperationalError, IntegrityError) as e:
+            except (OperationalError, IntegrityError, StaleDataError) as e:
                 calibre_db.session.rollback()
                 log.error_or_exception("Database error: %s", e)
                 return json.dumps({'success': False})
@@ -1190,7 +1191,7 @@ def upload_single_file(file_request, book, book_id):
                     calibre_db.session.add(db_format)
                     calibre_db.session.commit()
                     calibre_db.update_title_sort(config)
-                except (OperationalError, IntegrityError) as e:
+                except (OperationalError, IntegrityError, StaleDataError) as e:
                     calibre_db.session.rollback()
                     log.error_or_exception("Database error: {}".format(e))
                     flash(_(u"Database error: %(error)s.", error=e.orig), category="error")

cps/gdriveutils.py

@@ -33,6 +33,7 @@ try:
 except ImportError:
     from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.exc import OperationalError, InvalidRequestError, IntegrityError
+from sqlalchemy.orm.exc import StaleDataError
 from sqlalchemy.sql.expression import text
 
 try:
@@ -318,7 +319,7 @@ def getFolderId(path, drive):
                 session.commit()
         else:
             currentFolderId = storedPathName.gdrive_id
-    except (OperationalError, IntegrityError) as ex:
+    except (OperationalError, IntegrityError, StaleDataError) as ex:
         log.error_or_exception('Database error: {}'.format(ex))
         session.rollback()
     except ApiRequestError as ex:

cps/helper.py

@@ -322,12 +322,12 @@ def edit_book_read_status(book_id, read_status=None):
         try:
             calibre_db.update_title_sort(config)
             book = calibre_db.get_filtered_book(book_id)
-            read_status = getattr(book, 'custom_column_' + str(config.config_read_column))
+            book_read_status = getattr(book, 'custom_column_' + str(config.config_read_column))
-            if len(read_status):
+            if len(book_read_status):
                 if read_status is None:
-                    read_status[0].value = not read_status[0].value
+                    book_read_status[0].value = not book_read_status[0].value
                 else:
-                    read_status[0].value = read_status is True
+                    book_read_status[0].value = read_status is True
                 calibre_db.session.commit()
             else:
                 cc_class = db.cc_classes[config.config_read_column]
@@ -336,8 +336,8 @@ def edit_book_read_status(book_id, read_status=None):
                 calibre_db.session.commit()
         except (KeyError, AttributeError, IndexError):
             log.error(
-                "Custom Column No.{} is not existing in calibre database".format(config.config_read_column))
+                "Custom Column No.{} does not exist in calibre database".format(config.config_read_column))
-            return "Custom Column No.{} is not existing in calibre database".format(config.config_read_column)
+            return "Custom Column No.{} does not exist in calibre database".format(config.config_read_column)
         except (OperationalError, InvalidRequestError) as ex:
             calibre_db.session.rollback()
             log.error(u"Read status could not set: {}".format(ex))

cps/kobo_auth.py

@@ -107,7 +107,7 @@ def generate_auth_token(user_id):
 
     for book in books:
         formats = [data.format for data in book.data]
-        if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats:
+        if 'KEPUB' not in formats and config.config_kepubifypath and 'EPUB' in formats:
             helper.convert_book_format(book.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)
 
     return render_title_template(

cps/render_template.py

@@ -101,19 +101,6 @@ def get_sidebar_config(kwargs=None):
              "show_text": _('Show Books List'), "config_show": content})
     return sidebar, simple
 
-'''def get_readbooks_ids():
-    if not config.config_read_column:
-        readBooks = ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id))\
-            .filter(ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED).all()
-        return frozenset([x.book_id for x in readBooks])
-    else:
-        try:
-            readBooks = calibre_db.session.query(db.cc_classes[config.config_read_column])\
-                .filter(db.cc_classes[config.config_read_column].value == True).all()
-            return frozenset([x.book for x in readBooks])
-        except (KeyError, AttributeError, IndexError):
-            log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column))
-            return []'''
 
 # Returns the template for rendering and includes the instance name
 def render_title_template(*args, **kwargs):

cps/search.py

@@ -22,7 +22,7 @@ from flask import session as flask_session
 from flask_login import current_user
 from flask_babel import format_date
 from flask_babel import gettext as _
-from sqlalchemy.sql.expression import func, not_, and_, or_, text
+from sqlalchemy.sql.expression import func, not_, and_, or_, text, true
 from sqlalchemy.sql.functions import coalesce
 
 from . import logger, db, calibre_db, config, ub
@@ -119,32 +119,23 @@ def adv_search_ratings(q, rating_high, rating_low):
     return q
 
 
-def adv_search_read_status(q, read_status):
+def adv_search_read_status(read_status):
-    if read_status:
+    if not config.config_read_column:
-        if config.config_read_column:
-            try:
         if read_status == "True":
-                    q = q.join(db.cc_classes[config.config_read_column], isouter=True) \
+            db_filter = and_(ub.ReadBook.user_id == int(current_user.id),
-                        .filter(db.cc_classes[config.config_read_column].value == True)
-                else:
-                    q = q.join(db.cc_classes[config.config_read_column], isouter=True) \
-                        .filter(coalesce(db.cc_classes[config.config_read_column].value, False) != True)
-            except (KeyError, AttributeError):
-                log.error(u"Custom Column No.%d is not existing in calibre database", config.config_read_column)
-                flash(_("Custom Column No.%(column)d is not existing in calibre database",
-                        column=config.config_read_column),
-                      category="error")
-                return q
-        else:
-            if read_status == "True":
-                q = q.join(ub.ReadBook, db.Books.id == ub.ReadBook.book_id, isouter=True) \
-                    .filter(ub.ReadBook.user_id == int(current_user.id),
                              ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED)
         else:
-                q = q.join(ub.ReadBook, db.Books.id == ub.ReadBook.book_id, isouter=True) \
+            db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED
-                    .filter(ub.ReadBook.user_id == int(current_user.id),
+    else:
-                            coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED)
+        try:
-    return q
+            if read_status == "True":
+                db_filter = db.cc_classes[config.config_read_column].value == True
+            else:
+                db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True
+        except (KeyError, AttributeError, IndexError):
+            log.error("Custom Column No.{} does not exist in calibre database".format(config.config_read_column))
+            return true()
+    return db_filter
 
 
 def adv_search_extension(q, include_extension_inputs, exclude_extension_inputs):
@@ -238,23 +229,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None):
 
     cc = calibre_db.get_cc_columns(config, filter_config_custom_read=True)
     calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase)
-    if not config.config_read_column:
+    query = calibre_db.generate_linked_query(config.config_read_column, db.Books)
-        query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, ub.ReadBook).select_from(db.Books)
-                 .outerjoin(ub.ReadBook, and_(db.Books.id == ub.ReadBook.book_id,
-                                              int(current_user.id) == ub.ReadBook.user_id)))
-    else:
-        try:
-            read_column = cc[config.config_read_column]
-            query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, read_column.value)
-                     .select_from(db.Books)
-                     .outerjoin(read_column, read_column.book == db.Books.id))
-        except (KeyError, AttributeError):
-            log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column)
-            # Skip linking read column
-            query = calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, None)
-    query = query.outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id,
-                                                  int(current_user.id) == ub.ArchivedBook.user_id))
-
     q = query.outerjoin(db.books_series_link, db.Books.id == db.books_series_link.c.book)\
         .outerjoin(db.Series)\
         .filter(calibre_db.common_filters(True))
@@ -324,7 +299,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None):
             q = q.filter(func.datetime(db.Books.pubdate) > func.datetime(pub_start))
         if pub_end:
             q = q.filter(func.datetime(db.Books.pubdate) < func.datetime(pub_end))
-        q = adv_search_read_status(q, read_status)
+        q = q.filter(adv_search_read_status(read_status))
         if publisher:
             q = q.filter(db.Books.publishers.any(func.lower(db.Publishers.name).ilike("%" + publisher + "%")))
         q = adv_search_tag(q, tags['include_tag'], tags['exclude_tag'])

cps/tasks/convert.py

@@ -188,6 +188,8 @@ class TaskConvert(CalibreTask):
         log.info("ebook converter failed with error while converting book")
         if not error_message:
             error_message = N_('Ebook converter failed with unknown error')
+        else:
+            log.error(error_message)
         self._handleError(error_message)
         return
 
@@ -273,7 +275,10 @@ class TaskConvert(CalibreTask):
         return N_("Convert")
 
     def __str__(self):
+        if self.ereader_mail:
             return "Convert {} {}".format(self.book_id, self.ereader_mail)
+        else:
+            return "Convert {}".format(self.book_id)
 
     @property
     def is_cancellable(self):

cps/web.py

@@ -715,9 +715,9 @@ def render_read_books(page, are_read, as_xml=False, order=None):
             else:
                 db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True
         except (KeyError, AttributeError, IndexError):
-            log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column))
+            log.error("Custom Column No.{} does not exist in calibre database".format(config.config_read_column))
             if not as_xml:
-                flash(_("Custom Column No.%(column)d is not existing in calibre database",
+                flash(_("Custom Column No.%(column)d does not exist in calibre database",
                         column=config.config_read_column),
                       category="error")
                 return redirect(url_for("web.index"))

optional-requirements.txt

@@ -41,4 +41,4 @@ natsort>=2.2.0,<8.2.0
 comicapi>=2.2.0,<2.3.0
 
 # Kobo integration
-jsonschema>=3.2.0,<4.6.0
+jsonschema>=3.2.0,<4.7.0

setup.cfg

@@ -42,7 +42,7 @@ install_requires =
 	werkzeug<2.1.0
 	Babel>=1.3,<3.0
 	Flask-Babel>=0.11.1,<2.1.0
-	Flask-Login>=0.3.2,<0.6.1
+	Flask-Login>=0.3.2,<0.6.2
 	Flask-Principal>=0.3.2,<0.5.1
 	backports_abc>=0.4
 	Flask>=1.0.2,<2.1.0
@@ -97,5 +97,5 @@ comics =
 	natsort>=2.2.0,<8.2.0
 	comicapi>=2.2.0,<2.3.0
 kobo = 
-	jsonschema>=3.2.0,<4.5.0
+	jsonschema>=3.2.0,<4.6.0