diff --git a/cps/__init__.py b/cps/__init__.py index 31614c2a..24267b1f 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -102,7 +102,7 @@ if wtf_present: else: csrf = None -calibre_db = db.CalibreDB() +calibre_db = db.CalibreDB(app) web_server = WebServer() @@ -146,9 +146,7 @@ def create_app(): lm.anonymous_user = ub.Anonymous lm.session_protection = 'strong' if config.config_session == 1 else "basic" - db.CalibreDB.update_config(config) - db.CalibreDB.setup_db(config.config_calibre_dir, cli_param.settings_path) - calibre_db.init_db() + db.CalibreDB.update_config(config, config.config_calibre_dir, cli_param.settings_path) updater_thread.init_updater(config, web_server) # Perform dry run of updater and exit afterward diff --git a/cps/admin.py b/cps/admin.py index 6076e86d..b56b5b5c 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -144,7 +144,6 @@ def shutdown(): show_text = {} if task in (0, 1): # valid commandos received # close all database connections - calibre_db.dispose() ub.dispose() if task == 0: @@ -1767,10 +1766,10 @@ def _db_configuration_update_helper(): config.config_allowed_column_value = "" config.config_read_column = 0 _config_string(to_save, "config_calibre_dir") - calibre_db.update_config(config) + calibre_db.update_config(config, config.config_calibre_dir, ub.app_DB_path) if not os.access(os.path.join(config.config_calibre_dir, "metadata.db"), os.W_OK): flash(_("DB is not Writeable"), category="warning") - calibre_db.update_config(config) + calibre_db.update_config(config, config.config_calibre_dir, ub.app_DB_path) config.save() return _db_configuration_result(None, gdrive_error) diff --git a/cps/config_sql.py b/cps/config_sql.py index 3607f0af..7e7d033d 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -405,11 +405,13 @@ class ConfigSQL(object): return self.config_calibre_split_dir if self.config_calibre_split_dir else self.config_calibre_dir def store_calibre_uuid(self, calibre_db, Library_table): + from . import app try: - calibre_uuid = calibre_db.session.query(Library_table).one_or_none() - if self.config_calibre_uuid != calibre_uuid.uuid: - self.config_calibre_uuid = calibre_uuid.uuid - self.save() + with app.app_context(): + calibre_uuid = calibre_db.session.query(Library_table).one_or_none() + if self.config_calibre_uuid != calibre_uuid.uuid: + self.config_calibre_uuid = calibre_uuid.uuid + self.save() except AttributeError: pass diff --git a/cps/db.py b/cps/db.py index c0a3fc97..47ea6a5c 100644 --- a/cps/db.py +++ b/cps/db.py @@ -23,7 +23,7 @@ import json from datetime import datetime, timezone from urllib.parse import quote import unidecode -from weakref import WeakSet +# from weakref import WeakSet from uuid import uuid4 from sqlite3 import OperationalError as sqliteOperationalError @@ -45,7 +45,7 @@ from sqlalchemy.ext.associationproxy import association_proxy from .cw_login import current_user from flask_babel import gettext as _ from flask_babel import get_locale -from flask import flash +from flask import flash, g, Flask from . import logger, ub, isoLanguages from .pagination import Pagination @@ -528,34 +528,25 @@ class AlchemyEncoder(json.JSONEncoder): class CalibreDB: - _init = False - engine = None config = None - session_factory = None - # This is a WeakSet so that references here don't keep other CalibreDB - # instances alive once they reach the end of their respective scopes - instances = WeakSet() + config_calibre_dir = None + app_db_path = None - def __init__(self, expire_on_commit=True, init=False): + def __init__(self, _app: Flask=None): # , expire_on_commit=True, init=False): """ Initialize a new CalibreDB session """ - self.session = None - if init: - self.init_db(expire_on_commit) + self.Session = None + #if init: + # self.init_db(expire_on_commit) + if _app is not None: + self.init_app(_app) - def init_db(self, expire_on_commit=True): - if self._init: - self.init_session(expire_on_commit) - - self.instances.add(self) - - def init_session(self, expire_on_commit=True): - self.session = self.session_factory() - self.session.expire_on_commit = expire_on_commit - self.create_functions(self.config) + def init_app(self, _app): + _app.teardown_appcontext(self.teardown) @classmethod def setup_db_cc_classes(cls, cc): + global cc_classes cc_ids = [] books_custom_column_links = {} for row in cc: @@ -623,8 +614,6 @@ class CalibreDB: secondary=books_custom_column_links[cc_id[0]], backref='books')) - return cc_classes - @classmethod def check_valid_db(cls, config_calibre_dir, app_db_path, config_calibre_uuid): if not config_calibre_dir: @@ -644,7 +633,6 @@ class CalibreDB: local_session = scoped_session(sessionmaker()) local_session.configure(bind=connection) database_uuid = local_session().query(Library_Id).one_or_none() - # local_session.dispose() check_engine.connect() db_change = config_calibre_uuid != database_uuid.uuid @@ -652,13 +640,30 @@ class CalibreDB: return False, False return True, db_change + def teardown(self, exception): + ctx = g.get("lib_sql") + if ctx: + ctx.close() + + @property + def session(self): + # connect or get active connection + if not g.get("lib_sql"): + g.lib_sql = self.connect() + return g.lib_sql + @classmethod - def update_config(cls, config): + def update_config(cls, config, config_calibre_dir, app_db_path): cls.config = config + cls.config_calibre_dir = config_calibre_dir + cls.app_db_path = app_db_path + + + def connect(self): + return self.setup_db(self.config_calibre_dir, self.app_db_path) @classmethod def setup_db(cls, config_calibre_dir, app_db_path): - cls.dispose() if not config_calibre_dir: cls.config.invalidate() @@ -670,17 +675,17 @@ class CalibreDB: return None try: - cls.engine = create_engine('sqlite://', + engine = create_engine('sqlite://', echo=False, isolation_level="SERIALIZABLE", connect_args={'check_same_thread': False}, poolclass=StaticPool) - with cls.engine.begin() as connection: + with engine.begin() as connection: connection.execute(text('PRAGMA cache_size = 10000;')) connection.execute(text("attach database '{}' as calibre;".format(dbpath))) connection.execute(text("attach database '{}' as app_settings;".format(app_db_path))) - conn = cls.engine.connect() + conn = engine.connect() # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302 except Exception as ex: cls.config.invalidate(ex) @@ -696,13 +701,10 @@ class CalibreDB: log.error_or_exception(e) return None - cls.session_factory = scoped_session(sessionmaker(autocommit=False, - autoflush=True, - bind=cls.engine, future=True)) - for inst in cls.instances: - inst.init_session() + return scoped_session(sessionmaker(autocommit=False, + autoflush=False, + bind=engine, future=True)) - cls._init = True def get_book(self, book_id): return self.session.query(Books).filter(Books.id == book_id).first() @@ -1066,41 +1068,9 @@ class CalibreDB: except sqliteOperationalError: pass - @classmethod - def dispose(cls): - # global session - - for inst in cls.instances: - old_session = inst.session - inst.session = None - if old_session: - try: - old_session.close() - except Exception: - pass - if old_session.bind: - try: - old_session.bind.dispose() - except Exception: - pass - - for attr in list(Books.__dict__.keys()): - if attr.startswith("custom_column_"): - setattr(Books, attr, None) - - for db_class in cc_classes.values(): - Base.metadata.remove(db_class.__table__) - cc_classes.clear() - - for table in reversed(Base.metadata.sorted_tables): - name = table.key - if name.startswith("custom_column_") or name.startswith("books_custom_column_"): - if table is not None: - Base.metadata.remove(table) - def reconnect_db(self, config, app_db_path): - self.dispose() - self.engine.dispose() + # self.dispose() + # self.engine.dispose() self.setup_db(config.config_calibre_dir, app_db_path) self.update_config(config) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 08eb63ce..9b39c6c2 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -28,7 +28,7 @@ from sqlalchemy.exc import SQLAlchemyError from flask_babel import lazy_gettext as N_ from cps.services.worker import CalibreTask -from cps import db +from cps import db, app from cps import logger, config from cps.subproc_wrapper import process_open from flask_babel import gettext as _ @@ -62,157 +62,159 @@ class TaskConvert(CalibreTask): def run(self, worker_thread): self.worker_thread = worker_thread if config.config_use_google_drive: - worker_db = db.CalibreDB(expire_on_commit=False, init=True) - cur_book = worker_db.get_book(self.book_id) - self.title = cur_book.title - data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) - df = gdriveutils.getFileFromEbooksFolder(cur_book.path, - data.name + "." + self.settings['old_book_format'].lower()) - df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") - if df: - datafile_cover = None - datafile = os.path.join(config.get_book_path(), - cur_book.path, - data.name + "." + self.settings['old_book_format'].lower()) - if df_cover: - datafile_cover = os.path.join(config.get_book_path(), - cur_book.path, "cover.jpg") - if not os.path.exists(os.path.join(config.get_book_path(), cur_book.path)): - os.makedirs(os.path.join(config.get_book_path(), cur_book.path)) - df.GetContentFile(datafile) - if df_cover: - df_cover.GetContentFile(datafile_cover) - worker_db.session.close() - else: - # ToDo Include cover in error handling - error_message = _("%(format)s not found on Google Drive: %(fn)s", - format=self.settings['old_book_format'], - fn=data.name + "." + self.settings['old_book_format'].lower()) - worker_db.session.close() + with app.app_context(): + worker_db = db.CalibreDB(app) + cur_book = worker_db.get_book(self.book_id) + self.title = cur_book.title + data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) + df = gdriveutils.getFileFromEbooksFolder(cur_book.path, + data.name + "." + self.settings['old_book_format'].lower()) + df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") + if df: + datafile_cover = None + datafile = os.path.join(config.get_book_path(), + cur_book.path, + data.name + "." + self.settings['old_book_format'].lower()) + if df_cover: + datafile_cover = os.path.join(config.get_book_path(), + cur_book.path, "cover.jpg") + if not os.path.exists(os.path.join(config.get_book_path(), cur_book.path)): + os.makedirs(os.path.join(config.get_book_path(), cur_book.path)) + df.GetContentFile(datafile) + if df_cover: + df_cover.GetContentFile(datafile_cover) + # worker_db.session.close() + else: + # ToDo Include cover in error handling + error_message = _("%(format)s not found on Google Drive: %(fn)s", + format=self.settings['old_book_format'], + fn=data.name + "." + self.settings['old_book_format'].lower()) + # worker_db.session.close() return self._handleError(error_message) - filename = self._convert_ebook_format() - if config.config_use_google_drive: - os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) - if df_cover: - os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) - - if filename: + filename = self._convert_ebook_format() if config.config_use_google_drive: - # Upload files to gdrive - gdriveutils.updateGdriveCalibreFromLocal() - self._handleSuccess() - if self.ereader_mail: - # if we're sending to E-Reader after converting, create a one-off task and run it immediately - # todo: figure out how to incorporate this into the progress - try: - EmailText = N_(u"%(book)s send to E-Reader", book=escape(self.title)) - for email in self.ereader_mail.split(','): - email = strip_whitespaces(email) - worker_thread.add(self.user, TaskEmail(self.settings['subject'], - self.results["path"], - filename, - self.settings, - email, - EmailText, - self.settings['body'], - id=self.book_id, - internal=True) - ) - except Exception as ex: - return self._handleError(str(ex)) + os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) + if df_cover: + os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) + + if filename: + if config.config_use_google_drive: + # Upload files to gdrive + gdriveutils.updateGdriveCalibreFromLocal() + self._handleSuccess() + if self.ereader_mail: + # if we're sending to E-Reader after converting, create a one-off task and run it immediately + # todo: figure out how to incorporate this into the progress + try: + EmailText = N_(u"%(book)s send to E-Reader", book=escape(self.title)) + for email in self.ereader_mail.split(','): + email = strip_whitespaces(email) + worker_thread.add(self.user, TaskEmail(self.settings['subject'], + self.results["path"], + filename, + self.settings, + email, + EmailText, + self.settings['body'], + id=self.book_id, + internal=True) + ) + except Exception as ex: + return self._handleError(str(ex)) def _convert_ebook_format(self): error_message = None - local_db = db.CalibreDB(expire_on_commit=False, init=True) - file_path = self.file_path - book_id = self.book_id - format_old_ext = '.' + self.settings['old_book_format'].lower() - format_new_ext = '.' + self.settings['new_book_format'].lower() + with app.app_context(): + local_db = db.CalibreDB(app) + file_path = self.file_path + book_id = self.book_id + format_old_ext = '.' + self.settings['old_book_format'].lower() + format_new_ext = '.' + self.settings['new_book_format'].lower() - # check to see if destination format already exists - or if book is in database - # if it does - mark the conversion task as complete and return a success - # this will allow to send to E-Reader workflow to continue to work - if os.path.isfile(file_path + format_new_ext) or\ - local_db.get_book_format(self.book_id, self.settings['new_book_format']): - log.info("Book id %d already converted to %s", book_id, format_new_ext) - cur_book = local_db.get_book(book_id) - self.title = cur_book.title - self.results['path'] = cur_book.path - self.results['title'] = self.title - new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id)\ - .filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none() - if not new_format: - new_format = db.Data(name=os.path.basename(file_path), - book_format=self.settings['new_book_format'].upper(), - book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) - try: - local_db.session.merge(new_format) - local_db.session.commit() - except SQLAlchemyError as e: - local_db.session.rollback() - log.error("Database error: %s", e) - local_db.session.close() - self._handleError(N_("Oops! Database Error: %(error)s.", error=e)) - return - self._handleSuccess() - local_db.session.close() - return os.path.basename(file_path + format_new_ext) - else: - log.info("Book id %d - target format of %s does not exist. Moving forward with convert.", - book_id, - format_new_ext) - - if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub': - check, error_message = self._convert_kepubify(file_path, - format_old_ext, - format_new_ext) - else: - # check if calibre converter-executable is existing - if not os.path.exists(config.config_converterpath): - self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath)) - return - has_cover = local_db.get_book(book_id).has_cover - check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover) - - if check == 0: - cur_book = local_db.get_book(book_id) - if os.path.isfile(file_path + format_new_ext): - new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id) \ + # check to see if destination format already exists - or if book is in database + # if it does - mark the conversion task as complete and return a success + # this will allow to send to E-Reader workflow to continue to work + if os.path.isfile(file_path + format_new_ext) or\ + local_db.get_book_format(self.book_id, self.settings['new_book_format']): + log.info("Book id %d already converted to %s", book_id, format_new_ext) + cur_book = local_db.get_book(book_id) + self.title = cur_book.title + self.results['path'] = cur_book.path + self.results['title'] = self.title + new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id)\ .filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none() if not new_format: - new_format = db.Data(name=cur_book.data[0].name, + new_format = db.Data(name=os.path.basename(file_path), book_format=self.settings['new_book_format'].upper(), book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) try: local_db.session.merge(new_format) local_db.session.commit() - if self.settings['new_book_format'].upper() in ['KEPUB', 'EPUB', 'EPUB3']: - ub_session = init_db_thread() - remove_synced_book(book_id, True, ub_session) - ub_session.close() except SQLAlchemyError as e: local_db.session.rollback() log.error("Database error: %s", e) local_db.session.close() - self._handleError(error_message) + self._handleError(N_("Oops! Database Error: %(error)s.", error=e)) return - self.results['path'] = cur_book.path - self.title = cur_book.title - self.results['title'] = self.title - if not config.config_use_google_drive: self._handleSuccess() - return os.path.basename(file_path + format_new_ext) + local_db.session.close() + return os.path.basename(file_path + format_new_ext) else: - error_message = N_('%(format)s format not found on disk', format=format_new_ext.upper()) - local_db.session.close() - log.info("ebook converter failed with error while converting book") - if not error_message: - error_message = N_('Ebook converter failed with unknown error') - else: - log.error(error_message) - self._handleError(error_message) - return + log.info("Book id %d - target format of %s does not exist. Moving forward with convert.", + book_id, + format_new_ext) + + if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub': + check, error_message = self._convert_kepubify(file_path, + format_old_ext, + format_new_ext) + else: + # check if calibre converter-executable is existing + if not os.path.exists(config.config_converterpath): + self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath)) + return + has_cover = local_db.get_book(book_id).has_cover + check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover) + + if check == 0: + cur_book = local_db.get_book(book_id) + if os.path.isfile(file_path + format_new_ext): + new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id) \ + .filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none() + if not new_format: + new_format = db.Data(name=cur_book.data[0].name, + book_format=self.settings['new_book_format'].upper(), + book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) + try: + local_db.session.merge(new_format) + local_db.session.commit() + if self.settings['new_book_format'].upper() in ['KEPUB', 'EPUB', 'EPUB3']: + ub_session = init_db_thread() + remove_synced_book(book_id, True, ub_session) + ub_session.close() + except SQLAlchemyError as e: + local_db.session.rollback() + log.error("Database error: %s", e) + local_db.session.close() + self._handleError(error_message) + return + self.results['path'] = cur_book.path + self.title = cur_book.title + self.results['title'] = self.title + if not config.config_use_google_drive: + self._handleSuccess() + return os.path.basename(file_path + format_new_ext) + else: + error_message = N_('%(format)s format not found on disk', format=format_new_ext.upper()) + local_db.session.close() + log.info("ebook converter failed with error while converting book") + if not error_message: + error_message = N_('Ebook converter failed with unknown error') + else: + log.error(error_message) + self._handleError(error_message) + return def _convert_kepubify(self, file_path, format_old_ext, format_new_ext): if config.config_embed_metadata and config.config_binariesdir: diff --git a/cps/tasks/database.py b/cps/tasks/database.py index c9c30d43..e5f630c6 100644 --- a/cps/tasks/database.py +++ b/cps/tasks/database.py @@ -18,7 +18,7 @@ from flask_babel import lazy_gettext as N_ -from cps import config, logger, db, ub +from cps import config, logger, db, ub, app from cps.services.worker import CalibreTask @@ -26,11 +26,13 @@ class TaskReconnectDatabase(CalibreTask): def __init__(self, task_message=N_('Reconnecting Calibre database')): super(TaskReconnectDatabase, self).__init__(task_message) self.log = logger.create() - self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True) + # self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True) def run(self, worker_thread): - self.calibre_db.reconnect_db(config, ub.app_DB_path) - self.calibre_db.session.close() + with app.app_context(): + calibre_db = db.CalibreDB(app) + calibre_db.reconnect_db(config, ub.app_DB_path) + # self.calibre_db.session.close() self._handleSuccess() @property diff --git a/cps/tasks/metadata_backup.py b/cps/tasks/metadata_backup.py index 5e0bb96a..dc5b8404 100644 --- a/cps/tasks/metadata_backup.py +++ b/cps/tasks/metadata_backup.py @@ -19,7 +19,7 @@ import os from lxml import etree -from cps import config, db, gdriveutils, logger +from cps import config, db, gdriveutils, logger, app from cps.services.worker import CalibreTask from flask_babel import lazy_gettext as N_ @@ -34,7 +34,7 @@ class TaskBackupMetadata(CalibreTask): task_message=N_('Backing up Metadata')): super(TaskBackupMetadata, self).__init__(task_message) self.log = logger.create() - self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True) + # self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True) self.export_language = export_language self.translated_title = translated_title self.set_dirty = set_dirty @@ -46,47 +46,51 @@ class TaskBackupMetadata(CalibreTask): self.backup_metadata() def set_all_books_dirty(self): - try: - books = self.calibre_db.session.query(db.Books).all() - for book in books: - self.calibre_db.set_metadata_dirty(book.id) - self.calibre_db.session.commit() - self._handleSuccess() - except Exception as ex: - self.log.debug('Error adding book for backup: ' + str(ex)) - self._handleError('Error adding book for backup: ' + str(ex)) - self.calibre_db.session.rollback() - self.calibre_db.session.close() + with app.app_context(): + calibre_db = db.CalibreDB(app) + try: + books = calibre_db.session.query(db.Books).all() + for book in books: + calibre_db.set_metadata_dirty(book.id) + calibre_db.session.commit() + self._handleSuccess() + except Exception as ex: + self.log.debug('Error adding book for backup: ' + str(ex)) + self._handleError('Error adding book for backup: ' + str(ex)) + calibre_db.session.rollback() + # self.calibre_db.session.close() def backup_metadata(self): - try: - metadata_backup = self.calibre_db.session.query(db.Metadata_Dirtied).all() - custom_columns = (self.calibre_db.session.query(db.CustomColumns) - .filter(db.CustomColumns.mark_for_delete == 0) - .filter(db.CustomColumns.datatype.notin_(db.cc_exceptions)) - .order_by(db.CustomColumns.label).all()) - count = len(metadata_backup) - i = 0 - for backup in metadata_backup: - book = self.calibre_db.session.query(db.Books).filter(db.Books.id == backup.book).one_or_none() - self.calibre_db.session.query(db.Metadata_Dirtied).filter( - db.Metadata_Dirtied.book == backup.book).delete() - self.calibre_db.session.commit() - if book: - self.open_metadata(book, custom_columns) - else: - self.log.error("Book {} not found in database".format(backup.book)) - i += 1 - self.progress = (1.0 / count) * i - self._handleSuccess() - self.calibre_db.session.close() + with app.app_context(): + try: + calibre_db = db.CalibreDB(app) + metadata_backup = calibre_db.session.query(db.Metadata_Dirtied).all() + custom_columns = (calibre_db.session.query(db.CustomColumns) + .filter(db.CustomColumns.mark_for_delete == 0) + .filter(db.CustomColumns.datatype.notin_(db.cc_exceptions)) + .order_by(db.CustomColumns.label).all()) + count = len(metadata_backup) + i = 0 + for backup in metadata_backup: + book = calibre_db.session.query(db.Books).filter(db.Books.id == backup.book).one_or_none() + calibre_db.session.query(db.Metadata_Dirtied).filter( + db.Metadata_Dirtied.book == backup.book).delete() + calibre_db.session.commit() + if book: + self.open_metadata(book, custom_columns) + else: + self.log.error("Book {} not found in database".format(backup.book)) + i += 1 + self.progress = (1.0 / count) * i + self._handleSuccess() + # self.calibre_db.session.close() - except Exception as ex: - b = "NaN" if not hasattr(book, 'id') else book.id - self.log.debug('Error creating metadata backup for book {}: '.format(b) + str(ex)) - self._handleError('Error creating metadata backup: ' + str(ex)) - self.calibre_db.session.rollback() - self.calibre_db.session.close() + except Exception as ex: + b = "NaN" if not hasattr(book, 'id') else book.id + self.log.debug('Error creating metadata backup for book {}: '.format(b) + str(ex)) + self._handleError('Error creating metadata backup: ' + str(ex)) + calibre_db.session.rollback() + # self.calibre_db.session.close() def open_metadata(self, book, custom_columns): # package = self.create_new_metadata_backup(book, custom_columns) diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index a7658c02..358f4ad1 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -23,10 +23,11 @@ from io import BytesIO from datetime import datetime, timezone from .. import constants -from cps import config, db, fs, gdriveutils, logger, ub +from cps import config, db, fs, gdriveutils, logger, ub, app from cps.services.worker import CalibreTask, STAT_CANCELLED, STAT_ENDED from sqlalchemy import func, text, or_ from flask_babel import lazy_gettext as N_ + try: from wand.image import Image use_IM = True @@ -113,9 +114,10 @@ class TaskGenerateCoverThumbnails(CalibreTask): @staticmethod def get_books_with_covers(book_id=-1): filter_exp = (db.Books.id == book_id) if book_id != -1 else True - calibre_db = db.CalibreDB(expire_on_commit=False, init=True) - books_cover = calibre_db.session.query(db.Books).filter(db.Books.has_cover == 1).filter(filter_exp).all() - calibre_db.session.close() + with app.app_context(): + calibre_db = db.CalibreDB(app) #, expire_on_commit=False, init=True) + books_cover = calibre_db.session.query(db.Books).filter(db.Books.has_cover == 1).filter(filter_exp).all() + # calibre_db.session.close() return books_cover def get_book_cover_thumbnails(self, book_id): @@ -246,7 +248,7 @@ class TaskGenerateSeriesThumbnails(CalibreTask): super(TaskGenerateSeriesThumbnails, self).__init__(task_message) self.log = logger.create() self.app_db_session = ub.get_new_session_instance() - self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True) + # self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True) self.cache = fs.FileSystem() self.resolutions = [ constants.COVER_THUMBNAIL_SMALL, @@ -254,58 +256,60 @@ class TaskGenerateSeriesThumbnails(CalibreTask): ] def run(self, worker_thread): - if self.calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED: - self.message = 'Scanning Series' - all_series = self.get_series_with_four_plus_books() - count = len(all_series) + with app.app_context(): + calibre_db = db.CalibreDB(app) + if calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED: + self.message = 'Scanning Series' + all_series = self.get_series_with_four_plus_books(calibre_db) + count = len(all_series) - total_generated = 0 - for i, series in enumerate(all_series): - generated = 0 - series_thumbnails = self.get_series_thumbnails(series.id) - series_books = self.get_series_books(series.id) + total_generated = 0 + for i, series in enumerate(all_series): + generated = 0 + series_thumbnails = self.get_series_thumbnails(series.id) + series_books = self.get_series_books(series.id, calibre_db) - # Generate new thumbnails for missing covers - resolutions = list(map(lambda t: t.resolution, series_thumbnails)) - missing_resolutions = list(set(self.resolutions).difference(resolutions)) - for resolution in missing_resolutions: - generated += 1 - self.create_series_thumbnail(series, series_books, resolution) - - # Replace outdated or missing thumbnails - for thumbnail in series_thumbnails: - if any(book.last_modified > thumbnail.generated_at for book in series_books): + # Generate new thumbnails for missing covers + resolutions = list(map(lambda t: t.resolution, series_thumbnails)) + missing_resolutions = list(set(self.resolutions).difference(resolutions)) + for resolution in missing_resolutions: generated += 1 - self.update_series_thumbnail(series_books, thumbnail) + self.create_series_thumbnail(series, series_books, resolution) - elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS): - generated += 1 - self.update_series_thumbnail(series_books, thumbnail) + # Replace outdated or missing thumbnails + for thumbnail in series_thumbnails: + if any(book.last_modified > thumbnail.generated_at for book in series_books): + generated += 1 + self.update_series_thumbnail(series_books, thumbnail) - # Increment the progress - self.progress = (1.0 / count) * i + elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS): + generated += 1 + self.update_series_thumbnail(series_books, thumbnail) - if generated > 0: - total_generated += generated - self.message = N_('Generated {0} series thumbnails').format(total_generated) + # Increment the progress + self.progress = (1.0 / count) * i - # Check if job has been cancelled or ended - if self.stat == STAT_CANCELLED: - self.log.info(f'GenerateSeriesThumbnails task has been cancelled.') - return + if generated > 0: + total_generated += generated + self.message = N_('Generated {0} series thumbnails').format(total_generated) - if self.stat == STAT_ENDED: - self.log.info(f'GenerateSeriesThumbnails task has been ended.') - return + # Check if job has been cancelled or ended + if self.stat == STAT_CANCELLED: + self.log.info(f'GenerateSeriesThumbnails task has been cancelled.') + return - if total_generated == 0: - self.self_cleanup = True + if self.stat == STAT_ENDED: + self.log.info(f'GenerateSeriesThumbnails task has been ended.') + return - self._handleSuccess() - self.app_db_session.remove() + if total_generated == 0: + self.self_cleanup = True - def get_series_with_four_plus_books(self): - return self.calibre_db.session \ + self._handleSuccess() + self.app_db_session.remove() + + def get_series_with_four_plus_books(self, calibre_db): + return calibre_db.session \ .query(db.Series) \ .join(db.books_series_link) \ .join(db.Books) \ @@ -314,8 +318,8 @@ class TaskGenerateSeriesThumbnails(CalibreTask): .having(func.count('book_series_link') > 3) \ .all() - def get_series_books(self, series_id): - return self.calibre_db.session \ + def get_series_books(self, series_id, calibre_db): + return calibre_db.session \ .query(db.Books) \ .join(db.books_series_link) \ .join(db.Series) \ @@ -461,13 +465,15 @@ class TaskClearCoverThumbnailCache(CalibreTask): def run(self, worker_thread): if self.app_db_session: - if self.book_id == 0: # delete superfluous thumbnails - calibre_db = db.CalibreDB(expire_on_commit=False, init=True) - thumbnails = (calibre_db.session.query(ub.Thumbnail) - .join(db.Books, ub.Thumbnail.entity_id == db.Books.id, isouter=True) - .filter(db.Books.id==None) - .all()) - calibre_db.session.close() + # delete superfluous thumbnails + if self.book_id == 0: + with app.app_context(): + calibre_db = db.CalibreDB(app) + thumbnails = (calibre_db.session.query(ub.Thumbnail) + .join(db.Books, ub.Thumbnail.entity_id == db.Books.id, isouter=True) + .filter(db.Books.id==None) + .all()) + # calibre_db.session.close() elif self.book_id > 0: # make sure single book is selected thumbnails = self.get_thumbnails_for_book(self.book_id) if self.book_id < 0: diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 8443cf4a..234bfb96 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2024-11-20 18:23:06

+

Start Time: 2024-11-25 20:45:11

-

Stop Time: 2024-11-21 01:38:12

+

Stop Time: 2024-11-26 02:45:09

-

Duration: 6h 7 min

+

Duration: 4h 59 min

@@ -234,12 +234,12 @@ - + TestBackupMetadata 21 - 21 - 0 - 0 + 16 + 4 + 1 0 Detail @@ -248,20 +248,67 @@ - +
TestBackupMetadata - test_backup_all
- PASS + +
+ FAIL +
+ + + + - +
TestBackupMetadata - test_backup_change_book_author
- PASS + +
+ FAIL +
+ + + + @@ -275,11 +322,31 @@ - +
TestBackupMetadata - test_backup_change_book_identifier
- PASS + +
+ FAIL +
+ + + + @@ -302,11 +369,31 @@ - +
TestBackupMetadata - test_backup_change_book_publishing_date
- PASS + +
+ ERROR +
+ + + + @@ -338,11 +425,32 @@ - +
TestBackupMetadata - test_backup_change_book_title
- PASS + +
+ FAIL +
+ + + + @@ -438,13 +546,13 @@ - + TestBackupMetadataGdrive 1 + 0 + 0 1 0 - 0 - 0 Detail @@ -452,31 +560,307 @@ - +
TestBackupMetadataGdrive - test_backup_gdrive
- PASS + +
+ ERROR +
+ + + + - - TestCli - 13 - 13 + + _ErrorHolder + 3 0 0 + 3 0 - Detail + Detail - + + +
tearDownClass (test_backup_metadata_gdrive)
+ + +
+ ERROR +
+ + + + + + + + + + +
tearDownClass (test_edit_additional_books)
+ + +
+ ERROR +
+ + + + + + + + + + +
tearDownClass (test_readonly_db)
+ + +
+ ERROR +
+ + + + + + + + + + + TestCli + 13 + 8 + 3 + 2 + 0 + + Detail + + + + + +
TestCli - test_already_started
@@ -485,7 +869,7 @@ - +
TestCli - test_bind_to_single_interface
@@ -494,7 +878,7 @@ - +
TestCli - test_change_password
@@ -503,7 +887,7 @@ - +
TestCli - test_cli_SSL_files
@@ -512,7 +896,7 @@ - +
TestCli - test_cli_different_folder
@@ -521,7 +905,7 @@ - +
TestCli - test_cli_different_settings_database
@@ -530,7 +914,7 @@ - +
TestCli - test_dryrun_update
@@ -539,16 +923,36 @@ - +
TestCli - test_enable_reconnect
- PASS + +
+ FAIL +
+ + + + - +
TestCli - test_environ_port_setting
@@ -557,89 +961,325 @@ - +
TestCli - test_logfile
- PASS + +
+ ERROR +
+ + + + - +
TestCli - test_no_database
- PASS + +
+ FAIL +
+ + + + - +
TestCli - test_settingsdb_not_writeable
- PASS + +
+ FAIL +
+ + + + - +
TestCli - test_writeonly_static_files
- PASS + +
+ ERROR +
+ + + + - + TestCliGdrivedb 4 - 4 - 0 0 + 3 + 1 0 - Detail + Detail - +
TestCliGdrivedb - test_cli_gdrive_folder
- PASS + +
+ ERROR +
+ + + + - +
TestCliGdrivedb - test_cli_gdrive_location
- PASS + +
+ FAIL +
+ + + + - +
TestCliGdrivedb - test_gdrive_db_nonwrite
- PASS + +
+ FAIL +
+ + + + - +
TestCliGdrivedb - test_no_database
- PASS + +
+ FAIL +
+ + + + @@ -653,13 +1293,13 @@ 0 0 - Detail + Detail - +
TestCoverEditBooks - test_invalid_jpg_hdd
@@ -668,7 +1308,7 @@ - +
TestCoverEditBooks - test_upload_jpg
@@ -686,13 +1326,13 @@ 0 0 - Detail + Detail - +
TestDeleteDatabase - test_delete_books_in_database
@@ -702,30 +1342,50 @@ - + TestEbookConvertCalibre 15 - 15 - 0 - 0 + 6 + 2 + 7 0 - Detail + Detail - +
TestEbookConvertCalibre - test_calibre_log
- PASS + +
+ ERROR +
+ + + + - +
TestEbookConvertCalibre - test_convert_deactivate
@@ -734,70 +1394,214 @@ - +
TestEbookConvertCalibre - test_convert_email
- PASS + +
+ ERROR +
+ + + + - +
TestEbookConvertCalibre - test_convert_failed_and_email
- PASS + +
+ ERROR +
+ + + + - +
TestEbookConvertCalibre - test_convert_only
- PASS + +
+ FAIL +
+ + + + - +
TestEbookConvertCalibre - test_convert_options
- PASS + +
+ ERROR +
+ + + + - +
TestEbookConvertCalibre - test_convert_parameter
- PASS + +
+ FAIL +
+ + + + - +
TestEbookConvertCalibre - test_convert_wrong_excecutable
- PASS + +
+ ERROR +
+ + + + - +
TestEbookConvertCalibre - test_convert_xss
- PASS + +
+ ERROR +
+ + + + - +
TestEbookConvertCalibre - test_email_failed
@@ -806,7 +1610,7 @@ - +
TestEbookConvertCalibre - test_email_only
@@ -815,7 +1619,7 @@ - +
TestEbookConvertCalibre - test_kindle_send_not_configured
@@ -824,7 +1628,7 @@ - +
TestEbookConvertCalibre - test_ssl_smtp_setup_error
@@ -833,7 +1637,7 @@ - +
TestEbookConvertCalibre - test_starttls_smtp_setup_error
@@ -842,67 +1646,173 @@ - +
TestEbookConvertCalibre - test_user_convert_xss
- PASS + +
+ ERROR +
+ + + + - + TestEbookConvertCalibreGDrive 7 - 7 - 0 - 0 + 3 + 3 + 1 0 - Detail + Detail - +
TestEbookConvertCalibreGDrive - test_convert_email
- PASS + +
+ ERROR +
+ + + + - +
TestEbookConvertCalibreGDrive - test_convert_failed_and_email
- PASS + +
+ FAIL +
+ + + + - +
TestEbookConvertCalibreGDrive - test_convert_only
- PASS + +
+ FAIL +
+ + + + - +
TestEbookConvertCalibreGDrive - test_convert_parameter
- PASS + +
+ FAIL +
+ + + + - +
TestEbookConvertCalibreGDrive - test_email_failed
@@ -911,7 +1821,7 @@ - +
TestEbookConvertCalibreGDrive - test_email_only
@@ -920,7 +1830,7 @@ - +
TestEbookConvertCalibreGDrive - test_thumbnail_cache
@@ -930,21 +1840,21 @@ - + TestEbookConvertKepubify 4 - 4 - 0 + 2 0 + 2 0 - Detail + Detail - +
TestEbookConvertKepubify - test_convert_deactivate
@@ -953,16 +1863,36 @@ - +
TestEbookConvertKepubify - test_convert_only
- PASS + +
+ ERROR +
+ + + + - +
TestEbookConvertKepubify - test_convert_wrong_excecutable
@@ -971,31 +1901,51 @@ - +
TestEbookConvertKepubify - test_kobo_kepub_formats
- PASS + +
+ ERROR +
+ + + + - + TestEbookConvertGDriveKepubify 3 - 3 - 0 + 2 + 1 0 0 - Detail + Detail - +
TestEbookConvertGDriveKepubify - test_convert_deactivate
@@ -1004,16 +1954,38 @@ - +
TestEbookConvertGDriveKepubify - test_convert_only
- PASS + +
+ FAIL +
+ + + + - +
TestEbookConvertGDriveKepubify - test_convert_wrong_excecutable
@@ -1023,21 +1995,21 @@ - + TestEditAdditionalBooks 20 - 18 - 0 + 15 0 + 3 2 - Detail + Detail - +
TestEditAdditionalBooks - test_cbz_comicinfo
@@ -1046,7 +2018,7 @@ - +
TestEditAdditionalBooks - test_change_upload_formats
@@ -1055,7 +2027,7 @@ - +
TestEditAdditionalBooks - test_delete_book
@@ -1064,7 +2036,7 @@ - +
TestEditAdditionalBooks - test_delete_role
@@ -1073,7 +2045,7 @@ - +
TestEditAdditionalBooks - test_details_popup
@@ -1082,7 +2054,7 @@ - +
TestEditAdditionalBooks - test_edit_book_identifier
@@ -1091,7 +2063,7 @@ - +
TestEditAdditionalBooks - test_edit_book_identifier_capital
@@ -1100,7 +2072,7 @@ - +
TestEditAdditionalBooks - test_edit_book_identifier_standard
@@ -1109,7 +2081,7 @@ - +
TestEditAdditionalBooks - test_edit_special_book_identifier
@@ -1118,7 +2090,7 @@ - +
TestEditAdditionalBooks - test_title_sort
@@ -1127,7 +2099,7 @@ - +
TestEditAdditionalBooks - test_upload_cbz_coverformats
@@ -1136,7 +2108,7 @@ - +
TestEditAdditionalBooks - test_upload_edit_role
@@ -1145,7 +2117,7 @@ - +
TestEditAdditionalBooks - test_upload_metadata_cb7
@@ -1154,7 +2126,7 @@ - +
TestEditAdditionalBooks - test_upload_metadata_cbr
@@ -1163,7 +2135,7 @@ - +
TestEditAdditionalBooks - test_upload_metadata_cbt
@@ -1172,19 +2144,19 @@ - +
TestEditAdditionalBooks - test_writeonly_calibre_database
- SKIP + SKIP
-