update Tests

Per request db connection

cps/__init__.py

@@ -102,7 +102,7 @@ if wtf_present:
 else:
     csrf = None
 
-calibre_db = db.CalibreDB()
+calibre_db = db.CalibreDB(app)
 
 web_server = WebServer()
 
@@ -146,9 +146,7 @@ def create_app():
     lm.anonymous_user = ub.Anonymous
     lm.session_protection = 'strong' if config.config_session == 1 else "basic"
 
-    db.CalibreDB.update_config(config)
+    db.CalibreDB.update_config(config, config.config_calibre_dir, cli_param.settings_path)
-    db.CalibreDB.setup_db(config.config_calibre_dir, cli_param.settings_path)
-    calibre_db.init_db()
 
     updater_thread.init_updater(config, web_server)
     # Perform dry run of updater and exit afterward

cps/admin.py

@@ -144,7 +144,6 @@ def shutdown():
     show_text = {}
     if task in (0, 1):  # valid commandos received
         # close all database connections
-        calibre_db.dispose()
         ub.dispose()
 
         if task == 0:
@@ -1767,10 +1766,10 @@ def _db_configuration_update_helper():
             config.config_allowed_column_value = ""
             config.config_read_column = 0
         _config_string(to_save, "config_calibre_dir")
-        calibre_db.update_config(config)
+        calibre_db.update_config(config, config.config_calibre_dir, ub.app_DB_path)
         if not os.access(os.path.join(config.config_calibre_dir, "metadata.db"), os.W_OK):
             flash(_("DB is not Writeable"), category="warning")
-    calibre_db.update_config(config)
+    calibre_db.update_config(config, config.config_calibre_dir, ub.app_DB_path)
     config.save()
     return _db_configuration_result(None, gdrive_error)
 

cps/config_sql.py

@@ -405,7 +405,9 @@ class ConfigSQL(object):
         return self.config_calibre_split_dir if self.config_calibre_split_dir else self.config_calibre_dir
 
     def store_calibre_uuid(self, calibre_db, Library_table):
+        from . import app
         try:
+            with app.app_context():
                 calibre_uuid = calibre_db.session.query(Library_table).one_or_none()
                 if self.config_calibre_uuid != calibre_uuid.uuid:
                     self.config_calibre_uuid = calibre_uuid.uuid

cps/db.py

@@ -23,7 +23,7 @@ import json
 from datetime import datetime, timezone
 from urllib.parse import quote
 import unidecode
-from weakref import WeakSet
+# from weakref import WeakSet
 from uuid import uuid4
 
 from sqlite3 import OperationalError as sqliteOperationalError
@@ -45,7 +45,7 @@ from sqlalchemy.ext.associationproxy import association_proxy
 from .cw_login import current_user
 from flask_babel import gettext as _
 from flask_babel import get_locale
-from flask import flash
+from flask import flash, g, Flask
 
 from . import logger, ub, isoLanguages
 from .pagination import Pagination
@@ -528,34 +528,25 @@ class AlchemyEncoder(json.JSONEncoder):
 
 
 class CalibreDB:
-    _init = False
-    engine = None
     config = None
-    session_factory = None
+    config_calibre_dir = None
-    # This is a WeakSet so that references here don't keep other CalibreDB
+    app_db_path = None
-    # instances alive once they reach the end of their respective scopes
-    instances = WeakSet()
 
-    def __init__(self, expire_on_commit=True, init=False):
+    def __init__(self, _app: Flask=None):  # , expire_on_commit=True, init=False):
         """ Initialize a new CalibreDB session
         """
-        self.session = None
+        self.Session = None
-        if init:
+        #if init:
-            self.init_db(expire_on_commit)
+        #    self.init_db(expire_on_commit)
+        if _app is not None:
+            self.init_app(_app)
 
-    def init_db(self, expire_on_commit=True):
+    def init_app(self, _app):
-        if self._init:
+        _app.teardown_appcontext(self.teardown)
-            self.init_session(expire_on_commit)
-
-        self.instances.add(self)
-
-    def init_session(self, expire_on_commit=True):
-        self.session = self.session_factory()
-        self.session.expire_on_commit = expire_on_commit
-        self.create_functions(self.config)
 
     @classmethod
     def setup_db_cc_classes(cls, cc):
+        global cc_classes
         cc_ids = []
         books_custom_column_links = {}
         for row in cc:
@@ -623,8 +614,6 @@ class CalibreDB:
                                      secondary=books_custom_column_links[cc_id[0]],
                                      backref='books'))
 
-        return cc_classes
-
     @classmethod
     def check_valid_db(cls, config_calibre_dir, app_db_path, config_calibre_uuid):
         if not config_calibre_dir:
@@ -644,7 +633,6 @@ class CalibreDB:
                 local_session = scoped_session(sessionmaker())
                 local_session.configure(bind=connection)
                 database_uuid = local_session().query(Library_Id).one_or_none()
-                # local_session.dispose()
 
             check_engine.connect()
             db_change = config_calibre_uuid != database_uuid.uuid
@@ -652,13 +640,30 @@ class CalibreDB:
             return False, False
         return True, db_change
 
+    def teardown(self, exception):
+        ctx = g.get("lib_sql")
+        if ctx:
+            ctx.close()
+
+    @property
+    def session(self):
+        # connect or get active connection
+        if not g.get("lib_sql"):
+            g.lib_sql = self.connect()
+        return g.lib_sql
+
     @classmethod
-    def update_config(cls, config):
+    def update_config(cls, config, config_calibre_dir, app_db_path):
         cls.config = config
+        cls.config_calibre_dir = config_calibre_dir
+        cls.app_db_path = app_db_path
+
+
+    def connect(self):
+        return self.setup_db(self.config_calibre_dir, self.app_db_path)
 
     @classmethod
     def setup_db(cls, config_calibre_dir, app_db_path):
-        cls.dispose()
 
         if not config_calibre_dir:
             cls.config.invalidate()
@@ -670,17 +675,17 @@ class CalibreDB:
             return None
 
         try:
-            cls.engine = create_engine('sqlite://',
+            engine = create_engine('sqlite://',
                                        echo=False,
                                        isolation_level="SERIALIZABLE",
                                        connect_args={'check_same_thread': False},
                                        poolclass=StaticPool)
-            with cls.engine.begin() as connection:
+            with engine.begin() as connection:
                 connection.execute(text('PRAGMA cache_size = 10000;'))
                 connection.execute(text("attach database '{}' as calibre;".format(dbpath)))
                 connection.execute(text("attach database '{}' as app_settings;".format(app_db_path)))
 
-            conn = cls.engine.connect()
+            conn = engine.connect()
             # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
         except Exception as ex:
             cls.config.invalidate(ex)
@@ -696,13 +701,10 @@ class CalibreDB:
                 log.error_or_exception(e)
                 return None
 
-        cls.session_factory = scoped_session(sessionmaker(autocommit=False,
+        return scoped_session(sessionmaker(autocommit=False,
-                                                          autoflush=True,
+                                           autoflush=False,
-                                                          bind=cls.engine, future=True))
+                                           bind=engine, future=True))
-        for inst in cls.instances:
-            inst.init_session()
 
-        cls._init = True
 
     def get_book(self, book_id):
         return self.session.query(Books).filter(Books.id == book_id).first()
@@ -1066,41 +1068,9 @@ class CalibreDB:
         except sqliteOperationalError:
             pass
 
-    @classmethod
-    def dispose(cls):
-        # global session
-
-        for inst in cls.instances:
-            old_session = inst.session
-            inst.session = None
-            if old_session:
-                try:
-                    old_session.close()
-                except Exception:
-                    pass
-                if old_session.bind:
-                    try:
-                        old_session.bind.dispose()
-                    except Exception:
-                        pass
-
-        for attr in list(Books.__dict__.keys()):
-            if attr.startswith("custom_column_"):
-                setattr(Books, attr, None)
-
-        for db_class in cc_classes.values():
-            Base.metadata.remove(db_class.__table__)
-        cc_classes.clear()
-
-        for table in reversed(Base.metadata.sorted_tables):
-            name = table.key
-            if name.startswith("custom_column_") or name.startswith("books_custom_column_"):
-                if table is not None:
-                    Base.metadata.remove(table)
-
     def reconnect_db(self, config, app_db_path):
-        self.dispose()
+        # self.dispose()
-        self.engine.dispose()
+        # self.engine.dispose()
         self.setup_db(config.config_calibre_dir, app_db_path)
         self.update_config(config)
 

cps/tasks/convert.py

@@ -28,7 +28,7 @@ from sqlalchemy.exc import SQLAlchemyError
 from flask_babel import lazy_gettext as N_
 
 from cps.services.worker import CalibreTask
-from cps import db
+from cps import db, app
 from cps import logger, config
 from cps.subproc_wrapper import process_open
 from flask_babel import gettext as _
@@ -62,7 +62,8 @@ class TaskConvert(CalibreTask):
     def run(self, worker_thread):
         self.worker_thread = worker_thread
         if config.config_use_google_drive:
-            worker_db = db.CalibreDB(expire_on_commit=False, init=True)
+            with app.app_context():
+                worker_db = db.CalibreDB(app)
                 cur_book = worker_db.get_book(self.book_id)
                 self.title = cur_book.title
                 data = worker_db.get_book_format(self.book_id, self.settings['old_book_format'])
@@ -82,13 +83,13 @@ class TaskConvert(CalibreTask):
                     df.GetContentFile(datafile)
                     if df_cover:
                         df_cover.GetContentFile(datafile_cover)
-                worker_db.session.close()
+                    # worker_db.session.close()
                 else:
                     # ToDo Include cover in error handling
                     error_message = _("%(format)s not found on Google Drive: %(fn)s",
                                       format=self.settings['old_book_format'],
                                       fn=data.name + "." + self.settings['old_book_format'].lower())
-                worker_db.session.close()
+                    # worker_db.session.close()
                 return self._handleError(error_message)
 
             filename = self._convert_ebook_format()
@@ -124,7 +125,8 @@ class TaskConvert(CalibreTask):
 
     def _convert_ebook_format(self):
         error_message = None
-        local_db = db.CalibreDB(expire_on_commit=False, init=True)
+        with app.app_context():
+            local_db = db.CalibreDB(app)
             file_path = self.file_path
             book_id = self.book_id
             format_old_ext = '.' + self.settings['old_book_format'].lower()

cps/tasks/database.py

@@ -18,7 +18,7 @@
 
 from flask_babel import lazy_gettext as N_
 
-from cps import config, logger, db, ub
+from cps import config, logger, db, ub, app
 from cps.services.worker import CalibreTask
 
 
@@ -26,11 +26,13 @@ class TaskReconnectDatabase(CalibreTask):
     def __init__(self, task_message=N_('Reconnecting Calibre database')):
         super(TaskReconnectDatabase, self).__init__(task_message)
         self.log = logger.create()
-        self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
+        # self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
 
     def run(self, worker_thread):
-        self.calibre_db.reconnect_db(config, ub.app_DB_path)
+        with app.app_context():
-        self.calibre_db.session.close()
+            calibre_db = db.CalibreDB(app)
+            calibre_db.reconnect_db(config, ub.app_DB_path)
+        # self.calibre_db.session.close()
         self._handleSuccess()
 
     @property

cps/tasks/metadata_backup.py

@@ -19,7 +19,7 @@
 import os
 from lxml import etree
 
-from cps import config, db, gdriveutils, logger
+from cps import config, db, gdriveutils, logger, app
 from cps.services.worker import CalibreTask
 from flask_babel import lazy_gettext as N_
 
@@ -34,7 +34,7 @@ class TaskBackupMetadata(CalibreTask):
                  task_message=N_('Backing up Metadata')):
         super(TaskBackupMetadata, self).__init__(task_message)
         self.log = logger.create()
-        self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
+        # self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
         self.export_language = export_language
         self.translated_title = translated_title
         self.set_dirty = set_dirty
@@ -46,32 +46,36 @@ class TaskBackupMetadata(CalibreTask):
             self.backup_metadata()
 
     def set_all_books_dirty(self):
+        with app.app_context():
+            calibre_db = db.CalibreDB(app)
             try:
-            books = self.calibre_db.session.query(db.Books).all()
+                books = calibre_db.session.query(db.Books).all()
                 for book in books:
-                self.calibre_db.set_metadata_dirty(book.id)
+                    calibre_db.set_metadata_dirty(book.id)
-            self.calibre_db.session.commit()
+                calibre_db.session.commit()
                 self._handleSuccess()
             except Exception as ex:
                 self.log.debug('Error adding book for backup: ' + str(ex))
                 self._handleError('Error adding book for backup: ' + str(ex))
-            self.calibre_db.session.rollback()
+                calibre_db.session.rollback()
-        self.calibre_db.session.close()
+        # self.calibre_db.session.close()
 
     def backup_metadata(self):
+        with app.app_context():
             try:
-            metadata_backup = self.calibre_db.session.query(db.Metadata_Dirtied).all()
+                calibre_db = db.CalibreDB(app)
-            custom_columns = (self.calibre_db.session.query(db.CustomColumns)
+                metadata_backup = calibre_db.session.query(db.Metadata_Dirtied).all()
+                custom_columns = (calibre_db.session.query(db.CustomColumns)
                                   .filter(db.CustomColumns.mark_for_delete == 0)
                                   .filter(db.CustomColumns.datatype.notin_(db.cc_exceptions))
                                   .order_by(db.CustomColumns.label).all())
                 count = len(metadata_backup)
                 i = 0
                 for backup in metadata_backup:
-                book = self.calibre_db.session.query(db.Books).filter(db.Books.id == backup.book).one_or_none()
+                    book = calibre_db.session.query(db.Books).filter(db.Books.id == backup.book).one_or_none()
-                self.calibre_db.session.query(db.Metadata_Dirtied).filter(
+                    calibre_db.session.query(db.Metadata_Dirtied).filter(
                         db.Metadata_Dirtied.book == backup.book).delete()
-                self.calibre_db.session.commit()
+                    calibre_db.session.commit()
                     if book:
                         self.open_metadata(book, custom_columns)
                     else:
@@ -79,14 +83,14 @@ class TaskBackupMetadata(CalibreTask):
                     i += 1
                     self.progress = (1.0 / count) * i
                 self._handleSuccess()
-            self.calibre_db.session.close()
+                # self.calibre_db.session.close()
 
             except Exception as ex:
                 b = "NaN" if not hasattr(book, 'id') else book.id
                 self.log.debug('Error creating metadata backup for book {}: '.format(b) + str(ex))
                 self._handleError('Error creating metadata backup: ' + str(ex))
-            self.calibre_db.session.rollback()
+                calibre_db.session.rollback()
-            self.calibre_db.session.close()
+                # self.calibre_db.session.close()
 
     def open_metadata(self, book, custom_columns):
         # package = self.create_new_metadata_backup(book, custom_columns)

cps/tasks/thumbnail.py

@@ -23,10 +23,11 @@ from io import BytesIO
 from datetime import datetime, timezone
 
 from .. import constants
-from cps import config, db, fs, gdriveutils, logger, ub
+from cps import config, db, fs, gdriveutils, logger, ub, app
 from cps.services.worker import CalibreTask, STAT_CANCELLED, STAT_ENDED
 from sqlalchemy import func, text, or_
 from flask_babel import lazy_gettext as N_
+
 try:
     from wand.image import Image
     use_IM = True
@@ -113,9 +114,10 @@ class TaskGenerateCoverThumbnails(CalibreTask):
     @staticmethod
     def get_books_with_covers(book_id=-1):
         filter_exp = (db.Books.id == book_id) if book_id != -1 else True
-        calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
+        with app.app_context():
+            calibre_db = db.CalibreDB(app) #, expire_on_commit=False, init=True)
             books_cover = calibre_db.session.query(db.Books).filter(db.Books.has_cover == 1).filter(filter_exp).all()
-        calibre_db.session.close()
+            # calibre_db.session.close()
         return books_cover
 
     def get_book_cover_thumbnails(self, book_id):
@@ -246,7 +248,7 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
         super(TaskGenerateSeriesThumbnails, self).__init__(task_message)
         self.log = logger.create()
         self.app_db_session = ub.get_new_session_instance()
-        self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
+        # self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
         self.cache = fs.FileSystem()
         self.resolutions = [
             constants.COVER_THUMBNAIL_SMALL,
@@ -254,16 +256,18 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
         ]
 
     def run(self, worker_thread):
-        if self.calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED:
+        with app.app_context():
+            calibre_db = db.CalibreDB(app)
+            if calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED:
                 self.message = 'Scanning Series'
-            all_series = self.get_series_with_four_plus_books()
+                all_series = self.get_series_with_four_plus_books(calibre_db)
                 count = len(all_series)
 
                 total_generated = 0
                 for i, series in enumerate(all_series):
                     generated = 0
                     series_thumbnails = self.get_series_thumbnails(series.id)
-                series_books = self.get_series_books(series.id)
+                    series_books = self.get_series_books(series.id, calibre_db)
 
                     # Generate new thumbnails for missing covers
                     resolutions = list(map(lambda t: t.resolution, series_thumbnails))
@@ -304,8 +308,8 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
             self._handleSuccess()
             self.app_db_session.remove()
 
-    def get_series_with_four_plus_books(self):
+    def get_series_with_four_plus_books(self, calibre_db):
-        return self.calibre_db.session \
+        return calibre_db.session \
             .query(db.Series) \
             .join(db.books_series_link) \
             .join(db.Books) \
@@ -314,8 +318,8 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
             .having(func.count('book_series_link') > 3) \
             .all()
 
-    def get_series_books(self, series_id):
+    def get_series_books(self, series_id, calibre_db):
-        return self.calibre_db.session \
+        return calibre_db.session \
             .query(db.Books) \
             .join(db.books_series_link) \
             .join(db.Series) \
@@ -461,13 +465,15 @@ class TaskClearCoverThumbnailCache(CalibreTask):
 
     def run(self, worker_thread):
         if self.app_db_session:
-            if self.book_id == 0:  # delete superfluous thumbnails
+            # delete superfluous thumbnails
-                calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
+            if self.book_id == 0:
+                with app.app_context():
+                    calibre_db = db.CalibreDB(app)
                     thumbnails = (calibre_db.session.query(ub.Thumbnail)
                                   .join(db.Books, ub.Thumbnail.entity_id == db.Books.id, isouter=True)
                                   .filter(db.Books.id==None)
                                   .all())
-                calibre_db.session.close()
+                    # calibre_db.session.close()
             elif self.book_id > 0:  # make sure single book is selected
                 thumbnails = self.get_thumbnails_for_book(self.book_id)
             if self.book_id < 0: