mirrors/calibre-web
1
0
Fork 0
mirror of https://github.com/janeczku/calibre-web synced 2024-12-12 19:20:30 +00:00
Code Issues Releases Wiki Activity

update Tests

Browse Source 
Per request db connection
This commit is contained in:
Ozzie Isaacs 2024-11-26 06:13:10 +01:00
parent 7c6347442f
commit 190ee09e60
9 changed files with 4528 additions and 1071 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cps/__init__.py
View File

@ -102,7 +102,7 @@ if wtf_present:
else:
csrf = None
calibre_db = db.CalibreDB()
calibre_db = db.CalibreDB(app)
web_server = WebServer()
@ -146,9 +146,7 @@ def create_app():
lm.anonymous_user = ub.Anonymous
lm.session_protection = 'strong' if config.config_session == 1 else "basic"
db.CalibreDB.update_config(config)
db.CalibreDB.setup_db(config.config_calibre_dir, cli_param.settings_path)
calibre_db.init_db()
db.CalibreDB.update_config(config, config.config_calibre_dir, cli_param.settings_path)
updater_thread.init_updater(config, web_server)
# Perform dry run of updater and exit afterward

5
cps/admin.py
View File

@ -144,7 +144,6 @@ def shutdown():
show_text = {}
if task in (0, 1): # valid commandos received
# close all database connections
calibre_db.dispose()
ub.dispose()
if task == 0:
@ -1767,10 +1766,10 @@ def _db_configuration_update_helper():
config.config_allowed_column_value = ""
config.config_read_column = 0
_config_string(to_save, "config_calibre_dir")
calibre_db.update_config(config)
calibre_db.update_config(config, config.config_calibre_dir, ub.app_DB_path)
if not os.access(os.path.join(config.config_calibre_dir, "metadata.db"), os.W_OK):
flash(_("DB is not Writeable"), category="warning")
calibre_db.update_config(config)
calibre_db.update_config(config, config.config_calibre_dir, ub.app_DB_path)
config.save()
return _db_configuration_result(None, gdrive_error)

10
cps/config_sql.py
View File

@ -405,11 +405,13 @@ class ConfigSQL(object):
return self.config_calibre_split_dir if self.config_calibre_split_dir else self.config_calibre_dir
def store_calibre_uuid(self, calibre_db, Library_table):
from . import app
try:
calibre_uuid = calibre_db.session.query(Library_table).one_or_none()
if self.config_calibre_uuid != calibre_uuid.uuid:
self.config_calibre_uuid = calibre_uuid.uuid
self.save()
with app.app_context():
calibre_uuid = calibre_db.session.query(Library_table).one_or_none()
if self.config_calibre_uuid != calibre_uuid.uuid:
self.config_calibre_uuid = calibre_uuid.uuid
self.save()
except AttributeError:
pass

110
cps/db.py
View File

@ -23,7 +23,7 @@ import json
from datetime import datetime, timezone
from urllib.parse import quote
import unidecode
from weakref import WeakSet
# from weakref import WeakSet
from uuid import uuid4
from sqlite3 import OperationalError as sqliteOperationalError
@ -45,7 +45,7 @@ from sqlalchemy.ext.associationproxy import association_proxy
from .cw_login import current_user
from flask_babel import gettext as _
from flask_babel import get_locale
from flask import flash
from flask import flash, g, Flask
from . import logger, ub, isoLanguages
from .pagination import Pagination
@ -528,34 +528,25 @@ class AlchemyEncoder(json.JSONEncoder):
class CalibreDB:
_init = False
engine = None
config = None
session_factory = None
# This is a WeakSet so that references here don't keep other CalibreDB
# instances alive once they reach the end of their respective scopes
instances = WeakSet()
config_calibre_dir = None
app_db_path = None
def __init__(self, expire_on_commit=True, init=False):
def __init__(self, _app: Flask=None): # , expire_on_commit=True, init=False):
""" Initialize a new CalibreDB session
"""
self.session = None
if init:
self.init_db(expire_on_commit)
self.Session = None
#if init:
# self.init_db(expire_on_commit)
if _app is not None:
self.init_app(_app)
def init_db(self, expire_on_commit=True):
if self._init:
self.init_session(expire_on_commit)
self.instances.add(self)
def init_session(self, expire_on_commit=True):
self.session = self.session_factory()
self.session.expire_on_commit = expire_on_commit
self.create_functions(self.config)
def init_app(self, _app):
_app.teardown_appcontext(self.teardown)
@classmethod
def setup_db_cc_classes(cls, cc):
global cc_classes
cc_ids = []
books_custom_column_links = {}
for row in cc:
@ -623,8 +614,6 @@ class CalibreDB:
secondary=books_custom_column_links[cc_id[0]],
backref='books'))
return cc_classes
@classmethod
def check_valid_db(cls, config_calibre_dir, app_db_path, config_calibre_uuid):
if not config_calibre_dir:
@ -644,7 +633,6 @@ class CalibreDB:
local_session = scoped_session(sessionmaker())
local_session.configure(bind=connection)
database_uuid = local_session().query(Library_Id).one_or_none()
# local_session.dispose()
check_engine.connect()
db_change = config_calibre_uuid != database_uuid.uuid
@ -652,13 +640,30 @@ class CalibreDB:
return False, False
return True, db_change
def teardown(self, exception):
ctx = g.get("lib_sql")
if ctx:
ctx.close()
@property
def session(self):
# connect or get active connection
if not g.get("lib_sql"):
g.lib_sql = self.connect()
return g.lib_sql
@classmethod
def update_config(cls, config):
def update_config(cls, config, config_calibre_dir, app_db_path):
cls.config = config
cls.config_calibre_dir = config_calibre_dir
cls.app_db_path = app_db_path
def connect(self):
return self.setup_db(self.config_calibre_dir, self.app_db_path)
@classmethod
def setup_db(cls, config_calibre_dir, app_db_path):
cls.dispose()
if not config_calibre_dir:
cls.config.invalidate()
@ -670,17 +675,17 @@ class CalibreDB:
return None
try:
cls.engine = create_engine('sqlite://',
engine = create_engine('sqlite://',
echo=False,
isolation_level="SERIALIZABLE",
connect_args={'check_same_thread': False},
poolclass=StaticPool)
with cls.engine.begin() as connection:
with engine.begin() as connection:
connection.execute(text('PRAGMA cache_size = 10000;'))
connection.execute(text("attach database '{}' as calibre;".format(dbpath)))
connection.execute(text("attach database '{}' as app_settings;".format(app_db_path)))
conn = cls.engine.connect()
conn = engine.connect()
# conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
except Exception as ex:
cls.config.invalidate(ex)
@ -696,13 +701,10 @@ class CalibreDB:
log.error_or_exception(e)
return None
cls.session_factory = scoped_session(sessionmaker(autocommit=False,
autoflush=True,
bind=cls.engine, future=True))
for inst in cls.instances:
inst.init_session()
return scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=engine, future=True))
cls._init = True
def get_book(self, book_id):
return self.session.query(Books).filter(Books.id == book_id).first()
@ -1066,41 +1068,9 @@ class CalibreDB:
except sqliteOperationalError:
pass
@classmethod
def dispose(cls):
# global session
for inst in cls.instances:
old_session = inst.session
inst.session = None
if old_session:
try:
old_session.close()
except Exception:
pass
if old_session.bind:
try:
old_session.bind.dispose()
except Exception:
pass
for attr in list(Books.__dict__.keys()):
if attr.startswith("custom_column_"):
setattr(Books, attr, None)
for db_class in cc_classes.values():
Base.metadata.remove(db_class.__table__)
cc_classes.clear()
for table in reversed(Base.metadata.sorted_tables):
name = table.key
if name.startswith("custom_column_") or name.startswith("books_custom_column_"):
if table is not None:
Base.metadata.remove(table)
def reconnect_db(self, config, app_db_path):
self.dispose()
self.engine.dispose()
# self.dispose()
# self.engine.dispose()
self.setup_db(config.config_calibre_dir, app_db_path)
self.update_config(config)

264
cps/tasks/convert.py
View File

@ -28,7 +28,7 @@ from sqlalchemy.exc import SQLAlchemyError
from flask_babel import lazy_gettext as N_
from cps.services.worker import CalibreTask
from cps import db
from cps import db, app
from cps import logger, config
from cps.subproc_wrapper import process_open
from flask_babel import gettext as _
@ -62,157 +62,159 @@ class TaskConvert(CalibreTask):
def run(self, worker_thread):
self.worker_thread = worker_thread
if config.config_use_google_drive:
worker_db = db.CalibreDB(expire_on_commit=False, init=True)
cur_book = worker_db.get_book(self.book_id)
self.title = cur_book.title
data = worker_db.get_book_format(self.book_id, self.settings['old_book_format'])
df = gdriveutils.getFileFromEbooksFolder(cur_book.path,
data.name + "." + self.settings['old_book_format'].lower())
df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg")
if df:
datafile_cover = None
datafile = os.path.join(config.get_book_path(),
cur_book.path,
data.name + "." + self.settings['old_book_format'].lower())
if df_cover:
datafile_cover = os.path.join(config.get_book_path(),
cur_book.path, "cover.jpg")
if not os.path.exists(os.path.join(config.get_book_path(), cur_book.path)):
os.makedirs(os.path.join(config.get_book_path(), cur_book.path))
df.GetContentFile(datafile)
if df_cover:
df_cover.GetContentFile(datafile_cover)
worker_db.session.close()
else:
# ToDo Include cover in error handling
error_message = _("%(format)s not found on Google Drive: %(fn)s",
format=self.settings['old_book_format'],
fn=data.name + "." + self.settings['old_book_format'].lower())
worker_db.session.close()
with app.app_context():
worker_db = db.CalibreDB(app)
cur_book = worker_db.get_book(self.book_id)
self.title = cur_book.title
data = worker_db.get_book_format(self.book_id, self.settings['old_book_format'])
df = gdriveutils.getFileFromEbooksFolder(cur_book.path,
data.name + "." + self.settings['old_book_format'].lower())
df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg")
if df:
datafile_cover = None
datafile = os.path.join(config.get_book_path(),
cur_book.path,
data.name + "." + self.settings['old_book_format'].lower())
if df_cover:
datafile_cover = os.path.join(config.get_book_path(),
cur_book.path, "cover.jpg")
if not os.path.exists(os.path.join(config.get_book_path(), cur_book.path)):
os.makedirs(os.path.join(config.get_book_path(), cur_book.path))
df.GetContentFile(datafile)
if df_cover:
df_cover.GetContentFile(datafile_cover)
# worker_db.session.close()
else:
# ToDo Include cover in error handling
error_message = _("%(format)s not found on Google Drive: %(fn)s",
format=self.settings['old_book_format'],
fn=data.name + "." + self.settings['old_book_format'].lower())
# worker_db.session.close()
return self._handleError(error_message)
filename = self._convert_ebook_format()
if config.config_use_google_drive:
os.remove(self.file_path + '.' + self.settings['old_book_format'].lower())
if df_cover:
os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg"))
if filename:
filename = self._convert_ebook_format()
if config.config_use_google_drive:
# Upload files to gdrive
gdriveutils.updateGdriveCalibreFromLocal()
self._handleSuccess()
if self.ereader_mail:
# if we're sending to E-Reader after converting, create a one-off task and run it immediately
# todo: figure out how to incorporate this into the progress
try:
EmailText = N_(u"%(book)s send to E-Reader", book=escape(self.title))
for email in self.ereader_mail.split(','):
email = strip_whitespaces(email)
worker_thread.add(self.user, TaskEmail(self.settings['subject'],
self.results["path"],
filename,
self.settings,
email,
EmailText,
self.settings['body'],
id=self.book_id,
internal=True)
)
except Exception as ex:
return self._handleError(str(ex))
os.remove(self.file_path + '.' + self.settings['old_book_format'].lower())
if df_cover:
os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg"))
if filename:
if config.config_use_google_drive:
# Upload files to gdrive
gdriveutils.updateGdriveCalibreFromLocal()
self._handleSuccess()
if self.ereader_mail:
# if we're sending to E-Reader after converting, create a one-off task and run it immediately
# todo: figure out how to incorporate this into the progress
try:
EmailText = N_(u"%(book)s send to E-Reader", book=escape(self.title))
for email in self.ereader_mail.split(','):
email = strip_whitespaces(email)
worker_thread.add(self.user, TaskEmail(self.settings['subject'],
self.results["path"],
filename,
self.settings,
email,
EmailText,
self.settings['body'],
id=self.book_id,
internal=True)
)
except Exception as ex:
return self._handleError(str(ex))
def _convert_ebook_format(self):
error_message = None
local_db = db.CalibreDB(expire_on_commit=False, init=True)
file_path = self.file_path
book_id = self.book_id
format_old_ext = '.' + self.settings['old_book_format'].lower()
format_new_ext = '.' + self.settings['new_book_format'].lower()
with app.app_context():
local_db = db.CalibreDB(app)
file_path = self.file_path
book_id = self.book_id
format_old_ext = '.' + self.settings['old_book_format'].lower()
format_new_ext = '.' + self.settings['new_book_format'].lower()
# check to see if destination format already exists - or if book is in database
# if it does - mark the conversion task as complete and return a success
# this will allow to send to E-Reader workflow to continue to work
if os.path.isfile(file_path + format_new_ext) or\
local_db.get_book_format(self.book_id, self.settings['new_book_format']):
log.info("Book id %d already converted to %s", book_id, format_new_ext)
cur_book = local_db.get_book(book_id)
self.title = cur_book.title
self.results['path'] = cur_book.path
self.results['title'] = self.title
new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id)\
.filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none()
if not new_format:
new_format = db.Data(name=os.path.basename(file_path),
book_format=self.settings['new_book_format'].upper(),
book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext))
try:
local_db.session.merge(new_format)
local_db.session.commit()
except SQLAlchemyError as e:
local_db.session.rollback()
log.error("Database error: %s", e)
local_db.session.close()
self._handleError(N_("Oops! Database Error: %(error)s.", error=e))
return
self._handleSuccess()
local_db.session.close()
return os.path.basename(file_path + format_new_ext)
else:
log.info("Book id %d - target format of %s does not exist. Moving forward with convert.",
book_id,
format_new_ext)
if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub':
check, error_message = self._convert_kepubify(file_path,
format_old_ext,
format_new_ext)
else:
# check if calibre converter-executable is existing
if not os.path.exists(config.config_converterpath):
self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
return
has_cover = local_db.get_book(book_id).has_cover
check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover)
if check == 0:
cur_book = local_db.get_book(book_id)
if os.path.isfile(file_path + format_new_ext):
new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id) \
# check to see if destination format already exists - or if book is in database
# if it does - mark the conversion task as complete and return a success
# this will allow to send to E-Reader workflow to continue to work
if os.path.isfile(file_path + format_new_ext) or\
local_db.get_book_format(self.book_id, self.settings['new_book_format']):
log.info("Book id %d already converted to %s", book_id, format_new_ext)
cur_book = local_db.get_book(book_id)
self.title = cur_book.title
self.results['path'] = cur_book.path
self.results['title'] = self.title
new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id)\
.filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none()
if not new_format:
new_format = db.Data(name=cur_book.data[0].name,
new_format = db.Data(name=os.path.basename(file_path),
book_format=self.settings['new_book_format'].upper(),
book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext))
try:
local_db.session.merge(new_format)
local_db.session.commit()
if self.settings['new_book_format'].upper() in ['KEPUB', 'EPUB', 'EPUB3']:
ub_session = init_db_thread()
remove_synced_book(book_id, True, ub_session)
ub_session.close()
except SQLAlchemyError as e:
local_db.session.rollback()
log.error("Database error: %s", e)
local_db.session.close()
self._handleError(error_message)
self._handleError(N_("Oops! Database Error: %(error)s.", error=e))
return
self.results['path'] = cur_book.path
self.title = cur_book.title
self.results['title'] = self.title
if not config.config_use_google_drive:
self._handleSuccess()
return os.path.basename(file_path + format_new_ext)
local_db.session.close()
return os.path.basename(file_path + format_new_ext)
else:
error_message = N_('%(format)s format not found on disk', format=format_new_ext.upper())
local_db.session.close()
log.info("ebook converter failed with error while converting book")
if not error_message:
error_message = N_('Ebook converter failed with unknown error')
else:
log.error(error_message)
self._handleError(error_message)
return
log.info("Book id %d - target format of %s does not exist. Moving forward with convert.",
book_id,
format_new_ext)
if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub':
check, error_message = self._convert_kepubify(file_path,
format_old_ext,
format_new_ext)
else:
# check if calibre converter-executable is existing
if not os.path.exists(config.config_converterpath):
self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
return
has_cover = local_db.get_book(book_id).has_cover
check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover)
if check == 0:
cur_book = local_db.get_book(book_id)
if os.path.isfile(file_path + format_new_ext):
new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id) \
.filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none()
if not new_format:
new_format = db.Data(name=cur_book.data[0].name,
book_format=self.settings['new_book_format'].upper(),
book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext))
try:
local_db.session.merge(new_format)
local_db.session.commit()
if self.settings['new_book_format'].upper() in ['KEPUB', 'EPUB', 'EPUB3']:
ub_session = init_db_thread()
remove_synced_book(book_id, True, ub_session)
ub_session.close()
except SQLAlchemyError as e:
local_db.session.rollback()
log.error("Database error: %s", e)
local_db.session.close()
self._handleError(error_message)
return
self.results['path'] = cur_book.path
self.title = cur_book.title
self.results['title'] = self.title
if not config.config_use_google_drive:
self._handleSuccess()
return os.path.basename(file_path + format_new_ext)
else:
error_message = N_('%(format)s format not found on disk', format=format_new_ext.upper())
local_db.session.close()
log.info("ebook converter failed with error while converting book")
if not error_message:
error_message = N_('Ebook converter failed with unknown error')
else:
log.error(error_message)
self._handleError(error_message)
return
def _convert_kepubify(self, file_path, format_old_ext, format_new_ext):
if config.config_embed_metadata and config.config_binariesdir:

10
cps/tasks/database.py
View File

@ -18,7 +18,7 @@
from flask_babel import lazy_gettext as N_
from cps import config, logger, db, ub
from cps import config, logger, db, ub, app
from cps.services.worker import CalibreTask
@ -26,11 +26,13 @@ class TaskReconnectDatabase(CalibreTask):
def __init__(self, task_message=N_('Reconnecting Calibre database')):
super(TaskReconnectDatabase, self).__init__(task_message)
self.log = logger.create()
self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
# self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
def run(self, worker_thread):
self.calibre_db.reconnect_db(config, ub.app_DB_path)
self.calibre_db.session.close()
with app.app_context():
calibre_db = db.CalibreDB(app)
calibre_db.reconnect_db(config, ub.app_DB_path)
# self.calibre_db.session.close()
self._handleSuccess()
@property

84
cps/tasks/metadata_backup.py
View File

@ -19,7 +19,7 @@
import os
from lxml import etree
from cps import config, db, gdriveutils, logger
from cps import config, db, gdriveutils, logger, app
from cps.services.worker import CalibreTask
from flask_babel import lazy_gettext as N_
@ -34,7 +34,7 @@ class TaskBackupMetadata(CalibreTask):
task_message=N_('Backing up Metadata')):
super(TaskBackupMetadata, self).__init__(task_message)
self.log = logger.create()
self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
# self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
self.export_language = export_language
self.translated_title = translated_title
self.set_dirty = set_dirty
@ -46,47 +46,51 @@ class TaskBackupMetadata(CalibreTask):
self.backup_metadata()
def set_all_books_dirty(self):
try:
books = self.calibre_db.session.query(db.Books).all()
for book in books:
self.calibre_db.set_metadata_dirty(book.id)
self.calibre_db.session.commit()
self._handleSuccess()
except Exception as ex:
self.log.debug('Error adding book for backup: ' + str(ex))
self._handleError('Error adding book for backup: ' + str(ex))
self.calibre_db.session.rollback()
self.calibre_db.session.close()
with app.app_context():
calibre_db = db.CalibreDB(app)
try:
books = calibre_db.session.query(db.Books).all()
for book in books:
calibre_db.set_metadata_dirty(book.id)
calibre_db.session.commit()
self._handleSuccess()
except Exception as ex:
self.log.debug('Error adding book for backup: ' + str(ex))
self._handleError('Error adding book for backup: ' + str(ex))
calibre_db.session.rollback()
# self.calibre_db.session.close()
def backup_metadata(self):
try:
metadata_backup = self.calibre_db.session.query(db.Metadata_Dirtied).all()
custom_columns = (self.calibre_db.session.query(db.CustomColumns)
.filter(db.CustomColumns.mark_for_delete == 0)
.filter(db.CustomColumns.datatype.notin_(db.cc_exceptions))
.order_by(db.CustomColumns.label).all())
count = len(metadata_backup)
i = 0
for backup in metadata_backup:
book = self.calibre_db.session.query(db.Books).filter(db.Books.id == backup.book).one_or_none()
self.calibre_db.session.query(db.Metadata_Dirtied).filter(
db.Metadata_Dirtied.book == backup.book).delete()
self.calibre_db.session.commit()
if book:
self.open_metadata(book, custom_columns)
else:
self.log.error("Book {} not found in database".format(backup.book))
i += 1
self.progress = (1.0 / count) * i
self._handleSuccess()
self.calibre_db.session.close()
with app.app_context():
try:
calibre_db = db.CalibreDB(app)
metadata_backup = calibre_db.session.query(db.Metadata_Dirtied).all()
custom_columns = (calibre_db.session.query(db.CustomColumns)
.filter(db.CustomColumns.mark_for_delete == 0)
.filter(db.CustomColumns.datatype.notin_(db.cc_exceptions))
.order_by(db.CustomColumns.label).all())
count = len(metadata_backup)
i = 0
for backup in metadata_backup:
book = calibre_db.session.query(db.Books).filter(db.Books.id == backup.book).one_or_none()
calibre_db.session.query(db.Metadata_Dirtied).filter(
db.Metadata_Dirtied.book == backup.book).delete()
calibre_db.session.commit()
if book:
self.open_metadata(book, custom_columns)
else:
self.log.error("Book {} not found in database".format(backup.book))
i += 1
self.progress = (1.0 / count) * i
self._handleSuccess()
# self.calibre_db.session.close()
except Exception as ex:
b = "NaN" if not hasattr(book, 'id') else book.id
self.log.debug('Error creating metadata backup for book {}: '.format(b) + str(ex))
self._handleError('Error creating metadata backup: ' + str(ex))
self.calibre_db.session.rollback()
self.calibre_db.session.close()
except Exception as ex:
b = "NaN" if not hasattr(book, 'id') else book.id
self.log.debug('Error creating metadata backup for book {}: '.format(b) + str(ex))
self._handleError('Error creating metadata backup: ' + str(ex))
calibre_db.session.rollback()
# self.calibre_db.session.close()
def open_metadata(self, book, custom_columns):
# package = self.create_new_metadata_backup(book, custom_columns)

116
cps/tasks/thumbnail.py
View File

@ -23,10 +23,11 @@ from io import BytesIO
from datetime import datetime, timezone
from .. import constants
from cps import config, db, fs, gdriveutils, logger, ub
from cps import config, db, fs, gdriveutils, logger, ub, app
from cps.services.worker import CalibreTask, STAT_CANCELLED, STAT_ENDED
from sqlalchemy import func, text, or_
from flask_babel import lazy_gettext as N_
try:
from wand.image import Image
use_IM = True
@ -113,9 +114,10 @@ class TaskGenerateCoverThumbnails(CalibreTask):
@staticmethod
def get_books_with_covers(book_id=-1):
filter_exp = (db.Books.id == book_id) if book_id != -1 else True
calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
books_cover = calibre_db.session.query(db.Books).filter(db.Books.has_cover == 1).filter(filter_exp).all()
calibre_db.session.close()
with app.app_context():
calibre_db = db.CalibreDB(app) #, expire_on_commit=False, init=True)
books_cover = calibre_db.session.query(db.Books).filter(db.Books.has_cover == 1).filter(filter_exp).all()
# calibre_db.session.close()
return books_cover
def get_book_cover_thumbnails(self, book_id):
@ -246,7 +248,7 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
super(TaskGenerateSeriesThumbnails, self).__init__(task_message)
self.log = logger.create()
self.app_db_session = ub.get_new_session_instance()
self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
# self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
self.cache = fs.FileSystem()
self.resolutions = [
constants.COVER_THUMBNAIL_SMALL,
@ -254,58 +256,60 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
]
def run(self, worker_thread):
if self.calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED:
self.message = 'Scanning Series'
all_series = self.get_series_with_four_plus_books()
count = len(all_series)
with app.app_context():
calibre_db = db.CalibreDB(app)
if calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED:
self.message = 'Scanning Series'
all_series = self.get_series_with_four_plus_books(calibre_db)
count = len(all_series)
total_generated = 0
for i, series in enumerate(all_series):
generated = 0
series_thumbnails = self.get_series_thumbnails(series.id)
series_books = self.get_series_books(series.id)
total_generated = 0
for i, series in enumerate(all_series):
generated = 0
series_thumbnails = self.get_series_thumbnails(series.id)
series_books = self.get_series_books(series.id, calibre_db)
# Generate new thumbnails for missing covers
resolutions = list(map(lambda t: t.resolution, series_thumbnails))
missing_resolutions = list(set(self.resolutions).difference(resolutions))
for resolution in missing_resolutions:
generated += 1
self.create_series_thumbnail(series, series_books, resolution)
# Replace outdated or missing thumbnails
for thumbnail in series_thumbnails:
if any(book.last_modified > thumbnail.generated_at for book in series_books):
# Generate new thumbnails for missing covers
resolutions = list(map(lambda t: t.resolution, series_thumbnails))
missing_resolutions = list(set(self.resolutions).difference(resolutions))
for resolution in missing_resolutions:
generated += 1
self.update_series_thumbnail(series_books, thumbnail)
self.create_series_thumbnail(series, series_books, resolution)
elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS):
generated += 1
self.update_series_thumbnail(series_books, thumbnail)
# Replace outdated or missing thumbnails
for thumbnail in series_thumbnails:
if any(book.last_modified > thumbnail.generated_at for book in series_books):
generated += 1
self.update_series_thumbnail(series_books, thumbnail)
# Increment the progress
self.progress = (1.0 / count) * i
elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS):
generated += 1
self.update_series_thumbnail(series_books, thumbnail)
if generated > 0:
total_generated += generated
self.message = N_('Generated {0} series thumbnails').format(total_generated)
# Increment the progress
self.progress = (1.0 / count) * i
# Check if job has been cancelled or ended
if self.stat == STAT_CANCELLED:
self.log.info(f'GenerateSeriesThumbnails task has been cancelled.')
return
if generated > 0:
total_generated += generated
self.message = N_('Generated {0} series thumbnails').format(total_generated)
if self.stat == STAT_ENDED:
self.log.info(f'GenerateSeriesThumbnails task has been ended.')
return
# Check if job has been cancelled or ended
if self.stat == STAT_CANCELLED:
self.log.info(f'GenerateSeriesThumbnails task has been cancelled.')
return
if total_generated == 0:
self.self_cleanup = True
if self.stat == STAT_ENDED:
self.log.info(f'GenerateSeriesThumbnails task has been ended.')
return
self._handleSuccess()
self.app_db_session.remove()
if total_generated == 0:
self.self_cleanup = True
def get_series_with_four_plus_books(self):
return self.calibre_db.session \
self._handleSuccess()
self.app_db_session.remove()
def get_series_with_four_plus_books(self, calibre_db):
return calibre_db.session \
.query(db.Series) \
.join(db.books_series_link) \
.join(db.Books) \
@ -314,8 +318,8 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
.having(func.count('book_series_link') > 3) \
.all()
def get_series_books(self, series_id):
return self.calibre_db.session \
def get_series_books(self, series_id, calibre_db):
return calibre_db.session \
.query(db.Books) \
.join(db.books_series_link) \
.join(db.Series) \
@ -461,13 +465,15 @@ class TaskClearCoverThumbnailCache(CalibreTask):
def run(self, worker_thread):
if self.app_db_session:
if self.book_id == 0: # delete superfluous thumbnails
calibre_db = db.CalibreDB(expire_on_commit=False, init=True)
thumbnails = (calibre_db.session.query(ub.Thumbnail)
.join(db.Books, ub.Thumbnail.entity_id == db.Books.id, isouter=True)
.filter(db.Books.id==None)
.all())
calibre_db.session.close()
# delete superfluous thumbnails
if self.book_id == 0:
with app.app_context():
calibre_db = db.CalibreDB(app)
thumbnails = (calibre_db.session.query(ub.Thumbnail)
.join(db.Books, ub.Thumbnail.entity_id == db.Books.id, isouter=True)
.filter(db.Books.id==None)
.all())
# calibre_db.session.close()
elif self.book_id > 0: # make sure single book is selected
thumbnails = self.get_thumbnails_for_book(self.book_id)
if self.book_id < 0:

4994
test/Calibre-Web TestSummary_Linux.html
View File

File diff suppressed because it is too large Load Diff