From 774b9ae12d43542caeafdfd181fdf4dbe44c21c0 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Sat, 19 Dec 2020 00:49:36 -0600 Subject: [PATCH 001/112] Added thumbnail task and database table --- .gitignore | 1 + cps.py | 4 ++ cps/constants.py | 1 + cps/helper.py | 9 ++- cps/tasks/thumbnail.py | 154 +++++++++++++++++++++++++++++++++++++++++ cps/thumbnails.py | 63 +++++++++++++++++ cps/ub.py | 36 +++++++++- 7 files changed, 264 insertions(+), 4 deletions(-) create mode 100644 cps/tasks/thumbnail.py create mode 100644 cps/thumbnails.py diff --git a/.gitignore b/.gitignore index f06dcd44..cef58094 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ vendor/ # calibre-web *.db *.log +cps/cache .idea/ *.bak diff --git a/cps.py b/cps.py index 50ab0076..e90a38d9 100755 --- a/cps.py +++ b/cps.py @@ -43,6 +43,7 @@ from cps.gdrive import gdrive from cps.editbooks import editbook from cps.remotelogin import remotelogin from cps.error_handler import init_errorhandler +from cps.thumbnails import generate_thumbnails try: from cps.kobo import kobo, get_kobo_activated @@ -78,6 +79,9 @@ def main(): app.register_blueprint(kobo_auth) if oauth_available: app.register_blueprint(oauth) + + generate_thumbnails() + success = web_server.start() sys.exit(0 if success else 1) diff --git a/cps/constants.py b/cps/constants.py index c1bcbe59..0a9f9cd5 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -33,6 +33,7 @@ else: STATIC_DIR = os.path.join(BASE_DIR, 'cps', 'static') TEMPLATES_DIR = os.path.join(BASE_DIR, 'cps', 'templates') TRANSLATIONS_DIR = os.path.join(BASE_DIR, 'cps', 'translations') +CACHE_DIR = os.path.join(BASE_DIR, 'cps', 'cache') if HOME_CONFIG: home_dir = os.path.join(os.path.expanduser("~"),".calibre-web") diff --git a/cps/helper.py b/cps/helper.py index da5ea2b3..6fc6b02a 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -551,6 +551,11 @@ def get_book_cover_with_uuid(book_uuid, def get_book_cover_internal(book, use_generic_cover_on_failure): if book and book.has_cover: + # if thumbnails.cover_thumbnail_exists_for_book(book): + # thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first() + # return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename) + # else: + # WorkerThread.add(None, TaskThumbnail(book, _(u'Generating cover thumbnail for: ' + book.title))) if config.config_use_google_drive: try: if not gd.is_gdrive_ready(): @@ -561,8 +566,8 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): else: log.error('%s/cover.jpg not found on Google Drive', book.path) return get_cover_on_failure(use_generic_cover_on_failure) - except Exception as e: - log.debug_or_exception(e) + except Exception as ex: + log.debug_or_exception(ex) return get_cover_on_failure(use_generic_cover_on_failure) else: cover_file_path = os.path.join(config.config_calibre_dir, book.path) diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py new file mode 100644 index 00000000..c452ab41 --- /dev/null +++ b/cps/tasks/thumbnail.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import os + +from cps import config, db, gdriveutils, logger, ub +from cps.constants import CACHE_DIR as _CACHE_DIR +from cps.services.worker import CalibreTask +from datetime import datetime, timedelta +from sqlalchemy import func + +try: + from wand.image import Image + use_IM = True +except (ImportError, RuntimeError) as e: + use_IM = False + +THUMBNAIL_RESOLUTION_1X = 1.0 +THUMBNAIL_RESOLUTION_2X = 2.0 + + +class TaskThumbnail(CalibreTask): + def __init__(self, limit=100, task_message=u'Generating cover thumbnails'): + super(TaskThumbnail, self).__init__(task_message) + self.limit = limit + self.log = logger.create() + self.app_db_session = ub.get_new_session_instance() + self.worker_db = db.CalibreDB(expire_on_commit=False) + + def run(self, worker_thread): + if self.worker_db.session and use_IM: + thumbnails = self.get_thumbnail_book_ids() + thumbnail_book_ids = list(map(lambda t: t.book_id, thumbnails)) + self.log.info(','.join([str(elem) for elem in thumbnail_book_ids])) + self.log.info(len(thumbnail_book_ids)) + + books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids) + + count = len(books_without_thumbnails) + for i, book in enumerate(books_without_thumbnails): + thumbnails = self.get_thumbnails_for_book(thumbnails, book) + if thumbnails: + for thumbnail in thumbnails: + self.update_book_thumbnail(book, thumbnail) + + else: + self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_1X) + self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_2X) + + self.progress = (1.0 / count) * i + + self._handleSuccess() + self.app_db_session.close() + + def get_thumbnail_book_ids(self): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .group_by(ub.Thumbnail.book_id)\ + .having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\ + .all() + + def get_books_without_thumbnails(self, thumbnail_book_ids): + return self.worker_db.session\ + .query(db.Books)\ + .filter(db.Books.has_cover == 1)\ + .filter(db.Books.id.notin_(thumbnail_book_ids))\ + .limit(self.limit)\ + .all() + + def get_thumbnails_for_book(self, thumbnails, book): + results = list() + for thumbnail in thumbnails: + if thumbnail.book_id == book.id: + results.append(thumbnail) + + return results + + def update_book_thumbnail(self, book, thumbnail): + thumbnail.expiration = datetime.utcnow() + timedelta(days=30) + + try: + self.app_db_session.commit() + self.generate_book_thumbnail(book, thumbnail) + except Exception as ex: + self._handleError(u'Error updating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def create_book_thumbnail(self, book, resolution): + thumbnail = ub.Thumbnail() + thumbnail.book_id = book.id + thumbnail.resolution = resolution + + self.app_db_session.add(thumbnail) + try: + self.app_db_session.commit() + self.generate_book_thumbnail(book, thumbnail) + except Exception as ex: + self._handleError(u'Error creating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def generate_book_thumbnail(self, book, thumbnail): + if book and thumbnail: + if config.config_use_google_drive: + self.log.info('google drive thumbnail') + else: + book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + if os.path.isfile(book_cover_filepath): + with Image(filename=book_cover_filepath) as img: + height = self.get_thumbnail_height(thumbnail) + if img.height > height: + width = self.get_thumbnail_width(height, img) + img.resize(width=width, height=height, filter='lanczos') + img.save(filename=self.get_thumbnail_cache_path(thumbnail)) + + def get_thumbnail_height(self, thumbnail): + return int(225 * thumbnail.resolution) + + def get_thumbnail_width(self, height, img): + percent = (height / float(img.height)) + return int((float(img.width) * float(percent))) + + def get_thumbnail_cache_dir(self): + if not os.path.isdir(_CACHE_DIR): + os.makedirs(_CACHE_DIR) + + if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')): + os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails')) + + return os.path.join(_CACHE_DIR, 'thumbnails') + + def get_thumbnail_cache_path(self, thumbnail): + if thumbnail: + return os.path.join(self.get_thumbnail_cache_dir(), thumbnail.filename) + return None + + @property + def name(self): + return "Thumbnail" diff --git a/cps/thumbnails.py b/cps/thumbnails.py new file mode 100644 index 00000000..6ccff56f --- /dev/null +++ b/cps/thumbnails.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import os + +from . import logger, ub +from .constants import CACHE_DIR as _CACHE_DIR +from .services.worker import WorkerThread +from .tasks.thumbnail import TaskThumbnail + +from datetime import datetime + +THUMBNAIL_RESOLUTION_1X = 1.0 +THUMBNAIL_RESOLUTION_2X = 2.0 + +log = logger.create() + + +def get_thumbnail_cache_dir(): + if not os.path.isdir(_CACHE_DIR): + os.makedirs(_CACHE_DIR) + + if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')): + os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails')) + + return os.path.join(_CACHE_DIR, 'thumbnails') + + +def get_thumbnail_cache_path(thumbnail): + if thumbnail: + return os.path.join(get_thumbnail_cache_dir(), thumbnail.filename) + + return None + + +def cover_thumbnail_exists_for_book(book): + if book and book.has_cover: + thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first() + if thumbnail and thumbnail.expiration > datetime.utcnow(): + thumbnail_path = get_thumbnail_cache_path(thumbnail) + return thumbnail_path and os.path.isfile(thumbnail_path) + + return False + + +def generate_thumbnails(): + WorkerThread.add(None, TaskThumbnail()) diff --git a/cps/ub.py b/cps/ub.py index dbc3b419..4500160f 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -40,13 +40,14 @@ except ImportError: oauth_support = False from sqlalchemy import create_engine, exc, exists, event from sqlalchemy import Column, ForeignKey -from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON +from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON, Numeric from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.orm import backref, relationship, sessionmaker, Session, scoped_session from werkzeug.security import generate_password_hash -from . import constants +from . import cli, constants session = None @@ -434,6 +435,28 @@ class RemoteAuthToken(Base): return '' % self.id +class Thumbnail(Base): + __tablename__ = 'thumbnail' + + id = Column(Integer, primary_key=True) + book_id = Column(Integer) + uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True) + format = Column(String, default='jpeg') + resolution = Column(Numeric(precision=2, scale=1, asdecimal=False), default=1.0) + expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30)) + + @hybrid_property + def extension(self): + if self.format == 'jpeg': + return 'jpg' + else: + return self.format + + @hybrid_property + def filename(self): + return self.uuid + '.' + self.extension + + # Migrate database to current version, has to be updated after every database change. Currently migration from # everywhere to current should work. Migration is done by checking if relevant columns are existing, and than adding # rows with SQL commands @@ -451,6 +474,8 @@ def migrate_Database(session): KoboStatistics.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "archived_book"): ArchivedBook.__table__.create(bind=engine) + if not engine.dialect.has_table(engine.connect(), "thumbnail"): + Thumbnail.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "registration"): ReadBook.__table__.create(bind=engine) with engine.connect() as conn: @@ -676,6 +701,13 @@ def init_db(app_db_path): create_anonymous_user(session) +def get_new_session_instance(): + new_engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False) + new_session = scoped_session(sessionmaker()) + new_session.configure(bind=new_engine) + return new_session + + def dispose(): global session From 21fce9a5b5b004c1fcf16aa74e96d27f7e4bde0f Mon Sep 17 00:00:00 2001 From: mmonkey Date: Sat, 19 Dec 2020 02:58:40 -0600 Subject: [PATCH 002/112] Added background scheduler and scheduled thumbnail generation job --- cps.py | 4 --- cps/__init__.py | 9 ++++- cps/services/background_scheduler.py | 52 ++++++++++++++++++++++++++++ cps/tasks/thumbnail.py | 12 +++---- cps/thumbnails.py | 4 --- requirements.txt | 1 + 6 files changed, 66 insertions(+), 16 deletions(-) create mode 100644 cps/services/background_scheduler.py diff --git a/cps.py b/cps.py index e90a38d9..50ab0076 100755 --- a/cps.py +++ b/cps.py @@ -43,7 +43,6 @@ from cps.gdrive import gdrive from cps.editbooks import editbook from cps.remotelogin import remotelogin from cps.error_handler import init_errorhandler -from cps.thumbnails import generate_thumbnails try: from cps.kobo import kobo, get_kobo_activated @@ -79,9 +78,6 @@ def main(): app.register_blueprint(kobo_auth) if oauth_available: app.register_blueprint(oauth) - - generate_thumbnails() - success = web_server.start() sys.exit(0 if success else 1) diff --git a/cps/__init__.py b/cps/__init__.py index 1a7dc868..fa85e15c 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -36,6 +36,8 @@ from flask_principal import Principal from . import config_sql, logger, cache_buster, cli, ub, db from .reverseproxy import ReverseProxied from .server import WebServer +from .services.background_scheduler import BackgroundScheduler +from .tasks.thumbnail import TaskThumbnail mimetypes.init() @@ -95,7 +97,7 @@ def create_app(): app.instance_path = app.instance_path.decode('utf-8') if os.environ.get('FLASK_DEBUG'): - cache_buster.init_cache_busting(app) + cache_buster.init_cache_busting(app) log.info('Starting Calibre Web...') Principal(app) @@ -115,8 +117,13 @@ def create_app(): config.config_goodreads_api_secret, config.config_use_goodreads) + scheduler = BackgroundScheduler() + # Generate 100 book cover thumbnails every 5 minutes + scheduler.add_task(user=None, task=lambda: TaskThumbnail(config=config, limit=100), trigger='interval', minutes=5) + return app + @babel.localeselector def get_locale(): # if a user is logged in, use the locale from the user settings diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py new file mode 100644 index 00000000..efa57379 --- /dev/null +++ b/cps/services/background_scheduler.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import atexit + +from .. import logger +from .worker import WorkerThread +from apscheduler.schedulers.background import BackgroundScheduler as BScheduler + + +class BackgroundScheduler: + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super(BackgroundScheduler, cls).__new__(cls) + + scheduler = BScheduler() + atexit.register(lambda: scheduler.shutdown()) + + cls.log = logger.create() + cls.scheduler = scheduler + cls.scheduler.start() + + return cls._instance + + def add(self, func, trigger, **trigger_args): + self.scheduler.add_job(func=func, trigger=trigger, **trigger_args) + + def add_task(self, user, task, trigger, **trigger_args): + def scheduled_task(): + worker_task = task() + self.log.info('Running scheduled task in background: ' + worker_task.name + ': ' + worker_task.message) + WorkerThread.add(user, worker_task) + + self.add(func=scheduled_task, trigger=trigger, **trigger_args) diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index c452ab41..378b688e 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -19,7 +19,7 @@ from __future__ import division, print_function, unicode_literals import os -from cps import config, db, gdriveutils, logger, ub +from cps import db, logger, ub from cps.constants import CACHE_DIR as _CACHE_DIR from cps.services.worker import CalibreTask from datetime import datetime, timedelta @@ -36,8 +36,9 @@ THUMBNAIL_RESOLUTION_2X = 2.0 class TaskThumbnail(CalibreTask): - def __init__(self, limit=100, task_message=u'Generating cover thumbnails'): + def __init__(self, config, limit=100, task_message=u'Generating cover thumbnails'): super(TaskThumbnail, self).__init__(task_message) + self.config = config self.limit = limit self.log = logger.create() self.app_db_session = ub.get_new_session_instance() @@ -47,9 +48,6 @@ class TaskThumbnail(CalibreTask): if self.worker_db.session and use_IM: thumbnails = self.get_thumbnail_book_ids() thumbnail_book_ids = list(map(lambda t: t.book_id, thumbnails)) - self.log.info(','.join([str(elem) for elem in thumbnail_book_ids])) - self.log.info(len(thumbnail_book_ids)) - books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids) count = len(books_without_thumbnails) @@ -116,10 +114,10 @@ class TaskThumbnail(CalibreTask): def generate_book_thumbnail(self, book, thumbnail): if book and thumbnail: - if config.config_use_google_drive: + if self.config.config_use_google_drive: self.log.info('google drive thumbnail') else: - book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + book_cover_filepath = os.path.join(self.config.config_calibre_dir, book.path, 'cover.jpg') if os.path.isfile(book_cover_filepath): with Image(filename=book_cover_filepath) as img: height = self.get_thumbnail_height(thumbnail) diff --git a/cps/thumbnails.py b/cps/thumbnails.py index 6ccff56f..89e68f50 100644 --- a/cps/thumbnails.py +++ b/cps/thumbnails.py @@ -57,7 +57,3 @@ def cover_thumbnail_exists_for_book(book): return thumbnail_path and os.path.isfile(thumbnail_path) return False - - -def generate_thumbnails(): - WorkerThread.add(None, TaskThumbnail()) diff --git a/requirements.txt b/requirements.txt index f154dd5b..9d0a8654 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +APScheduler==3.6.3 Babel>=1.3, <2.9 Flask-Babel>=0.11.1,<2.1.0 Flask-Login>=0.3.2,<0.5.1 From e48bdf9d5a038d169046c0e5352d3c9d074769c6 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Sun, 20 Dec 2020 03:11:21 -0600 Subject: [PATCH 003/112] Display thumbnails on the frontend, generate thumbnails from google drive --- cps.py | 5 ++++ cps/__init__.py | 6 ----- cps/helper.py | 37 ++++++++++++++++++++--------- cps/schedule.py | 34 +++++++++++++++++++++++++++ cps/tasks/thumbnail.py | 44 +++++++++++++++++++++++++---------- cps/templates/author.html | 2 +- cps/templates/book_cover.html | 8 +++++++ cps/templates/book_edit.html | 4 +++- cps/templates/detail.html | 3 ++- cps/templates/discover.html | 3 ++- cps/templates/fragment.html | 1 + cps/templates/grid.html | 3 ++- cps/templates/index.html | 5 ++-- cps/templates/layout.html | 1 + cps/templates/search.html | 3 ++- cps/templates/shelf.html | 3 ++- cps/thumbnails.py | 9 ++----- cps/ub.py | 4 ++-- cps/web.py | 11 ++++----- 19 files changed, 133 insertions(+), 53 deletions(-) create mode 100644 cps/schedule.py create mode 100644 cps/templates/book_cover.html diff --git a/cps.py b/cps.py index 50ab0076..d63771eb 100755 --- a/cps.py +++ b/cps.py @@ -43,6 +43,7 @@ from cps.gdrive import gdrive from cps.editbooks import editbook from cps.remotelogin import remotelogin from cps.error_handler import init_errorhandler +from cps.schedule import register_jobs try: from cps.kobo import kobo, get_kobo_activated @@ -78,6 +79,10 @@ def main(): app.register_blueprint(kobo_auth) if oauth_available: app.register_blueprint(oauth) + + # Register scheduled jobs + register_jobs() + success = web_server.start() sys.exit(0 if success else 1) diff --git a/cps/__init__.py b/cps/__init__.py index fa85e15c..6a6d361a 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -36,8 +36,6 @@ from flask_principal import Principal from . import config_sql, logger, cache_buster, cli, ub, db from .reverseproxy import ReverseProxied from .server import WebServer -from .services.background_scheduler import BackgroundScheduler -from .tasks.thumbnail import TaskThumbnail mimetypes.init() @@ -117,10 +115,6 @@ def create_app(): config.config_goodreads_api_secret, config.config_use_goodreads) - scheduler = BackgroundScheduler() - # Generate 100 book cover thumbnails every 5 minutes - scheduler.add_task(user=None, task=lambda: TaskThumbnail(config=config, limit=100), trigger='interval', minutes=5) - return app diff --git a/cps/helper.py b/cps/helper.py index 6fc6b02a..d3420a11 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -52,7 +52,7 @@ except ImportError: from . import calibre_db from .tasks.convert import TaskConvert -from . import logger, config, get_locale, db, ub +from . import logger, config, get_locale, db, thumbnails, ub from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR from .subproc_wrapper import process_wait @@ -538,24 +538,27 @@ def get_cover_on_failure(use_generic_cover): return None -def get_book_cover(book_id): +def get_book_cover(book_id, resolution=1): book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) - return get_book_cover_internal(book, use_generic_cover_on_failure=True) + return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) -def get_book_cover_with_uuid(book_uuid, - use_generic_cover_on_failure=True): +def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True): book = calibre_db.get_book_by_uuid(book_uuid) return get_book_cover_internal(book, use_generic_cover_on_failure) -def get_book_cover_internal(book, use_generic_cover_on_failure): +def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=1, disable_thumbnail=False): if book and book.has_cover: - # if thumbnails.cover_thumbnail_exists_for_book(book): - # thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first() - # return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename) - # else: - # WorkerThread.add(None, TaskThumbnail(book, _(u'Generating cover thumbnail for: ' + book.title))) + + # Send the book cover thumbnail if it exists in cache + if not disable_thumbnail: + thumbnail = get_book_cover_thumbnail(book, resolution) + if thumbnail: + if os.path.isfile(thumbnails.get_thumbnail_cache_path(thumbnail)): + return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename) + + # Send the book cover from Google Drive if configured if config.config_use_google_drive: try: if not gd.is_gdrive_ready(): @@ -569,6 +572,8 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): except Exception as ex: log.debug_or_exception(ex) return get_cover_on_failure(use_generic_cover_on_failure) + + # Send the book cover from the Calibre directory else: cover_file_path = os.path.join(config.config_calibre_dir, book.path) if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")): @@ -579,6 +584,16 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): return get_cover_on_failure(use_generic_cover_on_failure) +def get_book_cover_thumbnail(book, resolution=1): + if book and book.has_cover: + return ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id == book.id)\ + .filter(ub.Thumbnail.resolution == resolution)\ + .filter(ub.Thumbnail.expiration > datetime.utcnow())\ + .first() + + # saves book cover from url def save_cover_from_url(url, book_path): try: diff --git a/cps/schedule.py b/cps/schedule.py new file mode 100644 index 00000000..5d2c94b9 --- /dev/null +++ b/cps/schedule.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals + +from . import logger +from .services.background_scheduler import BackgroundScheduler +from .tasks.thumbnail import TaskThumbnail + +log = logger.create() + + +def register_jobs(): + scheduler = BackgroundScheduler() + + # Generate 100 book cover thumbnails every 5 minutes + scheduler.add_task(user=None, task=lambda: TaskThumbnail(limit=100), trigger='interval', minutes=5) + + # TODO: validate thumbnail scheduled task diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index 378b688e..4e0c6db4 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -19,11 +19,13 @@ from __future__ import division, print_function, unicode_literals import os -from cps import db, logger, ub +from cps import config, db, gdriveutils, logger, ub from cps.constants import CACHE_DIR as _CACHE_DIR from cps.services.worker import CalibreTask +from cps.thumbnails import THUMBNAIL_RESOLUTION_1X, THUMBNAIL_RESOLUTION_2X from datetime import datetime, timedelta from sqlalchemy import func +from urllib.request import urlopen try: from wand.image import Image @@ -31,14 +33,10 @@ try: except (ImportError, RuntimeError) as e: use_IM = False -THUMBNAIL_RESOLUTION_1X = 1.0 -THUMBNAIL_RESOLUTION_2X = 2.0 - class TaskThumbnail(CalibreTask): - def __init__(self, config, limit=100, task_message=u'Generating cover thumbnails'): + def __init__(self, limit=100, task_message=u'Generating cover thumbnails'): super(TaskThumbnail, self).__init__(task_message) - self.config = config self.limit = limit self.log = logger.create() self.app_db_session = ub.get_new_session_instance() @@ -114,17 +112,39 @@ class TaskThumbnail(CalibreTask): def generate_book_thumbnail(self, book, thumbnail): if book and thumbnail: - if self.config.config_use_google_drive: - self.log.info('google drive thumbnail') - else: - book_cover_filepath = os.path.join(self.config.config_calibre_dir, book.path, 'cover.jpg') - if os.path.isfile(book_cover_filepath): - with Image(filename=book_cover_filepath) as img: + if config.config_use_google_drive: + if not gdriveutils.is_gdrive_ready(): + raise Exception('Google Drive is configured but not ready') + + web_content_link = gdriveutils.get_cover_via_gdrive(book.path) + if not web_content_link: + raise Exception('Google Drive cover url not found') + + stream = None + try: + stream = urlopen(web_content_link) + with Image(file=stream) as img: height = self.get_thumbnail_height(thumbnail) if img.height > height: width = self.get_thumbnail_width(height, img) img.resize(width=width, height=height, filter='lanczos') img.save(filename=self.get_thumbnail_cache_path(thumbnail)) + except Exception as ex: + # Bubble exception to calling function + raise ex + finally: + stream.close() + else: + book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + if not os.path.isfile(book_cover_filepath): + raise Exception('Book cover file not found') + + with Image(filename=book_cover_filepath) as img: + height = self.get_thumbnail_height(thumbnail) + if img.height > height: + width = self.get_thumbnail_width(height, img) + img.resize(width=width, height=height, filter='lanczos') + img.save(filename=self.get_thumbnail_cache_path(thumbnail)) def get_thumbnail_height(self, thumbnail): return int(225 * thumbnail.resolution) diff --git a/cps/templates/author.html b/cps/templates/author.html index 7887aa4a..24ce876a 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -36,7 +36,7 @@
diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html new file mode 100644 index 00000000..878c14a8 --- /dev/null +++ b/cps/templates/book_cover.html @@ -0,0 +1,8 @@ +{% macro book_cover_image(book_id, book_title) -%} + {{ book_title }} +{%- endmacro %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 003b33f9..881fa8ff 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -1,9 +1,11 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %} {% if book %}
- {{ book.title }} + {{ book_cover_image(book.id, book.title) }} +
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index 503d1dbd..d3615563 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,7 +4,8 @@
- {{ entry.title }} + {{ book_cover_image(entry.id, entry.title) }} +
diff --git a/cps/templates/discover.html b/cps/templates/discover.html index 3c858feb..33bafbbe 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -8,7 +9,7 @@
{% if entry.has_cover is defined %} - {{ entry.title }} + {{ book_cover_image(entry.id, entry.title) }} {% endif %}
diff --git a/cps/templates/fragment.html b/cps/templates/fragment.html index 1421ea6a..901dd193 100644 --- a/cps/templates/fragment.html +++ b/cps/templates/fragment.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %}
{% block body %}{% endblock %}
diff --git a/cps/templates/grid.html b/cps/templates/grid.html index ce2c05ac..bc3ca4a2 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}

{{_(title)}}

@@ -28,7 +29,7 @@
diff --git a/cps/templates/index.html b/cps/templates/index.html index 1db73c89..c536884f 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %} {% if g.user.show_detail_random() %} @@ -8,7 +9,7 @@
@@ -82,7 +83,7 @@
diff --git a/cps/templates/layout.html b/cps/templates/layout.html index 3b89a7ce..5df471a9 100644 --- a/cps/templates/layout.html +++ b/cps/templates/layout.html @@ -1,4 +1,5 @@ {% from 'modal_dialogs.html' import restrict_modal, delete_book, filechooser_modal %} +{% from 'book_cover.html' import book_cover_image %} diff --git a/cps/templates/search.html b/cps/templates/search.html index aedb6f45..56b12154 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -43,7 +44,7 @@
{% if entry.has_cover is defined %} - {{ entry.title }} + {{ book_cover_image(entry.id, entry.title) }} {% endif %}
diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index f7e3c1ae..7a678ea6 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -30,7 +31,7 @@
diff --git a/cps/thumbnails.py b/cps/thumbnails.py index 89e68f50..ea7aac86 100644 --- a/cps/thumbnails.py +++ b/cps/thumbnails.py @@ -21,13 +21,11 @@ import os from . import logger, ub from .constants import CACHE_DIR as _CACHE_DIR -from .services.worker import WorkerThread -from .tasks.thumbnail import TaskThumbnail from datetime import datetime -THUMBNAIL_RESOLUTION_1X = 1.0 -THUMBNAIL_RESOLUTION_2X = 2.0 +THUMBNAIL_RESOLUTION_1X = 1 +THUMBNAIL_RESOLUTION_2X = 2 log = logger.create() @@ -35,17 +33,14 @@ log = logger.create() def get_thumbnail_cache_dir(): if not os.path.isdir(_CACHE_DIR): os.makedirs(_CACHE_DIR) - if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')): os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails')) - return os.path.join(_CACHE_DIR, 'thumbnails') def get_thumbnail_cache_path(thumbnail): if thumbnail: return os.path.join(get_thumbnail_cache_dir(), thumbnail.filename) - return None diff --git a/cps/ub.py b/cps/ub.py index 4500160f..0b5a65e7 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -40,7 +40,7 @@ except ImportError: oauth_support = False from sqlalchemy import create_engine, exc, exists, event from sqlalchemy import Column, ForeignKey -from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON, Numeric +from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm.attributes import flag_modified @@ -442,7 +442,7 @@ class Thumbnail(Base): book_id = Column(Integer) uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True) format = Column(String, default='jpeg') - resolution = Column(Numeric(precision=2, scale=1, asdecimal=False), default=1.0) + resolution = Column(SmallInteger, default=1) expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30)) @hybrid_property diff --git a/cps/web.py b/cps/web.py index 4baf82cb..27a5849b 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1171,14 +1171,17 @@ def advanced_search_form(): @web.route("/cover/") +@web.route("/cover//") @login_required_if_no_ano -def get_cover(book_id): - return get_book_cover(book_id) +def get_cover(book_id, resolution=1): + return get_book_cover(book_id, resolution) + @web.route("/robots.txt") def get_robots(): return send_from_directory(constants.STATIC_DIR, "robots.txt") + @web.route("/show//", defaults={'anyname': 'None'}) @web.route("/show///") @login_required_if_no_ano @@ -1205,7 +1208,6 @@ def serve_book(book_id, book_format, anyname): return send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + book_format) - @web.route("/download//", defaults={'anyname': 'None'}) @web.route("/download///") @login_required_if_no_ano @@ -1387,9 +1389,6 @@ def logout(): return redirect(url_for('web.login')) - - - # ################################### Users own configuration ######################################################### From 541fc7e14ebe3716fce58593a2ff3f5b23403e3d Mon Sep 17 00:00:00 2001 From: mmonkey Date: Tue, 22 Dec 2020 17:49:21 -0600 Subject: [PATCH 004/112] fixed thumbnail generate tasks, added thumbnail cleanup task, added reconnect db scheduled job --- cps/fs.py | 61 ++++++++++++++ cps/helper.py | 7 +- cps/schedule.py | 18 ++++- cps/services/worker.py | 17 +++- cps/tasks/thumbnail.py | 177 +++++++++++++++++++++++++++++++---------- cps/thumbnails.py | 54 ------------- cps/ub.py | 28 ++++--- 7 files changed, 245 insertions(+), 117 deletions(-) create mode 100644 cps/fs.py delete mode 100644 cps/thumbnails.py diff --git a/cps/fs.py b/cps/fs.py new file mode 100644 index 00000000..699d5991 --- /dev/null +++ b/cps/fs.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +from .constants import CACHE_DIR +from os import listdir, makedirs, remove +from os.path import isdir, isfile, join +from shutil import rmtree + +CACHE_TYPE_THUMBNAILS = 'thumbnails' + + +class FileSystem: + _instance = None + _cache_dir = CACHE_DIR + + def __new__(cls): + if cls._instance is None: + cls._instance = super(FileSystem, cls).__new__(cls) + return cls._instance + + def get_cache_dir(self, cache_type=None): + if not isdir(self._cache_dir): + makedirs(self._cache_dir) + + if cache_type and not isdir(join(self._cache_dir, cache_type)): + makedirs(join(self._cache_dir, cache_type)) + + return join(self._cache_dir, cache_type) if cache_type else self._cache_dir + + def get_cache_file_path(self, filename, cache_type=None): + return join(self.get_cache_dir(cache_type), filename) if filename else None + + def list_cache_files(self, cache_type=None): + path = self.get_cache_dir(cache_type) + return [file for file in listdir(path) if isfile(join(path, file))] + + def delete_cache_dir(self, cache_type=None): + if not cache_type and isdir(self._cache_dir): + rmtree(self._cache_dir) + if cache_type and isdir(join(self._cache_dir, cache_type)): + rmtree(join(self._cache_dir, cache_type)) + + def delete_cache_file(self, filename, cache_type=None): + if isfile(join(self.get_cache_dir(cache_type), filename)): + remove(join(self.get_cache_dir(cache_type), filename)) diff --git a/cps/helper.py b/cps/helper.py index d3420a11..271ab3e9 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -52,7 +52,7 @@ except ImportError: from . import calibre_db from .tasks.convert import TaskConvert -from . import logger, config, get_locale, db, thumbnails, ub +from . import logger, config, get_locale, db, fs, ub from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR from .subproc_wrapper import process_wait @@ -555,8 +555,9 @@ def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=1, di if not disable_thumbnail: thumbnail = get_book_cover_thumbnail(book, resolution) if thumbnail: - if os.path.isfile(thumbnails.get_thumbnail_cache_path(thumbnail)): - return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename) + cache = fs.FileSystem() + if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename) # Send the book cover from Google Drive if configured if config.config_use_google_drive: diff --git a/cps/schedule.py b/cps/schedule.py index 5d2c94b9..5c658e41 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -18,9 +18,9 @@ from __future__ import division, print_function, unicode_literals -from . import logger +from . import config, db, logger, ub from .services.background_scheduler import BackgroundScheduler -from .tasks.thumbnail import TaskThumbnail +from .tasks.thumbnail import TaskCleanupCoverThumbnailCache, TaskGenerateCoverThumbnails log = logger.create() @@ -29,6 +29,16 @@ def register_jobs(): scheduler = BackgroundScheduler() # Generate 100 book cover thumbnails every 5 minutes - scheduler.add_task(user=None, task=lambda: TaskThumbnail(limit=100), trigger='interval', minutes=5) + scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='interval', minutes=5) - # TODO: validate thumbnail scheduled task + # Cleanup book cover cache every day at 4am + scheduler.add_task(user=None, task=lambda: TaskCleanupCoverThumbnailCache(), trigger='cron', hour=4) + + # Reconnect metadata.db every 4 hours + scheduler.add(func=reconnect_db_job, trigger='interval', hours=4) + + +def reconnect_db_job(): + log.info('Running background task: reconnect to calibre database') + calibre_db = db.CalibreDB() + calibre_db.reconnect_db(config, ub.app_DB_path) diff --git a/cps/services/worker.py b/cps/services/worker.py index 072674a0..2b6816db 100644 --- a/cps/services/worker.py +++ b/cps/services/worker.py @@ -35,7 +35,6 @@ def _get_main_thread(): raise Exception("main thread not found?!") - class ImprovedQueue(queue.Queue): def to_list(self): """ @@ -45,7 +44,8 @@ class ImprovedQueue(queue.Queue): with self.mutex: return list(self.queue) -#Class for all worker tasks in the background + +# Class for all worker tasks in the background class WorkerThread(threading.Thread): _instance = None @@ -127,6 +127,10 @@ class WorkerThread(threading.Thread): # CalibreTask.start() should wrap all exceptions in it's own error handling item.task.start(self) + # remove self_cleanup tasks from list + if item.task.self_cleanup: + self.dequeued.remove(item) + self.queue.task_done() @@ -141,6 +145,7 @@ class CalibreTask: self.end_time = None self.message = message self.id = uuid.uuid4() + self.self_cleanup = False @abc.abstractmethod def run(self, worker_thread): @@ -209,6 +214,14 @@ class CalibreTask: # todo: throw error if outside of [0,1] self._progress = x + @property + def self_cleanup(self): + return self._self_cleanup + + @self_cleanup.setter + def self_cleanup(self, is_self_cleanup): + self._self_cleanup = is_self_cleanup + def _handleError(self, error_message): self.stat = STAT_FAIL self.progress = 1 diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index 4e0c6db4..f61eb4a7 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -19,13 +19,15 @@ from __future__ import division, print_function, unicode_literals import os -from cps import config, db, gdriveutils, logger, ub -from cps.constants import CACHE_DIR as _CACHE_DIR +from cps import config, db, fs, gdriveutils, logger, ub from cps.services.worker import CalibreTask -from cps.thumbnails import THUMBNAIL_RESOLUTION_1X, THUMBNAIL_RESOLUTION_2X from datetime import datetime, timedelta from sqlalchemy import func -from urllib.request import urlopen + +try: + from urllib.request import urlopen +except ImportError as e: + from urllib2 import urlopen try: from wand.image import Image @@ -33,73 +35,92 @@ try: except (ImportError, RuntimeError) as e: use_IM = False +THUMBNAIL_RESOLUTION_1X = 1 +THUMBNAIL_RESOLUTION_2X = 2 -class TaskThumbnail(CalibreTask): + +class TaskGenerateCoverThumbnails(CalibreTask): def __init__(self, limit=100, task_message=u'Generating cover thumbnails'): - super(TaskThumbnail, self).__init__(task_message) + super(TaskGenerateCoverThumbnails, self).__init__(task_message) + self.self_cleanup = True self.limit = limit self.log = logger.create() self.app_db_session = ub.get_new_session_instance() - self.worker_db = db.CalibreDB(expire_on_commit=False) + self.calibre_db = db.CalibreDB(expire_on_commit=False) + self.cache = fs.FileSystem() + self.resolutions = [ + THUMBNAIL_RESOLUTION_1X, + THUMBNAIL_RESOLUTION_2X + ] def run(self, worker_thread): - if self.worker_db.session and use_IM: - thumbnails = self.get_thumbnail_book_ids() - thumbnail_book_ids = list(map(lambda t: t.book_id, thumbnails)) + if self.calibre_db.session and use_IM: + expired_thumbnails = self.get_expired_thumbnails() + thumbnail_book_ids = self.get_thumbnail_book_ids() books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids) count = len(books_without_thumbnails) for i, book in enumerate(books_without_thumbnails): - thumbnails = self.get_thumbnails_for_book(thumbnails, book) - if thumbnails: - for thumbnail in thumbnails: - self.update_book_thumbnail(book, thumbnail) - - else: - self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_1X) - self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_2X) + for resolution in self.resolutions: + expired_thumbnail = self.get_expired_thumbnail_for_book_and_resolution( + book, + resolution, + expired_thumbnails + ) + if expired_thumbnail: + self.update_book_thumbnail(book, expired_thumbnail) + else: + self.create_book_thumbnail(book, resolution) self.progress = (1.0 / count) * i self._handleSuccess() - self.app_db_session.close() + self.app_db_session.remove() + + def get_expired_thumbnails(self): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.expiration < datetime.utcnow())\ + .all() def get_thumbnail_book_ids(self): return self.app_db_session\ - .query(ub.Thumbnail)\ + .query(ub.Thumbnail.book_id)\ .group_by(ub.Thumbnail.book_id)\ .having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\ - .all() + .distinct() def get_books_without_thumbnails(self, thumbnail_book_ids): - return self.worker_db.session\ + return self.calibre_db.session\ .query(db.Books)\ .filter(db.Books.has_cover == 1)\ .filter(db.Books.id.notin_(thumbnail_book_ids))\ .limit(self.limit)\ .all() - def get_thumbnails_for_book(self, thumbnails, book): - results = list() - for thumbnail in thumbnails: - if thumbnail.book_id == book.id: - results.append(thumbnail) + def get_expired_thumbnail_for_book_and_resolution(self, book, resolution, expired_thumbnails): + for thumbnail in expired_thumbnails: + if thumbnail.book_id == book.id and thumbnail.resolution == resolution: + return thumbnail - return results + return None def update_book_thumbnail(self, book, thumbnail): + thumbnail.generated_at = datetime.utcnow() thumbnail.expiration = datetime.utcnow() + timedelta(days=30) try: self.app_db_session.commit() self.generate_book_thumbnail(book, thumbnail) except Exception as ex: + self.log.info(u'Error updating book thumbnail: ' + str(ex)) self._handleError(u'Error updating book thumbnail: ' + str(ex)) self.app_db_session.rollback() def create_book_thumbnail(self, book, resolution): thumbnail = ub.Thumbnail() thumbnail.book_id = book.id + thumbnail.format = 'jpeg' thumbnail.resolution = resolution self.app_db_session.add(thumbnail) @@ -107,6 +128,7 @@ class TaskThumbnail(CalibreTask): self.app_db_session.commit() self.generate_book_thumbnail(book, thumbnail) except Exception as ex: + self.log.info(u'Error creating book thumbnail: ' + str(ex)) self._handleError(u'Error creating book thumbnail: ' + str(ex)) self.app_db_session.rollback() @@ -128,9 +150,12 @@ class TaskThumbnail(CalibreTask): if img.height > height: width = self.get_thumbnail_width(height, img) img.resize(width=width, height=height, filter='lanczos') - img.save(filename=self.get_thumbnail_cache_path(thumbnail)) + img.format = thumbnail.format + filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + img.save(filename=filename) except Exception as ex: # Bubble exception to calling function + self.log.info(u'Error generating thumbnail file: ' + str(ex)) raise ex finally: stream.close() @@ -144,7 +169,9 @@ class TaskThumbnail(CalibreTask): if img.height > height: width = self.get_thumbnail_width(height, img) img.resize(width=width, height=height, filter='lanczos') - img.save(filename=self.get_thumbnail_cache_path(thumbnail)) + img.format = thumbnail.format + filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + img.save(filename=filename) def get_thumbnail_height(self, thumbnail): return int(225 * thumbnail.resolution) @@ -153,20 +180,88 @@ class TaskThumbnail(CalibreTask): percent = (height / float(img.height)) return int((float(img.width) * float(percent))) - def get_thumbnail_cache_dir(self): - if not os.path.isdir(_CACHE_DIR): - os.makedirs(_CACHE_DIR) + @property + def name(self): + return "GenerateCoverThumbnails" - if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')): - os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails')) - return os.path.join(_CACHE_DIR, 'thumbnails') +class TaskCleanupCoverThumbnailCache(CalibreTask): + def __init__(self, task_message=u'Validating cover thumbnail cache'): + super(TaskCleanupCoverThumbnailCache, self).__init__(task_message) + self.log = logger.create() + self.app_db_session = ub.get_new_session_instance() + self.calibre_db = db.CalibreDB(expire_on_commit=False) + self.cache = fs.FileSystem() - def get_thumbnail_cache_path(self, thumbnail): - if thumbnail: - return os.path.join(self.get_thumbnail_cache_dir(), thumbnail.filename) - return None + def run(self, worker_thread): + cached_thumbnail_files = self.cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS) + + # Expire thumbnails in the database if the cached file is missing + # This case will happen if a user deletes the cache dir or cached files + if self.app_db_session: + self.expire_missing_thumbnails(cached_thumbnail_files) + self.progress = 0.33 + + # Delete thumbnails in the database if the book has been removed + # This case will happen if a book is removed in Calibre and the metadata.db file is updated in the filesystem + if self.app_db_session and self.calibre_db: + book_ids = self.get_book_ids() + self.delete_thumbnails_for_missing_books(book_ids) + self.progress = 0.66 + + # Delete extraneous cached thumbnail files + # This case will happen if a book was deleted and the thumbnail OR the metadata.db file was changed externally + if self.app_db_session: + db_thumbnail_files = self.get_thumbnail_filenames() + self.delete_extraneous_thumbnail_files(cached_thumbnail_files, db_thumbnail_files) + + self._handleSuccess() + self.app_db_session.remove() + + def expire_missing_thumbnails(self, filenames): + try: + self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.filename.notin_(filenames))\ + .update({"expiration": datetime.utcnow()}, synchronize_session=False) + self.app_db_session.commit() + except Exception as ex: + self.log.info(u'Error expiring thumbnails for missing cache files: ' + str(ex)) + self._handleError(u'Error expiring thumbnails for missing cache files: ' + str(ex)) + self.app_db_session.rollback() + + def get_book_ids(self): + results = self.calibre_db.session\ + .query(db.Books.id)\ + .filter(db.Books.has_cover == 1)\ + .distinct() + + return [value for value, in results] + + def delete_thumbnails_for_missing_books(self, book_ids): + try: + self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.notin_(book_ids))\ + .delete(synchronize_session=False) + self.app_db_session.commit() + except Exception as ex: + self.log.info(str(ex)) + self._handleError(u'Error deleting thumbnails for missing books: ' + str(ex)) + self.app_db_session.rollback() + + def get_thumbnail_filenames(self): + results = self.app_db_session\ + .query(ub.Thumbnail.filename)\ + .all() + + return [thumbnail for thumbnail, in results] + + def delete_extraneous_thumbnail_files(self, cached_thumbnail_files, db_thumbnail_files): + extraneous_files = list(set(cached_thumbnail_files).difference(db_thumbnail_files)) + for file in extraneous_files: + self.cache.delete_cache_file(file, fs.CACHE_TYPE_THUMBNAILS) @property def name(self): - return "Thumbnail" + return "CleanupCoverThumbnailCache" diff --git a/cps/thumbnails.py b/cps/thumbnails.py deleted file mode 100644 index ea7aac86..00000000 --- a/cps/thumbnails.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- - -# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) -# Copyright (C) 2020 mmonkey -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from __future__ import division, print_function, unicode_literals -import os - -from . import logger, ub -from .constants import CACHE_DIR as _CACHE_DIR - -from datetime import datetime - -THUMBNAIL_RESOLUTION_1X = 1 -THUMBNAIL_RESOLUTION_2X = 2 - -log = logger.create() - - -def get_thumbnail_cache_dir(): - if not os.path.isdir(_CACHE_DIR): - os.makedirs(_CACHE_DIR) - if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')): - os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails')) - return os.path.join(_CACHE_DIR, 'thumbnails') - - -def get_thumbnail_cache_path(thumbnail): - if thumbnail: - return os.path.join(get_thumbnail_cache_dir(), thumbnail.filename) - return None - - -def cover_thumbnail_exists_for_book(book): - if book and book.has_cover: - thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first() - if thumbnail and thumbnail.expiration > datetime.utcnow(): - thumbnail_path = get_thumbnail_cache_path(thumbnail) - return thumbnail_path and os.path.isfile(thumbnail_path) - - return False diff --git a/cps/ub.py b/cps/ub.py index 0b5a65e7..30abd728 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -18,6 +18,7 @@ # along with this program. If not, see . from __future__ import division, print_function, unicode_literals +import atexit import os import sys import datetime @@ -42,12 +43,11 @@ from sqlalchemy import create_engine, exc, exists, event from sqlalchemy import Column, ForeignKey from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.orm import backref, relationship, sessionmaker, Session, scoped_session from werkzeug.security import generate_password_hash -from . import cli, constants +from . import cli, constants, logger session = None @@ -435,6 +435,14 @@ class RemoteAuthToken(Base): return '' % self.id +def filename(context): + file_format = context.get_current_parameters()['format'] + if file_format == 'jpeg': + return context.get_current_parameters()['uuid'] + '.jpg' + else: + return context.get_current_parameters()['uuid'] + '.' + file_format + + class Thumbnail(Base): __tablename__ = 'thumbnail' @@ -443,19 +451,10 @@ class Thumbnail(Base): uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True) format = Column(String, default='jpeg') resolution = Column(SmallInteger, default=1) + filename = Column(String, default=filename) + generated_at = Column(DateTime, default=lambda: datetime.datetime.utcnow()) expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30)) - @hybrid_property - def extension(self): - if self.format == 'jpeg': - return 'jpg' - else: - return self.format - - @hybrid_property - def filename(self): - return self.uuid + '.' + self.extension - # Migrate database to current version, has to be updated after every database change. Currently migration from # everywhere to current should work. Migration is done by checking if relevant columns are existing, and than adding @@ -705,6 +704,9 @@ def get_new_session_instance(): new_engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False) new_session = scoped_session(sessionmaker()) new_session.configure(bind=new_engine) + + atexit.register(lambda: new_session.remove() if new_session else True) + return new_session From 626051e4892ef42d65e16f01932ae8ce48342c22 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Wed, 23 Dec 2020 03:25:25 -0600 Subject: [PATCH 005/112] Added clear cache button to admin settings, updated cache busting for book cover images --- cps/admin.py | 19 +++++++- cps/editbooks.py | 2 + cps/helper.py | 23 +++++++--- cps/jinjia.py | 6 +++ cps/schedule.py | 12 +---- cps/static/css/caliBlur.css | 82 ++++++++++++++++++++++++++++------- cps/static/js/main.js | 12 +++++ cps/tasks/database.py | 49 +++++++++++++++++++++ cps/tasks/thumbnail.py | 58 ++++++++++++++++++++++++- cps/templates/admin.html | 36 +++++++++++---- cps/templates/author.html | 2 +- cps/templates/book_cover.html | 8 ++-- cps/templates/book_edit.html | 3 +- cps/templates/detail.html | 3 +- cps/templates/discover.html | 2 +- cps/templates/grid.html | 2 +- cps/templates/index.html | 4 +- cps/templates/search.html | 2 +- cps/templates/shelf.html | 2 +- cps/web.py | 8 +++- 20 files changed, 278 insertions(+), 57 deletions(-) create mode 100644 cps/tasks/database.py diff --git a/cps/admin.py b/cps/admin.py index 8b3ca247..dbc1b708 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -38,7 +38,7 @@ from sqlalchemy import and_ from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError from sqlalchemy.sql.expression import func, or_ -from . import constants, logger, helper, services +from . import constants, logger, helper, services, fs from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash from .gdriveutils import is_gdrive_ready, gdrive_support @@ -157,6 +157,23 @@ def shutdown(): return json.dumps(showtext), 400 +@admi.route("/clear-cache") +@login_required +@admin_required +def clear_cache(): + cache_type = request.args.get('cache_type'.strip()) + showtext = {} + + if cache_type == fs.CACHE_TYPE_THUMBNAILS: + log.info('clearing cover thumbnail cache') + showtext['text'] = _(u'Cleared cover thumbnail cache') + helper.clear_cover_thumbnail_cache() + return json.dumps(showtext) + + showtext['text'] = _(u'Unknown command') + return json.dumps(showtext) + + @admi.route("/admin/view") @login_required @admin_required diff --git a/cps/editbooks.py b/cps/editbooks.py index 08ee93b1..6d26ebca 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -595,6 +595,7 @@ def upload_cover(request, book): abort(403) ret, message = helper.save_cover(requested_file, book.path) if ret is True: + helper.clear_cover_thumbnail_cache(book.id) return True else: flash(message, category="error") @@ -684,6 +685,7 @@ def edit_book(book_id): if result is True: book.has_cover = 1 modif_date = True + helper.clear_cover_thumbnail_cache(book.id) else: flash(error, category="error") diff --git a/cps/helper.py b/cps/helper.py index 271ab3e9..0b0c675f 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -58,6 +58,7 @@ from .constants import STATIC_DIR as _STATIC_DIR from .subproc_wrapper import process_wait from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS from .tasks.mail import TaskEmail +from .tasks.thumbnail import TaskClearCoverThumbnailCache log = logger.create() @@ -525,6 +526,7 @@ def update_dir_stucture(book_id, calibrepath, first_author=None, orignal_filepat def delete_book(book, calibrepath, book_format): + clear_cover_thumbnail_cache(book.id) if config.config_use_google_drive: return delete_book_gdrive(book, book_format) else: @@ -538,9 +540,9 @@ def get_cover_on_failure(use_generic_cover): return None -def get_book_cover(book_id, resolution=1): +def get_book_cover(book_id): book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) - return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) + return get_book_cover_internal(book, use_generic_cover_on_failure=True) def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True): @@ -548,11 +550,19 @@ def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True): return get_book_cover_internal(book, use_generic_cover_on_failure) -def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=1, disable_thumbnail=False): +def get_cached_book_cover(cache_id): + parts = cache_id.split('_') + book_uuid = parts[0] if len(parts) else None + resolution = parts[2] if len(parts) > 2 else None + book = calibre_db.get_book_by_uuid(book_uuid) if book_uuid else None + return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) + + +def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None): if book and book.has_cover: # Send the book cover thumbnail if it exists in cache - if not disable_thumbnail: + if resolution: thumbnail = get_book_cover_thumbnail(book, resolution) if thumbnail: cache = fs.FileSystem() @@ -585,7 +595,7 @@ def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=1, di return get_cover_on_failure(use_generic_cover_on_failure) -def get_book_cover_thumbnail(book, resolution=1): +def get_book_cover_thumbnail(book, resolution): if book and book.has_cover: return ub.session\ .query(ub.Thumbnail)\ @@ -846,3 +856,6 @@ def get_download_link(book_id, book_format, client): else: abort(404) + +def clear_cover_thumbnail_cache(book_id=None): + WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id)) diff --git a/cps/jinjia.py b/cps/jinjia.py index 688d1fba..bf81c059 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -128,8 +128,14 @@ def formatseriesindex_filter(series_index): return series_index return 0 + @jinjia.app_template_filter('uuidfilter') def uuidfilter(var): return uuid4() +@jinjia.app_template_filter('book_cover_cache_id') +def book_cover_cache_id(book, resolution=None): + timestamp = int(book.last_modified.timestamp() * 1000) + cache_bust = str(book.uuid) + '_' + str(timestamp) + return cache_bust if not resolution else cache_bust + '_' + str(resolution) diff --git a/cps/schedule.py b/cps/schedule.py index 5c658e41..f349a231 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -18,12 +18,10 @@ from __future__ import division, print_function, unicode_literals -from . import config, db, logger, ub from .services.background_scheduler import BackgroundScheduler +from .tasks.database import TaskReconnectDatabase from .tasks.thumbnail import TaskCleanupCoverThumbnailCache, TaskGenerateCoverThumbnails -log = logger.create() - def register_jobs(): scheduler = BackgroundScheduler() @@ -35,10 +33,4 @@ def register_jobs(): scheduler.add_task(user=None, task=lambda: TaskCleanupCoverThumbnailCache(), trigger='cron', hour=4) # Reconnect metadata.db every 4 hours - scheduler.add(func=reconnect_db_job, trigger='interval', hours=4) - - -def reconnect_db_job(): - log.info('Running background task: reconnect to calibre database') - calibre_db = db.CalibreDB() - calibre_db.reconnect_db(config, ub.app_DB_path) + scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='interval', hours=4) diff --git a/cps/static/css/caliBlur.css b/cps/static/css/caliBlur.css index d085608d..da5d2933 100644 --- a/cps/static/css/caliBlur.css +++ b/cps/static/css/caliBlur.css @@ -5167,7 +5167,7 @@ body.login > div.navbar.navbar-default.navbar-static-top > div > div.navbar-head pointer-events: none } -#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #deleteButton, #deleteModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover { +#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #ClearCacheDialog:hover:before, #deleteButton, #deleteModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover { cursor: pointer } @@ -5254,7 +5254,7 @@ body.admin > div.container-fluid > div > div.col-sm-10 > div.container-fluid > d margin-bottom: 20px } -body.admin:not(.modal-open) .btn-default { +body.admin .btn-default { margin-bottom: 10px } @@ -5485,7 +5485,7 @@ body.admin.modal-open .navbar { z-index: 0 !important } -#RestartDialog, #ShutdownDialog, #StatusDialog, #deleteModal { +#RestartDialog, #ShutdownDialog, #StatusDialog, #ClearCacheDialog, #deleteModal { top: 0; overflow: hidden; padding-top: 70px; @@ -5495,7 +5495,7 @@ body.admin.modal-open .navbar { background: rgba(0, 0, 0, .5) } -#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #deleteModal:before { +#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #ClearCacheDialog:before, #deleteModal:before { content: "\E208"; padding-right: 10px; display: block; @@ -5517,18 +5517,18 @@ body.admin.modal-open .navbar { z-index: 99 } -#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before { +#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #ClearCacheDialog.in:before, #deleteModal.in:before { -webkit-transform: translate(0, 0); -ms-transform: translate(0, 0); transform: translate(0, 0) } -#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog { +#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #ClearCacheDialog > .modal-dialog, #deleteModal > .modal-dialog { width: 450px; margin: auto } -#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { +#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #ClearCacheDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { max-height: calc(100% - 90px); -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, .5); box-shadow: 0 5px 15px rgba(0, 0, 0, .5); @@ -5539,7 +5539,7 @@ body.admin.modal-open .navbar { width: 450px } -#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header { +#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header { padding: 15px 20px; border-radius: 3px 3px 0 0; line-height: 1.71428571; @@ -5552,7 +5552,7 @@ body.admin.modal-open .navbar { text-align: left } -#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before { +#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before { padding-right: 10px; font-size: 18px; color: #999; @@ -5576,6 +5576,11 @@ body.admin.modal-open .navbar { font-family: plex-icons-new, serif } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:before { + content: "\EA15"; + font-family: plex-icons-new, serif +} + #deleteModal > .modal-dialog > .modal-content > .modal-header:before { content: "\EA6D"; font-family: plex-icons-new, serif @@ -5599,6 +5604,12 @@ body.admin.modal-open .navbar { font-size: 20px } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:after { + content: "Clear Cover Thumbnail Cache"; + display: inline-block; + font-size: 20px +} + #deleteModal > .modal-dialog > .modal-content > .modal-header:after { content: "Delete Book"; display: inline-block; @@ -5629,7 +5640,17 @@ body.admin.modal-open .navbar { text-align: left } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p { +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body { + padding: 20px 20px 10px; + font-size: 16px; + line-height: 1.6em; + font-family: Open Sans Regular, Helvetica Neue, Helvetica, Arial, sans-serif; + color: #eee; + background: #282828; + text-align: left +} + +#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p { padding: 20px 20px 0 0; font-size: 16px; line-height: 1.6em; @@ -5638,7 +5659,7 @@ body.admin.modal-open .navbar { background: #282828 } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { +#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { float: right; z-index: 9; position: relative; @@ -5674,6 +5695,18 @@ body.admin.modal-open .navbar { border-radius: 3px } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > #clear_cache { + float: right; + z-index: 9; + position: relative; + margin: 25px 0 0 10px; + min-width: 80px; + padding: 10px 18px; + font-size: 16px; + line-height: 1.33; + border-radius: 3px +} + #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-danger { float: right; z-index: 9; @@ -5694,11 +5727,15 @@ body.admin.modal-open .navbar { margin: 55px 0 0 10px } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache) { + margin: 25px 0 0 10px +} + #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { margin: 0 0 0 10px } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover { +#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover { background-color: hsla(0, 0%, 100%, .3) } @@ -5732,6 +5769,21 @@ body.admin.modal-open .navbar { box-shadow: 0 5px 15px rgba(0, 0, 0, .5) } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body:after { + content: ''; + position: absolute; + width: 100%; + height: 72px; + background-color: #323232; + border-radius: 0 0 3px 3px; + left: 0; + margin-top: 10px; + z-index: 0; + border-top: 1px solid #222; + -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, .5); + box-shadow: 0 5px 15px rgba(0, 0, 0, .5) +} + #deleteButton { position: fixed; top: 60px; @@ -7322,11 +7374,11 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. background-color: transparent !important } - #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog { + #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #ClearCacheDialog > .modal-dialog, #deleteModal > .modal-dialog { max-width: calc(100vw - 40px) } - #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { + #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #ClearCacheDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { max-width: calc(100vw - 40px); left: 0 } @@ -7476,7 +7528,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. padding: 30px 15px } - #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before { + #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #ClearCacheDialog.in:before, #deleteModal.in:before { left: auto; right: 34px } diff --git a/cps/static/js/main.js b/cps/static/js/main.js index 3fbaed88..d8c1863f 100644 --- a/cps/static/js/main.js +++ b/cps/static/js/main.js @@ -405,6 +405,18 @@ $(function() { } }); }); + $("#clear_cache").click(function () { + $("#spinner3").show(); + $.ajax({ + dataType: "json", + url: window.location.pathname + "/../../clear-cache", + data: {"cache_type":"thumbnails"}, + success: function(data) { + $("#spinner3").hide(); + $("#ClearCacheDialog").modal("hide"); + } + }); + }); // Init all data control handlers to default $("input[data-control]").trigger("change"); diff --git a/cps/tasks/database.py b/cps/tasks/database.py new file mode 100644 index 00000000..11f0186d --- /dev/null +++ b/cps/tasks/database.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals + +from cps import config, logger +from cps.services.worker import CalibreTask + +try: + from urllib.request import urlopen +except ImportError as e: + from urllib2 import urlopen + + +class TaskReconnectDatabase(CalibreTask): + def __init__(self, task_message=u'Reconnecting Calibre database'): + super(TaskReconnectDatabase, self).__init__(task_message) + self.log = logger.create() + self.listen_address = config.get_config_ipaddress() + self.listen_port = config.config_port + + def run(self, worker_thread): + address = self.listen_address if self.listen_address else 'localhost' + port = self.listen_port if self.listen_port else 8083 + + try: + urlopen('http://' + address + ':' + str(port) + '/reconnect') + self._handleSuccess() + except Exception as ex: + self._handleError(u'Unable to reconnect Calibre database: ' + str(ex)) + + @property + def name(self): + return "Reconnect Database" diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index f61eb4a7..b541bc70 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -42,7 +42,6 @@ THUMBNAIL_RESOLUTION_2X = 2 class TaskGenerateCoverThumbnails(CalibreTask): def __init__(self, limit=100, task_message=u'Generating cover thumbnails'): super(TaskGenerateCoverThumbnails, self).__init__(task_message) - self.self_cleanup = True self.limit = limit self.log = logger.create() self.app_db_session = ub.get_new_session_instance() @@ -186,7 +185,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): class TaskCleanupCoverThumbnailCache(CalibreTask): - def __init__(self, task_message=u'Validating cover thumbnail cache'): + def __init__(self, task_message=u'Cleaning up cover thumbnail cache'): super(TaskCleanupCoverThumbnailCache, self).__init__(task_message) self.log = logger.create() self.app_db_session = ub.get_new_session_instance() @@ -265,3 +264,58 @@ class TaskCleanupCoverThumbnailCache(CalibreTask): @property def name(self): return "CleanupCoverThumbnailCache" + + +class TaskClearCoverThumbnailCache(CalibreTask): + def __init__(self, book_id=None, task_message=u'Clearing cover thumbnail cache'): + super(TaskClearCoverThumbnailCache, self).__init__(task_message) + self.log = logger.create() + self.book_id = book_id + self.app_db_session = ub.get_new_session_instance() + self.cache = fs.FileSystem() + + def run(self, worker_thread): + if self.app_db_session: + if self.book_id: + thumbnails = self.get_thumbnails_for_book(self.book_id) + for thumbnail in thumbnails: + self.expire_and_delete_thumbnail(thumbnail) + else: + self.expire_and_delete_all_thumbnails() + + self._handleSuccess() + self.app_db_session.remove() + + def get_thumbnails_for_book(self, book_id): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id == book_id)\ + .all() + + def expire_and_delete_thumbnail(self, thumbnail): + thumbnail.expiration = datetime.utcnow() + + try: + self.app_db_session.commit() + self.cache.delete_cache_file(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + except Exception as ex: + self.log.info(u'Error expiring book thumbnail: ' + str(ex)) + self._handleError(u'Error expiring book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def expire_and_delete_all_thumbnails(self): + self.app_db_session\ + .query(ub.Thumbnail)\ + .update({'expiration': datetime.utcnow()}) + + try: + self.app_db_session.commit() + self.cache.delete_cache_dir(fs.CACHE_TYPE_THUMBNAILS) + except Exception as ex: + self.log.info(u'Error expiring book thumbnails: ' + str(ex)) + self._handleError(u'Error expiring book thumbnails: ' + str(ex)) + self.app_db_session.rollback() + + @property + def name(self): + return "ClearCoverThumbnailCache" diff --git a/cps/templates/admin.html b/cps/templates/admin.html index 1ef64157..e3d20db3 100644 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -139,15 +139,18 @@
-
+

{{_('Administration')}}

- - -
-
-
{{_('Reconnect Calibre Database')}}
-
{{_('Restart')}}
-
{{_('Shutdown')}}
+ + +
+
+
{{_('Reconnect Calibre Database')}}
+
{{_('Clear Cover Thumbnail Cache')}}
+
+
+
{{_('Restart')}}
+
{{_('Shutdown')}}
@@ -226,4 +229,21 @@
+ {% endblock %} diff --git a/cps/templates/author.html b/cps/templates/author.html index 24ce876a..3cf3fb4b 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -36,7 +36,7 @@
diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html index 878c14a8..3f0ed154 100644 --- a/cps/templates/book_cover.html +++ b/cps/templates/book_cover.html @@ -1,8 +1,8 @@ -{% macro book_cover_image(book_id, book_title) -%} +{% macro book_cover_image(book, book_title) -%} {{ book_title }} {%- endmacro %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 881fa8ff..369b8d05 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -4,8 +4,7 @@ {% if book %}
- {{ book_cover_image(book.id, book.title) }} - + {{ book_cover_image(book, book.title) }}
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index d3615563..cdf6ab2b 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,8 +4,7 @@
- {{ book_cover_image(entry.id, entry.title) }} - + {{ book_cover_image(entry, entry.title) }}
diff --git a/cps/templates/discover.html b/cps/templates/discover.html index 33bafbbe..5d4666f6 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -9,7 +9,7 @@ diff --git a/cps/templates/grid.html b/cps/templates/grid.html index bc3ca4a2..67594b4e 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -29,7 +29,7 @@
diff --git a/cps/templates/index.html b/cps/templates/index.html index c536884f..531a535c 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -9,7 +9,7 @@
@@ -83,7 +83,7 @@
diff --git a/cps/templates/search.html b/cps/templates/search.html index 56b12154..8e0cf668 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -44,7 +44,7 @@ diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index 7a678ea6..bebc0b1f 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -31,7 +31,7 @@
diff --git a/cps/web.py b/cps/web.py index 27a5849b..16c14fcf 100644 --- a/cps/web.py +++ b/cps/web.py @@ -50,7 +50,7 @@ from . import babel, db, ub, config, get_locale, app from . import calibre_db, shelf from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download from .helper import check_valid_domain, render_task_status, \ - get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \ + get_cc_columns, get_book_cover, get_cached_book_cover, get_download_link, send_mail, generate_random_password, \ send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password from .pagination import Pagination from .redirect import redirect_back @@ -1177,6 +1177,12 @@ def get_cover(book_id, resolution=1): return get_book_cover(book_id, resolution) +@web.route("/cached-cover/") +@login_required_if_no_ano +def get_cached_cover(cache_id): + return get_cached_book_cover(cache_id) + + @web.route("/robots.txt") def get_robots(): return send_from_directory(constants.STATIC_DIR, "robots.txt") From 242a2767a1e374a938de2f1b3fdb7cb2175c5fd1 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Thu, 24 Dec 2020 02:35:32 -0600 Subject: [PATCH 006/112] Added thumbnail urls to book cover srcsets with cache busting ids --- cps/db.py | 3 +- cps/helper.py | 38 +++++++++++++++ cps/jinjia.py | 16 +++++++ cps/schedule.py | 10 ++-- cps/tasks/thumbnail.py | 61 ++++++++++++++++++++---- cps/templates/author.html | 2 +- cps/templates/book_cover.html | 19 +++++--- cps/templates/book_edit.html | 2 +- cps/templates/detail.html | 2 +- cps/templates/discover.html | 2 +- cps/templates/grid.html | 2 +- cps/templates/index.html | 4 +- cps/templates/search.html | 2 +- cps/templates/shelf.html | 2 +- cps/web.py | 89 +++++++++++++++++++++++++---------- 15 files changed, 199 insertions(+), 55 deletions(-) diff --git a/cps/db.py b/cps/db.py index 2e428f72..8b8db10a 100644 --- a/cps/db.py +++ b/cps/db.py @@ -609,7 +609,8 @@ class CalibreDB(): randm = self.session.query(Books) \ .filter(self.common_filters(allow_show_archived)) \ .order_by(func.random()) \ - .limit(self.config.config_random_books) + .limit(self.config.config_random_books) \ + .all() else: randm = false() off = int(int(pagesize) * (page - 1)) diff --git a/cps/helper.py b/cps/helper.py index 0b0c675f..c33a69e1 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -533,6 +533,21 @@ def delete_book(book, calibrepath, book_format): return delete_book_file(book, calibrepath, book_format) +def get_thumbnails_for_books(books): + books_with_covers = list(filter(lambda b: b.has_cover, books)) + book_ids = list(map(lambda b: b.id, books_with_covers)) + return ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.in_(book_ids))\ + .filter(ub.Thumbnail.expiration > datetime.utcnow())\ + .all() + + +def get_thumbnails_for_book_series(series): + books = list(map(lambda s: s[0], series)) + return get_thumbnails_for_books(books) + + def get_cover_on_failure(use_generic_cover): if use_generic_cover: return send_from_directory(_STATIC_DIR, "generic_cover.jpg") @@ -558,6 +573,29 @@ def get_cached_book_cover(cache_id): return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) +def get_cached_book_cover_thumbnail(cache_id): + parts = cache_id.split('_') + thumbnail_uuid = parts[0] if len(parts) else None + thumbnail = None + if thumbnail_uuid: + thumbnail = ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.uuid == thumbnail_uuid)\ + .first() + + if thumbnail and thumbnail.expiration > datetime.utcnow(): + cache = fs.FileSystem() + if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename) + + elif thumbnail: + book = calibre_db.get_book(thumbnail.book_id) + return get_book_cover_internal(book, use_generic_cover_on_failure=True) + + else: + return get_cover_on_failure(True) + + def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None): if book and book.has_cover: diff --git a/cps/jinjia.py b/cps/jinjia.py index bf81c059..b2479adc 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -139,3 +139,19 @@ def book_cover_cache_id(book, resolution=None): timestamp = int(book.last_modified.timestamp() * 1000) cache_bust = str(book.uuid) + '_' + str(timestamp) return cache_bust if not resolution else cache_bust + '_' + str(resolution) + + +@jinjia.app_template_filter('get_book_thumbnails') +def get_book_thumbnails(book_id, thumbnails=None): + return list(filter(lambda t: t.book_id == book_id, thumbnails)) if book_id > -1 and thumbnails else list() + + +@jinjia.app_template_filter('get_book_thumbnail_srcset') +def get_book_thumbnail_srcset(thumbnails): + srcset = list() + for thumbnail in thumbnails: + timestamp = int(thumbnail.generated_at.timestamp() * 1000) + cache_id = str(thumbnail.uuid) + '_' + str(timestamp) + url = url_for('web.get_cached_cover_thumbnail', cache_id=cache_id) + srcset.append(url + ' ' + str(thumbnail.resolution) + 'x') + return ', '.join(srcset) diff --git a/cps/schedule.py b/cps/schedule.py index f349a231..7ee43410 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -20,17 +20,17 @@ from __future__ import division, print_function, unicode_literals from .services.background_scheduler import BackgroundScheduler from .tasks.database import TaskReconnectDatabase -from .tasks.thumbnail import TaskCleanupCoverThumbnailCache, TaskGenerateCoverThumbnails +from .tasks.thumbnail import TaskSyncCoverThumbnailCache, TaskGenerateCoverThumbnails def register_jobs(): scheduler = BackgroundScheduler() # Generate 100 book cover thumbnails every 5 minutes - scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='interval', minutes=5) + scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='cron', minute='*/5') - # Cleanup book cover cache every day at 4am - scheduler.add_task(user=None, task=lambda: TaskCleanupCoverThumbnailCache(), trigger='cron', hour=4) + # Cleanup book cover cache every 6 hours + scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='cron', minute='15', hour='*/6') # Reconnect metadata.db every 4 hours - scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='interval', hours=4) + scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', minute='5', hour='*/4') diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index b541bc70..e9df170e 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -59,6 +59,10 @@ class TaskGenerateCoverThumbnails(CalibreTask): books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids) count = len(books_without_thumbnails) + if count == 0: + # Do not display this task on the frontend if there are no covers to update + self.self_cleanup = True + for i, book in enumerate(books_without_thumbnails): for resolution in self.resolutions: expired_thumbnail = self.get_expired_thumbnail_for_book_and_resolution( @@ -71,6 +75,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): else: self.create_book_thumbnail(book, resolution) + self.message(u'Generating cover thumbnail {0} of {1}'.format(i, count)) self.progress = (1.0 / count) * i self._handleSuccess() @@ -181,12 +186,12 @@ class TaskGenerateCoverThumbnails(CalibreTask): @property def name(self): - return "GenerateCoverThumbnails" + return "ThumbnailsGenerate" -class TaskCleanupCoverThumbnailCache(CalibreTask): - def __init__(self, task_message=u'Cleaning up cover thumbnail cache'): - super(TaskCleanupCoverThumbnailCache, self).__init__(task_message) +class TaskSyncCoverThumbnailCache(CalibreTask): + def __init__(self, task_message=u'Syncing cover thumbnail cache'): + super(TaskSyncCoverThumbnailCache, self).__init__(task_message) self.log = logger.create() self.app_db_session = ub.get_new_session_instance() self.calibre_db = db.CalibreDB(expire_on_commit=False) @@ -199,14 +204,23 @@ class TaskCleanupCoverThumbnailCache(CalibreTask): # This case will happen if a user deletes the cache dir or cached files if self.app_db_session: self.expire_missing_thumbnails(cached_thumbnail_files) - self.progress = 0.33 + self.progress = 0.25 # Delete thumbnails in the database if the book has been removed # This case will happen if a book is removed in Calibre and the metadata.db file is updated in the filesystem if self.app_db_session and self.calibre_db: book_ids = self.get_book_ids() self.delete_thumbnails_for_missing_books(book_ids) - self.progress = 0.66 + self.progress = 0.50 + + # Expire thumbnails in the database if their corresponding book has been updated since they were generated + # This case will happen if the book was updated externally + if self.app_db_session and self.cache: + books = self.get_books_updated_in_the_last_day() + book_ids = list(map(lambda b: b.id, books)) + thumbnails = self.get_thumbnails_for_updated_books(book_ids) + self.expire_thumbnails_for_updated_book(books, thumbnails) + self.progress = 0.75 # Delete extraneous cached thumbnail files # This case will happen if a book was deleted and the thumbnail OR the metadata.db file was changed externally @@ -261,9 +275,40 @@ class TaskCleanupCoverThumbnailCache(CalibreTask): for file in extraneous_files: self.cache.delete_cache_file(file, fs.CACHE_TYPE_THUMBNAILS) + def get_books_updated_in_the_last_day(self): + return self.calibre_db.session\ + .query(db.Books)\ + .filter(db.Books.has_cover == 1)\ + .filter(db.Books.last_modified > datetime.utcnow() - timedelta(days=1, hours=1))\ + .all() + + def get_thumbnails_for_updated_books(self, book_ids): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.in_(book_ids))\ + .all() + + def expire_thumbnails_for_updated_book(self, books, thumbnails): + thumbnail_ids = list() + for book in books: + for thumbnail in thumbnails: + if thumbnail.book_id == book.id and thumbnail.generated_at < book.last_modified: + thumbnail_ids.append(thumbnail.id) + + try: + self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.id.in_(thumbnail_ids)) \ + .update({"expiration": datetime.utcnow()}, synchronize_session=False) + self.app_db_session.commit() + except Exception as ex: + self.log.info(u'Error expiring thumbnails for updated books: ' + str(ex)) + self._handleError(u'Error expiring thumbnails for updated books: ' + str(ex)) + self.app_db_session.rollback() + @property def name(self): - return "CleanupCoverThumbnailCache" + return "ThumbnailsSync" class TaskClearCoverThumbnailCache(CalibreTask): @@ -318,4 +363,4 @@ class TaskClearCoverThumbnailCache(CalibreTask): @property def name(self): - return "ClearCoverThumbnailCache" + return "ThumbnailsClear" diff --git a/cps/templates/author.html b/cps/templates/author.html index 3cf3fb4b..e5acdd2f 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -36,7 +36,7 @@
diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html index 3f0ed154..c5281797 100644 --- a/cps/templates/book_cover.html +++ b/cps/templates/book_cover.html @@ -1,8 +1,13 @@ -{% macro book_cover_image(book, book_title) -%} - {{ book_title }} +{% macro book_cover_image(book, thumbnails) -%} + {%- set book_title = book.title if book.title else book.name -%} + {% set srcset = thumbnails|get_book_thumbnail_srcset if thumbnails|length else '' %} + {%- if srcset|length -%} + {{ book_title }} + {%- else -%} + {{ book_title }} + {%- endif -%} {%- endmacro %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 369b8d05..c538d5ca 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -4,7 +4,7 @@ {% if book %}
- {{ book_cover_image(book, book.title) }} + {{ book_cover_image(book, book.id|get_book_thumbnails(thumbnails)) }}
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index cdf6ab2b..671186c7 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,7 +4,7 @@
- {{ book_cover_image(entry, entry.title) }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
diff --git a/cps/templates/discover.html b/cps/templates/discover.html index 5d4666f6..c5c12db2 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -9,7 +9,7 @@ diff --git a/cps/templates/grid.html b/cps/templates/grid.html index 67594b4e..0f669d51 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -29,7 +29,7 @@
diff --git a/cps/templates/index.html b/cps/templates/index.html index 531a535c..2be15ba7 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -9,7 +9,7 @@
@@ -83,7 +83,7 @@
diff --git a/cps/templates/search.html b/cps/templates/search.html index 8e0cf668..a5871afb 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -44,7 +44,7 @@ diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index bebc0b1f..cb55c50c 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -31,7 +31,7 @@
diff --git a/cps/web.py b/cps/web.py index 16c14fcf..c400b96e 100644 --- a/cps/web.py +++ b/cps/web.py @@ -49,9 +49,10 @@ from . import constants, logger, isoLanguages, services from . import babel, db, ub, config, get_locale, app from . import calibre_db, shelf from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download -from .helper import check_valid_domain, render_task_status, \ - get_cc_columns, get_book_cover, get_cached_book_cover, get_download_link, send_mail, generate_random_password, \ - send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password +from .helper import check_valid_domain, render_task_status, get_cc_columns, get_book_cover, get_cached_book_cover, \ + get_cached_book_cover_thumbnail, get_thumbnails_for_books, get_thumbnails_for_book_series, get_download_link, \ + send_mail, generate_random_password, send_registration_mail, check_send_to_kindle, check_read_formats, \ + tags_filters, reset_password from .pagination import Pagination from .redirect import redirect_back from .usermanagement import login_required_if_no_ano @@ -386,16 +387,18 @@ def render_books_list(data, sort, book_id, page): db.Books, db.Books.ratings.any(db.Ratings.rating > 9), order) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - id=book_id, title=_(u"Top Rated Books"), page="rated") + id=book_id, title=_(u"Top Rated Books"), page="rated", thumbnails=thumbnails) else: abort(404) elif data == "discover": if current_user.check_visibility(constants.SIDEBAR_RANDOM): entries, __, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, [func.randomblob(2)]) pagination = Pagination(1, config.config_books_per_page, config.config_books_per_page) + thumbnails = get_thumbnails_for_books(entries) return render_title_template('discover.html', entries=entries, pagination=pagination, id=book_id, - title=_(u"Discover (Random Books)"), page="discover") + title=_(u"Discover (Random Books)"), page="discover", thumbnails=thumbnails) else: abort(404) elif data == "unread": @@ -433,8 +436,9 @@ def render_books_list(data, sort, book_id, page): else: website = data or "newest" entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, order) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=_(u"Books"), page=website) + title=_(u"Books"), page=website, thumbnails=thumbnails) def render_hot_books(page): @@ -458,8 +462,9 @@ def render_hot_books(page): ub.delete_download(book.Downloads.book_id) numBooks = entries.__len__() pagination = Pagination(page, config.config_books_per_page, numBooks) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=_(u"Hot Books (Most Downloaded)"), page="hot") + title=_(u"Hot Books (Most Downloaded)"), page="hot", thumbnails=thumbnails) else: abort(404) @@ -490,12 +495,14 @@ def render_downloaded_books(page, order): .filter(db.Books.id == book.id).first(): ub.delete_download(book.id) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, title=_(u"Downloaded books by %(user)s",user=current_user.nickname), - page="download") + page="download", + thumbnails=thumbnails) else: abort(404) @@ -521,9 +528,10 @@ def render_author_books(page, author_id, order): author_info = services.goodreads_support.get_author_info(author_name) other_books = services.goodreads_support.get_other_books(author_info, entries) + thumbnails = get_thumbnails_for_books(entries) return render_title_template('author.html', entries=entries, pagination=pagination, id=author_id, title=_(u"Author: %(name)s", name=author_name), author=author_info, - other_books=other_books, page="author") + other_books=other_books, page="author", thumbnails=thumbnails) def render_publisher_books(page, book_id, order): @@ -535,8 +543,10 @@ def render_publisher_books(page, book_id, order): [db.Series.name, order[0], db.Books.series_index], db.books_series_link, db.Series) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, - title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher") + title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher", + thumbnails=thumbnails) else: abort(404) @@ -548,8 +558,10 @@ def render_series_books(page, book_id, order): db.Books, db.Books.series.any(db.Series.id == book_id), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"Series: %(serie)s", serie=name.name), page="series") + title=_(u"Series: %(serie)s", serie=name.name), page="series", + thumbnails=thumbnails) else: abort(404) @@ -561,8 +573,10 @@ def render_ratings_books(page, book_id, order): db.Books.ratings.any(db.Ratings.id == book_id), [order[0]]) if name and name.rating <= 10: + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings") + title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings", + thumbnails=thumbnails) else: abort(404) @@ -574,8 +588,10 @@ def render_formats_books(page, book_id, order): db.Books, db.Books.data.any(db.Data.format == book_id.upper()), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"File format: %(format)s", format=name.format), page="formats") + title=_(u"File format: %(format)s", format=name.format), page="formats", + thumbnails=thumbnails) else: abort(404) @@ -588,8 +604,10 @@ def render_category_books(page, book_id, order): db.Books.tags.any(db.Tags.id == book_id), [order[0], db.Series.name, db.Books.series_index], db.books_series_link, db.Series) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, - title=_(u"Category: %(name)s", name=name.name), page="category") + title=_(u"Category: %(name)s", name=name.name), page="category", + thumbnails=thumbnails) else: abort(404) @@ -607,8 +625,9 @@ def render_language_books(page, name, order): db.Books, db.Books.languages.any(db.Languages.lang_code == name), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name, - title=_(u"Language: %(name)s", name=lang_name), page="language") + title=_(u"Language: %(name)s", name=lang_name), page="language", thumbnails=thumbnails) def render_read_books(page, are_read, as_xml=False, order=None): @@ -652,8 +671,10 @@ def render_read_books(page, are_read, as_xml=False, order=None): else: name = _(u'Unread Books') + ' (' + str(pagination.total_count) + ')' pagename = "unread" + + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename) + title=name, page=pagename, thumbnails=thumbnails) def render_archived_books(page, order): @@ -676,8 +697,9 @@ def render_archived_books(page, order): name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')' pagename = "archived" + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename) + title=name, page=pagename, thumbnails=thumbnails) def render_prepare_search_form(cc): @@ -710,6 +732,7 @@ def render_prepare_search_form(cc): def render_search_results(term, offset=None, order=None, limit=None): entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit) + thumbnails = get_thumbnails_for_books(entries) return render_title_template('search.html', searchterm=term, pagination=pagination, @@ -718,7 +741,8 @@ def render_search_results(term, offset=None, order=None, limit=None): entries=entries, result_count=result_count, title=_(u"Search"), - page="search") + page="search", + thumbnails=thumbnails) # ################################### View Books list ################################################################## @@ -748,6 +772,7 @@ def books_table(): return render_title_template('book_table.html', title=_(u"Books List"), page="book_table", visiblility=visibility) + @web.route("/ajax/listbooks") @login_required def list_books(): @@ -780,6 +805,7 @@ def list_books(): response.headers["Content-Type"] = "application/json; charset=utf-8" return response + @web.route("/ajax/table_settings", methods=['POST']) @login_required def update_table_settings(): @@ -834,6 +860,7 @@ def publisher_list(): charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \ .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all() + return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, title=_(u"Publishers"), page="publisherlist", data="publisher") else: @@ -865,8 +892,10 @@ def series_list(): .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all() + thumbnails = get_thumbnails_for_book_series(entries) return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=charlist, - title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view") + title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view", + thumbnails=thumbnails) else: abort(404) @@ -1150,13 +1179,16 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): else: offset = 0 limit_all = result_count + + thumbnails = get_thumbnails_for_books(entries) return render_title_template('search.html', adv_searchterm=searchterm, pagination=pagination, entries=q[offset:limit_all], result_count=result_count, - title=_(u"Advanced Search"), page="advsearch") - + title=_(u"Advanced Search"), + page="advsearch", + thumbnails=thumbnails) @web.route("/advsearch", methods=['GET']) @@ -1171,10 +1203,9 @@ def advanced_search_form(): @web.route("/cover/") -@web.route("/cover//") @login_required_if_no_ano -def get_cover(book_id, resolution=1): - return get_book_cover(book_id, resolution) +def get_cover(book_id): + return get_book_cover(book_id) @web.route("/cached-cover/") @@ -1183,6 +1214,12 @@ def get_cached_cover(cache_id): return get_cached_book_cover(cache_id) +@web.route("/cached-cover-thumbnail/") +@login_required_if_no_ano +def get_cached_cover_thumbnail(cache_id): + return get_cached_book_cover_thumbnail(cache_id) + + @web.route("/robots.txt") def get_robots(): return send_from_directory(constants.STATIC_DIR, "robots.txt") @@ -1591,6 +1628,7 @@ def show_book(book_id): if media_format.format.lower() in constants.EXTENSIONS_AUDIO: audioentries.append(media_format.format.lower()) + thumbnails = get_thumbnails_for_books([entries]) return render_title_template('detail.html', entry=entries, audioentries=audioentries, @@ -1602,7 +1640,8 @@ def show_book(book_id): is_archived=is_archived, kindle_list=kindle_list, reader_list=reader_list, - page="book") + page="book", + thumbnails=thumbnails) else: log.debug(u"Error opening eBook. File does not exist or file is not accessible") flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error") From eef21759cd56c28560ee2007726fab18c8595df5 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Thu, 24 Dec 2020 03:00:26 -0600 Subject: [PATCH 007/112] Fix generate thumbnail task messages, don't load thumbnails when the cache file has been deleted --- cps/helper.py | 4 ++++ cps/tasks/thumbnail.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cps/helper.py b/cps/helper.py index c33a69e1..add1b067 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -536,9 +536,13 @@ def delete_book(book, calibrepath, book_format): def get_thumbnails_for_books(books): books_with_covers = list(filter(lambda b: b.has_cover, books)) book_ids = list(map(lambda b: b.id, books_with_covers)) + cache = fs.FileSystem() + thumbnail_files = cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS) + return ub.session\ .query(ub.Thumbnail)\ .filter(ub.Thumbnail.book_id.in_(book_ids))\ + .filter(ub.Thumbnail.filename.in_(thumbnail_files))\ .filter(ub.Thumbnail.expiration > datetime.utcnow())\ .all() diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index e9df170e..70ddc06b 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -75,7 +75,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): else: self.create_book_thumbnail(book, resolution) - self.message(u'Generating cover thumbnail {0} of {1}'.format(i, count)) + self.message = u'Generating cover thumbnail {0} of {1}'.format(i + 1, count) self.progress = (1.0 / count) * i self._handleSuccess() From 8cc06683df25684793d1d6154a74a3b11408d980 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Mon, 4 Jan 2021 12:28:05 -0600 Subject: [PATCH 008/112] only python3 supported now --- cps/tasks/database.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cps/tasks/database.py b/cps/tasks/database.py index 11f0186d..62208c0c 100644 --- a/cps/tasks/database.py +++ b/cps/tasks/database.py @@ -20,11 +20,7 @@ from __future__ import division, print_function, unicode_literals from cps import config, logger from cps.services.worker import CalibreTask - -try: - from urllib.request import urlopen -except ImportError as e: - from urllib2 import urlopen +from urllib.request import urlopen class TaskReconnectDatabase(CalibreTask): From 2c8d055ca4e9b92842615516834bb2472348b15e Mon Sep 17 00:00:00 2001 From: mmonkey Date: Mon, 4 Jan 2021 12:36:40 -0600 Subject: [PATCH 009/112] support python2.7 for the mean time --- cps/tasks/database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cps/tasks/database.py b/cps/tasks/database.py index 62208c0c..11f0186d 100644 --- a/cps/tasks/database.py +++ b/cps/tasks/database.py @@ -20,7 +20,11 @@ from __future__ import division, print_function, unicode_literals from cps import config, logger from cps.services.worker import CalibreTask -from urllib.request import urlopen + +try: + from urllib.request import urlopen +except ImportError as e: + from urllib2 import urlopen class TaskReconnectDatabase(CalibreTask): From dd30ac4fbdc9699dc4462b1a9a7efeddb53ffe3a Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 20 Mar 2021 11:32:50 +0100 Subject: [PATCH 010/112] Added thumbnails Merge remote-tracking branch 'cover/thumbnails' into development # Conflicts: # cps/admin.py # cps/templates/layout.html # cps/ub.py # cps/web.py Update join for sqlalchemy 1.4 --- .gitignore | 1 + cps.py | 5 + cps/__init__.py | 3 +- cps/admin.py | 19 +- cps/constants.py | 1 + cps/db.py | 13 +- cps/editbooks.py | 2 + cps/fs.py | 61 +++++ cps/helper.py | 89 ++++++- cps/jinjia.py | 22 ++ cps/schedule.py | 36 +++ cps/services/background_scheduler.py | 52 ++++ cps/services/worker.py | 17 +- cps/shelf.py | 2 +- cps/static/css/caliBlur.css | 82 ++++-- cps/static/js/main.js | 12 + cps/tasks/database.py | 49 ++++ cps/tasks/thumbnail.py | 366 +++++++++++++++++++++++++++ cps/templates/admin.html | 36 ++- cps/templates/author.html | 2 +- cps/templates/book_cover.html | 13 + cps/templates/book_edit.html | 3 +- cps/templates/detail.html | 2 +- cps/templates/discover.html | 3 +- cps/templates/fragment.html | 1 + cps/templates/grid.html | 3 +- cps/templates/index.html | 5 +- cps/templates/layout.html | 1 + cps/templates/search.html | 3 +- cps/templates/shelf.html | 3 +- cps/ub.py | 34 +++ cps/web.py | 96 +++++-- requirements.txt | 1 + 33 files changed, 965 insertions(+), 73 deletions(-) create mode 100644 cps/fs.py create mode 100644 cps/schedule.py create mode 100644 cps/services/background_scheduler.py create mode 100644 cps/tasks/database.py create mode 100644 cps/tasks/thumbnail.py create mode 100644 cps/templates/book_cover.html diff --git a/.gitignore b/.gitignore index 109de4ef..903cfd36 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ vendor/ # calibre-web *.db *.log +cps/cache .idea/ *.bak diff --git a/cps.py b/cps.py index 737b0d97..19ca89b8 100755 --- a/cps.py +++ b/cps.py @@ -43,6 +43,7 @@ from cps.gdrive import gdrive from cps.editbooks import editbook from cps.remotelogin import remotelogin from cps.error_handler import init_errorhandler +from cps.schedule import register_jobs try: from cps.kobo import kobo, get_kobo_activated @@ -78,6 +79,10 @@ def main(): app.register_blueprint(kobo_auth) if oauth_available: app.register_blueprint(oauth) + + # Register scheduled jobs + register_jobs() + success = web_server.start() sys.exit(0 if success else 1) diff --git a/cps/__init__.py b/cps/__init__.py index 627cca0b..30029428 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -96,7 +96,7 @@ def create_app(): app.instance_path = app.instance_path.decode('utf-8') if os.environ.get('FLASK_DEBUG'): - cache_buster.init_cache_busting(app) + cache_buster.init_cache_busting(app) log.info('Starting Calibre Web...') if sys.version_info < (3, 0): @@ -121,6 +121,7 @@ def create_app(): return app + @babel.localeselector def get_locale(): # if a user is logged in, use the locale from the user settings diff --git a/cps/admin.py b/cps/admin.py index 78cfebf1..cd548bfb 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -40,7 +40,7 @@ from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError from sqlalchemy.sql.expression import func, or_ -from . import constants, logger, helper, services, isoLanguages +from . import constants, logger, helper, services, isoLanguages, fs from .cli import filepicker from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash @@ -164,6 +164,23 @@ def shutdown(): return json.dumps(showtext), 400 +@admi.route("/clear-cache") +@login_required +@admin_required +def clear_cache(): + cache_type = request.args.get('cache_type'.strip()) + showtext = {} + + if cache_type == fs.CACHE_TYPE_THUMBNAILS: + log.info('clearing cover thumbnail cache') + showtext['text'] = _(u'Cleared cover thumbnail cache') + helper.clear_cover_thumbnail_cache() + return json.dumps(showtext) + + showtext['text'] = _(u'Unknown command') + return json.dumps(showtext) + + @admi.route("/admin/view") @login_required @admin_required diff --git a/cps/constants.py b/cps/constants.py index e9c26cb1..0eb94709 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -37,6 +37,7 @@ else: STATIC_DIR = os.path.join(BASE_DIR, 'cps', 'static') TEMPLATES_DIR = os.path.join(BASE_DIR, 'cps', 'templates') TRANSLATIONS_DIR = os.path.join(BASE_DIR, 'cps', 'translations') +CACHE_DIR = os.path.join(BASE_DIR, 'cps', 'cache') if HOME_CONFIG: home_dir = os.path.join(os.path.expanduser("~"),".calibre-web") diff --git a/cps/db.py b/cps/db.py index ac59ac2b..f43cc811 100644 --- a/cps/db.py +++ b/cps/db.py @@ -620,15 +620,18 @@ class CalibreDB(): randm = self.session.query(Books) \ .filter(self.common_filters(allow_show_archived)) \ .order_by(func.random()) \ - .limit(self.config.config_random_books) + .limit(self.config.config_random_books) \ + .all() else: randm = false() off = int(int(pagesize) * (page - 1)) - query = self.session.query(database) \ - .filter(db_filter) \ + query = self.session.query(database) + if len(join) == 3: + query = query.join(join[0], join[1]).join(join[2], isouter=True) + elif len(join) == 2: + query = query.join(join[0], join[1], isouter=True) + query = query.filter(db_filter)\ .filter(self.common_filters(allow_show_archived)) - if len(join): - query = query.join(*join, isouter=True) entries = list() pagination = list() try: diff --git a/cps/editbooks.py b/cps/editbooks.py index 28cad5c5..b7f496d0 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -614,6 +614,7 @@ def upload_cover(request, book): abort(403) ret, message = helper.save_cover(requested_file, book.path) if ret is True: + helper.clear_cover_thumbnail_cache(book.id) return True else: flash(message, category="error") @@ -710,6 +711,7 @@ def edit_book(book_id): if result is True: book.has_cover = 1 modif_date = True + helper.clear_cover_thumbnail_cache(book.id) else: flash(error, category="error") diff --git a/cps/fs.py b/cps/fs.py new file mode 100644 index 00000000..699d5991 --- /dev/null +++ b/cps/fs.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +from .constants import CACHE_DIR +from os import listdir, makedirs, remove +from os.path import isdir, isfile, join +from shutil import rmtree + +CACHE_TYPE_THUMBNAILS = 'thumbnails' + + +class FileSystem: + _instance = None + _cache_dir = CACHE_DIR + + def __new__(cls): + if cls._instance is None: + cls._instance = super(FileSystem, cls).__new__(cls) + return cls._instance + + def get_cache_dir(self, cache_type=None): + if not isdir(self._cache_dir): + makedirs(self._cache_dir) + + if cache_type and not isdir(join(self._cache_dir, cache_type)): + makedirs(join(self._cache_dir, cache_type)) + + return join(self._cache_dir, cache_type) if cache_type else self._cache_dir + + def get_cache_file_path(self, filename, cache_type=None): + return join(self.get_cache_dir(cache_type), filename) if filename else None + + def list_cache_files(self, cache_type=None): + path = self.get_cache_dir(cache_type) + return [file for file in listdir(path) if isfile(join(path, file))] + + def delete_cache_dir(self, cache_type=None): + if not cache_type and isdir(self._cache_dir): + rmtree(self._cache_dir) + if cache_type and isdir(join(self._cache_dir, cache_type)): + rmtree(join(self._cache_dir, cache_type)) + + def delete_cache_file(self, filename, cache_type=None): + if isfile(join(self.get_cache_dir(cache_type), filename)): + remove(join(self.get_cache_dir(cache_type), filename)) diff --git a/cps/helper.py b/cps/helper.py index e18ae33b..e3c79dea 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -52,12 +52,13 @@ except ImportError: from . import calibre_db from .tasks.convert import TaskConvert -from . import logger, config, get_locale, db, ub +from . import logger, config, get_locale, db, fs, ub from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR from .subproc_wrapper import process_wait from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS from .tasks.mail import TaskEmail +from .tasks.thumbnail import TaskClearCoverThumbnailCache log = logger.create() @@ -514,12 +515,32 @@ def update_dir_stucture(book_id, calibrepath, first_author=None, orignal_filepat def delete_book(book, calibrepath, book_format): + clear_cover_thumbnail_cache(book.id) if config.config_use_google_drive: return delete_book_gdrive(book, book_format) else: return delete_book_file(book, calibrepath, book_format) +def get_thumbnails_for_books(books): + books_with_covers = list(filter(lambda b: b.has_cover, books)) + book_ids = list(map(lambda b: b.id, books_with_covers)) + cache = fs.FileSystem() + thumbnail_files = cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS) + + return ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.in_(book_ids))\ + .filter(ub.Thumbnail.filename.in_(thumbnail_files))\ + .filter(ub.Thumbnail.expiration > datetime.utcnow())\ + .all() + + +def get_thumbnails_for_book_series(series): + books = list(map(lambda s: s[0], series)) + return get_thumbnails_for_books(books) + + def get_cover_on_failure(use_generic_cover): if use_generic_cover: return send_from_directory(_STATIC_DIR, "generic_cover.jpg") @@ -532,14 +553,54 @@ def get_book_cover(book_id): return get_book_cover_internal(book, use_generic_cover_on_failure=True) -def get_book_cover_with_uuid(book_uuid, - use_generic_cover_on_failure=True): +def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True): book = calibre_db.get_book_by_uuid(book_uuid) return get_book_cover_internal(book, use_generic_cover_on_failure) -def get_book_cover_internal(book, use_generic_cover_on_failure): +def get_cached_book_cover(cache_id): + parts = cache_id.split('_') + book_uuid = parts[0] if len(parts) else None + resolution = parts[2] if len(parts) > 2 else None + book = calibre_db.get_book_by_uuid(book_uuid) if book_uuid else None + return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) + + +def get_cached_book_cover_thumbnail(cache_id): + parts = cache_id.split('_') + thumbnail_uuid = parts[0] if len(parts) else None + thumbnail = None + if thumbnail_uuid: + thumbnail = ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.uuid == thumbnail_uuid)\ + .first() + + if thumbnail and thumbnail.expiration > datetime.utcnow(): + cache = fs.FileSystem() + if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename) + + elif thumbnail: + book = calibre_db.get_book(thumbnail.book_id) + return get_book_cover_internal(book, use_generic_cover_on_failure=True) + + else: + return get_cover_on_failure(True) + + +def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None): if book and book.has_cover: + + # Send the book cover thumbnail if it exists in cache + if resolution: + thumbnail = get_book_cover_thumbnail(book, resolution) + if thumbnail: + cache = fs.FileSystem() + if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename) + + # Send the book cover from Google Drive if configured if config.config_use_google_drive: try: if not gd.is_gdrive_ready(): @@ -550,9 +611,11 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): else: log.error('%s/cover.jpg not found on Google Drive', book.path) return get_cover_on_failure(use_generic_cover_on_failure) - except Exception as e: - log.debug_or_exception(e) + except Exception as ex: + log.debug_or_exception(ex) return get_cover_on_failure(use_generic_cover_on_failure) + + # Send the book cover from the Calibre directory else: cover_file_path = os.path.join(config.config_calibre_dir, book.path) if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")): @@ -563,6 +626,16 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): return get_cover_on_failure(use_generic_cover_on_failure) +def get_book_cover_thumbnail(book, resolution): + if book and book.has_cover: + return ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id == book.id)\ + .filter(ub.Thumbnail.resolution == resolution)\ + .filter(ub.Thumbnail.expiration > datetime.utcnow())\ + .first() + + # saves book cover from url def save_cover_from_url(url, book_path): try: @@ -820,3 +893,7 @@ def get_download_link(book_id, book_format, client): return do_download_file(book, book_format, client, data1, headers) else: abort(404) + + +def clear_cover_thumbnail_cache(book_id=None): + WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id)) diff --git a/cps/jinjia.py b/cps/jinjia.py index 688d1fba..b2479adc 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -128,8 +128,30 @@ def formatseriesindex_filter(series_index): return series_index return 0 + @jinjia.app_template_filter('uuidfilter') def uuidfilter(var): return uuid4() +@jinjia.app_template_filter('book_cover_cache_id') +def book_cover_cache_id(book, resolution=None): + timestamp = int(book.last_modified.timestamp() * 1000) + cache_bust = str(book.uuid) + '_' + str(timestamp) + return cache_bust if not resolution else cache_bust + '_' + str(resolution) + + +@jinjia.app_template_filter('get_book_thumbnails') +def get_book_thumbnails(book_id, thumbnails=None): + return list(filter(lambda t: t.book_id == book_id, thumbnails)) if book_id > -1 and thumbnails else list() + + +@jinjia.app_template_filter('get_book_thumbnail_srcset') +def get_book_thumbnail_srcset(thumbnails): + srcset = list() + for thumbnail in thumbnails: + timestamp = int(thumbnail.generated_at.timestamp() * 1000) + cache_id = str(thumbnail.uuid) + '_' + str(timestamp) + url = url_for('web.get_cached_cover_thumbnail', cache_id=cache_id) + srcset.append(url + ' ' + str(thumbnail.resolution) + 'x') + return ', '.join(srcset) diff --git a/cps/schedule.py b/cps/schedule.py new file mode 100644 index 00000000..7ee43410 --- /dev/null +++ b/cps/schedule.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals + +from .services.background_scheduler import BackgroundScheduler +from .tasks.database import TaskReconnectDatabase +from .tasks.thumbnail import TaskSyncCoverThumbnailCache, TaskGenerateCoverThumbnails + + +def register_jobs(): + scheduler = BackgroundScheduler() + + # Generate 100 book cover thumbnails every 5 minutes + scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='cron', minute='*/5') + + # Cleanup book cover cache every 6 hours + scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='cron', minute='15', hour='*/6') + + # Reconnect metadata.db every 4 hours + scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', minute='5', hour='*/4') diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py new file mode 100644 index 00000000..efa57379 --- /dev/null +++ b/cps/services/background_scheduler.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import atexit + +from .. import logger +from .worker import WorkerThread +from apscheduler.schedulers.background import BackgroundScheduler as BScheduler + + +class BackgroundScheduler: + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super(BackgroundScheduler, cls).__new__(cls) + + scheduler = BScheduler() + atexit.register(lambda: scheduler.shutdown()) + + cls.log = logger.create() + cls.scheduler = scheduler + cls.scheduler.start() + + return cls._instance + + def add(self, func, trigger, **trigger_args): + self.scheduler.add_job(func=func, trigger=trigger, **trigger_args) + + def add_task(self, user, task, trigger, **trigger_args): + def scheduled_task(): + worker_task = task() + self.log.info('Running scheduled task in background: ' + worker_task.name + ': ' + worker_task.message) + WorkerThread.add(user, worker_task) + + self.add(func=scheduled_task, trigger=trigger, **trigger_args) diff --git a/cps/services/worker.py b/cps/services/worker.py index 072674a0..2b6816db 100644 --- a/cps/services/worker.py +++ b/cps/services/worker.py @@ -35,7 +35,6 @@ def _get_main_thread(): raise Exception("main thread not found?!") - class ImprovedQueue(queue.Queue): def to_list(self): """ @@ -45,7 +44,8 @@ class ImprovedQueue(queue.Queue): with self.mutex: return list(self.queue) -#Class for all worker tasks in the background + +# Class for all worker tasks in the background class WorkerThread(threading.Thread): _instance = None @@ -127,6 +127,10 @@ class WorkerThread(threading.Thread): # CalibreTask.start() should wrap all exceptions in it's own error handling item.task.start(self) + # remove self_cleanup tasks from list + if item.task.self_cleanup: + self.dequeued.remove(item) + self.queue.task_done() @@ -141,6 +145,7 @@ class CalibreTask: self.end_time = None self.message = message self.id = uuid.uuid4() + self.self_cleanup = False @abc.abstractmethod def run(self, worker_thread): @@ -209,6 +214,14 @@ class CalibreTask: # todo: throw error if outside of [0,1] self._progress = x + @property + def self_cleanup(self): + return self._self_cleanup + + @self_cleanup.setter + def self_cleanup(self, is_self_cleanup): + self._self_cleanup = is_self_cleanup + def _handleError(self, error_message): self.stat = STAT_FAIL self.progress = 1 diff --git a/cps/shelf.py b/cps/shelf.py index 5c6037ac..7b00c32b 100644 --- a/cps/shelf.py +++ b/cps/shelf.py @@ -403,7 +403,7 @@ def render_show_shelf(shelf_type, shelf_id, page_no, sort_param): db.Books, ub.BookShelf.shelf == shelf_id, [ub.BookShelf.order.asc()], - ub.BookShelf,ub.BookShelf.book_id == db.Books.id) + ub.BookShelf, ub.BookShelf.book_id == db.Books.id) # delete chelf entries where book is not existent anymore, can happen if book is deleted outside calibre-web wrong_entries = calibre_db.session.query(ub.BookShelf)\ .join(db.Books, ub.BookShelf.book_id == db.Books.id, isouter=True)\ diff --git a/cps/static/css/caliBlur.css b/cps/static/css/caliBlur.css index 14e5c286..aa747c0b 100644 --- a/cps/static/css/caliBlur.css +++ b/cps/static/css/caliBlur.css @@ -5148,7 +5148,7 @@ body.login > div.navbar.navbar-default.navbar-static-top > div > div.navbar-head pointer-events: none } -#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #deleteButton, #deleteModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover { +#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #ClearCacheDialog:hover:before, #deleteButton, #deleteModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover { cursor: pointer } @@ -5235,7 +5235,7 @@ body.admin > div.container-fluid > div > div.col-sm-10 > div.container-fluid > d margin-bottom: 20px } -body.admin:not(.modal-open) .btn-default { +body.admin .btn-default { margin-bottom: 10px } @@ -5466,7 +5466,7 @@ body.admin.modal-open .navbar { z-index: 0 !important } -#RestartDialog, #ShutdownDialog, #StatusDialog, #deleteModal { +#RestartDialog, #ShutdownDialog, #StatusDialog, #ClearCacheDialog, #deleteModal { top: 0; overflow: hidden; padding-top: 70px; @@ -5476,7 +5476,7 @@ body.admin.modal-open .navbar { background: rgba(0, 0, 0, .5) } -#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #deleteModal:before { +#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #ClearCacheDialog:before, #deleteModal:before { content: "\E208"; padding-right: 10px; display: block; @@ -5498,18 +5498,18 @@ body.admin.modal-open .navbar { z-index: 99 } -#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before { +#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #ClearCacheDialog.in:before, #deleteModal.in:before { -webkit-transform: translate(0, 0); -ms-transform: translate(0, 0); transform: translate(0, 0) } -#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog { +#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #ClearCacheDialog > .modal-dialog, #deleteModal > .modal-dialog { width: 450px; margin: auto } -#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { +#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #ClearCacheDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { max-height: calc(100% - 90px); -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, .5); box-shadow: 0 5px 15px rgba(0, 0, 0, .5); @@ -5520,7 +5520,7 @@ body.admin.modal-open .navbar { width: 450px } -#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header { +#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header { padding: 15px 20px; border-radius: 3px 3px 0 0; line-height: 1.71428571; @@ -5533,7 +5533,7 @@ body.admin.modal-open .navbar { text-align: left } -#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before { +#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before { padding-right: 10px; font-size: 18px; color: #999; @@ -5557,6 +5557,11 @@ body.admin.modal-open .navbar { font-family: plex-icons-new, serif } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:before { + content: "\EA15"; + font-family: plex-icons-new, serif +} + #deleteModal > .modal-dialog > .modal-content > .modal-header:before { content: "\EA6D"; font-family: plex-icons-new, serif @@ -5580,6 +5585,12 @@ body.admin.modal-open .navbar { font-size: 20px } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:after { + content: "Clear Cover Thumbnail Cache"; + display: inline-block; + font-size: 20px +} + #deleteModal > .modal-dialog > .modal-content > .modal-header:after { content: "Delete Book"; display: inline-block; @@ -5610,7 +5621,17 @@ body.admin.modal-open .navbar { text-align: left } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p { +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body { + padding: 20px 20px 10px; + font-size: 16px; + line-height: 1.6em; + font-family: Open Sans Regular, Helvetica Neue, Helvetica, Arial, sans-serif; + color: #eee; + background: #282828; + text-align: left +} + +#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p { padding: 20px 20px 0 0; font-size: 16px; line-height: 1.6em; @@ -5619,7 +5640,7 @@ body.admin.modal-open .navbar { background: #282828 } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { +#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { float: right; z-index: 9; position: relative; @@ -5655,6 +5676,18 @@ body.admin.modal-open .navbar { border-radius: 3px } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > #clear_cache { + float: right; + z-index: 9; + position: relative; + margin: 25px 0 0 10px; + min-width: 80px; + padding: 10px 18px; + font-size: 16px; + line-height: 1.33; + border-radius: 3px +} + #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-danger { float: right; z-index: 9; @@ -5675,11 +5708,15 @@ body.admin.modal-open .navbar { margin: 55px 0 0 10px } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache) { + margin: 25px 0 0 10px +} + #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { margin: 0 0 0 10px } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover { +#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover { background-color: hsla(0, 0%, 100%, .3) } @@ -5713,6 +5750,21 @@ body.admin.modal-open .navbar { box-shadow: 0 5px 15px rgba(0, 0, 0, .5) } +#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body:after { + content: ''; + position: absolute; + width: 100%; + height: 72px; + background-color: #323232; + border-radius: 0 0 3px 3px; + left: 0; + margin-top: 10px; + z-index: 0; + border-top: 1px solid #222; + -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, .5); + box-shadow: 0 5px 15px rgba(0, 0, 0, .5) +} + #deleteButton { position: fixed; top: 60px; @@ -7299,11 +7351,11 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. background-color: transparent !important } - #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog { + #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #ClearCacheDialog > .modal-dialog, #deleteModal > .modal-dialog { max-width: calc(100vw - 40px) } - #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { + #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #ClearCacheDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { max-width: calc(100vw - 40px); left: 0 } @@ -7453,7 +7505,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. padding: 30px 15px } - #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before { + #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #ClearCacheDialog.in:before, #deleteModal.in:before { left: auto; right: 34px } diff --git a/cps/static/js/main.js b/cps/static/js/main.js index 834b9b30..51d6095d 100644 --- a/cps/static/js/main.js +++ b/cps/static/js/main.js @@ -430,6 +430,18 @@ $(function() { } }); }); + $("#clear_cache").click(function () { + $("#spinner3").show(); + $.ajax({ + dataType: "json", + url: window.location.pathname + "/../../clear-cache", + data: {"cache_type":"thumbnails"}, + success: function(data) { + $("#spinner3").hide(); + $("#ClearCacheDialog").modal("hide"); + } + }); + }); // Init all data control handlers to default $("input[data-control]").trigger("change"); diff --git a/cps/tasks/database.py b/cps/tasks/database.py new file mode 100644 index 00000000..11f0186d --- /dev/null +++ b/cps/tasks/database.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals + +from cps import config, logger +from cps.services.worker import CalibreTask + +try: + from urllib.request import urlopen +except ImportError as e: + from urllib2 import urlopen + + +class TaskReconnectDatabase(CalibreTask): + def __init__(self, task_message=u'Reconnecting Calibre database'): + super(TaskReconnectDatabase, self).__init__(task_message) + self.log = logger.create() + self.listen_address = config.get_config_ipaddress() + self.listen_port = config.config_port + + def run(self, worker_thread): + address = self.listen_address if self.listen_address else 'localhost' + port = self.listen_port if self.listen_port else 8083 + + try: + urlopen('http://' + address + ':' + str(port) + '/reconnect') + self._handleSuccess() + except Exception as ex: + self._handleError(u'Unable to reconnect Calibre database: ' + str(ex)) + + @property + def name(self): + return "Reconnect Database" diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py new file mode 100644 index 00000000..70ddc06b --- /dev/null +++ b/cps/tasks/thumbnail.py @@ -0,0 +1,366 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import os + +from cps import config, db, fs, gdriveutils, logger, ub +from cps.services.worker import CalibreTask +from datetime import datetime, timedelta +from sqlalchemy import func + +try: + from urllib.request import urlopen +except ImportError as e: + from urllib2 import urlopen + +try: + from wand.image import Image + use_IM = True +except (ImportError, RuntimeError) as e: + use_IM = False + +THUMBNAIL_RESOLUTION_1X = 1 +THUMBNAIL_RESOLUTION_2X = 2 + + +class TaskGenerateCoverThumbnails(CalibreTask): + def __init__(self, limit=100, task_message=u'Generating cover thumbnails'): + super(TaskGenerateCoverThumbnails, self).__init__(task_message) + self.limit = limit + self.log = logger.create() + self.app_db_session = ub.get_new_session_instance() + self.calibre_db = db.CalibreDB(expire_on_commit=False) + self.cache = fs.FileSystem() + self.resolutions = [ + THUMBNAIL_RESOLUTION_1X, + THUMBNAIL_RESOLUTION_2X + ] + + def run(self, worker_thread): + if self.calibre_db.session and use_IM: + expired_thumbnails = self.get_expired_thumbnails() + thumbnail_book_ids = self.get_thumbnail_book_ids() + books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids) + + count = len(books_without_thumbnails) + if count == 0: + # Do not display this task on the frontend if there are no covers to update + self.self_cleanup = True + + for i, book in enumerate(books_without_thumbnails): + for resolution in self.resolutions: + expired_thumbnail = self.get_expired_thumbnail_for_book_and_resolution( + book, + resolution, + expired_thumbnails + ) + if expired_thumbnail: + self.update_book_thumbnail(book, expired_thumbnail) + else: + self.create_book_thumbnail(book, resolution) + + self.message = u'Generating cover thumbnail {0} of {1}'.format(i + 1, count) + self.progress = (1.0 / count) * i + + self._handleSuccess() + self.app_db_session.remove() + + def get_expired_thumbnails(self): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.expiration < datetime.utcnow())\ + .all() + + def get_thumbnail_book_ids(self): + return self.app_db_session\ + .query(ub.Thumbnail.book_id)\ + .group_by(ub.Thumbnail.book_id)\ + .having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\ + .distinct() + + def get_books_without_thumbnails(self, thumbnail_book_ids): + return self.calibre_db.session\ + .query(db.Books)\ + .filter(db.Books.has_cover == 1)\ + .filter(db.Books.id.notin_(thumbnail_book_ids))\ + .limit(self.limit)\ + .all() + + def get_expired_thumbnail_for_book_and_resolution(self, book, resolution, expired_thumbnails): + for thumbnail in expired_thumbnails: + if thumbnail.book_id == book.id and thumbnail.resolution == resolution: + return thumbnail + + return None + + def update_book_thumbnail(self, book, thumbnail): + thumbnail.generated_at = datetime.utcnow() + thumbnail.expiration = datetime.utcnow() + timedelta(days=30) + + try: + self.app_db_session.commit() + self.generate_book_thumbnail(book, thumbnail) + except Exception as ex: + self.log.info(u'Error updating book thumbnail: ' + str(ex)) + self._handleError(u'Error updating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def create_book_thumbnail(self, book, resolution): + thumbnail = ub.Thumbnail() + thumbnail.book_id = book.id + thumbnail.format = 'jpeg' + thumbnail.resolution = resolution + + self.app_db_session.add(thumbnail) + try: + self.app_db_session.commit() + self.generate_book_thumbnail(book, thumbnail) + except Exception as ex: + self.log.info(u'Error creating book thumbnail: ' + str(ex)) + self._handleError(u'Error creating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def generate_book_thumbnail(self, book, thumbnail): + if book and thumbnail: + if config.config_use_google_drive: + if not gdriveutils.is_gdrive_ready(): + raise Exception('Google Drive is configured but not ready') + + web_content_link = gdriveutils.get_cover_via_gdrive(book.path) + if not web_content_link: + raise Exception('Google Drive cover url not found') + + stream = None + try: + stream = urlopen(web_content_link) + with Image(file=stream) as img: + height = self.get_thumbnail_height(thumbnail) + if img.height > height: + width = self.get_thumbnail_width(height, img) + img.resize(width=width, height=height, filter='lanczos') + img.format = thumbnail.format + filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + img.save(filename=filename) + except Exception as ex: + # Bubble exception to calling function + self.log.info(u'Error generating thumbnail file: ' + str(ex)) + raise ex + finally: + stream.close() + else: + book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + if not os.path.isfile(book_cover_filepath): + raise Exception('Book cover file not found') + + with Image(filename=book_cover_filepath) as img: + height = self.get_thumbnail_height(thumbnail) + if img.height > height: + width = self.get_thumbnail_width(height, img) + img.resize(width=width, height=height, filter='lanczos') + img.format = thumbnail.format + filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + img.save(filename=filename) + + def get_thumbnail_height(self, thumbnail): + return int(225 * thumbnail.resolution) + + def get_thumbnail_width(self, height, img): + percent = (height / float(img.height)) + return int((float(img.width) * float(percent))) + + @property + def name(self): + return "ThumbnailsGenerate" + + +class TaskSyncCoverThumbnailCache(CalibreTask): + def __init__(self, task_message=u'Syncing cover thumbnail cache'): + super(TaskSyncCoverThumbnailCache, self).__init__(task_message) + self.log = logger.create() + self.app_db_session = ub.get_new_session_instance() + self.calibre_db = db.CalibreDB(expire_on_commit=False) + self.cache = fs.FileSystem() + + def run(self, worker_thread): + cached_thumbnail_files = self.cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS) + + # Expire thumbnails in the database if the cached file is missing + # This case will happen if a user deletes the cache dir or cached files + if self.app_db_session: + self.expire_missing_thumbnails(cached_thumbnail_files) + self.progress = 0.25 + + # Delete thumbnails in the database if the book has been removed + # This case will happen if a book is removed in Calibre and the metadata.db file is updated in the filesystem + if self.app_db_session and self.calibre_db: + book_ids = self.get_book_ids() + self.delete_thumbnails_for_missing_books(book_ids) + self.progress = 0.50 + + # Expire thumbnails in the database if their corresponding book has been updated since they were generated + # This case will happen if the book was updated externally + if self.app_db_session and self.cache: + books = self.get_books_updated_in_the_last_day() + book_ids = list(map(lambda b: b.id, books)) + thumbnails = self.get_thumbnails_for_updated_books(book_ids) + self.expire_thumbnails_for_updated_book(books, thumbnails) + self.progress = 0.75 + + # Delete extraneous cached thumbnail files + # This case will happen if a book was deleted and the thumbnail OR the metadata.db file was changed externally + if self.app_db_session: + db_thumbnail_files = self.get_thumbnail_filenames() + self.delete_extraneous_thumbnail_files(cached_thumbnail_files, db_thumbnail_files) + + self._handleSuccess() + self.app_db_session.remove() + + def expire_missing_thumbnails(self, filenames): + try: + self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.filename.notin_(filenames))\ + .update({"expiration": datetime.utcnow()}, synchronize_session=False) + self.app_db_session.commit() + except Exception as ex: + self.log.info(u'Error expiring thumbnails for missing cache files: ' + str(ex)) + self._handleError(u'Error expiring thumbnails for missing cache files: ' + str(ex)) + self.app_db_session.rollback() + + def get_book_ids(self): + results = self.calibre_db.session\ + .query(db.Books.id)\ + .filter(db.Books.has_cover == 1)\ + .distinct() + + return [value for value, in results] + + def delete_thumbnails_for_missing_books(self, book_ids): + try: + self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.notin_(book_ids))\ + .delete(synchronize_session=False) + self.app_db_session.commit() + except Exception as ex: + self.log.info(str(ex)) + self._handleError(u'Error deleting thumbnails for missing books: ' + str(ex)) + self.app_db_session.rollback() + + def get_thumbnail_filenames(self): + results = self.app_db_session\ + .query(ub.Thumbnail.filename)\ + .all() + + return [thumbnail for thumbnail, in results] + + def delete_extraneous_thumbnail_files(self, cached_thumbnail_files, db_thumbnail_files): + extraneous_files = list(set(cached_thumbnail_files).difference(db_thumbnail_files)) + for file in extraneous_files: + self.cache.delete_cache_file(file, fs.CACHE_TYPE_THUMBNAILS) + + def get_books_updated_in_the_last_day(self): + return self.calibre_db.session\ + .query(db.Books)\ + .filter(db.Books.has_cover == 1)\ + .filter(db.Books.last_modified > datetime.utcnow() - timedelta(days=1, hours=1))\ + .all() + + def get_thumbnails_for_updated_books(self, book_ids): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.in_(book_ids))\ + .all() + + def expire_thumbnails_for_updated_book(self, books, thumbnails): + thumbnail_ids = list() + for book in books: + for thumbnail in thumbnails: + if thumbnail.book_id == book.id and thumbnail.generated_at < book.last_modified: + thumbnail_ids.append(thumbnail.id) + + try: + self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.id.in_(thumbnail_ids)) \ + .update({"expiration": datetime.utcnow()}, synchronize_session=False) + self.app_db_session.commit() + except Exception as ex: + self.log.info(u'Error expiring thumbnails for updated books: ' + str(ex)) + self._handleError(u'Error expiring thumbnails for updated books: ' + str(ex)) + self.app_db_session.rollback() + + @property + def name(self): + return "ThumbnailsSync" + + +class TaskClearCoverThumbnailCache(CalibreTask): + def __init__(self, book_id=None, task_message=u'Clearing cover thumbnail cache'): + super(TaskClearCoverThumbnailCache, self).__init__(task_message) + self.log = logger.create() + self.book_id = book_id + self.app_db_session = ub.get_new_session_instance() + self.cache = fs.FileSystem() + + def run(self, worker_thread): + if self.app_db_session: + if self.book_id: + thumbnails = self.get_thumbnails_for_book(self.book_id) + for thumbnail in thumbnails: + self.expire_and_delete_thumbnail(thumbnail) + else: + self.expire_and_delete_all_thumbnails() + + self._handleSuccess() + self.app_db_session.remove() + + def get_thumbnails_for_book(self, book_id): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id == book_id)\ + .all() + + def expire_and_delete_thumbnail(self, thumbnail): + thumbnail.expiration = datetime.utcnow() + + try: + self.app_db_session.commit() + self.cache.delete_cache_file(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + except Exception as ex: + self.log.info(u'Error expiring book thumbnail: ' + str(ex)) + self._handleError(u'Error expiring book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def expire_and_delete_all_thumbnails(self): + self.app_db_session\ + .query(ub.Thumbnail)\ + .update({'expiration': datetime.utcnow()}) + + try: + self.app_db_session.commit() + self.cache.delete_cache_dir(fs.CACHE_TYPE_THUMBNAILS) + except Exception as ex: + self.log.info(u'Error expiring book thumbnails: ' + str(ex)) + self._handleError(u'Error expiring book thumbnails: ' + str(ex)) + self.app_db_session.rollback() + + @property + def name(self): + return "ThumbnailsClear" diff --git a/cps/templates/admin.html b/cps/templates/admin.html index 576652d4..20d0802c 100644 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -142,15 +142,18 @@
-
+

{{_('Administration')}}

- - -
-
-
{{_('Reconnect Calibre Database')}}
-
{{_('Restart')}}
-
{{_('Shutdown')}}
+ + +
+
+
{{_('Reconnect Calibre Database')}}
+
{{_('Clear Cover Thumbnail Cache')}}
+
+
+
{{_('Restart')}}
+
{{_('Shutdown')}}
@@ -231,4 +234,21 @@
+ {% endblock %} diff --git a/cps/templates/author.html b/cps/templates/author.html index 4e32db80..990f60ad 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -37,7 +37,7 @@
- + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html new file mode 100644 index 00000000..c5281797 --- /dev/null +++ b/cps/templates/book_cover.html @@ -0,0 +1,13 @@ +{% macro book_cover_image(book, thumbnails) -%} + {%- set book_title = book.title if book.title else book.name -%} + {% set srcset = thumbnails|get_book_thumbnail_srcset if thumbnails|length else '' %} + {%- if srcset|length -%} + {{ book_title }} + {%- else -%} + {{ book_title }} + {%- endif -%} +{%- endmacro %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index c0fc141e..091beffc 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -1,9 +1,10 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %} {% if book %}
- {{ book.title }} + {{ book_cover_image(book, book.id|get_book_thumbnails(thumbnails)) }}
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index 342cce53..3e2b65ac 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,7 +4,7 @@
- {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
diff --git a/cps/templates/discover.html b/cps/templates/discover.html index d57994b4..02eff658 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -9,7 +10,7 @@ {% if entry.has_cover is defined %} - {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/fragment.html b/cps/templates/fragment.html index 1421ea6a..901dd193 100644 --- a/cps/templates/fragment.html +++ b/cps/templates/fragment.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %}
{% block body %}{% endblock %}
diff --git a/cps/templates/grid.html b/cps/templates/grid.html index 9724e31d..2eecd6c6 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}

{{_(title)}}

@@ -29,7 +30,7 @@
- {{ entry[0].name }} + {{ book_cover_image(entry[0], entry[0].id|get_book_thumbnails(thumbnails)) }} {{entry.count}} diff --git a/cps/templates/index.html b/cps/templates/index.html index d300fc65..579f19f7 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %} {% if g.user.show_detail_random() %} @@ -9,7 +10,7 @@
- {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} @@ -86,7 +87,7 @@
- {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/layout.html b/cps/templates/layout.html index 318140fa..fef082bb 100644 --- a/cps/templates/layout.html +++ b/cps/templates/layout.html @@ -1,4 +1,5 @@ {% from 'modal_dialogs.html' import restrict_modal, delete_book, filechooser_modal, delete_confirm_modal, change_confirm_modal %} +{% from 'book_cover.html' import book_cover_image %} diff --git a/cps/templates/search.html b/cps/templates/search.html index d11f3ec8..428655ea 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -44,7 +45,7 @@ {% if entry.has_cover is defined %} - {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index 1ad79dbd..e49a008f 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -31,7 +32,7 @@
- {{ entry.title }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/ub.py b/cps/ub.py index ecf98679..af53e4f8 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -18,6 +18,7 @@ # along with this program. If not, see . from __future__ import division, print_function, unicode_literals +import atexit import os import sys import datetime @@ -441,6 +442,27 @@ class RemoteAuthToken(Base): return '' % self.id +def filename(context): + file_format = context.get_current_parameters()['format'] + if file_format == 'jpeg': + return context.get_current_parameters()['uuid'] + '.jpg' + else: + return context.get_current_parameters()['uuid'] + '.' + file_format + + +class Thumbnail(Base): + __tablename__ = 'thumbnail' + + id = Column(Integer, primary_key=True) + book_id = Column(Integer) + uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True) + format = Column(String, default='jpeg') + resolution = Column(SmallInteger, default=1) + filename = Column(String, default=filename) + generated_at = Column(DateTime, default=lambda: datetime.datetime.utcnow()) + expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30)) + + # Add missing tables during migration of database def add_missing_tables(engine, session): if not engine.dialect.has_table(engine.connect(), "book_read_link"): @@ -455,6 +477,8 @@ def add_missing_tables(engine, session): KoboStatistics.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "archived_book"): ArchivedBook.__table__.create(bind=engine) + if not engine.dialect.has_table(engine.connect(), "thumbnail"): + Thumbnail.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "registration"): Registration.__table__.create(bind=engine) with engine.connect() as conn: @@ -725,6 +749,16 @@ def init_db(app_db_path): sys.exit(3) +def get_new_session_instance(): + new_engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False) + new_session = scoped_session(sessionmaker()) + new_session.configure(bind=new_engine) + + atexit.register(lambda: new_session.remove() if new_session else True) + + return new_session + + def dispose(): global session diff --git a/cps/web.py b/cps/web.py index ad8cd5da..5bf226ab 100644 --- a/cps/web.py +++ b/cps/web.py @@ -50,9 +50,10 @@ from . import constants, logger, isoLanguages, services from . import babel, db, ub, config, get_locale, app from . import calibre_db from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download -from .helper import check_valid_domain, render_task_status, \ - get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \ - send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password +from .helper import check_valid_domain, render_task_status, get_cc_columns, get_book_cover, get_cached_book_cover, \ + get_cached_book_cover_thumbnail, get_thumbnails_for_books, get_thumbnails_for_book_series, get_download_link, \ + send_mail, generate_random_password, send_registration_mail, check_send_to_kindle, check_read_formats, \ + tags_filters, reset_password from .pagination import Pagination from .redirect import redirect_back from .usermanagement import login_required_if_no_ano @@ -411,8 +412,9 @@ def render_books_list(data, sort, book_id, page): else: website = data or "newest" entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, order) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=_(u"Books"), page=website) + title=_(u"Books"), page=website, thumbnails=thumbnails) def render_rated_books(page, book_id, order): @@ -457,8 +459,9 @@ def render_hot_books(page): ub.delete_download(book.Downloads.book_id) numBooks = entries.__len__() pagination = Pagination(page, config.config_books_per_page, numBooks) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=_(u"Hot Books (Most Downloaded)"), page="hot") + title=_(u"Hot Books (Most Downloaded)"), page="hot", thumbnails=thumbnails) else: abort(404) @@ -482,12 +485,14 @@ def render_downloaded_books(page, order): .filter(db.Books.id == book.id).first(): ub.delete_download(book.id) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, title=_(u"Downloaded books by %(user)s",user=current_user.nickname), - page="download") + page="download", + thumbnails=thumbnails) else: abort(404) @@ -498,6 +503,7 @@ def render_author_books(page, author_id, order): db.Books.authors.any(db.Authors.id == author_id), [order[0], db.Series.name, db.Books.series_index], db.books_series_link, + db.Books.id==db.books_series_link.c.book, db.Series) if entries is None or not len(entries): flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), @@ -513,9 +519,10 @@ def render_author_books(page, author_id, order): author_info = services.goodreads_support.get_author_info(author_name) other_books = services.goodreads_support.get_other_books(author_info, entries) + thumbnails = get_thumbnails_for_books(entries) return render_title_template('author.html', entries=entries, pagination=pagination, id=author_id, title=_(u"Author: %(name)s", name=author_name), author=author_info, - other_books=other_books, page="author") + other_books=other_books, page="author", thumbnails=thumbnails) def render_publisher_books(page, book_id, order): @@ -526,9 +533,12 @@ def render_publisher_books(page, book_id, order): db.Books.publishers.any(db.Publishers.id == book_id), [db.Series.name, order[0], db.Books.series_index], db.books_series_link, + db.Books.id == db.books_series_link.c.book, db.Series) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, - title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher") + title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher", + thumbnails=thumbnails) else: abort(404) @@ -540,8 +550,10 @@ def render_series_books(page, book_id, order): db.Books, db.Books.series.any(db.Series.id == book_id), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"Series: %(serie)s", serie=name.name), page="series") + title=_(u"Series: %(serie)s", serie=name.name), page="series", + thumbnails=thumbnails) else: abort(404) @@ -553,8 +565,10 @@ def render_ratings_books(page, book_id, order): db.Books.ratings.any(db.Ratings.id == book_id), [order[0]]) if name and name.rating <= 10: + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings") + title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings", + thumbnails=thumbnails) else: abort(404) @@ -566,8 +580,10 @@ def render_formats_books(page, book_id, order): db.Books, db.Books.data.any(db.Data.format == book_id.upper()), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"File format: %(format)s", format=name.format), page="formats") + title=_(u"File format: %(format)s", format=name.format), page="formats", + thumbnails=thumbnails) else: abort(404) @@ -579,9 +595,13 @@ def render_category_books(page, book_id, order): db.Books, db.Books.tags.any(db.Tags.id == book_id), [order[0], db.Series.name, db.Books.series_index], - db.books_series_link, db.Series) + db.books_series_link, + db.Books.id == db.books_series_link.c.book, + db.Series) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, - title=_(u"Category: %(name)s", name=name.name), page="category") + title=_(u"Category: %(name)s", name=name.name), page="category", + thumbnails=thumbnails) else: abort(404) @@ -599,8 +619,9 @@ def render_language_books(page, name, order): db.Books, db.Books.languages.any(db.Languages.lang_code == name), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name, - title=_(u"Language: %(name)s", name=lang_name), page="language") + title=_(u"Language: %(name)s", name=lang_name), page="language", thumbnails=thumbnails) def render_read_books(page, are_read, as_xml=False, order=None): @@ -644,8 +665,10 @@ def render_read_books(page, are_read, as_xml=False, order=None): else: name = _(u'Unread Books') + ' (' + str(pagination.total_count) + ')' pagename = "unread" + + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename) + title=name, page=pagename, thumbnails=thumbnails) def render_archived_books(page, order): @@ -668,8 +691,9 @@ def render_archived_books(page, order): name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')' pagename = "archived" + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename) + title=name, page=pagename, thumbnails=thumbnails) def render_prepare_search_form(cc): @@ -702,6 +726,7 @@ def render_prepare_search_form(cc): def render_search_results(term, offset=None, order=None, limit=None): entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit) + thumbnails = get_thumbnails_for_books(entries) return render_title_template('search.html', searchterm=term, pagination=pagination, @@ -710,7 +735,8 @@ def render_search_results(term, offset=None, order=None, limit=None): entries=entries, result_count=result_count, title=_(u"Search"), - page="search") + page="search", + thumbnails=thumbnails) # ################################### View Books list ################################################################## @@ -740,6 +766,7 @@ def books_table(): return render_title_template('book_table.html', title=_(u"Books List"), page="book_table", visiblility=visibility) + @web.route("/ajax/listbooks") @login_required def list_books(): @@ -772,6 +799,7 @@ def list_books(): response.headers["Content-Type"] = "application/json; charset=utf-8" return response + @web.route("/ajax/table_settings", methods=['POST']) @login_required def update_table_settings(): @@ -826,6 +854,7 @@ def publisher_list(): charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \ .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all() + return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, title=_(u"Publishers"), page="publisherlist", data="publisher") else: @@ -857,8 +886,10 @@ def series_list(): .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all() + thumbnails = get_thumbnails_for_book_series(entries) return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=charlist, - title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view") + title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view", + thumbnails=thumbnails) else: abort(404) @@ -1235,13 +1266,16 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): else: offset = 0 limit_all = result_count + + thumbnails = get_thumbnails_for_books(entries) return render_title_template('search.html', adv_searchterm=searchterm, pagination=pagination, entries=q[offset:limit_all], result_count=result_count, - title=_(u"Advanced Search"), page="advsearch") - + title=_(u"Advanced Search"), + page="advsearch", + thumbnails=thumbnails) @web.route("/advsearch", methods=['GET']) @@ -1260,10 +1294,24 @@ def advanced_search_form(): def get_cover(book_id): return get_book_cover(book_id) + +@web.route("/cached-cover/") +@login_required_if_no_ano +def get_cached_cover(cache_id): + return get_cached_book_cover(cache_id) + + +@web.route("/cached-cover-thumbnail/") +@login_required_if_no_ano +def get_cached_cover_thumbnail(cache_id): + return get_cached_book_cover_thumbnail(cache_id) + + @web.route("/robots.txt") def get_robots(): return send_from_directory(constants.STATIC_DIR, "robots.txt") + @web.route("/show//", defaults={'anyname': 'None'}) @web.route("/show///") @login_required_if_no_ano @@ -1293,7 +1341,6 @@ def serve_book(book_id, book_format, anyname): return send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + book_format) - @web.route("/download//", defaults={'anyname': 'None'}) @web.route("/download///") @login_required_if_no_ano @@ -1481,9 +1528,6 @@ def logout(): return redirect(url_for('web.login')) - - - # ################################### Users own configuration ######################################################### def change_profile_email(to_save, kobo_support, local_oauth_check, oauth_status): if "email" in to_save and to_save["email"] != current_user.email: @@ -1683,6 +1727,7 @@ def show_book(book_id): if media_format.format.lower() in constants.EXTENSIONS_AUDIO: audioentries.append(media_format.format.lower()) + thumbnails = get_thumbnails_for_books([entries]) return render_title_template('detail.html', entry=entries, audioentries=audioentries, @@ -1694,7 +1739,8 @@ def show_book(book_id): is_archived=is_archived, kindle_list=kindle_list, reader_list=reader_list, - page="book") + page="book", + thumbnails=thumbnails) else: log.debug(u"Error opening eBook. File does not exist or file is not accessible") flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error") diff --git a/requirements.txt b/requirements.txt index ca11eff9..5a26381e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +APScheduler>=3.6.3, <3.8.0 Babel>=1.3, <2.9 Flask-Babel>=0.11.1,<2.1.0 Flask-Login>=0.3.2,<0.5.1 From 1e7a2c400ba6c6a6b661c1a086d85a95cd813806 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 20 Mar 2021 14:06:15 +0100 Subject: [PATCH 011/112] Fixed misstyping --- cps/config_sql.py | 2 +- cps/db.py | 2 +- cps/gdriveutils.py | 2 +- cps/ub.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cps/config_sql.py b/cps/config_sql.py index 17f1c613..78e6c9c9 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -24,7 +24,7 @@ import sys from sqlalchemy import exc, Column, String, Integer, SmallInteger, Boolean, BLOB, JSON from sqlalchemy.exc import OperationalError try: - # Compability with sqlalchemy 2.0 + # Compatibility with sqlalchemy 2.0 from sqlalchemy.orm import declarative_base except ImportError: from sqlalchemy.ext.declarative import declarative_base diff --git a/cps/db.py b/cps/db.py index f43cc811..883eec05 100644 --- a/cps/db.py +++ b/cps/db.py @@ -32,7 +32,7 @@ from sqlalchemy.orm import relationship, sessionmaker, scoped_session from sqlalchemy.orm.collections import InstrumentedList from sqlalchemy.ext.declarative import DeclarativeMeta try: - # Compability with sqlalchemy 2.0 + # Compatibility with sqlalchemy 2.0 from sqlalchemy.orm import declarative_base except ImportError: from sqlalchemy.ext.declarative import declarative_base diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 5f970554..c7a95ca5 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -29,7 +29,7 @@ from sqlalchemy import Column, UniqueConstraint from sqlalchemy import String, Integer from sqlalchemy.orm import sessionmaker, scoped_session try: - # Compability with sqlalchemy 2.0 + # Compatibility with sqlalchemy 2.0 from sqlalchemy.orm import declarative_base except ImportError: from sqlalchemy.ext.declarative import declarative_base diff --git a/cps/ub.py b/cps/ub.py index af53e4f8..4a76ea49 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -45,7 +45,7 @@ from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.sql.expression import func try: - # Compability with sqlalchemy 2.0 + # Compatibility with sqlalchemy 2.0 from sqlalchemy.orm import declarative_base except ImportError: from sqlalchemy.ext.declarative import declarative_base From 9a08bcd2bc0a5280cbb293d1306b7c4e2ad6d792 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Thu, 16 Sep 2021 23:20:11 -0500 Subject: [PATCH 012/112] Started addressing some PR comments --- cps/admin.py | 1 - cps/templates/book_cover.html | 3 ++- cps/updater.py | 2 +- cps/web.py | 26 ++++++++++++++------------ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 32d66a6f..98535320 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -41,7 +41,6 @@ from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError from sqlalchemy.sql.expression import func, or_, text from . import constants, logger, helper, services, isoLanguages, fs -from .cli import filepicker from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ valid_email, check_username diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html index c5281797..67e7c24b 100644 --- a/cps/templates/book_cover.html +++ b/cps/templates/book_cover.html @@ -1,5 +1,6 @@ -{% macro book_cover_image(book, thumbnails) -%} +{% macro book_cover_image(book, thumbnails, title) -%} {%- set book_title = book.title if book.title else book.name -%} + {%- set book_title = title if title else book_title -%} {% set srcset = thumbnails|get_book_thumbnail_srcset if thumbnails|length else '' %} {%- if srcset|length -%} Date: Fri, 17 Sep 2021 01:42:56 -0500 Subject: [PATCH 013/112] Review feedback fixes --- cps/helper.py | 5 +++-- cps/jinjia.py | 2 ++ cps/schedule.py | 7 +++++-- cps/services/background_scheduler.py | 4 ++-- cps/tasks/thumbnail.py | 1 + cps/templates/book_cover.html | 2 +- cps/ub.py | 2 +- cps/web.py | 22 +++++++++++----------- 8 files changed, 26 insertions(+), 19 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 758d2531..6f22a701 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -554,13 +554,14 @@ def get_thumbnails_for_books(books): cache = fs.FileSystem() thumbnail_files = cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS) - return ub.session\ + thumbnails = ub.session\ .query(ub.Thumbnail)\ .filter(ub.Thumbnail.book_id.in_(book_ids))\ - .filter(ub.Thumbnail.filename.in_(thumbnail_files))\ .filter(ub.Thumbnail.expiration > datetime.utcnow())\ .all() + return list(filter(lambda t: t.filename in thumbnail_files, thumbnails)) + def get_thumbnails_for_book_series(series): books = list(map(lambda s: s[0], series)) diff --git a/cps/jinjia.py b/cps/jinjia.py index ac6d3d33..5f86478c 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -129,10 +129,12 @@ def formatseriesindex_filter(series_index): return series_index return 0 + @jinjia.app_template_filter('escapedlink') def escapedlink_filter(url, text): return "{}".format(url, escape(text)) + @jinjia.app_template_filter('uuidfilter') def uuidfilter(var): return uuid4() diff --git a/cps/schedule.py b/cps/schedule.py index 7ee43410..8ab9c732 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -18,6 +18,7 @@ from __future__ import division, print_function, unicode_literals +from datetime import datetime from .services.background_scheduler import BackgroundScheduler from .tasks.database import TaskReconnectDatabase from .tasks.thumbnail import TaskSyncCoverThumbnailCache, TaskGenerateCoverThumbnails @@ -26,8 +27,10 @@ from .tasks.thumbnail import TaskSyncCoverThumbnailCache, TaskGenerateCoverThumb def register_jobs(): scheduler = BackgroundScheduler() - # Generate 100 book cover thumbnails every 5 minutes - scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='cron', minute='*/5') + # Generate up to 1000 book covers daily + generate_thumbnails_task = scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=1000), + trigger='interval', days=1) + generate_thumbnails_task.modify(next_run_time=datetime.now()) # Cleanup book cover cache every 6 hours scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='cron', minute='15', hour='*/6') diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py index efa57379..122168e8 100644 --- a/cps/services/background_scheduler.py +++ b/cps/services/background_scheduler.py @@ -41,7 +41,7 @@ class BackgroundScheduler: return cls._instance def add(self, func, trigger, **trigger_args): - self.scheduler.add_job(func=func, trigger=trigger, **trigger_args) + return self.scheduler.add_job(func=func, trigger=trigger, **trigger_args) def add_task(self, user, task, trigger, **trigger_args): def scheduled_task(): @@ -49,4 +49,4 @@ class BackgroundScheduler: self.log.info('Running scheduled task in background: ' + worker_task.name + ': ' + worker_task.message) WorkerThread.add(user, worker_task) - self.add(func=scheduled_task, trigger=trigger, **trigger_args) + return self.add(func=scheduled_task, trigger=trigger, **trigger_args) diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index 70ddc06b..fed12e8b 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -37,6 +37,7 @@ except (ImportError, RuntimeError) as e: THUMBNAIL_RESOLUTION_1X = 1 THUMBNAIL_RESOLUTION_2X = 2 +THUMBNAIL_RESOLUTION_3X = 3 class TaskGenerateCoverThumbnails(CalibreTask): diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html index 67e7c24b..884ff4cd 100644 --- a/cps/templates/book_cover.html +++ b/cps/templates/book_cover.html @@ -1,4 +1,4 @@ -{% macro book_cover_image(book, thumbnails, title) -%} +{% macro book_cover_image(book, thumbnails, title=None) -%} {%- set book_title = book.title if book.title else book.name -%} {%- set book_title = title if title else book_title -%} {% set srcset = thumbnails|get_book_thumbnail_srcset if thumbnails|length else '' %} diff --git a/cps/ub.py b/cps/ub.py index 9d3a5845..e0bee4ef 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -532,7 +532,7 @@ class Thumbnail(Base): resolution = Column(SmallInteger, default=1) filename = Column(String, default=filename) generated_at = Column(DateTime, default=lambda: datetime.datetime.utcnow()) - expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30)) + expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=90)) # Add missing tables during migration of database diff --git a/cps/web.py b/cps/web.py index 64bdd8e4..f43bc6ad 100644 --- a/cps/web.py +++ b/cps/web.py @@ -416,7 +416,7 @@ def render_books_list(data, sort, book_id, page): db.Books.id == db.books_series_link.c.book, db.Series) - thumbnails = get_thumbnails_for_books(entries + random if random else entries) + thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, title=_(u"Books"), page=website, thumbnails=thumbnails) @@ -467,7 +467,7 @@ def render_hot_books(page): ub.delete_download(book.Downloads.book_id) numBooks = entries.__len__() pagination = Pagination(page, config.config_books_per_page, numBooks) - thumbnails = get_thumbnails_for_books(entries + random if random else entries) + thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, title=_(u"Hot Books (Most Downloaded)"), page="hot", thumbnails=thumbnails) else: @@ -498,7 +498,7 @@ def render_downloaded_books(page, order, user_id): ub.delete_download(book.id) user = ub.session.query(ub.User).filter(ub.User.id == user_id).first() - thumbnails = get_thumbnails_for_books(entries + random if random else entries) + thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, @@ -551,7 +551,7 @@ def render_publisher_books(page, book_id, order): db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) - thumbnails = get_thumbnails_for_books(entries + random if random else entries) + thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher", thumbnails=thumbnails) @@ -566,7 +566,7 @@ def render_series_books(page, book_id, order): db.Books, db.Books.series.any(db.Series.id == book_id), [order[0]]) - thumbnails = get_thumbnails_for_books(entries + random if random else entries) + thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"Series: %(serie)s", serie=name.name), page="series", thumbnails=thumbnails) @@ -581,7 +581,7 @@ def render_ratings_books(page, book_id, order): db.Books.ratings.any(db.Ratings.id == book_id), [order[0]]) if name and name.rating <= 10: - thumbnails = get_thumbnails_for_books(entries + random if random else entries) + thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings", thumbnails=thumbnails) @@ -596,7 +596,7 @@ def render_formats_books(page, book_id, order): db.Books, db.Books.data.any(db.Data.format == book_id.upper()), [order[0]]) - thumbnails = get_thumbnails_for_books(entries + random if random else entries) + thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"File format: %(format)s", format=name.format), page="formats", thumbnails=thumbnails) @@ -614,7 +614,7 @@ def render_category_books(page, book_id, order): db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) - thumbnails = get_thumbnails_for_books(entries + random if random else entries) + thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, title=_(u"Category: %(name)s", name=name.name), page="category", thumbnails=thumbnails) @@ -635,7 +635,7 @@ def render_language_books(page, name, order): db.Books, db.Books.languages.any(db.Languages.lang_code == name), [order[0]]) - thumbnails = get_thumbnails_for_books(entries + random if random else entries) + thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name, title=_(u"Language: %(name)s", name=lang_name), page="language", thumbnails=thumbnails) @@ -688,7 +688,7 @@ def render_read_books(page, are_read, as_xml=False, order=None): name = _(u'Unread Books') + ' (' + str(pagination.total_count) + ')' pagename = "unread" - thumbnails = get_thumbnails_for_books(entries + random if random else entries) + thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, title=name, page=pagename, thumbnails=thumbnails) @@ -713,7 +713,7 @@ def render_archived_books(page, order): name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')' pagename = "archived" - thumbnails = get_thumbnails_for_books(entries + random if random else entries) + thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, title=name, page=pagename, thumbnails=thumbnails) From 8bee2b9552f3ad2e0b3b04b1f3302d67e98b4614 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Sun, 19 Sep 2021 22:45:19 -0500 Subject: [PATCH 014/112] Added CACHE_DIR env variable, graceful handling when APScheduler is not installed --- cps/constants.py | 5 ++++- cps/fs.py | 8 ++++++++ cps/helper.py | 2 ++ cps/schedule.py | 15 +++++++-------- cps/services/background_scheduler.py | 26 +++++++++++++++++++------- 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/cps/constants.py b/cps/constants.py index 1e44796a..6a6b8f2b 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -40,7 +40,10 @@ else: STATIC_DIR = os.path.join(BASE_DIR, 'cps', 'static') TEMPLATES_DIR = os.path.join(BASE_DIR, 'cps', 'templates') TRANSLATIONS_DIR = os.path.join(BASE_DIR, 'cps', 'translations') -CACHE_DIR = os.path.join(BASE_DIR, 'cps', 'cache') + +# Cache dir - use CACHE_DIR environment variable, otherwise use the default directory: cps/cache +DEFAULT_CACHE_DIR = os.path.join(BASE_DIR, 'cps', 'cache') +CACHE_DIR = os.environ.get("CACHE_DIR", DEFAULT_CACHE_DIR) if HOME_CONFIG: home_dir = os.path.join(os.path.expanduser("~"),".calibre-web") diff --git a/cps/fs.py b/cps/fs.py index 699d5991..8528e39c 100644 --- a/cps/fs.py +++ b/cps/fs.py @@ -50,6 +50,14 @@ class FileSystem: path = self.get_cache_dir(cache_type) return [file for file in listdir(path) if isfile(join(path, file))] + def list_existing_cache_files(self, filenames, cache_type=None): + path = self.get_cache_dir(cache_type) + return [file for file in listdir(path) if isfile(join(path, file)) and file in filenames] + + def list_missing_cache_files(self, filenames, cache_type=None): + path = self.get_cache_dir(cache_type) + return [file for file in listdir(path) if isfile(join(path, file)) and file not in filenames] + def delete_cache_dir(self, cache_type=None): if not cache_type and isdir(self._cache_dir): rmtree(self._cache_dir) diff --git a/cps/helper.py b/cps/helper.py index 6f22a701..688bc615 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -405,6 +405,7 @@ def update_dir_structure_file(book_id, calibrepath, first_author, orignal_filepa src=new_path, dest=new_name, error=str(ex)) return False + def update_dir_structure_gdrive(book_id, first_author): error = False book = calibre_db.get_book(book_id) @@ -505,6 +506,7 @@ def uniq(inpt): output.append(x) return output + def check_email(email): email = valid_email(email) if ub.session.query(ub.User).filter(func.lower(ub.User.email) == email.lower()).first(): diff --git a/cps/schedule.py b/cps/schedule.py index 8ab9c732..649b65a3 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -27,13 +27,12 @@ from .tasks.thumbnail import TaskSyncCoverThumbnailCache, TaskGenerateCoverThumb def register_jobs(): scheduler = BackgroundScheduler() - # Generate up to 1000 book covers daily - generate_thumbnails_task = scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=1000), - trigger='interval', days=1) - generate_thumbnails_task.modify(next_run_time=datetime.now()) + if scheduler: + # Reconnect metadata.db once every 12 hours + scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='interval', hours=12) - # Cleanup book cover cache every 6 hours - scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='cron', minute='15', hour='*/6') + # Cleanup book cover cache once every 24 hours + scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='interval', days=1) - # Reconnect metadata.db every 4 hours - scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', minute='5', hour='*/4') + # Generate all missing book cover thumbnails once every 24 hours + scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(), trigger='interval', days=1) diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py index 122168e8..f2688b7f 100644 --- a/cps/services/background_scheduler.py +++ b/cps/services/background_scheduler.py @@ -21,13 +21,23 @@ import atexit from .. import logger from .worker import WorkerThread -from apscheduler.schedulers.background import BackgroundScheduler as BScheduler + +try: + from apscheduler.schedulers.background import BackgroundScheduler as BScheduler + use_APScheduler = True +except (ImportError, RuntimeError) as e: + use_APScheduler = False + log = logger.create() + log.info(f'APScheduler not found. Unable to schedule tasks.') class BackgroundScheduler: _instance = None def __new__(cls): + if not use_APScheduler: + return False + if cls._instance is None: cls._instance = super(BackgroundScheduler, cls).__new__(cls) @@ -41,12 +51,14 @@ class BackgroundScheduler: return cls._instance def add(self, func, trigger, **trigger_args): - return self.scheduler.add_job(func=func, trigger=trigger, **trigger_args) + if use_APScheduler: + return self.scheduler.add_job(func=func, trigger=trigger, **trigger_args) def add_task(self, user, task, trigger, **trigger_args): - def scheduled_task(): - worker_task = task() - self.log.info('Running scheduled task in background: ' + worker_task.name + ': ' + worker_task.message) - WorkerThread.add(user, worker_task) + if use_APScheduler: + def scheduled_task(): + worker_task = task() + self.log.info('Running scheduled task in background: ' + worker_task.name + ': ' + worker_task.message) + WorkerThread.add(user, worker_task) - return self.add(func=scheduled_task, trigger=trigger, **trigger_args) + return self.add(func=scheduled_task, trigger=trigger, **trigger_args) From 524ed07a6c3daf6c8e746a9b5e69f76742fe2301 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Tue, 21 Sep 2021 23:39:00 -0500 Subject: [PATCH 015/112] Handle read only access to cache dir gracefully. minor cleanup --- cps.py | 3 +- cps/fs.py | 44 ++++++++++++++++++++++------ cps/schedule.py | 12 +++++--- cps/services/background_scheduler.py | 4 +-- 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/cps.py b/cps.py index 82137bbc..fe006551 100755 --- a/cps.py +++ b/cps.py @@ -44,7 +44,7 @@ from cps.editbooks import editbook from cps.remotelogin import remotelogin from cps.search_metadata import meta from cps.error_handler import init_errorhandler -from cps.schedule import register_jobs +from cps.schedule import register_jobs, register_startup_jobs try: from cps.kobo import kobo, get_kobo_activated @@ -83,6 +83,7 @@ def main(): # Register scheduled jobs register_jobs() + # register_startup_jobs() success = web_server.start() sys.exit(0 if success else 1) diff --git a/cps/fs.py b/cps/fs.py index 8528e39c..4f835fa6 100644 --- a/cps/fs.py +++ b/cps/fs.py @@ -17,6 +17,7 @@ # along with this program. If not, see . from __future__ import division, print_function, unicode_literals +from . import logger from .constants import CACHE_DIR from os import listdir, makedirs, remove from os.path import isdir, isfile, join @@ -32,16 +33,26 @@ class FileSystem: def __new__(cls): if cls._instance is None: cls._instance = super(FileSystem, cls).__new__(cls) + cls.log = logger.create() return cls._instance def get_cache_dir(self, cache_type=None): if not isdir(self._cache_dir): - makedirs(self._cache_dir) + try: + makedirs(self._cache_dir) + except OSError: + self.log.info(f'Failed to create path {self._cache_dir} (Permission denied).') + return False - if cache_type and not isdir(join(self._cache_dir, cache_type)): - makedirs(join(self._cache_dir, cache_type)) + path = join(self._cache_dir, cache_type) + if cache_type and not isdir(path): + try: + makedirs(path) + except OSError: + self.log.info(f'Failed to create path {path} (Permission denied).') + return False - return join(self._cache_dir, cache_type) if cache_type else self._cache_dir + return path if cache_type else self._cache_dir def get_cache_file_path(self, filename, cache_type=None): return join(self.get_cache_dir(cache_type), filename) if filename else None @@ -60,10 +71,25 @@ class FileSystem: def delete_cache_dir(self, cache_type=None): if not cache_type and isdir(self._cache_dir): - rmtree(self._cache_dir) - if cache_type and isdir(join(self._cache_dir, cache_type)): - rmtree(join(self._cache_dir, cache_type)) + try: + rmtree(self._cache_dir) + except OSError: + self.log.info(f'Failed to delete path {self._cache_dir} (Permission denied).') + return False + + path = join(self._cache_dir, cache_type) + if cache_type and isdir(path): + try: + rmtree(path) + except OSError: + self.log.info(f'Failed to delete path {path} (Permission denied).') + return False def delete_cache_file(self, filename, cache_type=None): - if isfile(join(self.get_cache_dir(cache_type), filename)): - remove(join(self.get_cache_dir(cache_type), filename)) + path = join(self.get_cache_dir(cache_type), filename) + if isfile(path): + try: + remove(path) + except OSError: + self.log.info(f'Failed to delete path {path} (Permission denied).') + return False diff --git a/cps/schedule.py b/cps/schedule.py index 649b65a3..d427d908 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -18,8 +18,8 @@ from __future__ import division, print_function, unicode_literals -from datetime import datetime from .services.background_scheduler import BackgroundScheduler +from .services.worker import WorkerThread from .tasks.database import TaskReconnectDatabase from .tasks.thumbnail import TaskSyncCoverThumbnailCache, TaskGenerateCoverThumbnails @@ -29,10 +29,14 @@ def register_jobs(): if scheduler: # Reconnect metadata.db once every 12 hours - scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='interval', hours=12) + scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', hour='4,16') # Cleanup book cover cache once every 24 hours - scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='interval', days=1) + scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='cron', hour=4) # Generate all missing book cover thumbnails once every 24 hours - scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(), trigger='interval', days=1) + scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(), trigger='cron', hour=4) + + +def register_startup_jobs(): + WorkerThread.add(None, TaskGenerateCoverThumbnails()) diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py index f2688b7f..1b588ac2 100644 --- a/cps/services/background_scheduler.py +++ b/cps/services/background_scheduler.py @@ -28,7 +28,7 @@ try: except (ImportError, RuntimeError) as e: use_APScheduler = False log = logger.create() - log.info(f'APScheduler not found. Unable to schedule tasks.') + log.info('APScheduler not found. Unable to schedule tasks.') class BackgroundScheduler: @@ -58,7 +58,7 @@ class BackgroundScheduler: if use_APScheduler: def scheduled_task(): worker_task = task() - self.log.info('Running scheduled task in background: ' + worker_task.name + ': ' + worker_task.message) + self.log.info(f'Running scheduled task in background: {worker_task.name} - {worker_task.message}') WorkerThread.add(user, worker_task) return self.add(func=scheduled_task, trigger=trigger, **trigger_args) From be28a9131553b7f29df6c712096530bdb62ae122 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Fri, 24 Sep 2021 03:11:14 -0500 Subject: [PATCH 016/112] Simplified all of the thumbnail generation and loading. --- cps/fs.py | 14 +- cps/helper.py | 61 +-------- cps/jinjia.py | 26 ++-- cps/schedule.py | 5 +- cps/tasks/thumbnail.py | 250 +++++++--------------------------- cps/templates/author.html | 3 +- cps/templates/book_cover.html | 19 ++- cps/templates/book_edit.html | 3 +- cps/templates/detail.html | 3 +- cps/templates/discover.html | 3 +- cps/templates/grid.html | 3 +- cps/templates/index.html | 6 +- cps/templates/search.html | 3 +- cps/templates/shelf.html | 3 +- cps/ub.py | 2 +- cps/web.py | 83 +++-------- 16 files changed, 111 insertions(+), 376 deletions(-) diff --git a/cps/fs.py b/cps/fs.py index 4f835fa6..30ab552a 100644 --- a/cps/fs.py +++ b/cps/fs.py @@ -57,17 +57,9 @@ class FileSystem: def get_cache_file_path(self, filename, cache_type=None): return join(self.get_cache_dir(cache_type), filename) if filename else None - def list_cache_files(self, cache_type=None): - path = self.get_cache_dir(cache_type) - return [file for file in listdir(path) if isfile(join(path, file))] - - def list_existing_cache_files(self, filenames, cache_type=None): - path = self.get_cache_dir(cache_type) - return [file for file in listdir(path) if isfile(join(path, file)) and file in filenames] - - def list_missing_cache_files(self, filenames, cache_type=None): - path = self.get_cache_dir(cache_type) - return [file for file in listdir(path) if isfile(join(path, file)) and file not in filenames] + def get_cache_file_exists(self, filename, cache_type=None): + path = self.get_cache_file_path(filename, cache_type) + return isfile(path) def delete_cache_dir(self, cache_type=None): if not cache_type and isdir(self._cache_dir): diff --git a/cps/helper.py b/cps/helper.py index 688bc615..1652be23 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -35,7 +35,7 @@ from babel.units import format_unit from flask import send_from_directory, make_response, redirect, abort, url_for from flask_babel import gettext as _ from flask_login import current_user -from sqlalchemy.sql.expression import true, false, and_, text, func +from sqlalchemy.sql.expression import true, false, and_, or_, text, func from werkzeug.datastructures import Headers from werkzeug.security import generate_password_hash from markupsafe import escape @@ -550,26 +550,6 @@ def delete_book(book, calibrepath, book_format): return delete_book_file(book, calibrepath, book_format) -def get_thumbnails_for_books(books): - books_with_covers = list(filter(lambda b: b.has_cover, books)) - book_ids = list(map(lambda b: b.id, books_with_covers)) - cache = fs.FileSystem() - thumbnail_files = cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS) - - thumbnails = ub.session\ - .query(ub.Thumbnail)\ - .filter(ub.Thumbnail.book_id.in_(book_ids))\ - .filter(ub.Thumbnail.expiration > datetime.utcnow())\ - .all() - - return list(filter(lambda t: t.filename in thumbnail_files, thumbnails)) - - -def get_thumbnails_for_book_series(series): - books = list(map(lambda s: s[0], series)) - return get_thumbnails_for_books(books) - - def get_cover_on_failure(use_generic_cover): if use_generic_cover: return send_from_directory(_STATIC_DIR, "generic_cover.jpg") @@ -577,9 +557,9 @@ def get_cover_on_failure(use_generic_cover): return None -def get_book_cover(book_id): +def get_book_cover(book_id, resolution=None): book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) - return get_book_cover_internal(book, use_generic_cover_on_failure=True) + return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True): @@ -587,37 +567,6 @@ def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True): return get_book_cover_internal(book, use_generic_cover_on_failure) -def get_cached_book_cover(cache_id): - parts = cache_id.split('_') - book_uuid = parts[0] if len(parts) else None - resolution = parts[2] if len(parts) > 2 else None - book = calibre_db.get_book_by_uuid(book_uuid) if book_uuid else None - return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) - - -def get_cached_book_cover_thumbnail(cache_id): - parts = cache_id.split('_') - thumbnail_uuid = parts[0] if len(parts) else None - thumbnail = None - if thumbnail_uuid: - thumbnail = ub.session\ - .query(ub.Thumbnail)\ - .filter(ub.Thumbnail.uuid == thumbnail_uuid)\ - .first() - - if thumbnail and thumbnail.expiration > datetime.utcnow(): - cache = fs.FileSystem() - if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): - return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename) - - elif thumbnail: - book = calibre_db.get_book(thumbnail.book_id) - return get_book_cover_internal(book, use_generic_cover_on_failure=True) - - else: - return get_cover_on_failure(True) - - def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None): if book and book.has_cover: @@ -626,7 +575,7 @@ def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None) thumbnail = get_book_cover_thumbnail(book, resolution) if thumbnail: cache = fs.FileSystem() - if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): + if cache.get_cache_file_exists(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename) # Send the book cover from Google Drive if configured @@ -661,7 +610,7 @@ def get_book_cover_thumbnail(book, resolution): .query(ub.Thumbnail)\ .filter(ub.Thumbnail.book_id == book.id)\ .filter(ub.Thumbnail.resolution == resolution)\ - .filter(ub.Thumbnail.expiration > datetime.utcnow())\ + .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow()))\ .first() diff --git a/cps/jinjia.py b/cps/jinjia.py index 5f86478c..8c8b72a9 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -33,6 +33,7 @@ from flask_babel import get_locale from flask_login import current_user from markupsafe import escape from . import logger +from .tasks.thumbnail import THUMBNAIL_RESOLUTION_1X, THUMBNAIL_RESOLUTION_2X, THUMBNAIL_RESOLUTION_3X jinjia = Blueprint('jinjia', __name__) @@ -140,24 +141,17 @@ def uuidfilter(var): return uuid4() -@jinjia.app_template_filter('book_cover_cache_id') -def book_cover_cache_id(book, resolution=None): +@jinjia.app_template_filter('last_modified') +def book_cover_cache_id(book): timestamp = int(book.last_modified.timestamp() * 1000) - cache_bust = str(book.uuid) + '_' + str(timestamp) - return cache_bust if not resolution else cache_bust + '_' + str(resolution) + return str(timestamp) -@jinjia.app_template_filter('get_book_thumbnails') -def get_book_thumbnails(book_id, thumbnails=None): - return list(filter(lambda t: t.book_id == book_id, thumbnails)) if book_id > -1 and thumbnails else list() - - -@jinjia.app_template_filter('get_book_thumbnail_srcset') -def get_book_thumbnail_srcset(thumbnails): +@jinjia.app_template_filter('get_cover_srcset') +def get_cover_srcset(book): srcset = list() - for thumbnail in thumbnails: - timestamp = int(thumbnail.generated_at.timestamp() * 1000) - cache_id = str(thumbnail.uuid) + '_' + str(timestamp) - url = url_for('web.get_cached_cover_thumbnail', cache_id=cache_id) - srcset.append(url + ' ' + str(thumbnail.resolution) + 'x') + for resolution in [THUMBNAIL_RESOLUTION_1X, THUMBNAIL_RESOLUTION_2X, THUMBNAIL_RESOLUTION_3X]: + timestamp = int(book.last_modified.timestamp() * 1000) + url = url_for('web.get_cover', book_id=book.id, resolution=resolution, cache_bust=str(timestamp)) + srcset.append(f'{url} {resolution}x') return ', '.join(srcset) diff --git a/cps/schedule.py b/cps/schedule.py index d427d908..8c350bd5 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -21,7 +21,7 @@ from __future__ import division, print_function, unicode_literals from .services.background_scheduler import BackgroundScheduler from .services.worker import WorkerThread from .tasks.database import TaskReconnectDatabase -from .tasks.thumbnail import TaskSyncCoverThumbnailCache, TaskGenerateCoverThumbnails +from .tasks.thumbnail import TaskGenerateCoverThumbnails def register_jobs(): @@ -31,9 +31,6 @@ def register_jobs(): # Reconnect metadata.db once every 12 hours scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', hour='4,16') - # Cleanup book cover cache once every 24 hours - scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='cron', hour=4) - # Generate all missing book cover thumbnails once every 24 hours scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(), trigger='cron', hour=4) diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index fed12e8b..688f6e4b 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -22,7 +22,7 @@ import os from cps import config, db, fs, gdriveutils, logger, ub from cps.services.worker import CalibreTask from datetime import datetime, timedelta -from sqlalchemy import func +from sqlalchemy import or_ try: from urllib.request import urlopen @@ -41,9 +41,8 @@ THUMBNAIL_RESOLUTION_3X = 3 class TaskGenerateCoverThumbnails(CalibreTask): - def __init__(self, limit=100, task_message=u'Generating cover thumbnails'): + def __init__(self, task_message=u'Generating cover thumbnails'): super(TaskGenerateCoverThumbnails, self).__init__(task_message) - self.limit = limit self.log = logger.create() self.app_db_session = ub.get_new_session_instance() self.calibre_db = db.CalibreDB(expire_on_commit=False) @@ -55,74 +54,51 @@ class TaskGenerateCoverThumbnails(CalibreTask): def run(self, worker_thread): if self.calibre_db.session and use_IM: - expired_thumbnails = self.get_expired_thumbnails() - thumbnail_book_ids = self.get_thumbnail_book_ids() - books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids) + books_with_covers = self.get_books_with_covers() + count = len(books_with_covers) - count = len(books_without_thumbnails) - if count == 0: - # Do not display this task on the frontend if there are no covers to update - self.self_cleanup = True + updated = 0 + generated = 0 + for i, book in enumerate(books_with_covers): + book_cover_thumbnails = self.get_book_cover_thumbnails(book.id) - for i, book in enumerate(books_without_thumbnails): - for resolution in self.resolutions: - expired_thumbnail = self.get_expired_thumbnail_for_book_and_resolution( - book, - resolution, - expired_thumbnails - ) - if expired_thumbnail: - self.update_book_thumbnail(book, expired_thumbnail) - else: - self.create_book_thumbnail(book, resolution) + # Generate new thumbnails for missing covers + resolutions = list(map(lambda t: t.resolution, book_cover_thumbnails)) + missing_resolutions = list(set(self.resolutions).difference(resolutions)) + for resolution in missing_resolutions: + generated += 1 + self.create_book_cover_thumbnail(book, resolution) - self.message = u'Generating cover thumbnail {0} of {1}'.format(i + 1, count) + # Replace outdated or missing thumbnails + for thumbnail in book_cover_thumbnails: + if book.last_modified > thumbnail.generated_at: + updated += 1 + self.update_book_cover_thumbnail(book, thumbnail) + + elif not self.cache.get_cache_file_exists(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): + updated += 1 + self.update_book_cover_thumbnail(book, thumbnail) + + self.message = u'Processing book {0} of {1}'.format(i + 1, count) self.progress = (1.0 / count) * i self._handleSuccess() self.app_db_session.remove() - def get_expired_thumbnails(self): - return self.app_db_session\ - .query(ub.Thumbnail)\ - .filter(ub.Thumbnail.expiration < datetime.utcnow())\ - .all() - - def get_thumbnail_book_ids(self): - return self.app_db_session\ - .query(ub.Thumbnail.book_id)\ - .group_by(ub.Thumbnail.book_id)\ - .having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\ - .distinct() - - def get_books_without_thumbnails(self, thumbnail_book_ids): + def get_books_with_covers(self): return self.calibre_db.session\ .query(db.Books)\ .filter(db.Books.has_cover == 1)\ - .filter(db.Books.id.notin_(thumbnail_book_ids))\ - .limit(self.limit)\ .all() - def get_expired_thumbnail_for_book_and_resolution(self, book, resolution, expired_thumbnails): - for thumbnail in expired_thumbnails: - if thumbnail.book_id == book.id and thumbnail.resolution == resolution: - return thumbnail + def get_book_cover_thumbnails(self, book_id): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id == book_id)\ + .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow()))\ + .all() - return None - - def update_book_thumbnail(self, book, thumbnail): - thumbnail.generated_at = datetime.utcnow() - thumbnail.expiration = datetime.utcnow() + timedelta(days=30) - - try: - self.app_db_session.commit() - self.generate_book_thumbnail(book, thumbnail) - except Exception as ex: - self.log.info(u'Error updating book thumbnail: ' + str(ex)) - self._handleError(u'Error updating book thumbnail: ' + str(ex)) - self.app_db_session.rollback() - - def create_book_thumbnail(self, book, resolution): + def create_book_cover_thumbnail(self, book, resolution): thumbnail = ub.Thumbnail() thumbnail.book_id = book.id thumbnail.format = 'jpeg' @@ -137,6 +113,18 @@ class TaskGenerateCoverThumbnails(CalibreTask): self._handleError(u'Error creating book thumbnail: ' + str(ex)) self.app_db_session.rollback() + def update_book_cover_thumbnail(self, book, thumbnail): + thumbnail.generated_at = datetime.utcnow() + + try: + self.app_db_session.commit() + self.cache.delete_cache_file(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + self.generate_book_thumbnail(book, thumbnail) + except Exception as ex: + self.log.info(u'Error updating book thumbnail: ' + str(ex)) + self._handleError(u'Error updating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + def generate_book_thumbnail(self, book, thumbnail): if book and thumbnail: if config.config_use_google_drive: @@ -190,128 +178,6 @@ class TaskGenerateCoverThumbnails(CalibreTask): return "ThumbnailsGenerate" -class TaskSyncCoverThumbnailCache(CalibreTask): - def __init__(self, task_message=u'Syncing cover thumbnail cache'): - super(TaskSyncCoverThumbnailCache, self).__init__(task_message) - self.log = logger.create() - self.app_db_session = ub.get_new_session_instance() - self.calibre_db = db.CalibreDB(expire_on_commit=False) - self.cache = fs.FileSystem() - - def run(self, worker_thread): - cached_thumbnail_files = self.cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS) - - # Expire thumbnails in the database if the cached file is missing - # This case will happen if a user deletes the cache dir or cached files - if self.app_db_session: - self.expire_missing_thumbnails(cached_thumbnail_files) - self.progress = 0.25 - - # Delete thumbnails in the database if the book has been removed - # This case will happen if a book is removed in Calibre and the metadata.db file is updated in the filesystem - if self.app_db_session and self.calibre_db: - book_ids = self.get_book_ids() - self.delete_thumbnails_for_missing_books(book_ids) - self.progress = 0.50 - - # Expire thumbnails in the database if their corresponding book has been updated since they were generated - # This case will happen if the book was updated externally - if self.app_db_session and self.cache: - books = self.get_books_updated_in_the_last_day() - book_ids = list(map(lambda b: b.id, books)) - thumbnails = self.get_thumbnails_for_updated_books(book_ids) - self.expire_thumbnails_for_updated_book(books, thumbnails) - self.progress = 0.75 - - # Delete extraneous cached thumbnail files - # This case will happen if a book was deleted and the thumbnail OR the metadata.db file was changed externally - if self.app_db_session: - db_thumbnail_files = self.get_thumbnail_filenames() - self.delete_extraneous_thumbnail_files(cached_thumbnail_files, db_thumbnail_files) - - self._handleSuccess() - self.app_db_session.remove() - - def expire_missing_thumbnails(self, filenames): - try: - self.app_db_session\ - .query(ub.Thumbnail)\ - .filter(ub.Thumbnail.filename.notin_(filenames))\ - .update({"expiration": datetime.utcnow()}, synchronize_session=False) - self.app_db_session.commit() - except Exception as ex: - self.log.info(u'Error expiring thumbnails for missing cache files: ' + str(ex)) - self._handleError(u'Error expiring thumbnails for missing cache files: ' + str(ex)) - self.app_db_session.rollback() - - def get_book_ids(self): - results = self.calibre_db.session\ - .query(db.Books.id)\ - .filter(db.Books.has_cover == 1)\ - .distinct() - - return [value for value, in results] - - def delete_thumbnails_for_missing_books(self, book_ids): - try: - self.app_db_session\ - .query(ub.Thumbnail)\ - .filter(ub.Thumbnail.book_id.notin_(book_ids))\ - .delete(synchronize_session=False) - self.app_db_session.commit() - except Exception as ex: - self.log.info(str(ex)) - self._handleError(u'Error deleting thumbnails for missing books: ' + str(ex)) - self.app_db_session.rollback() - - def get_thumbnail_filenames(self): - results = self.app_db_session\ - .query(ub.Thumbnail.filename)\ - .all() - - return [thumbnail for thumbnail, in results] - - def delete_extraneous_thumbnail_files(self, cached_thumbnail_files, db_thumbnail_files): - extraneous_files = list(set(cached_thumbnail_files).difference(db_thumbnail_files)) - for file in extraneous_files: - self.cache.delete_cache_file(file, fs.CACHE_TYPE_THUMBNAILS) - - def get_books_updated_in_the_last_day(self): - return self.calibre_db.session\ - .query(db.Books)\ - .filter(db.Books.has_cover == 1)\ - .filter(db.Books.last_modified > datetime.utcnow() - timedelta(days=1, hours=1))\ - .all() - - def get_thumbnails_for_updated_books(self, book_ids): - return self.app_db_session\ - .query(ub.Thumbnail)\ - .filter(ub.Thumbnail.book_id.in_(book_ids))\ - .all() - - def expire_thumbnails_for_updated_book(self, books, thumbnails): - thumbnail_ids = list() - for book in books: - for thumbnail in thumbnails: - if thumbnail.book_id == book.id and thumbnail.generated_at < book.last_modified: - thumbnail_ids.append(thumbnail.id) - - try: - self.app_db_session\ - .query(ub.Thumbnail)\ - .filter(ub.Thumbnail.id.in_(thumbnail_ids)) \ - .update({"expiration": datetime.utcnow()}, synchronize_session=False) - self.app_db_session.commit() - except Exception as ex: - self.log.info(u'Error expiring thumbnails for updated books: ' + str(ex)) - self._handleError(u'Error expiring thumbnails for updated books: ' + str(ex)) - self.app_db_session.rollback() - - @property - def name(self): - return "ThumbnailsSync" - - class TaskClearCoverThumbnailCache(CalibreTask): def __init__(self, book_id=None, task_message=u'Clearing cover thumbnail cache'): super(TaskClearCoverThumbnailCache, self).__init__(task_message) @@ -325,9 +191,9 @@ class TaskClearCoverThumbnailCache(CalibreTask): if self.book_id: thumbnails = self.get_thumbnails_for_book(self.book_id) for thumbnail in thumbnails: - self.expire_and_delete_thumbnail(thumbnail) + self.delete_thumbnail(thumbnail) else: - self.expire_and_delete_all_thumbnails() + self.delete_all_thumbnails() self._handleSuccess() self.app_db_session.remove() @@ -338,29 +204,19 @@ class TaskClearCoverThumbnailCache(CalibreTask): .filter(ub.Thumbnail.book_id == book_id)\ .all() - def expire_and_delete_thumbnail(self, thumbnail): - thumbnail.expiration = datetime.utcnow() - + def delete_thumbnail(self, thumbnail): try: - self.app_db_session.commit() self.cache.delete_cache_file(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) except Exception as ex: - self.log.info(u'Error expiring book thumbnail: ' + str(ex)) - self._handleError(u'Error expiring book thumbnail: ' + str(ex)) - self.app_db_session.rollback() - - def expire_and_delete_all_thumbnails(self): - self.app_db_session\ - .query(ub.Thumbnail)\ - .update({'expiration': datetime.utcnow()}) + self.log.info(u'Error deleting book thumbnail: ' + str(ex)) + self._handleError(u'Error deleting book thumbnail: ' + str(ex)) + def delete_all_thumbnails(self): try: - self.app_db_session.commit() self.cache.delete_cache_dir(fs.CACHE_TYPE_THUMBNAILS) except Exception as ex: - self.log.info(u'Error expiring book thumbnails: ' + str(ex)) - self._handleError(u'Error expiring book thumbnails: ' + str(ex)) - self.app_db_session.rollback() + self.log.info(u'Error deleting book thumbnails: ' + str(ex)) + self._handleError(u'Error deleting book thumbnails: ' + str(ex)) @property def name(self): diff --git a/cps/templates/author.html b/cps/templates/author.html index 3a4e6c57..2ff1ce7a 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -37,8 +37,7 @@
- {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} - + {{ book_cover_image(entry, title=author.name|safe) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html index 884ff4cd..7d3a84f4 100644 --- a/cps/templates/book_cover.html +++ b/cps/templates/book_cover.html @@ -1,14 +1,11 @@ -{% macro book_cover_image(book, thumbnails, title=None) -%} +{% macro book_cover_image(book, title=None) -%} {%- set book_title = book.title if book.title else book.name -%} {%- set book_title = title if title else book_title -%} - {% set srcset = thumbnails|get_book_thumbnail_srcset if thumbnails|length else '' %} - {%- if srcset|length -%} - {{ book_title }} - {%- else -%} - {{ book_title }} - {%- endif -%} + {% set srcset = book|get_cover_srcset %} + {{ book_title }} {%- endmacro %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index bce329d0..8525b8e3 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -4,8 +4,7 @@ {% if book %}
- {{ book_cover_image(book, book.id|get_book_thumbnails(thumbnails)) }} - + {{ book_cover_image(book) }}
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index ea870d4c..b538ff9a 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,8 +4,7 @@
- {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} - + {{ book_cover_image(entry) }}
diff --git a/cps/templates/discover.html b/cps/templates/discover.html index ecfc83d0..01cd393d 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -10,8 +10,7 @@ {% if entry.has_cover is defined %} - {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} - + {{ book_cover_image(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/grid.html b/cps/templates/grid.html index 5a3c1e01..45fb53bf 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -29,8 +29,7 @@
- {{ book_cover_image(entry[0], entry[0].id|get_book_thumbnails(thumbnails)) }} - + {{ book_cover_image(entry[0], title=entry[0].series[0].name|shortentitle) }} {{entry.count}} diff --git a/cps/templates/index.html b/cps/templates/index.html index c38ba01e..e47c6c70 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -10,8 +10,7 @@
- {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} - + {{ book_cover_image(entry) }} {% if entry.id in read_book_ids %}{% endif %} @@ -88,8 +87,7 @@
- {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} - + {{ book_cover_image(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/search.html b/cps/templates/search.html index cf9c4fe7..872b6dc1 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -45,8 +45,7 @@ {% if entry.has_cover is defined %} - {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} - + {{ book_cover_image(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index cb08be90..0e5865c9 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -32,8 +32,7 @@
- {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }} - + {{ book_cover_image(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/ub.py b/cps/ub.py index e0bee4ef..f108a1ce 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -532,7 +532,7 @@ class Thumbnail(Base): resolution = Column(SmallInteger, default=1) filename = Column(String, default=filename) generated_at = Column(DateTime, default=lambda: datetime.datetime.utcnow()) - expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=90)) + expiration = Column(DateTime, nullable=True) # Add missing tables during migration of database diff --git a/cps/web.py b/cps/web.py index f43bc6ad..8f78ff32 100644 --- a/cps/web.py +++ b/cps/web.py @@ -51,7 +51,6 @@ from . import babel, db, ub, config, get_locale, app from . import calibre_db from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download from .helper import check_valid_domain, render_task_status, check_email, check_username, \ - get_cached_book_cover, get_cached_book_cover_thumbnail, get_thumbnails_for_books, get_thumbnails_for_book_series, \ get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \ send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password, valid_email from .pagination import Pagination @@ -415,10 +414,8 @@ def render_books_list(data, sort, book_id, page): db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) - - thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=_(u"Books"), page=website, thumbnails=thumbnails) + title=_(u"Books"), page=website) def render_rated_books(page, book_id, order): @@ -467,9 +464,8 @@ def render_hot_books(page): ub.delete_download(book.Downloads.book_id) numBooks = entries.__len__() pagination = Pagination(page, config.config_books_per_page, numBooks) - thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=_(u"Hot Books (Most Downloaded)"), page="hot", thumbnails=thumbnails) + title=_(u"Hot Books (Most Downloaded)"), page="hot") else: abort(404) @@ -497,16 +493,13 @@ def render_downloaded_books(page, order, user_id): .filter(db.Books.id == book.id).first(): ub.delete_download(book.id) user = ub.session.query(ub.User).filter(ub.User.id == user_id).first() - - thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=user_id, title=_(u"Downloaded books by %(user)s",user=user.name), - page="download", - thumbnails=thumbnails) + page="download") else: abort(404) @@ -535,10 +528,9 @@ def render_author_books(page, author_id, order): author_info = services.goodreads_support.get_author_info(author_name) other_books = services.goodreads_support.get_other_books(author_info, entries) - thumbnails = get_thumbnails_for_books(entries) return render_title_template('author.html', entries=entries, pagination=pagination, id=author_id, title=_(u"Author: %(name)s", name=author_name), author=author_info, - other_books=other_books, page="author", thumbnails=thumbnails) + other_books=other_books, page="author") def render_publisher_books(page, book_id, order): @@ -551,10 +543,8 @@ def render_publisher_books(page, book_id, order): db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) - thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, - title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher", - thumbnails=thumbnails) + title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher") else: abort(404) @@ -566,10 +556,8 @@ def render_series_books(page, book_id, order): db.Books, db.Books.series.any(db.Series.id == book_id), [order[0]]) - thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"Series: %(serie)s", serie=name.name), page="series", - thumbnails=thumbnails) + title=_(u"Series: %(serie)s", serie=name.name), page="series") else: abort(404) @@ -581,10 +569,8 @@ def render_ratings_books(page, book_id, order): db.Books.ratings.any(db.Ratings.id == book_id), [order[0]]) if name and name.rating <= 10: - thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings", - thumbnails=thumbnails) + title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings") else: abort(404) @@ -596,10 +582,8 @@ def render_formats_books(page, book_id, order): db.Books, db.Books.data.any(db.Data.format == book_id.upper()), [order[0]]) - thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"File format: %(format)s", format=name.format), page="formats", - thumbnails=thumbnails) + title=_(u"File format: %(format)s", format=name.format), page="formats") else: abort(404) @@ -614,10 +598,8 @@ def render_category_books(page, book_id, order): db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) - thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, - title=_(u"Category: %(name)s", name=name.name), page="category", - thumbnails=thumbnails) + title=_(u"Category: %(name)s", name=name.name), page="category") else: abort(404) @@ -635,9 +617,8 @@ def render_language_books(page, name, order): db.Books, db.Books.languages.any(db.Languages.lang_code == name), [order[0]]) - thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name, - title=_(u"Language: %(name)s", name=lang_name), page="language", thumbnails=thumbnails) + title=_(u"Language: %(name)s", name=lang_name), page="language") def render_read_books(page, are_read, as_xml=False, order=None): @@ -687,10 +668,8 @@ def render_read_books(page, are_read, as_xml=False, order=None): else: name = _(u'Unread Books') + ' (' + str(pagination.total_count) + ')' pagename = "unread" - - thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename, thumbnails=thumbnails) + title=name, page=pagename) def render_archived_books(page, order): @@ -713,9 +692,8 @@ def render_archived_books(page, order): name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')' pagename = "archived" - thumbnails = get_thumbnails_for_books(entries + random if type(random) is list else entries) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename, thumbnails=thumbnails) + title=name, page=pagename) def render_prepare_search_form(cc): @@ -752,7 +730,6 @@ def render_prepare_search_form(cc): def render_search_results(term, offset=None, order=None, limit=None): join = db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit, *join) - thumbnails = get_thumbnails_for_books(entries) return render_title_template('search.html', searchterm=term, pagination=pagination, @@ -761,8 +738,7 @@ def render_search_results(term, offset=None, order=None, limit=None): entries=entries, result_count=result_count, title=_(u"Search"), - page="search", - thumbnails=thumbnails) + page="search") # ################################### View Books list ################################################################## @@ -973,10 +949,9 @@ def series_list(): .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all() - thumbnails = get_thumbnails_for_book_series(entries) return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=charlist, title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view", - order=order_no, thumbnails=thumbnails) + order=order_no) else: abort(404) @@ -1392,17 +1367,13 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): else: offset = 0 limit_all = result_count - - entries = q[offset:limit_all] - thumbnails = get_thumbnails_for_books(entries) return render_title_template('search.html', adv_searchterm=searchterm, pagination=pagination, - entries=entries, + entries=q[offset:limit_all], result_count=result_count, title=_(u"Advanced Search"), - page="advsearch", - thumbnails=thumbnails) + page="advsearch") @web.route("/advsearch", methods=['GET']) @@ -1417,21 +1388,11 @@ def advanced_search_form(): @web.route("/cover/") +@web.route("/cover//") +@web.route("/cover///") @login_required_if_no_ano -def get_cover(book_id): - return get_book_cover(book_id) - - -@web.route("/cached-cover/") -@login_required_if_no_ano -def get_cached_cover(cache_id): - return get_cached_book_cover(cache_id) - - -@web.route("/cached-cover-thumbnail/") -@login_required_if_no_ano -def get_cached_cover_thumbnail(cache_id): - return get_cached_book_cover_thumbnail(cache_id) +def get_cover(book_id, resolution=None, cache_bust=None): + return get_book_cover(book_id, resolution) @web.route("/robots.txt") @@ -1841,7 +1802,6 @@ def show_book(book_id): if media_format.format.lower() in constants.EXTENSIONS_AUDIO: audioentries.append(media_format.format.lower()) - thumbnails = get_thumbnails_for_books([entries]) return render_title_template('detail.html', entry=entries, audioentries=audioentries, @@ -1853,8 +1813,7 @@ def show_book(book_id): is_archived=is_archived, kindle_list=kindle_list, reader_list=reader_list, - page="book", - thumbnails=thumbnails) + page="book") else: log.debug(u"Oops! Selected book title is unavailable. File does not exist or is not accessible") flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), From 0bd544704d2060efa544d176b582826838d306ec Mon Sep 17 00:00:00 2001 From: mmonkey Date: Sat, 25 Sep 2021 03:04:38 -0500 Subject: [PATCH 017/112] Added series cover thumbnail generation. Better cache file handling. --- cps/admin.py | 2 +- cps/constants.py | 13 ++ cps/fs.py | 19 ++- cps/helper.py | 57 ++++++- cps/jinjia.py | 43 ++++-- cps/schedule.py | 3 +- cps/tasks/thumbnail.py | 280 ++++++++++++++++++++++++++++++---- cps/templates/author.html | 2 +- cps/templates/book_cover.html | 11 -- cps/templates/book_edit.html | 4 +- cps/templates/detail.html | 2 +- cps/templates/discover.html | 4 +- cps/templates/fragment.html | 2 +- cps/templates/grid.html | 4 +- cps/templates/image.html | 23 +++ cps/templates/index.html | 6 +- cps/templates/layout.html | 2 +- cps/templates/search.html | 4 +- cps/templates/shelf.html | 4 +- cps/ub.py | 5 +- cps/web.py | 32 +++- 21 files changed, 430 insertions(+), 92 deletions(-) delete mode 100644 cps/templates/book_cover.html create mode 100644 cps/templates/image.html diff --git a/cps/admin.py b/cps/admin.py index 98535320..92c8bd70 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -166,7 +166,7 @@ def clear_cache(): cache_type = request.args.get('cache_type'.strip()) showtext = {} - if cache_type == fs.CACHE_TYPE_THUMBNAILS: + if cache_type == constants.CACHE_TYPE_THUMBNAILS: log.info('clearing cover thumbnail cache') showtext['text'] = _(u'Cleared cover thumbnail cache') helper.clear_cover_thumbnail_cache() diff --git a/cps/constants.py b/cps/constants.py index 6a6b8f2b..306d2872 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -169,6 +169,19 @@ NIGHTLY_VERSION[1] = '$Format:%cI$' # NIGHTLY_VERSION[0] = 'bb7d2c6273ae4560e83950d36d64533343623a57' # NIGHTLY_VERSION[1] = '2018-09-09T10:13:08+02:00' +# CACHE +CACHE_TYPE_THUMBNAILS = 'thumbnails' + +# Thumbnail Types +THUMBNAIL_TYPE_COVER = 1 +THUMBNAIL_TYPE_SERIES = 2 +THUMBNAIL_TYPE_AUTHOR = 3 + +# Thumbnails Sizes +COVER_THUMBNAIL_ORIGINAL = 0 +COVER_THUMBNAIL_SMALL = 1 +COVER_THUMBNAIL_MEDIUM = 2 +COVER_THUMBNAIL_LARGE = 3 # clean-up the module namespace del sys, os, namedtuple diff --git a/cps/fs.py b/cps/fs.py index 30ab552a..0171a5d5 100644 --- a/cps/fs.py +++ b/cps/fs.py @@ -19,12 +19,10 @@ from __future__ import division, print_function, unicode_literals from . import logger from .constants import CACHE_DIR -from os import listdir, makedirs, remove +from os import makedirs, remove from os.path import isdir, isfile, join from shutil import rmtree -CACHE_TYPE_THUMBNAILS = 'thumbnails' - class FileSystem: _instance = None @@ -54,8 +52,19 @@ class FileSystem: return path if cache_type else self._cache_dir + def get_cache_file_dir(self, filename, cache_type=None): + path = join(self.get_cache_dir(cache_type), filename[:2]) + if not isdir(path): + try: + makedirs(path) + except OSError: + self.log.info(f'Failed to create path {path} (Permission denied).') + return False + + return path + def get_cache_file_path(self, filename, cache_type=None): - return join(self.get_cache_dir(cache_type), filename) if filename else None + return join(self.get_cache_file_dir(filename, cache_type), filename) if filename else None def get_cache_file_exists(self, filename, cache_type=None): path = self.get_cache_file_path(filename, cache_type) @@ -78,7 +87,7 @@ class FileSystem: return False def delete_cache_file(self, filename, cache_type=None): - path = join(self.get_cache_dir(cache_type), filename) + path = self.get_cache_file_path(filename, cache_type) if isfile(path): try: remove(path) diff --git a/cps/helper.py b/cps/helper.py index 1652be23..2f2df0e0 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -55,7 +55,7 @@ from . import calibre_db from .tasks.convert import TaskConvert from . import logger, config, get_locale, db, fs, ub from . import gdriveutils as gd -from .constants import STATIC_DIR as _STATIC_DIR +from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES from .subproc_wrapper import process_wait from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS from .tasks.mail import TaskEmail @@ -575,8 +575,9 @@ def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None) thumbnail = get_book_cover_thumbnail(book, resolution) if thumbnail: cache = fs.FileSystem() - if cache.get_cache_file_exists(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): - return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename) + if cache.get_cache_file_exists(thumbnail.filename, CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_file_dir(thumbnail.filename, CACHE_TYPE_THUMBNAILS), + thumbnail.filename) # Send the book cover from Google Drive if configured if config.config_use_google_drive: @@ -606,14 +607,54 @@ def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None) def get_book_cover_thumbnail(book, resolution): if book and book.has_cover: - return ub.session\ - .query(ub.Thumbnail)\ - .filter(ub.Thumbnail.book_id == book.id)\ - .filter(ub.Thumbnail.resolution == resolution)\ - .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow()))\ + return ub.session \ + .query(ub.Thumbnail) \ + .filter(ub.Thumbnail.type == THUMBNAIL_TYPE_COVER) \ + .filter(ub.Thumbnail.entity_id == book.id) \ + .filter(ub.Thumbnail.resolution == resolution) \ + .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow())) \ .first() +def get_series_thumbnail_on_failure(series_id, resolution): + book = calibre_db.session \ + .query(db.Books) \ + .join(db.books_series_link) \ + .join(db.Series) \ + .filter(db.Series.id == series_id) \ + .filter(db.Books.has_cover == 1) \ + .first() + + return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) + + +def get_series_cover_thumbnail(series_id, resolution=None): + return get_series_cover_internal(series_id, resolution) + + +def get_series_cover_internal(series_id, resolution=None): + # Send the series thumbnail if it exists in cache + if resolution: + thumbnail = get_series_thumbnail(series_id, resolution) + if thumbnail: + cache = fs.FileSystem() + if cache.get_cache_file_exists(thumbnail.filename, CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_file_dir(thumbnail.filename, CACHE_TYPE_THUMBNAILS), + thumbnail.filename) + + return get_series_thumbnail_on_failure(series_id, resolution) + + +def get_series_thumbnail(series_id, resolution): + return ub.session \ + .query(ub.Thumbnail) \ + .filter(ub.Thumbnail.type == THUMBNAIL_TYPE_SERIES) \ + .filter(ub.Thumbnail.entity_id == series_id) \ + .filter(ub.Thumbnail.resolution == resolution) \ + .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow())) \ + .first() + + # saves book cover from url def save_cover_from_url(url, book_path): try: diff --git a/cps/jinjia.py b/cps/jinjia.py index 8c8b72a9..1a58416a 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -32,9 +32,7 @@ from flask import Blueprint, request, url_for from flask_babel import get_locale from flask_login import current_user from markupsafe import escape -from . import logger -from .tasks.thumbnail import THUMBNAIL_RESOLUTION_1X, THUMBNAIL_RESOLUTION_2X, THUMBNAIL_RESOLUTION_3X - +from . import constants, logger jinjia = Blueprint('jinjia', __name__) log = logger.create() @@ -141,17 +139,44 @@ def uuidfilter(var): return uuid4() +@jinjia.app_template_filter('cache_timestamp') +def cache_timestamp(rolling_period='month'): + if rolling_period == 'day': + return str(int(datetime.datetime.today().replace(hour=1, minute=1).timestamp())) + elif rolling_period == 'year': + return str(int(datetime.datetime.today().replace(day=1).timestamp())) + else: + return str(int(datetime.datetime.today().replace(month=1, day=1).timestamp())) + + @jinjia.app_template_filter('last_modified') -def book_cover_cache_id(book): - timestamp = int(book.last_modified.timestamp() * 1000) - return str(timestamp) +def book_last_modified(book): + return str(int(book.last_modified.timestamp())) @jinjia.app_template_filter('get_cover_srcset') def get_cover_srcset(book): srcset = list() - for resolution in [THUMBNAIL_RESOLUTION_1X, THUMBNAIL_RESOLUTION_2X, THUMBNAIL_RESOLUTION_3X]: - timestamp = int(book.last_modified.timestamp() * 1000) - url = url_for('web.get_cover', book_id=book.id, resolution=resolution, cache_bust=str(timestamp)) + resolutions = { + constants.COVER_THUMBNAIL_SMALL: 'sm', + constants.COVER_THUMBNAIL_MEDIUM: 'md', + constants.COVER_THUMBNAIL_LARGE: 'lg' + } + for resolution, shortname in resolutions.items(): + url = url_for('web.get_cover', book_id=book.id, resolution=shortname, c=book_last_modified(book)) + srcset.append(f'{url} {resolution}x') + return ', '.join(srcset) + + +@jinjia.app_template_filter('get_series_srcset') +def get_cover_srcset(series): + srcset = list() + resolutions = { + constants.COVER_THUMBNAIL_SMALL: 'sm', + constants.COVER_THUMBNAIL_MEDIUM: 'md', + constants.COVER_THUMBNAIL_LARGE: 'lg' + } + for resolution, shortname in resolutions.items(): + url = url_for('web.get_series_cover', series_id=series.id, resolution=shortname, c=cache_timestamp()) srcset.append(f'{url} {resolution}x') return ', '.join(srcset) diff --git a/cps/schedule.py b/cps/schedule.py index 8c350bd5..dc153b9a 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -21,7 +21,7 @@ from __future__ import division, print_function, unicode_literals from .services.background_scheduler import BackgroundScheduler from .services.worker import WorkerThread from .tasks.database import TaskReconnectDatabase -from .tasks.thumbnail import TaskGenerateCoverThumbnails +from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails def register_jobs(): @@ -37,3 +37,4 @@ def register_jobs(): def register_startup_jobs(): WorkerThread.add(None, TaskGenerateCoverThumbnails()) + # WorkerThread.add(None, TaskGenerateSeriesThumbnails()) diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index 688f6e4b..152b8772 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -19,10 +19,11 @@ from __future__ import division, print_function, unicode_literals import os +from .. import constants from cps import config, db, fs, gdriveutils, logger, ub from cps.services.worker import CalibreTask from datetime import datetime, timedelta -from sqlalchemy import or_ +from sqlalchemy import func, text, or_ try: from urllib.request import urlopen @@ -35,9 +36,34 @@ try: except (ImportError, RuntimeError) as e: use_IM = False -THUMBNAIL_RESOLUTION_1X = 1 -THUMBNAIL_RESOLUTION_2X = 2 -THUMBNAIL_RESOLUTION_3X = 3 + +def get_resize_height(resolution): + return int(225 * resolution) + + +def get_resize_width(resolution, original_width, original_height): + height = get_resize_height(resolution) + percent = (height / float(original_height)) + width = int((float(original_width) * float(percent))) + return width if width % 2 == 0 else width + 1 + + +def get_best_fit(width, height, image_width, image_height): + resize_width = int(width / 2.0) + resize_height = int(height / 2.0) + aspect_ratio = image_width / image_height + + # If this image's aspect ratio is different than the first image, then resize this image + # to fill the width and height of the first image + if aspect_ratio < width / height: + resize_width = int(width / 2.0) + resize_height = image_height * int(width / 2.0) / image_width + + elif aspect_ratio > width / height: + resize_width = image_width * int(height / 2.0) / image_height + resize_height = int(height / 2.0) + + return {'width': resize_width, 'height': resize_height} class TaskGenerateCoverThumbnails(CalibreTask): @@ -48,8 +74,8 @@ class TaskGenerateCoverThumbnails(CalibreTask): self.calibre_db = db.CalibreDB(expire_on_commit=False) self.cache = fs.FileSystem() self.resolutions = [ - THUMBNAIL_RESOLUTION_1X, - THUMBNAIL_RESOLUTION_2X + constants.COVER_THUMBNAIL_SMALL, + constants.COVER_THUMBNAIL_MEDIUM ] def run(self, worker_thread): @@ -75,7 +101,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): updated += 1 self.update_book_cover_thumbnail(book, thumbnail) - elif not self.cache.get_cache_file_exists(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): + elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS): updated += 1 self.update_book_cover_thumbnail(book, thumbnail) @@ -86,21 +112,23 @@ class TaskGenerateCoverThumbnails(CalibreTask): self.app_db_session.remove() def get_books_with_covers(self): - return self.calibre_db.session\ - .query(db.Books)\ - .filter(db.Books.has_cover == 1)\ + return self.calibre_db.session \ + .query(db.Books) \ + .filter(db.Books.has_cover == 1) \ .all() def get_book_cover_thumbnails(self, book_id): - return self.app_db_session\ - .query(ub.Thumbnail)\ - .filter(ub.Thumbnail.book_id == book_id)\ - .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow()))\ + return self.app_db_session \ + .query(ub.Thumbnail) \ + .filter(ub.Thumbnail.type == constants.THUMBNAIL_TYPE_COVER) \ + .filter(ub.Thumbnail.entity_id == book_id) \ + .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow())) \ .all() def create_book_cover_thumbnail(self, book, resolution): thumbnail = ub.Thumbnail() - thumbnail.book_id = book.id + thumbnail.type = constants.THUMBNAIL_TYPE_COVER + thumbnail.entity_id = book.id thumbnail.format = 'jpeg' thumbnail.resolution = resolution @@ -118,7 +146,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): try: self.app_db_session.commit() - self.cache.delete_cache_file(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) self.generate_book_thumbnail(book, thumbnail) except Exception as ex: self.log.info(u'Error updating book thumbnail: ' + str(ex)) @@ -144,7 +172,8 @@ class TaskGenerateCoverThumbnails(CalibreTask): width = self.get_thumbnail_width(height, img) img.resize(width=width, height=height, filter='lanczos') img.format = thumbnail.format - filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + filename = self.cache.get_cache_file_path(thumbnail.filename, + constants.CACHE_TYPE_THUMBNAILS) img.save(filename=filename) except Exception as ex: # Bubble exception to calling function @@ -158,26 +187,212 @@ class TaskGenerateCoverThumbnails(CalibreTask): raise Exception('Book cover file not found') with Image(filename=book_cover_filepath) as img: - height = self.get_thumbnail_height(thumbnail) + height = get_resize_height(thumbnail.resolution) if img.height > height: - width = self.get_thumbnail_width(height, img) + width = get_resize_width(thumbnail.resolution, img.width, img.height) img.resize(width=width, height=height, filter='lanczos') img.format = thumbnail.format - filename = self.cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + filename = self.cache.get_cache_file_path(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) img.save(filename=filename) - def get_thumbnail_height(self, thumbnail): - return int(225 * thumbnail.resolution) - - def get_thumbnail_width(self, height, img): - percent = (height / float(img.height)) - return int((float(img.width) * float(percent))) - @property def name(self): return "ThumbnailsGenerate" +class TaskGenerateSeriesThumbnails(CalibreTask): + def __init__(self, task_message=u'Generating series thumbnails'): + super(TaskGenerateSeriesThumbnails, self).__init__(task_message) + self.log = logger.create() + self.app_db_session = ub.get_new_session_instance() + self.calibre_db = db.CalibreDB(expire_on_commit=False) + self.cache = fs.FileSystem() + self.resolutions = [ + constants.COVER_THUMBNAIL_SMALL, + constants.COVER_THUMBNAIL_MEDIUM + ] + + # get all series + # get all books in series with covers and count >= 4 books + # get the dimensions from the first book in the series & pop the first book from the series list of books + # randomly select three other books in the series + + # resize the covers in the sequence? + # create an image sequence from the 4 selected books of the series + # join pairs of books in the series with wand's concat + # join the two sets of pairs with wand's + + def run(self, worker_thread): + if self.calibre_db.session and use_IM: + all_series = self.get_series_with_four_plus_books() + count = len(all_series) + + updated = 0 + generated = 0 + for i, series in enumerate(all_series): + series_thumbnails = self.get_series_thumbnails(series.id) + series_books = self.get_series_books(series.id) + + # Generate new thumbnails for missing covers + resolutions = list(map(lambda t: t.resolution, series_thumbnails)) + missing_resolutions = list(set(self.resolutions).difference(resolutions)) + for resolution in missing_resolutions: + generated += 1 + self.create_series_thumbnail(series, series_books, resolution) + + # Replace outdated or missing thumbnails + for thumbnail in series_thumbnails: + if any(book.last_modified > thumbnail.generated_at for book in series_books): + updated += 1 + self.update_series_thumbnail(series_books, thumbnail) + + elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS): + updated += 1 + self.update_series_thumbnail(series_books, thumbnail) + + self.message = u'Processing series {0} of {1}'.format(i + 1, count) + self.progress = (1.0 / count) * i + + self._handleSuccess() + self.app_db_session.remove() + + def get_series_with_four_plus_books(self): + return self.calibre_db.session \ + .query(db.Series) \ + .join(db.books_series_link) \ + .join(db.Books) \ + .filter(db.Books.has_cover == 1) \ + .group_by(text('books_series_link.series')) \ + .having(func.count('book_series_link') > 3) \ + .all() + + def get_series_books(self, series_id): + return self.calibre_db.session \ + .query(db.Books) \ + .join(db.books_series_link) \ + .join(db.Series) \ + .filter(db.Books.has_cover == 1) \ + .filter(db.Series.id == series_id) \ + .all() + + def get_series_thumbnails(self, series_id): + return self.app_db_session \ + .query(ub.Thumbnail) \ + .filter(ub.Thumbnail.type == constants.THUMBNAIL_TYPE_SERIES) \ + .filter(ub.Thumbnail.entity_id == series_id) \ + .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow())) \ + .all() + + def create_series_thumbnail(self, series, series_books, resolution): + thumbnail = ub.Thumbnail() + thumbnail.type = constants.THUMBNAIL_TYPE_SERIES + thumbnail.entity_id = series.id + thumbnail.format = 'jpeg' + thumbnail.resolution = resolution + + self.app_db_session.add(thumbnail) + try: + self.app_db_session.commit() + self.generate_series_thumbnail(series_books, thumbnail) + except Exception as ex: + self.log.info(u'Error creating book thumbnail: ' + str(ex)) + self._handleError(u'Error creating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def update_series_thumbnail(self, series_books, thumbnail): + thumbnail.generated_at = datetime.utcnow() + + try: + self.app_db_session.commit() + self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) + self.generate_series_thumbnail(series_books, thumbnail) + except Exception as ex: + self.log.info(u'Error updating book thumbnail: ' + str(ex)) + self._handleError(u'Error updating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def generate_series_thumbnail(self, series_books, thumbnail): + books = series_books[:4] + + top = 0 + left = 0 + width = 0 + height = 0 + with Image() as canvas: + for book in books: + if config.config_use_google_drive: + if not gdriveutils.is_gdrive_ready(): + raise Exception('Google Drive is configured but not ready') + + web_content_link = gdriveutils.get_cover_via_gdrive(book.path) + if not web_content_link: + raise Exception('Google Drive cover url not found') + + stream = None + try: + stream = urlopen(web_content_link) + with Image(file=stream) as img: + # Use the first image in this set to determine the width and height to scale the + # other images in this set + if width == 0 or height == 0: + width = get_resize_width(thumbnail.resolution, img.width, img.height) + height = get_resize_height(thumbnail.resolution) + canvas.blank(width, height) + + dimensions = get_best_fit(width, height, img.width, img.height) + + # resize and crop the image + img.resize(width=int(dimensions['width']), height=int(dimensions['height']), filter='lanczos') + img.crop(width=int(width / 2.0), height=int(height / 2.0), gravity='center') + + # add the image to the canvas + canvas.composite(img, left, top) + + except Exception as ex: + self.log.info(u'Error generating thumbnail file: ' + str(ex)) + raise ex + finally: + stream.close() + + book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + if not os.path.isfile(book_cover_filepath): + raise Exception('Book cover file not found') + + with Image(filename=book_cover_filepath) as img: + # Use the first image in this set to determine the width and height to scale the + # other images in this set + if width == 0 or height == 0: + width = get_resize_width(thumbnail.resolution, img.width, img.height) + height = get_resize_height(thumbnail.resolution) + canvas.blank(width, height) + + dimensions = get_best_fit(width, height, img.width, img.height) + + # resize and crop the image + img.resize(width=int(dimensions['width']), height=int(dimensions['height']), filter='lanczos') + img.crop(width=int(width / 2.0), height=int(height / 2.0), gravity='center') + + # add the image to the canvas + canvas.composite(img, left, top) + + # set the coordinates for the next iteration + if left == 0 and top == 0: + left = int(width / 2.0) + elif left == int(width / 2.0) and top == 0: + left = 0 + top = int(height / 2.0) + else: + left = int(width / 2.0) + + canvas.format = thumbnail.format + filename = self.cache.get_cache_file_path(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) + canvas.save(filename=filename) + + @property + def name(self): + return "SeriesThumbnailGenerate" + + class TaskClearCoverThumbnailCache(CalibreTask): def __init__(self, book_id=None, task_message=u'Clearing cover thumbnail cache'): super(TaskClearCoverThumbnailCache, self).__init__(task_message) @@ -199,21 +414,22 @@ class TaskClearCoverThumbnailCache(CalibreTask): self.app_db_session.remove() def get_thumbnails_for_book(self, book_id): - return self.app_db_session\ - .query(ub.Thumbnail)\ - .filter(ub.Thumbnail.book_id == book_id)\ + return self.app_db_session \ + .query(ub.Thumbnail) \ + .filter(ub.Thumbnail.type == constants.THUMBNAIL_TYPE_COVER) \ + .filter(ub.Thumbnail.entity_id == book_id) \ .all() def delete_thumbnail(self, thumbnail): try: - self.cache.delete_cache_file(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS) + self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) except Exception as ex: self.log.info(u'Error deleting book thumbnail: ' + str(ex)) self._handleError(u'Error deleting book thumbnail: ' + str(ex)) def delete_all_thumbnails(self): try: - self.cache.delete_cache_dir(fs.CACHE_TYPE_THUMBNAILS) + self.cache.delete_cache_dir(constants.CACHE_TYPE_THUMBNAILS) except Exception as ex: self.log.info(u'Error deleting book thumbnails: ' + str(ex)) self._handleError(u'Error deleting book thumbnails: ' + str(ex)) diff --git a/cps/templates/author.html b/cps/templates/author.html index 2ff1ce7a..df891d5b 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -37,7 +37,7 @@
- {{ book_cover_image(entry, title=author.name|safe) }} + {{ image.book_cover(entry, title=author.name|safe) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html deleted file mode 100644 index 7d3a84f4..00000000 --- a/cps/templates/book_cover.html +++ /dev/null @@ -1,11 +0,0 @@ -{% macro book_cover_image(book, title=None) -%} - {%- set book_title = book.title if book.title else book.name -%} - {%- set book_title = title if title else book_title -%} - {% set srcset = book|get_cover_srcset %} - {{ book_title }} -{%- endmacro %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 8525b8e3..b5f5df7a 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -1,10 +1,10 @@ -{% from 'book_cover.html' import book_cover_image %} +{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %} {% if book %}
- {{ book_cover_image(book) }} + {{ image.book_cover(book) }}
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index b538ff9a..7985691a 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,7 +4,7 @@
- {{ book_cover_image(entry) }} + {{ image.book_cover(entry) }}
diff --git a/cps/templates/discover.html b/cps/templates/discover.html index 01cd393d..d697ff0a 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -1,4 +1,4 @@ -{% from 'book_cover.html' import book_cover_image %} +{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %}
@@ -10,7 +10,7 @@ {% if entry.has_cover is defined %} - {{ book_cover_image(entry) }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/fragment.html b/cps/templates/fragment.html index 901dd193..f2e94fb2 100644 --- a/cps/templates/fragment.html +++ b/cps/templates/fragment.html @@ -1,4 +1,4 @@ -{% from 'book_cover.html' import book_cover_image %} +{% import 'image.html' as image %}
{% block body %}{% endblock %}
diff --git a/cps/templates/grid.html b/cps/templates/grid.html index 45fb53bf..b3395fcb 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -1,4 +1,4 @@ -{% from 'book_cover.html' import book_cover_image %} +{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %}

{{_(title)}}

@@ -29,7 +29,7 @@
- {{ book_cover_image(entry[0], title=entry[0].series[0].name|shortentitle) }} + {{ image.series(entry[0].series[0], title=entry[0].series[0].name|shortentitle) }} {{entry.count}} diff --git a/cps/templates/image.html b/cps/templates/image.html new file mode 100644 index 00000000..ae57727b --- /dev/null +++ b/cps/templates/image.html @@ -0,0 +1,23 @@ +{% macro book_cover(book, title=None, alt=None) -%} + {%- set image_title = book.title if book.title else book.name -%} + {%- set image_title = title if title else image_title -%} + {%- set image_alt = alt if alt else image_title -%} + {% set srcset = book|get_cover_srcset %} + {{ image_alt }} +{%- endmacro %} + +{% macro series(series, title=None, alt=None) -%} + {%- set image_alt = alt if alt else image_title -%} + {% set srcset = series|get_series_srcset %} + {{ book_title }} +{%- endmacro %} diff --git a/cps/templates/index.html b/cps/templates/index.html index e47c6c70..709d6ce7 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -1,4 +1,4 @@ -{% from 'book_cover.html' import book_cover_image %} +{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %} {% if g.user.show_detail_random() %} @@ -10,7 +10,7 @@
- {{ book_cover_image(entry) }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} @@ -87,7 +87,7 @@
- {{ book_cover_image(entry) }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/layout.html b/cps/templates/layout.html index 63dc7469..4a664ec1 100644 --- a/cps/templates/layout.html +++ b/cps/templates/layout.html @@ -1,5 +1,5 @@ {% from 'modal_dialogs.html' import restrict_modal, delete_book, filechooser_modal, delete_confirm_modal, change_confirm_modal %} -{% from 'book_cover.html' import book_cover_image %} +{% import 'image.html' as image %} diff --git a/cps/templates/search.html b/cps/templates/search.html index 872b6dc1..15ff344f 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -1,4 +1,4 @@ -{% from 'book_cover.html' import book_cover_image %} +{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %}
@@ -45,7 +45,7 @@ {% if entry.has_cover is defined %} - {{ book_cover_image(entry) }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index 0e5865c9..18dbde8b 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -1,4 +1,4 @@ -{% from 'book_cover.html' import book_cover_image %} +{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %}
@@ -32,7 +32,7 @@
- {{ book_cover_image(entry) }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/ub.py b/cps/ub.py index f108a1ce..7666f0f2 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -526,10 +526,11 @@ class Thumbnail(Base): __tablename__ = 'thumbnail' id = Column(Integer, primary_key=True) - book_id = Column(Integer) + entity_id = Column(Integer) uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True) format = Column(String, default='jpeg') - resolution = Column(SmallInteger, default=1) + type = Column(SmallInteger, default=constants.THUMBNAIL_TYPE_COVER) + resolution = Column(SmallInteger, default=constants.COVER_THUMBNAIL_SMALL) filename = Column(String, default=filename) generated_at = Column(DateTime, default=lambda: datetime.datetime.utcnow()) expiration = Column(DateTime, nullable=True) diff --git a/cps/web.py b/cps/web.py index 8f78ff32..88a5c0ab 100644 --- a/cps/web.py +++ b/cps/web.py @@ -50,8 +50,8 @@ from . import constants, logger, isoLanguages, services from . import babel, db, ub, config, get_locale, app from . import calibre_db from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download -from .helper import check_valid_domain, render_task_status, check_email, check_username, \ - get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \ +from .helper import check_valid_domain, render_task_status, check_email, check_username, get_cc_columns, \ + get_book_cover, get_series_cover_thumbnail, get_download_link, send_mail, generate_random_password, \ send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password, valid_email from .pagination import Pagination from .redirect import redirect_back @@ -1388,11 +1388,31 @@ def advanced_search_form(): @web.route("/cover/") -@web.route("/cover//") -@web.route("/cover///") +@web.route("/cover//") @login_required_if_no_ano -def get_cover(book_id, resolution=None, cache_bust=None): - return get_book_cover(book_id, resolution) +def get_cover(book_id, resolution=None): + resolutions = { + 'og': constants.COVER_THUMBNAIL_ORIGINAL, + 'sm': constants.COVER_THUMBNAIL_SMALL, + 'md': constants.COVER_THUMBNAIL_MEDIUM, + 'lg': constants.COVER_THUMBNAIL_LARGE, + } + cover_resolution = resolutions.get(resolution, None) + return get_book_cover(book_id, cover_resolution) + + +@web.route("/series_cover/") +@web.route("/series_cover//") +@login_required_if_no_ano +def get_series_cover(series_id, resolution=None): + resolutions = { + 'og': constants.COVER_THUMBNAIL_ORIGINAL, + 'sm': constants.COVER_THUMBNAIL_SMALL, + 'md': constants.COVER_THUMBNAIL_MEDIUM, + 'lg': constants.COVER_THUMBNAIL_LARGE, + } + cover_resolution = resolutions.get(resolution, None) + return get_series_cover_thumbnail(series_id, cover_resolution) @web.route("/robots.txt") From 26071d4e7ad1943f91904804fc1b5464866d748b Mon Sep 17 00:00:00 2001 From: mmonkey Date: Sun, 26 Sep 2021 02:02:48 -0500 Subject: [PATCH 018/112] Added Scheduled Tasks Settings --- cps/admin.py | 62 +++++++++++++++++++--------- cps/config_sql.py | 15 +++++-- cps/schedule.py | 19 +++++---- cps/services/background_scheduler.py | 24 +++++++---- cps/tasks/thumbnail.py | 14 +------ cps/templates/admin.html | 43 +++++++++++-------- cps/templates/schedule_edit.html | 38 +++++++++++++++++ 7 files changed, 147 insertions(+), 68 deletions(-) create mode 100644 cps/templates/schedule_edit.html diff --git a/cps/admin.py b/cps/admin.py index 92c8bd70..bd292ba3 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -159,23 +159,6 @@ def shutdown(): return json.dumps(showtext), 400 -@admi.route("/clear-cache") -@login_required -@admin_required -def clear_cache(): - cache_type = request.args.get('cache_type'.strip()) - showtext = {} - - if cache_type == constants.CACHE_TYPE_THUMBNAILS: - log.info('clearing cover thumbnail cache') - showtext['text'] = _(u'Cleared cover thumbnail cache') - helper.clear_cover_thumbnail_cache() - return json.dumps(showtext) - - showtext['text'] = _(u'Unknown command') - return json.dumps(showtext) - - @admi.route("/admin/view") @login_required @admin_required @@ -205,6 +188,7 @@ def admin(): feature_support=feature_support, kobo_support=kobo_support, title=_(u"Admin page"), page="admin") + @admi.route("/admin/dbconfig", methods=["GET", "POST"]) @login_required @admin_required @@ -245,6 +229,7 @@ def ajax_db_config(): def calibreweb_alive(): return "", 200 + @admi.route("/admin/viewconfig") @login_required @admin_required @@ -257,6 +242,7 @@ def view_configuration(): restrictColumns=restrict_columns, title=_(u"UI Configuration"), page="uiconfig") + @admi.route("/admin/usertable") @login_required @admin_required @@ -339,6 +325,7 @@ def list_users(): response.headers["Content-Type"] = "application/json; charset=utf-8" return response + @admi.route("/ajax/deleteuser", methods=['POST']) @login_required @admin_required @@ -372,6 +359,7 @@ def delete_user(): success.extend(errors) return Response(json.dumps(success), mimetype='application/json') + @admi.route("/ajax/getlocale") @login_required @admin_required @@ -517,6 +505,7 @@ def update_table_settings(): return "Invalid request", 400 return "" + def check_valid_read_column(column): if column != "0": if not calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.id == column) \ @@ -524,6 +513,7 @@ def check_valid_read_column(column): return False return True + def check_valid_restricted_column(column): if column != "0": if not calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.id == column) \ @@ -532,7 +522,6 @@ def check_valid_restricted_column(column): return True - @admi.route("/admin/viewconfig", methods=["POST"]) @login_required @admin_required @@ -564,7 +553,6 @@ def update_view_configuration(): _config_int(to_save, "config_books_per_page") _config_int(to_save, "config_authors_max") - config.config_default_role = constants.selected_roles(to_save) config.config_default_role &= ~constants.ROLE_ANONYMOUS @@ -1210,6 +1198,7 @@ def _db_configuration_update_helper(): config.save() return _db_configuration_result(None, gdrive_error) + def _configuration_update_helper(): reboot_required = False to_save = request.form.to_dict() @@ -1299,6 +1288,7 @@ def _configuration_update_helper(): return _configuration_result(None, reboot_required) + def _configuration_result(error_flash=None, reboot=False): resp = {} if error_flash: @@ -1388,6 +1378,7 @@ def _handle_new_user(to_save, content, languages, translations, kobo_support): log.error("Settings DB is not Writeable") flash(_("Settings DB is not Writeable"), category="error") + def _delete_user(content): if ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN, ub.User.id != content.id).count(): @@ -1572,6 +1563,39 @@ def update_mailsettings(): return edit_mailsettings() +@admi.route("/admin/scheduledtasks") +@login_required +@admin_required +def edit_scheduledtasks(): + content = config.get_scheduled_task_settings() + return render_title_template("schedule_edit.html", content=content, title=_(u"Edit Scheduled Tasks Settings")) + + +@admi.route("/admin/scheduledtasks", methods=["POST"]) +@login_required +@admin_required +def update_scheduledtasks(): + to_save = request.form.to_dict() + _config_int(to_save, "schedule_start_time") + _config_int(to_save, "schedule_end_time") + _config_checkbox(to_save, "schedule_generate_book_covers") + _config_checkbox(to_save, "schedule_generate_series_covers") + + try: + config.save() + flash(_(u"Scheduled tasks settings updated"), category="success") + except IntegrityError as ex: + ub.session.rollback() + log.error("An unknown error occurred while saving scheduled tasks settings") + flash(_(u"An unknown error occurred. Please try again later."), category="error") + except OperationalError: + ub.session.rollback() + log.error("Settings DB is not Writeable") + flash(_("Settings DB is not Writeable"), category="error") + + return edit_scheduledtasks() + + @admi.route("/admin/user/", methods=["GET", "POST"]) @login_required @admin_required diff --git a/cps/config_sql.py b/cps/config_sql.py index 88107f9b..4841f03d 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -133,13 +133,18 @@ class _Settings(_Base): config_calibre = Column(String) config_rarfile_location = Column(String, default=None) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) - config_unicode_filename =Column(Boolean, default=False) + config_unicode_filename = Column(Boolean, default=False) config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) config_reverse_proxy_login_header_name = Column(String) config_allow_reverse_proxy_header_login = Column(Boolean, default=False) + schedule_start_time = Column(Integer, default=4) + schedule_end_time = Column(Integer, default=6) + schedule_generate_book_covers = Column(Boolean, default=False) + schedule_generate_series_covers = Column(Boolean, default=False) + def __repr__(self): return self.__class__.__name__ @@ -170,7 +175,6 @@ class _ConfigSQL(object): if change: self.save() - def _read_from_storage(self): if self._settings is None: log.debug("_ConfigSQL._read_from_storage") @@ -254,6 +258,8 @@ class _ConfigSQL(object): return bool((self.mail_server != constants.DEFAULT_MAIL_SERVER and self.mail_server_type == 0) or (self.mail_gmail_token != {} and self.mail_server_type == 1)) + def get_scheduled_task_settings(self): + return {k:v for k, v in self.__dict__.items() if k.startswith('schedule_')} def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None): """Possibly updates a field of this object. @@ -289,7 +295,6 @@ class _ConfigSQL(object): storage[k] = v return storage - def load(self): '''Load all configuration values from the underlying storage.''' s = self._read_from_storage() # type: _Settings @@ -407,6 +412,7 @@ def autodetect_calibre_binary(): return element return "" + def autodetect_unrar_binary(): if sys.platform == "win32": calibre_path = ["C:\\program files\\WinRar\\unRAR.exe", @@ -418,6 +424,7 @@ def autodetect_unrar_binary(): return element return "" + def autodetect_kepubify_binary(): if sys.platform == "win32": calibre_path = ["C:\\program files\\kepubify\\kepubify-windows-64Bit.exe", @@ -429,6 +436,7 @@ def autodetect_kepubify_binary(): return element return "" + def _migrate_database(session): # make sure the table is created, if it does not exist _Base.metadata.create_all(session.bind) @@ -452,6 +460,7 @@ def load_configuration(session): # session.commit() return conf + def get_flask_session_key(session): flask_settings = session.query(_Flask_Settings).one_or_none() if flask_settings == None: diff --git a/cps/schedule.py b/cps/schedule.py index dc153b9a..2cddaecb 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -19,7 +19,6 @@ from __future__ import division, print_function, unicode_literals from .services.background_scheduler import BackgroundScheduler -from .services.worker import WorkerThread from .tasks.database import TaskReconnectDatabase from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails @@ -28,13 +27,19 @@ def register_jobs(): scheduler = BackgroundScheduler() if scheduler: - # Reconnect metadata.db once every 12 hours - scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', hour='4,16') + # Reconnect Calibre database (metadata.db) + scheduler.schedule_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', hour='4,16') - # Generate all missing book cover thumbnails once every 24 hours - scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(), trigger='cron', hour=4) + # Generate all missing book cover thumbnails + scheduler.schedule_task(user=None, task=lambda: TaskGenerateCoverThumbnails(), trigger='cron', hour=4) + + # Generate all missing series thumbnails + scheduler.schedule_task(user=None, task=lambda: TaskGenerateSeriesThumbnails(), trigger='cron', hour=4) def register_startup_jobs(): - WorkerThread.add(None, TaskGenerateCoverThumbnails()) - # WorkerThread.add(None, TaskGenerateSeriesThumbnails()) + scheduler = BackgroundScheduler() + + if scheduler: + scheduler.schedule_task_immediately(None, task=lambda: TaskGenerateCoverThumbnails()) + scheduler.schedule_task_immediately(None, task=lambda: TaskGenerateSeriesThumbnails()) diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py index 1b588ac2..ba578903 100644 --- a/cps/services/background_scheduler.py +++ b/cps/services/background_scheduler.py @@ -40,25 +40,31 @@ class BackgroundScheduler: if cls._instance is None: cls._instance = super(BackgroundScheduler, cls).__new__(cls) - - scheduler = BScheduler() - atexit.register(lambda: scheduler.shutdown()) - cls.log = logger.create() - cls.scheduler = scheduler + cls.scheduler = BScheduler() cls.scheduler.start() + atexit.register(lambda: cls.scheduler.shutdown()) + return cls._instance - def add(self, func, trigger, **trigger_args): + def _add(self, func, trigger, **trigger_args): if use_APScheduler: return self.scheduler.add_job(func=func, trigger=trigger, **trigger_args) - def add_task(self, user, task, trigger, **trigger_args): + # Expects a lambda expression for the task, so that the task isn't instantiated before the task is scheduled + def schedule_task(self, user, task, trigger, **trigger_args): if use_APScheduler: def scheduled_task(): worker_task = task() - self.log.info(f'Running scheduled task in background: {worker_task.name} - {worker_task.message}') WorkerThread.add(user, worker_task) - return self.add(func=scheduled_task, trigger=trigger, **trigger_args) + return self._add(func=scheduled_task, trigger=trigger, **trigger_args) + + # Expects a lambda expression for the task, so that the task isn't instantiated before the task is scheduled + def schedule_task_immediately(self, user, task): + if use_APScheduler: + def scheduled_task(): + WorkerThread.add(user, task()) + + return self._add(func=scheduled_task, trigger='date') diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index 152b8772..a220fd8c 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -167,9 +167,9 @@ class TaskGenerateCoverThumbnails(CalibreTask): try: stream = urlopen(web_content_link) with Image(file=stream) as img: - height = self.get_thumbnail_height(thumbnail) + height = get_resize_height(thumbnail.resolution) if img.height > height: - width = self.get_thumbnail_width(height, img) + width = get_resize_width(thumbnail.resolution, img.width, img.height) img.resize(width=width, height=height, filter='lanczos') img.format = thumbnail.format filename = self.cache.get_cache_file_path(thumbnail.filename, @@ -212,16 +212,6 @@ class TaskGenerateSeriesThumbnails(CalibreTask): constants.COVER_THUMBNAIL_MEDIUM ] - # get all series - # get all books in series with covers and count >= 4 books - # get the dimensions from the first book in the series & pop the first book from the series list of books - # randomly select three other books in the series - - # resize the covers in the sequence? - # create an image sequence from the 4 selected books of the series - # join pairs of books in the series with wand's concat - # join the two sets of pairs with wand's - def run(self, worker_thread): if self.calibre_db.session and use_IM: all_series = self.get_series_with_four_plus_books() diff --git a/cps/templates/admin.html b/cps/templates/admin.html index 597ba103..ec0fc84e 100644 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -156,6 +156,31 @@
+
+
+

{{_('Scheduled Tasks')}}

+
+
+
{{_('Time at which tasks start to run')}}
+
{{config.schedule_start_time}}:00
+
+
+
{{_('Time at which tasks stop running')}}
+
{{config.schedule_end_time}}:00
+
+
+
{{_('Generate book cover thumbnails')}}
+
{{ display_bool_setting(config.schedule_generate_book_covers) }}
+
+
+
{{_('Generate series cover thumbnails')}}
+
{{ display_bool_setting(config.schedule_generate_series_covers) }}
+
+
+ {{_('Edit Scheduled Tasks Settings')}} +
+
+

{{_('Administration')}}

{{_('Download Debug Package')}} @@ -163,7 +188,6 @@
{{_('Reconnect Calibre Database')}}
-
{{_('Clear Cover Thumbnail Cache')}}
{{_('Restart')}}
@@ -248,21 +272,4 @@
- {% endblock %} diff --git a/cps/templates/schedule_edit.html b/cps/templates/schedule_edit.html new file mode 100644 index 00000000..f4e72224 --- /dev/null +++ b/cps/templates/schedule_edit.html @@ -0,0 +1,38 @@ +{% extends "layout.html" %} +{% block header %} + + +{% endblock %} +{% block body %} +
+

{{title}}

+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ + {{_('Cancel')}} +
+
+{% endblock %} From 46205a1f83ec6a81c57d5dd3348ea583f9201063 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Wed, 29 Sep 2021 02:40:12 -0500 Subject: [PATCH 019/112] Made long running tasks cancellable. Added cancel button to cancellable tasks in the task list. Added APP_MODE env variable for determining if the app is running in development, test, or production. --- cps.py | 8 +- cps/admin.py | 23 +++++- cps/constants.py | 7 +- cps/db.py | 6 +- cps/helper.py | 17 +++- cps/schedule.py | 61 ++++++++++++--- cps/services/background_scheduler.py | 35 ++++++--- cps/services/worker.py | 35 +++++++-- cps/static/css/caliBlur.css | 93 +++++++++------------- cps/static/js/main.js | 12 --- cps/static/js/table.js | 34 +++++++- cps/tasks/convert.py | 4 + cps/tasks/database.py | 4 + cps/tasks/mail.py | 6 +- cps/tasks/thumbnail.py | 111 +++++++++++++++++++-------- cps/tasks/upload.py | 4 + cps/templates/admin.html | 2 +- cps/templates/schedule_edit.html | 6 +- cps/templates/tasks.html | 27 +++++++ cps/web.py | 4 +- 20 files changed, 346 insertions(+), 153 deletions(-) diff --git a/cps.py b/cps.py index fe006551..20a27c71 100755 --- a/cps.py +++ b/cps.py @@ -44,7 +44,7 @@ from cps.editbooks import editbook from cps.remotelogin import remotelogin from cps.search_metadata import meta from cps.error_handler import init_errorhandler -from cps.schedule import register_jobs, register_startup_jobs +from cps.schedule import register_scheduled_tasks, register_startup_tasks try: from cps.kobo import kobo, get_kobo_activated @@ -81,9 +81,9 @@ def main(): if oauth_available: app.register_blueprint(oauth) - # Register scheduled jobs - register_jobs() - # register_startup_jobs() + # Register scheduled tasks + register_scheduled_tasks() + register_startup_tasks() success = web_server.start() sys.exit(0 if success else 1) diff --git a/cps/admin.py b/cps/admin.py index bd292ba3..62b3dbe0 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -40,12 +40,13 @@ from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError from sqlalchemy.sql.expression import func, or_, text -from . import constants, logger, helper, services, isoLanguages, fs -from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils +from . import constants, logger, helper, services, isoLanguages +from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils, schedule from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ valid_email, check_username from .gdriveutils import is_gdrive_ready, gdrive_support from .render_template import render_title_template, get_sidebar_config +from .services.worker import WorkerThread from . import debug_info, _BABEL_TRANSLATIONS try: @@ -1568,7 +1569,7 @@ def update_mailsettings(): @admin_required def edit_scheduledtasks(): content = config.get_scheduled_task_settings() - return render_title_template("schedule_edit.html", content=content, title=_(u"Edit Scheduled Tasks Settings")) + return render_title_template("schedule_edit.html", config=content, title=_(u"Edit Scheduled Tasks Settings")) @admi.route("/admin/scheduledtasks", methods=["POST"]) @@ -1584,6 +1585,12 @@ def update_scheduledtasks(): try: config.save() flash(_(u"Scheduled tasks settings updated"), category="success") + + # Cancel any running tasks + schedule.end_scheduled_tasks() + + # Re-register tasks with new settings + schedule.register_scheduled_tasks() except IntegrityError as ex: ub.session.rollback() log.error("An unknown error occurred while saving scheduled tasks settings") @@ -1869,3 +1876,13 @@ def extract_dynamic_field_from_filter(user, filtr): def extract_user_identifier(user, filtr): dynamic_field = extract_dynamic_field_from_filter(user, filtr) return extract_user_data_from_field(user, dynamic_field) + + +@admi.route("/ajax/canceltask", methods=['POST']) +@login_required +@admin_required +def cancel_task(): + task_id = request.get_json().get('task_id', None) + worker = WorkerThread.get_instance() + worker.end_task(task_id) + return "" diff --git a/cps/constants.py b/cps/constants.py index 306d2872..a92a0029 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -24,10 +24,13 @@ from sqlalchemy import __version__ as sql_version sqlalchemy_version2 = ([int(x) for x in sql_version.split('.')] >= [2,0,0]) +# APP_MODE - production, development, or test +APP_MODE = os.environ.get('APP_MODE', 'production') + # if installed via pip this variable is set to true (empty file with name .HOMEDIR present) HOME_CONFIG = os.path.isfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), '.HOMEDIR')) -#In executables updater is not available, so variable is set to False there +# In executables updater is not available, so variable is set to False there UPDATER_AVAILABLE = True # Base dir is parent of current file, necessary if called from different folder @@ -43,7 +46,7 @@ TRANSLATIONS_DIR = os.path.join(BASE_DIR, 'cps', 'translations') # Cache dir - use CACHE_DIR environment variable, otherwise use the default directory: cps/cache DEFAULT_CACHE_DIR = os.path.join(BASE_DIR, 'cps', 'cache') -CACHE_DIR = os.environ.get("CACHE_DIR", DEFAULT_CACHE_DIR) +CACHE_DIR = os.environ.get('CACHE_DIR', DEFAULT_CACHE_DIR) if HOME_CONFIG: home_dir = os.path.join(os.path.expanduser("~"),".calibre-web") diff --git a/cps/db.py b/cps/db.py index 296db7a4..e4cea100 100644 --- a/cps/db.py +++ b/cps/db.py @@ -443,11 +443,11 @@ class CalibreDB(): """ self.session = None if self._init: - self.initSession(expire_on_commit) + self.init_session(expire_on_commit) self.instances.add(self) - def initSession(self, expire_on_commit=True): + def init_session(self, expire_on_commit=True): self.session = self.session_factory() self.session.expire_on_commit = expire_on_commit self.update_title_sort(self.config) @@ -593,7 +593,7 @@ class CalibreDB(): autoflush=True, bind=cls.engine)) for inst in cls.instances: - inst.initSession() + inst.init_session() cls._init = True return True diff --git a/cps/helper.py b/cps/helper.py index 2f2df0e0..a221dbb7 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -57,7 +57,8 @@ from . import logger, config, get_locale, db, fs, ub from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES from .subproc_wrapper import process_wait -from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS +from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS, STAT_ENDED, \ + STAT_CANCELLED from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache @@ -838,12 +839,22 @@ def render_task_status(tasklist): ret['status'] = _(u'Started') elif task.stat == STAT_FINISH_SUCCESS: ret['status'] = _(u'Finished') + elif task.stat == STAT_ENDED: + ret['status'] = _(u'Ended') + elif task.stat == STAT_CANCELLED: + ret['status'] = _(u'Cancelled') else: ret['status'] = _(u'Unknown Status') - ret['taskMessage'] = "{}: {}".format(_(task.name), task.message) + ret['taskMessage'] = "{}: {}".format(_(task.name), task.message) if task.message else _(task.name) ret['progress'] = "{} %".format(int(task.progress * 100)) ret['user'] = escape(user) # prevent xss + + # Hidden fields + ret['id'] = task.id + ret['stat'] = task.stat + ret['is_cancellable'] = task.is_cancellable + renderedtasklist.append(ret) return renderedtasklist @@ -914,5 +925,5 @@ def get_download_link(book_id, book_format, client): abort(404) -def clear_cover_thumbnail_cache(book_id=None): +def clear_cover_thumbnail_cache(book_id): WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id)) diff --git a/cps/schedule.py b/cps/schedule.py index 2cddaecb..2bb7878f 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -17,29 +17,70 @@ # along with this program. If not, see . from __future__ import division, print_function, unicode_literals +import datetime +from . import config, constants from .services.background_scheduler import BackgroundScheduler from .tasks.database import TaskReconnectDatabase from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails +from .services.worker import WorkerThread -def register_jobs(): +def get_scheduled_tasks(reconnect=True): + tasks = list() + + # Reconnect Calibre database (metadata.db) + if reconnect: + tasks.append(lambda: TaskReconnectDatabase()) + + # Generate all missing book cover thumbnails + if config.schedule_generate_book_covers: + tasks.append(lambda: TaskGenerateCoverThumbnails()) + + # Generate all missing series thumbnails + if config.schedule_generate_series_covers: + tasks.append(lambda: TaskGenerateSeriesThumbnails()) + + return tasks + + +def end_scheduled_tasks(): + worker = WorkerThread.get_instance() + for __, __, __, task in worker.tasks: + if task.scheduled and task.is_cancellable: + worker.end_task(task.id) + + +def register_scheduled_tasks(): scheduler = BackgroundScheduler() if scheduler: - # Reconnect Calibre database (metadata.db) - scheduler.schedule_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', hour='4,16') + # Remove all existing jobs + scheduler.remove_all_jobs() - # Generate all missing book cover thumbnails - scheduler.schedule_task(user=None, task=lambda: TaskGenerateCoverThumbnails(), trigger='cron', hour=4) + start = config.schedule_start_time + end = config.schedule_end_time - # Generate all missing series thumbnails - scheduler.schedule_task(user=None, task=lambda: TaskGenerateSeriesThumbnails(), trigger='cron', hour=4) + # Register scheduled tasks + if start != end: + scheduler.schedule_tasks(tasks=get_scheduled_tasks(), trigger='cron', hour=start) + scheduler.schedule(func=end_scheduled_tasks, trigger='cron', hour=end) + + # Kick-off tasks, if they should currently be running + now = datetime.datetime.now().hour + if start <= now < end: + scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False)) -def register_startup_jobs(): +def register_startup_tasks(): scheduler = BackgroundScheduler() if scheduler: - scheduler.schedule_task_immediately(None, task=lambda: TaskGenerateCoverThumbnails()) - scheduler.schedule_task_immediately(None, task=lambda: TaskGenerateSeriesThumbnails()) + start = config.schedule_start_time + end = config.schedule_end_time + now = datetime.datetime.now().hour + + # Run scheduled tasks immediately for development and testing + # Ignore tasks that should currently be running, as these will be added when registering scheduled tasks + if constants.APP_MODE in ['development', 'test'] and not (start <= now < end): + scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False)) diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py index ba578903..971b0bf7 100644 --- a/cps/services/background_scheduler.py +++ b/cps/services/background_scheduler.py @@ -48,23 +48,38 @@ class BackgroundScheduler: return cls._instance - def _add(self, func, trigger, **trigger_args): + def schedule(self, func, trigger, **trigger_args): if use_APScheduler: return self.scheduler.add_job(func=func, trigger=trigger, **trigger_args) - # Expects a lambda expression for the task, so that the task isn't instantiated before the task is scheduled - def schedule_task(self, user, task, trigger, **trigger_args): + # Expects a lambda expression for the task + def schedule_task(self, task, user=None, trigger='cron', **trigger_args): if use_APScheduler: def scheduled_task(): worker_task = task() + worker_task.scheduled = True WorkerThread.add(user, worker_task) + return self.schedule(func=scheduled_task, trigger=trigger, **trigger_args) - return self._add(func=scheduled_task, trigger=trigger, **trigger_args) - - # Expects a lambda expression for the task, so that the task isn't instantiated before the task is scheduled - def schedule_task_immediately(self, user, task): + # Expects a list of lambda expressions for the tasks + def schedule_tasks(self, tasks, user=None, trigger='cron', **trigger_args): if use_APScheduler: - def scheduled_task(): - WorkerThread.add(user, task()) + for task in tasks: + self.schedule_task(task, user=user, trigger=trigger, **trigger_args) - return self._add(func=scheduled_task, trigger='date') + # Expects a lambda expression for the task + def schedule_task_immediately(self, task, user=None): + if use_APScheduler: + def immediate_task(): + WorkerThread.add(user, task()) + return self.schedule(func=immediate_task, trigger='date') + + # Expects a list of lambda expressions for the tasks + def schedule_tasks_immediately(self, tasks, user=None): + if use_APScheduler: + for task in tasks: + self.schedule_task_immediately(task, user) + + # Remove all jobs + def remove_all_jobs(self): + self.scheduler.remove_all_jobs() diff --git a/cps/services/worker.py b/cps/services/worker.py index 97068f74..04a4c056 100644 --- a/cps/services/worker.py +++ b/cps/services/worker.py @@ -21,6 +21,8 @@ STAT_WAITING = 0 STAT_FAIL = 1 STAT_STARTED = 2 STAT_FINISH_SUCCESS = 3 +STAT_ENDED = 4 +STAT_CANCELLED = 5 # Only retain this many tasks in dequeued list TASK_CLEANUP_TRIGGER = 20 @@ -50,7 +52,7 @@ class WorkerThread(threading.Thread): _instance = None @classmethod - def getInstance(cls): + def get_instance(cls): if cls._instance is None: cls._instance = WorkerThread() return cls._instance @@ -67,12 +69,13 @@ class WorkerThread(threading.Thread): @classmethod def add(cls, user, task): - ins = cls.getInstance() + ins = cls.get_instance() ins.num += 1 - log.debug("Add Task for user: {}: {}".format(user, task)) + username = user if user is not None else 'System' + log.debug("Add Task for user: {}: {}".format(username, task)) ins.queue.put(QueuedTask( num=ins.num, - user=user, + user=username, added=datetime.now(), task=task, )) @@ -134,6 +137,12 @@ class WorkerThread(threading.Thread): self.queue.task_done() + def end_task(self, task_id): + ins = self.get_instance() + for __, __, __, task in ins.tasks: + if str(task.id) == str(task_id) and task.is_cancellable: + task.stat = STAT_CANCELLED if task.stat == STAT_WAITING else STAT_ENDED + class CalibreTask: __metaclass__ = abc.ABCMeta @@ -147,10 +156,11 @@ class CalibreTask: self.message = message self.id = uuid.uuid4() self.self_cleanup = False + self._scheduled = False @abc.abstractmethod def run(self, worker_thread): - """Provides the caller some human-readable name for this class""" + """The main entry-point for this task""" raise NotImplementedError @abc.abstractmethod @@ -158,6 +168,11 @@ class CalibreTask: """Provides the caller some human-readable name for this class""" raise NotImplementedError + @abc.abstractmethod + def is_cancellable(self): + """Does this task gracefully handle being cancelled (STAT_ENDED, STAT_CANCELLED)?""" + raise NotImplementedError + def start(self, *args): self.start_time = datetime.now() self.stat = STAT_STARTED @@ -208,7 +223,7 @@ class CalibreTask: We have a separate dictating this because there may be certain tasks that want to override this """ # By default, we're good to clean a task if it's "Done" - return self.stat in (STAT_FINISH_SUCCESS, STAT_FAIL) + return self.stat in (STAT_FINISH_SUCCESS, STAT_FAIL, STAT_ENDED, STAT_CANCELLED) '''@progress.setter def progress(self, x): @@ -226,6 +241,14 @@ class CalibreTask: def self_cleanup(self, is_self_cleanup): self._self_cleanup = is_self_cleanup + @property + def scheduled(self): + return self._scheduled + + @scheduled.setter + def scheduled(self, is_scheduled): + self._scheduled = is_scheduled + def _handleError(self, error_message): self.stat = STAT_FAIL self.progress = 1 diff --git a/cps/static/css/caliBlur.css b/cps/static/css/caliBlur.css index b4fa6045..b2b35423 100644 --- a/cps/static/css/caliBlur.css +++ b/cps/static/css/caliBlur.css @@ -5150,7 +5150,7 @@ body.login > div.navbar.navbar-default.navbar-static-top > div > div.navbar-head pointer-events: none } -#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #ClearCacheDialog:hover:before, #deleteButton, #deleteModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover { +#DeleteDomain:hover:before, #RestartDialog:hover:before, #ShutdownDialog:hover:before, #StatusDialog:hover:before, #deleteButton, #deleteModal:hover:before, #cancelTaskModal:hover:before, body.mailset > div.container-fluid > div > div.col-sm-10 > div.discover td > a:hover { cursor: pointer } @@ -5237,6 +5237,10 @@ body.admin > div.container-fluid > div > div.col-sm-10 > div.container-fluid > d margin-bottom: 20px } +body.admin > div.container-fluid div.scheduled_tasks_details { + margin-bottom: 20px +} + body.admin .btn-default { margin-bottom: 10px } @@ -5468,7 +5472,7 @@ body.admin.modal-open .navbar { z-index: 0 !important } -#RestartDialog, #ShutdownDialog, #StatusDialog, #ClearCacheDialog, #deleteModal { +#RestartDialog, #ShutdownDialog, #StatusDialog, #deleteModal, #cancelTaskModal { top: 0; overflow: hidden; padding-top: 70px; @@ -5478,7 +5482,7 @@ body.admin.modal-open .navbar { background: rgba(0, 0, 0, .5) } -#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #ClearCacheDialog:before, #deleteModal:before { +#RestartDialog:before, #ShutdownDialog:before, #StatusDialog:before, #deleteModal:before, #cancelTaskModal:before { content: "\E208"; padding-right: 10px; display: block; @@ -5500,18 +5504,18 @@ body.admin.modal-open .navbar { z-index: 99 } -#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #ClearCacheDialog.in:before, #deleteModal.in:before { +#RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before, #cancelTaskModal.in:before { -webkit-transform: translate(0, 0); -ms-transform: translate(0, 0); transform: translate(0, 0) } -#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #ClearCacheDialog > .modal-dialog, #deleteModal > .modal-dialog { +#RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog, #cancelTaskModal > .modal-dialog { width: 450px; margin: auto } -#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #ClearCacheDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { +#RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content, #cancelTaskModal > .modal-dialog > .modal-content { max-height: calc(100% - 90px); -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, .5); box-shadow: 0 5px 15px rgba(0, 0, 0, .5); @@ -5522,7 +5526,7 @@ body.admin.modal-open .navbar { width: 450px } -#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header { +#RestartDialog > .modal-dialog > .modal-content > .modal-header, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header, #StatusDialog > .modal-dialog > .modal-content > .modal-header, #deleteModal > .modal-dialog > .modal-content > .modal-header, #cancelTaskModal > .modal-dialog > .modal-content > .modal-header { padding: 15px 20px; border-radius: 3px 3px 0 0; line-height: 1.71428571; @@ -5535,7 +5539,7 @@ body.admin.modal-open .navbar { text-align: left } -#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before { +#RestartDialog > .modal-dialog > .modal-content > .modal-header:before, #ShutdownDialog > .modal-dialog > .modal-content > .modal-header:before, #StatusDialog > .modal-dialog > .modal-content > .modal-header:before, #deleteModal > .modal-dialog > .modal-content > .modal-header:before, #cancelTaskModal > .modal-dialog > .modal-content > .modal-header:before { padding-right: 10px; font-size: 18px; color: #999; @@ -5559,12 +5563,12 @@ body.admin.modal-open .navbar { font-family: plex-icons-new, serif } -#ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:before { - content: "\EA15"; +#deleteModal > .modal-dialog > .modal-content > .modal-header:before { + content: "\EA6D"; font-family: plex-icons-new, serif } -#deleteModal > .modal-dialog > .modal-content > .modal-header:before { +#cancelTaskModal > .modal-dialog > .modal-content > .modal-header:before { content: "\EA6D"; font-family: plex-icons-new, serif } @@ -5587,19 +5591,19 @@ body.admin.modal-open .navbar { font-size: 20px } -#ClearCacheDialog > .modal-dialog > .modal-content > .modal-header:after { - content: "Clear Cover Thumbnail Cache"; - display: inline-block; - font-size: 20px -} - #deleteModal > .modal-dialog > .modal-content > .modal-header:after { content: "Delete Book"; display: inline-block; font-size: 20px } -#StatusDialog > .modal-dialog > .modal-content > .modal-header > span, #deleteModal > .modal-dialog > .modal-content > .modal-header > span, #loader > center > img, .rating-mobile { +#cancelTaskModal > .modal-dialog > .modal-content > .modal-header:after { + content: "Delete Book"; + display: inline-block; + font-size: 20px +} + +#StatusDialog > .modal-dialog > .modal-content > .modal-header > span, #deleteModal > .modal-dialog > .modal-content > .modal-header > span, #cancelTaskModal > .modal-dialog > .modal-content > .modal-header > span, #loader > center > img, .rating-mobile { display: none } @@ -5613,7 +5617,7 @@ body.admin.modal-open .navbar { text-align: left } -#ShutdownDialog > .modal-dialog > .modal-content > .modal-body, #StatusDialog > .modal-dialog > .modal-content > .modal-body, #deleteModal > .modal-dialog > .modal-content > .modal-body { +#ShutdownDialog > .modal-dialog > .modal-content > .modal-body, #StatusDialog > .modal-dialog > .modal-content > .modal-body, #deleteModal > .modal-dialog > .modal-content > .modal-body, #cancelTaskModal > .modal-dialog > .modal-content > .modal-body { padding: 20px 20px 40px; font-size: 16px; line-height: 1.6em; @@ -5623,17 +5627,7 @@ body.admin.modal-open .navbar { text-align: left } -#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body { - padding: 20px 20px 10px; - font-size: 16px; - line-height: 1.6em; - font-family: Open Sans Regular, Helvetica Neue, Helvetica, Arial, sans-serif; - color: #eee; - background: #282828; - text-align: left -} - -#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p { +#RestartDialog > .modal-dialog > .modal-content > .modal-body > p, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > p, #StatusDialog > .modal-dialog > .modal-content > .modal-body > p, #deleteModal > .modal-dialog > .modal-content > .modal-body > p, #cancelTaskModal > .modal-dialog > .modal-content > .modal-body > p { padding: 20px 20px 0 0; font-size: 16px; line-height: 1.6em; @@ -5642,7 +5636,7 @@ body.admin.modal-open .navbar { background: #282828 } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { +#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart), #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown), #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default, #cancelTaskModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { float: right; z-index: 9; position: relative; @@ -5678,11 +5672,11 @@ body.admin.modal-open .navbar { border-radius: 3px } -#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > #clear_cache { +#deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-danger { float: right; z-index: 9; position: relative; - margin: 25px 0 0 10px; + margin: 0 0 0 10px; min-width: 80px; padding: 10px 18px; font-size: 16px; @@ -5690,7 +5684,7 @@ body.admin.modal-open .navbar { border-radius: 3px } -#deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-danger { +#cancelTaskModal > .modal-dialog > .modal-content > .modal-footer > .btn-danger { float: right; z-index: 9; position: relative; @@ -5710,15 +5704,15 @@ body.admin.modal-open .navbar { margin: 55px 0 0 10px } -#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache) { - margin: 25px 0 0 10px -} - #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { margin: 0 0 0 10px } -#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #ClearCacheDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#clear_cache):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover { +#cancelTaskModal > .modal-dialog > .modal-content > .modal-footer > .btn-default { + margin: 0 0 0 10px +} + +#RestartDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#restart):hover, #ShutdownDialog > .modal-dialog > .modal-content > .modal-body > .btn-default:not(#shutdown):hover, #deleteModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover, #cancelTaskModal > .modal-dialog > .modal-content > .modal-footer > .btn-default:hover { background-color: hsla(0, 0%, 100%, .3) } @@ -5752,21 +5746,6 @@ body.admin.modal-open .navbar { box-shadow: 0 5px 15px rgba(0, 0, 0, .5) } -#ClearCacheDialog > .modal-dialog > .modal-content > .modal-body:after { - content: ''; - position: absolute; - width: 100%; - height: 72px; - background-color: #323232; - border-radius: 0 0 3px 3px; - left: 0; - margin-top: 10px; - z-index: 0; - border-top: 1px solid #222; - -webkit-box-shadow: 0 5px 15px rgba(0, 0, 0, .5); - box-shadow: 0 5px 15px rgba(0, 0, 0, .5) -} - #deleteButton { position: fixed; top: 60px; @@ -7355,11 +7334,11 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. background-color: transparent !important } - #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #ClearCacheDialog > .modal-dialog, #deleteModal > .modal-dialog { + #RestartDialog > .modal-dialog, #ShutdownDialog > .modal-dialog, #StatusDialog > .modal-dialog, #deleteModal > .modal-dialog, #cancelTaskModal > .modal-dialog { max-width: calc(100vw - 40px) } - #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #ClearCacheDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content { + #RestartDialog > .modal-dialog > .modal-content, #ShutdownDialog > .modal-dialog > .modal-content, #StatusDialog > .modal-dialog > .modal-content, #deleteModal > .modal-dialog > .modal-content, #cancelTaskModal > .modal-dialog > .modal-content { max-width: calc(100vw - 40px); left: 0 } @@ -7509,7 +7488,7 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div. padding: 30px 15px } - #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #ClearCacheDialog.in:before, #deleteModal.in:before { + #RestartDialog.in:before, #ShutdownDialog.in:before, #StatusDialog.in:before, #deleteModal.in:before, #cancelTaskModal.in:before { left: auto; right: 34px } diff --git a/cps/static/js/main.js b/cps/static/js/main.js index 5537d189..988e3b9f 100644 --- a/cps/static/js/main.js +++ b/cps/static/js/main.js @@ -454,18 +454,6 @@ $(function() { } }); }); - $("#clear_cache").click(function () { - $("#spinner3").show(); - $.ajax({ - dataType: "json", - url: window.location.pathname + "/../../clear-cache", - data: {"cache_type":"thumbnails"}, - success: function(data) { - $("#spinner3").hide(); - $("#ClearCacheDialog").modal("hide"); - } - }); - }); // Init all data control handlers to default $("input[data-control]").trigger("change"); diff --git a/cps/static/js/table.js b/cps/static/js/table.js index a55ec5d1..dc4ab4ab 100644 --- a/cps/static/js/table.js +++ b/cps/static/js/table.js @@ -15,7 +15,7 @@ * along with this program. If not, see . */ -/* exported TableActions, RestrictionActions, EbookActions, responseHandler */ +/* exported TableActions, RestrictionActions, EbookActions, TaskActions, responseHandler */ /* global getPath, confirmDialog */ var selections = []; @@ -42,6 +42,24 @@ $(function() { }, 1000); } + $("#cancel_task_confirm").click(function() { + //get data-id attribute of the clicked element + var taskId = $(this).data("task-id"); + $.ajax({ + method: "post", + contentType: "application/json; charset=utf-8", + dataType: "json", + url: window.location.pathname + "/../ajax/canceltask", + data: JSON.stringify({"task_id": taskId}), + }); + }); + //triggered when modal is about to be shown + $("#cancelTaskModal").on("show.bs.modal", function(e) { + //get data-id attribute of the clicked element and store in button + var taskId = $(e.relatedTarget).data("task-id"); + $(e.currentTarget).find("#cancel_task_confirm").data("task-id", taskId); + }); + $("#books-table").on("check.bs.table check-all.bs.table uncheck.bs.table uncheck-all.bs.table", function (e, rowsAfter, rowsBefore) { var rows = rowsAfter; @@ -576,6 +594,7 @@ function handle_header_buttons () { $(".header_select").removeAttr("disabled"); } } + /* Function for deleting domain restrictions */ function TableActions (value, row) { return [ @@ -613,6 +632,19 @@ function UserActions (value, row) { ].join(""); } +/* Function for cancelling tasks */ +function TaskActions (value, row) { + var cancellableStats = [0, 1, 2]; + if (row.id && row.is_cancellable && cancellableStats.includes(row.stat)) { + return [ + "
", + "", + "
" + ].join(""); + } + return ''; +} + /* Function for keeping checked rows */ function responseHandler(res) { $.each(res.rows, function (i, row) { diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 56cc7076..f150a397 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -234,3 +234,7 @@ class TaskConvert(CalibreTask): @property def name(self): return "Convert" + + @property + def is_cancellable(self): + return False diff --git a/cps/tasks/database.py b/cps/tasks/database.py index 11f0186d..0441d564 100644 --- a/cps/tasks/database.py +++ b/cps/tasks/database.py @@ -47,3 +47,7 @@ class TaskReconnectDatabase(CalibreTask): @property def name(self): return "Reconnect Database" + + @property + def is_cancellable(self): + return False diff --git a/cps/tasks/mail.py b/cps/tasks/mail.py index 292114d5..24064bd3 100644 --- a/cps/tasks/mail.py +++ b/cps/tasks/mail.py @@ -162,7 +162,6 @@ class TaskEmail(CalibreTask): log.debug_or_exception(ex) self._handleError(u'Error sending e-mail: {}'.format(ex)) - def send_standard_email(self, msg): use_ssl = int(self.settings.get('mail_use_ssl', 0)) timeout = 600 # set timeout to 5mins @@ -218,7 +217,6 @@ class TaskEmail(CalibreTask): self.asyncSMTP = None self._progress = x - @classmethod def _get_attachment(cls, bookpath, filename): """Get file as MIMEBase message""" @@ -260,5 +258,9 @@ class TaskEmail(CalibreTask): def name(self): return "E-mail" + @property + def is_cancellable(self): + return False + def __str__(self): return "{}, {}".format(self.name, self.subject) diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index a220fd8c..d147f10d 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) -# Copyright (C) 2020 mmonkey +# Copyright (C) 2020 monkey # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,8 +21,8 @@ import os from .. import constants from cps import config, db, fs, gdriveutils, logger, ub -from cps.services.worker import CalibreTask -from datetime import datetime, timedelta +from cps.services.worker import CalibreTask, STAT_CANCELLED, STAT_ENDED +from datetime import datetime from sqlalchemy import func, text, or_ try: @@ -67,7 +67,7 @@ def get_best_fit(width, height, image_width, image_height): class TaskGenerateCoverThumbnails(CalibreTask): - def __init__(self, task_message=u'Generating cover thumbnails'): + def __init__(self, task_message=''): super(TaskGenerateCoverThumbnails, self).__init__(task_message) self.log = logger.create() self.app_db_session = ub.get_new_session_instance() @@ -79,13 +79,14 @@ class TaskGenerateCoverThumbnails(CalibreTask): ] def run(self, worker_thread): - if self.calibre_db.session and use_IM: + if self.calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED: + self.message = 'Scanning Books' books_with_covers = self.get_books_with_covers() count = len(books_with_covers) - updated = 0 - generated = 0 + total_generated = 0 for i, book in enumerate(books_with_covers): + generated = 0 book_cover_thumbnails = self.get_book_cover_thumbnails(book.id) # Generate new thumbnails for missing covers @@ -98,16 +99,32 @@ class TaskGenerateCoverThumbnails(CalibreTask): # Replace outdated or missing thumbnails for thumbnail in book_cover_thumbnails: if book.last_modified > thumbnail.generated_at: - updated += 1 + generated += 1 self.update_book_cover_thumbnail(book, thumbnail) elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS): - updated += 1 + generated += 1 self.update_book_cover_thumbnail(book, thumbnail) - self.message = u'Processing book {0} of {1}'.format(i + 1, count) + # Increment the progress self.progress = (1.0 / count) * i + if generated > 0: + total_generated += generated + self.message = u'Generated {0} cover thumbnails'.format(total_generated) + + # Check if job has been cancelled or ended + if self.stat == STAT_CANCELLED: + self.log.info(f'GenerateCoverThumbnails task has been cancelled.') + return + + if self.stat == STAT_ENDED: + self.log.info(f'GenerateCoverThumbnails task has been ended.') + return + + if total_generated == 0: + self.self_cleanup = True + self._handleSuccess() self.app_db_session.remove() @@ -180,7 +197,8 @@ class TaskGenerateCoverThumbnails(CalibreTask): self.log.info(u'Error generating thumbnail file: ' + str(ex)) raise ex finally: - stream.close() + if stream is not None: + stream.close() else: book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') if not os.path.isfile(book_cover_filepath): @@ -197,11 +215,15 @@ class TaskGenerateCoverThumbnails(CalibreTask): @property def name(self): - return "ThumbnailsGenerate" + return 'GenerateCoverThumbnails' + + @property + def is_cancellable(self): + return True class TaskGenerateSeriesThumbnails(CalibreTask): - def __init__(self, task_message=u'Generating series thumbnails'): + def __init__(self, task_message=''): super(TaskGenerateSeriesThumbnails, self).__init__(task_message) self.log = logger.create() self.app_db_session = ub.get_new_session_instance() @@ -209,17 +231,18 @@ class TaskGenerateSeriesThumbnails(CalibreTask): self.cache = fs.FileSystem() self.resolutions = [ constants.COVER_THUMBNAIL_SMALL, - constants.COVER_THUMBNAIL_MEDIUM + constants.COVER_THUMBNAIL_MEDIUM, ] def run(self, worker_thread): - if self.calibre_db.session and use_IM: + if self.calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED: + self.message = 'Scanning Series' all_series = self.get_series_with_four_plus_books() count = len(all_series) - updated = 0 - generated = 0 + total_generated = 0 for i, series in enumerate(all_series): + generated = 0 series_thumbnails = self.get_series_thumbnails(series.id) series_books = self.get_series_books(series.id) @@ -233,16 +256,32 @@ class TaskGenerateSeriesThumbnails(CalibreTask): # Replace outdated or missing thumbnails for thumbnail in series_thumbnails: if any(book.last_modified > thumbnail.generated_at for book in series_books): - updated += 1 + generated += 1 self.update_series_thumbnail(series_books, thumbnail) elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS): - updated += 1 + generated += 1 self.update_series_thumbnail(series_books, thumbnail) - self.message = u'Processing series {0} of {1}'.format(i + 1, count) + # Increment the progress self.progress = (1.0 / count) * i + if generated > 0: + total_generated += generated + self.message = u'Generated {0} series thumbnails'.format(total_generated) + + # Check if job has been cancelled or ended + if self.stat == STAT_CANCELLED: + self.log.info(f'GenerateSeriesThumbnails task has been cancelled.') + return + + if self.stat == STAT_ENDED: + self.log.info(f'GenerateSeriesThumbnails task has been ended.') + return + + if total_generated == 0: + self.self_cleanup = True + self._handleSuccess() self.app_db_session.remove() @@ -302,7 +341,8 @@ class TaskGenerateSeriesThumbnails(CalibreTask): self.app_db_session.rollback() def generate_series_thumbnail(self, series_books, thumbnail): - books = series_books[:4] + # Get the last four books in the series based on series_index + books = sorted(series_books, key=lambda b: float(b.series_index), reverse=True)[:4] top = 0 left = 0 @@ -342,7 +382,8 @@ class TaskGenerateSeriesThumbnails(CalibreTask): self.log.info(u'Error generating thumbnail file: ' + str(ex)) raise ex finally: - stream.close() + if stream is not None: + stream.close() book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') if not os.path.isfile(book_cover_filepath): @@ -380,11 +421,15 @@ class TaskGenerateSeriesThumbnails(CalibreTask): @property def name(self): - return "SeriesThumbnailGenerate" + return 'GenerateSeriesThumbnails' + + @property + def is_cancellable(self): + return True class TaskClearCoverThumbnailCache(CalibreTask): - def __init__(self, book_id=None, task_message=u'Clearing cover thumbnail cache'): + def __init__(self, book_id, task_message=u'Clearing cover thumbnail cache'): super(TaskClearCoverThumbnailCache, self).__init__(task_message) self.log = logger.create() self.book_id = book_id @@ -397,8 +442,6 @@ class TaskClearCoverThumbnailCache(CalibreTask): thumbnails = self.get_thumbnails_for_book(self.book_id) for thumbnail in thumbnails: self.delete_thumbnail(thumbnail) - else: - self.delete_all_thumbnails() self._handleSuccess() self.app_db_session.remove() @@ -411,19 +454,19 @@ class TaskClearCoverThumbnailCache(CalibreTask): .all() def delete_thumbnail(self, thumbnail): + thumbnail.expiration = datetime.utcnow() + try: + self.app_db_session.commit() self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) except Exception as ex: self.log.info(u'Error deleting book thumbnail: ' + str(ex)) self._handleError(u'Error deleting book thumbnail: ' + str(ex)) - def delete_all_thumbnails(self): - try: - self.cache.delete_cache_dir(constants.CACHE_TYPE_THUMBNAILS) - except Exception as ex: - self.log.info(u'Error deleting book thumbnails: ' + str(ex)) - self._handleError(u'Error deleting book thumbnails: ' + str(ex)) - @property def name(self): - return "ThumbnailsClear" + return 'ThumbnailsClear' + + @property + def is_cancellable(self): + return False diff --git a/cps/tasks/upload.py b/cps/tasks/upload.py index d7ef34c2..9f58bf16 100644 --- a/cps/tasks/upload.py +++ b/cps/tasks/upload.py @@ -16,3 +16,7 @@ class TaskUpload(CalibreTask): @property def name(self): return "Upload" + + @property + def is_cancellable(self): + return False diff --git a/cps/templates/admin.html b/cps/templates/admin.html index ec0fc84e..3fd55ae2 100644 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -159,7 +159,7 @@

{{_('Scheduled Tasks')}}

-
+
{{_('Time at which tasks start to run')}}
{{config.schedule_start_time}}:00
diff --git a/cps/templates/schedule_edit.html b/cps/templates/schedule_edit.html index f4e72224..71bb2d1a 100644 --- a/cps/templates/schedule_edit.html +++ b/cps/templates/schedule_edit.html @@ -11,7 +11,7 @@
@@ -19,12 +19,12 @@
- +
diff --git a/cps/templates/tasks.html b/cps/templates/tasks.html index c13ddff9..b36a6daa 100644 --- a/cps/templates/tasks.html +++ b/cps/templates/tasks.html @@ -16,6 +16,9 @@ {{_('Progress')}} {{_('Run Time')}} {{_('Start Time')}} + {% if g.user.role_admin() %} + {{_('Actions')}} + {% endif %} @@ -23,6 +26,30 @@
{% endblock %} +{% block modal %} +{{ delete_book() }} +{% if g.user.role_admin() %} + +{% endif %} +{% endblock %} {% block js %} diff --git a/cps/web.py b/cps/web.py index 88a5c0ab..c5ad2265 100644 --- a/cps/web.py +++ b/cps/web.py @@ -124,7 +124,7 @@ def viewer_required(f): @web.route("/ajax/emailstat") @login_required def get_email_status_json(): - tasks = WorkerThread.getInstance().tasks + tasks = WorkerThread.get_instance().tasks return jsonify(render_task_status(tasks)) @@ -1055,7 +1055,7 @@ def category_list(): @login_required def get_tasks_status(): # if current user admin, show all email, otherwise only own emails - tasks = WorkerThread.getInstance().tasks + tasks = WorkerThread.get_instance().tasks answer = render_task_status(tasks) return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks") From 9e7f69e38a2958d63d1c0a1299687d82ff686417 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Wed, 29 Sep 2021 03:01:28 -0500 Subject: [PATCH 020/112] Updated series cache timeout to one day --- cps/templates/image.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/templates/image.html b/cps/templates/image.html index ae57727b..728384a1 100644 --- a/cps/templates/image.html +++ b/cps/templates/image.html @@ -16,7 +16,7 @@ {% set srcset = series|get_series_srcset %} {{ book_title }} From cd3791f5f4b85581705117dcaff9f18bbe052336 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Thu, 30 Sep 2021 01:43:31 -0500 Subject: [PATCH 021/112] Always use full-sized image for book edit and details pages --- cps/__init__.py | 3 +-- cps/admin.py | 2 +- cps/templates/book_edit.html | 4 ++-- cps/templates/detail.html | 3 ++- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cps/__init__.py b/cps/__init__.py index 838da5b4..af4a82e8 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -114,9 +114,8 @@ def create_app(): if os.environ.get('FLASK_DEBUG'): cache_buster.init_cache_busting(app) - cache_buster.init_cache_busting(app) - log.info('Starting Calibre Web...') + Principal(app) lm.init_app(app) app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session)) diff --git a/cps/admin.py b/cps/admin.py index 62b3dbe0..4d168aa0 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -40,7 +40,7 @@ from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError from sqlalchemy.sql.expression import func, or_, text -from . import constants, logger, helper, services, isoLanguages +from . import constants, logger, helper, services from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils, schedule from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ valid_email, check_username diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index b5f5df7a..12d3bd07 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -1,10 +1,10 @@ -{% import 'image.html' as image %} {% extends "layout.html" %} {% block body %} {% if book %}
- {{ image.book_cover(book) }} + +
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index 7985691a..19cd079e 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,7 +4,8 @@
- {{ image.book_cover(entry) }} + +
From 341632376768b10664b78f367c2d948861181e02 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Wed, 26 Jan 2022 23:56:10 -0600 Subject: [PATCH 022/112] Removed title tags from cover images --- cps/templates/author.html | 2 +- cps/templates/discover.html | 2 +- cps/templates/grid.html | 2 +- cps/templates/image.html | 7 ++----- cps/templates/index.html | 4 ++-- cps/templates/search.html | 2 +- cps/templates/shelf.html | 2 +- 7 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cps/templates/author.html b/cps/templates/author.html index 449efa53..9f124876 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -37,7 +37,7 @@
- {{ image.book_cover(entry, title=author.name|safe) }} + {{ image.book_cover(entry, alt=author.name|safe) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/discover.html b/cps/templates/discover.html index a39ba674..4842a7ea 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -10,7 +10,7 @@ {% if entry.has_cover is defined %} - {{ image.book_cover(entry, alt=entry.title) }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/grid.html b/cps/templates/grid.html index 7d3add5e..638b7245 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -28,7 +28,7 @@
- {{ image.series(entry[0].series[0], title=entry[0].series[0].name|shortentitle) }} + {{ image.series(entry[0].series[0], alt=entry[0].series[0].name|shortentitle) }} {{entry.count}} diff --git a/cps/templates/image.html b/cps/templates/image.html index 728384a1..0bdba9a5 100644 --- a/cps/templates/image.html +++ b/cps/templates/image.html @@ -1,23 +1,20 @@ -{% macro book_cover(book, title=None, alt=None) -%} +{% macro book_cover(book, alt=None) -%} {%- set image_title = book.title if book.title else book.name -%} - {%- set image_title = title if title else image_title -%} {%- set image_alt = alt if alt else image_title -%} {% set srcset = book|get_cover_srcset %} {{ image_alt }} {%- endmacro %} -{% macro series(series, title=None, alt=None) -%} +{% macro series(series, alt=None) -%} {%- set image_alt = alt if alt else image_title -%} {% set srcset = series|get_series_srcset %} {{ book_title }} {%- endmacro %} diff --git a/cps/templates/index.html b/cps/templates/index.html index ef872ccc..ff7ff115 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -10,7 +10,7 @@
- {{ image.book_cover(entry, alt=entry.title) }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} @@ -92,7 +92,7 @@
- {{ image.book_cover(entry, alt=entry.title) }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} diff --git a/cps/templates/search.html b/cps/templates/search.html index 1bc741ca..12966348 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -46,7 +46,7 @@ {% if entry.Books.has_cover is defined %} - {{ image.book_cover(entry.Books, alt=entry.Books.title) }} + {{ image.book_cover(entry.Books) }} {% if entry.Books.id in read_book_ids %}{% endif %} diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index 13a119c0..693f6b4a 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -35,7 +35,7 @@
- {{ image.book_cover(entry, alt=entry.title) }} + {{ image.book_cover(entry) }} {% if entry.id in read_book_ids %}{% endif %} From 50bb74d748f89668fe4abaac81b0c36ae77e66df Mon Sep 17 00:00:00 2001 From: mmonkey Date: Thu, 27 Jan 2022 00:35:45 -0600 Subject: [PATCH 023/112] Add CSRF support for schedule task settings, fixed details page not loading --- cps/schedule.py | 1 - cps/services/background_scheduler.py | 1 - cps/templates/detail.html | 2 +- cps/templates/schedule_edit.html | 1 + 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cps/schedule.py b/cps/schedule.py index 2bb7878f..39c9bc09 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from __future__ import division, print_function, unicode_literals import datetime from . import config, constants diff --git a/cps/services/background_scheduler.py b/cps/services/background_scheduler.py index 971b0bf7..415b2962 100644 --- a/cps/services/background_scheduler.py +++ b/cps/services/background_scheduler.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from __future__ import division, print_function, unicode_literals import atexit from .. import logger diff --git a/cps/templates/detail.html b/cps/templates/detail.html index 109b04d2..3bcf5cc6 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -5,7 +5,7 @@
- +
diff --git a/cps/templates/schedule_edit.html b/cps/templates/schedule_edit.html index 71bb2d1a..f989baf9 100644 --- a/cps/templates/schedule_edit.html +++ b/cps/templates/schedule_edit.html @@ -7,6 +7,7 @@

{{title}}

+
@@ -48,7 +48,7 @@ {% endif %}
- +
diff --git a/cps/templates/book_table.html b/cps/templates/book_table.html index 4b379b37..9ba173bb 100644 --- a/cps/templates/book_table.html +++ b/cps/templates/book_table.html @@ -6,7 +6,7 @@ data-escape="true" {% if g.user.role_edit() %} data-editable-type="text" - data-editable-url="{{ url_for('editbook.edit_list_book', param=parameter)}}" + data-editable-url="{{ url_for('edit-book.edit_list_book', param=parameter)}}" data-editable-title="{{ edit_text }}" data-edit="true" {% if validate %}data-edit-validate="{{ _('This Field is Required') }}" {% endif %} @@ -66,30 +66,30 @@ {{ text_table_row('authors', _('Enter Authors'),_('Authors'), true, true) }} {{ text_table_row('tags', _('Enter Categories'),_('Categories'), false, true) }} {{ text_table_row('series', _('Enter Series'),_('Series'), false, true) }} - {{_('Series Index')}} + {{_('Series Index')}} {{ text_table_row('languages', _('Enter Languages'),_('Languages'), false, true) }} {{ text_table_row('publishers', _('Enter Publishers'),_('Publishers'), false, true) }} - {{_('Comments')}} + {{_('Comments')}} {% if g.user.check_visibility(32768) %} {{ book_checkbox_row('is_archived', _('Archiv Status'), false)}} {% endif %} {{ book_checkbox_row('read_status', _('Read Status'), false)}} {% for c in cc %} {% if c.datatype == "int" %} - {{c.name}} + {{c.name}} {% elif c.datatype == "rating" %} - {{c.name}} + {{c.name}} {% elif c.datatype == "float" %} - {{c.name}} + {{c.name}} {% elif c.datatype == "enumeration" %} - {{c.name}} + {{c.name}} {% elif c.datatype in ["datetime"] %} {% elif c.datatype == "text" %} {{ text_table_row('custom_column_' + c.id|string, _('Enter ') + c.name, c.name, false, false) }} {% elif c.datatype == "comments" %} - {{c.name}} + {{c.name}} {% elif c.datatype == "bool" %} {{ book_checkbox_row('custom_column_' + c.id|string, c.name, false)}} {% else %} diff --git a/cps/templates/detail.html b/cps/templates/detail.html index c2153db8..38005cb9 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -138,7 +138,7 @@

{% for identifier in entry.identifiers %} - {{identifier.formatType()}} + {{identifier.format_type()}} {%endfor%}

@@ -295,7 +295,7 @@ {% if g.user.role_edit() %} {% endif %} diff --git a/cps/templates/layout.html b/cps/templates/layout.html index ec69c91b..0d0df778 100644 --- a/cps/templates/layout.html +++ b/cps/templates/layout.html @@ -60,7 +60,7 @@ {% if g.user.is_authenticated or g.allow_anonymous %} {% if g.user.role_upload() and g.allow_upload %}
  • - +
    {{_('Upload')}}", methods=['POST']) @login_required def toggle_archived(book_id): - is_archived = change_archived_books(book_id, message="Book {} archivebit toggled".format(book_id)) + is_archived = change_archived_books(book_id, message="Book {} archive bit toggled".format(book_id)) if is_archived: remove_synced_book(book_id) return "" @@ -230,6 +236,7 @@ def get_comic_book(book_id, book_format, page): return "", 204 ''' + # ################################### Typeahead ################################################################## @@ -297,6 +304,12 @@ def get_matching_tags(): return json_dumps +def generate_char_list(data_colum, db_link): + return (calibre_db.session.query(func.upper(func.substr(data_colum, 1, 1)).label('char')) + .join(db_link).join(db.Books).filter(calibre_db.common_filters()) + .group_by(func.upper(func.substr(data_colum, 1, 1))).all()) + + def get_sort_function(sort_param, data): order = [db.Books.timestamp.desc()] if sort_param == 'stored': @@ -373,7 +386,7 @@ def render_books_list(data, sort_param, book_id, page): else: website = data or "newest" entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, order[0], - False, 0, + False, 0, db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) @@ -407,12 +420,13 @@ def render_discover_books(page, book_id): else: abort(404) + def render_hot_books(page, order): if current_user.check_visibility(constants.SIDEBAR_HOT): if order[1] not in ['hotasc', 'hotdesc']: - # Unary expression comparsion only working (for this expression) in sqlalchemy 1.4+ - #if not (order[0][0].compare(func.count(ub.Downloads.book_id).desc()) or - # order[0][0].compare(func.count(ub.Downloads.book_id).asc())): + # Unary expression comparsion only working (for this expression) in sqlalchemy 1.4+ + # if not (order[0][0].compare(func.count(ub.Downloads.book_id).desc()) or + # order[0][0].compare(func.count(ub.Downloads.book_id).asc())): order = [func.count(ub.Downloads.book_id).desc()], 'hotdesc' if current_user.show_detail_random(): random = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \ @@ -420,19 +434,19 @@ def render_hot_books(page, order): else: random = false() off = int(int(config.config_books_per_page) * (page - 1)) - all_books = ub.session.query(ub.Downloads, func.count(ub.Downloads.book_id))\ + all_books = ub.session.query(ub.Downloads, func.count(ub.Downloads.book_id)) \ .order_by(*order[0]).group_by(ub.Downloads.book_id) hot_books = all_books.offset(off).limit(config.config_books_per_page) entries = list() for book in hot_books: - downloadBook = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).filter( + download_book = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).filter( db.Books.id == book.Downloads.book_id).first() - if downloadBook: - entries.append(downloadBook) + if download_book: + entries.append(download_book) else: ub.delete_download(book.Downloads.book_id) - numBooks = entries.__len__() - pagination = Pagination(page, config.config_books_per_page, numBooks) + num_books = entries.__len__() + pagination = Pagination(page, config.config_books_per_page, num_books) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, title=_(u"Hot Books (Most Downloaded)"), page="hot", order=order[1]) else: @@ -462,8 +476,8 @@ def render_downloaded_books(page, order, user_id): db.Series, ub.Downloads, db.Books.id == ub.Downloads.book_id) for book in entries: - if not calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \ - .filter(db.Books.id == book.id).first(): + if not calibre_db.session.query(db.Books).\ + filter(calibre_db.common_filters()).filter(db.Books.id == book.id).first(): ub.delete_download(book.id) user = ub.session.query(ub.User).filter(ub.User.id == user_id).first() return render_title_template('index.html', @@ -471,7 +485,7 @@ def render_downloaded_books(page, order, user_id): entries=entries, pagination=pagination, id=user_id, - title=_(u"Downloaded books by %(user)s",user=user.name), + title=_(u"Downloaded books by %(user)s", user=user.name), page="download", order=order[1]) else: @@ -639,29 +653,27 @@ def render_read_books(page, are_read, as_xml=False, order=None): column=config.config_read_column), category="error") return redirect(url_for("web.index")) - return [] # ToDo: Handle error Case for opds + return [] # ToDo: Handle error Case for opds if as_xml: return entries, pagination else: if are_read: name = _(u'Read Books') + ' (' + str(pagination.total_count) + ')' - pagename = "read" + page_name = "read" else: name = _(u'Unread Books') + ' (' + str(pagination.total_count) + ')' - pagename = "unread" + page_name = "unread" return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename, order=order[1]) + title=name, page=page_name, order=order[1]) def render_archived_books(page, sort_param): order = sort_param[0] or [] - archived_books = ( - ub.session.query(ub.ArchivedBook) - .filter(ub.ArchivedBook.user_id == int(current_user.id)) - .filter(ub.ArchivedBook.is_archived == True) - .all() - ) + archived_books = (ub.session.query(ub.ArchivedBook) + .filter(ub.ArchivedBook.user_id == int(current_user.id)) + .filter(ub.ArchivedBook.is_archived == True) + .all()) archived_book_ids = [archived_book.book_id for archived_book in archived_books] archived_filter = db.Books.id.in_(archived_book_ids) @@ -674,40 +686,40 @@ def render_archived_books(page, sort_param): False, 0) name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')' - pagename = "archived" + page_name = "archived" return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename, order=sort_param[1]) + title=name, page=page_name, order=sort_param[1]) def render_prepare_search_form(cc): # prepare data for search-form - tags = calibre_db.session.query(db.Tags)\ - .join(db.books_tags_link)\ - .join(db.Books)\ + tags = calibre_db.session.query(db.Tags) \ + .join(db.books_tags_link) \ + .join(db.Books) \ .filter(calibre_db.common_filters()) \ - .group_by(text('books_tags_link.tag'))\ + .group_by(text('books_tags_link.tag')) \ .order_by(db.Tags.name).all() - series = calibre_db.session.query(db.Series)\ - .join(db.books_series_link)\ - .join(db.Books)\ + series = calibre_db.session.query(db.Series) \ + .join(db.books_series_link) \ + .join(db.Books) \ .filter(calibre_db.common_filters()) \ - .group_by(text('books_series_link.series'))\ - .order_by(db.Series.name)\ + .group_by(text('books_series_link.series')) \ + .order_by(db.Series.name) \ .filter(calibre_db.common_filters()).all() - shelves = ub.session.query(ub.Shelf)\ - .filter(or_(ub.Shelf.is_public == 1, ub.Shelf.user_id == int(current_user.id)))\ + shelves = ub.session.query(ub.Shelf) \ + .filter(or_(ub.Shelf.is_public == 1, ub.Shelf.user_id == int(current_user.id))) \ .order_by(ub.Shelf.name).all() - extensions = calibre_db.session.query(db.Data)\ - .join(db.Books)\ + extensions = calibre_db.session.query(db.Data) \ + .join(db.Books) \ .filter(calibre_db.common_filters()) \ - .group_by(db.Data.format)\ + .group_by(db.Data.format) \ .order_by(db.Data.format).all() if current_user.filter_language() == u"all": languages = calibre_db.speaking_language() else: languages = None return render_title_template('search_form.html', tags=tags, languages=languages, extensions=extensions, - series=series,shelves=shelves, title=_(u"Advanced Search"), cc=cc, page="advsearch") + series=series, shelves=shelves, title=_(u"Advanced Search"), cc=cc, page="advsearch") def render_search_results(term, offset=None, order=None, limit=None): @@ -716,7 +728,6 @@ def render_search_results(term, offset=None, order=None, limit=None): offset, order, limit, - False, config.config_read_column, *join) return render_title_template('search.html', @@ -759,12 +770,13 @@ def books_table(): return render_title_template('book_table.html', title=_(u"Books List"), cc=cc, page="book_table", visiblility=visibility) + @web.route("/ajax/listbooks") @login_required def list_books(): off = int(request.args.get("offset") or 0) limit = int(request.args.get("limit") or config.config_books_per_page) - search = request.args.get("search") + search_param = request.args.get("search") sort_param = request.args.get("sort", "id") order = request.args.get("order", "").lower() state = None @@ -784,8 +796,8 @@ def list_books(): elif sort_param == "authors": order = [db.Authors.name.asc(), db.Series.name, db.Books.series_index] if order == "asc" \ else [db.Authors.name.desc(), db.Series.name.desc(), db.Books.series_index.desc()] - join = db.books_authors_link, db.Books.id == db.books_authors_link.c.book, db.Authors, \ - db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series + join = db.books_authors_link, db.Books.id == db.books_authors_link.c.book, db.Authors, db.books_series_link, \ + db.Books.id == db.books_series_link.c.book, db.Series elif sort_param == "languages": order = [db.Languages.lang_code.asc()] if order == "asc" else [db.Languages.lang_code.desc()] join = db.books_languages_link, db.Books.id == db.books_languages_link.c.book, db.Languages @@ -794,10 +806,11 @@ def list_books(): elif not state: order = [db.Books.timestamp.desc()] - total_count = filtered_count = calibre_db.session.query(db.Books).filter(calibre_db.common_filters(allow_show_archived=True)).count() + total_count = filtered_count = calibre_db.session.query(db.Books).filter( + calibre_db.common_filters(allow_show_archived=True)).count() if state is not None: - if search: - books = calibre_db.search_query(search, config.config_read_column).all() + if search_param: + books = calibre_db.search_query(search_param, config.config_read_column).all() filtered_count = len(books) else: if not config.config_read_column: @@ -818,15 +831,14 @@ def list_books(): # Skip linking read column and return None instead of read status books = calibre_db.session.query(db.Books, None, ub.ArchivedBook.is_archived) books = (books.outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id)) + int(current_user.id) == ub.ArchivedBook.user_id)) .filter(calibre_db.common_filters(allow_show_archived=True)).all()) entries = calibre_db.get_checkbox_sorted(books, state, off, limit, order, True) - elif search: - entries, filtered_count, __ = calibre_db.get_search_results(search, + elif search_param: + entries, filtered_count, __ = calibre_db.get_search_results(search_param, off, - [order,''], + [order, ''], limit, - True, config.config_read_column, *join) else: @@ -845,9 +857,9 @@ def list_books(): val = entry[0] val.read_status = entry[1] == ub.ReadBook.STATUS_FINISHED val.is_archived = entry[2] is True - for index in range(0, len(val.languages)): - val.languages[index].language_name = isoLanguages.get_language_name(get_locale(), val.languages[ - index].lang_code) + for lang_index in range(0, len(val.languages)): + val.languages[lang_index].language_name = isoLanguages.get_language_name(get_locale(), val.languages[ + lang_index].lang_code) result.append(val) table_entries = {'totalNotFiltered': total_count, 'total': filtered_count, "rows": result} @@ -857,6 +869,7 @@ def list_books(): response.headers["Content-Type"] = "application/json; charset=utf-8" return response + @web.route("/ajax/table_settings", methods=['POST']) @login_required def update_table_settings(): @@ -886,19 +899,18 @@ def author_list(): entries = calibre_db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \ .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(text('books_authors_link.author')).order_by(order).all() - charlist = calibre_db.session.query(func.upper(func.substr(db.Authors.sort, 1, 1)).label('char')) \ - .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \ - .group_by(func.upper(func.substr(db.Authors.sort, 1, 1))).all() + char_list = generate_char_list(db.Authors.sort, db.books_authors_link) # If not creating a copy, readonly databases can not display authornames with "|" in it as changing the name # starts a change session - autor_copy = copy.deepcopy(entries) - for entry in autor_copy: + author_copy = copy.deepcopy(entries) + for entry in author_copy: entry.Authors.name = entry.Authors.name.replace('|', ',') - return render_title_template('list.html', entries=autor_copy, folder='web.books_list', charlist=charlist, + return render_title_template('list.html', entries=author_copy, folder='web.books_list', charlist=char_list, title=u"Authors", page="authorlist", data='author', order=order_no) else: abort(404) + @web.route("/downloadlist") @login_required_if_no_ano def download_list(): @@ -909,12 +921,12 @@ def download_list(): order = ub.User.name.asc() order_no = 1 if current_user.check_visibility(constants.SIDEBAR_DOWNLOAD) and current_user.role_admin(): - entries = ub.session.query(ub.User, func.count(ub.Downloads.book_id).label('count'))\ + entries = ub.session.query(ub.User, func.count(ub.Downloads.book_id).label('count')) \ .join(ub.Downloads).group_by(ub.Downloads.user_id).order_by(order).all() - charlist = ub.session.query(func.upper(func.substr(ub.User.name, 1, 1)).label('char')) \ + char_list = ub.session.query(func.upper(func.substr(ub.User.name, 1, 1)).label('char')) \ .filter(ub.User.role.op('&')(constants.ROLE_ANONYMOUS) != constants.ROLE_ANONYMOUS) \ .group_by(func.upper(func.substr(ub.User.name, 1, 1))).all() - return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, + return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list, title=_(u"Downloads"), page="downloadlist", data="download", order=order_no) else: abort(404) @@ -933,10 +945,8 @@ def publisher_list(): entries = calibre_db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \ .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(text('books_publishers_link.publisher')).order_by(order).all() - charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \ - .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \ - .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all() - return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, + char_list = generate_char_list(db.Publishers.name, db.books_publishers_link) + return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list, title=_(u"Publishers"), page="publisherlist", data="publisher", order=order_no) else: abort(404) @@ -952,25 +962,19 @@ def series_list(): else: order = db.Series.sort.asc() order_no = 1 + char_list = generate_char_list(db.Series.sort, db.books_series_link) if current_user.get_view_property('series', 'series_view') == 'list': entries = calibre_db.session.query(db.Series, func.count('books_series_link.book').label('count')) \ .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(text('books_series_link.series')).order_by(order).all() - charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \ - .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \ - .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all() - return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, + return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list, title=_(u"Series"), page="serieslist", data="series", order=order_no) else: entries = calibre_db.session.query(db.Books, func.count('books_series_link').label('count'), func.max(db.Books.series_index), db.Books.id) \ - .join(db.books_series_link).join(db.Series).filter(calibre_db.common_filters())\ + .join(db.books_series_link).join(db.Series).filter(calibre_db.common_filters()) \ .group_by(text('books_series_link.series')).order_by(order).all() - charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \ - .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \ - .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all() - - return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=charlist, + return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=char_list, title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view", order=order_no) else: @@ -988,7 +992,7 @@ def ratings_list(): order = db.Ratings.rating.asc() order_no = 1 entries = calibre_db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'), - (db.Ratings.rating / 2).label('name')) \ + (db.Ratings.rating / 2).label('name')) \ .join(db.books_ratings_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(text('books_ratings_link.rating')).order_by(order).all() return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(), @@ -1023,14 +1027,14 @@ def formats_list(): def language_overview(): if current_user.check_visibility(constants.SIDEBAR_LANGUAGE) and current_user.filter_language() == u"all": order_no = 0 if current_user.get_view_property('language', 'dir') == 'desc' else 1 - charlist = list() + char_list = list() languages = calibre_db.speaking_language(reverse_order=not order_no, with_count=True) for lang in languages: upper_lang = lang[0].name[0].upper() - if upper_lang not in charlist: - charlist.append(upper_lang) + if upper_lang not in char_list: + char_list.append(upper_lang) return render_title_template('languages.html', languages=languages, - charlist=charlist, title=_(u"Languages"), page="langlist", + charlist=char_list, title=_(u"Languages"), page="langlist", data="language", order=order_no) else: abort(404) @@ -1049,10 +1053,8 @@ def category_list(): entries = calibre_db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \ .join(db.books_tags_link).join(db.Books).order_by(order).filter(calibre_db.common_filters()) \ .group_by(text('books_tags_link.tag')).all() - charlist = calibre_db.session.query(func.upper(func.substr(db.Tags.name, 1, 1)).label('char')) \ - .join(db.books_tags_link).join(db.Books).filter(calibre_db.common_filters()) \ - .group_by(func.upper(func.substr(db.Tags.name, 1, 1))).all() - return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, + char_list = generate_char_list(db.Tags.name, db.books_tags_link) + return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list, title=_(u"Categories"), page="catlist", data="category", order=order_no) else: abort(404) @@ -1176,7 +1178,15 @@ def adv_search_read_status(q, read_status): return q -def adv_search_extension(q, include_extension_inputs, exclude_extension_inputs): +def adv_search_text(q, include_inputs, exclude_inputs, data_value): + for inp in include_inputs: + q = q.filter(db.Books.data.any(data_value == inp)) + for excl in exclude_inputs: + q = q.filter(not_(db.Books.data.any(data_value == excl))) + return q + + +'''def adv_search_extension(q, include_extension_inputs, exclude_extension_inputs): for extension in include_extension_inputs: q = q.filter(db.Books.data.any(db.Data.format == extension)) for extension in exclude_extension_inputs: @@ -1197,15 +1207,17 @@ def adv_search_serie(q, include_series_inputs, exclude_series_inputs): q = q.filter(db.Books.series.any(db.Series.id == serie)) for serie in exclude_series_inputs: q = q.filter(not_(db.Books.series.any(db.Series.id == serie))) - return q + return q''' + def adv_search_shelf(q, include_shelf_inputs, exclude_shelf_inputs): - q = q.outerjoin(ub.BookShelf, db.Books.id == ub.BookShelf.book_id)\ + q = q.outerjoin(ub.BookShelf, db.Books.id == ub.BookShelf.book_id) \ .filter(or_(ub.BookShelf.shelf == None, ub.BookShelf.shelf.notin_(exclude_shelf_inputs))) if len(include_shelf_inputs) > 0: q = q.filter(ub.BookShelf.shelf.in_(include_shelf_inputs)) return q + def extend_search_term(searchterm, author_name, book_title, @@ -1232,7 +1244,7 @@ def extend_search_term(searchterm, format='medium', locale=get_locale())]) except ValueError: pub_end = u"" - elements = {'tag': db.Tags, 'serie':db.Series, 'shelf':ub.Shelf} + elements = {'tag': db.Tags, 'serie': db.Series, 'shelf': ub.Shelf} for key, db_element in elements.items(): tag_names = calibre_db.session.query(db_element).filter(db_element.id.in_(tags['include_' + key])).all() searchterm.extend(tag.name for tag in tag_names) @@ -1284,8 +1296,8 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): query = query.outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, int(current_user.id) == ub.ArchivedBook.user_id)) - q = query.outerjoin(db.books_series_link, db.Books.id == db.books_series_link.c.book)\ - .outerjoin(db.Series)\ + q = query.outerjoin(db.books_series_link, db.Books.id == db.books_series_link.c.book) \ + .outerjoin(db.Series) \ .filter(calibre_db.common_filters(True)) # parse multiselects to a complete dict @@ -1311,43 +1323,43 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): if publisher: publisher = publisher.strip().lower() - searchterm = [] + search_term = [] cc_present = False for c in cc: if c.datatype == "datetime": column_start = term.get('custom_column_' + str(c.id) + '_start') column_end = term.get('custom_column_' + str(c.id) + '_end') if column_start: - searchterm.extend([u"{} >= {}".format(c.name, - format_date(datetime.strptime(column_start, "%Y-%m-%d").date(), - format='medium', - locale=get_locale()) - )]) + search_term.extend([u"{} >= {}".format(c.name, + format_date(datetime.strptime(column_start, "%Y-%m-%d").date(), + format='medium', + locale=get_locale()) + )]) cc_present = True if column_end: - searchterm.extend([u"{} <= {}".format(c.name, - format_date(datetime.strptime(column_end, "%Y-%m-%d").date(), - format='medium', - locale=get_locale()) - )]) + search_term.extend([u"{} <= {}".format(c.name, + format_date(datetime.strptime(column_end, "%Y-%m-%d").date(), + format='medium', + locale=get_locale()) + )]) cc_present = True elif term.get('custom_column_' + str(c.id)): - searchterm.extend([(u"{}: {}".format(c.name, term.get('custom_column_' + str(c.id))))]) + search_term.extend([(u"{}: {}".format(c.name, term.get('custom_column_' + str(c.id))))]) cc_present = True - - if any(tags.values()) or author_name or book_title or publisher or pub_start or pub_end or rating_low \ - or rating_high or description or cc_present or read_status: - searchterm, pub_start, pub_end = extend_search_term(searchterm, - author_name, - book_title, - publisher, - pub_start, - pub_end, - tags, - rating_high, - rating_low, - read_status) + if any(tags.values()) or author_name or book_title or \ + publisher or pub_start or pub_end or rating_low or rating_high \ + or description or cc_present or read_status: + search_term, pub_start, pub_end = extend_search_term(search_term, + author_name, + book_title, + publisher, + pub_start, + pub_end, + tags, + rating_high, + rating_low, + read_status) # q = q.filter() if author_name: q = q.filter(db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + author_name + "%"))) @@ -1360,12 +1372,12 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): q = adv_search_read_status(q, read_status) if publisher: q = q.filter(db.Books.publishers.any(func.lower(db.Publishers.name).ilike("%" + publisher + "%"))) - q = adv_search_tag(q, tags['include_tag'], tags['exclude_tag']) - q = adv_search_serie(q, tags['include_serie'], tags['exclude_serie']) + q = adv_search_text(q, tags['include_tag'], tags['exclude_tag'], db.Tags.id) + q = adv_search_text(q, tags['include_serie'], tags['exclude_serie'], db.Series.id) + q = adv_search_text(q, tags['include_extension'], tags['exclude_extension'], db.Data.format) q = adv_search_shelf(q, tags['include_shelf'], tags['exclude_shelf']) - q = adv_search_extension(q, tags['include_extension'], tags['exclude_extension']) - q = adv_search_language(q, tags['include_language'], tags['exclude_language']) - q = adv_search_ratings(q, rating_high, rating_low) + q = adv_search_language(q, tags['include_language'], tags['exclude_language'], ) + q = adv_search_ratings(q, rating_high, rating_low, ) if description: q = q.filter(db.Books.comments.any(func.lower(db.Comments.text).ilike("%" + description + "%"))) @@ -1390,7 +1402,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): limit_all = result_count entries = calibre_db.order_authors(q[offset:limit_all], list_return=True, combined=True) return render_title_template('search.html', - adv_searchterm=searchterm, + adv_searchterm=search_term, pagination=pagination, entries=entries, result_count=result_count, @@ -1414,10 +1426,12 @@ def advanced_search_form(): def get_cover(book_id): return get_book_cover(book_id) + @web.route("/robots.txt") def get_robots(): return send_from_directory(constants.STATIC_DIR, "robots.txt") + @web.route("/show//", defaults={'anyname': 'None'}) @web.route("/show///") @login_required_if_no_ano @@ -1561,7 +1575,7 @@ def login(): category="success") return redirect_back(url_for("web.index")) elif login_result is None and user and check_password_hash(str(user.password), form['password']) \ - and user.name != "Guest": + and user.name != "Guest": login_user(user, remember=bool(form.get('remember_me'))) ub.store_user_session() log.info("Local Fallback Login as: '%s'", user.name) @@ -1573,23 +1587,23 @@ def login(): log.info(error) flash(_(u"Could not login: %(message)s", message=error), category="error") else: - ip_Address = request.headers.get('X-Forwarded-For', request.remote_addr) - log.warning('LDAP Login failed for user "%s" IP-address: %s', form['username'], ip_Address) + ip_address = request.headers.get('X-Forwarded-For', request.remote_addr) + log.warning('LDAP Login failed for user "%s" IP-address: %s', form['username'], ip_address) flash(_(u"Wrong Username or Password"), category="error") else: - ip_Address = request.headers.get('X-Forwarded-For', request.remote_addr) + ip_address = request.headers.get('X-Forwarded-For', request.remote_addr) if 'forgot' in form and form['forgot'] == 'forgot': if user is not None and user.name != "Guest": ret, __ = reset_password(user.id) if ret == 1: flash(_(u"New Password was send to your email address"), category="info") - log.info('Password reset for user "%s" IP-address: %s', form['username'], ip_Address) + log.info('Password reset for user "%s" IP-address: %s', form['username'], ip_address) else: log.error(u"An unknown error occurred. Please try again later") flash(_(u"An unknown error occurred. Please try again later."), category="error") else: flash(_(u"Please enter valid username to reset password"), category="error") - log.warning('Username missing for password reset IP-address: %s', ip_Address) + log.warning('Username missing for password reset IP-address: %s', ip_address) else: if user and check_password_hash(str(user.password), form['password']) and user.name != "Guest": login_user(user, remember=bool(form.get('remember_me'))) @@ -1599,7 +1613,7 @@ def login(): config.config_is_initial = False return redirect_back(url_for("web.index")) else: - log.warning('Login failed for user "%s" IP-address: %s', form['username'], ip_Address) + log.warning('Login failed for user "%s" IP-address: %s', form['username'], ip_address) flash(_(u"Wrong Username or Password"), category="error") next_url = request.args.get('next', default=url_for("web.index"), type=str) @@ -1617,7 +1631,7 @@ def login(): @login_required def logout(): if current_user is not None and current_user.is_authenticated: - ub.delete_user_session(current_user.id, flask_session.get('_id',"")) + ub.delete_user_session(current_user.id, flask_session.get('_id', "")) logout_user() if feature_support['oauth'] and (config.config_login_type == 2 or config.config_login_type == 3): logout_oauth_user() @@ -1639,7 +1653,7 @@ def change_profile(kobo_support, local_oauth_check, oauth_status, translations, current_user.email = check_email(to_save["email"]) if current_user.role_admin(): if to_save.get("name", current_user.name) != current_user.name: - # Query User name, if not existing, change + # Query username, if not existing, change current_user.name = check_username(to_save["name"]) current_user.random_books = 1 if to_save.get("show_random") == "on" else 0 if to_save.get("default_language"): @@ -1693,7 +1707,7 @@ def change_profile(kobo_support, local_oauth_check, oauth_status, translations, @login_required def profile(): languages = calibre_db.speaking_language() - translations = babel.list_translations() + [LC('en')] + translations = babel.list_translations() + [Locale('en')] kobo_support = feature_support['kobo'] and config.config_kobo_sync if feature_support['oauth'] and config.config_login_type == 2: oauth_status = get_oauth_status() @@ -1727,7 +1741,8 @@ def read_book(book_id, book_format): book.ordered_authors = calibre_db.order_authors([book], False) if not book: - flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), category="error") + flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), + category="error") log.debug(u"Oops! Selected book title is unavailable. File does not exist or is not accessible") return redirect(url_for("web.index")) @@ -1768,7 +1783,8 @@ def read_book(book_id, book_format): return render_title_template('readcbr.html', comicfile=all_name, title=title, extension=fileExt) log.debug(u"Oops! Selected book title is unavailable. File does not exist or is not accessible") - flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), category="error") + flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), + category="error") return redirect(url_for("web.index")) @@ -1782,14 +1798,14 @@ def show_book(book_id): entry = entries[0] entry.read_status = read_book == ub.ReadBook.STATUS_FINISHED entry.is_archived = archived_book - for index in range(0, len(entry.languages)): - entry.languages[index].language_name = isoLanguages.get_language_name(get_locale(), entry.languages[ - index].lang_code) + for lang_index in range(0, len(entry.languages)): + entry.languages[lang_index].language_name = isoLanguages.get_language_name(get_locale(), entry.languages[ + lang_index].lang_code) cc = get_cc_columns(filter_config_custom_read=True) - book_in_shelfs = [] + book_in_shelves = [] shelfs = ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).all() for sh in shelfs: - book_in_shelfs.append(sh.shelf) + book_in_shelves.append(sh.shelf) entry.tags = sort(entry.tags, key=lambda tag: tag.name) @@ -1806,9 +1822,9 @@ def show_book(book_id): return render_title_template('detail.html', entry=entry, cc=cc, - is_xhr=request.headers.get('X-Requested-With')=='XMLHttpRequest', + is_xhr=request.headers.get('X-Requested-With') == 'XMLHttpRequest', title=entry.title, - books_shelfs=book_in_shelfs, + books_shelfs=book_in_shelves, page="book") else: log.debug(u"Oops! Selected book title is unavailable. File does not exist or is not accessible") diff --git a/optional-requirements.txt b/optional-requirements.txt index 04f7bb0c..aea1efb7 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -1,5 +1,5 @@ # GDrive Integration -google-api-python-client>=1.7.11,<2.37.0 +google-api-python-client>=1.7.11,<2.41.0 gevent>20.6.0,<22.0.0 greenlet>=0.4.17,<1.2.0 httplib2>=0.9.2,<0.21.0 @@ -12,8 +12,8 @@ PyYAML>=3.12 rsa>=3.4.2,<4.9.0 # Gmail -google-auth-oauthlib>=0.4.3,<0.5.0 -google-api-python-client>=1.7.11,<2.37.0 +google-auth-oauthlib>=0.4.3,<0.6.0 +google-api-python-client>=1.7.11,<2.41.0 # goodreads goodreads>=0.3.2,<0.4.0 @@ -29,7 +29,7 @@ SQLAlchemy-Utils>=0.33.5,<0.39.0 # metadata extraction rarfile>=3.2 -scholarly>=1.2.0,<1.6 +scholarly>=1.2.0,<1.7 markdown2>=2.0.0,<2.5.0 html2text>=2020.1.16,<2022.1.1 python-dateutil>=2.1,<2.9.0 diff --git a/requirements.txt b/requirements.txt index d9bed7bb..de08fbda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ SQLAlchemy>=1.3.0,<1.5.0 tornado>=4.1,<6.2 Wand>=0.4.4,<0.7.0 unidecode>=0.04.19,<1.4.0 -lxml>=3.8.0,<4.8.0 +lxml>=3.8.0,<4.9.0 flask-wtf>=0.14.2,<1.1.0 chardet>=3.0.0,<4.1.0 +advocate>=1.0.0,<1.1.0 From 8f3bb2e3380e93ef940b6aeaefd7713b50277a72 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Mon, 14 Mar 2022 17:12:35 +0100 Subject: [PATCH 056/112] Bugfixes from testrun --- cps/admin.py | 8 +- cps/opds.py | 10 +- cps/web.py | 71 +- test/Calibre-Web TestSummary_Linux.html | 986 +++++++++++++++--------- 4 files changed, 669 insertions(+), 406 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 5d8ed9c1..fedee340 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -863,10 +863,10 @@ def delete_restriction(res_type, user_id): usr = current_user if element['id'].startswith('a'): usr.allowed_tags = restriction_deletion(element, usr.list_allowed_tags) - ub.session_commit("Deleted allowed tags of user {}: {}".format(usr.name, usr.list_allowed_tags)) + ub.session_commit("Deleted allowed tags of user {}: {}".format(usr.name, element['Element'])) elif element['id'].startswith('d'): usr.denied_tags = restriction_deletion(element, usr.list_denied_tags) - ub.session_commit("Deleted denied tags of user {}: {}".format(usr.name, usr.list_allowed_tags)) + ub.session_commit("Deleted denied tag of user {}: {}".format(usr.name, element['Element'])) elif res_type == 3: # Columns per user if isinstance(user_id, int): usr = ub.session.query(ub.User).filter(ub.User.id == int(user_id)).first() @@ -875,12 +875,12 @@ def delete_restriction(res_type, user_id): if element['id'].startswith('a'): usr.allowed_column_value = restriction_deletion(element, usr.list_allowed_column_values) ub.session_commit("Deleted allowed columns of user {}: {}".format(usr.name, - usr.list_allowed_column_values)) + usr.list_allowed_column_values())) elif element['id'].startswith('d'): usr.denied_column_value = restriction_deletion(element, usr.list_denied_column_values) ub.session_commit("Deleted denied columns of user {}: {}".format(usr.name, - usr.list_denied_column_values)) + usr.list_denied_column_values())) return "" diff --git a/cps/opds.py b/cps/opds.py index c0637c6b..c8ccdea9 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -26,7 +26,7 @@ from functools import wraps from flask import Blueprint, request, render_template, Response, g, make_response, abort from flask_login import current_user -from sqlalchemy.sql.expression import func, text, or_, and_, true +from sqlalchemy.sql.expression import func, text, or_, and_, any_, true from werkzeug.security import check_password_hash from . import constants, logger, config, db, calibre_db, ub, services, get_locale, isoLanguages from .helper import get_download_link, get_book_cover @@ -34,7 +34,7 @@ from .pagination import Pagination from .web import render_read_books from .usermanagement import load_user_from_request from flask_babel import gettext as _ - +from sqlalchemy.orm import InstrumentedAttribute opds = Blueprint('opds', __name__) log = logger.create() @@ -298,7 +298,7 @@ def feed_ratingindex(): @opds.route("/opds/ratings/") @requires_basic_auth_if_no_ano def feed_ratings(book_id): - return render_xml_dataset(db.Tags, book_id) + return render_xml_dataset(db.Ratings, book_id) @opds.route("/opds/formats") @@ -493,7 +493,7 @@ def render_xml_dataset(data_table, book_id): off = request.args.get("offset") or 0 entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, - data_table.any(data_table.id == book_id), + getattr(db.Books, data_table.__tablename__).any(data_table.id == book_id), [db.Books.timestamp.desc()]) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -502,7 +502,7 @@ def render_element_index(database_column, linked_table, folder): shift = 0 off = int(request.args.get("offset") or 0) entries = calibre_db.session.query(func.upper(func.substr(database_column, 1, 1)).label('id')) - if linked_table: + if linked_table is not None: entries = entries.join(linked_table).join(db.Books) entries = entries.filter(calibre_db.common_filters()).group_by(func.upper(func.substr(database_column, 1, 1))).all() elements = [] diff --git a/cps/web.py b/cps/web.py index 6f2f1af1..b1f27035 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1129,27 +1129,6 @@ def adv_search_custom_columns(cc, term, q): return q -def adv_search_language(q, include_languages_inputs, exclude_languages_inputs): - if current_user.filter_language() != "all": - q = q.filter(db.Books.languages.any(db.Languages.lang_code == current_user.filter_language())) - else: - for language in include_languages_inputs: - q = q.filter(db.Books.languages.any(db.Languages.id == language)) - for language in exclude_languages_inputs: - q = q.filter(not_(db.Books.series.any(db.Languages.id == language))) - return q - - -def adv_search_ratings(q, rating_high, rating_low): - if rating_high: - rating_high = int(rating_high) * 2 - q = q.filter(db.Books.ratings.any(db.Ratings.rating <= rating_high)) - if rating_low: - rating_low = int(rating_low) * 2 - q = q.filter(db.Books.ratings.any(db.Ratings.rating >= rating_low)) - return q - - def adv_search_read_status(q, read_status): if read_status: if config.config_read_column: @@ -1178,38 +1157,32 @@ def adv_search_read_status(q, read_status): return q -def adv_search_text(q, include_inputs, exclude_inputs, data_value): +def adv_search_language(q, include_languages_inputs, exclude_languages_inputs): + if current_user.filter_language() != "all": + q = q.filter(db.Books.languages.any(db.Languages.lang_code == current_user.filter_language())) + else: + return adv_search_text(q, include_languages_inputs, exclude_languages_inputs, db.Languages.id) + return q + + +def adv_search_ratings(q, rating_high, rating_low): + if rating_high: + rating_high = int(rating_high) * 2 + q = q.filter(db.Books.ratings.any(db.Ratings.rating <= rating_high)) + if rating_low: + rating_low = int(rating_low) * 2 + q = q.filter(db.Books.ratings.any(db.Ratings.rating >= rating_low)) + return q + + +def adv_search_text(q, include_inputs, exclude_inputs, data_table): for inp in include_inputs: - q = q.filter(db.Books.data.any(data_value == inp)) + q = q.filter(getattr(db.Books, data_table.class_.__tablename__).any(data_table == inp)) for excl in exclude_inputs: - q = q.filter(not_(db.Books.data.any(data_value == excl))) + q = q.filter(not_(getattr(db.Books, data_table.class_.__tablename__).any(data_table == excl))) return q -'''def adv_search_extension(q, include_extension_inputs, exclude_extension_inputs): - for extension in include_extension_inputs: - q = q.filter(db.Books.data.any(db.Data.format == extension)) - for extension in exclude_extension_inputs: - q = q.filter(not_(db.Books.data.any(db.Data.format == extension))) - return q - - -def adv_search_tag(q, include_tag_inputs, exclude_tag_inputs): - for tag in include_tag_inputs: - q = q.filter(db.Books.tags.any(db.Tags.id == tag)) - for tag in exclude_tag_inputs: - q = q.filter(not_(db.Books.tags.any(db.Tags.id == tag))) - return q - - -def adv_search_serie(q, include_series_inputs, exclude_series_inputs): - for serie in include_series_inputs: - q = q.filter(db.Books.series.any(db.Series.id == serie)) - for serie in exclude_series_inputs: - q = q.filter(not_(db.Books.series.any(db.Series.id == serie))) - return q''' - - def adv_search_shelf(q, include_shelf_inputs, exclude_shelf_inputs): q = q.outerjoin(ub.BookShelf, db.Books.id == ub.BookShelf.book_id) \ .filter(or_(ub.BookShelf.shelf == None, ub.BookShelf.shelf.notin_(exclude_shelf_inputs))) @@ -1376,7 +1349,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): q = adv_search_text(q, tags['include_serie'], tags['exclude_serie'], db.Series.id) q = adv_search_text(q, tags['include_extension'], tags['exclude_extension'], db.Data.format) q = adv_search_shelf(q, tags['include_shelf'], tags['exclude_shelf']) - q = adv_search_language(q, tags['include_language'], tags['exclude_language'], ) + q = adv_search_language(q, tags['include_language'], tags['exclude_language']) q = adv_search_ratings(q, rating_high, rating_low, ) if description: diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 580cf3c9..b2de4c81 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
    -

    Start Time: 2022-03-12 22:57:13

    +

    Start Time: 2022-03-13 21:32:38

    -

    Stop Time: 2022-03-13 03:46:17

    +

    Stop Time: 2022-03-14 02:20:13

    -

    Duration: 4h 0 min

    +

    Duration: 3h 59 min

    @@ -363,11 +363,11 @@ - + TestCoverEditBooks 2 - 2 - 0 + 1 + 1 0 0 @@ -386,11 +386,31 @@ - +
    TestCoverEditBooks - test_upload_jpg
    - PASS + +
    + FAIL +
    + + + + @@ -681,11 +701,11 @@ - + TestEbookConvertGDriveKepubify 3 - 3 - 0 + 2 + 1 0 0 @@ -704,11 +724,33 @@ - +
    TestEbookConvertGDriveKepubify - test_convert_only
    - PASS + +
    + FAIL +
    + + + + @@ -1586,12 +1628,12 @@ - + TestEditBooksOnGdrive 20 - 17 + 16 3 - 0 + 1 0 Detail @@ -1609,11 +1651,41 @@ - +
    TestEditBooksOnGdrive - test_edit_author
    - PASS + +
    + ERROR +
    + + + + @@ -1761,7 +1833,7 @@
  • Traceback (most recent call last):
    -  File "/home/ozzie/Development/calibre-web-test/test/test_edit_ebooks_gdrive.py", line 865, in test_upload_book_epub
    +  File "/home/ozzie/Development/calibre-web-test/test/test_edit_ebooks_gdrive.py", line 868, in test_upload_book_epub
         self.assertEqual('8936', resp.headers['Content-Length'])
     AssertionError: '8936' != '1103'
     - 8936
    @@ -1801,7 +1873,7 @@ AssertionError: '8936' != '1103'
                         
    Traceback (most recent call last):
    -  File "/home/ozzie/Development/calibre-web-test/test/test_edit_ebooks_gdrive.py", line 778, in test_upload_cover_hdd
    +  File "/home/ozzie/Development/calibre-web-test/test/test_edit_ebooks_gdrive.py", line 781, in test_upload_cover_hdd
         self.assertGreater(diff('original.png', 'jpeg.png', delete_diff_file=True), 0.02)
     AssertionError: 0.0 not greater than 0.02
    @@ -1830,9 +1902,9 @@ AssertionError: 0.0 not greater than 0.02
    Traceback (most recent call last):
    -  File "/home/ozzie/Development/calibre-web-test/test/test_edit_ebooks_gdrive.py", line 947, in test_watch_metadata
    +  File "/home/ozzie/Development/calibre-web-test/test/test_edit_ebooks_gdrive.py", line 950, in test_watch_metadata
         self.assertNotIn('series', book)
    -AssertionError: 'series' unexpectedly found in {'id': 5, 'reader': [], 'title': 'testbook', 'author': ['John Döe'], 'rating': 0, 'languages': ['English'], 'identifier': [], 'cover': '/cover/5?edit=4a23d5c4-e97a-42e1-bd43-351fb1de43df', 'tag': [], 'publisher': ['Randomhäus'], 'pubdate': 'Jan 19, 2017', 'comment': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit.Aenean commodo ligula eget dolor.Aenean massa.Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus.Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem.Nulla consequat massa quis enim.Donec pede justo, fringilla vel, aliquet nec, vulputate', 'add_shelf': [], 'del_shelf': [], 'edit_enable': True, 'kindle': None, 'kindlebtn': None, 'download': ['EPUB (6.7 kB)'], 'read': False, 'archived': False, 'series_all': 'Book 1 of test', 'series_index': '1', 'series': 'test', 'cust_columns': []}
    +AssertionError: 'series' unexpectedly found in {'id': 5, 'reader': [], 'title': 'testbook', 'author': ['John Döe'], 'rating': 0, 'languages': ['English'], 'identifier': [], 'cover': '/cover/5?edit=f66a16a8-b9e1-446d-b1ad-21b11df67753', 'tag': [], 'publisher': ['Randomhäus'], 'pubdate': 'Jan 19, 2017', 'comment': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit.Aenean commodo ligula eget dolor.Aenean massa.Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus.Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem.Nulla consequat massa quis enim.Donec pede justo, fringilla vel, aliquet nec, vulputate', 'add_shelf': [], 'del_shelf': [], 'edit_enable': True, 'kindle': None, 'kindlebtn': None, 'download': ['EPUB (6.7 kB)'], 'read': False, 'archived': False, 'series_all': 'Book 1 of test', 'series_index': '1', 'series': 'test', 'cust_columns': []}
    @@ -2891,12 +2963,12 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - + TestOPDSFeed 23 - 23 - 0 - 0 + 14 + 4 + 5 0 Detail @@ -2905,20 +2977,62 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
    TestOPDSFeed - test_opds
    - PASS + +
    + FAIL +
    + + + + - +
    TestOPDSFeed - test_opds_author
    - PASS + +
    + ERROR +
    + + + + @@ -2977,11 +3091,31 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
    TestOPDSFeed - test_opds_guest_user
    - PASS + +
    + FAIL +
    + + + + @@ -3004,20 +3138,62 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
    TestOPDSFeed - test_opds_non_admin
    - PASS + +
    + FAIL +
    + + + + - +
    TestOPDSFeed - test_opds_publisher
    - PASS + +
    + ERROR +
    + + + + @@ -3031,11 +3207,33 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
    TestOPDSFeed - test_opds_ratings
    - PASS + +
    + ERROR +
    + + + + @@ -3049,20 +3247,62 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
    TestOPDSFeed - test_opds_search
    - PASS + +
    + FAIL +
    + + + + - +
    TestOPDSFeed - test_opds_series
    - PASS + +
    + ERROR +
    + + + + @@ -3076,11 +3316,33 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
    TestOPDSFeed - test_opds_tags
    - PASS + +
    + ERROR +
    + + + + @@ -3113,12 +3375,12 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - - TestUploadPDF - 1 + + _ErrorHolder 1 0 0 + 1 0 Detail @@ -3127,144 +3389,34 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - + -
    TestUploadPDF - test_upload_invalid_pdf
    - - PASS - - - - - - - TestReader - 5 - 4 - 1 - 0 - 0 - - Detail - - - - - - - -
    TestReader - test_comic_reader
    - - PASS - - - - - - -
    TestReader - test_epub_reader
    - - PASS - - - - - - -
    TestReader - test_pdf_reader
    - - PASS - - - - - - -
    TestReader - test_sound_listener
    +
    tearDownClass (test_opds_feed)
    - FAIL + ERROR
    - diff --git a/cps/templates/discover.html b/cps/templates/discover.html index 852d6210..900dad8c 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -7,7 +7,7 @@
    - +

    {{entry.title|shortentitle}}

    diff --git a/cps/templates/email_edit.html b/cps/templates/email_edit.html index 9f23f78b..2a844209 100644 --- a/cps/templates/email_edit.html +++ b/cps/templates/email_edit.html @@ -69,7 +69,7 @@ {% endif %} {{_('Back')}}

    - {% if g.allow_registration %} + {% if g.allow_registration and not simple%}

    {{_('Allowed Domains (Whitelist)')}}

    diff --git a/cps/templates/index.html b/cps/templates/index.html index 28807343..a4547705 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -7,7 +7,7 @@ {% for entry in random %}
    - +

    {{entry.title|shortentitle}}

    @@ -24,7 +24,7 @@ {% if not loop.first %} & {% endif %} - {{author.name.replace('|',',')|shortentitle(30)}} + {{author.name.replace('|',',')|shortentitle(30)}} {% if loop.last %} (...) {% endif %} @@ -89,7 +89,7 @@ {% for entry in entries %}

    - +

    {{entry.title|shortentitle}}

    diff --git a/cps/templates/layout.html b/cps/templates/layout.html index 0d0df778..46f03da1 100644 --- a/cps/templates/layout.html +++ b/cps/templates/layout.html @@ -69,7 +69,7 @@

    {% endif %} - {% if not g.user.is_anonymous %} + {% if not g.user.is_anonymous and not simple%}
  • {% endif %} {% if g.user.role_admin() %} diff --git a/cps/templates/search.html b/cps/templates/search.html index 4f8d607b..77c01c80 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -43,7 +43,7 @@
    - +

    {{entry.Books.title|shortentitle}}

    diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index e5cb2ffb..a7b5bcf0 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -32,7 +32,7 @@ {% for entry in entries %}

    - +

    {{entry.title|shortentitle}}

    diff --git a/cps/templates/user_edit.html b/cps/templates/user_edit.html index de7a4fb3..b489cbaf 100644 --- a/cps/templates/user_edit.html +++ b/cps/templates/user_edit.html @@ -83,7 +83,7 @@

    - {% if ( g.user and g.user.role_admin() and not new_user ) %} + {% if ( g.user and g.user.role_admin() and not new_user ) and not simple %} {{_('Add Allowed/Denied Tags')}} {{_('Add allowed/Denied Custom Column Values')}} {% endif %} @@ -131,7 +131,7 @@
    {% endif %} {% endif %} - {% if kobo_support and not content.role_anonymous() %} + {% if kobo_support and not content.role_anonymous() and not simple%}
    From 32a3c45ee0f7e13bd61075f32a4dcebc415585a1 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 26 Mar 2022 19:35:56 +0100 Subject: [PATCH 071/112] Refactored load read status for web access and opds access Refactored and removed discover html page Bugfix show author Bugfix open dialog in author page Fix for #2341 (advanced search with linked read column and read column having a higher number than number of available custom columns) --- cps/db.py | 53 +- cps/opds.py | 68 +- cps/render_template.py | 6 +- cps/shelf.py | 2 +- cps/templates/author.html | 32 +- cps/templates/discover.html | 65 -- cps/templates/feed.xml | 32 +- cps/templates/index.html | 61 +- cps/templates/search.html | 2 +- cps/templates/shelf.html | 26 +- cps/templates/shelfdown.html | 18 +- cps/web.py | 141 ++- test/Calibre-Web TestSummary_Linux.html | 1042 ++++++++++++++--------- 13 files changed, 846 insertions(+), 702 deletions(-) delete mode 100644 cps/templates/discover.html diff --git a/cps/db.py b/cps/db.py index f2fe9b78..3b193e1a 100644 --- a/cps/db.py +++ b/cps/db.py @@ -680,6 +680,25 @@ class CalibreDB: return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter, pos_content_cc_filter, ~neg_content_cc_filter, archived_filter) + def generate_linked_query(self, config_read_column, database): + if not config_read_column: + query = (self.session.query(database, ub.ArchivedBook.is_archived, ub.ReadBook.read_status) + .select_from(Books) + .outerjoin(ub.ReadBook, + and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == Books.id))) + else: + try: + read_column = cc_classes[config_read_column] + query = (self.session.query(database, ub.ArchivedBook.is_archived, read_column.value) + .select_from(Books) + .outerjoin(read_column, read_column.book == Books.id)) + except (KeyError, AttributeError, IndexError): + log.error("Custom Column No.{} is not existing in calibre database".format(config_read_column)) + # Skip linking read column and return None instead of read status + query = self.session.query(database, None, ub.ArchivedBook.is_archived) + return query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id, + int(current_user.id) == ub.ArchivedBook.user_id)) + @staticmethod def get_checkbox_sorted(inputlist, state, offset, limit, order, combo=False): outcome = list() @@ -709,30 +728,14 @@ class CalibreDB: join_archive_read, config_read_column, *join): pagesize = pagesize or self.config.config_books_per_page if current_user.show_detail_random(): - randm = self.session.query(Books) \ - .filter(self.common_filters(allow_show_archived)) \ - .order_by(func.random()) \ - .limit(self.config.config_random_books).all() + random_query = self.generate_linked_query(config_read_column, database) + randm = (random_query.filter(self.common_filters(allow_show_archived)) + .order_by(func.random()) + .limit(self.config.config_random_books).all()) else: randm = false() if join_archive_read: - if not config_read_column: - query = (self.session.query(database, ub.ReadBook.read_status, ub.ArchivedBook.is_archived) - .select_from(Books) - .outerjoin(ub.ReadBook, - and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == Books.id))) - else: - try: - read_column = cc_classes[config_read_column] - query = (self.session.query(database, read_column.value, ub.ArchivedBook.is_archived) - .select_from(Books) - .outerjoin(read_column, read_column.book == Books.id)) - except (KeyError, AttributeError, IndexError): - log.error("Custom Column No.{} is not existing in calibre database".format(read_column)) - # Skip linking read column and return None instead of read status - query = self.session.query(database, None, ub.ArchivedBook.is_archived) - query = query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id)) + query = self.generate_linked_query(config_read_column, database) else: query = self.session.query(database) off = int(int(pagesize) * (page - 1)) @@ -830,21 +833,23 @@ class CalibreDB: authorterms = re.split("[, ]+", term) for authorterm in authorterms: q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%"))) - if not config_read_column: + query = self.generate_linked_query(config_read_column, Books) + '''if not config_read_column: query = (self.session.query(Books, ub.ArchivedBook.is_archived, ub.ReadBook).select_from(Books) .outerjoin(ub.ReadBook, and_(Books.id == ub.ReadBook.book_id, int(current_user.id) == ub.ReadBook.user_id))) else: try: read_column = cc_classes[config_read_column] - query = (self.session.query(Books, ub.ArchivedBook.is_archived, read_column.value).select_from(Books) + query = (self.session.query(Books, ub.ArchivedBook.is_archived, read_column.value) + .select_from(Books) .outerjoin(read_column, read_column.book == Books.id)) except (KeyError, AttributeError, IndexError): log.error("Custom Column No.{} is not existing in calibre database".format(config_read_column)) # Skip linking read column query = self.session.query(Books, ub.ArchivedBook.is_archived, None) query = query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id)) + int(current_user.id) == ub.ArchivedBook.user_id))''' if len(join) == 6: query = query.outerjoin(join[0], join[1]).outerjoin(join[2]).outerjoin(join[3], join[4]).outerjoin(join[5]) diff --git a/cps/opds.py b/cps/opds.py index 180fcacb..702ffe1e 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -26,7 +26,8 @@ from functools import wraps from flask import Blueprint, request, render_template, Response, g, make_response, abort from flask_login import current_user -from sqlalchemy.sql.expression import func, text, or_, and_, any_, true +from sqlalchemy.sql.expression import func, text, or_, and_, true +from sqlalchemy.exc import InvalidRequestError, OperationalError from werkzeug.security import check_password_hash from . import constants, logger, config, db, calibre_db, ub, services, get_locale, isoLanguages from .helper import get_download_link, get_book_cover @@ -108,7 +109,8 @@ def feed_letter_books(book_id): entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, letter, - [db.Books.sort]) + [db.Books.sort], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -118,15 +120,16 @@ def feed_letter_books(book_id): def feed_new(): off = request.args.get("offset") or 0 entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, - db.Books, True, [db.Books.timestamp.desc()]) + db.Books, True, [db.Books.timestamp.desc()], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @opds.route("/opds/discover") @requires_basic_auth_if_no_ano def feed_discover(): - entries = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).order_by(func.random())\ - .limit(config.config_books_per_page) + query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + entries = query.filter(calibre_db.common_filters()).order_by(func.random()).limit(config.config_books_per_page) pagination = Pagination(1, config.config_books_per_page, int(config.config_books_per_page)) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -137,7 +140,8 @@ def feed_best_rated(): off = request.args.get("offset") or 0 entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, db.Books.ratings.any(db.Ratings.rating > 9), - [db.Books.timestamp.desc()]) + [db.Books.timestamp.desc()], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -150,11 +154,11 @@ def feed_hot(): hot_books = all_books.offset(off).limit(config.config_books_per_page) entries = list() for book in hot_books: - download_book = calibre_db.get_book(book.Downloads.book_id) + query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + download_book = query.filter(calibre_db.common_filters()).filter( + book.Downloads.book_id == db.Books.id).first() if download_book: - entries.append( - calibre_db.get_filtered_book(book.Downloads.book_id) - ) + entries.append(download_book) else: ub.delete_download(book.Downloads.book_id) num_books = entries.__len__() @@ -270,7 +274,8 @@ def feed_series(book_id): entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, db.Books.series.any(db.Series.id == book_id), - [db.Books.series_index]) + [db.Books.series_index], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -324,7 +329,8 @@ def feed_format(book_id): entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, db.Books.data.any(db.Data.format == book_id.upper()), - [db.Books.timestamp.desc()]) + [db.Books.timestamp.desc()], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -351,7 +357,8 @@ def feed_languages(book_id): entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, db.Books.languages.any(db.Languages.id == book_id), - [db.Books.timestamp.desc()]) + [db.Books.timestamp.desc()], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -381,13 +388,25 @@ def feed_shelf(book_id): result = list() # user is allowed to access shelf if shelf: - books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == book_id).order_by( - ub.BookShelf.order.asc()).all() - for book in books_in_shelf: - cur_book = calibre_db.get_book(book.book_id) - result.append(cur_book) - pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, - len(result)) + result, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + config.config_books_per_page, + db.Books, + ub.BookShelf.shelf == shelf.id, + [ub.BookShelf.order.asc()], + True, config.config_read_column, + ub.BookShelf, ub.BookShelf.book_id == db.Books.id) + # delete shelf entries where book is not existent anymore, can happen if book is deleted outside calibre-web + wrong_entries = calibre_db.session.query(ub.BookShelf) \ + .join(db.Books, ub.BookShelf.book_id == db.Books.id, isouter=True) \ + .filter(db.Books.id == None).all() + for entry in wrong_entries: + log.info('Not existing book {} in {} deleted'.format(entry.book_id, shelf)) + try: + ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == entry.book_id).delete() + ub.session.commit() + except (OperationalError, InvalidRequestError) as e: + ub.session.rollback() + log.error_or_exception("Settings Database error: {}".format(e)) return render_xml_template('feed.xml', entries=result, pagination=pagination) @@ -451,8 +470,7 @@ def feed_search(term): entries, __, ___ = calibre_db.get_search_results(term, config_read_column=config.config_read_column) entries_count = len(entries) if len(entries) > 0 else 1 pagination = Pagination(1, entries_count, entries_count) - items = [entry[0] for entry in entries] - return render_xml_template('feed.xml', searchterm=term, entries=items, pagination=pagination) + return render_xml_template('feed.xml', searchterm=term, entries=entries, pagination=pagination) else: return render_xml_template('feed.xml', searchterm="") @@ -493,14 +511,16 @@ def render_xml_dataset(data_table, book_id): entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, getattr(db.Books, data_table.__tablename__).any(data_table.id == book_id), - [db.Books.timestamp.desc()]) + [db.Books.timestamp.desc()], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) def render_element_index(database_column, linked_table, folder): shift = 0 off = int(request.args.get("offset") or 0) - entries = calibre_db.session.query(func.upper(func.substr(database_column, 1, 1)).label('id')) + entries = calibre_db.session.query(func.upper(func.substr(database_column, 1, 1)).label('id'), None, None) + # query = calibre_db.generate_linked_query(config.config_read_column, db.Books) if linked_table is not None: entries = entries.join(linked_table).join(db.Books) entries = entries.filter(calibre_db.common_filters()).group_by(func.upper(func.substr(database_column, 1, 1))).all() diff --git a/cps/render_template.py b/cps/render_template.py index 91118049..1bc5454d 100644 --- a/cps/render_template.py +++ b/cps/render_template.py @@ -101,7 +101,7 @@ def get_sidebar_config(kwargs=None): "show_text": _('Show Books List'), "config_show": content}) return sidebar, simple -def get_readbooks_ids(): +'''def get_readbooks_ids(): if not config.config_read_column: readBooks = ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id))\ .filter(ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED).all() @@ -113,11 +113,11 @@ def get_readbooks_ids(): return frozenset([x.book for x in readBooks]) except (KeyError, AttributeError, IndexError): log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) - return [] + return []''' # Returns the template for rendering and includes the instance name def render_title_template(*args, **kwargs): sidebar, simple = get_sidebar_config(kwargs) return render_template(instance=config.config_calibre_web_title, sidebar=sidebar, simple=simple, - accept=constants.EXTENSIONS_UPLOAD, read_book_ids=get_readbooks_ids(), + accept=constants.EXTENSIONS_UPLOAD, # read_book_ids=get_readbooks_ids(), *args, **kwargs) diff --git a/cps/shelf.py b/cps/shelf.py index 0bf12164..35f2941d 100644 --- a/cps/shelf.py +++ b/cps/shelf.py @@ -439,7 +439,7 @@ def render_show_shelf(shelf_type, shelf_id, page_no, sort_param): db.Books, ub.BookShelf.shelf == shelf_id, [ub.BookShelf.order.asc()], - False, 0, + True, config.config_read_column, ub.BookShelf, ub.BookShelf.book_id == db.Books.id) # delete chelf entries where book is not existent anymore, can happen if book is deleted outside calibre-web wrong_entries = calibre_db.session.query(ub.BookShelf) \ diff --git a/cps/templates/author.html b/cps/templates/author.html index b691d398..f7aeb3e1 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -31,23 +31,22 @@
    - {% if entries[0] %} {% for entry in entries %}
    - -

    {{entry.title|shortentitle}}

    +
    +

    {{entry.Books.title|shortentitle}}

    - {% for author in entry.ordered_authors %} + {% for author in entry.Books.authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & @@ -63,23 +62,23 @@ {{author.name.replace('|',',')|shortentitle(30)}} {% endif %} {% endfor %} - {% for format in entry.data %} + {% for format in entry.Books.data %} {% if format.format|lower in g.constants.EXTENSIONS_AUDIO %} {% endif %} {% endfor %}

    - {% if entry.series.__len__() > 0 %} + {% if entry.Books.series.__len__() > 0 %}

    - - {{entry.series[0].name}} + + {{entry.Books.series[0].name}} - ({{entry.series_index|formatseriesindex}}) + ({{entry.Books.series_index|formatseriesindex}})

    {% endif %} - {% if entry.ratings.__len__() > 0 %} + {% if entry.Books.ratings.__len__() > 0 %}
    - {% for number in range((entry.ratings[0].rating/2)|int(2)) %} + {% for number in range((entry.Books.ratings[0].rating/2)|int(2)) %} {% if loop.last and loop.index < 5 %} {% for numer in range(5 - loop.index) %} @@ -92,7 +91,6 @@
    {% endfor %} - {% endif %}
    @@ -110,7 +108,7 @@

    {{entry.title|shortentitle}}

    - {% for author in entry.ordered_authors %} + {% for author in entry.authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {{author.name.replace('|',',')}} {% if loop.last %} diff --git a/cps/templates/discover.html b/cps/templates/discover.html deleted file mode 100644 index 900dad8c..00000000 --- a/cps/templates/discover.html +++ /dev/null @@ -1,65 +0,0 @@ -{% extends "layout.html" %} -{% block body %} -

    -

    {{title}}

    -
    - {% for entry in entries %} -
    -
    - {% if entry.has_cover is defined %} - - - {{ entry.title }} - {% if entry.id in read_book_ids %}{% endif %} - - - {% endif %} -
    -
    - -

    {{entry.title|shortentitle}}

    -
    -

    - {% for author in entry.ordered_authors %} - {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} - {% if not loop.first %} - & - {% endif %} - {{author.name.replace('|',',')|shortentitle(30)}} - {% if loop.last %} - (...) - {% endif %} - {% else %} - {% if not loop.first %} - & - {% endif %} - {{author.name.replace('|',',')|shortentitle(30)}} - {% endif %} - {% endfor %} -

    - {% if entry.series.__len__() > 0 %} -

    - - {{entry.series[0].name}} - - ({{entry.series_index|formatseriesindex}}) -

    - {% endif %} - {% if entry.ratings.__len__() > 0 %} -
    - {% for number in range((entry.ratings[0].rating/2)|int(2)) %} - - {% if loop.last and loop.index < 5 %} - {% for numer in range(5 - loop.index) %} - - {% endfor %} - {% endif %} - {% endfor %} -
    - {% endif %} -
    -
    - {% endfor %} -
    -
    -{% endblock %} diff --git a/cps/templates/feed.xml b/cps/templates/feed.xml index 9073142e..940fb0da 100644 --- a/cps/templates/feed.xml +++ b/cps/templates/feed.xml @@ -40,35 +40,35 @@ {% if entries and entries[0] %} {% for entry in entries %} - {{entry.title}} - urn:uuid:{{entry.uuid}} - {{entry.atom_timestamp}} - {% if entry.authors.__len__() > 0 %} + {{entry.Books.title}} + urn:uuid:{{entry.Books.uuid}} + {{entry.Books.atom_timestamp}} + {% if entry.Books.authors.__len__() > 0 %} - {{entry.authors[0].name}} + {{entry.Books.authors[0].name}} {% endif %} - {% if entry.publishers.__len__() > 0 %} + {% if entry.Books.publishers.__len__() > 0 %} - {{entry.publishers[0].name}} + {{entry.Books.publishers[0].name}} {% endif %} - {% for lang in entry.languages %} + {% for lang in entry.Books.languages %} {{lang.lang_code}} {% endfor %} - {% for tag in entry.tags %} + {% for tag in entry.Books.tags %} {% endfor %} - {% if entry.comments[0] %}{{entry.comments[0].text|striptags}}{% endif %} - {% if entry.has_cover %} - - + {% if entry.Books.comments[0] %}{{entry.Books.comments[0].text|striptags}}{% endif %} + {% if entry.Books.has_cover %} + + {% endif %} - {% for format in entry.data %} - + {% for format in entry.Books.data %} + {% endfor %} {% endfor %} diff --git a/cps/templates/index.html b/cps/templates/index.html index a4547705..4cbf520e 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -1,25 +1,25 @@ {% extends "layout.html" %} {% block body %} -{% if g.user.show_detail_random() %} +{% if g.user.show_detail_random() and page != "discover" %}

    {{_('Discover (Random Books)')}}

    {% for entry in random %}
    - -

    {{entry.title|shortentitle}}

    +
    +

    {{entry.Books.title|shortentitle}}

    - {% for author in entry.ordered_authors %} + {% for author in entry.Books.authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & @@ -36,17 +36,17 @@ {% endif %} {% endfor %}

    - {% if entry.series.__len__() > 0 %} + {% if entry.Books.series.__len__() > 0 %}

    - - {{entry.series[0].name}} + + {{entry.Books.series[0].name}} - ({{entry.series_index|formatseriesindex}}) + ({{entry.Books.series_index|formatseriesindex}})

    {% endif %} - {% if entry.ratings.__len__() > 0 %} + {% if entry.Books.ratings.__len__() > 0 %}
    - {% for number in range((entry.ratings[0].rating/2)|int(2)) %} + {% for number in range((entry.Books.ratings[0].rating/2)|int(2)) %} {% if loop.last and loop.index < 5 %} {% for numer in range(5 - loop.index) %} @@ -64,6 +64,7 @@ {% endif %}

    {{title}}

    + {% if page != 'discover' %} - + {% endif %}
    {% if entries[0] %} {% for entry in entries %}
    - -

    {{entry.title|shortentitle}}

    +
    +

    {{entry.Books.title|shortentitle}}

    - {% for author in entry.ordered_authors %} + {% for author in entry.Books.authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & @@ -117,27 +118,27 @@ {{author.name.replace('|',',')|shortentitle(30)}} {% endif %} {% endfor %} - {% for format in entry.data %} + {% for format in entry.Books.data %} {% if format.format|lower in g.constants.EXTENSIONS_AUDIO %} {% endif %} {%endfor%}

    - {% if entry.series.__len__() > 0 %} + {% if entry.Books.series.__len__() > 0 %}

    {% if page != "series" %} - - {{entry.series[0].name}} + + {{entry.Books.series[0].name}} {% else %} - {{entry.series[0].name}} + {{entry.Books.series[0].name}} {% endif %} - ({{entry.series_index|formatseriesindex}}) + ({{entry.Books.series_index|formatseriesindex}})

    {% endif %} - {% if entry.ratings.__len__() > 0 %} + {% if entry.Books.ratings.__len__() > 0 %}
    - {% for number in range((entry.ratings[0].rating/2)|int(2)) %} + {% for number in range((entry.Books.ratings[0].rating/2)|int(2)) %} {% if loop.last and loop.index < 5 %} {% for numer in range(5 - loop.index) %} diff --git a/cps/templates/search.html b/cps/templates/search.html index 77c01c80..318d06f1 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -46,7 +46,7 @@ {{ entry.Books.title }} - {% if entry.Books.id in read_book_ids %}{% endif %} + {% if entry[2] == True %}{% endif %} {% endif %} diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index a7b5bcf0..0e9b0bd5 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -32,19 +32,19 @@ {% for entry in entries %}
    - -

    {{entry.title|shortentitle}}

    +
    +

    {{entry.Books.title|shortentitle}}

    - {% for author in entry.ordered_authors %} + {% for author in entry.Books.authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & @@ -61,17 +61,17 @@ {% endif %} {% endfor %}

    - {% if entry.series.__len__() > 0 %} + {% if entry.Books.series.__len__() > 0 %}

    - - {{entry.series[0].name}} + + {{entry.Books.series[0].name}} - ({{entry.series_index|formatseriesindex}}) + ({{entry.Books.series_index|formatseriesindex}})

    {% endif %} - {% if entry.ratings.__len__() > 0 %} + {% if entry.Books.ratings.__len__() > 0 %}
    - {% for number in range((entry.ratings[0].rating/2)|int(2)) %} + {% for number in range((entry.Books.ratings[0].rating/2)|int(2)) %} {% if loop.last and loop.index < 5 %} {% for numer in range(5 - loop.index) %} diff --git a/cps/templates/shelfdown.html b/cps/templates/shelfdown.html index c800dca7..f1a0b137 100644 --- a/cps/templates/shelfdown.html +++ b/cps/templates/shelfdown.html @@ -35,31 +35,31 @@
    -

    {{entry.title|shortentitle}}

    +

    {{entry.Books.title|shortentitle}}

    - {% for author in entry.ordered_authors %} + {% for author in entry.Books.authors %} {{author.name.replace('|',',')}} {% if not loop.last %} & {% endif %} {% endfor %}

    - {% if entry.series.__len__() > 0 %} + {% if entry.Books.series.__len__() > 0 %}

    - - {{entry.series[0].name}} + + {{entry.Books.series[0].name}} - ({{entry.series_index}}) + ({{entry.Books.series_index}})

    {% endif %}
    {% if g.user.role_download() %} - {% if entry.data|length %} + {% if entry.Books.data|length %}
    - {% for format in entry.data %} - + {% for format in entry.Books.data %} + {{format.format}} ({{ format.uncompressed_size|filesizeformat }}) {% endfor %} diff --git a/cps/web.py b/cps/web.py index d21ba630..525fccbe 100644 --- a/cps/web.py +++ b/cps/web.py @@ -85,7 +85,10 @@ except ImportError: def add_security_headers(resp): csp = "default-src 'self'" csp += ''.join([' ' + host for host in config.config_trustedhosts.strip().split(',')]) - csp += " 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; img-src 'self' data:" + csp += " 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; img-src 'self' " + if request.path.startswith("/author/") and config.config_use_goodreads: + csp += "images.gr-assets.com i.gr-assets.com s.gr-assets.com" + csp += " data:" resp.headers['Content-Security-Policy'] = csp if request.endpoint == "edit-book.show_edit_book" or config.config_use_google_drive: resp.headers['Content-Security-Policy'] += " *" @@ -350,7 +353,7 @@ def render_books_list(data, sort_param, book_id, page): if data == "rated": return render_rated_books(page, book_id, order=order) elif data == "discover": - return render_discover_books(page, book_id) + return render_discover_books(book_id) elif data == "unread": return render_read_books(page, False, order=order) elif data == "read": @@ -386,7 +389,7 @@ def render_books_list(data, sort_param, book_id, page): else: website = data or "newest" entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, order[0], - False, 0, + True, config.config_read_column, db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) @@ -400,7 +403,7 @@ def render_rated_books(page, book_id, order): db.Books, db.Books.ratings.any(db.Ratings.rating > 9), order[0], - False, 0, + True, config.config_read_column, db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) @@ -411,11 +414,13 @@ def render_rated_books(page, book_id, order): abort(404) -def render_discover_books(page, book_id): +def render_discover_books(book_id): if current_user.check_visibility(constants.SIDEBAR_RANDOM): - entries, __, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, [func.randomblob(2)]) + entries, __, ___ = calibre_db.fill_indexpage(1, 0, db.Books, True, [func.randomblob(2)], + join_archive_read=True, + config_read_column=config.config_read_column) pagination = Pagination(1, config.config_books_per_page, config.config_books_per_page) - return render_title_template('discover.html', entries=entries, pagination=pagination, id=book_id, + return render_title_template('index.html', random=false(), entries=entries, pagination=pagination, id=book_id, title=_(u"Discover (Random Books)"), page="discover") else: abort(404) @@ -429,18 +434,22 @@ def render_hot_books(page, order): # order[0][0].compare(func.count(ub.Downloads.book_id).asc())): order = [func.count(ub.Downloads.book_id).desc()], 'hotdesc' if current_user.show_detail_random(): - random = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \ - .order_by(func.random()).limit(config.config_random_books) + random_query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + random = (random_query.filter(calibre_db.common_filters()) + .order_by(func.random()) + .limit(config.config_random_books).all()) else: random = false() + off = int(int(config.config_books_per_page) * (page - 1)) all_books = ub.session.query(ub.Downloads, func.count(ub.Downloads.book_id)) \ .order_by(*order[0]).group_by(ub.Downloads.book_id) hot_books = all_books.offset(off).limit(config.config_books_per_page) entries = list() for book in hot_books: - download_book = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).filter( - db.Books.id == book.Downloads.book_id).first() + query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + download_book = query.filter(calibre_db.common_filters()).filter( + book.Downloads.book_id == db.Books.id).first() if download_book: entries.append(download_book) else: @@ -459,26 +468,20 @@ def render_downloaded_books(page, order, user_id): else: user_id = current_user.id if current_user.check_visibility(constants.SIDEBAR_DOWNLOAD): - if current_user.show_detail_random(): - random = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \ - .order_by(func.random()).limit(config.config_random_books) - else: - random = false() - - entries, __, pagination = calibre_db.fill_indexpage(page, + entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, ub.Downloads.user_id == user_id, order[0], - False, 0, + True, config.config_read_column, db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series, ub.Downloads, db.Books.id == ub.Downloads.book_id) for book in entries: - if not calibre_db.session.query(db.Books).\ - filter(calibre_db.common_filters()).filter(db.Books.id == book.id).first(): - ub.delete_download(book.id) + if not (calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) + .filter(db.Books.id == book.Books.id).first()): + ub.delete_download(book.Books.id) user = ub.session.query(ub.User).filter(ub.User.id == user_id).first() return render_title_template('index.html', random=random, @@ -497,9 +500,9 @@ def render_author_books(page, author_id, order): db.Books, db.Books.authors.any(db.Authors.id == author_id), [order[0][0], db.Series.name, db.Books.series_index], - False, 0, + True, config.config_read_column, db.books_series_link, - db.Books.id == db.books_series_link.c.book, + db.books_series_link.c.book == db.Books.id, db.Series) if entries is None or not len(entries): flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), @@ -515,7 +518,8 @@ def render_author_books(page, author_id, order): other_books = [] if services.goodreads_support and config.config_use_goodreads: author_info = services.goodreads_support.get_author_info(author_name) - other_books = services.goodreads_support.get_other_books(author_info, entries) + book_entries = [entry.Books for entry in entries] + other_books = services.goodreads_support.get_other_books(author_info, book_entries) return render_title_template('author.html', entries=entries, pagination=pagination, id=author_id, title=_(u"Author: %(name)s", name=author_name), author=author_info, other_books=other_books, page="author", order=order[1]) @@ -528,7 +532,7 @@ def render_publisher_books(page, book_id, order): db.Books, db.Books.publishers.any(db.Publishers.id == book_id), [db.Series.name, order[0][0], db.Books.series_index], - False, 0, + True, config.config_read_column, db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) @@ -546,7 +550,8 @@ def render_series_books(page, book_id, order): entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, db.Books.series.any(db.Series.id == book_id), - [order[0][0]]) + [order[0][0]], + True, config.config_read_column) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"Series: %(serie)s", serie=name.name), page="series", order=order[1]) else: @@ -558,7 +563,8 @@ def render_ratings_books(page, book_id, order): entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, db.Books.ratings.any(db.Ratings.id == book_id), - [order[0][0]]) + [order[0][0]], + True, config.config_read_column) if name and name.rating <= 10: return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), @@ -574,7 +580,8 @@ def render_formats_books(page, book_id, order): entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, db.Books.data.any(db.Data.format == book_id.upper()), - [order[0][0]]) + [order[0][0]], + True, config.config_read_column) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"File format: %(format)s", format=name.format), page="formats", @@ -590,7 +597,7 @@ def render_category_books(page, book_id, order): db.Books, db.Books.tags.any(db.Tags.id == book_id), [order[0][0], db.Series.name, db.Books.series_index], - False, 0, + True, config.config_read_column, db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) @@ -609,7 +616,8 @@ def render_language_books(page, name, order): entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, db.Books.languages.any(db.Languages.lang_code == name), - [order[0][0]]) + [order[0][0]], + True, config.config_read_column) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name, title=_(u"Language: %(name)s", name=lang_name), page="language", order=order[1]) @@ -622,30 +630,12 @@ def render_read_books(page, are_read, as_xml=False, order=None): ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED) else: db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED - entries, random, pagination = calibre_db.fill_indexpage(page, 0, - db.Books, - db_filter, - sort_param, - False, 0, - db.books_series_link, - db.Books.id == db.books_series_link.c.book, - db.Series, - ub.ReadBook, db.Books.id == ub.ReadBook.book_id) else: try: if are_read: db_filter = db.cc_classes[config.config_read_column].value == True else: db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True - entries, random, pagination = calibre_db.fill_indexpage(page, 0, - db.Books, - db_filter, - sort_param, - False, 0, - db.books_series_link, - db.Books.id == db.books_series_link.c.book, - db.Series, - db.cc_classes[config.config_read_column]) except (KeyError, AttributeError, IndexError): log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) if not as_xml: @@ -655,6 +645,15 @@ def render_read_books(page, are_read, as_xml=False, order=None): return redirect(url_for("web.index")) return [] # ToDo: Handle error Case for opds + entries, random, pagination = calibre_db.fill_indexpage(page, 0, + db.Books, + db_filter, + sort_param, + True, config.config_read_column, + db.books_series_link, + db.Books.id == db.books_series_link.c.book, + db.Series) + if as_xml: return entries, pagination else: @@ -683,7 +682,7 @@ def render_archived_books(page, sort_param): archived_filter, order, True, - False, 0) + True, config.config_read_column) name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')' page_name = "archived" @@ -723,7 +722,7 @@ def render_prepare_search_form(cc): def render_search_results(term, offset=None, order=None, limit=None): - join = db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series + join = db.books_series_link, db.books_series_link.c.book == db.Books.id, db.Series entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, @@ -813,27 +812,8 @@ def list_books(): books = calibre_db.search_query(search_param, config.config_read_column).all() filtered_count = len(books) else: - if not config.config_read_column: - books = (calibre_db.session.query(db.Books, ub.ReadBook.read_status, ub.ArchivedBook.is_archived) - .select_from(db.Books) - .outerjoin(ub.ReadBook, - and_(ub.ReadBook.user_id == int(current_user.id), - ub.ReadBook.book_id == db.Books.id))) - else: - read_column = "" - try: - read_column = db.cc_classes[config.config_read_column] - books = (calibre_db.session.query(db.Books, read_column.value, ub.ArchivedBook.is_archived) - .select_from(db.Books) - .outerjoin(read_column, read_column.book == db.Books.id)) - except (KeyError, AttributeError, IndexError): - log.error( - "Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) - # Skip linking read column and return None instead of read status - books = calibre_db.session.query(db.Books, None, ub.ArchivedBook.is_archived) - books = (books.outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id)) - .filter(calibre_db.common_filters(allow_show_archived=True)).all()) + query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + books = query.filter(calibre_db.common_filters(allow_show_archived=True)).all() entries = calibre_db.get_checkbox_sorted(books, state, off, limit, order, True) elif search_param: entries, filtered_count, __ = calibre_db.get_search_results(search_param, @@ -856,8 +836,8 @@ def list_books(): result = list() for entry in entries: val = entry[0] - val.read_status = entry[1] == ub.ReadBook.STATUS_FINISHED - val.is_archived = entry[2] is True + val.is_archived = entry[1] is True + val.read_status = entry[2] == ub.ReadBook.STATUS_FINISHED for lang_index in range(0, len(val.languages)): val.languages[lang_index].language_name = isoLanguages.get_language_name(get_locale(), val.languages[ lang_index].lang_code) @@ -1254,24 +1234,25 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): cc = get_cc_columns(filter_config_custom_read=True) calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase) - if not config.config_read_column: + query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + '''if not config.config_read_column: query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, ub.ReadBook).select_from(db.Books) .outerjoin(ub.ReadBook, and_(db.Books.id == ub.ReadBook.book_id, int(current_user.id) == ub.ReadBook.user_id))) else: try: - read_column = cc[config.config_read_column] + read_column = db.cc_classes[config.config_read_column] query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, read_column.value) - .select_from(db.Books) - .outerjoin(read_column, read_column.book == db.Books.id)) + .select_from(db.Books) + .outerjoin(read_column, read_column.book == db.Books.id)) except (KeyError, AttributeError, IndexError): log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) # Skip linking read column query = calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, None) query = query.outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id)) + int(current_user.id) == ub.ArchivedBook.user_id))''' - q = query.outerjoin(db.books_series_link, db.Books.id == db.books_series_link.c.book) \ + q = query.outerjoin(db.books_series_link, db.books_series_link.c.book == db.Books.id) \ .outerjoin(db.Series) \ .filter(calibre_db.common_filters(True)) diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 7be8da35..4427548a 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
    -

    Start Time: 2022-03-19 22:04:05

    +

    Start Time: 2022-03-26 21:40:01

    -

    Stop Time: 2022-03-20 02:50:09

    +

    Stop Time: 2022-03-27 04:18:33

    -

    Duration: 3h 58 min

    +

    Duration: 4h 50 min

    @@ -891,11 +891,11 @@ - + TestEditBooks 36 - 35 - 0 + 34 + 1 0 1 @@ -1237,11 +1237,31 @@ - +
    TestEditBooks - test_upload_cover_hdd
    - PASS + +
    + FAIL +
    + + + + @@ -1589,8 +1609,8 @@ TestEditBooksOnGdrive 20 - 16 - 4 + 18 + 2 0 0 @@ -1735,31 +1755,11 @@ - +
    TestEditBooksOnGdrive - test_edit_title
    - -
    - FAIL -
    - - - - + PASS @@ -1833,31 +1833,11 @@ AssertionError: 0.0 not greater than 0.02 - +
    TestEditBooksOnGdrive - test_watch_metadata
    - -
    - FAIL -
    - - - - + PASS @@ -2031,11 +2011,11 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - + TestErrorReadColumn 2 - 2 - 0 + 1 + 1 0 0 @@ -2054,11 +2034,87 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
    TestErrorReadColumn - test_invalid_custom_read_column
    - PASS + +
    + FAIL +
    + + + + + + + + + + + _ErrorHolder + 1 + 0 + 0 + 1 + 0 + + Detail + + + + + + + +
    tearDownClass (test_error_read_column)
    + + +
    + ERROR +
    + + + + @@ -2072,13 +2128,13 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're 0 1 - Detail + Detail - +
    TestFilePicker - test_filepicker_limited_file
    @@ -2087,19 +2143,19 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
    TestFilePicker - test_filepicker_new_file
    - SKIP + SKIP
    -