diff --git a/.gitignore b/.gitignore index f06dcd44..cef58094 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ vendor/ # calibre-web *.db *.log +cps/cache .idea/ *.bak diff --git a/cps.py b/cps.py index 50ab0076..e90a38d9 100755 --- a/cps.py +++ b/cps.py @@ -43,6 +43,7 @@ from cps.gdrive import gdrive from cps.editbooks import editbook from cps.remotelogin import remotelogin from cps.error_handler import init_errorhandler +from cps.thumbnails import generate_thumbnails try: from cps.kobo import kobo, get_kobo_activated @@ -78,6 +79,9 @@ def main(): app.register_blueprint(kobo_auth) if oauth_available: app.register_blueprint(oauth) + + generate_thumbnails() + success = web_server.start() sys.exit(0 if success else 1) diff --git a/cps/constants.py b/cps/constants.py index c1bcbe59..0a9f9cd5 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -33,6 +33,7 @@ else: STATIC_DIR = os.path.join(BASE_DIR, 'cps', 'static') TEMPLATES_DIR = os.path.join(BASE_DIR, 'cps', 'templates') TRANSLATIONS_DIR = os.path.join(BASE_DIR, 'cps', 'translations') +CACHE_DIR = os.path.join(BASE_DIR, 'cps', 'cache') if HOME_CONFIG: home_dir = os.path.join(os.path.expanduser("~"),".calibre-web") diff --git a/cps/helper.py b/cps/helper.py index da5ea2b3..6fc6b02a 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -551,6 +551,11 @@ def get_book_cover_with_uuid(book_uuid, def get_book_cover_internal(book, use_generic_cover_on_failure): if book and book.has_cover: + # if thumbnails.cover_thumbnail_exists_for_book(book): + # thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first() + # return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename) + # else: + # WorkerThread.add(None, TaskThumbnail(book, _(u'Generating cover thumbnail for: ' + book.title))) if config.config_use_google_drive: try: if not gd.is_gdrive_ready(): @@ -561,8 +566,8 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): else: log.error('%s/cover.jpg not found on Google Drive', book.path) return get_cover_on_failure(use_generic_cover_on_failure) - except Exception as e: - log.debug_or_exception(e) + except Exception as ex: + log.debug_or_exception(ex) return get_cover_on_failure(use_generic_cover_on_failure) else: cover_file_path = os.path.join(config.config_calibre_dir, book.path) diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py new file mode 100644 index 00000000..c452ab41 --- /dev/null +++ b/cps/tasks/thumbnail.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import os + +from cps import config, db, gdriveutils, logger, ub +from cps.constants import CACHE_DIR as _CACHE_DIR +from cps.services.worker import CalibreTask +from datetime import datetime, timedelta +from sqlalchemy import func + +try: + from wand.image import Image + use_IM = True +except (ImportError, RuntimeError) as e: + use_IM = False + +THUMBNAIL_RESOLUTION_1X = 1.0 +THUMBNAIL_RESOLUTION_2X = 2.0 + + +class TaskThumbnail(CalibreTask): + def __init__(self, limit=100, task_message=u'Generating cover thumbnails'): + super(TaskThumbnail, self).__init__(task_message) + self.limit = limit + self.log = logger.create() + self.app_db_session = ub.get_new_session_instance() + self.worker_db = db.CalibreDB(expire_on_commit=False) + + def run(self, worker_thread): + if self.worker_db.session and use_IM: + thumbnails = self.get_thumbnail_book_ids() + thumbnail_book_ids = list(map(lambda t: t.book_id, thumbnails)) + self.log.info(','.join([str(elem) for elem in thumbnail_book_ids])) + self.log.info(len(thumbnail_book_ids)) + + books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids) + + count = len(books_without_thumbnails) + for i, book in enumerate(books_without_thumbnails): + thumbnails = self.get_thumbnails_for_book(thumbnails, book) + if thumbnails: + for thumbnail in thumbnails: + self.update_book_thumbnail(book, thumbnail) + + else: + self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_1X) + self.create_book_thumbnail(book, THUMBNAIL_RESOLUTION_2X) + + self.progress = (1.0 / count) * i + + self._handleSuccess() + self.app_db_session.close() + + def get_thumbnail_book_ids(self): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .group_by(ub.Thumbnail.book_id)\ + .having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\ + .all() + + def get_books_without_thumbnails(self, thumbnail_book_ids): + return self.worker_db.session\ + .query(db.Books)\ + .filter(db.Books.has_cover == 1)\ + .filter(db.Books.id.notin_(thumbnail_book_ids))\ + .limit(self.limit)\ + .all() + + def get_thumbnails_for_book(self, thumbnails, book): + results = list() + for thumbnail in thumbnails: + if thumbnail.book_id == book.id: + results.append(thumbnail) + + return results + + def update_book_thumbnail(self, book, thumbnail): + thumbnail.expiration = datetime.utcnow() + timedelta(days=30) + + try: + self.app_db_session.commit() + self.generate_book_thumbnail(book, thumbnail) + except Exception as ex: + self._handleError(u'Error updating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def create_book_thumbnail(self, book, resolution): + thumbnail = ub.Thumbnail() + thumbnail.book_id = book.id + thumbnail.resolution = resolution + + self.app_db_session.add(thumbnail) + try: + self.app_db_session.commit() + self.generate_book_thumbnail(book, thumbnail) + except Exception as ex: + self._handleError(u'Error creating book thumbnail: ' + str(ex)) + self.app_db_session.rollback() + + def generate_book_thumbnail(self, book, thumbnail): + if book and thumbnail: + if config.config_use_google_drive: + self.log.info('google drive thumbnail') + else: + book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + if os.path.isfile(book_cover_filepath): + with Image(filename=book_cover_filepath) as img: + height = self.get_thumbnail_height(thumbnail) + if img.height > height: + width = self.get_thumbnail_width(height, img) + img.resize(width=width, height=height, filter='lanczos') + img.save(filename=self.get_thumbnail_cache_path(thumbnail)) + + def get_thumbnail_height(self, thumbnail): + return int(225 * thumbnail.resolution) + + def get_thumbnail_width(self, height, img): + percent = (height / float(img.height)) + return int((float(img.width) * float(percent))) + + def get_thumbnail_cache_dir(self): + if not os.path.isdir(_CACHE_DIR): + os.makedirs(_CACHE_DIR) + + if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')): + os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails')) + + return os.path.join(_CACHE_DIR, 'thumbnails') + + def get_thumbnail_cache_path(self, thumbnail): + if thumbnail: + return os.path.join(self.get_thumbnail_cache_dir(), thumbnail.filename) + return None + + @property + def name(self): + return "Thumbnail" diff --git a/cps/thumbnails.py b/cps/thumbnails.py new file mode 100644 index 00000000..6ccff56f --- /dev/null +++ b/cps/thumbnails.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import os + +from . import logger, ub +from .constants import CACHE_DIR as _CACHE_DIR +from .services.worker import WorkerThread +from .tasks.thumbnail import TaskThumbnail + +from datetime import datetime + +THUMBNAIL_RESOLUTION_1X = 1.0 +THUMBNAIL_RESOLUTION_2X = 2.0 + +log = logger.create() + + +def get_thumbnail_cache_dir(): + if not os.path.isdir(_CACHE_DIR): + os.makedirs(_CACHE_DIR) + + if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')): + os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails')) + + return os.path.join(_CACHE_DIR, 'thumbnails') + + +def get_thumbnail_cache_path(thumbnail): + if thumbnail: + return os.path.join(get_thumbnail_cache_dir(), thumbnail.filename) + + return None + + +def cover_thumbnail_exists_for_book(book): + if book and book.has_cover: + thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first() + if thumbnail and thumbnail.expiration > datetime.utcnow(): + thumbnail_path = get_thumbnail_cache_path(thumbnail) + return thumbnail_path and os.path.isfile(thumbnail_path) + + return False + + +def generate_thumbnails(): + WorkerThread.add(None, TaskThumbnail()) diff --git a/cps/ub.py b/cps/ub.py index dbc3b419..4500160f 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -40,13 +40,14 @@ except ImportError: oauth_support = False from sqlalchemy import create_engine, exc, exists, event from sqlalchemy import Column, ForeignKey -from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON +from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON, Numeric from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.orm import backref, relationship, sessionmaker, Session, scoped_session from werkzeug.security import generate_password_hash -from . import constants +from . import cli, constants session = None @@ -434,6 +435,28 @@ class RemoteAuthToken(Base): return '' % self.id +class Thumbnail(Base): + __tablename__ = 'thumbnail' + + id = Column(Integer, primary_key=True) + book_id = Column(Integer) + uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True) + format = Column(String, default='jpeg') + resolution = Column(Numeric(precision=2, scale=1, asdecimal=False), default=1.0) + expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30)) + + @hybrid_property + def extension(self): + if self.format == 'jpeg': + return 'jpg' + else: + return self.format + + @hybrid_property + def filename(self): + return self.uuid + '.' + self.extension + + # Migrate database to current version, has to be updated after every database change. Currently migration from # everywhere to current should work. Migration is done by checking if relevant columns are existing, and than adding # rows with SQL commands @@ -451,6 +474,8 @@ def migrate_Database(session): KoboStatistics.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "archived_book"): ArchivedBook.__table__.create(bind=engine) + if not engine.dialect.has_table(engine.connect(), "thumbnail"): + Thumbnail.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "registration"): ReadBook.__table__.create(bind=engine) with engine.connect() as conn: @@ -676,6 +701,13 @@ def init_db(app_db_path): create_anonymous_user(session) +def get_new_session_instance(): + new_engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False) + new_session = scoped_session(sessionmaker()) + new_session.configure(bind=new_engine) + return new_session + + def dispose(): global session