From e48bdf9d5a038d169046c0e5352d3c9d074769c6 Mon Sep 17 00:00:00 2001 From: mmonkey Date: Sun, 20 Dec 2020 03:11:21 -0600 Subject: [PATCH] Display thumbnails on the frontend, generate thumbnails from google drive --- cps.py | 5 ++++ cps/__init__.py | 6 ----- cps/helper.py | 37 ++++++++++++++++++++--------- cps/schedule.py | 34 +++++++++++++++++++++++++++ cps/tasks/thumbnail.py | 44 +++++++++++++++++++++++++---------- cps/templates/author.html | 2 +- cps/templates/book_cover.html | 8 +++++++ cps/templates/book_edit.html | 4 +++- cps/templates/detail.html | 3 ++- cps/templates/discover.html | 3 ++- cps/templates/fragment.html | 1 + cps/templates/grid.html | 3 ++- cps/templates/index.html | 5 ++-- cps/templates/layout.html | 1 + cps/templates/search.html | 3 ++- cps/templates/shelf.html | 3 ++- cps/thumbnails.py | 9 ++----- cps/ub.py | 4 ++-- cps/web.py | 11 ++++----- 19 files changed, 133 insertions(+), 53 deletions(-) create mode 100644 cps/schedule.py create mode 100644 cps/templates/book_cover.html diff --git a/cps.py b/cps.py index 50ab0076..d63771eb 100755 --- a/cps.py +++ b/cps.py @@ -43,6 +43,7 @@ from cps.gdrive import gdrive from cps.editbooks import editbook from cps.remotelogin import remotelogin from cps.error_handler import init_errorhandler +from cps.schedule import register_jobs try: from cps.kobo import kobo, get_kobo_activated @@ -78,6 +79,10 @@ def main(): app.register_blueprint(kobo_auth) if oauth_available: app.register_blueprint(oauth) + + # Register scheduled jobs + register_jobs() + success = web_server.start() sys.exit(0 if success else 1) diff --git a/cps/__init__.py b/cps/__init__.py index fa85e15c..6a6d361a 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -36,8 +36,6 @@ from flask_principal import Principal from . import config_sql, logger, cache_buster, cli, ub, db from .reverseproxy import ReverseProxied from .server import WebServer -from .services.background_scheduler import BackgroundScheduler -from .tasks.thumbnail import TaskThumbnail mimetypes.init() @@ -117,10 +115,6 @@ def create_app(): config.config_goodreads_api_secret, config.config_use_goodreads) - scheduler = BackgroundScheduler() - # Generate 100 book cover thumbnails every 5 minutes - scheduler.add_task(user=None, task=lambda: TaskThumbnail(config=config, limit=100), trigger='interval', minutes=5) - return app diff --git a/cps/helper.py b/cps/helper.py index 6fc6b02a..d3420a11 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -52,7 +52,7 @@ except ImportError: from . import calibre_db from .tasks.convert import TaskConvert -from . import logger, config, get_locale, db, ub +from . import logger, config, get_locale, db, thumbnails, ub from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR from .subproc_wrapper import process_wait @@ -538,24 +538,27 @@ def get_cover_on_failure(use_generic_cover): return None -def get_book_cover(book_id): +def get_book_cover(book_id, resolution=1): book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) - return get_book_cover_internal(book, use_generic_cover_on_failure=True) + return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) -def get_book_cover_with_uuid(book_uuid, - use_generic_cover_on_failure=True): +def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True): book = calibre_db.get_book_by_uuid(book_uuid) return get_book_cover_internal(book, use_generic_cover_on_failure) -def get_book_cover_internal(book, use_generic_cover_on_failure): +def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=1, disable_thumbnail=False): if book and book.has_cover: - # if thumbnails.cover_thumbnail_exists_for_book(book): - # thumbnail = ub.session.query(ub.Thumbnail).filter(ub.Thumbnail.book_id == book.id).first() - # return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename) - # else: - # WorkerThread.add(None, TaskThumbnail(book, _(u'Generating cover thumbnail for: ' + book.title))) + + # Send the book cover thumbnail if it exists in cache + if not disable_thumbnail: + thumbnail = get_book_cover_thumbnail(book, resolution) + if thumbnail: + if os.path.isfile(thumbnails.get_thumbnail_cache_path(thumbnail)): + return send_from_directory(thumbnails.get_thumbnail_cache_dir(), thumbnail.filename) + + # Send the book cover from Google Drive if configured if config.config_use_google_drive: try: if not gd.is_gdrive_ready(): @@ -569,6 +572,8 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): except Exception as ex: log.debug_or_exception(ex) return get_cover_on_failure(use_generic_cover_on_failure) + + # Send the book cover from the Calibre directory else: cover_file_path = os.path.join(config.config_calibre_dir, book.path) if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")): @@ -579,6 +584,16 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): return get_cover_on_failure(use_generic_cover_on_failure) +def get_book_cover_thumbnail(book, resolution=1): + if book and book.has_cover: + return ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id == book.id)\ + .filter(ub.Thumbnail.resolution == resolution)\ + .filter(ub.Thumbnail.expiration > datetime.utcnow())\ + .first() + + # saves book cover from url def save_cover_from_url(url, book_path): try: diff --git a/cps/schedule.py b/cps/schedule.py new file mode 100644 index 00000000..5d2c94b9 --- /dev/null +++ b/cps/schedule.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2020 mmonkey +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals + +from . import logger +from .services.background_scheduler import BackgroundScheduler +from .tasks.thumbnail import TaskThumbnail + +log = logger.create() + + +def register_jobs(): + scheduler = BackgroundScheduler() + + # Generate 100 book cover thumbnails every 5 minutes + scheduler.add_task(user=None, task=lambda: TaskThumbnail(limit=100), trigger='interval', minutes=5) + + # TODO: validate thumbnail scheduled task diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index 378b688e..4e0c6db4 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -19,11 +19,13 @@ from __future__ import division, print_function, unicode_literals import os -from cps import db, logger, ub +from cps import config, db, gdriveutils, logger, ub from cps.constants import CACHE_DIR as _CACHE_DIR from cps.services.worker import CalibreTask +from cps.thumbnails import THUMBNAIL_RESOLUTION_1X, THUMBNAIL_RESOLUTION_2X from datetime import datetime, timedelta from sqlalchemy import func +from urllib.request import urlopen try: from wand.image import Image @@ -31,14 +33,10 @@ try: except (ImportError, RuntimeError) as e: use_IM = False -THUMBNAIL_RESOLUTION_1X = 1.0 -THUMBNAIL_RESOLUTION_2X = 2.0 - class TaskThumbnail(CalibreTask): - def __init__(self, config, limit=100, task_message=u'Generating cover thumbnails'): + def __init__(self, limit=100, task_message=u'Generating cover thumbnails'): super(TaskThumbnail, self).__init__(task_message) - self.config = config self.limit = limit self.log = logger.create() self.app_db_session = ub.get_new_session_instance() @@ -114,17 +112,39 @@ class TaskThumbnail(CalibreTask): def generate_book_thumbnail(self, book, thumbnail): if book and thumbnail: - if self.config.config_use_google_drive: - self.log.info('google drive thumbnail') - else: - book_cover_filepath = os.path.join(self.config.config_calibre_dir, book.path, 'cover.jpg') - if os.path.isfile(book_cover_filepath): - with Image(filename=book_cover_filepath) as img: + if config.config_use_google_drive: + if not gdriveutils.is_gdrive_ready(): + raise Exception('Google Drive is configured but not ready') + + web_content_link = gdriveutils.get_cover_via_gdrive(book.path) + if not web_content_link: + raise Exception('Google Drive cover url not found') + + stream = None + try: + stream = urlopen(web_content_link) + with Image(file=stream) as img: height = self.get_thumbnail_height(thumbnail) if img.height > height: width = self.get_thumbnail_width(height, img) img.resize(width=width, height=height, filter='lanczos') img.save(filename=self.get_thumbnail_cache_path(thumbnail)) + except Exception as ex: + # Bubble exception to calling function + raise ex + finally: + stream.close() + else: + book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + if not os.path.isfile(book_cover_filepath): + raise Exception('Book cover file not found') + + with Image(filename=book_cover_filepath) as img: + height = self.get_thumbnail_height(thumbnail) + if img.height > height: + width = self.get_thumbnail_width(height, img) + img.resize(width=width, height=height, filter='lanczos') + img.save(filename=self.get_thumbnail_cache_path(thumbnail)) def get_thumbnail_height(self, thumbnail): return int(225 * thumbnail.resolution) diff --git a/cps/templates/author.html b/cps/templates/author.html index 7887aa4a..24ce876a 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -36,7 +36,7 @@
- + {{ book_cover_image(entry.id, entry.title) }}
diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html new file mode 100644 index 00000000..878c14a8 --- /dev/null +++ b/cps/templates/book_cover.html @@ -0,0 +1,8 @@ +{% macro book_cover_image(book_id, book_title) -%} + {{ book_title }} +{%- endmacro %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 003b33f9..881fa8ff 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -1,9 +1,11 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %} {% if book %}
- {{ book.title }} + {{ book_cover_image(book.id, book.title) }} +
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index 503d1dbd..d3615563 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,7 +4,8 @@
- {{ entry.title }} + {{ book_cover_image(entry.id, entry.title) }} +
diff --git a/cps/templates/discover.html b/cps/templates/discover.html index 3c858feb..33bafbbe 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -8,7 +9,7 @@
{% if entry.has_cover is defined %} - {{ entry.title }} + {{ book_cover_image(entry.id, entry.title) }} {% endif %}
diff --git a/cps/templates/fragment.html b/cps/templates/fragment.html index 1421ea6a..901dd193 100644 --- a/cps/templates/fragment.html +++ b/cps/templates/fragment.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %}
{% block body %}{% endblock %}
diff --git a/cps/templates/grid.html b/cps/templates/grid.html index ce2c05ac..bc3ca4a2 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}

{{_(title)}}

@@ -28,7 +29,7 @@
- {{ entry[0].name }} + {{ book_cover_image(entry[0].id, entry[0].name) }} {{entry.count}}
diff --git a/cps/templates/index.html b/cps/templates/index.html index 1db73c89..c536884f 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %} {% if g.user.show_detail_random() %} @@ -8,7 +9,7 @@
- {{ entry.title }} + {{ book_cover_image(entry.id, entry.title) }}
@@ -82,7 +83,7 @@
- {{ entry.title }} + {{ book_cover_image(entry.id, entry.title) }}
diff --git a/cps/templates/layout.html b/cps/templates/layout.html index 3b89a7ce..5df471a9 100644 --- a/cps/templates/layout.html +++ b/cps/templates/layout.html @@ -1,4 +1,5 @@ {% from 'modal_dialogs.html' import restrict_modal, delete_book, filechooser_modal %} +{% from 'book_cover.html' import book_cover_image %} diff --git a/cps/templates/search.html b/cps/templates/search.html index aedb6f45..56b12154 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -43,7 +44,7 @@
{% if entry.has_cover is defined %} - {{ entry.title }} + {{ book_cover_image(entry.id, entry.title) }} {% endif %}
diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index f7e3c1ae..7a678ea6 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -1,3 +1,4 @@ +{% from 'book_cover.html' import book_cover_image %} {% extends "layout.html" %} {% block body %}
@@ -30,7 +31,7 @@
- {{ entry.title }} + {{ book_cover_image(entry.id, entry.title) }}
diff --git a/cps/thumbnails.py b/cps/thumbnails.py index 89e68f50..ea7aac86 100644 --- a/cps/thumbnails.py +++ b/cps/thumbnails.py @@ -21,13 +21,11 @@ import os from . import logger, ub from .constants import CACHE_DIR as _CACHE_DIR -from .services.worker import WorkerThread -from .tasks.thumbnail import TaskThumbnail from datetime import datetime -THUMBNAIL_RESOLUTION_1X = 1.0 -THUMBNAIL_RESOLUTION_2X = 2.0 +THUMBNAIL_RESOLUTION_1X = 1 +THUMBNAIL_RESOLUTION_2X = 2 log = logger.create() @@ -35,17 +33,14 @@ log = logger.create() def get_thumbnail_cache_dir(): if not os.path.isdir(_CACHE_DIR): os.makedirs(_CACHE_DIR) - if not os.path.isdir(os.path.join(_CACHE_DIR, 'thumbnails')): os.makedirs(os.path.join(_CACHE_DIR, 'thumbnails')) - return os.path.join(_CACHE_DIR, 'thumbnails') def get_thumbnail_cache_path(thumbnail): if thumbnail: return os.path.join(get_thumbnail_cache_dir(), thumbnail.filename) - return None diff --git a/cps/ub.py b/cps/ub.py index 4500160f..0b5a65e7 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -40,7 +40,7 @@ except ImportError: oauth_support = False from sqlalchemy import create_engine, exc, exists, event from sqlalchemy import Column, ForeignKey -from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON, Numeric +from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm.attributes import flag_modified @@ -442,7 +442,7 @@ class Thumbnail(Base): book_id = Column(Integer) uuid = Column(String, default=lambda: str(uuid.uuid4()), unique=True) format = Column(String, default='jpeg') - resolution = Column(Numeric(precision=2, scale=1, asdecimal=False), default=1.0) + resolution = Column(SmallInteger, default=1) expiration = Column(DateTime, default=lambda: datetime.datetime.utcnow() + datetime.timedelta(days=30)) @hybrid_property diff --git a/cps/web.py b/cps/web.py index 4baf82cb..27a5849b 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1171,14 +1171,17 @@ def advanced_search_form(): @web.route("/cover/") +@web.route("/cover//") @login_required_if_no_ano -def get_cover(book_id): - return get_book_cover(book_id) +def get_cover(book_id, resolution=1): + return get_book_cover(book_id, resolution) + @web.route("/robots.txt") def get_robots(): return send_from_directory(constants.STATIC_DIR, "robots.txt") + @web.route("/show//", defaults={'anyname': 'None'}) @web.route("/show///") @login_required_if_no_ano @@ -1205,7 +1208,6 @@ def serve_book(book_id, book_format, anyname): return send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + book_format) - @web.route("/download//", defaults={'anyname': 'None'}) @web.route("/download///") @login_required_if_no_ano @@ -1387,9 +1389,6 @@ def logout(): return redirect(url_for('web.login')) - - - # ################################### Users own configuration #########################################################