diff --git a/cps/db.py b/cps/db.py index 2e428f72..8b8db10a 100644 --- a/cps/db.py +++ b/cps/db.py @@ -609,7 +609,8 @@ class CalibreDB(): randm = self.session.query(Books) \ .filter(self.common_filters(allow_show_archived)) \ .order_by(func.random()) \ - .limit(self.config.config_random_books) + .limit(self.config.config_random_books) \ + .all() else: randm = false() off = int(int(pagesize) * (page - 1)) diff --git a/cps/helper.py b/cps/helper.py index 0b0c675f..c33a69e1 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -533,6 +533,21 @@ def delete_book(book, calibrepath, book_format): return delete_book_file(book, calibrepath, book_format) +def get_thumbnails_for_books(books): + books_with_covers = list(filter(lambda b: b.has_cover, books)) + book_ids = list(map(lambda b: b.id, books_with_covers)) + return ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.in_(book_ids))\ + .filter(ub.Thumbnail.expiration > datetime.utcnow())\ + .all() + + +def get_thumbnails_for_book_series(series): + books = list(map(lambda s: s[0], series)) + return get_thumbnails_for_books(books) + + def get_cover_on_failure(use_generic_cover): if use_generic_cover: return send_from_directory(_STATIC_DIR, "generic_cover.jpg") @@ -558,6 +573,29 @@ def get_cached_book_cover(cache_id): return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) +def get_cached_book_cover_thumbnail(cache_id): + parts = cache_id.split('_') + thumbnail_uuid = parts[0] if len(parts) else None + thumbnail = None + if thumbnail_uuid: + thumbnail = ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.uuid == thumbnail_uuid)\ + .first() + + if thumbnail and thumbnail.expiration > datetime.utcnow(): + cache = fs.FileSystem() + if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename) + + elif thumbnail: + book = calibre_db.get_book(thumbnail.book_id) + return get_book_cover_internal(book, use_generic_cover_on_failure=True) + + else: + return get_cover_on_failure(True) + + def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None): if book and book.has_cover: diff --git a/cps/jinjia.py b/cps/jinjia.py index bf81c059..b2479adc 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -139,3 +139,19 @@ def book_cover_cache_id(book, resolution=None): timestamp = int(book.last_modified.timestamp() * 1000) cache_bust = str(book.uuid) + '_' + str(timestamp) return cache_bust if not resolution else cache_bust + '_' + str(resolution) + + +@jinjia.app_template_filter('get_book_thumbnails') +def get_book_thumbnails(book_id, thumbnails=None): + return list(filter(lambda t: t.book_id == book_id, thumbnails)) if book_id > -1 and thumbnails else list() + + +@jinjia.app_template_filter('get_book_thumbnail_srcset') +def get_book_thumbnail_srcset(thumbnails): + srcset = list() + for thumbnail in thumbnails: + timestamp = int(thumbnail.generated_at.timestamp() * 1000) + cache_id = str(thumbnail.uuid) + '_' + str(timestamp) + url = url_for('web.get_cached_cover_thumbnail', cache_id=cache_id) + srcset.append(url + ' ' + str(thumbnail.resolution) + 'x') + return ', '.join(srcset) diff --git a/cps/schedule.py b/cps/schedule.py index f349a231..7ee43410 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -20,17 +20,17 @@ from __future__ import division, print_function, unicode_literals from .services.background_scheduler import BackgroundScheduler from .tasks.database import TaskReconnectDatabase -from .tasks.thumbnail import TaskCleanupCoverThumbnailCache, TaskGenerateCoverThumbnails +from .tasks.thumbnail import TaskSyncCoverThumbnailCache, TaskGenerateCoverThumbnails def register_jobs(): scheduler = BackgroundScheduler() # Generate 100 book cover thumbnails every 5 minutes - scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='interval', minutes=5) + scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='cron', minute='*/5') - # Cleanup book cover cache every day at 4am - scheduler.add_task(user=None, task=lambda: TaskCleanupCoverThumbnailCache(), trigger='cron', hour=4) + # Cleanup book cover cache every 6 hours + scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='cron', minute='15', hour='*/6') # Reconnect metadata.db every 4 hours - scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='interval', hours=4) + scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', minute='5', hour='*/4') diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index b541bc70..e9df170e 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -59,6 +59,10 @@ class TaskGenerateCoverThumbnails(CalibreTask): books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids) count = len(books_without_thumbnails) + if count == 0: + # Do not display this task on the frontend if there are no covers to update + self.self_cleanup = True + for i, book in enumerate(books_without_thumbnails): for resolution in self.resolutions: expired_thumbnail = self.get_expired_thumbnail_for_book_and_resolution( @@ -71,6 +75,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): else: self.create_book_thumbnail(book, resolution) + self.message(u'Generating cover thumbnail {0} of {1}'.format(i, count)) self.progress = (1.0 / count) * i self._handleSuccess() @@ -181,12 +186,12 @@ class TaskGenerateCoverThumbnails(CalibreTask): @property def name(self): - return "GenerateCoverThumbnails" + return "ThumbnailsGenerate" -class TaskCleanupCoverThumbnailCache(CalibreTask): - def __init__(self, task_message=u'Cleaning up cover thumbnail cache'): - super(TaskCleanupCoverThumbnailCache, self).__init__(task_message) +class TaskSyncCoverThumbnailCache(CalibreTask): + def __init__(self, task_message=u'Syncing cover thumbnail cache'): + super(TaskSyncCoverThumbnailCache, self).__init__(task_message) self.log = logger.create() self.app_db_session = ub.get_new_session_instance() self.calibre_db = db.CalibreDB(expire_on_commit=False) @@ -199,14 +204,23 @@ class TaskCleanupCoverThumbnailCache(CalibreTask): # This case will happen if a user deletes the cache dir or cached files if self.app_db_session: self.expire_missing_thumbnails(cached_thumbnail_files) - self.progress = 0.33 + self.progress = 0.25 # Delete thumbnails in the database if the book has been removed # This case will happen if a book is removed in Calibre and the metadata.db file is updated in the filesystem if self.app_db_session and self.calibre_db: book_ids = self.get_book_ids() self.delete_thumbnails_for_missing_books(book_ids) - self.progress = 0.66 + self.progress = 0.50 + + # Expire thumbnails in the database if their corresponding book has been updated since they were generated + # This case will happen if the book was updated externally + if self.app_db_session and self.cache: + books = self.get_books_updated_in_the_last_day() + book_ids = list(map(lambda b: b.id, books)) + thumbnails = self.get_thumbnails_for_updated_books(book_ids) + self.expire_thumbnails_for_updated_book(books, thumbnails) + self.progress = 0.75 # Delete extraneous cached thumbnail files # This case will happen if a book was deleted and the thumbnail OR the metadata.db file was changed externally @@ -261,9 +275,40 @@ class TaskCleanupCoverThumbnailCache(CalibreTask): for file in extraneous_files: self.cache.delete_cache_file(file, fs.CACHE_TYPE_THUMBNAILS) + def get_books_updated_in_the_last_day(self): + return self.calibre_db.session\ + .query(db.Books)\ + .filter(db.Books.has_cover == 1)\ + .filter(db.Books.last_modified > datetime.utcnow() - timedelta(days=1, hours=1))\ + .all() + + def get_thumbnails_for_updated_books(self, book_ids): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.in_(book_ids))\ + .all() + + def expire_thumbnails_for_updated_book(self, books, thumbnails): + thumbnail_ids = list() + for book in books: + for thumbnail in thumbnails: + if thumbnail.book_id == book.id and thumbnail.generated_at < book.last_modified: + thumbnail_ids.append(thumbnail.id) + + try: + self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.id.in_(thumbnail_ids)) \ + .update({"expiration": datetime.utcnow()}, synchronize_session=False) + self.app_db_session.commit() + except Exception as ex: + self.log.info(u'Error expiring thumbnails for updated books: ' + str(ex)) + self._handleError(u'Error expiring thumbnails for updated books: ' + str(ex)) + self.app_db_session.rollback() + @property def name(self): - return "CleanupCoverThumbnailCache" + return "ThumbnailsSync" class TaskClearCoverThumbnailCache(CalibreTask): @@ -318,4 +363,4 @@ class TaskClearCoverThumbnailCache(CalibreTask): @property def name(self): - return "ClearCoverThumbnailCache" + return "ThumbnailsClear" diff --git a/cps/templates/author.html b/cps/templates/author.html index 3cf3fb4b..e5acdd2f 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -36,7 +36,7 @@