diff --git a/cps/db.py b/cps/db.py index 2e428f72..8b8db10a 100644 --- a/cps/db.py +++ b/cps/db.py @@ -609,7 +609,8 @@ class CalibreDB(): randm = self.session.query(Books) \ .filter(self.common_filters(allow_show_archived)) \ .order_by(func.random()) \ - .limit(self.config.config_random_books) + .limit(self.config.config_random_books) \ + .all() else: randm = false() off = int(int(pagesize) * (page - 1)) diff --git a/cps/helper.py b/cps/helper.py index 0b0c675f..c33a69e1 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -533,6 +533,21 @@ def delete_book(book, calibrepath, book_format): return delete_book_file(book, calibrepath, book_format) +def get_thumbnails_for_books(books): + books_with_covers = list(filter(lambda b: b.has_cover, books)) + book_ids = list(map(lambda b: b.id, books_with_covers)) + return ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.in_(book_ids))\ + .filter(ub.Thumbnail.expiration > datetime.utcnow())\ + .all() + + +def get_thumbnails_for_book_series(series): + books = list(map(lambda s: s[0], series)) + return get_thumbnails_for_books(books) + + def get_cover_on_failure(use_generic_cover): if use_generic_cover: return send_from_directory(_STATIC_DIR, "generic_cover.jpg") @@ -558,6 +573,29 @@ def get_cached_book_cover(cache_id): return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution) +def get_cached_book_cover_thumbnail(cache_id): + parts = cache_id.split('_') + thumbnail_uuid = parts[0] if len(parts) else None + thumbnail = None + if thumbnail_uuid: + thumbnail = ub.session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.uuid == thumbnail_uuid)\ + .first() + + if thumbnail and thumbnail.expiration > datetime.utcnow(): + cache = fs.FileSystem() + if cache.get_cache_file_path(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS): + return send_from_directory(cache.get_cache_dir(fs.CACHE_TYPE_THUMBNAILS), thumbnail.filename) + + elif thumbnail: + book = calibre_db.get_book(thumbnail.book_id) + return get_book_cover_internal(book, use_generic_cover_on_failure=True) + + else: + return get_cover_on_failure(True) + + def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None): if book and book.has_cover: diff --git a/cps/jinjia.py b/cps/jinjia.py index bf81c059..b2479adc 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -139,3 +139,19 @@ def book_cover_cache_id(book, resolution=None): timestamp = int(book.last_modified.timestamp() * 1000) cache_bust = str(book.uuid) + '_' + str(timestamp) return cache_bust if not resolution else cache_bust + '_' + str(resolution) + + +@jinjia.app_template_filter('get_book_thumbnails') +def get_book_thumbnails(book_id, thumbnails=None): + return list(filter(lambda t: t.book_id == book_id, thumbnails)) if book_id > -1 and thumbnails else list() + + +@jinjia.app_template_filter('get_book_thumbnail_srcset') +def get_book_thumbnail_srcset(thumbnails): + srcset = list() + for thumbnail in thumbnails: + timestamp = int(thumbnail.generated_at.timestamp() * 1000) + cache_id = str(thumbnail.uuid) + '_' + str(timestamp) + url = url_for('web.get_cached_cover_thumbnail', cache_id=cache_id) + srcset.append(url + ' ' + str(thumbnail.resolution) + 'x') + return ', '.join(srcset) diff --git a/cps/schedule.py b/cps/schedule.py index f349a231..7ee43410 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -20,17 +20,17 @@ from __future__ import division, print_function, unicode_literals from .services.background_scheduler import BackgroundScheduler from .tasks.database import TaskReconnectDatabase -from .tasks.thumbnail import TaskCleanupCoverThumbnailCache, TaskGenerateCoverThumbnails +from .tasks.thumbnail import TaskSyncCoverThumbnailCache, TaskGenerateCoverThumbnails def register_jobs(): scheduler = BackgroundScheduler() # Generate 100 book cover thumbnails every 5 minutes - scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='interval', minutes=5) + scheduler.add_task(user=None, task=lambda: TaskGenerateCoverThumbnails(limit=100), trigger='cron', minute='*/5') - # Cleanup book cover cache every day at 4am - scheduler.add_task(user=None, task=lambda: TaskCleanupCoverThumbnailCache(), trigger='cron', hour=4) + # Cleanup book cover cache every 6 hours + scheduler.add_task(user=None, task=lambda: TaskSyncCoverThumbnailCache(), trigger='cron', minute='15', hour='*/6') # Reconnect metadata.db every 4 hours - scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='interval', hours=4) + scheduler.add_task(user=None, task=lambda: TaskReconnectDatabase(), trigger='cron', minute='5', hour='*/4') diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index b541bc70..e9df170e 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -59,6 +59,10 @@ class TaskGenerateCoverThumbnails(CalibreTask): books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids) count = len(books_without_thumbnails) + if count == 0: + # Do not display this task on the frontend if there are no covers to update + self.self_cleanup = True + for i, book in enumerate(books_without_thumbnails): for resolution in self.resolutions: expired_thumbnail = self.get_expired_thumbnail_for_book_and_resolution( @@ -71,6 +75,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): else: self.create_book_thumbnail(book, resolution) + self.message(u'Generating cover thumbnail {0} of {1}'.format(i, count)) self.progress = (1.0 / count) * i self._handleSuccess() @@ -181,12 +186,12 @@ class TaskGenerateCoverThumbnails(CalibreTask): @property def name(self): - return "GenerateCoverThumbnails" + return "ThumbnailsGenerate" -class TaskCleanupCoverThumbnailCache(CalibreTask): - def __init__(self, task_message=u'Cleaning up cover thumbnail cache'): - super(TaskCleanupCoverThumbnailCache, self).__init__(task_message) +class TaskSyncCoverThumbnailCache(CalibreTask): + def __init__(self, task_message=u'Syncing cover thumbnail cache'): + super(TaskSyncCoverThumbnailCache, self).__init__(task_message) self.log = logger.create() self.app_db_session = ub.get_new_session_instance() self.calibre_db = db.CalibreDB(expire_on_commit=False) @@ -199,14 +204,23 @@ class TaskCleanupCoverThumbnailCache(CalibreTask): # This case will happen if a user deletes the cache dir or cached files if self.app_db_session: self.expire_missing_thumbnails(cached_thumbnail_files) - self.progress = 0.33 + self.progress = 0.25 # Delete thumbnails in the database if the book has been removed # This case will happen if a book is removed in Calibre and the metadata.db file is updated in the filesystem if self.app_db_session and self.calibre_db: book_ids = self.get_book_ids() self.delete_thumbnails_for_missing_books(book_ids) - self.progress = 0.66 + self.progress = 0.50 + + # Expire thumbnails in the database if their corresponding book has been updated since they were generated + # This case will happen if the book was updated externally + if self.app_db_session and self.cache: + books = self.get_books_updated_in_the_last_day() + book_ids = list(map(lambda b: b.id, books)) + thumbnails = self.get_thumbnails_for_updated_books(book_ids) + self.expire_thumbnails_for_updated_book(books, thumbnails) + self.progress = 0.75 # Delete extraneous cached thumbnail files # This case will happen if a book was deleted and the thumbnail OR the metadata.db file was changed externally @@ -261,9 +275,40 @@ class TaskCleanupCoverThumbnailCache(CalibreTask): for file in extraneous_files: self.cache.delete_cache_file(file, fs.CACHE_TYPE_THUMBNAILS) + def get_books_updated_in_the_last_day(self): + return self.calibre_db.session\ + .query(db.Books)\ + .filter(db.Books.has_cover == 1)\ + .filter(db.Books.last_modified > datetime.utcnow() - timedelta(days=1, hours=1))\ + .all() + + def get_thumbnails_for_updated_books(self, book_ids): + return self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.book_id.in_(book_ids))\ + .all() + + def expire_thumbnails_for_updated_book(self, books, thumbnails): + thumbnail_ids = list() + for book in books: + for thumbnail in thumbnails: + if thumbnail.book_id == book.id and thumbnail.generated_at < book.last_modified: + thumbnail_ids.append(thumbnail.id) + + try: + self.app_db_session\ + .query(ub.Thumbnail)\ + .filter(ub.Thumbnail.id.in_(thumbnail_ids)) \ + .update({"expiration": datetime.utcnow()}, synchronize_session=False) + self.app_db_session.commit() + except Exception as ex: + self.log.info(u'Error expiring thumbnails for updated books: ' + str(ex)) + self._handleError(u'Error expiring thumbnails for updated books: ' + str(ex)) + self.app_db_session.rollback() + @property def name(self): - return "CleanupCoverThumbnailCache" + return "ThumbnailsSync" class TaskClearCoverThumbnailCache(CalibreTask): @@ -318,4 +363,4 @@ class TaskClearCoverThumbnailCache(CalibreTask): @property def name(self): - return "ClearCoverThumbnailCache" + return "ThumbnailsClear" diff --git a/cps/templates/author.html b/cps/templates/author.html index 3cf3fb4b..e5acdd2f 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -36,7 +36,7 @@
- {{ book_cover_image(entry, entry.title) }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
diff --git a/cps/templates/book_cover.html b/cps/templates/book_cover.html index 3f0ed154..c5281797 100644 --- a/cps/templates/book_cover.html +++ b/cps/templates/book_cover.html @@ -1,8 +1,13 @@ -{% macro book_cover_image(book, book_title) -%} - {{ book_title }} +{% macro book_cover_image(book, thumbnails) -%} + {%- set book_title = book.title if book.title else book.name -%} + {% set srcset = thumbnails|get_book_thumbnail_srcset if thumbnails|length else '' %} + {%- if srcset|length -%} + {{ book_title }} + {%- else -%} + {{ book_title }} + {%- endif -%} {%- endmacro %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index 369b8d05..c538d5ca 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -4,7 +4,7 @@ {% if book %}
- {{ book_cover_image(book, book.title) }} + {{ book_cover_image(book, book.id|get_book_thumbnails(thumbnails)) }}
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index cdf6ab2b..671186c7 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,7 +4,7 @@
- {{ book_cover_image(entry, entry.title) }} + {{ book_cover_image(entry, entry.id|get_book_thumbnails(thumbnails)) }}
diff --git a/cps/templates/discover.html b/cps/templates/discover.html index 5d4666f6..c5c12db2 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -9,7 +9,7 @@ diff --git a/cps/templates/grid.html b/cps/templates/grid.html index 67594b4e..0f669d51 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -29,7 +29,7 @@
diff --git a/cps/templates/index.html b/cps/templates/index.html index 531a535c..2be15ba7 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -9,7 +9,7 @@
@@ -83,7 +83,7 @@
diff --git a/cps/templates/search.html b/cps/templates/search.html index 8e0cf668..a5871afb 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -44,7 +44,7 @@ diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index bebc0b1f..cb55c50c 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -31,7 +31,7 @@
diff --git a/cps/web.py b/cps/web.py index 16c14fcf..c400b96e 100644 --- a/cps/web.py +++ b/cps/web.py @@ -49,9 +49,10 @@ from . import constants, logger, isoLanguages, services from . import babel, db, ub, config, get_locale, app from . import calibre_db, shelf from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download -from .helper import check_valid_domain, render_task_status, \ - get_cc_columns, get_book_cover, get_cached_book_cover, get_download_link, send_mail, generate_random_password, \ - send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password +from .helper import check_valid_domain, render_task_status, get_cc_columns, get_book_cover, get_cached_book_cover, \ + get_cached_book_cover_thumbnail, get_thumbnails_for_books, get_thumbnails_for_book_series, get_download_link, \ + send_mail, generate_random_password, send_registration_mail, check_send_to_kindle, check_read_formats, \ + tags_filters, reset_password from .pagination import Pagination from .redirect import redirect_back from .usermanagement import login_required_if_no_ano @@ -386,16 +387,18 @@ def render_books_list(data, sort, book_id, page): db.Books, db.Books.ratings.any(db.Ratings.rating > 9), order) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - id=book_id, title=_(u"Top Rated Books"), page="rated") + id=book_id, title=_(u"Top Rated Books"), page="rated", thumbnails=thumbnails) else: abort(404) elif data == "discover": if current_user.check_visibility(constants.SIDEBAR_RANDOM): entries, __, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, [func.randomblob(2)]) pagination = Pagination(1, config.config_books_per_page, config.config_books_per_page) + thumbnails = get_thumbnails_for_books(entries) return render_title_template('discover.html', entries=entries, pagination=pagination, id=book_id, - title=_(u"Discover (Random Books)"), page="discover") + title=_(u"Discover (Random Books)"), page="discover", thumbnails=thumbnails) else: abort(404) elif data == "unread": @@ -433,8 +436,9 @@ def render_books_list(data, sort, book_id, page): else: website = data or "newest" entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, order) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=_(u"Books"), page=website) + title=_(u"Books"), page=website, thumbnails=thumbnails) def render_hot_books(page): @@ -458,8 +462,9 @@ def render_hot_books(page): ub.delete_download(book.Downloads.book_id) numBooks = entries.__len__() pagination = Pagination(page, config.config_books_per_page, numBooks) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=_(u"Hot Books (Most Downloaded)"), page="hot") + title=_(u"Hot Books (Most Downloaded)"), page="hot", thumbnails=thumbnails) else: abort(404) @@ -490,12 +495,14 @@ def render_downloaded_books(page, order): .filter(db.Books.id == book.id).first(): ub.delete_download(book.id) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, title=_(u"Downloaded books by %(user)s",user=current_user.nickname), - page="download") + page="download", + thumbnails=thumbnails) else: abort(404) @@ -521,9 +528,10 @@ def render_author_books(page, author_id, order): author_info = services.goodreads_support.get_author_info(author_name) other_books = services.goodreads_support.get_other_books(author_info, entries) + thumbnails = get_thumbnails_for_books(entries) return render_title_template('author.html', entries=entries, pagination=pagination, id=author_id, title=_(u"Author: %(name)s", name=author_name), author=author_info, - other_books=other_books, page="author") + other_books=other_books, page="author", thumbnails=thumbnails) def render_publisher_books(page, book_id, order): @@ -535,8 +543,10 @@ def render_publisher_books(page, book_id, order): [db.Series.name, order[0], db.Books.series_index], db.books_series_link, db.Series) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, - title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher") + title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher", + thumbnails=thumbnails) else: abort(404) @@ -548,8 +558,10 @@ def render_series_books(page, book_id, order): db.Books, db.Books.series.any(db.Series.id == book_id), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"Series: %(serie)s", serie=name.name), page="series") + title=_(u"Series: %(serie)s", serie=name.name), page="series", + thumbnails=thumbnails) else: abort(404) @@ -561,8 +573,10 @@ def render_ratings_books(page, book_id, order): db.Books.ratings.any(db.Ratings.id == book_id), [order[0]]) if name and name.rating <= 10: + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings") + title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), page="ratings", + thumbnails=thumbnails) else: abort(404) @@ -574,8 +588,10 @@ def render_formats_books(page, book_id, order): db.Books, db.Books.data.any(db.Data.format == book_id.upper()), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, - title=_(u"File format: %(format)s", format=name.format), page="formats") + title=_(u"File format: %(format)s", format=name.format), page="formats", + thumbnails=thumbnails) else: abort(404) @@ -588,8 +604,10 @@ def render_category_books(page, book_id, order): db.Books.tags.any(db.Tags.id == book_id), [order[0], db.Series.name, db.Books.series_index], db.books_series_link, db.Series) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id, - title=_(u"Category: %(name)s", name=name.name), page="category") + title=_(u"Category: %(name)s", name=name.name), page="category", + thumbnails=thumbnails) else: abort(404) @@ -607,8 +625,9 @@ def render_language_books(page, name, order): db.Books, db.Books.languages.any(db.Languages.lang_code == name), [order[0]]) + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name, - title=_(u"Language: %(name)s", name=lang_name), page="language") + title=_(u"Language: %(name)s", name=lang_name), page="language", thumbnails=thumbnails) def render_read_books(page, are_read, as_xml=False, order=None): @@ -652,8 +671,10 @@ def render_read_books(page, are_read, as_xml=False, order=None): else: name = _(u'Unread Books') + ' (' + str(pagination.total_count) + ')' pagename = "unread" + + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename) + title=name, page=pagename, thumbnails=thumbnails) def render_archived_books(page, order): @@ -676,8 +697,9 @@ def render_archived_books(page, order): name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')' pagename = "archived" + thumbnails = get_thumbnails_for_books(entries + random) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename) + title=name, page=pagename, thumbnails=thumbnails) def render_prepare_search_form(cc): @@ -710,6 +732,7 @@ def render_prepare_search_form(cc): def render_search_results(term, offset=None, order=None, limit=None): entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit) + thumbnails = get_thumbnails_for_books(entries) return render_title_template('search.html', searchterm=term, pagination=pagination, @@ -718,7 +741,8 @@ def render_search_results(term, offset=None, order=None, limit=None): entries=entries, result_count=result_count, title=_(u"Search"), - page="search") + page="search", + thumbnails=thumbnails) # ################################### View Books list ################################################################## @@ -748,6 +772,7 @@ def books_table(): return render_title_template('book_table.html', title=_(u"Books List"), page="book_table", visiblility=visibility) + @web.route("/ajax/listbooks") @login_required def list_books(): @@ -780,6 +805,7 @@ def list_books(): response.headers["Content-Type"] = "application/json; charset=utf-8" return response + @web.route("/ajax/table_settings", methods=['POST']) @login_required def update_table_settings(): @@ -834,6 +860,7 @@ def publisher_list(): charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \ .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all() + return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist, title=_(u"Publishers"), page="publisherlist", data="publisher") else: @@ -865,8 +892,10 @@ def series_list(): .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \ .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all() + thumbnails = get_thumbnails_for_book_series(entries) return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=charlist, - title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view") + title=_(u"Series"), page="serieslist", data="series", bodyClass="grid-view", + thumbnails=thumbnails) else: abort(404) @@ -1150,13 +1179,16 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): else: offset = 0 limit_all = result_count + + thumbnails = get_thumbnails_for_books(entries) return render_title_template('search.html', adv_searchterm=searchterm, pagination=pagination, entries=q[offset:limit_all], result_count=result_count, - title=_(u"Advanced Search"), page="advsearch") - + title=_(u"Advanced Search"), + page="advsearch", + thumbnails=thumbnails) @web.route("/advsearch", methods=['GET']) @@ -1171,10 +1203,9 @@ def advanced_search_form(): @web.route("/cover/") -@web.route("/cover//") @login_required_if_no_ano -def get_cover(book_id, resolution=1): - return get_book_cover(book_id, resolution) +def get_cover(book_id): + return get_book_cover(book_id) @web.route("/cached-cover/") @@ -1183,6 +1214,12 @@ def get_cached_cover(cache_id): return get_cached_book_cover(cache_id) +@web.route("/cached-cover-thumbnail/") +@login_required_if_no_ano +def get_cached_cover_thumbnail(cache_id): + return get_cached_book_cover_thumbnail(cache_id) + + @web.route("/robots.txt") def get_robots(): return send_from_directory(constants.STATIC_DIR, "robots.txt") @@ -1591,6 +1628,7 @@ def show_book(book_id): if media_format.format.lower() in constants.EXTENSIONS_AUDIO: audioentries.append(media_format.format.lower()) + thumbnails = get_thumbnails_for_books([entries]) return render_title_template('detail.html', entry=entries, audioentries=audioentries, @@ -1602,7 +1640,8 @@ def show_book(book_id): is_archived=is_archived, kindle_list=kindle_list, reader_list=reader_list, - page="book") + page="book", + thumbnails=thumbnails) else: log.debug(u"Error opening eBook. File does not exist or file is not accessible") flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")