Simplified all of the thumbnail generation and loading.

cps/tasks/thumbnail.py

@@ -22,7 +22,7 @@ import os
 from cps import config, db, fs, gdriveutils, logger, ub
 from cps.services.worker import CalibreTask
 from datetime import datetime, timedelta
-from sqlalchemy import func
+from sqlalchemy import or_
 
 try:
     from urllib.request import urlopen
@@ -41,9 +41,8 @@ THUMBNAIL_RESOLUTION_3X = 3
 
 
 class TaskGenerateCoverThumbnails(CalibreTask):
-    def __init__(self, limit=100, task_message=u'Generating cover thumbnails'):
+    def __init__(self, task_message=u'Generating cover thumbnails'):
         super(TaskGenerateCoverThumbnails, self).__init__(task_message)
-        self.limit = limit
         self.log = logger.create()
         self.app_db_session = ub.get_new_session_instance()
         self.calibre_db = db.CalibreDB(expire_on_commit=False)
@@ -55,74 +54,51 @@ class TaskGenerateCoverThumbnails(CalibreTask):
 
     def run(self, worker_thread):
         if self.calibre_db.session and use_IM:
-            expired_thumbnails = self.get_expired_thumbnails()
-            thumbnail_book_ids = self.get_thumbnail_book_ids()
-            books_without_thumbnails = self.get_books_without_thumbnails(thumbnail_book_ids)
+            books_with_covers = self.get_books_with_covers()
+            count = len(books_with_covers)
 
-            count = len(books_without_thumbnails)
-            if count == 0:
-                # Do not display this task on the frontend if there are no covers to update
-                self.self_cleanup = True
+            updated = 0
+            generated = 0
+            for i, book in enumerate(books_with_covers):
+                book_cover_thumbnails = self.get_book_cover_thumbnails(book.id)
 
-            for i, book in enumerate(books_without_thumbnails):
-                for resolution in self.resolutions:
-                    expired_thumbnail = self.get_expired_thumbnail_for_book_and_resolution(
-                        book,
-                        resolution,
-                        expired_thumbnails
-                    )
-                    if expired_thumbnail:
-                        self.update_book_thumbnail(book, expired_thumbnail)
-                    else:
-                        self.create_book_thumbnail(book, resolution)
+                # Generate new thumbnails for missing covers
+                resolutions = list(map(lambda t: t.resolution, book_cover_thumbnails))
+                missing_resolutions = list(set(self.resolutions).difference(resolutions))
+                for resolution in missing_resolutions:
+                    generated += 1
+                    self.create_book_cover_thumbnail(book, resolution)
 
-                self.message = u'Generating cover thumbnail {0} of {1}'.format(i + 1, count)
+                # Replace outdated or missing thumbnails
+                for thumbnail in book_cover_thumbnails:
+                    if book.last_modified > thumbnail.generated_at:
+                        updated += 1
+                        self.update_book_cover_thumbnail(book, thumbnail)
+
+                    elif not self.cache.get_cache_file_exists(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS):
+                        updated += 1
+                        self.update_book_cover_thumbnail(book, thumbnail)
+
+                self.message = u'Processing book {0} of {1}'.format(i + 1, count)
                 self.progress = (1.0 / count) * i
 
         self._handleSuccess()
         self.app_db_session.remove()
 
-    def get_expired_thumbnails(self):
-        return self.app_db_session\
-            .query(ub.Thumbnail)\
-            .filter(ub.Thumbnail.expiration < datetime.utcnow())\
-            .all()
-
-    def get_thumbnail_book_ids(self):
-        return self.app_db_session\
-            .query(ub.Thumbnail.book_id)\
-            .group_by(ub.Thumbnail.book_id)\
-            .having(func.min(ub.Thumbnail.expiration) > datetime.utcnow())\
-            .distinct()
-
-    def get_books_without_thumbnails(self, thumbnail_book_ids):
+    def get_books_with_covers(self):
         return self.calibre_db.session\
             .query(db.Books)\
             .filter(db.Books.has_cover == 1)\
-            .filter(db.Books.id.notin_(thumbnail_book_ids))\
-            .limit(self.limit)\
             .all()
 
-    def get_expired_thumbnail_for_book_and_resolution(self, book, resolution, expired_thumbnails):
-        for thumbnail in expired_thumbnails:
-            if thumbnail.book_id == book.id and thumbnail.resolution == resolution:
-                return thumbnail
+    def get_book_cover_thumbnails(self, book_id):
+        return self.app_db_session\
+            .query(ub.Thumbnail)\
+            .filter(ub.Thumbnail.book_id == book_id)\
+            .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow()))\
+            .all()
 
-        return None
-
-    def update_book_thumbnail(self, book, thumbnail):
-        thumbnail.generated_at = datetime.utcnow()
-        thumbnail.expiration = datetime.utcnow() + timedelta(days=30)
-
-        try:
-            self.app_db_session.commit()
-            self.generate_book_thumbnail(book, thumbnail)
-        except Exception as ex:
-            self.log.info(u'Error updating book thumbnail: ' + str(ex))
-            self._handleError(u'Error updating book thumbnail: ' + str(ex))
-            self.app_db_session.rollback()
-
-    def create_book_thumbnail(self, book, resolution):
+    def create_book_cover_thumbnail(self, book, resolution):
         thumbnail = ub.Thumbnail()
         thumbnail.book_id = book.id
         thumbnail.format = 'jpeg'
@@ -137,6 +113,18 @@ class TaskGenerateCoverThumbnails(CalibreTask):
             self._handleError(u'Error creating book thumbnail: ' + str(ex))
             self.app_db_session.rollback()
 
+    def update_book_cover_thumbnail(self, book, thumbnail):
+        thumbnail.generated_at = datetime.utcnow()
+
+        try:
+            self.app_db_session.commit()
+            self.cache.delete_cache_file(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS)
+            self.generate_book_thumbnail(book, thumbnail)
+        except Exception as ex:
+            self.log.info(u'Error updating book thumbnail: ' + str(ex))
+            self._handleError(u'Error updating book thumbnail: ' + str(ex))
+            self.app_db_session.rollback()
+
     def generate_book_thumbnail(self, book, thumbnail):
         if book and thumbnail:
             if config.config_use_google_drive:
@@ -190,128 +178,6 @@ class TaskGenerateCoverThumbnails(CalibreTask):
         return "ThumbnailsGenerate"
 
 
-class TaskSyncCoverThumbnailCache(CalibreTask):
-    def __init__(self, task_message=u'Syncing cover thumbnail cache'):
-        super(TaskSyncCoverThumbnailCache, self).__init__(task_message)
-        self.log = logger.create()
-        self.app_db_session = ub.get_new_session_instance()
-        self.calibre_db = db.CalibreDB(expire_on_commit=False)
-        self.cache = fs.FileSystem()
-
-    def run(self, worker_thread):
-        cached_thumbnail_files = self.cache.list_cache_files(fs.CACHE_TYPE_THUMBNAILS)
-
-        # Expire thumbnails in the database if the cached file is missing
-        # This case will happen if a user deletes the cache dir or cached files
-        if self.app_db_session:
-            self.expire_missing_thumbnails(cached_thumbnail_files)
-            self.progress = 0.25
-
-        # Delete thumbnails in the database if the book has been removed
-        # This case will happen if a book is removed in Calibre and the metadata.db file is updated in the filesystem
-        if self.app_db_session and self.calibre_db:
-            book_ids = self.get_book_ids()
-            self.delete_thumbnails_for_missing_books(book_ids)
-            self.progress = 0.50
-
-        # Expire thumbnails in the database if their corresponding book has been updated since they were generated
-        # This case will happen if the book was updated externally
-        if self.app_db_session and self.cache:
-            books = self.get_books_updated_in_the_last_day()
-            book_ids = list(map(lambda b: b.id, books))
-            thumbnails = self.get_thumbnails_for_updated_books(book_ids)
-            self.expire_thumbnails_for_updated_book(books, thumbnails)
-            self.progress = 0.75
-
-        # Delete extraneous cached thumbnail files
-        # This case will happen if a book was deleted and the thumbnail OR the metadata.db file was changed externally
-        if self.app_db_session:
-            db_thumbnail_files = self.get_thumbnail_filenames()
-            self.delete_extraneous_thumbnail_files(cached_thumbnail_files, db_thumbnail_files)
-
-        self._handleSuccess()
-        self.app_db_session.remove()
-
-    def expire_missing_thumbnails(self, filenames):
-        try:
-            self.app_db_session\
-                .query(ub.Thumbnail)\
-                .filter(ub.Thumbnail.filename.notin_(filenames))\
-                .update({"expiration": datetime.utcnow()}, synchronize_session=False)
-            self.app_db_session.commit()
-        except Exception as ex:
-            self.log.info(u'Error expiring thumbnails for missing cache files: ' + str(ex))
-            self._handleError(u'Error expiring thumbnails for missing cache files: ' + str(ex))
-            self.app_db_session.rollback()
-
-    def get_book_ids(self):
-        results = self.calibre_db.session\
-            .query(db.Books.id)\
-            .filter(db.Books.has_cover == 1)\
-            .distinct()
-
-        return [value for value, in results]
-
-    def delete_thumbnails_for_missing_books(self, book_ids):
-        try:
-            self.app_db_session\
-                .query(ub.Thumbnail)\
-                .filter(ub.Thumbnail.book_id.notin_(book_ids))\
-                .delete(synchronize_session=False)
-            self.app_db_session.commit()
-        except Exception as ex:
-            self.log.info(str(ex))
-            self._handleError(u'Error deleting thumbnails for missing books: ' + str(ex))
-            self.app_db_session.rollback()
-
-    def get_thumbnail_filenames(self):
-        results = self.app_db_session\
-            .query(ub.Thumbnail.filename)\
-            .all()
-
-        return [thumbnail for thumbnail, in results]
-
-    def delete_extraneous_thumbnail_files(self, cached_thumbnail_files, db_thumbnail_files):
-        extraneous_files = list(set(cached_thumbnail_files).difference(db_thumbnail_files))
-        for file in extraneous_files:
-            self.cache.delete_cache_file(file, fs.CACHE_TYPE_THUMBNAILS)
-
-    def get_books_updated_in_the_last_day(self):
-        return self.calibre_db.session\
-            .query(db.Books)\
-            .filter(db.Books.has_cover == 1)\
-            .filter(db.Books.last_modified > datetime.utcnow() - timedelta(days=1, hours=1))\
-            .all()
-
-    def get_thumbnails_for_updated_books(self, book_ids):
-        return self.app_db_session\
-            .query(ub.Thumbnail)\
-            .filter(ub.Thumbnail.book_id.in_(book_ids))\
-            .all()
-
-    def expire_thumbnails_for_updated_book(self, books, thumbnails):
-        thumbnail_ids = list()
-        for book in books:
-            for thumbnail in thumbnails:
-                if thumbnail.book_id == book.id and thumbnail.generated_at < book.last_modified:
-                    thumbnail_ids.append(thumbnail.id)
-
-        try:
-            self.app_db_session\
-                .query(ub.Thumbnail)\
-                .filter(ub.Thumbnail.id.in_(thumbnail_ids)) \
-                .update({"expiration": datetime.utcnow()}, synchronize_session=False)
-            self.app_db_session.commit()
-        except Exception as ex:
-            self.log.info(u'Error expiring thumbnails for updated books: ' + str(ex))
-            self._handleError(u'Error expiring thumbnails for updated books: ' + str(ex))
-            self.app_db_session.rollback()
-
-    @property
-    def name(self):
-        return "ThumbnailsSync"
-
-
 class TaskClearCoverThumbnailCache(CalibreTask):
     def __init__(self, book_id=None, task_message=u'Clearing cover thumbnail cache'):
         super(TaskClearCoverThumbnailCache, self).__init__(task_message)
@@ -325,9 +191,9 @@ class TaskClearCoverThumbnailCache(CalibreTask):
             if self.book_id:
                 thumbnails = self.get_thumbnails_for_book(self.book_id)
                 for thumbnail in thumbnails:
-                    self.expire_and_delete_thumbnail(thumbnail)
+                    self.delete_thumbnail(thumbnail)
             else:
-                self.expire_and_delete_all_thumbnails()
+                self.delete_all_thumbnails()
 
         self._handleSuccess()
         self.app_db_session.remove()
@@ -338,29 +204,19 @@ class TaskClearCoverThumbnailCache(CalibreTask):
             .filter(ub.Thumbnail.book_id == book_id)\
             .all()
 
-    def expire_and_delete_thumbnail(self, thumbnail):
-        thumbnail.expiration = datetime.utcnow()
-
+    def delete_thumbnail(self, thumbnail):
         try:
-            self.app_db_session.commit()
             self.cache.delete_cache_file(thumbnail.filename, fs.CACHE_TYPE_THUMBNAILS)
         except Exception as ex:
-            self.log.info(u'Error expiring book thumbnail: ' + str(ex))
-            self._handleError(u'Error expiring book thumbnail: ' + str(ex))
-            self.app_db_session.rollback()
-
-    def expire_and_delete_all_thumbnails(self):
-        self.app_db_session\
-            .query(ub.Thumbnail)\
-            .update({'expiration': datetime.utcnow()})
+            self.log.info(u'Error deleting book thumbnail: ' + str(ex))
+            self._handleError(u'Error deleting book thumbnail: ' + str(ex))
 
+    def delete_all_thumbnails(self):
         try:
-            self.app_db_session.commit()
             self.cache.delete_cache_dir(fs.CACHE_TYPE_THUMBNAILS)
         except Exception as ex:
-            self.log.info(u'Error expiring book thumbnails: ' + str(ex))
-            self._handleError(u'Error expiring book thumbnails: ' + str(ex))
-            self.app_db_session.rollback()
+            self.log.info(u'Error deleting book thumbnails: ' + str(ex))
+            self._handleError(u'Error deleting book thumbnails: ' + str(ex))
 
     @property
     def name(self):