Merge remote-tracking branch 'origin/cover_thumbnail' into cover_thumbnail

cps.py

@@ -77,7 +77,7 @@ def main():
         app.register_blueprint(oauth)
 
     # Register scheduled tasks
-    register_scheduled_tasks()
+    register_scheduled_tasks()  # ToDo only reconnect if reconnect is enabled
     register_startup_tasks()
 
     success = web_server.start()

cps/__init__.py

@@ -179,12 +179,6 @@ def get_locale():
     return negotiate_locale(preferred or ['en'], _BABEL_TRANSLATIONS)
 
 
-@babel.timezoneselector
-def get_timezone():
-    user = getattr(g, 'user', None)
-    return user.timezone if user else None
-
-
 from .updater import Updater
 updater_thread = Updater()
 

cps/about.py

@@ -69,9 +69,9 @@ _VERSIONS.update(uploader.get_versions(False))
 
 
 def collect_stats():
-    _VERSIONS['ebook converter'] = _(converter.get_calibre_version())
-    _VERSIONS['unrar'] = _(converter.get_unrar_version())
-    _VERSIONS['kepubify'] = _(converter.get_kepubify_version())
+    _VERSIONS['ebook converter'] = converter.get_calibre_version()
+    _VERSIONS['unrar'] = converter.get_unrar_version()
+    _VERSIONS['kepubify'] = converter.get_kepubify_version()
     return _VERSIONS
 
 

cps/admin.py

@@ -24,13 +24,12 @@ import os
 import re
 import base64
 import json
-import time
 import operator
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, time
 from functools import wraps
 
 from babel import Locale
-from babel.dates import format_datetime
+from babel.dates import format_datetime, format_time, format_timedelta
 from flask import Blueprint, flash, redirect, url_for, abort, request, make_response, send_from_directory, g, Response
 from flask_login import login_required, current_user, logout_user, confirm_login
 from flask_babel import gettext as _
@@ -44,7 +43,7 @@ from . import constants, logger, helper, services, cli
 from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils, \
     kobo_sync_status, schedule
 from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \
-    valid_email, check_username
+    valid_email, check_username, update_thumbnail_cache
 from .gdriveutils import is_gdrive_ready, gdrive_support
 from .render_template import render_title_template, get_sidebar_config
 from .services.worker import WorkerThread
@@ -58,7 +57,8 @@ feature_support = {
         'goodreads': bool(services.goodreads_support),
         'kobo':  bool(services.kobo),
         'updater': constants.UPDATER_AVAILABLE,
-        'gmail': bool(services.gmail)
+        'gmail': bool(services.gmail),
+        'scheduler': schedule.use_APScheduler
     }
 
 try:
@@ -169,10 +169,22 @@ def reconnect():
         abort(404)
 
 
+@admi.route("/ajax/updateThumbnails", methods=['POST'])
+@admin_required
+@login_required
+def update_thumbnails():
+    content = config.get_scheduled_task_settings()
+    if content['schedule_generate_book_covers']:
+        log.info("Update of Cover cache requested")
+        update_thumbnail_cache()
+    return ""
+
+
 @admi.route("/admin/view")
 @login_required
 @admin_required
 def admin():
+    locale = get_locale()
     version = updater_thread.get_current_version_info()
     if version is False:
         commit = _(u'Unknown')
@@ -187,15 +199,19 @@ def admin():
                     form_date -= timedelta(hours=int(commit[20:22]), minutes=int(commit[23:]))
                 elif commit[19] == '-':
                     form_date += timedelta(hours=int(commit[20:22]), minutes=int(commit[23:]))
-            commit = format_datetime(form_date - tz, format='short', locale=get_locale())
+            commit = format_datetime(form_date - tz, format='short', locale=locale)
         else:
             commit = version['version']
 
     all_user = ub.session.query(ub.User).all()
     email_settings = config.get_mail_settings()
-    kobo_support = feature_support['kobo'] and config.config_kobo_sync
+    schedule_time = format_time(time(hour=config.schedule_start_time), format="short", locale=locale)
+    t = timedelta(hours=config.schedule_duration // 60, minutes=config.schedule_duration % 60)
+    schedule_duration = format_timedelta(t, format="short", threshold=.99, locale=locale)
+
     return render_title_template("admin.html", allUser=all_user, email=email_settings, config=config, commit=commit,
-                                 feature_support=feature_support, kobo_support=kobo_support,
+                                 feature_support=feature_support, schedule_time=schedule_time,
+                                 schedule_duration=schedule_duration,
                                  title=_(u"Admin page"), page="admin")
 
 
@@ -612,6 +628,8 @@ def load_dialogtexts(element_id):
         texts["main"] = _('Are you sure you want to change shelf sync behavior for the selected user(s)?')
     elif element_id == "db_submit":
         texts["main"] = _('Are you sure you want to change Calibre library location?')
+    elif element_id == "admin_refresh_cover_cache":
+        texts["main"] = _('Calibre-Web will search for updated Covers and update Cover Thumbnails, this may take a while?')
     elif element_id == "btnfullsync":
         texts["main"] = _("Are you sure you want delete Calibre-Web's sync database "
                           "to force a full sync with your Kobo Reader?")
@@ -1647,36 +1665,57 @@ def update_mailsettings():
 @admin_required
 def edit_scheduledtasks():
     content = config.get_scheduled_task_settings()
-    return render_title_template("schedule_edit.html", config=content, title=_(u"Edit Scheduled Tasks Settings"))
+    time_field = list()
+    duration_field = list()
+
+    locale = get_locale()
+    for n in range(24):
+        time_field.append((n , format_time(time(hour=n), format="short", locale=locale)))
+    for n in range(5, 65, 5):
+        t = timedelta(hours=n // 60, minutes=n % 60)
+        duration_field.append((n, format_timedelta(t, format="short", threshold=.99, locale=locale)))
+
+    return render_title_template("schedule_edit.html", config=content, starttime=time_field, duration=duration_field, title=_(u"Edit Scheduled Tasks Settings"))
 
 
 @admi.route("/admin/scheduledtasks", methods=["POST"])
 @login_required
 @admin_required
 def update_scheduledtasks():
+    error = False
     to_save = request.form.to_dict()
-    _config_int(to_save, "schedule_start_time")
-    _config_int(to_save, "schedule_end_time")
+    if 0 <= int(to_save.get("schedule_start_time")) <= 23:
+        _config_int(to_save, "schedule_start_time")
+    else:
+        flash(_(u"Invalid start time for task specified"), category="error")
+        error = True
+    if 0 < int(to_save.get("schedule_duration")) <= 60:
+        _config_int(to_save, "schedule_duration")
+    else:
+        flash(_(u"Invalid duration for task specified"), category="error")
+        error = True
     _config_checkbox(to_save, "schedule_generate_book_covers")
     _config_checkbox(to_save, "schedule_generate_series_covers")
+    _config_checkbox(to_save, "schedule_reconnect")
 
-    try:
-        config.save()
-        flash(_(u"Scheduled tasks settings updated"), category="success")
+    if not error:
+        try:
+            config.save()
+            flash(_(u"Scheduled tasks settings updated"), category="success")
 
-        # Cancel any running tasks
-        schedule.end_scheduled_tasks()
+            # Cancel any running tasks
+            schedule.end_scheduled_tasks()
 
-        # Re-register tasks with new settings
-        schedule.register_scheduled_tasks()
-    except IntegrityError as ex:
-        ub.session.rollback()
-        log.error("An unknown error occurred while saving scheduled tasks settings")
-        flash(_(u"An unknown error occurred. Please try again later."), category="error")
-    except OperationalError:
-        ub.session.rollback()
-        log.error("Settings DB is not Writeable")
-        flash(_("Settings DB is not Writeable"), category="error")
+            # Re-register tasks with new settings
+            schedule.register_scheduled_tasks(config.schedule_reconnect)
+        except IntegrityError:
+            ub.session.rollback()
+            log.error("An unknown error occurred while saving scheduled tasks settings")
+            flash(_(u"An unknown error occurred. Please try again later."), category="error")
+        except OperationalError:
+            ub.session.rollback()
+            log.error("Settings DB is not Writeable")
+            flash(_("Settings DB is not Writeable"), category="error")
 
     return edit_scheduledtasks()
 

cps/comic.py

@@ -130,7 +130,9 @@ def get_comic_info(tmp_file_path, original_file_name, original_file_extension, r
                 series=loaded_metadata.series or "",
                 series_id=loaded_metadata.issue or "",
                 languages=loaded_metadata.language,
-                publisher="")
+                publisher="",
+                pubdate="",
+                identifiers=[])
 
     return BookMeta(
         file_path=tmp_file_path,
@@ -143,4 +145,6 @@ def get_comic_info(tmp_file_path, original_file_name, original_file_extension, r
         series="",
         series_id="",
         languages="",
-        publisher="")
+        publisher="",
+        pubdate="",
+        identifiers=[])

cps/config_sql.py

@@ -142,9 +142,10 @@ class _Settings(_Base):
     config_allow_reverse_proxy_header_login = Column(Boolean, default=False)
 
     schedule_start_time = Column(Integer, default=4)
-    schedule_end_time = Column(Integer, default=6)
+    schedule_duration = Column(Integer, default=10)
     schedule_generate_book_covers = Column(Boolean, default=False)
     schedule_generate_series_covers = Column(Boolean, default=False)
+    schedule_reconnect = Column(Boolean, default=False)
 
     def __repr__(self):
         return self.__class__.__name__

cps/constants.py

@@ -161,7 +161,7 @@ def selected_roles(dictionary):
 
 # :rtype: BookMeta
 BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, '
-                                  'series_id, languages, publisher')
+                                  'series_id, languages, publisher, pubdate, identifiers')
 
 STABLE_VERSION = {'version': '0.6.19 Beta'}
 

cps/converter.py

@@ -18,7 +18,8 @@
 
 import os
 import re
-from flask_babel import gettext as _
+
+from flask_babel import lazy_gettext as N_
 
 from . import config, logger
 from .subproc_wrapper import process_wait
@@ -26,9 +27,9 @@ from .subproc_wrapper import process_wait
 
 log = logger.create()
 
-# _() necessary to make babel aware of string for translation
-_NOT_INSTALLED = _('not installed')
-_EXECUTION_ERROR = _('Execution permissions missing')
+# strings getting translated when used
+_NOT_INSTALLED = N_('not installed')
+_EXECUTION_ERROR = N_('Execution permissions missing')
 
 
 def _get_command_version(path, pattern, argument=None):

cps/db.py

@@ -25,6 +25,7 @@ from datetime import datetime
 from urllib.parse import quote
 import unidecode
 
+from sqlite3 import OperationalError as sqliteOperationalError
 from sqlalchemy import create_engine
 from sqlalchemy import Table, Column, ForeignKey, CheckConstraint
 from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float
@@ -903,9 +904,20 @@ class CalibreDB:
                     .join(books_languages_link).join(Books)\
                     .filter(self.common_filters(return_all_languages=return_all_languages)) \
                     .group_by(text('books_languages_link.lang_code')).all()
+            tags = list()
             for lang in languages:
-                lang[0].name = isoLanguages.get_language_name(get_locale(), lang[0].lang_code)
-            return sorted(languages, key=lambda x: x[0].name, reverse=reverse_order)
+                tag = Category(isoLanguages.get_language_name(get_locale(), lang[0].lang_code), lang[0].lang_code)
+                tags.append([tag, lang[1]])
+            # Append all books without language to list
+            if not return_all_languages:
+                no_lang_count = (self.session.query(Books)
+                                 .outerjoin(books_languages_link).outerjoin(Languages)
+                                 .filter(Languages.lang_code == None)
+                                 .filter(self.common_filters())
+                                 .count())
+                if no_lang_count:
+                    tags.append([Category(_("None"), "none"), no_lang_count])
+            return sorted(tags, key=lambda x: x[0].name, reverse=reverse_order)
         else:
             if not languages:
                 languages = self.session.query(Languages) \
@@ -929,7 +941,10 @@ class CalibreDB:
             return title.strip()
 
         conn = conn or self.session.connection().connection.connection
-        conn.create_function("title_sort", 1, _title_sort)
+        try:
+            conn.create_function("title_sort", 1, _title_sort)
+        except sqliteOperationalError:
+            pass
 
     @classmethod
     def dispose(cls):
@@ -977,3 +992,22 @@ def lcase(s):
         _log = logger.create()
         _log.error_or_exception(ex)
         return s.lower()
+
+
+class Category:
+    name = None
+    id = None
+    count = None
+    rating = None
+
+    def __init__(self, name, cat_id, rating=None):
+        self.name = name
+        self.id = cat_id
+        self.rating = rating
+        self.count = 1
+
+'''class Count:
+    count = None
+
+    def __init__(self, count):
+        self.count = count'''

cps/editbooks.py

@@ -25,7 +25,7 @@ from datetime import datetime
 import json
 from shutil import copyfile
 from uuid import uuid4
-from markupsafe import escape
+from markupsafe import escape  # dependency of flask
 from functools import wraps
 
 try:
@@ -35,9 +35,10 @@ except ImportError:
 
 from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
 from flask_babel import gettext as _
+from flask_babel import lazy_gettext as N_
 from flask_login import current_user, login_required
 from sqlalchemy.exc import OperationalError, IntegrityError
-from sqlite3 import OperationalError as sqliteOperationalError
+# from sqlite3 import OperationalError as sqliteOperationalError
 from . import constants, logger, isoLanguages, gdriveutils, uploader, helper, kobo_sync_status
 from . import config, get_locale, ub, db
 from . import calibre_db
@@ -241,7 +242,7 @@ def delete_book_ajax(book_id, book_format):
 
 
 def delete_whole_book(book_id, book):
-    # delete book from Shelfs, Downloads, Read list
+    # delete book from shelves, Downloads, Read list
     ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete()
     ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id).delete()
     ub.delete_download(book_id)
@@ -383,7 +384,7 @@ def render_edit_book(book_id):
     for authr in book.authors:
         author_names.append(authr.name.replace('|', ','))
 
-    # Option for showing convertbook button
+    # Option for showing convert_book button
     valid_source_formats = list()
     allowed_conversion_formats = list()
     kepub_possible = None
@@ -413,11 +414,11 @@ def render_edit_book(book_id):
 
 def edit_book_ratings(to_save, book):
     changed = False
-    if to_save.get("rating","").strip():
+    if to_save.get("rating", "").strip():
         old_rating = False
         if len(book.ratings) > 0:
             old_rating = book.ratings[0].rating
-        rating_x2 = int(float(to_save.get("rating","")) * 2)
+        rating_x2 = int(float(to_save.get("rating", "")) * 2)
         if rating_x2 != old_rating:
             changed = True
             is_rating = calibre_db.session.query(db.Ratings).filter(db.Ratings.rating == rating_x2).first()
@@ -622,8 +623,9 @@ def edit_cc_data(book_id, book, to_save, cc):
                                               'custom')
     return changed
 
+
 # returns None if no file is uploaded
-# returns False if an error occours, in all other cases the ebook metadata is returned
+# returns False if an error occurs, in all other cases the ebook metadata is returned
 def upload_single_file(file_request, book, book_id):
     # Check and handle Uploaded file
     requested_file = file_request.files.get('btn-upload-format', None)
@@ -676,11 +678,11 @@ def upload_single_file(file_request, book, book_id):
                     calibre_db.session.rollback()
                     log.error_or_exception("Database error: {}".format(e))
                     flash(_(u"Database error: %(error)s.", error=e.orig), category="error")
-                    return False # return redirect(url_for('web.show_book', book_id=book.id))
+                    return False  # return redirect(url_for('web.show_book', book_id=book.id))
 
             # Queue uploader info
             link = '<a href="{}">{}</a>'.format(url_for('web.show_book', book_id=book.id), escape(book.title))
-            upload_text = _(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=link)
+            upload_text = N_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=link)
             WorkerThread.add(current_user.name, TaskUpload(upload_text, escape(book.title)))
 
             return uploader.process(
@@ -688,6 +690,7 @@ def upload_single_file(file_request, book, book_id):
                 rarExecutable=config.config_rarfile_location)
     return None
 
+
 def upload_cover(cover_request, book):
     requested_file = cover_request.files.get('btn-upload-cover', None)
     if requested_file:
@@ -698,7 +701,7 @@ def upload_cover(cover_request, book):
                 return False
             ret, message = helper.save_cover(requested_file, book.path)
             if ret is True:
-                helper.clear_cover_thumbnail_cache(book.id)
+                helper.replace_cover_thumbnail_cache(book.id)
                 return True
             else:
                 flash(message, category="error")
@@ -739,6 +742,7 @@ def handle_author_on_edit(book, author_name, update_stored=True):
         change = True
     return input_authors, change, renamed
 
+
 @EditBook.route("/admin/book/<int:book_id>", methods=['GET'])
 @login_required_if_no_ano
 @edit_required
@@ -754,11 +758,11 @@ def edit_book(book_id):
     edit_error = False
 
     # create the function for sorting...
-    try:
-        calibre_db.update_title_sort(config)
-    except sqliteOperationalError as e:
-        log.error_or_exception(e)
-        calibre_db.session.rollback()
+    #try:
+    calibre_db.update_title_sort(config)
+    #except sqliteOperationalError as e:
+    #    log.error_or_exception(e)
+    #    calibre_db.session.rollback()
 
     book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
     # Book not found
@@ -815,6 +819,7 @@ def edit_book(book_id):
                 if result is True:
                     book.has_cover = 1
                     modify_date = True
+                    helper.replace_cover_thumbnail_cache(book.id)
                 else:
                     flash(error, category="error")
 
@@ -984,8 +989,13 @@ def create_book_on_upload(modify_date, meta):
     # combine path and normalize path from Windows systems
     path = os.path.join(author_dir, title_dir).replace('\\', '/')
 
+    try:
+        pubdate = datetime.strptime(meta.pubdate[:10], "%Y-%m-%d")
+    except ValueError:
+        pubdate = datetime(101, 1, 1)
+
     # Calibre adds books with utc as timezone
-    db_book = db.Books(title, "", sort_authors, datetime.utcnow(), datetime(101, 1, 1),
+    db_book = db.Books(title, "", sort_authors, datetime.utcnow(), pubdate,
                        '1', datetime.utcnow(), path, meta.cover, db_author, [], "")
 
     modify_date |= modify_database_object(input_authors, db_book.authors, db.Authors, calibre_db.session,
@@ -1018,6 +1028,16 @@ def create_book_on_upload(modify_date, meta):
 
     # flush content, get db_book.id available
     calibre_db.session.flush()
+
+    # Handle identifiers now that db_book.id is available
+    identifier_list = []
+    for type_key, type_value in meta.identifiers:
+        identifier_list.append(db.Identifiers(type_value, type_key, db_book.id))
+    modification, warning = modify_identifiers(identifier_list, db_book.identifiers, calibre_db.session)
+    if warning:
+        flash(_("Identifiers are not Case Sensitive, Overwriting Old Identifier"), category="warning")
+    modify_date |= modification
+
     return db_book, input_authors, title_dir, renamed_authors
 
 
@@ -1048,18 +1068,18 @@ def file_handling_on_upload(requested_file):
 def move_coverfile(meta, db_book):
     # move cover to final directory, including book id
     if meta.cover:
-        coverfile = meta.cover
+        cover_file = meta.cover
     else:
-        coverfile = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg')
-    new_coverpath = os.path.join(config.config_calibre_dir, db_book.path)
+        cover_file = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg')
+    new_cover_path = os.path.join(config.config_calibre_dir, db_book.path)
     try:
-        os.makedirs(new_coverpath, exist_ok=True)
-        copyfile(coverfile, os.path.join(new_coverpath, "cover.jpg"))
+        os.makedirs(new_cover_path, exist_ok=True)
+        copyfile(cover_file, os.path.join(new_cover_path, "cover.jpg"))
         if meta.cover:
             os.unlink(meta.cover)
     except OSError as e:
-        log.error("Failed to move cover file %s: %s", new_coverpath, e)
-        flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_coverpath,
+        log.error("Failed to move cover file %s: %s", new_cover_path, e)
+        flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_cover_path,
                 error=e),
               category="error")
 
@@ -1115,8 +1135,9 @@ def upload():
                 if error:
                     flash(error, category="error")
                 link = '<a href="{}">{}</a>'.format(url_for('web.show_book', book_id=book_id), escape(title))
-                upload_text = _(u"File %(file)s uploaded", file=link)
+                upload_text = N_(u"File %(file)s uploaded", file=link)
                 WorkerThread.add(current_user.name, TaskUpload(upload_text, escape(title)))
+                helper.add_book_to_thumbnail_cache(book_id)
 
                 if len(request.files.getlist("btn-upload")) < 2:
                     if current_user.role_edit() or current_user.role_admin():
@@ -1177,7 +1198,7 @@ def edit_list_book(param):
     vals = request.form.to_dict()
     book = calibre_db.get_book(vals['pk'])
     sort_param = ""
-    # ret = ""
+    ret = ""
     try:
         if param == 'series_index':
             edit_book_series_index(vals['value'], book)

cps/epub.py

@@ -63,13 +63,15 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
 
     epub_metadata = {}
 
-    for s in ['title', 'description', 'creator', 'language', 'subject']:
+    for s in ['title', 'description', 'creator', 'language', 'subject', 'publisher', 'date']:
         tmp = p.xpath('dc:%s/text()' % s, namespaces=ns)
         if len(tmp) > 0:
             if s == 'creator':
                 epub_metadata[s] = ' & '.join(split_authors(tmp))
             elif s == 'subject':
                 epub_metadata[s] = ', '.join(tmp)
+            elif s == 'date':
+                epub_metadata[s] = tmp[0][:10]
             else:
                 epub_metadata[s] = tmp[0]
         else:
@@ -78,6 +80,12 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
     if epub_metadata['subject'] == 'Unknown':
         epub_metadata['subject'] = ''
 
+    if epub_metadata['publisher'] == u'Unknown':
+        epub_metadata['publisher'] = ''
+
+    if epub_metadata['date'] == u'Unknown':
+        epub_metadata['date'] = ''
+
     if epub_metadata['description'] == u'Unknown':
         description = tree.xpath("//*[local-name() = 'description']/text()")
         if len(description) > 0:
@@ -92,6 +100,14 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
 
     cover_file = parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path)
 
+    identifiers = []
+    for node in p.xpath('dc:identifier', namespaces=ns):
+        identifier_name=node.attrib.values()[-1];
+        identifier_value=node.text;
+        if identifier_name in ('uuid','calibre'):
+            continue;
+        identifiers.append( [identifier_name, identifier_value] )
+
     if not epub_metadata['title']:
         title = original_file_name
     else:
@@ -108,7 +124,9 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
         series=epub_metadata['series'].encode('utf-8').decode('utf-8'),
         series_id=epub_metadata['series_id'].encode('utf-8').decode('utf-8'),
         languages=epub_metadata['language'],
-        publisher="")
+        publisher=epub_metadata['publisher'].encode('utf-8').decode('utf-8'),
+        pubdate=epub_metadata['date'],
+        identifiers=identifiers)
 
 
 def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path):

cps/fb2.py

@@ -77,4 +77,6 @@ def get_fb2_info(tmp_file_path, original_file_extension):
         series="",
         series_id="",
         languages="",
-        publisher="")
+        publisher="",
+        pubdate="",
+        identifiers=[])

cps/helper.py

@@ -33,6 +33,7 @@ from babel.dates import format_datetime
 from babel.units import format_unit
 from flask import send_from_directory, make_response, redirect, abort, url_for
 from flask_babel import gettext as _
+from flask_babel import lazy_gettext as N_
 from flask_login import current_user
 from sqlalchemy.sql.expression import true, false, and_, or_, text, func
 from sqlalchemy.exc import InvalidRequestError, OperationalError
@@ -53,14 +54,14 @@ except ImportError:
 
 from . import calibre_db, cli
 from .tasks.convert import TaskConvert
-from . import logger, config, get_locale, db, ub, kobo_sync_status, fs
+from . import logger, config, get_locale, db, ub, fs
 from . import gdriveutils as gd
 from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES
 from .subproc_wrapper import process_wait
 from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS, STAT_ENDED, \
     STAT_CANCELLED
 from .tasks.mail import TaskEmail
-from .tasks.thumbnail import TaskClearCoverThumbnailCache
+from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails
 
 log = logger.create()
 
@@ -111,9 +112,10 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
     return None
 
 
+# Texts are not lazy translated as they are supposed to get send out as is
 def send_test_mail(kindle_mail, user_name):
     WorkerThread.add(user_name, TaskEmail(_(u'Calibre-Web test e-mail'), None, None,
-                     config.get_mail_settings(), kindle_mail, _(u"Test e-mail"),
+                     config.get_mail_settings(), kindle_mail, N_(u"Test e-mail"),
                                           _(u'This e-mail has been sent via Calibre-Web.')))
     return
 
@@ -135,7 +137,7 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False):
         attachment=None,
         settings=config.get_mail_settings(),
         recipient=e_mail,
-        taskMessage=_(u"Registration e-mail for user: %(name)s", name=user_name),
+        task_message=N_(u"Registration e-mail for user: %(name)s", name=user_name),
         text=txt
     ))
     return
@@ -219,7 +221,7 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
         if entry.format.upper() == book_format.upper():
             converted_file_name = entry.name + '.' + book_format.lower()
             link = '<a href="{}">{}</a>'.format(url_for('web.show_book', book_id=book_id), escape(book.title))
-            email_text = _(u"%(book)s send to Kindle", book=link)
+            email_text = N_(u"%(book)s send to Kindle", book=link)
             WorkerThread.add(user_id, TaskEmail(_(u"Send to Kindle"), book.path, converted_file_name,
                              config.get_mail_settings(), kindle_mail,
                              email_text, _(u'This e-mail has been sent via Calibre-Web.')))
@@ -715,9 +717,10 @@ def get_book_cover(book_id, resolution=None):
     return get_book_cover_internal(book, use_generic_cover_on_failure=True, resolution=resolution)
 
 
-def get_book_cover_with_uuid(book_uuid, use_generic_cover_on_failure=True):
+# Called only by kobo sync -> cover not found should be answered with 404 and not with default cover
+def get_book_cover_with_uuid(book_uuid, resolution=None):
     book = calibre_db.get_book_by_uuid(book_uuid)
-    return get_book_cover_internal(book, use_generic_cover_on_failure)
+    return get_book_cover_internal(book, use_generic_cover_on_failure=False, resolution=resolution)
 
 
 def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None):
@@ -819,9 +822,6 @@ def save_cover_from_url(url, book_path):
             log.error("python modul advocate is not installed but is needed")
             return False, _("Python modul 'advocate' is not installed but is needed for cover downloads")
         img.raise_for_status()
-        #            # cover_processing()
-            # move_coverfile(meta, db_book)
-
         return save_cover(img, book_path)
     except (socket.gaierror,
             requests.exceptions.HTTPError,
@@ -990,7 +990,7 @@ def format_runtime(runtime):
 # helper function to apply localize status information in tasklist entries
 def render_task_status(tasklist):
     renderedtasklist = list()
-    for __, user, __, task in tasklist:
+    for __, user, __, task, __ in tasklist:
         if user == current_user.name or current_user.role_admin():
             ret = {}
             if task.start_time:
@@ -1014,12 +1014,12 @@ def render_task_status(tasklist):
                 else:
                     ret['status'] = _(u'Unknown Status')
 
-            ret['taskMessage'] = "{}: {}".format(_(task.name), task.message) if task.message else _(task.name)
+            ret['taskMessage'] = "{}: {}".format(task.name, task.message) if task.message else task.name
             ret['progress'] = "{} %".format(int(task.progress * 100))
             ret['user'] = escape(user)  # prevent xss
 
             # Hidden fields
-            ret['id'] = task.id
+            ret['task_id'] = task.id
             ret['stat'] = task.stat
             ret['is_cancellable'] = task.is_cancellable
 
@@ -1077,7 +1077,21 @@ def get_download_link(book_id, book_format, client):
 
 
 def clear_cover_thumbnail_cache(book_id):
-    WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id))
+    WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True)
+
+
+def replace_cover_thumbnail_cache(book_id):
+    WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True)
+    WorkerThread.add(None, TaskGenerateCoverThumbnails(book_id), hidden=True)
+
 
 def delete_thumbnail_cache():
     WorkerThread.add(None, TaskClearCoverThumbnailCache(-1))
+
+
+def add_book_to_thumbnail_cache(book_id):
+    WorkerThread.add(None, TaskGenerateCoverThumbnails(book_id), hidden=True)
+
+
+def update_thumbnail_cache():
+    WorkerThread.add(None, TaskGenerateCoverThumbnails())

cps/kobo.py

@@ -45,7 +45,7 @@ import requests
 
 
 from . import config, logger, kobo_auth, db, calibre_db, helper, shelf as shelf_lib, ub, csrf, kobo_sync_status
-from .constants import sqlalchemy_version2
+from .constants import sqlalchemy_version2, COVER_THUMBNAIL_SMALL
 from .helper import get_download_link
 from .services import SyncToken as SyncToken
 from .web import download_required
@@ -148,8 +148,8 @@ def HandleSyncRequest():
         sync_token.books_last_created = datetime.datetime.min
         sync_token.reading_state_last_modified = datetime.datetime.min
 
-    new_books_last_modified = sync_token.books_last_modified # needed for sync selected shelfs only
-    new_books_last_created = sync_token.books_last_created # needed to distinguish between new and changed entitlement
+    new_books_last_modified = sync_token.books_last_modified  # needed for sync selected shelfs only
+    new_books_last_created = sync_token.books_last_created  # needed to distinguish between new and changed entitlement
     new_reading_state_last_modified = sync_token.reading_state_last_modified
 
     new_archived_last_modified = datetime.datetime.min
@@ -176,18 +176,17 @@ def HandleSyncRequest():
                            .join(db.Data).outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id,
                                                                           ub.ArchivedBook.user_id == current_user.id))
                            .filter(db.Books.id.notin_(calibre_db.session.query(ub.KoboSyncedBooks.book_id)
-                                           .filter(ub.KoboSyncedBooks.user_id == current_user.id)))
-                .filter(ub.BookShelf.date_added > sync_token.books_last_modified)
-                .filter(db.Data.format.in_(KOBO_FORMATS))
-                .filter(calibre_db.common_filters(allow_show_archived=True))
-                .order_by(db.Books.id)
-                .order_by(ub.ArchivedBook.last_modified)
-                .join(ub.BookShelf, db.Books.id == ub.BookShelf.book_id)
-                .join(ub.Shelf)
-                .filter(ub.Shelf.user_id == current_user.id)
-                .filter(ub.Shelf.kobo_sync)
-                .distinct()
-        )
+                                                      .filter(ub.KoboSyncedBooks.user_id == current_user.id)))
+                           .filter(ub.BookShelf.date_added > sync_token.books_last_modified)
+                           .filter(db.Data.format.in_(KOBO_FORMATS))
+                           .filter(calibre_db.common_filters(allow_show_archived=True))
+                           .order_by(db.Books.id)
+                           .order_by(ub.ArchivedBook.last_modified)
+                           .join(ub.BookShelf, db.Books.id == ub.BookShelf.book_id)
+                           .join(ub.Shelf)
+                           .filter(ub.Shelf.user_id == current_user.id)
+                           .filter(ub.Shelf.kobo_sync)
+                           .distinct())
     else:
         if sqlalchemy_version2:
             changed_entries = select(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived)
@@ -196,16 +195,14 @@ def HandleSyncRequest():
                                                        ub.ArchivedBook.last_modified,
                                                        ub.ArchivedBook.is_archived)
         changed_entries = (changed_entries
-                   .join(db.Data).outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id,
-                                                                  ub.ArchivedBook.user_id == current_user.id))
-                   .filter(db.Books.id.notin_(calibre_db.session.query(ub.KoboSyncedBooks.book_id)
-                                              .filter(ub.KoboSyncedBooks.user_id == current_user.id)))
-                   .filter(calibre_db.common_filters(allow_show_archived=True))
-                   .filter(db.Data.format.in_(KOBO_FORMATS))
-                   .order_by(db.Books.last_modified)
-                   .order_by(db.Books.id)
-        )
-
+                           .join(db.Data).outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id,
+                                                                          ub.ArchivedBook.user_id == current_user.id))
+                           .filter(db.Books.id.notin_(calibre_db.session.query(ub.KoboSyncedBooks.book_id)
+                                                      .filter(ub.KoboSyncedBooks.user_id == current_user.id)))
+                           .filter(calibre_db.common_filters(allow_show_archived=True))
+                           .filter(db.Data.format.in_(KOBO_FORMATS))
+                           .order_by(db.Books.last_modified)
+                           .order_by(db.Books.id))
 
     reading_states_in_new_entitlements = []
     if sqlalchemy_version2:
@@ -215,7 +212,7 @@ def HandleSyncRequest():
     log.debug("Books to Sync: {}".format(len(books.all())))
     for book in books:
         formats = [data.format for data in book.Books.data]
-        if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats:
+        if 'KEPUB' not in formats and config.config_kepubifypath and 'EPUB' in formats:
             helper.convert_book_format(book.Books.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)
 
         kobo_reading_state = get_or_create_reading_state(book.Books.id)
@@ -262,7 +259,7 @@ def HandleSyncRequest():
             .columns(db.Books).first()
     else:
         max_change = changed_entries.from_self().filter(ub.ArchivedBook.is_archived)\
-            .filter(ub.ArchivedBook.user_id==current_user.id) \
+            .filter(ub.ArchivedBook.user_id == current_user.id) \
             .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()).first()
 
     max_change = max_change.last_modified if max_change else new_archived_last_modified
@@ -425,9 +422,9 @@ def get_author(book):
     author_list = []
     autor_roles = []
     for author in book.authors:
-        autor_roles.append({"Name":author.name})    #.encode('unicode-escape').decode('latin-1')
+        autor_roles.append({"Name": author.name})
         author_list.append(author.name)
-    return {"ContributorRoles": autor_roles, "Contributors":author_list}
+    return {"ContributorRoles": autor_roles, "Contributors": author_list}
 
 
 def get_publisher(book):
@@ -441,6 +438,7 @@ def get_series(book):
         return None
     return book.series[0].name
 
+
 def get_seriesindex(book):
     return book.series_index or 1
 
@@ -485,7 +483,7 @@ def get_metadata(book):
         "Language": "en",
         "PhoneticPronunciations": {},
         "PublicationDate": convert_to_kobo_timestamp_string(book.pubdate),
-        "Publisher": {"Imprint": "", "Name": get_publisher(book),},
+        "Publisher": {"Imprint": "", "Name": get_publisher(book), },
         "RevisionId": book_uuid,
         "Title": book.title,
         "WorkId": book_uuid,
@@ -504,6 +502,7 @@ def get_metadata(book):
 
     return metadata
 
+
 @csrf.exempt
 @kobo.route("/v1/library/tags", methods=["POST", "DELETE"])
 @requires_kobo_auth
@@ -718,7 +717,6 @@ def sync_shelves(sync_token, sync_results, only_kobo_shelves=False):
             *extra_filters
         ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc())
 
-
     for shelf in shelflist:
         if not shelf_lib.check_shelf_view_permissions(shelf):
             continue
@@ -764,6 +762,7 @@ def create_kobo_tag(shelf):
         )
     return {"Tag": tag}
 
+
 @csrf.exempt
 @kobo.route("/v1/library/<book_uuid>/state", methods=["GET", "PUT"])
 @requires_kobo_auth
@@ -808,7 +807,7 @@ def HandleStateRequest(book_uuid):
                 book_read = kobo_reading_state.book_read_link
                 new_book_read_status = get_ub_read_status(request_status_info["Status"])
                 if new_book_read_status == ub.ReadBook.STATUS_IN_PROGRESS \
-                    and new_book_read_status != book_read.read_status:
+                        and new_book_read_status != book_read.read_status:
                     book_read.times_started_reading += 1
                     book_read.last_time_started_reading = datetime.datetime.utcnow()
                 book_read.read_status = new_book_read_status
@@ -848,7 +847,7 @@ def get_ub_read_status(kobo_read_status):
 
 def get_or_create_reading_state(book_id):
     book_read = ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id,
-                                                          ub.ReadBook.user_id == int(current_user.id)).one_or_none()
+                                                     ub.ReadBook.user_id == int(current_user.id)).one_or_none()
     if not book_read:
         book_read = ub.ReadBook(user_id=current_user.id, book_id=book_id)
     if not book_read.kobo_reading_state:
@@ -912,13 +911,12 @@ def get_current_bookmark_response(current_bookmark):
         }
     return resp
 
+
 @kobo.route("/<book_uuid>/<width>/<height>/<isGreyscale>/image.jpg", defaults={'Quality': ""})
 @kobo.route("/<book_uuid>/<width>/<height>/<Quality>/<isGreyscale>/image.jpg")
 @requires_kobo_auth
-def HandleCoverImageRequest(book_uuid, width, height,Quality, isGreyscale):
-    book_cover = helper.get_book_cover_with_uuid(
-        book_uuid, use_generic_cover_on_failure=False
-    )
+def HandleCoverImageRequest(book_uuid, width, height, Quality, isGreyscale):
+    book_cover = helper.get_book_cover_with_uuid(book_uuid, resolution=COVER_THUMBNAIL_SMALL)
     if not book_cover:
         if config.config_kobo_proxy:
             log.debug("Cover for unknown book: %s proxied to kobo" % book_uuid)
@@ -991,8 +989,8 @@ def handle_getests():
     if config.config_kobo_proxy:
         return redirect_or_proxy_request()
     else:
-        testkey = request.headers.get("X-Kobo-userkey","")
-        return make_response(jsonify({"Result": "Success", "TestKey":testkey, "Tests": {}}))
+        testkey = request.headers.get("X-Kobo-userkey", "")
+        return make_response(jsonify({"Result": "Success", "TestKey": testkey, "Tests": {}}))
 
 
 @csrf.exempt
@@ -1022,7 +1020,7 @@ def make_calibre_web_auth_response():
     content = request.get_json()
     AccessToken = base64.b64encode(os.urandom(24)).decode('utf-8')
     RefreshToken = base64.b64encode(os.urandom(24)).decode('utf-8')
-    return  make_response(
+    return make_response(
         jsonify(
             {
                 "AccessToken": AccessToken,
@@ -1160,14 +1158,16 @@ def NATIVE_KOBO_RESOURCES():
         "eula_page": "https://www.kobo.com/termsofuse?style=onestore",
         "exchange_auth": "https://storeapi.kobo.com/v1/auth/exchange",
         "external_book": "https://storeapi.kobo.com/v1/products/books/external/{Ids}",
-        "facebook_sso_page": "https://authorize.kobo.com/signin/provider/Facebook/login?returnUrl=http://store.kobobooks.com/",
+        "facebook_sso_page":
+            "https://authorize.kobo.com/signin/provider/Facebook/login?returnUrl=http://store.kobobooks.com/",
         "featured_list": "https://storeapi.kobo.com/v1/products/featured/{FeaturedListId}",
         "featured_lists": "https://storeapi.kobo.com/v1/products/featured",
         "free_books_page": {
             "EN": "https://www.kobo.com/{region}/{language}/p/free-ebooks",
             "FR": "https://www.kobo.com/{region}/{language}/p/livres-gratuits",
             "IT": "https://www.kobo.com/{region}/{language}/p/libri-gratuiti",
-            "NL": "https://www.kobo.com/{region}/{language}/List/bekijk-het-overzicht-van-gratis-ebooks/QpkkVWnUw8sxmgjSlCbJRg",
+            "NL": "https://www.kobo.com/{region}/{language}/"
+                  "List/bekijk-het-overzicht-van-gratis-ebooks/QpkkVWnUw8sxmgjSlCbJRg",
             "PT": "https://www.kobo.com/{region}/{language}/p/livros-gratis",
         },
         "fte_feedback": "https://storeapi.kobo.com/v1/products/ftefeedback",
@@ -1192,7 +1192,8 @@ def NATIVE_KOBO_RESOURCES():
         "library_stack": "https://storeapi.kobo.com/v1/user/library/stacks/{LibraryItemId}",
         "library_sync": "https://storeapi.kobo.com/v1/library/sync",
         "love_dashboard_page": "https://store.kobobooks.com/{culture}/kobosuperpoints",
-        "love_points_redemption_page": "https://store.kobobooks.com/{culture}/KoboSuperPointsRedemption?productId={ProductId}",
+        "love_points_redemption_page":
+            "https://store.kobobooks.com/{culture}/KoboSuperPointsRedemption?productId={ProductId}",
         "magazine_landing_page": "https://store.kobobooks.com/emagazines",
         "notifications_registration_issue": "https://storeapi.kobo.com/v1/notifications/registration",
         "oauth_host": "https://oauth.kobo.com",
@@ -1208,7 +1209,8 @@ def NATIVE_KOBO_RESOURCES():
         "product_recommendations": "https://storeapi.kobo.com/v1/products/{ProductId}/recommendations",
         "product_reviews": "https://storeapi.kobo.com/v1/products/{ProductIds}/reviews",
         "products": "https://storeapi.kobo.com/v1/products",
-        "provider_external_sign_in_page": "https://authorize.kobo.com/ExternalSignIn/{providerName}?returnUrl=http://store.kobobooks.com/",
+        "provider_external_sign_in_page":
+            "https://authorize.kobo.com/ExternalSignIn/{providerName}?returnUrl=http://store.kobobooks.com/",
         "purchase_buy": "https://www.kobo.com/checkout/createpurchase/",
         "purchase_buy_templated": "https://www.kobo.com/{culture}/checkout/createpurchase/{ProductId}",
         "quickbuy_checkout": "https://storeapi.kobo.com/v1/store/quickbuy/{PurchaseId}/checkout",

cps/metadata_provider/amazon.py

@@ -19,11 +19,13 @@
 import concurrent.futures
 import requests
 from bs4 import BeautifulSoup as BS  # requirement
+from typing import List, Optional
 
 try:
     import cchardet #optional for better speed
 except ImportError:
     pass
+from cps import logger
 from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
 import cps.logger as logger
 
@@ -31,6 +33,9 @@ import cps.logger as logger
 from operator import itemgetter
 log = logger.create()
 
+log = logger.create()
+
+
 class Amazon(Metadata):
     __name__ = "Amazon"
     __id__ = "amazon"
@@ -49,17 +54,21 @@ class Amazon(Metadata):
 
     def search(
         self, query: str, generic_cover: str = "", locale: str = "en"
-    ):
+    ) -> Optional[List[MetaRecord]]:
         #timer=time()
         def inner(link, index) -> [dict, int]:
-            try:
-                with self.session as session:
-                    r = session.get(f"https://www.amazon.com{link}")
+            with self.session as session:
+                try:
+                    r = session.get(f"https://www.amazon.com/{link}")
                     r.raise_for_status()
-                    long_soup = BS(r.text, "lxml")  #~4sec :/
-                    soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-books-ppd_csm_instrumentation_wrapper"})
-                    if soup2 is None:
-                        return
+                except Exception as ex:
+                    log.warning(ex)
+                    return
+                long_soup = BS(r.text, "lxml")  #~4sec :/
+                soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-books-ppd_csm_instrumentation_wrapper"})
+                if soup2 is None:
+                    return
+                try:
                     match = MetaRecord(
                         title = "",
                         authors = "",
@@ -104,27 +113,29 @@ class Amazon(Metadata):
                     except (AttributeError, TypeError):
                         match.cover = ""
                     return match, index
-            except Exception as e:
-                log.error_or_exception(e)
-                return
+                except Exception as e:
+                    log.error_or_exception(e)
+                    return
 
         val = list()
-        try:
-            if self.active:
+        if self.active:
+            try:
                 results = self.session.get(
-                    f"https://www.amazon.com/s?k={query.replace(' ', '+')}"
-                    f"&i=digital-text&sprefix={query.replace(' ', '+')}"
+                    f"https://www.amazon.com/s?k={query.replace(' ', '+')}&i=digital-text&sprefix={query.replace(' ', '+')}"
                     f"%2Cdigital-text&ref=nb_sb_noss",
                     headers=self.headers)
                 results.raise_for_status()
-                soup = BS(results.text, 'html.parser')
-                links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in
-                              soup.findAll("div", attrs={"data-component-type": "s-search-result"})]
-                with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
-                    fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:5])}
-                    val = list(map(lambda x: x.result(), concurrent.futures.as_completed(fut)))
-            result = list(filter(lambda x: x, val))
-            return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance
-        except requests.exceptions.HTTPError as e:
-            log.error_or_exception(e)
-            return []
+            except requests.exceptions.HTTPError as e:
+                log.error_or_exception(e)
+                return None
+            except Exception as e:
+                log.warning(e)
+                return None
+            soup = BS(results.text, 'html.parser')
+            links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in
+                          soup.findAll("div", attrs={"data-component-type": "s-search-result"})]
+            with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+                fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:5])}
+                val = list(map(lambda x : x.result() ,concurrent.futures.as_completed(fut)))
+        result = list(filter(lambda x: x, val))
+        return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance

cps/metadata_provider/comicvine.py

@@ -21,8 +21,11 @@ from typing import Dict, List, Optional
 from urllib.parse import quote
 
 import requests
+from cps import logger
 from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
 
+log = logger.create()
+
 
 class ComicVine(Metadata):
     __name__ = "ComicVine"
@@ -46,10 +49,15 @@ class ComicVine(Metadata):
             if title_tokens:
                 tokens = [quote(t.encode("utf-8")) for t in title_tokens]
                 query = "%20".join(tokens)
-            result = requests.get(
-                f"{ComicVine.BASE_URL}{query}{ComicVine.QUERY_PARAMS}",
-                headers=ComicVine.HEADERS,
-            )
+            try:
+                result = requests.get(
+                    f"{ComicVine.BASE_URL}{query}{ComicVine.QUERY_PARAMS}",
+                    headers=ComicVine.HEADERS,
+                )
+                result.raise_for_status()
+            except Exception as e:
+                log.warning(e)
+                return None
             for result in result.json()["results"]:
                 match = self._parse_search_result(
                     result=result, generic_cover=generic_cover, locale=locale

cps/metadata_provider/douban.py

@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2022 xlivevil
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+import re
+from concurrent import futures
+from typing import List, Optional
+
+import requests
+from html2text import HTML2Text
+from lxml import etree
+
+from cps import logger
+from cps.services.Metadata import Metadata, MetaRecord, MetaSourceInfo
+
+log = logger.create()
+
+
+def html2text(html: str) -> str:
+
+    h2t = HTML2Text()
+    h2t.body_width = 0
+    h2t.single_line_break = True
+    h2t.emphasis_mark = "*"
+    return h2t.handle(html)
+
+
+class Douban(Metadata):
+    __name__ = "豆瓣"
+    __id__ = "douban"
+    DESCRIPTION = "豆瓣"
+    META_URL = "https://book.douban.com/"
+    SEARCH_URL = "https://www.douban.com/j/search"
+
+    ID_PATTERN = re.compile(r"sid: (?P<id>\d+),")
+    AUTHORS_PATTERN = re.compile(r"作者|译者")
+    PUBLISHER_PATTERN = re.compile(r"出版社")
+    SUBTITLE_PATTERN = re.compile(r"副标题")
+    PUBLISHED_DATE_PATTERN = re.compile(r"出版年")
+    SERIES_PATTERN = re.compile(r"丛书")
+    IDENTIFIERS_PATTERN = re.compile(r"ISBN|统一书号")
+
+    TITTLE_XPATH = "//span[@property='v:itemreviewed']"
+    COVER_XPATH = "//a[@class='nbg']"
+    INFO_XPATH = "//*[@id='info']//span[@class='pl']"
+    TAGS_XPATH = "//a[contains(@class, 'tag')]"
+    DESCRIPTION_XPATH = "//div[@id='link-report']//div[@class='intro']"
+    RATING_XPATH = "//div[@class='rating_self clearfix']/strong"
+
+    session = requests.Session()
+    session.headers = {
+        'user-agent':
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36 Edg/98.0.1108.56',
+    }
+
+    def search(
+        self, query: str, generic_cover: str = "", locale: str = "en"
+    ) -> Optional[List[MetaRecord]]:
+        if self.active:
+            log.debug(f"starting search {query} on douban")
+            if title_tokens := list(
+                self.get_title_tokens(query, strip_joiners=False)
+            ):
+                query = "+".join(title_tokens)
+
+            try:
+                r = self.session.get(
+                    self.SEARCH_URL, params={"cat": 1001, "q": query}
+                )
+                r.raise_for_status()
+
+            except Exception as e:
+                log.warning(e)
+                return None
+
+            results = r.json()
+            if results["total"] == 0:
+                return []
+
+            book_id_list = [
+                self.ID_PATTERN.search(item).group("id")
+                for item in results["items"][:10] if self.ID_PATTERN.search(item)
+            ]
+
+            with futures.ThreadPoolExecutor(max_workers=5) as executor:
+
+                fut = [
+                    executor.submit(self._parse_single_book, book_id, generic_cover)
+                    for book_id in book_id_list
+                ]
+                
+                val = [
+                    future.result() 
+                    for future in futures.as_completed(fut) if future.result()
+                ]
+
+        return val
+
+    def _parse_single_book(
+        self, id: str, generic_cover: str = ""
+    ) -> Optional[MetaRecord]:
+        url = f"https://book.douban.com/subject/{id}/"
+
+        try:
+            r = self.session.get(url)
+            r.raise_for_status()
+        except Exception as e:
+            log.warning(e)
+            return None
+
+        match = MetaRecord(
+            id=id,
+            title="",
+            authors=[],
+            url=url,
+            source=MetaSourceInfo(
+                id=self.__id__,
+                description=self.DESCRIPTION,
+                link=self.META_URL,
+            ),
+        )
+
+        html = etree.HTML(r.content.decode("utf8"))
+
+        match.title = html.xpath(self.TITTLE_XPATH)[0].text
+        match.cover = html.xpath(self.COVER_XPATH)[0].attrib["href"] or generic_cover
+        try:
+            rating_num = float(html.xpath(self.RATING_XPATH)[0].text.strip())
+        except Exception:
+            rating_num = 0
+        match.rating = int(-1 * rating_num // 2 * -1) if rating_num else 0
+
+        tag_elements = html.xpath(self.TAGS_XPATH)
+        if len(tag_elements):
+            match.tags = [tag_element.text for tag_element in tag_elements]
+
+        description_element = html.xpath(self.DESCRIPTION_XPATH)
+        if len(description_element):
+            match.description = html2text(etree.tostring(
+                description_element[-1], encoding="utf8").decode("utf8"))
+
+        info = html.xpath(self.INFO_XPATH)
+
+        for element in info:
+            text = element.text
+            if self.AUTHORS_PATTERN.search(text):
+                next = element.getnext()
+                while next is not None and next.tag != "br":
+                    match.authors.append(next.text)
+                    next = next.getnext()
+            elif self.PUBLISHER_PATTERN.search(text):
+                match.publisher = element.tail.strip()
+            elif self.SUBTITLE_PATTERN.search(text):
+                match.title = f'{match.title}:' + element.tail.strip()
+            elif self.PUBLISHED_DATE_PATTERN.search(text):
+                match.publishedDate = self._clean_date(element.tail.strip())
+            elif self.SUBTITLE_PATTERN.search(text):
+                match.series = element.getnext().text
+            elif i_type := self.IDENTIFIERS_PATTERN.search(text):
+                match.identifiers[i_type.group()] = element.tail.strip()
+
+        return match
+
+    
+    def _clean_date(self, date: str) -> str:
+        """
+        Clean up the date string to be in the format YYYY-MM-DD
+
+        Examples of possible patterns:
+            '2014-7-16', '1988年4月', '1995-04', '2021-8', '2020-12-1', '1996年',
+            '1972', '2004/11/01', '1959年3月北京第1版第1印'
+        """
+        year = date[:4]
+        moon = "01"
+        day = "01"
+
+        if len(date) > 5:
+            digit = []
+            ls = []
+            for i in range(5, len(date)):
+                if date[i].isdigit():
+                    digit.append(date[i])
+                elif digit:
+                    ls.append("".join(digit) if len(digit)==2 else f"0{digit[0]}")
+                    digit = []
+            if digit:
+                ls.append("".join(digit) if len(digit)==2 else f"0{digit[0]}")
+
+            moon = ls[0]
+            if len(ls)>1:
+                day = ls[1]     
+
+        return f"{year}-{moon}-{day}"

cps/metadata_provider/google.py

@@ -22,9 +22,12 @@ from urllib.parse import quote
 
 import requests
 
+from cps import logger
 from cps.isoLanguages import get_lang3, get_language_name
 from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
 
+log = logger.create()
+
 
 class Google(Metadata):
     __name__ = "Google"
@@ -45,7 +48,12 @@ class Google(Metadata):
             if title_tokens:
                 tokens = [quote(t.encode("utf-8")) for t in title_tokens]
                 query = "+".join(tokens)
-            results = requests.get(Google.SEARCH_URL + query)
+            try:
+                results = requests.get(Google.SEARCH_URL + query)
+                results.raise_for_status()
+            except Exception as e:
+                log.warning(e)
+                return None
             for result in results.json().get("items", []):
                 val.append(
                     self._parse_search_result(

cps/metadata_provider/lubimyczytac.py

@@ -27,9 +27,12 @@ from html2text import HTML2Text
 from lxml.html import HtmlElement, fromstring, tostring
 from markdown2 import Markdown
 
+from cps import logger
 from cps.isoLanguages import get_language_name
 from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
 
+log = logger.create()
+
 SYMBOLS_TO_TRANSLATE = (
     "öÖüÜóÓőŐúÚéÉáÁűŰíÍąĄćĆęĘłŁńŃóÓśŚźŹżŻ",
     "oOuUoOoOuUeEaAuUiIaAcCeElLnNoOsSzZzZ",
@@ -112,20 +115,23 @@ class LubimyCzytac(Metadata):
         self, query: str, generic_cover: str = "", locale: str = "en"
     ) -> Optional[List[MetaRecord]]:
         if self.active:
-            result = requests.get(self._prepare_query(title=query))
-            if result.text:
-                root = fromstring(result.text)
-                lc_parser = LubimyCzytacParser(root=root, metadata=self)
-                matches = lc_parser.parse_search_results()
-                if matches:
-                    with ThreadPool(processes=10) as pool:
-                        final_matches = pool.starmap(
-                            lc_parser.parse_single_book,
-                            [(match, generic_cover, locale) for match in matches],
-                        )
-                    return final_matches
-                return matches
-            return []
+            try:
+                result = requests.get(self._prepare_query(title=query))
+                result.raise_for_status()
+            except Exception as e:
+                log.warning(e)
+                return None
+            root = fromstring(result.text)
+            lc_parser = LubimyCzytacParser(root=root, metadata=self)
+            matches = lc_parser.parse_search_results()
+            if matches:
+                with ThreadPool(processes=10) as pool:
+                    final_matches = pool.starmap(
+                        lc_parser.parse_single_book,
+                        [(match, generic_cover, locale) for match in matches],
+                    )
+                return final_matches
+            return matches
 
     def _prepare_query(self, title: str) -> str:
         query = ""
@@ -202,7 +208,12 @@ class LubimyCzytacParser:
     def parse_single_book(
         self, match: MetaRecord, generic_cover: str, locale: str
     ) -> MetaRecord:
-        response = requests.get(match.url)
+        try:
+            response = requests.get(match.url)
+            response.raise_for_status()
+        except Exception as e:
+            log.warning(e)
+            return None
         self.root = fromstring(response.text)
         match.cover = self._parse_cover(generic_cover=generic_cover)
         match.description = self._parse_description()

cps/metadata_provider/scholar.py

@@ -28,8 +28,12 @@ try:
 except FakeUserAgentError:
     raise ImportError("No module named 'scholarly'")
 
+from cps import logger
 from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata
 
+log = logger.create()
+
+
 class scholar(Metadata):
     __name__ = "Google Scholar"
     __id__ = "googlescholar"
@@ -44,7 +48,11 @@ class scholar(Metadata):
             if title_tokens:
                 tokens = [quote(t.encode("utf-8")) for t in title_tokens]
                 query = " ".join(tokens)
-            scholar_gen = itertools.islice(scholarly.search_pubs(query), 10)
+            try:
+                scholar_gen = itertools.islice(scholarly.search_pubs(query), 10)
+            except Exception as e:
+                log.warning(e)
+                return None
             for result in scholar_gen:
                 match = self._parse_search_result(
                     result=result, generic_cover="", locale=locale

cps/schedule.py

@@ -19,38 +19,39 @@
 import datetime
 
 from . import config, constants
-from .services.background_scheduler import BackgroundScheduler
+from .services.background_scheduler import BackgroundScheduler, use_APScheduler
 from .tasks.database import TaskReconnectDatabase
-from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails
+from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails, TaskClearCoverThumbnailCache
 from .services.worker import WorkerThread
 
 
 def get_scheduled_tasks(reconnect=True):
     tasks = list()
-
+    # config.schedule_reconnect or
     # Reconnect Calibre database (metadata.db)
     if reconnect:
-        tasks.append([lambda: TaskReconnectDatabase(), 'reconnect'])
+        tasks.append([lambda: TaskReconnectDatabase(), 'reconnect', False])
 
     # Generate all missing book cover thumbnails
     if config.schedule_generate_book_covers:
-        tasks.append([lambda: TaskGenerateCoverThumbnails(), 'generate book covers'])
+        tasks.append([lambda: TaskClearCoverThumbnailCache(0), 'delete superfluous book covers', True])
+        tasks.append([lambda: TaskGenerateCoverThumbnails(), 'generate book covers', False])
 
     # Generate all missing series thumbnails
     if config.schedule_generate_series_covers:
-        tasks.append([lambda: TaskGenerateSeriesThumbnails(), 'generate book covers'])
+        tasks.append([lambda: TaskGenerateSeriesThumbnails(), 'generate book covers', False])
 
     return tasks
 
 
 def end_scheduled_tasks():
     worker = WorkerThread.get_instance()
-    for __, __, __, task in worker.tasks:
+    for __, __, __, task, __ in worker.tasks:
         if task.scheduled and task.is_cancellable:
             worker.end_task(task.id)
 
 
-def register_scheduled_tasks():
+def register_scheduled_tasks(reconnect=True):
     scheduler = BackgroundScheduler()
 
     if scheduler:
@@ -58,16 +59,17 @@ def register_scheduled_tasks():
         scheduler.remove_all_jobs()
 
         start = config.schedule_start_time
-        end = config.schedule_end_time
+        duration = config.schedule_duration
 
         # Register scheduled tasks
-        if start != end:
-            scheduler.schedule_tasks(tasks=get_scheduled_tasks(), trigger='cron', hour=start)
-            scheduler.schedule(func=end_scheduled_tasks, trigger='cron', name="end scheduled task", hour=end)
+        scheduler.schedule_tasks(tasks=get_scheduled_tasks(), trigger='cron', hour=start)
+        end_time = calclulate_end_time(start, duration)
+        scheduler.schedule(func=end_scheduled_tasks, trigger='cron', name="end scheduled task", hour=end_time.hour,
+                           minute=end_time.minute)
 
         # Kick-off tasks, if they should currently be running
-        if should_task_be_running(start, end):
-            scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False))
+        if should_task_be_running(start, duration):
+            scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(reconnect))
 
 
 def register_startup_tasks():
@@ -75,14 +77,21 @@ def register_startup_tasks():
 
     if scheduler:
         start = config.schedule_start_time
-        end = config.schedule_end_time
+        duration = config.schedule_duration
 
         # Run scheduled tasks immediately for development and testing
         # Ignore tasks that should currently be running, as these will be added when registering scheduled tasks
-        if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, end):
+        if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, duration):
             scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False))
 
 
-def should_task_be_running(start, end):
-    now = datetime.datetime.now().hour
-    return (start < end and start <= now < end) or (end < start <= now or now < end)
+def should_task_be_running(start, duration):
+    now = datetime.datetime.now()
+    start_time = datetime.datetime.now().replace(hour=start, minute=0, second=0, microsecond=0)
+    end_time = start_time + datetime.timedelta(hours=duration // 60, minutes=duration % 60)
+    return start_time < now < end_time
+
+def calclulate_end_time(start, duration):
+    start_time = datetime.datetime.now().replace(hour=start, minute=0)
+    return start_time + datetime.timedelta(hours=duration // 60, minutes=duration % 60)
+

cps/search_metadata.py

@@ -57,7 +57,7 @@ for f in modules:
         try:
             importlib.import_module("cps.metadata_provider." + a)
             new_list.append(a)
-        except ImportError as e:
+        except (ImportError, IndentationError, SyntaxError) as e:
             log.error("Import error for metadata source: {} - {}".format(a, e))
             pass
 
@@ -138,6 +138,6 @@ def metadata_search():
                 if active.get(c.__id__, True)
             }
             for future in concurrent.futures.as_completed(meta):
-                data.extend([asdict(x) for x in future.result()])
+                data.extend([asdict(x) for x in future.result() if x])
     # log.info({'Time elapsed {}'.format(current_milli_time()-start)})
     return Response(json.dumps(data), mimetype="application/json")

cps/services/background_scheduler.py

@@ -52,32 +52,32 @@ class BackgroundScheduler:
             return self.scheduler.add_job(func=func, trigger=trigger, name=name, **trigger_args)
 
     # Expects a lambda expression for the task
-    def schedule_task(self, task, user=None, name=None, trigger='cron', **trigger_args):
+    def schedule_task(self, task, user=None, name=None, hidden=False, trigger='cron', **trigger_args):
         if use_APScheduler:
             def scheduled_task():
                 worker_task = task()
                 worker_task.scheduled = True
-                WorkerThread.add(user, worker_task)
+                WorkerThread.add(user, worker_task, hidden=hidden)
             return self.schedule(func=scheduled_task, trigger=trigger, name=name, **trigger_args)
 
     # Expects a list of lambda expressions for the tasks
     def schedule_tasks(self, tasks, user=None, trigger='cron', **trigger_args):
         if use_APScheduler:
             for task in tasks:
-                self.schedule_task(task[0], user=user, trigger=trigger, name=task[1], **trigger_args)
+                self.schedule_task(task[0], user=user, trigger=trigger, name=task[1], hidden=task[2], **trigger_args)
 
     # Expects a lambda expression for the task
-    def schedule_task_immediately(self, task, user=None, name=None):
+    def schedule_task_immediately(self, task, user=None, name=None, hidden=False):
         if use_APScheduler:
             def immediate_task():
-                WorkerThread.add(user, task())
+                WorkerThread.add(user, task(), hidden)
             return self.schedule(func=immediate_task, trigger='date', name=name)
 
     # Expects a list of lambda expressions for the tasks
     def schedule_tasks_immediately(self, tasks, user=None):
         if use_APScheduler:
             for task in tasks:
-                self.schedule_task_immediately(task[0], user, name="immediately " + task[1])
+                self.schedule_task_immediately(task[0], user, name="immediately " + task[1], hidden=task[2])
 
     # Remove all jobs
     def remove_all_jobs(self):

cps/services/worker.py

@@ -43,7 +43,7 @@ STAT_CANCELLED = 5
 # Only retain this many tasks in dequeued list
 TASK_CLEANUP_TRIGGER = 20
 
-QueuedTask = namedtuple('QueuedTask', 'num, user, added, task')
+QueuedTask = namedtuple('QueuedTask', 'num, user, added, task, hidden')
 
 
 def _get_main_thread():
@@ -84,7 +84,7 @@ class WorkerThread(threading.Thread):
         self.start()
 
     @classmethod
-    def add(cls, user, task):
+    def add(cls, user, task, hidden=False):
         ins = cls.get_instance()
         ins.num += 1
         username = user if user is not None else 'System'
@@ -94,6 +94,7 @@ class WorkerThread(threading.Thread):
             user=username,
             added=datetime.now(),
             task=task,
+            hidden=hidden
         ))
 
     @property
@@ -114,10 +115,10 @@ class WorkerThread(threading.Thread):
             if delta > TASK_CLEANUP_TRIGGER:
                 ret = alive
             else:
-                # otherwise, lop off the oldest dead tasks until we hit the target trigger
-                ret = sorted(dead, key=lambda x: x.task.end_time)[-TASK_CLEANUP_TRIGGER:] + alive
+                # otherwise, loop off the oldest dead tasks until we hit the target trigger
+                ret = sorted(dead, key=lambda y: y.task.end_time)[-TASK_CLEANUP_TRIGGER:] + alive
 
-            self.dequeued = sorted(ret, key=lambda x: x.num)
+            self.dequeued = sorted(ret, key=lambda y: y.num)
 
     # Main thread loop starting the different tasks
     def run(self):
@@ -144,18 +145,18 @@ class WorkerThread(threading.Thread):
 
             # sometimes tasks (like Upload) don't actually have work to do and are created as already finished
             if item.task.stat is STAT_WAITING:
-                # CalibreTask.start() should wrap all exceptions in it's own error handling
+                # CalibreTask.start() should wrap all exceptions in its own error handling
                 item.task.start(self)
 
-            # remove self_cleanup tasks from list
-            if item.task.self_cleanup:
+            # remove self_cleanup tasks and hidden "System Tasks" from list
+            if item.task.self_cleanup or item.hidden:
                 self.dequeued.remove(item)
 
             self.queue.task_done()
 
     def end_task(self, task_id):
         ins = self.get_instance()
-        for __, __, __, task in ins.tasks:
+        for __, __, __, task, __ in ins.tasks:
             if str(task.id) == str(task_id) and task.is_cancellable:
                 task.stat = STAT_CANCELLED if task.stat == STAT_WAITING else STAT_ENDED
 
@@ -241,14 +242,6 @@ class CalibreTask:
         # By default, we're good to clean a task if it's "Done"
         return self.stat in (STAT_FINISH_SUCCESS, STAT_FAIL, STAT_ENDED, STAT_CANCELLED)
 
-    '''@progress.setter
-    def progress(self, x):        
-        if x > 1: 
-            x = 1
-        if x < 0: 
-            x = 0
-        self._progress = x'''
-
     @property
     def self_cleanup(self):
         return self._self_cleanup

cps/static/js/edit_books.js

@@ -33,7 +33,7 @@ $(".datepicker").datepicker({
     if (results) {
         pubDate = new Date(results[1], parseInt(results[2], 10) - 1, results[3]) || new Date(this.value);
         $(this).next('input')
-            .val(pubDate.toLocaleDateString(language))
+            .val(pubDate.toLocaleDateString(language.replaceAll("_","-")))
             .removeClass("hidden");
     }
 }).trigger("change");

cps/static/js/get_meta.js

@@ -92,14 +92,19 @@ $(function () {
                 data: {"query": keyword},
                 dataType: "json",
                 success: function success(data) {
-                    $("#meta-info").html("<ul id=\"book-list\" class=\"media-list\"></ul>");
-                    data.forEach(function(book) {
-                        var $book = $(templates.bookResult(book));
-                        $book.find("img").on("click", function () {
-                            populateForm(book);
+                    if (data.length) {
+                        $("#meta-info").html("<ul id=\"book-list\" class=\"media-list\"></ul>");
+                        data.forEach(function(book) {
+                            var $book = $(templates.bookResult(book));
+                            $book.find("img").on("click", function () {
+                                populateForm(book);
+                            });
+                            $("#book-list").append($book);
                         });
-                        $("#book-list").append($book);
-                    });
+                    }
+                    else {
+                        $("#meta-info").html("<p class=\"text-danger\">" + msg.no_result + "!</p>" + $("#meta-info")[0].innerHTML)
+                    }
                 },
                 error: function error() {
                     $("#meta-info").html("<p class=\"text-danger\">" + msg.search_error + "!</p>" + $("#meta-info")[0].innerHTML);

cps/static/js/main.js

@@ -474,6 +474,17 @@ $(function() {
             }
         });
     });
+    $("#admin_refresh_cover_cache").click(function() {
+        confirmDialog("admin_refresh_cover_cache", "GeneralChangeModal", 0, function () {
+            $.ajax({
+                method:"post",
+                contentType: "application/json; charset=utf-8",
+                dataType: "json",
+                url: getPath() + "/ajax/updateThumbnails",
+            });
+        });
+    });
+
     $("#restart_database").click(function() {
         $("#DialogHeader").addClass("hidden");
         $("#DialogFinished").addClass("hidden");

cps/static/js/table.js

@@ -550,7 +550,7 @@ $(function() {
 
     $("#user-table").on("click-cell.bs.table", function (field, value, row, $element) {
         if (value === "denied_column_value") {
-            ConfirmDialog("btndeluser", "GeneralDeleteModal", $element.id, user_handle);
+            confirmDialog("btndeluser", "GeneralDeleteModal", $element.id, user_handle);
         }
     });
 
@@ -641,9 +641,9 @@ function UserActions (value, row) {
 /* Function for cancelling tasks */
 function TaskActions (value, row) {
     var cancellableStats = [0, 1, 2];
-    if (row.id && row.is_cancellable && cancellableStats.includes(row.stat)) {
+    if (row.task_id && row.is_cancellable && cancellableStats.includes(row.stat)) {
         return [
-            "<div class=\"task-cancel\" data-toggle=\"modal\" data-target=\"#cancelTaskModal\" data-task-id=\"" + row.id + "\" title=\"Cancel\">",
+            "<div class=\"danger task-cancel\" data-toggle=\"modal\" data-target=\"#cancelTaskModal\" data-task-id=\"" + row.task_id + "\" title=\"Cancel\">",
             "<i class=\"glyphicon glyphicon-ban-circle\"></i>",
             "</div>"
         ].join("");

cps/tasks/convert.py

@@ -18,12 +18,12 @@
 
 import os
 import re
-
 from glob import glob
 from shutil import copyfile
 from markupsafe import escape
 
 from sqlalchemy.exc import SQLAlchemyError
+from flask_babel import lazy_gettext as N_
 
 from cps.services.worker import CalibreTask
 from cps import db
@@ -41,10 +41,10 @@ log = logger.create()
 
 
 class TaskConvert(CalibreTask):
-    def __init__(self, file_path, bookid, taskMessage, settings, kindle_mail, user=None):
-        super(TaskConvert, self).__init__(taskMessage)
+    def __init__(self, file_path, book_id, task_message, settings, kindle_mail, user=None):
+        super(TaskConvert, self).__init__(task_message)
         self.file_path = file_path
-        self.bookid = bookid
+        self.book_id = book_id
         self.title = ""
         self.settings = settings
         self.kindle_mail = kindle_mail
@@ -56,9 +56,9 @@ class TaskConvert(CalibreTask):
         self.worker_thread = worker_thread
         if config.config_use_google_drive:
             worker_db = db.CalibreDB(expire_on_commit=False)
-            cur_book = worker_db.get_book(self.bookid)
+            cur_book = worker_db.get_book(self.book_id)
             self.title = cur_book.title
-            data = worker_db.get_book_format(self.bookid, self.settings['old_book_format'])
+            data = worker_db.get_book_format(self.book_id, self.settings['old_book_format'])
             df = gdriveutils.getFileFromEbooksFolder(cur_book.path,
                                                      data.name + "." + self.settings['old_book_format'].lower())
             if df:
@@ -89,7 +89,7 @@ class TaskConvert(CalibreTask):
                 # if we're sending to kindle after converting, create a one-off task and run it immediately
                 # todo: figure out how to incorporate this into the progress
                 try:
-                    EmailText = _(u"%(book)s send to Kindle", book=escape(self.title))
+                    EmailText = N_(u"%(book)s send to Kindle", book=escape(self.title))
                     worker_thread.add(self.user, TaskEmail(self.settings['subject'],
                                                            self.results["path"],
                                                            filename,
@@ -106,7 +106,7 @@ class TaskConvert(CalibreTask):
         error_message = None
         local_db = db.CalibreDB(expire_on_commit=False)
         file_path = self.file_path
-        book_id = self.bookid
+        book_id = self.book_id
         format_old_ext = u'.' + self.settings['old_book_format'].lower()
         format_new_ext = u'.' + self.settings['new_book_format'].lower()
 
@@ -114,7 +114,7 @@ class TaskConvert(CalibreTask):
         # if it does - mark the conversion task as complete and return a success
         # this will allow send to kindle workflow to continue to work
         if os.path.isfile(file_path + format_new_ext) or\
-            local_db.get_book_format(self.bookid, self.settings['new_book_format']):
+                local_db.get_book_format(self.book_id, self.settings['new_book_format']):
             log.info("Book id %d already converted to %s", book_id, format_new_ext)
             cur_book = local_db.get_book(book_id)
             self.title = cur_book.title
@@ -133,7 +133,7 @@ class TaskConvert(CalibreTask):
                     local_db.session.rollback()
                     log.error("Database error: %s", e)
                     local_db.session.close()
-                    self._handleError(error_message)
+                    self._handleError(N_("Database error: %(error)s.", error=e))
                     return
                 self._handleSuccess()
                 local_db.session.close()
@@ -150,8 +150,7 @@ class TaskConvert(CalibreTask):
         else:
             # check if calibre converter-executable is existing
             if not os.path.exists(config.config_converterpath):
-                # ToDo Text is not translated
-                self._handleError(_(u"Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
+                self._handleError(N_(u"Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
                 return
             check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext)
 
@@ -184,11 +183,11 @@ class TaskConvert(CalibreTask):
                     self._handleSuccess()
                 return os.path.basename(file_path + format_new_ext)
             else:
-                error_message = _('%(format)s format not found on disk', format=format_new_ext.upper())
+                error_message = N_('%(format)s format not found on disk', format=format_new_ext.upper())
         local_db.session.close()
         log.info("ebook converter failed with error while converting book")
         if not error_message:
-            error_message = _('Ebook converter failed with unknown error')
+            error_message = N_('Ebook converter failed with unknown error')
         self._handleError(error_message)
         return
 
@@ -198,7 +197,7 @@ class TaskConvert(CalibreTask):
         try:
             p = process_open(command, quotes)
         except OSError as e:
-            return 1, _(u"Kepubify-converter failed: %(error)s", error=e)
+            return 1, N_(u"Kepubify-converter failed: %(error)s", error=e)
         self.progress = 0.01
         while True:
             nextline = p.stdout.readlines()
@@ -219,7 +218,7 @@ class TaskConvert(CalibreTask):
                 copyfile(converted_file[0], (file_path + format_new_ext))
                 os.unlink(converted_file[0])
             else:
-                return 1, _(u"Converted file not found or more than one file in folder %(folder)s",
+                return 1, N_(u"Converted file not found or more than one file in folder %(folder)s",
                             folder=os.path.dirname(file_path))
         return check, None
 
@@ -243,7 +242,7 @@ class TaskConvert(CalibreTask):
 
             p = process_open(command, quotes, newlines=False)
         except OSError as e:
-            return 1, _(u"Ebook-converter failed: %(error)s", error=e)
+            return 1, N_(u"Ebook-converter failed: %(error)s", error=e)
 
         while p.poll() is None:
             nextline = p.stdout.readline()
@@ -266,15 +265,15 @@ class TaskConvert(CalibreTask):
             ele = ele.decode('utf-8', errors="ignore").strip('\n')
             log.debug(ele)
             if not ele.startswith('Traceback') and not ele.startswith('  File'):
-                error_message = _("Calibre failed with error: %(error)s", error=ele)
+                error_message = N_("Calibre failed with error: %(error)s", error=ele)
         return check, error_message
 
     @property
     def name(self):
-        return "Convert"
+        return N_("Convert")
 
     def __str__(self):
-        return "Convert {} {}".format(self.bookid, self.kindle_mail)
+        return "Convert {} {}".format(self.book_id, self.kindle_mail)
 
     @property
     def is_cancellable(self):

cps/tasks/database.py

@@ -16,24 +16,22 @@
 #   You should have received a copy of the GNU General Public License
 #   along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-from __future__ import division, print_function, unicode_literals
+from urllib.request import urlopen
+
+from flask_babel import lazy_gettext as N_
 
 from cps import config, logger
 from cps.services.worker import CalibreTask
 
-try:
-    from urllib.request import urlopen
-except ImportError as e:
-    from urllib2 import urlopen
-
 
 class TaskReconnectDatabase(CalibreTask):
-    def __init__(self, task_message=u'Reconnecting Calibre database'):
+    def __init__(self, task_message=N_('Reconnecting Calibre database')):
         super(TaskReconnectDatabase, self).__init__(task_message)
         self.log = logger.create()
         self.listen_address = config.get_config_ipaddress()
         self.listen_port = config.config_port
 
+
     def run(self, worker_thread):
         address = self.listen_address if self.listen_address else 'localhost'
         port = self.listen_port if self.listen_port else 8083
@@ -42,7 +40,7 @@ class TaskReconnectDatabase(CalibreTask):
             urlopen('http://' + address + ':' + str(port) + '/reconnect')
             self._handleSuccess()
         except Exception as ex:
-            self._handleError(u'Unable to reconnect Calibre database: ' + str(ex))
+            self._handleError('Unable to reconnect Calibre database: ' + str(ex))
 
     @property
     def name(self):

cps/tasks/mail.py

@@ -26,9 +26,8 @@ from io import StringIO
 from email.message import EmailMessage
 from email.utils import parseaddr
 
-
-from email import encoders
-from email.utils import formatdate, make_msgid
+from flask_babel import lazy_gettext as N_
+from email.utils import formatdate
 from email.generator import Generator
 
 from cps.services.worker import CalibreTask
@@ -111,13 +110,13 @@ class EmailSSL(EmailBase, smtplib.SMTP_SSL):
 
 
 class TaskEmail(CalibreTask):
-    def __init__(self, subject, filepath, attachment, settings, recipient, taskMessage, text, internal=False):
-        super(TaskEmail, self).__init__(taskMessage)
+    def __init__(self, subject, filepath, attachment, settings, recipient, task_message, text, internal=False):
+        super(TaskEmail, self).__init__(task_message)
         self.subject = subject
         self.attachment = attachment
         self.settings = settings
         self.filepath = filepath
-        self.recipent = recipient
+        self.recipient = recipient
         self.text = text
         self.asyncSMTP = None
         self.results = dict()
@@ -139,7 +138,7 @@ class TaskEmail(CalibreTask):
         message = EmailMessage()
         # message = MIMEMultipart()
         message['From'] = self.settings["mail_from"]
-        message['To'] = self.recipent
+        message['To'] = self.recipient
         message['Subject'] = self.subject
         message['Date'] = formatdate(localtime=True)
         message['Message-Id'] = "{}@{}".format(uuid.uuid4(), self.get_msgid_domain()) # f"<{uuid.uuid4()}@{get_msgid_domain(from_)}>" # make_msgid('calibre-web')
@@ -212,7 +211,7 @@ class TaskEmail(CalibreTask):
         gen = Generator(fp, mangle_from_=False)
         gen.flatten(msg)
 
-        self.asyncSMTP.sendmail(self.settings["mail_from"], self.recipent, fp.getvalue())
+        self.asyncSMTP.sendmail(self.settings["mail_from"], self.recipient, fp.getvalue())
         self.asyncSMTP.quit()
         self._handleSuccess()
         log.debug("E-mail send successfully")
@@ -264,7 +263,7 @@ class TaskEmail(CalibreTask):
 
     @property
     def name(self):
-        return "E-mail"
+        return N_("E-mail")
 
     @property
     def is_cancellable(self):

cps/tasks/thumbnail.py

@@ -16,7 +16,6 @@
 #   You should have received a copy of the GNU General Public License
 #   along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-from __future__ import division, print_function, unicode_literals
 import os
 from urllib.request import urlopen
 
@@ -25,7 +24,7 @@ from cps import config, db, fs, gdriveutils, logger, ub
 from cps.services.worker import CalibreTask, STAT_CANCELLED, STAT_ENDED
 from datetime import datetime
 from sqlalchemy import func, text, or_
-
+from flask_babel import lazy_gettext as N_
 
 try:
     from wand.image import Image
@@ -50,7 +49,7 @@ def get_best_fit(width, height, image_width, image_height):
     resize_height = int(height / 2.0)
     aspect_ratio = image_width / image_height
 
-    # If this image's aspect ratio is different than the first image, then resize this image
+    # If this image's aspect ratio is different from the first image, then resize this image
     # to fill the width and height of the first image
     if aspect_ratio < width / height:
         resize_width = int(width / 2.0)
@@ -64,9 +63,10 @@ def get_best_fit(width, height, image_width, image_height):
 
 
 class TaskGenerateCoverThumbnails(CalibreTask):
-    def __init__(self, task_message=''):
+    def __init__(self, book_id=-1, task_message=''):
         super(TaskGenerateCoverThumbnails, self).__init__(task_message)
         self.log = logger.create()
+        self.book_id = book_id
         self.app_db_session = ub.get_new_session_instance()
         self.calibre_db = db.CalibreDB(expire_on_commit=False)
         self.cache = fs.FileSystem()
@@ -78,37 +78,21 @@ class TaskGenerateCoverThumbnails(CalibreTask):
     def run(self, worker_thread):
         if self.calibre_db.session and use_IM and self.stat != STAT_CANCELLED and self.stat != STAT_ENDED:
             self.message = 'Scanning Books'
-            books_with_covers = self.get_books_with_covers()
+            books_with_covers = self.get_books_with_covers(self.book_id)
             count = len(books_with_covers)
 
             total_generated = 0
             for i, book in enumerate(books_with_covers):
-                generated = 0
-                book_cover_thumbnails = self.get_book_cover_thumbnails(book.id)
 
                 # Generate new thumbnails for missing covers
-                resolutions = list(map(lambda t: t.resolution, book_cover_thumbnails))
-                missing_resolutions = list(set(self.resolutions).difference(resolutions))
-                for resolution in missing_resolutions:
-                    generated += 1
-                    self.create_book_cover_thumbnail(book, resolution)
-
-                # Replace outdated or missing thumbnails
-                for thumbnail in book_cover_thumbnails:
-                    if book.last_modified > thumbnail.generated_at:
-                        generated += 1
-                        self.update_book_cover_thumbnail(book, thumbnail)
-
-                    elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS):
-                        generated += 1
-                        self.update_book_cover_thumbnail(book, thumbnail)
+                generated = self.create_book_cover_thumbnails(book)
 
                 # Increment the progress
                 self.progress = (1.0 / count) * i
 
                 if generated > 0:
                     total_generated += generated
-                    self.message = u'Generated {0} cover thumbnails'.format(total_generated)
+                    self.message = N_(u'Generated %(count)s cover thumbnails', count=total_generated)
 
                 # Check if job has been cancelled or ended
                 if self.stat == STAT_CANCELLED:
@@ -125,10 +109,12 @@ class TaskGenerateCoverThumbnails(CalibreTask):
         self._handleSuccess()
         self.app_db_session.remove()
 
-    def get_books_with_covers(self):
+    def get_books_with_covers(self, book_id=-1):
+        filter_exp = (db.Books.id == book_id) if book_id != -1 else True
         return self.calibre_db.session \
             .query(db.Books) \
             .filter(db.Books.has_cover == 1) \
+            .filter(filter_exp) \
             .all()
 
     def get_book_cover_thumbnails(self, book_id):
@@ -139,7 +125,29 @@ class TaskGenerateCoverThumbnails(CalibreTask):
             .filter(or_(ub.Thumbnail.expiration.is_(None), ub.Thumbnail.expiration > datetime.utcnow())) \
             .all()
 
-    def create_book_cover_thumbnail(self, book, resolution):
+    def create_book_cover_thumbnails(self, book):
+        generated = 0
+        book_cover_thumbnails = self.get_book_cover_thumbnails(book.id)
+
+        # Generate new thumbnails for missing covers
+        resolutions = list(map(lambda t: t.resolution, book_cover_thumbnails))
+        missing_resolutions = list(set(self.resolutions).difference(resolutions))
+        for resolution in missing_resolutions:
+            generated += 1
+            self.create_book_cover_single_thumbnail(book, resolution)
+
+        # Replace outdated or missing thumbnails
+        for thumbnail in book_cover_thumbnails:
+            if book.last_modified > thumbnail.generated_at:
+                generated += 1
+                self.update_book_cover_thumbnail(book, thumbnail)
+
+            elif not self.cache.get_cache_file_exists(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS):
+                generated += 1
+                self.update_book_cover_thumbnail(book, thumbnail)
+        return generated
+
+    def create_book_cover_single_thumbnail(self, book, resolution):
         thumbnail = ub.Thumbnail()
         thumbnail.type = constants.THUMBNAIL_TYPE_COVER
         thumbnail.entity_id = book.id
@@ -151,8 +159,8 @@ class TaskGenerateCoverThumbnails(CalibreTask):
             self.app_db_session.commit()
             self.generate_book_thumbnail(book, thumbnail)
         except Exception as ex:
-            self.log.info(u'Error creating book thumbnail: ' + str(ex))
-            self._handleError(u'Error creating book thumbnail: ' + str(ex))
+            self.log.info('Error creating book thumbnail: ' + str(ex))
+            self._handleError('Error creating book thumbnail: ' + str(ex))
             self.app_db_session.rollback()
 
     def update_book_cover_thumbnail(self, book, thumbnail):
@@ -163,8 +171,8 @@ class TaskGenerateCoverThumbnails(CalibreTask):
             self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS)
             self.generate_book_thumbnail(book, thumbnail)
         except Exception as ex:
-            self.log.info(u'Error updating book thumbnail: ' + str(ex))
-            self._handleError(u'Error updating book thumbnail: ' + str(ex))
+            self.log.info('Error updating book thumbnail: ' + str(ex))
+            self._handleError('Error updating book thumbnail: ' + str(ex))
             self.app_db_session.rollback()
 
     def generate_book_thumbnail(self, book, thumbnail):
@@ -191,7 +199,7 @@ class TaskGenerateCoverThumbnails(CalibreTask):
                             img.save(filename=filename)
                 except Exception as ex:
                     # Bubble exception to calling function
-                    self.log.info(u'Error generating thumbnail file: ' + str(ex))
+                    self.log.info('Error generating thumbnail file: ' + str(ex))
                     raise ex
                 finally:
                     if stream is not None:
@@ -212,10 +220,13 @@ class TaskGenerateCoverThumbnails(CalibreTask):
 
     @property
     def name(self):
-        return 'GenerateCoverThumbnails'
+        return N_('Cover Thumbnails')
 
     def __str__(self):
-        return "GenerateCoverThumbnails"
+        if self.book_id > 0:
+            return "Add Cover Thumbnails for Book {}".format(self.book_id)
+        else:
+            return "Generate Cover Thumbnails"
 
     @property
     def is_cancellable(self):
@@ -268,7 +279,7 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
 
                 if generated > 0:
                     total_generated += generated
-                    self.message = u'Generated {0} series thumbnails'.format(total_generated)
+                    self.message = N_('Generated {0} series thumbnails').format(total_generated)
 
                 # Check if job has been cancelled or ended
                 if self.stat == STAT_CANCELLED:
@@ -324,8 +335,8 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
             self.app_db_session.commit()
             self.generate_series_thumbnail(series_books, thumbnail)
         except Exception as ex:
-            self.log.info(u'Error creating book thumbnail: ' + str(ex))
-            self._handleError(u'Error creating book thumbnail: ' + str(ex))
+            self.log.info('Error creating book thumbnail: ' + str(ex))
+            self._handleError('Error creating book thumbnail: ' + str(ex))
             self.app_db_session.rollback()
 
     def update_series_thumbnail(self, series_books, thumbnail):
@@ -336,8 +347,8 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
             self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS)
             self.generate_series_thumbnail(series_books, thumbnail)
         except Exception as ex:
-            self.log.info(u'Error updating book thumbnail: ' + str(ex))
-            self._handleError(u'Error updating book thumbnail: ' + str(ex))
+            self.log.info('Error updating book thumbnail: ' + str(ex))
+            self._handleError('Error updating book thumbnail: ' + str(ex))
             self.app_db_session.rollback()
 
     def generate_series_thumbnail(self, series_books, thumbnail):
@@ -380,7 +391,7 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
                             canvas.composite(img, left, top)
 
                     except Exception as ex:
-                        self.log.info(u'Error generating thumbnail file: ' + str(ex))
+                        self.log.info('Error generating thumbnail file: ' + str(ex))
                         raise ex
                     finally:
                         if stream is not None:
@@ -422,7 +433,7 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
 
     @property
     def name(self):
-        return 'GenerateSeriesThumbnails'
+        return N_('Cover Thumbnails')
 
     def __str__(self):
         return "GenerateSeriesThumbnails"
@@ -433,22 +444,28 @@ class TaskGenerateSeriesThumbnails(CalibreTask):
 
 
 class TaskClearCoverThumbnailCache(CalibreTask):
-    def __init__(self, book_id, task_message=u'Clearing cover thumbnail cache'):
+    def __init__(self, book_id, task_message=N_('Clearing cover thumbnail cache')):
         super(TaskClearCoverThumbnailCache, self).__init__(task_message)
         self.log = logger.create()
         self.book_id = book_id
+        self.calibre_db = db.CalibreDB(expire_on_commit=False)
         self.app_db_session = ub.get_new_session_instance()
         self.cache = fs.FileSystem()
 
     def run(self, worker_thread):
         if self.app_db_session:
-            if self.book_id > 0:    # make sure all thumbnails aren't getting deleted due to a bug
+            if self.book_id == 0:  # delete superfluous thumbnails
+                thumbnails = (self.calibre_db.session.query(ub.Thumbnail)
+                              .join(db.Books, ub.Thumbnail.entity_id == db.Books.id, isouter=True)
+                              .filter(db.Books.id == None)
+                              .all())
+            elif self.book_id > 0:  # make sure single book is selected
                 thumbnails = self.get_thumbnails_for_book(self.book_id)
+            if self.book_id < 0:
+                self.delete_all_thumbnails()
+            else:
                 for thumbnail in thumbnails:
                     self.delete_thumbnail(thumbnail)
-            else:
-                self.delete_all_thumbnails()
-
         self._handleSuccess()
         self.app_db_session.remove()
 
@@ -460,7 +477,6 @@ class TaskClearCoverThumbnailCache(CalibreTask):
             .all()
 
     def delete_thumbnail(self, thumbnail):
-        # thumbnail.expiration = datetime.utcnow()
         try:
             self.cache.delete_cache_file(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS)
             self.app_db_session \
@@ -470,8 +486,8 @@ class TaskClearCoverThumbnailCache(CalibreTask):
                 .delete()
             self.app_db_session.commit()
         except Exception as ex:
-            self.log.info(u'Error deleting book thumbnail: ' + str(ex))
-            self._handleError(u'Error deleting book thumbnail: ' + str(ex))
+            self.log.info('Error deleting book thumbnail: ' + str(ex))
+            self._handleError('Error deleting book thumbnail: ' + str(ex))
 
     def delete_all_thumbnails(self):
         try:
@@ -479,16 +495,17 @@ class TaskClearCoverThumbnailCache(CalibreTask):
             self.app_db_session.commit()
             self.cache.delete_cache_dir(constants.CACHE_TYPE_THUMBNAILS)
         except Exception as ex:
-            self.log.info(u'Error deleting thumbnail directory: ' + str(ex))
-            self._handleError(u'Error deleting thumbnail directory: ' + str(ex))
+            self.log.info('Error deleting thumbnail directory: ' + str(ex))
+            self._handleError('Error deleting thumbnail directory: ' + str(ex))
 
     @property
     def name(self):
-        return 'ThumbnailsClear'
+        return N_('Cover Thumbnails')
 
+    # needed for logging
     def __str__(self):
         if self.book_id > 0:
-            return "Delete Thumbnail cache for book " + str(self.book_id)
+            return "Replace/Delete Cover Thumbnails for book " + str(self.book_id)
         else:
             return "Delete Thumbnail cache directory"
 

cps/tasks/upload.py

@@ -17,11 +17,14 @@
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 from datetime import datetime
+
+from flask_babel import lazy_gettext as N_
+
 from cps.services.worker import CalibreTask, STAT_FINISH_SUCCESS
 
 class TaskUpload(CalibreTask):
-    def __init__(self, taskMessage, book_title):
-        super(TaskUpload, self).__init__(taskMessage)
+    def __init__(self, task_message, book_title):
+        super(TaskUpload, self).__init__(task_message)
         self.start_time = self.end_time = datetime.now()
         self.stat = STAT_FINISH_SUCCESS
         self.progress = 1
@@ -32,7 +35,7 @@ class TaskUpload(CalibreTask):
 
     @property
     def name(self):
-        return "Upload"
+        return N_("Upload")
 
     def __str__(self):
         return "Upload {}".format(self.book_title)

cps/templates/admin.html

@@ -161,32 +161,40 @@
       <a class="btn btn-default" id="view_config" href="{{url_for('admin.view_configuration')}}">{{_('Edit UI Configuration')}}</a>
     </div>
   </div>
-
+{%  if feature_support['scheduler'] %}
   <div class="row">
     <div class="col">
       <h2>{{_('Scheduled Tasks')}}</h2>
         <div class="col-xs-12 col-sm-12 scheduled_tasks_details">
           <div class="row">
             <div class="col-xs-6 col-sm-3">{{_('Time at which tasks start to run')}}</div>
-            <div class="col-xs-6 col-sm-3">{{config.schedule_start_time}}:00</div>
+            <div class="col-xs-6 col-sm-3">{{schedule_time}}</div>
           </div>
           <div class="row">
-            <div class="col-xs-6 col-sm-3">{{_('Time at which tasks stop running')}}</div>
-            <div class="col-xs-6 col-sm-3">{{config.schedule_end_time}}:00</div>
+            <div class="col-xs-6 col-sm-3">{{_('Maximum tasks duration')}}</div>
+            <div class="col-xs-6 col-sm-3">{{schedule_duration}}</div>
           </div>
           <div class="row">
             <div class="col-xs-6 col-sm-3">{{_('Generate book cover thumbnails')}}</div>
             <div class="col-xs-6 col-sm-3">{{ display_bool_setting(config.schedule_generate_book_covers) }}</div>
           </div>
-          <div class="row">
+          <!--div class="row">
             <div class="col-xs-6 col-sm-3">{{_('Generate series cover thumbnails')}}</div>
             <div class="col-xs-6 col-sm-3">{{ display_bool_setting(config.schedule_generate_series_covers) }}</div>
+          </div-->
+          <div class="row">
+            <div class="col-xs-6 col-sm-3">{{_('Reconnect to Calibre Library')}}</div>
+            <div class="col-xs-6 col-sm-3">{{ display_bool_setting(config.schedule_reconnect) }}</div>
           </div>
+
         </div>
       <a class="btn btn-default scheduledtasks" id="admin_edit_scheduled_tasks" href="{{url_for('admin.edit_scheduledtasks')}}">{{_('Edit Scheduled Tasks Settings')}}</a>
+      {% if config.schedule_generate_book_covers %}
+        <a class="btn btn-default" id="admin_refresh_cover_cache">{{_('Refresh Thumbnail Cover Cache')}}</a>
+      {%  endif %}
     </div>
   </div>
-
+{%  endif %}
   <div class="row form-group">
     <h2>{{_('Administration')}}</h2>
     <a class="btn btn-default" id="debug" href="{{url_for('admin.download_debug')}}">{{_('Download Debug Package')}}</a>
@@ -279,3 +287,6 @@
   </div>
 </div>
 {% endblock %}
+{% block modal %}
+{{ change_confirm_modal() }}
+{% endblock %}

cps/templates/languages.html

@@ -1,35 +0,0 @@
-{% extends "layout.html" %}
-{% block body %}
-  <h1>{{title}}</h1>
-  <div class="filterheader hidden-xs">
-    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
-    <div id="asc" data-order="{{ order }}" data-id="{{ data }}" class="btn btn-primary {% if order == 1 %} active{% endif%}"><span class="glyphicon glyphicon-sort-by-alphabet"></span></div>
-    <div id="desc" data-id="{{ data }}" class="btn btn-primary{% if order == 0 %} active{% endif%}"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></div>
-    {% if charlist|length %}
-    <div id="all" class="active btn btn-primary {% if charlist|length > 9 %}hidden-sm{% endif %}">{{_('All')}}</div>
-    {% endif %}
-    <div class="btn-group character {% if charlist|length > 9 %}hidden-sm{% endif %}" role="group">
-      {% for char in charlist%}
-      <div class="btn btn-primary char">{{char}}</div>
-      {% endfor %}
-    </div>
-  </div>
-  <div class="container">
-      <div div id="list" class="col-xs-12 col-sm-6">
-  {% for lang in languages %}
-      {% if loop.index0 == (loop.length/2)|int and loop.length > 20 %}
-      </div>
-      <div id="second" class="col-xs-12 col-sm-6">
-    {% endif %}
-    <div class="row" data-id="{{lang[0].name}}">
-        <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{lang[1]}}</span></div>
-        <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{{url_for('web.books_list', book_id=lang[0].lang_code, data=data, sort_param='stored')}}">{{lang[0].name}}</a></div>
-    </div>
-  {% endfor %}
-      </div>
-  </div>
-{% endblock %}
-{% block js %}
-<script src="{{ url_for('static', filename='js/filter_list.js') }}"></script>
-{% endblock %}
-

cps/templates/list.html

@@ -14,7 +14,7 @@
       {% endif %}
       <div class="btn-group character {% if charlist|length > 9 %}hidden-sm{% endif %}" role="group">
         {% for char in charlist%}
-        <div class="btn btn-primary char">{{char.char}}</div>
+        <div class="btn btn-primary char">{{char[0]}}</div>
         {% endfor %}
       </div>
 
@@ -29,8 +29,8 @@
         </div>
         <div id="second" class="col-xs-12 col-sm-6">
       {% endif %}
-      <div class="row" {% if entry[0].sort %}data-name="{{entry[0].name}}"{% endif %} data-id="{% if entry[0].sort %}{{entry[0].sort}}{% else %}{% if entry.name %}{{entry.name}}{% else %}{{entry[0].name}}{% endif %}{% endif %}">
-        <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry.count}}</span></div>
+      <div class="row" {% if entry[0].sort %}data-name="{{entry[0].name}}"{% endif %} data-id="{% if entry[0].sort %}{{entry[0].sort}}{% else %}{% if entry[0].format %}{{entry[0].format}}{% else %}{% if entry[0].rating %}{{entry[0].rating}}{% else %}{{entry[0].name}}{% endif %}{% endif %}{% endif %}">
+        <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry[1]}}</span></div>
         <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{% if entry.format %}{{url_for('web.books_list', data=data, sort_param='stored', book_id=entry.format )}}{% else %}{{url_for('web.books_list', data=data, sort_param='stored', book_id=entry[0].id )}}{% endif %}">
           {% if entry.name %}
           <div class="rating">

cps/templates/schedule_edit.html

@@ -11,16 +11,16 @@
     <div class="form-group">
       <label for="schedule_start_time">{{_('Time at which tasks start to run')}}</label>
       <select name="schedule_start_time" id="schedule_start_time" class="form-control">
-        {% for n in range(24) %}
-          <option value="{{n}}" {% if config.schedule_start_time == n %}selected{% endif %}>{{n}}{{_(':00')}}</option>
+        {% for n in starttime %}
+          <option value="{{n[0]}}" {% if config.schedule_start_time == n[0] %}selected{% endif %}>{{n[1]}}</option>
         {% endfor %}
       </select>
     </div>
     <div class="form-group">
-      <label for="schedule_end_time">{{_('Time at which tasks stop running')}}</label>
-      <select name="schedule_end_time" id="schedule_end_time" class="form-control">
-        {% for n in range(24) %}
-          <option value="{{n}}" {% if config.schedule_end_time == n %}selected{% endif %}>{{n}}{{_(':00')}}</option>
+      <label for="schedule_duration">{{_('Maximum tasks duration')}}</label>
+      <select name="schedule_duration" id="schedule_duration" class="form-control">
+        {% for n in duration %}
+          <option value="{{n[0]}}" {% if config.schedule_duration == n[0] %}selected{% endif %}>{{n[1]}}</option>
         {% endfor %}
       </select>
     </div>
@@ -28,10 +28,15 @@
       <input type="checkbox" id="schedule_generate_book_covers" name="schedule_generate_book_covers" {% if config.schedule_generate_book_covers %}checked{% endif %}>
       <label for="schedule_generate_book_covers">{{_('Generate Book Cover Thumbnails')}}</label>
     </div>
-    <div class="form-group">
+    <!--div class="form-group">
       <input type="checkbox" id="schedule_generate_series_covers" name="schedule_generate_series_covers" {% if config.schedule_generate_series_covers %}checked{% endif %}>
       <label for="schedule_generate_series_covers">{{_('Generate Series Cover Thumbnails')}}</label>
+    </div-->
+    <div class="form-group">
+      <input type="checkbox" id="schedule_reconnect" name="schedule_reconnect" {% if config.schedule_generate_book_covers %}checked{% endif %}>
+      <label for="schedule_reconnect">{{_('Reconnect to Calibre Library')}}</label>
     </div>
+
     <button type="submit" name="submit" value="submit" class="btn btn-default">{{_('Save')}}</button>
     <a href="{{ url_for('admin.admin') }}" id="email_back" class="btn btn-default">{{_('Cancel')}}</a>
   </form>

cps/templates/stats.html

@@ -39,7 +39,7 @@
   {% if version %}
     <tr>
       <th>{{library}}</th>
-      <td>{{_(version)}}</td>
+      <td>{{version}}</td>
     </tr>
   {% endif %}
   {% endfor %}

cps/uploader.py

@@ -107,52 +107,10 @@ def default_meta(tmp_file_path, original_file_name, original_file_extension):
         series="",
         series_id="",
         languages="",
-        publisher="")
-
-
-def parse_xmp(pdf_file):
-    """
-    Parse XMP Metadata and prepare for BookMeta object
-    """
-    try:
-        xmp_info = pdf_file.getXmpMetadata()
-    except Exception as ex:
-        log.debug('Can not read XMP metadata {}'.format(ex))
-        return None
-
-    if xmp_info:
-        try:
-            xmp_author = xmp_info.dc_creator # list
-        except AttributeError:
-            xmp_author = ['']
-
-        if xmp_info.dc_title:
-            xmp_title = xmp_info.dc_title['x-default']
-        else:
-            xmp_title = ''
-
-        if xmp_info.dc_description:
-            xmp_description = xmp_info.dc_description['x-default']
-        else:
-            xmp_description = ''
-
-        languages = []
-        try:
-            for i in xmp_info.dc_language:
-                #calibre-web currently only takes one language.
-                languages.append(isoLanguages.get_lang3(i))
-        except AttributeError:
-            languages.append('')
-
-        xmp_tags = ', '.join(xmp_info.dc_subject)
-        xmp_publisher = ', '.join(xmp_info.dc_publisher)
-
-        return {'author': xmp_author,
-                    'title': xmp_title,
-                    'subject': xmp_description,
-                    'tags': xmp_tags, 'languages': languages,
-                    'publisher': xmp_publisher
-                    }
+        publisher="",
+        pubdate="",
+        identifiers=[]
+        )
 
 
 def parse_xmp(pdf_file):
@@ -251,7 +209,9 @@ def pdf_meta(tmp_file_path, original_file_name, original_file_extension):
         series="",
         series_id="",
         languages=','.join(languages),
-        publisher=publisher)
+        publisher=publisher,
+        pubdate="",
+        identifiers=[])
 
 
 def pdf_preview(tmp_file_path, tmp_dir):

cps/web.py

@@ -307,10 +307,20 @@ def get_matching_tags():
     return json_dumps
 
 
-def generate_char_list(data_colum, db_link):
-    return (calibre_db.session.query(func.upper(func.substr(data_colum, 1, 1)).label('char'))
+def generate_char_list(entries): # data_colum, db_link):
+    char_list = list()
+    for entry in entries:
+        upper_char = entry[0].name[0].upper()
+        if upper_char not in char_list:
+            char_list.append(upper_char)
+    return char_list
+
+
+def query_char_list(data_colum, db_link):
+    results = (calibre_db.session.query(func.upper(func.substr(data_colum, 1, 1)).label('char'))
             .join(db_link).join(db.Books).filter(calibre_db.common_filters())
             .group_by(func.upper(func.substr(data_colum, 1, 1))).all())
+    return results
 
 
 def get_sort_function(sort_param, data):
@@ -526,50 +536,92 @@ def render_author_books(page, author_id, order):
 
 
 def render_publisher_books(page, book_id, order):
-    publisher = calibre_db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
-    if publisher:
+    if book_id == '-1':
         entries, random, pagination = calibre_db.fill_indexpage(page, 0,
                                                                 db.Books,
-                                                                db.Books.publishers.any(db.Publishers.id == book_id),
+                                                                db.Publishers.name == None,
                                                                 [db.Series.name, order[0][0], db.Books.series_index],
                                                                 True, config.config_read_column,
+                                                                db.books_publishers_link,
+                                                                db.Books.id == db.books_publishers_link.c.book,
+                                                                db.Publishers,
+                                                                db.books_series_link,
+                                                                db.Books.id == db.books_series_link.c.book,
+                                                                db.Series)
+        publisher = _("None")
+    else:
+        publisher = calibre_db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
+        if publisher:
+            entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                    db.Books,
+                                                                    db.Books.publishers.any(
+                                                                        db.Publishers.id == book_id),
+                                                                    [db.Series.name, order[0][0],
+                                                                     db.Books.series_index],
+                                                                    True, config.config_read_column,
+                                                                    db.books_series_link,
+                                                                    db.Books.id == db.books_series_link.c.book,
+                                                                    db.Series)
+            publisher = publisher.name
+        else:
+            abort(404)
+
+    return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
+                                 title=_(u"Publisher: %(name)s", name=publisher),
+                                 page="publisher",
+                                 order=order[1])
+
+
+def render_series_books(page, book_id, order):
+    if book_id == '-1':
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db.Series.name == None,
+                                                                [order[0][0]],
+                                                                True, config.config_read_column,
                                                                 db.books_series_link,
                                                                 db.Books.id == db.books_series_link.c.book,
                                                                 db.Series)
-        return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
-                                     title=_(u"Publisher: %(name)s", name=publisher.name),
-                                     page="publisher",
-                                     order=order[1])
+        series_name = _("None")
     else:
-        abort(404)
-
-
-def render_series_books(page, book_id, order):
-    name = calibre_db.session.query(db.Series).filter(db.Series.id == book_id).first()
-    if name:
-        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
-                                                                db.Books,
-                                                                db.Books.series.any(db.Series.id == book_id),
-                                                                [order[0][0]],
-                                                                True, config.config_read_column)
-        return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
-                                     title=_(u"Series: %(serie)s", serie=name.name), page="series", order=order[1])
-    else:
-        abort(404)
+        series_name = calibre_db.session.query(db.Series).filter(db.Series.id == book_id).first()
+        if series_name:
+            entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                    db.Books,
+                                                                    db.Books.series.any(db.Series.id == book_id),
+                                                                    [order[0][0]],
+                                                                    True, config.config_read_column)
+            series_name = series_name.name
+        else:
+            abort(404)
+    return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
+                                 title=_(u"Series: %(serie)s", serie=series_name), page="series", order=order[1])
 
 
 def render_ratings_books(page, book_id, order):
-    name = calibre_db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
-    entries, random, pagination = calibre_db.fill_indexpage(page, 0,
-                                                            db.Books,
-                                                            db.Books.ratings.any(db.Ratings.id == book_id),
-                                                            [order[0][0]],
-                                                            True, config.config_read_column)
-    if name and name.rating <= 10:
+    if book_id == '-1':
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db.Books.ratings == None,
+                                                                [order[0][0]],
+                                                                True, config.config_read_column,
+                                                                db.books_series_link,
+                                                                db.Books.id == db.books_series_link.c.book,
+                                                                db.Series)
+        title = _(u"Rating: None")
+        rating = -1
+    else:
+        name = calibre_db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db.Books.ratings.any(db.Ratings.id == book_id),
+                                                                [order[0][0]],
+                                                                True, config.config_read_column)
+        title = _(u"Rating: %(rating)s stars", rating=int(name.rating / 2))
+        rating = name.rating
+    if title and rating <= 10:
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
-                                     title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)),
-                                     page="ratings",
-                                     order=order[1])
+                                     title=title, page="ratings", order=order[1])
     else:
         abort(404)
 
@@ -591,33 +643,61 @@ def render_formats_books(page, book_id, order):
 
 
 def render_category_books(page, book_id, order):
-    name = calibre_db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
-    if name:
+    if book_id == '-1':
         entries, random, pagination = calibre_db.fill_indexpage(page, 0,
                                                                 db.Books,
-                                                                db.Books.tags.any(db.Tags.id == book_id),
+                                                                db.Tags.name == None,
                                                                 [order[0][0], db.Series.name, db.Books.series_index],
                                                                 True, config.config_read_column,
+                                                                db.books_tags_link,
+                                                                db.Books.id == db.books_tags_link.c.book,
+                                                                db.Tags,
                                                                 db.books_series_link,
                                                                 db.Books.id == db.books_series_link.c.book,
                                                                 db.Series)
-        return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
-                                     title=_(u"Category: %(name)s", name=name.name), page="category", order=order[1])
+        tagsname = _("None")
     else:
-        abort(404)
+        tagsname = calibre_db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
+        if tagsname:
+            entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                    db.Books,
+                                                                    db.Books.tags.any(db.Tags.id == book_id),
+                                                                    [order[0][0], db.Series.name,
+                                                                     db.Books.series_index],
+                                                                    True, config.config_read_column,
+                                                                    db.books_series_link,
+                                                                    db.Books.id == db.books_series_link.c.book,
+                                                                    db.Series)
+            tagsname = tagsname.name
+        else:
+            abort(404)
+    return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
+                                 title=_(u"Category: %(name)s", name=tagsname), page="category", order=order[1])
 
 
 def render_language_books(page, name, order):
     try:
-        lang_name = isoLanguages.get_language_name(get_locale(), name)
+        if name.lower() != "none":
+            lang_name = isoLanguages.get_language_name(get_locale(), name)
+        else:
+            lang_name = _("None")
     except KeyError:
         abort(404)
-
-    entries, random, pagination = calibre_db.fill_indexpage(page, 0,
-                                                            db.Books,
-                                                            db.Books.languages.any(db.Languages.lang_code == name),
-                                                            [order[0][0]],
-                                                            True, config.config_read_column)
+    if name == "none":
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db.Languages.lang_code == None,
+                                                                [order[0][0]],
+                                                                True, config.config_read_column,
+                                                                db.books_languages_link,
+                                                                db.Books.id == db.books_languages_link.c.book,
+                                                                db.Languages)
+    else:
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db.Books.languages.any(db.Languages.lang_code == name),
+                                                                [order[0][0]],
+                                                                True, config.config_read_column)
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name,
                                  title=_(u"Language: %(name)s", name=lang_name), page="language", order=order[1])
 
@@ -880,7 +960,7 @@ def author_list():
         entries = calibre_db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \
             .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_authors_link.author')).order_by(order).all()
-        char_list = generate_char_list(db.Authors.sort, db.books_authors_link)
+        char_list = query_char_list(db.Authors.sort, db.books_authors_link)
         # If not creating a copy, readonly databases can not display authornames with "|" in it as changing the name
         # starts a change session
         author_copy = copy.deepcopy(entries)
@@ -926,7 +1006,15 @@ def publisher_list():
         entries = calibre_db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \
             .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_publishers_link.publisher')).order_by(order).all()
-        char_list = generate_char_list(db.Publishers.name, db.books_publishers_link)
+        no_publisher_count = (calibre_db.session.query(db.Books)
+                           .outerjoin(db.books_publishers_link).outerjoin(db.Publishers)
+                           .filter(db.Publishers.name == None)
+                           .filter(calibre_db.common_filters())
+                           .count())
+        if no_publisher_count:
+            entries.append([db.Category(_("None"), "-1"), no_publisher_count])
+        entries = sorted(entries, key=lambda x: x[0].name, reverse=not order_no)
+        char_list = generate_char_list(entries)
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list,
                                      title=_(u"Publishers"), page="publisherlist", data="publisher", order=order_no)
     else:
@@ -943,11 +1031,19 @@ def series_list():
         else:
             order = db.Series.sort.asc()
             order_no = 1
-        char_list = generate_char_list(db.Series.sort, db.books_series_link)
+        char_list = query_char_list(db.Series.sort, db.books_series_link)
         if current_user.get_view_property('series', 'series_view') == 'list':
             entries = calibre_db.session.query(db.Series, func.count('books_series_link.book').label('count')) \
                 .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
                 .group_by(text('books_series_link.series')).order_by(order).all()
+            no_series_count = (calibre_db.session.query(db.Books)
+                            .outerjoin(db.books_series_link).outerjoin(db.Series)
+                            .filter(db.Series.name == None)
+                            .filter(calibre_db.common_filters())
+                            .count())
+            if no_series_count:
+                entries.append([db.Category(_("None"), "-1"), no_series_count])
+            entries = sorted(entries, key=lambda x: x[0].name, reverse=not order_no)
             return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list,
                                          title=_(u"Series"), page="serieslist", data="series", order=order_no)
         else:
@@ -976,6 +1072,13 @@ def ratings_list():
                                            (db.Ratings.rating / 2).label('name')) \
             .join(db.books_ratings_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_ratings_link.rating')).order_by(order).all()
+        no_rating_count = (calibre_db.session.query(db.Books)
+                           .outerjoin(db.books_ratings_link).outerjoin(db.Ratings)
+                           .filter(db.Ratings.rating == None)
+                           .filter(calibre_db.common_filters())
+                           .count())
+        entries.append([db.Category(_("None"), "-1", -1), no_rating_count])
+        entries = sorted(entries, key=lambda x: x[0].rating, reverse=not order_no)
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"Ratings list"), page="ratingslist", data="ratings", order=order_no)
     else:
@@ -997,6 +1100,12 @@ def formats_list():
                                            db.Data.format.label('format')) \
             .join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(db.Data.format).order_by(order).all()
+        no_format_count = (calibre_db.session.query(db.Books).outerjoin(db.Data)
+                           .filter(db.Data.format == None)
+                           .filter(calibre_db.common_filters())
+                           .count())
+        if no_format_count:
+            entries.append([db.Category(_("None"), "-1"), no_format_count])
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"File formats list"), page="formatslist", data="formats", order=order_no)
     else:
@@ -1008,15 +1117,10 @@ def formats_list():
 def language_overview():
     if current_user.check_visibility(constants.SIDEBAR_LANGUAGE) and current_user.filter_language() == u"all":
         order_no = 0 if current_user.get_view_property('language', 'dir') == 'desc' else 1
-        char_list = list()
         languages = calibre_db.speaking_language(reverse_order=not order_no, with_count=True)
-        for lang in languages:
-            upper_lang = lang[0].name[0].upper()
-            if upper_lang not in char_list:
-                char_list.append(upper_lang)
-        return render_title_template('languages.html', languages=languages,
-                                     charlist=char_list, title=_(u"Languages"), page="langlist",
-                                     data="language", order=order_no)
+        char_list = generate_char_list(languages)
+        return render_title_template('list.html', entries=languages, folder='web.books_list', charlist=char_list,
+                                     title=_(u"Languages"), page="langlist", data="language", order=order_no)
     else:
         abort(404)
 
@@ -1034,7 +1138,15 @@ def category_list():
         entries = calibre_db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \
             .join(db.books_tags_link).join(db.Books).order_by(order).filter(calibre_db.common_filters()) \
             .group_by(text('books_tags_link.tag')).all()
-        char_list = generate_char_list(db.Tags.name, db.books_tags_link)
+        no_tag_count = (calibre_db.session.query(db.Books)
+                         .outerjoin(db.books_tags_link).outerjoin(db.Tags)
+                        .filter(db.Tags.name == None)
+                         .filter(calibre_db.common_filters())
+                         .count())
+        if no_tag_count:
+            entries.append([db.Category(_("None"), "-1"), no_tag_count])
+        entries = sorted(entries, key=lambda x: x[0].name, reverse=not order_no)
+        char_list = generate_char_list(entries)
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=char_list,
                                      title=_(u"Categories"), page="catlist", data="category", order=order_no)
     else:

optional-requirements.txt

@@ -1,5 +1,5 @@
 # GDrive Integration
-google-api-python-client>=1.7.11,<2.44.0
+google-api-python-client>=1.7.11,<2.46.0
 gevent>20.6.0,<22.0.0
 greenlet>=0.4.17,<1.2.0
 httplib2>=0.9.2,<0.21.0
@@ -13,7 +13,7 @@ rsa>=3.4.2,<4.9.0
 
 # Gmail
 google-auth-oauthlib>=0.4.3,<0.6.0
-google-api-python-client>=1.7.11,<2.44.0
+google-api-python-client>=1.7.11,<2.46.0
 
 # goodreads
 goodreads>=0.3.2,<0.4.0

requirements.txt

@@ -2,7 +2,7 @@ APScheduler>=3.6.3,<3.10.0
 werkzeug<2.1.0
 Babel>=1.3,<3.0
 Flask-Babel>=0.11.1,<2.1.0
-Flask-Login>=0.3.2,<0.5.1
+Flask-Login>=0.3.2,<0.6.1
 Flask-Principal>=0.3.2,<0.5.1
 backports_abc>=0.4
 Flask>=1.0.2,<2.1.0

setup.cfg

@@ -42,7 +42,7 @@ install_requires =
 	werkzeug<2.1.0
 	Babel>=1.3,<3.0
 	Flask-Babel>=0.11.1,<2.1.0
-	Flask-Login>=0.3.2,<0.5.1
+	Flask-Login>=0.3.2,<0.6.1
 	Flask-Principal>=0.3.2,<0.5.1
 	backports_abc>=0.4
 	Flask>=1.0.2,<2.1.0