Merge branch 'master' into Develop

cps/__init__.py

@@ -58,8 +58,8 @@ mimetypes.add_type('application/epub+zip', '.kepub')
 mimetypes.add_type('application/fb2+zip', '.fb2')
 mimetypes.add_type('application/x-mobipocket-ebook', '.mobi')
 mimetypes.add_type('application/octet-stream', '.prc')
-mimetypes.add_type('application/vnd.amazon.ebook', '.azw')
-mimetypes.add_type('application/x-mobi8-ebook', '.azw3')
+mimetypes.add_type('application/x-mobipocket-ebook', '.azw')
+mimetypes.add_type('application/x-mobipocket-ebook', '.azw3')
 mimetypes.add_type('application/x-cbr', '.cbr')
 mimetypes.add_type('application/x-cbz', '.cbz')
 mimetypes.add_type('application/x-tar', '.cbt')
@@ -77,7 +77,7 @@ mimetypes.add_type('audio/ogg', '.ogg')
 mimetypes.add_type('application/ogg', '.oga')
 mimetypes.add_type('text/css', '.css')
 mimetypes.add_type('application/x-ms-reader', '.lit')
-mimetypes.add_type('text/javascript; charset=UTF-8', '.js')
+mimetypes.add_type('text/javascript', '.js')
 mimetypes.add_type('text/rtf', '.rtf')
 
 log = logger.create()

cps/admin.py

@@ -425,7 +425,7 @@ def table_get_locale():
     current_locale = get_locale()
     for loc in locale:
         ret.append({'value': str(loc), 'text': loc.get_language_name(current_locale)})
-    return json.dumps(ret)
+    return json.dumps(sorted(ret, key=lambda x: x['text']))
 
 
 @admi.route("/ajax/getdefaultlanguage")
@@ -437,7 +437,7 @@ def table_get_default_lang():
     ret.append({'value': 'all', 'text': _('Show All')})
     for lang in languages:
         ret.append({'value': lang.lang_code, 'text': lang.name})
-    return json.dumps(ret)
+    return json.dumps(sorted(ret, key=lambda x: x['text']))
 
 
 @admi.route("/ajax/editlistusers/<param>", methods=['POST'])

cps/cover.py

@@ -36,7 +36,7 @@ def cover_processing(tmp_file_path, img, extension):
         if use_IM:
             with Image(blob=img) as imgc:
                 imgc.format = 'jpeg'
-                imgc.transform_colorspace('rgb')
+                imgc.transform_colorspace('srgb')
                 imgc.save(filename=tmp_cover_name)
                 return tmp_cover_name
         else:

cps/cw_babel.py

@@ -34,7 +34,7 @@ def get_user_locale_language(user_language):
 
 
 def get_available_locale():
-    return [Locale('en')] + babel.list_translations()
+    return sorted(babel.list_translations(), key=lambda x: x.display_name.lower())
 
 
 def get_available_translations():

cps/epub.py

@@ -56,7 +56,7 @@ def get_epub_layout(book, book_data):
         p = tree.xpath('/pkg:package/pkg:metadata', namespaces=default_ns)[0]
 
         layout = p.xpath('pkg:meta[@property="rendition:layout"]/text()', namespaces=default_ns)
-    except (etree.XMLSyntaxError, KeyError, IndexError, OSError) as e:
+    except (etree.XMLSyntaxError, KeyError, IndexError, OSError, UnicodeDecodeError) as e:
         log.error("Could not parse epub metadata of book {} during kobo sync: {}".format(book.id, e))
         layout = []
 

cps/helper.py

@@ -237,7 +237,7 @@ def send_mail(book_id, book_format, convert, ereader_mail, calibrepath, user_id)
     return _("The requested file could not be read. Maybe wrong permissions?")
 
 
-def get_valid_filename(value, replace_whitespace=True, chars=128):
+def get_valid_filename(value, replace_whitespace=True, chars=128, force_unidecode=False):
     """
     Returns the given string converted to a string that can be used for a clean
     filename. Limits num characters to 128 max.
@@ -245,7 +245,7 @@ def get_valid_filename(value, replace_whitespace=True, chars=128):
     if value[-1:] == '.':
         value = value[:-1]+'_'
     value = value.replace("/", "_").replace(":", "_").strip('\0')
-    if config.config_unicode_filename:
+    if config.config_unicode_filename or force_unidecode:
         value = (unidecode.unidecode(value))
     if replace_whitespace:
         #  *+:\"/<>? are replaced by _
@@ -891,7 +891,7 @@ def save_cover(img, book_path):
             else:
                 imgc = Image(blob=io.BytesIO(img.content))
             imgc.format = 'jpeg'
-            imgc.transform_colorspace("rgb")
+            imgc.transform_colorspace("srgb")
             img = imgc
         except (BlobError, MissingDelegateError):
             log.error("Invalid cover file content")
@@ -1091,11 +1091,14 @@ def get_download_link(book_id, book_format, client):
             file_name = book.title
             if len(book.authors) > 0:
                 file_name = file_name + ' - ' + book.authors[0].name
-            file_name = get_valid_filename(file_name, replace_whitespace=False)
+            if client == "kindle":
+                file_name = get_valid_filename(file_name, replace_whitespace=False, force_unidecode=True)
+            else:
+                file_name = quote(get_valid_filename(file_name, replace_whitespace=False))
             headers = Headers()
             headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream")
-            headers["Content-Disposition"] = "attachment; filename=%s.%s; filename*=UTF-8''%s.%s" % (
-                quote(file_name), book_format, quote(file_name), book_format)
+            headers["Content-Disposition"] = ('attachment; filename="{}.{}"; filename*=UTF-8\'\'{}.{}').format(
+                file_name, book_format, file_name, book_format)
             return do_download_file(book, book_format, client, data1, headers)
     else:
         log.error("Book id {} not found for downloading".format(book_id))

cps/kobo.py

@@ -106,10 +106,12 @@ def make_request_to_kobo_store(sync_token=None):
     return store_response
 
 
-def redirect_or_proxy_request():
+def redirect_or_proxy_request(auth=False):
     if config.config_kobo_proxy:
+        try:
             if request.method == "GET":
-            return redirect(get_store_url_for_current_request(), 307)
+                alfa = redirect(get_store_url_for_current_request(), 307)
+                return alfa
             else:
                 # The Kobo device turns other request types into GET requests on redirects,
                 # so we instead proxy to the Kobo store ourselves.
@@ -122,7 +124,10 @@ def redirect_or_proxy_request():
                 return make_response(
                     store_response.content, store_response.status_code, response_headers.items()
                 )
-    else:
+        except Exception as e:
+            log.error("Failed to receive or parse response from Kobo's endpoint: {}".format(e))
+            if auth:
+                return make_calibre_web_auth_response()
     return make_response(jsonify({}))
 
 
@@ -1042,7 +1047,7 @@ def HandleAuthRequest():
     log.debug('Kobo Auth request')
     if config.config_kobo_proxy:
         try:
-            return redirect_or_proxy_request()
+            return redirect_or_proxy_request(auth=True)
         except Exception:
             log.error("Failed to receive or parse response from Kobo's auth endpoint. Falling back to un-proxied mode.")
     return make_calibre_web_auth_response()

cps/logger.py

@@ -29,7 +29,7 @@ from .constants import CONFIG_DIR as _CONFIG_DIR
 ACCESS_FORMATTER_GEVENT  = Formatter("%(message)s")
 ACCESS_FORMATTER_TORNADO = Formatter("[%(asctime)s] %(message)s")
 
-FORMATTER           = Formatter("[%(asctime)s] %(levelname)5s {%(name)s:%(lineno)d} %(message)s")
+FORMATTER           = Formatter("[%(asctime)s] %(levelname)5s {%(filename)s:%(lineno)d} %(message)s")
 DEFAULT_LOG_LEVEL   = logging.INFO
 DEFAULT_LOG_FILE    = os.path.join(_CONFIG_DIR, "calibre-web.log")
 DEFAULT_ACCESS_LOG  = os.path.join(_CONFIG_DIR, "access.log")
@@ -42,18 +42,12 @@ logging.addLevelName(logging.CRITICAL, "CRIT")
 
 class _Logger(logging.Logger):
 
-    def error_or_exception(self, message, stacklevel=2, *args, **kwargs):
+    def error_or_exception(self, message, stacklevel=1, *args, **kwargs):
         is_debug = self.getEffectiveLevel() <= logging.DEBUG
-        if sys.version_info > (3, 7):
-            if is_debug:
+        if not is_debug:
             self.exception(message, stacklevel=stacklevel, *args, **kwargs)
         else:
             self.error(message, stacklevel=stacklevel, *args, **kwargs)
-        else:
-            if is_debug:
-                self.exception(message, stack_info=True, *args, **kwargs)
-            else:
-                self.error(message, *args, **kwargs)
 
     def debug_no_auth(self, message, *args, **kwargs):
         message = message.strip("\r\n")

cps/static/css/caliBlur.css

@@ -7153,12 +7153,11 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
     }
 
     body.editbook > div.container-fluid > div.row-fluid > div.col-sm-10 > div.col-sm-3, body.upload > div.container-fluid > div.row-fluid > div.col-sm-10 > div.col-sm-3 {
-        max-width: 130px;
-        width: 130px;
-        height: 180px;
-        margin: 0;
+        position: relative;
+        max-width: unset;
+        width: 100%;
+        height: unset;
         padding: 15px;
-        position: absolute
     }
 
     body.editbook > div.container-fluid > div.row-fluid > div.col-sm-10 > form > div.col-sm-9, body.upload > div.container-fluid > div.row-fluid > div.col-sm-10 > form > div.col-sm-9 {
@@ -7167,10 +7166,6 @@ body.edituser.admin > div.container-fluid > div.row-fluid > div.col-sm-10 > div.
         width: 100%
     }
 
-    body.editbook > div.container-fluid > div.row-fluid > div.col-sm-10 > form > div.col-sm-9 > .form-group:nth-child(1), body.editbook > div.container-fluid > div.row-fluid > div.col-sm-10 > form > div.col-sm-9 > .form-group:nth-child(2), body.upload > div.container-fluid > div.row-fluid > div.col-sm-10 > form > div.col-sm-9 > .form-group:nth-child(1), body.upload > div.container-fluid > div.row-fluid > div.col-sm-10 > form > div.col-sm-9 > .form-group:nth-child(2) {
-        padding-left: 120px
-    }
-
     #deleteButton, body.editbook > div.container-fluid > div.row-fluid > div.col-sm-10 > div.col-sm-3 > div.text-center > #delete, body.upload > div.container-fluid > div.row-fluid > div.col-sm-10 > div.col-sm-3 > div.text-center > #delete {
         top: 48px;
         height: 42px

cps/static/js/libs/pdf.js

@@ -16667,7 +16667,7 @@ const PDFWorkerUtil = {
 {
   if (isNodeJS) {
     PDFWorkerUtil.isWorkerDisabled = true;
-    GlobalWorkerOptions.workerSrc ||= "./pdf.worker.mjs";
+    GlobalWorkerOptions.workerSrc ||= "./pdf.worker.js";
   }
   PDFWorkerUtil.isSameOrigin = function (baseUrl, otherUrl) {
     let base;

cps/static/js/libs/viewer.js

@@ -828,7 +828,7 @@ const defaultOptions = {
     kind: OptionKind.WORKER
   },
   workerSrc: {
-    value: "../build/pdf.worker.mjs",
+    value: "../build/pdf.worker.js",
     kind: OptionKind.WORKER
   }
 };

cps/tasks/mail.py

@@ -25,7 +25,7 @@ import mimetypes
 
 from io import StringIO
 from email.message import EmailMessage
-from email.utils import formatdate, parseaddr
+from email.utils import formatdate, parseaddr, make_msgid
 from email.generator import Generator
 from flask_babel import lazy_gettext as N_
 
@@ -35,7 +35,7 @@ from cps.embed_helper import do_calibre_export
 from cps import logger, config
 from cps import gdriveutils
 from cps.string_helper import strip_whitespaces
-import uuid
+
 
 log = logger.create()
 
@@ -56,7 +56,7 @@ class EmailBase:
 
     def send(self, strg):
         """Send `strg' to the server."""
-        log.debug_no_auth('send: {}'.format(strg[:300]))
+        log.debug_no_auth('send: {}'.format(strg[:300]), stacklevel=2)
         if hasattr(self, 'sock') and self.sock:
             try:
                 if self.transferSize:
@@ -142,7 +142,7 @@ class TaskEmail(CalibreTask):
         message['To'] = self.recipient
         message['Subject'] = self.subject
         message['Date'] = formatdate(localtime=True)
-        message['Message-Id'] = "{}@{}".format(uuid.uuid4(), self.get_msgid_domain())
+        message['Message-ID'] = make_msgid(domain=self.get_msgid_domain())
         message.set_content(self.text.encode('UTF-8'), "text", "plain")
         if self.attachment:
             data = self._get_attachment(self.filepath, self.attachment)
@@ -169,10 +169,14 @@ class TaskEmail(CalibreTask):
             else:
                 self.send_gmail_email(msg)
         except MemoryError as e:
-            log.error_or_exception(e, stacklevel=3)
+            log.error_or_exception(e, stacklevel=2)
             self._handleError('MemoryError sending e-mail: {}'.format(str(e)))
+        except (smtplib.SMTPRecipientsRefused) as e:
+            log.error_or_exception(e, stacklevel=2)
+            self._handleError('Smtplib Error sending e-mail: {}'.format(
+                (list(e.args[0].values())[0][1]).decode('utf-8)').replace("\n", '. ')))
         except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
-            log.error_or_exception(e, stacklevel=3)
+            log.error_or_exception(e, stacklevel=2)
             if hasattr(e, "smtp_error"):
                 text = e.smtp_error.decode('utf-8').replace("\n", '. ')
             elif hasattr(e, "message"):
@@ -183,10 +187,10 @@ class TaskEmail(CalibreTask):
                 text = ''
             self._handleError('Smtplib Error sending e-mail: {}'.format(text))
         except (socket.error) as e:
-            log.error_or_exception(e, stacklevel=3)
+            log.error_or_exception(e, stacklevel=2)
             self._handleError('Socket Error sending e-mail: {}'.format(e.strerror))
         except Exception as ex:
-            log.error_or_exception(ex, stacklevel=3)
+            log.error_or_exception(ex, stacklevel=2)
             self._handleError('Error sending e-mail: {}'.format(ex))
 
     def send_standard_email(self, msg):
@@ -269,7 +273,7 @@ class TaskEmail(CalibreTask):
                 if config.config_binariesdir and config.config_embed_metadata:
                     os.remove(datafile)
             except IOError as e:
-                log.error_or_exception(e, stacklevel=3)
+                log.error_or_exception(e, stacklevel=2)
                 log.error('The requested file could not be read. Maybe wrong permissions?')
                 return None
         return data

cps/templates/config_db.html

@@ -62,7 +62,7 @@
       {% endif %}
     {% endif %}
     <div class="col-sm-12">
-      <div id="db_submit" name="submit" class="btn btn-default">{{_('Save')}}</div>
+      <button id="db_submit" type="submit" name="submit" class="btn btn-default">{{_('Save')}}</button>
       <a href="{{ url_for('admin.admin') }}" id="config_back" class="btn btn-default">{{_('Cancel')}}</a>
     </div>
   </form>

cps/templates/readpdf.html

@@ -35,7 +35,7 @@ See https://github.com/adobe-type-tools/cmap-resources
     <link rel="stylesheet" href="{{ url_for('static', filename='css/libs/viewer.css') }}">
 	  <!-- This snippet is used in production (included from viewer.html) -->
 	  <link rel="resource" type="application/l10n" href="{{ url_for('static', filename='locale/locale.json') }}">
-	  <script src="{{ url_for('static', filename='js/libs/pdf.mjs') }}" type="module"></script>
+	  <script src="{{ url_for('static', filename='js/libs/pdf.js') }}" type="module"></script>
 
     <script type="text/javascript">
     window.addEventListener('webviewerloaded', function() {
@@ -46,12 +46,12 @@ See https://github.com/adobe-type-tools/cmap-resources
         PDFViewerApplicationOptions.set('cMapUrl', "{{ url_for('static', filename='cmaps/') }}");
         PDFViewerApplicationOptions.set('sidebarViewOnLoad', 0);
         PDFViewerApplicationOptions.set('imageResourcesPath', "{{ url_for('static', filename='css/images/') }}");
-        PDFViewerApplicationOptions.set('workerSrc', "{{ url_for('static', filename='js/libs/pdf.worker.mjs') }}");
+        PDFViewerApplicationOptions.set('workerSrc', "{{ url_for('static', filename='js/libs/pdf.worker.js') }}");
         PDFViewerApplicationOptions.set('defaultUrl',"{{ url_for('web.serve_book', book_id=pdffile, book_format='pdf') }}")
     });
     </script>
 
-    <script src="{{ url_for('static', filename='js/libs/viewer.mjs') }}" type="module">></script>
+    <script src="{{ url_for('static', filename='js/libs/viewer.js') }}" type="module">></script>
 
   </head>
 

cps/web.py

@@ -60,7 +60,7 @@ from .services.worker import WorkerThread
 from .tasks_status import render_task_status
 from .usermanagement import user_login_required
 from .string_helper import strip_whitespaces
-
+import traceback
 
 feature_support = {
     'ldap': bool(services.ldap),
@@ -1242,7 +1242,12 @@ def serve_book(book_id, book_format, anyname):
 @login_required_if_no_ano
 @download_required
 def download_link(book_id, book_format, anyname):
-    client = "kobo" if "Kobo" in request.headers.get('User-Agent') else ""
+    if "kindle" in request.headers.get('User-Agent').lower():
+        client = "kindle"
+    elif "Kobo" in request.headers.get('User-Agent').lower():
+        client = "kobo"
+    else:
+        client = ""
     return get_download_link(book_id, book_format, client)
 
 

optional-requirements.txt

@@ -1,11 +1,11 @@
 # GDrive Integration
 google-api-python-client>=1.7.11,<2.200.0
-gevent>20.6.0,<24.3.0
+gevent>20.6.0,<24.12.0
 greenlet>=0.4.17,<3.2.0
 httplib2>=0.9.2,<0.23.0
 oauth2client>=4.0.0,<4.1.4
 uritemplate>=3.0.0,<4.2.0
-pyasn1-modules>=0.0.8,<0.5.0
+pyasn1-modules>=0.0.8,<0.7.0
 pyasn1>=0.1.9,<0.7.0
 PyDrive2>=1.3.1,<1.22.0
 PyYAML>=3.12,<6.1
@@ -17,23 +17,23 @@ google-api-python-client>=1.7.11,<2.200.0
 
 # goodreads
 goodreads>=0.3.2,<0.4.0
-python-Levenshtein>=0.12.0,<0.27.0
+python-Levenshtein>=0.12.0,<0.28.0
 
 # ldap login
 python-ldap>=3.0.0,<3.5.0
 Flask-SimpleLDAP>=1.4.0,<2.1.0
 
 # oauth
-Flask-Dance>=2.0.0,<7.1.0
+Flask-Dance>=2.0.0,<7.2.0
 SQLAlchemy-Utils>=0.33.5,<0.42.0
 
 # metadata extraction
 rarfile>=3.2,<5.0
 scholarly>=1.2.0,<1.8
-markdown2>=2.0.0,<2.5.0
-html2text>=2020.1.16,<2024.2.26
+markdown2>=2.0.0,<2.6.0
+html2text>=2020.1.16,<2025.2.26
 python-dateutil>=2.1,<2.10.0
-beautifulsoup4>=4.0.1,<4.13.0
+beautifulsoup4>=4.0.1,<4.14.0
 faust-cchardet>=2.1.18,<2.1.20
 py7zr>=0.15.0,<0.21.0
 mutagen>=1.40.0,<1.50.0

pyproject.toml

@@ -12,12 +12,11 @@ classifiers = [
     "Development Status :: 5 - Production/Stable",
     "License :: OSI Approved :: GNU Affero General Public License v3",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.6",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Operating System :: OS Independent",
 ]
 keywords = [

requirements.txt

@@ -1,26 +1,26 @@
-APScheduler>=3.6.3,<3.11.0
+APScheduler>=3.6.3,<3.12.0
 Babel>=1.3,<3.0
-Flask-Babel>=0.11.1,<4.1.0
+Flask-Babel>=3.0.0,<4.1.0
 Flask-Principal>=0.3.2,<0.5.1
-Flask>=1.0.2,<3.1.0
+Flask>=1.0.2,<3.2.0
 iso-639>=0.4.5,<0.5.0;python_version<'3.12'
 pycountry>=20.0.0,<25.0.0;python_version>='3.12'
-PyPDF>=3.15.6,<5.1.0
+PyPDF>=3.15.6,<5.5.0
 pytz>=2016.10
 requests>=2.32.0,<2.33.0
 SQLAlchemy>=1.3.0,<2.1.0
-tornado>=6.3,<6.5
+tornado>=6.4.2,<6.6
 Wand>=0.4.4,<0.7.0
 unidecode>=0.04.19,<1.4.0
-lxml>=4.9.1,<5.3.0
+lxml>=4.9.1,<5.4.0
 flask-wtf>=0.14.2,<1.3.0
 chardet>=3.0.0,<5.3.0
 netifaces-plus>=0.12.0,<0.13.0
 urllib3>=1.22,<3.0
-Flask-Limiter>=2.3.0,<3.9.0
-regex>=2022.3.2,<2024.6.25
+Flask-Limiter>=2.3.0,<3.13.0
+regex>=2022.3.2,<2025.3.20
 bleach>=6.0.0,<6.2.0
 python-magic>=0.4.27,<0.5.0
 python-magic-bin>=0.4.0,<0.5.0;sys_platform=='win32'
 flask-httpAuth>=4.4.0,<5.0.0
-cryptography>=30.0.0,<44.0.0
+cryptography>=43.0.4,<45.0.0