From 2f515b725b7cec83caa558d84dd2895d9ce01766 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 20 Jul 2024 18:14:54 +0200 Subject: [PATCH 1/7] Schedules reconnect is now working even if reconnect route is not available ( #3107) --- cps/tasks/database.py | 18 +++++------------- cps/tasks/thumbnail.py | 1 - requirements.txt | 3 +-- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/cps/tasks/database.py b/cps/tasks/database.py index e5aa26da..c9c30d43 100644 --- a/cps/tasks/database.py +++ b/cps/tasks/database.py @@ -16,11 +16,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from urllib.request import urlopen - from flask_babel import lazy_gettext as N_ -from cps import config, logger +from cps import config, logger, db, ub from cps.services.worker import CalibreTask @@ -28,18 +26,12 @@ class TaskReconnectDatabase(CalibreTask): def __init__(self, task_message=N_('Reconnecting Calibre database')): super(TaskReconnectDatabase, self).__init__(task_message) self.log = logger.create() - self.listen_address = config.get_config_ipaddress() - self.listen_port = config.config_port + self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True) def run(self, worker_thread): - address = self.listen_address if self.listen_address else 'localhost' - port = self.listen_port if self.listen_port else 8083 - - try: - urlopen('http://' + address + ':' + str(port) + '/reconnect') - self._handleSuccess() - except Exception as ex: - self._handleError('Unable to reconnect Calibre database: ' + str(ex)) + self.calibre_db.reconnect_db(config, ub.app_DB_path) + self.calibre_db.session.close() + self._handleSuccess() @property def name(self): diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index 2921ffc9..d2179dd9 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -70,7 +70,6 @@ class TaskGenerateCoverThumbnails(CalibreTask): self.log = logger.create() self.book_id = book_id self.app_db_session = ub.get_new_session_instance() - # self.calibre_db = db.CalibreDB(expire_on_commit=False, init=True) self.cache = fs.FileSystem() self.resolutions = [ constants.COVER_THUMBNAIL_SMALL, diff --git a/requirements.txt b/requirements.txt index fd6a808c..0e3f86f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,6 @@ Werkzeug<3.0.0 APScheduler>=3.6.3,<3.11.0 Babel>=1.3,<3.0 Flask-Babel>=0.11.1,<4.1.0 -# Flask-Login>=0.3.2,<0.6.4 Flask-Principal>=0.3.2,<0.5.1 Flask>=1.0.2,<3.1.0 iso-639>=0.4.5,<0.5.0 @@ -21,4 +20,4 @@ Flask-Limiter>=2.3.0,<3.6.0 regex>=2022.3.2,<2024.6.25 bleach>=6.0.0,<6.2.0 python-magic>=0.4.27,<0.5.0 -flask-httpAuth>=4.4.0 +flask-httpAuth>=4.4.0,<5.0.0 From e33e4ac4acf4b8acd5baed0610502c5544afae1a Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 21 Jul 2024 09:44:32 +0200 Subject: [PATCH 2/7] Bugfix upload book format with language to book without language --- cps/editbooks.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index 84299c63..5f58cf81 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -134,8 +134,9 @@ def edit_book(book_id): # handle upload other formats from local disk meta = upload_single_file(request, book, book_id) # only merge metadata if file was uploaded and no error occurred (meta equals not false or none) + upload_format = False if meta: - merge_metadata(to_save, meta) + upload_format = merge_metadata(to_save, meta) # handle upload covers from local disk cover_upload_success = upload_cover(request, book) if cover_upload_success: @@ -179,7 +180,7 @@ def edit_book(book_id): modify_date |= edit_book_publisher(to_save['publisher'], book) # handle book languages try: - modify_date |= edit_book_languages(to_save['languages'], book) + modify_date |= edit_book_languages(to_save['languages'], book, upload_format) except ValueError as e: flash(str(e), category="error") edit_error = True @@ -575,6 +576,10 @@ def merge_metadata(to_save, meta): to_save['author_name'] = '' if to_save.get('book_title', "") == _('Unknown'): to_save['book_title'] = '' + if not to_save["languages"] and meta.languages: + upload_language = True + else: + upload_language = False for s_field, m_field in [ ('tags', 'tags'), ('author_name', 'author'), ('series', 'series'), ('series_index', 'series_id'), ('languages', 'languages'), @@ -582,7 +587,7 @@ def merge_metadata(to_save, meta): to_save[s_field] = to_save[s_field] or getattr(meta, m_field, '') to_save["description"] = to_save["description"] or Markup( getattr(meta, 'description', '')).unescape() - + return upload_language def identifier_list(to_save, book): """Generate a list of Identifiers from form information""" From e3b895755d2d96031166695ceee96c7f9e626cf5 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 20 Jul 2024 13:58:53 +0200 Subject: [PATCH 3/7] Update test results --- test/Calibre-Web TestSummary_Linux.html | 124 ++++++------------------ 1 file changed, 32 insertions(+), 92 deletions(-) diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 5b79eaf6..288c7443 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2024-07-11 20:34:31

+

Start Time: 2024-07-18 20:53:44

-

Stop Time: 2024-07-12 03:34:37

+

Stop Time: 2024-07-19 03:48:09

-

Duration: 5h 47 min

+

Duration: 5h 43 min

@@ -2009,7 +2009,7 @@ 
Traceback (most recent call last):
   File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_metadata.py", line 90, in test_load_metadata
-    elif 'https://amazon.com/' == results[20]['source']:
+    elif len(results)>19 and 'https://amazon.com/' == results[20]['source']:
 IndexError: list index out of range
@@ -2021,11 +2021,11 @@ IndexError: list index out of range - + TestEditBooksOnGdrive 18 - 16 - 2 + 18 + 0 0 0 @@ -2152,31 +2152,11 @@ IndexError: list index out of range - +
TestEditBooksOnGdrive - test_edit_rating
- -
- FAIL -
- - - - + PASS @@ -2208,31 +2188,11 @@ AssertionError: 1 != 0 - +
TestEditBooksOnGdrive - test_watch_metadata
- -
- FAIL -
- - - - + PASS @@ -3340,11 +3300,11 @@ AssertionError: False is not true - + TestLogin 19 - 18 - 1 + 19 + 0 0 0 @@ -3498,31 +3458,11 @@ AssertionError: False is not true - +
TestLogin - test_proxy_login_multi_user
- -
- FAIL -
- - - - + PASS @@ -5674,8 +5614,8 @@ ModuleNotFoundError: No module named 'build_release' Total 499 - 484 - 3 + 487 + 0 2 10   @@ -5705,7 +5645,7 @@ ModuleNotFoundError: No module named 'build_release' Platform - Linux 6.5.0-41-generic #41~22.04.2-Ubuntu SMP PREEMPT_DYNAMIC Mon Jun 3 11:32:55 UTC 2 x86_64 x86_64 + Linux 6.5.0-44-generic #44~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Tue Jun 18 14:36:16 UTC 2 x86_64 x86_64 Basic @@ -5758,14 +5698,14 @@ ModuleNotFoundError: No module named 'build_release' - Flask-Limiter - 3.5.1 + Flask-HTTPAuth + 4.8.0 Basic - Flask-Login - 0.6.3 + Flask-Limiter + 3.5.1 Basic @@ -5891,7 +5831,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestBackupMetadataGdrive @@ -5921,7 +5861,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestCliGdrivedb @@ -5951,7 +5891,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestEbookConvertCalibreGDrive @@ -5981,7 +5921,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestEbookConvertGDriveKepubify @@ -6029,7 +5969,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestEditAuthorsGdrive @@ -6065,7 +6005,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestEditBooksOnGdrive @@ -6107,7 +6047,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestEmbedMetadataGdrive @@ -6137,7 +6077,7 @@ ModuleNotFoundError: No module named 'build_release' PyDrive2 - 1.19.0 + 1.20.0 TestSetupGdrive @@ -6209,7 +6149,7 @@ ModuleNotFoundError: No module named 'build_release' From a0728b07d05785e60995656cf894c86ff96874ff Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 21 Jul 2024 13:46:22 +0200 Subject: [PATCH 4/7] Fix wrong quotes on calibre binary call for windows --- cps/embed_helper.py | 2 +- cps/tasks/convert.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cps/embed_helper.py b/cps/embed_helper.py index 71de216d..54db47ef 100644 --- a/cps/embed_helper.py +++ b/cps/embed_helper.py @@ -28,7 +28,7 @@ log = logger.create() def do_calibre_export(book_id, book_format): try: - quotes = [3, 5, 7, 9] + quotes = [4, 6] tmp_dir = get_temp_dir() calibredb_binarypath = get_calibre_binarypath("calibredb") temp_file_name = str(uuid4()) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index e6af356f..3bef81a9 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -255,7 +255,7 @@ class TaskConvert(CalibreTask): try: # path_tmp_opf = self._embed_metadata() if config.config_embed_metadata: - quotes = [3, 5] + quotes = [5] tmp_dir = get_temp_dir() calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) my_env = os.environ.copy() From f8adcbb69138a625d478b42808920bdbd3e4a7c6 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Thu, 25 Jul 2024 21:41:40 +0200 Subject: [PATCH 5/7] Make light theme default theme on epub reader Fix csp Header for reader and google drive (#3108) Testrun --- cps/helper.py | 4 +- cps/static/js/reading/epub.js | 2 +- cps/web.py | 18 +- test/Calibre-Web TestSummary_Linux.html | 1083 ++++++++++++++--------- 4 files changed, 667 insertions(+), 440 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 08fa1387..37302075 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -463,8 +463,8 @@ def rename_author_path(first_author, old_author_dir, renamed_author, calibre_pat except OSError as ex: log.error("Rename author from: %s to %s: %s", old_author_path, new_author_path, ex) log.debug(ex, exc_info=True) - return _("Rename author from: '%(src)s' to '%(dest)s' failed with error: %(error)s", - src=old_author_path, dest=new_author_path, error=str(ex)) + raise Exception(_("Rename author from: '%(src)s' to '%(dest)s' failed with error: %(error)s", + src=old_author_path, dest=new_author_path, error=str(ex))) return new_authordir # Moves files in file storage during author/title rename, or from temp dir to file storage diff --git a/cps/static/js/reading/epub.js b/cps/static/js/reading/epub.js index 5047b1d3..a552da05 100644 --- a/cps/static/js/reading/epub.js +++ b/cps/static/js/reading/epub.js @@ -79,6 +79,6 @@ var reader; } // Default settings load - const theme = localStorage.getItem("calibre.reader.theme") ?? Object.keys(themes)[0]; + const theme = localStorage.getItem("calibre.reader.theme") ?? "lightTheme"; selectTheme(theme); })(); diff --git a/cps/web.py b/cps/web.py index 7da22e3a..2519ebd5 100644 --- a/cps/web.py +++ b/cps/web.py @@ -89,21 +89,21 @@ except ImportError: def add_security_headers(resp): default_src = ([host.strip() for host in config.config_trustedhosts.split(',') if host] + ["'self'", "'unsafe-inline'", "'unsafe-eval'"]) - csp = "default-src " + ' '.join(default_src) + "; " - csp += "font-src 'self' data:" + csp = "default-src " + ' '.join(default_src) + if request.endpoint == "web.read_book" and config.config_use_google_drive: + csp +=" blob: " + csp += "; font-src 'self' data:" if request.endpoint == "web.read_book": - csp += " blob:" + csp += " blob: " csp += "; img-src 'self'" if request.path.startswith("/author/") and config.config_use_goodreads: csp += " images.gr-assets.com i.gr-assets.com s.gr-assets.com" csp += " data:" if request.endpoint == "edit-book.show_edit_book" or config.config_use_google_drive: - csp += " *;" - elif request.endpoint == "web.read_book": - csp += " blob:; style-src-elem 'self' blob: 'unsafe-inline';" - else: - csp += ";" - csp += " object-src 'none';" + csp += " *" + if request.endpoint == "web.read_book": + csp += " blob: ; style-src-elem 'self' blob: 'unsafe-inline'" + csp += "; object-src 'none';" resp.headers['Content-Security-Policy'] = csp resp.headers['X-Content-Type-Options'] = 'nosniff' resp.headers['X-Frame-Options'] = 'SAMEORIGIN' diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 288c7443..1bdb3396 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2024-07-18 20:53:44

+

Start Time: 2024-07-24 19:37:58

-

Stop Time: 2024-07-19 03:48:09

+

Stop Time: 2024-07-25 02:52:34

-

Duration: 5h 43 min

+

Duration: 6h 1 min

@@ -1791,15 +1791,15 @@ - - TestEditBooksList - 19 - 19 - 0 - 0 + + TestEditAuthorsSmb + 9 + 3 + 4 + 2 0 - Detail + Detail @@ -1807,52 +1807,132 @@ -
TestEditBooksList - test_booklist_xss
+
TestEditAuthorsSmb - test_change_capital_co_author
PASS - + -
TestEditBooksList - test_bookslist_edit_author
+
TestEditAuthorsSmb - test_change_capital_one_author_one_book
+ + +
+ FAIL +
+ + + - PASS - + -
TestEditBooksList - test_bookslist_edit_categories
+
TestEditAuthorsSmb - test_change_capital_one_author_two_books
+ + +
+ FAIL +
+ + + - PASS - + -
TestEditBooksList - test_bookslist_edit_comment
+
TestEditAuthorsSmb - test_change_capital_one_author_two_books_coauthor
+ + +
+ FAIL +
+ + + - PASS - + -
TestEditBooksList - test_bookslist_edit_cust_category
+
TestEditAuthorsSmb - test_change_capital_rename_co_author
+ + +
+ FAIL +
+ + + - PASS -
TestEditBooksList - test_bookslist_edit_cust_comment
+
TestEditAuthorsSmb - test_change_capital_rename_two_co_authors
PASS @@ -1860,6 +1940,153 @@ + +
TestEditAuthorsSmb - test_rename_author_emphasis_mark_onupload
+ + PASS + + + + + + +
TestEditAuthorsSmb - test_rename_capital_on_upload
+ + +
+ ERROR +
+ + + + + + + + + + +
TestEditAuthorsSmb - test_rename_tag_emphasis_mark_onupload
+ + +
+ ERROR +
+ + + + + + + + + + + TestEditBooksList + 19 + 19 + 0 + 0 + 0 + + Detail + + + + + + + +
TestEditBooksList - test_booklist_xss
+ + PASS + + + + + + +
TestEditBooksList - test_bookslist_edit_author
+ + PASS + + + + + + +
TestEditBooksList - test_bookslist_edit_categories
+ + PASS + + + + + + +
TestEditBooksList - test_bookslist_edit_comment
+ + PASS + + + + + + +
TestEditBooksList - test_bookslist_edit_cust_category
+ + PASS + + + + + + +
TestEditBooksList - test_bookslist_edit_cust_comment
+ + PASS + + + + +
TestEditBooksList - test_bookslist_edit_cust_enum
@@ -1868,7 +2095,7 @@ - +
TestEditBooksList - test_bookslist_edit_cust_float
@@ -1877,7 +2104,7 @@ - +
TestEditBooksList - test_bookslist_edit_cust_int
@@ -1886,7 +2113,7 @@ - +
TestEditBooksList - test_bookslist_edit_cust_ratings
@@ -1895,7 +2122,7 @@ - +
TestEditBooksList - test_bookslist_edit_cust_text
@@ -1904,7 +2131,7 @@ - +
TestEditBooksList - test_bookslist_edit_languages
@@ -1913,7 +2140,7 @@ - +
TestEditBooksList - test_bookslist_edit_publisher
@@ -1922,7 +2149,7 @@ - +
TestEditBooksList - test_bookslist_edit_series
@@ -1931,7 +2158,7 @@ - +
TestEditBooksList - test_bookslist_edit_seriesindex
@@ -1940,7 +2167,7 @@ - +
TestEditBooksList - test_bookslist_edit_title
@@ -1949,7 +2176,7 @@ - +
TestEditBooksList - test_list_visibility
@@ -1958,7 +2185,7 @@ - +
TestEditBooksList - test_restricted_rights
@@ -1967,7 +2194,7 @@ - +
TestEditBooksList - test_search_books_list
@@ -1977,40 +2204,40 @@ - + TestLoadMetadata 1 0 - 0 1 0 + 0 - Detail + Detail - +
TestLoadMetadata - test_load_metadata
- ERROR + FAIL
-