From 0ce41aef56c718e37114ba1ece97c1ab5bed2267 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 5 Feb 2022 19:05:14 +0100 Subject: [PATCH 1/6] Detect missing database file also in gdrive mode --- cps/config_sql.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cps/config_sql.py b/cps/config_sql.py index ebc4ca24..5b94fc02 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -304,9 +304,8 @@ class _ConfigSQL(object): have_metadata_db = bool(self.config_calibre_dir) if have_metadata_db: - if not self.config_use_google_drive: - db_file = os.path.join(self.config_calibre_dir, 'metadata.db') - have_metadata_db = os.path.isfile(db_file) + db_file = os.path.join(self.config_calibre_dir, 'metadata.db') + have_metadata_db = os.path.isfile(db_file) self.db_configured = have_metadata_db constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip().lower() for x in self.config_upload_formats.split(',')] if os.environ.get('FLASK_DEBUG'): From 41f89af9595afc8fe617cdca4f316bb32ccb615b Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 6 Feb 2022 14:23:35 +0100 Subject: [PATCH 2/6] Log error in case gdrive.db folder can't be found --- cps/gdriveutils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index eae175a0..57f272ff 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -196,7 +196,8 @@ def migrate(): if not os.path.exists(cli.gdpath): try: Base.metadata.create_all(engine) - except Exception: + except Exception as ex: + log.error("Error connect to database: {} - {}".format(cli.gdpath, ex)) raise migrate() From 411c13977f030566bdf9505dfce63f5b580f40f9 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 6 Feb 2022 16:22:28 +0100 Subject: [PATCH 3/6] Fix for #2294 (cover files in epubs with property as property in manifest are now found) --- .github/FUNDING.yml | 1 + cps/epub.py | 73 ++++++++++++++++++++++++--------------------- 2 files changed, 40 insertions(+), 34 deletions(-) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..c3d0309e --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: ["https://PayPal.Me/calibreweb",] diff --git a/cps/epub.py b/cps/epub.py index cbbdcbbd..b436a755 100644 --- a/cps/epub.py +++ b/cps/epub.py @@ -25,15 +25,14 @@ from .helper import split_authors from .constants import BookMeta - -def extractCover(zipFile, coverFile, coverpath, tmp_file_name): - if coverFile is None: +def extract_cover(zip_file, cover_file, cover_path, tmp_file_name): + if cover_file is None: return None else: - zipCoverPath = os.path.join(coverpath, coverFile).replace('\\', '/') - cf = zipFile.read(zipCoverPath) + zip_cover_path = os.path.join(cover_path, cover_file).replace('\\', '/') + cf = zip_file.read(zip_cover_path) prefix = os.path.splitext(tmp_file_name)[0] - tmp_cover_name = prefix + '.' + os.path.basename(zipCoverPath) + tmp_cover_name = prefix + '.' + os.path.basename(zip_cover_path) image = open(tmp_cover_name, 'wb') image.write(cf) image.close() @@ -47,12 +46,12 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): 'dc': 'http://purl.org/dc/elements/1.1/' } - epubZip = zipfile.ZipFile(tmp_file_path) + epub_zip = zipfile.ZipFile(tmp_file_path) - txt = epubZip.read('META-INF/container.xml') + txt = epub_zip.read('META-INF/container.xml') tree = etree.fromstring(txt) cfname = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0] - cf = epubZip.read(cfname) + cf = epub_zip.read(cfname) tree = etree.fromstring(cf) coverpath = os.path.dirname(cfname) @@ -86,9 +85,9 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): lang = epub_metadata['language'].split('-', 1)[0].lower() epub_metadata['language'] = isoLanguages.get_lang3(lang) - epub_metadata = parse_epbub_series(ns, tree, epub_metadata) + epub_metadata = parse_epub_series(ns, tree, epub_metadata) - coverfile = parse_ebpub_cover(ns, tree, epubZip, coverpath, tmp_file_path) + cover_file = parse_epub_cover(ns, tree, epub_zip, coverpath, tmp_file_path) if not epub_metadata['title']: title = original_file_name @@ -100,7 +99,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): extension=original_file_extension, title=title.encode('utf-8').decode('utf-8'), author=epub_metadata['creator'].encode('utf-8').decode('utf-8'), - cover=coverfile, + cover=cover_file, description=epub_metadata['description'], tags=epub_metadata['subject'].encode('utf-8').decode('utf-8'), series=epub_metadata['series'].encode('utf-8').decode('utf-8'), @@ -108,37 +107,43 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): languages=epub_metadata['language'], publisher="") -def parse_ebpub_cover(ns, tree, epubZip, coverpath, tmp_file_path): - coversection = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns) - coverfile = None - if len(coversection) > 0: - coverfile = extractCover(epubZip, coversection[0], coverpath, tmp_file_path) + +def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path): + cover_section = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns) + cover_file = None + if len(cover_section) > 0: + cover_file = extract_cover(epub_zip, cover_section[0], cover_path, tmp_file_path) else: meta_cover = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='cover']/@content", namespaces=ns) if len(meta_cover) > 0: - coversection = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='"+meta_cover[0]+"']/@href", namespaces=ns) + cover_section = tree.xpath( + "/pkg:package/pkg:manifest/pkg:item[@id='"+meta_cover[0]+"']/@href", namespaces=ns) + if not cover_section: + cover_section = tree.xpath( + "/pkg:package/pkg:manifest/pkg:item[@properties='" + meta_cover[0] + "']/@href", namespaces=ns) else: - coversection = tree.xpath("/pkg:package/pkg:guide/pkg:reference/@href", namespaces=ns) - if len(coversection) > 0: - filetype = coversection[0].rsplit('.', 1)[-1] + cover_section = tree.xpath("/pkg:package/pkg:guide/pkg:reference/@href", namespaces=ns) + if len(cover_section) > 0: + filetype = cover_section[0].rsplit('.', 1)[-1] if filetype == "xhtml" or filetype == "html": # if cover is (x)html format - markup = epubZip.read(os.path.join(coverpath, coversection[0])) - markupTree = etree.fromstring(markup) + markup = epub_zip.read(os.path.join(cover_path, cover_section[0])) + markup_tree = etree.fromstring(markup) # no matter xhtml or html with no namespace - imgsrc = markupTree.xpath("//*[local-name() = 'img']/@src") + img_src = markup_tree.xpath("//*[local-name() = 'img']/@src") # Alternative image source - if not len(imgsrc): - imgsrc = markupTree.xpath("//attribute::*[contains(local-name(), 'href')]") - if len(imgsrc): - # imgsrc maybe startwith "../"" so fullpath join then relpath to cwd - filename = os.path.relpath(os.path.join(os.path.dirname(os.path.join(coverpath, coversection[0])), - imgsrc[0])) - coverfile = extractCover(epubZip, filename, "", tmp_file_path) + if not len(img_src): + img_src = markup_tree.xpath("//attribute::*[contains(local-name(), 'href')]") + if len(img_src): + # img_src maybe start with "../"" so fullpath join then relpath to cwd + filename = os.path.relpath(os.path.join(os.path.dirname(os.path.join(cover_path, cover_section[0])), + img_src[0])) + cover_file = extract_cover(epub_zip, filename, "", tmp_file_path) else: - coverfile = extractCover(epubZip, coversection[0], coverpath, tmp_file_path) - return coverfile + cover_file = extract_cover(epub_zip, cover_section[0], cover_path, tmp_file_path) + return cover_file -def parse_epbub_series(ns, tree, epub_metadata): + +def parse_epub_series(ns, tree, epub_metadata): series = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series']/@content", namespaces=ns) if len(series) > 0: epub_metadata['series'] = series[0] From 0c3c0c06649629eae615a9db978eb189e03edfe7 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 6 Feb 2022 16:33:12 +0100 Subject: [PATCH 4/6] Improved logging of book title on upload --- cps/editbooks.py | 4 ++-- cps/tasks/upload.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index 4948c2dc..9fce1272 100755 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -670,7 +670,7 @@ def upload_single_file(request, book, book_id): # Queue uploader info link = '{}'.format(url_for('web.show_book', book_id=book.id), escape(book.title)) uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=link) - WorkerThread.add(current_user.name, TaskUpload(uploadText)) + WorkerThread.add(current_user.name, TaskUpload(uploadText), escape(book.title)) return uploader.process( saved_filename, *os.path.splitext(requested_file.filename), @@ -1092,7 +1092,7 @@ def upload(): flash(error, category="error") link = '{}'.format(url_for('web.show_book', book_id=book_id), escape(title)) uploadText = _(u"File %(file)s uploaded", file=link) - WorkerThread.add(current_user.name, TaskUpload(uploadText)) + WorkerThread.add(current_user.name, TaskUpload(uploadText, escape(title))) if len(request.files.getlist("btn-upload")) < 2: if current_user.role_edit() or current_user.role_admin(): diff --git a/cps/tasks/upload.py b/cps/tasks/upload.py index 2a667c28..e0bb0094 100644 --- a/cps/tasks/upload.py +++ b/cps/tasks/upload.py @@ -20,11 +20,12 @@ from datetime import datetime from cps.services.worker import CalibreTask, STAT_FINISH_SUCCESS class TaskUpload(CalibreTask): - def __init__(self, taskMessage): + def __init__(self, taskMessage, book_title): super(TaskUpload, self).__init__(taskMessage) self.start_time = self.end_time = datetime.now() self.stat = STAT_FINISH_SUCCESS self.progress = 1 + self.book_title = book_title def run(self, worker_thread): """Upload task doesn't have anything to do, it's simply a way to add information to the task list""" @@ -34,4 +35,4 @@ class TaskUpload(CalibreTask): return "Upload" def __str__(self): - return "Upload {}".format(self.message) + return "Upload {}".format(self.book_title) From 3bb41aca6de1e84b5d0e862224943719eb0ad4da Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 6 Feb 2022 18:57:58 +0100 Subject: [PATCH 5/6] Bugfix for already present mobi file during convert (fixes #1900) --- cps/tasks/convert.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 596bf95c..90513b09 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -116,8 +116,20 @@ class TaskConvert(CalibreTask): log.info("Book id %d already converted to %s", book_id, format_new_ext) cur_book = local_db.get_book(book_id) self.title = cur_book.title - self.results['path'] = file_path + self.results['path'] = cur_book.path self.results['title'] = self.title + new_format = db.Data(name=os.path.basename(file_path), + book_format=self.settings['new_book_format'].upper(), + book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) + try: + local_db.session.merge(new_format) + local_db.session.commit() + except SQLAlchemyError as e: + local_db.session.rollback() + log.error("Database error: %s", e) + local_db.session.close() + self._handleError(error_message) + return self._handleSuccess() local_db.session.close() return os.path.basename(file_path + format_new_ext) From 7c623941de04f8c3612facf187b27eb28d29238b Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Mon, 7 Feb 2022 13:55:18 +0100 Subject: [PATCH 6/6] Bugfixes after testrun Enabled re-encode of bookformats --- cps/editbooks.py | 2 +- cps/tasks/convert.py | 62 +++---- test/Calibre-Web TestSummary_Linux.html | 210 +++++++++++++++--------- 3 files changed, 167 insertions(+), 107 deletions(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index 9fce1272..fcf043a5 100755 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -670,7 +670,7 @@ def upload_single_file(request, book, book_id): # Queue uploader info link = '{}'.format(url_for('web.show_book', book_id=book.id), escape(book.title)) uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=link) - WorkerThread.add(current_user.name, TaskUpload(uploadText), escape(book.title)) + WorkerThread.add(current_user.name, TaskUpload(uploadText, escape(book.title))) return uploader.process( saved_filename, *os.path.splitext(requested_file.filename), diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 90513b09..c8b8229c 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -118,21 +118,24 @@ class TaskConvert(CalibreTask): self.title = cur_book.title self.results['path'] = cur_book.path self.results['title'] = self.title - new_format = db.Data(name=os.path.basename(file_path), - book_format=self.settings['new_book_format'].upper(), - book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) - try: - local_db.session.merge(new_format) - local_db.session.commit() - except SQLAlchemyError as e: - local_db.session.rollback() - log.error("Database error: %s", e) + new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id)\ + .filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none() + if not new_format: + new_format = db.Data(name=os.path.basename(file_path), + book_format=self.settings['new_book_format'].upper(), + book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) + try: + local_db.session.merge(new_format) + local_db.session.commit() + except SQLAlchemyError as e: + local_db.session.rollback() + log.error("Database error: %s", e) + local_db.session.close() + self._handleError(error_message) + return + self._handleSuccess() local_db.session.close() - self._handleError(error_message) - return - self._handleSuccess() - local_db.session.close() - return os.path.basename(file_path + format_new_ext) + return os.path.basename(file_path + format_new_ext) else: log.info("Book id %d - target format of %s does not exist. Moving forward with convert.", book_id, @@ -153,22 +156,25 @@ class TaskConvert(CalibreTask): if check == 0: cur_book = local_db.get_book(book_id) if os.path.isfile(file_path + format_new_ext): - new_format = db.Data(name=cur_book.data[0].name, + new_format = local_db.session.query(db.Data).filter(db.Data.book == book_id) \ + .filter(db.Data.format == self.settings['new_book_format'].upper()).one_or_none() + if not new_format: + new_format = db.Data(name=cur_book.data[0].name, book_format=self.settings['new_book_format'].upper(), book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext)) - try: - local_db.session.merge(new_format) - local_db.session.commit() - if self.settings['new_book_format'].upper() in ['KEPUB', 'EPUB', 'EPUB3']: - ub_session = ini() - remove_synced_book(book_id, True, ub_session) - ub_session.close() - except SQLAlchemyError as e: - local_db.session.rollback() - log.error("Database error: %s", e) - local_db.session.close() - self._handleError(error_message) - return + try: + local_db.session.merge(new_format) + local_db.session.commit() + if self.settings['new_book_format'].upper() in ['KEPUB', 'EPUB', 'EPUB3']: + ub_session = ini() + remove_synced_book(book_id, True, ub_session) + ub_session.close() + except SQLAlchemyError as e: + local_db.session.rollback() + log.error("Database error: %s", e) + local_db.session.close() + self._handleError(error_message) + return self.results['path'] = cur_book.path self.title = cur_book.title self.results['title'] = self.title diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index bd12238c..1be6540c 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2022-02-01 20:32:05

+

Start Time: 2022-02-06 21:09:18

-

Stop Time: 2022-02-02 00:57:41

+

Stop Time: 2022-02-07 01:33:46

-

Duration: 3h 39 min

+

Duration: 3h 38 min

@@ -639,11 +639,11 @@ - + TestEbookConvertKepubify 3 - 3 - 0 + 2 + 1 0 0 @@ -662,11 +662,33 @@ - +
TestEbookConvertKepubify - test_convert_only
- PASS + +
+ FAIL +
+ + + + @@ -681,11 +703,11 @@ - + TestEbookConvertGDriveKepubify 3 - 3 - 0 + 2 + 1 0 0 @@ -704,11 +726,33 @@ - +
TestEbookConvertGDriveKepubify - test_convert_only
- PASS + +
+ FAIL +
+ + + + @@ -723,13 +767,13 @@ - + TestEditAdditionalBooks 19 - 17 - 0 + 15 0 2 + 2 Detail @@ -764,20 +808,70 @@ - +
TestEditAdditionalBooks - test_delete_role
- PASS + +
+ ERROR +
+ + + + - +
TestEditAdditionalBooks - test_details_popup
- PASS + +
+ ERROR +
+ + + + @@ -1351,11 +1445,11 @@ - + TestEditAuthorsGdrive 6 - 5 - 1 + 6 + 0 0 0 @@ -1410,31 +1504,11 @@ - +
TestEditAuthorsGdrive - test_rename_capital_on_upload
- -
- FAIL -
- - - - + PASS @@ -1617,13 +1691,13 @@ AssertionError: <selenium.webdriver.remote.webelement.WebElement (session= - + TestLoadMetadata 1 - 0 1 0 0 + 0 Detail @@ -1631,31 +1705,11 @@ AssertionError: <selenium.webdriver.remote.webelement.WebElement (session= - +
TestLoadMetadata - test_load_metadata
- -
- FAIL -
- - - - + PASS @@ -4572,9 +4626,9 @@ AssertionError: 0.0 not greater than or equal to 0.05 Total 404 - 395 + 393 + 2 2 - 0 7   @@ -4741,7 +4795,7 @@ AssertionError: 0.0 not greater than or equal to 0.05 httplib2 - 0.20.2 + 0.20.4 TestCliGdrivedb @@ -4771,7 +4825,7 @@ AssertionError: 0.0 not greater than or equal to 0.05 httplib2 - 0.20.2 + 0.20.4 TestEbookConvertCalibreGDrive @@ -4801,7 +4855,7 @@ AssertionError: 0.0 not greater than or equal to 0.05 httplib2 - 0.20.2 + 0.20.4 TestEbookConvertGDriveKepubify @@ -4843,7 +4897,7 @@ AssertionError: 0.0 not greater than or equal to 0.05 httplib2 - 0.20.2 + 0.20.4 TestEditAuthorsGdrive @@ -4879,7 +4933,7 @@ AssertionError: 0.0 not greater than or equal to 0.05 httplib2 - 0.20.2 + 0.20.4 TestEditBooksOnGdrive @@ -4921,7 +4975,7 @@ AssertionError: 0.0 not greater than or equal to 0.05 httplib2 - 0.20.2 + 0.20.4 TestSetupGdrive @@ -5005,7 +5059,7 @@ AssertionError: 0.0 not greater than or equal to 0.05