diff --git a/cps/admin.py b/cps/admin.py index 570ab8e0..97b41165 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -40,7 +40,6 @@ from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError from sqlalchemy.sql.expression import func, or_, text from . import constants, logger, helper, services -# from .cli import filepicker from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ valid_email, check_username diff --git a/cps/converter.py b/cps/converter.py index 2ff73666..f37168c7 100644 --- a/cps/converter.py +++ b/cps/converter.py @@ -39,9 +39,7 @@ def _get_command_version(path, pattern, argument=None): if argument: command.append(argument) try: - for line in process_wait(command): - if re.search(pattern, line): - return line + return process_wait(command, pattern=pattern).string except Exception as ex: log.warning("%s: %s", path, ex) return _EXECUTION_ERROR diff --git a/cps/debug_info.py b/cps/debug_info.py index cfa549f9..dd5e858e 100644 --- a/cps/debug_info.py +++ b/cps/debug_info.py @@ -22,10 +22,6 @@ import glob import zipfile import json from io import BytesIO -try: - from StringIO import StringIO -except ImportError: - from io import StringIO import os @@ -38,9 +34,9 @@ log = logger.create() def assemble_logfiles(file_name): log_list = sorted(glob.glob(file_name + '*'), reverse=True) - wfd = StringIO() + wfd = BytesIO() for f in log_list: - with open(f, 'r') as fd: + with open(f, 'rb') as fd: shutil.copyfileobj(fd, wfd) wfd.seek(0) if int(__version__.split('.')[0]) < 2: diff --git a/cps/helper.py b/cps/helper.py index d567e9b3..b6ea6760 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -711,12 +711,11 @@ def check_unrar(unrarLocation): if sys.version_info < (3, 0): unrarLocation = unrarLocation.encode(sys.getfilesystemencoding()) unrarLocation = [unrarLocation] - for lines in process_wait(unrarLocation): - value = re.search('UNRAR (.*) freeware', lines, re.IGNORECASE) - if value: - version = value.group(1) - log.debug("unrar version %s", version) - break + value = process_wait(unrarLocation, pattern='UNRAR (.*) freeware') + if value: + version = value.group(1) + log.debug("unrar version %s", version) + except (OSError, UnicodeDecodeError) as err: log.debug_or_exception(err) return _('Error excecuting UnRar') diff --git a/cps/jinjia.py b/cps/jinjia.py index de34cc86..70a6090e 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -113,10 +113,8 @@ def yesno(value, yes, no): @jinjia.app_template_filter('formatfloat') def formatfloat(value, decimals=1): - formatedstring = '%d' % value - if (value % 1) != 0: - formatedstring = ('%s.%d' % (formatedstring, (value % 1) * 10**decimals)).rstrip('0') - return formatedstring + value = 0 if not value else value + return ('{0:.' + str(decimals) + 'f}').format(value).rstrip('0').rstrip('.') @jinjia.app_template_filter('formatseriesindex') diff --git a/cps/kobo.py b/cps/kobo.py index f91adb00..11381170 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -44,6 +44,8 @@ from werkzeug.datastructures import Headers from sqlalchemy import func from sqlalchemy.sql.expression import and_, or_ from sqlalchemy.exc import StatementError +from sqlalchemy import __version__ as sql_version +from sqlalchemy.sql import select import requests from . import config, logger, kobo_auth, db, calibre_db, helper, shelf as shelf_lib, ub @@ -64,6 +66,7 @@ kobo_auth.register_url_value_preprocessor(kobo) log = logger.create() +sql2 = ([int(x) for x in sql_version.split('.')] >= [2,0,0]) def get_store_url_for_current_request(): # Programmatically modify the current url to point to the official Kobo store @@ -136,6 +139,7 @@ def convert_to_kobo_timestamp_string(timestamp): def HandleSyncRequest(): sync_token = SyncToken.SyncToken.from_headers(request.headers) log.info("Kobo library sync request received.") + log.debug("SyncToken: {}".format(sync_token)) if not current_app.wsgi_app.is_proxied: log.debug('Kobo: Received unproxied request, changed request port to external server port') @@ -153,14 +157,19 @@ def HandleSyncRequest(): calibre_db.reconnect_db(config, ub.app_DB_path) only_kobo_shelves = current_user.kobo_only_shelves_sync - # calibre_db.session.query(ub.Shelf).filter(ub.Shelf.user_id == current_user.id).filter(ub.Shelf.kobo_sync).count() > 0 if only_kobo_shelves: - changed_entries = ( - calibre_db.session.query(db.Books, + if sql2: + changed_entries = select(db.Books, ub.ArchivedBook.last_modified, ub.BookShelf.date_added, ub.ArchivedBook.is_archived) + else: + changed_entries = calibre_db.session.query(db.Books, + ub.ArchivedBook.last_modified, + ub.BookShelf.date_added, + ub.ArchivedBook.is_archived) + changed_entries = (changed_entries .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id) .filter(or_(db.Books.last_modified > sync_token.books_last_modified, ub.BookShelf.date_added > sync_token.books_last_modified)) @@ -174,8 +183,13 @@ def HandleSyncRequest(): .distinct() ) else: - changed_entries = ( - calibre_db.session.query(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived) + if sql2: + changed_entries = select(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived) + else: + changed_entries = calibre_db.session.query(db.Books, + ub.ArchivedBook.last_modified, + ub.ArchivedBook.is_archived) + changed_entries = (changed_entries .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id) .filter(db.Books.last_modified > sync_token.books_last_modified) .filter(calibre_db.common_filters()) @@ -188,7 +202,11 @@ def HandleSyncRequest(): changed_entries = changed_entries.filter(db.Books.id > sync_token.books_last_id) reading_states_in_new_entitlements = [] - for book in changed_entries.limit(SYNC_ITEM_LIMIT): + if sql2: + books = calibre_db.session.execute(changed_entries.limit(SYNC_ITEM_LIMIT)) + else: + books = changed_entries.limit(SYNC_ITEM_LIMIT) + for book in books: formats = [data.format for data in book.Books.data] if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats: helper.convert_book_format(book.Books.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name) @@ -228,18 +246,28 @@ def HandleSyncRequest(): new_books_last_created = max(ts_created, new_books_last_created) - max_change = changed_entries.from_self().filter(ub.ArchivedBook.is_archived)\ - .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()).first() + if sql2: + max_change = calibre_db.session.execute(changed_entries + .filter(ub.ArchivedBook.is_archived) + .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()))\ + .columns(db.Books).first() + else: + max_change = changed_entries.from_self().filter(ub.ArchivedBook.is_archived) \ + .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()).first() max_change = max_change.last_modified if max_change else new_archived_last_modified new_archived_last_modified = max(new_archived_last_modified, max_change) # no. of books returned - book_count = changed_entries.count() - + if sql2: + entries = calibre_db.session.execute(changed_entries).all() + book_count = len(entries) + else: + entries = changed_entries.all() + book_count = changed_entries.count() # last entry: - books_last_id = changed_entries.all()[-1].Books.id or -1 if book_count else -1 + books_last_id = entries[-1].Books.id or -1 if book_count else -1 # generate reading state data changed_reading_states = ub.session.query(ub.KoboReadingState) @@ -303,6 +331,7 @@ def generate_sync_response(sync_token, sync_results, set_cont=False): extra_headers["x-kobo-sync"] = "continue" sync_token.to_headers(extra_headers) + log.debug("Kobo Sync Content: {}".format(sync_results)) response = make_response(jsonify(sync_results), extra_headers) return response @@ -668,12 +697,23 @@ def sync_shelves(sync_token, sync_results, only_kobo_shelves=False): }) extra_filters.append(ub.Shelf.kobo_sync) - for shelf in ub.session.query(ub.Shelf).outerjoin(ub.BookShelf).filter( - or_(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified, - func.datetime(ub.BookShelf.date_added) > sync_token.tags_last_modified), - ub.Shelf.user_id == current_user.id, - *extra_filters - ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc()): # .columns(ub.Shelf): + if sql2: + shelflist = ub.session.execute(select(ub.Shelf).outerjoin(ub.BookShelf).filter( + or_(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified, + func.datetime(ub.BookShelf.date_added) > sync_token.tags_last_modified), + ub.Shelf.user_id == current_user.id, + *extra_filters + ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc())).columns(ub.Shelf) + else: + shelflist = ub.session.query(ub.Shelf).outerjoin(ub.BookShelf).filter( + or_(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified, + func.datetime(ub.BookShelf.date_added) > sync_token.tags_last_modified), + ub.Shelf.user_id == current_user.id, + *extra_filters + ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc()) + + + for shelf in shelflist: if not shelf_lib.check_shelf_view_permissions(shelf): continue diff --git a/cps/services/SyncToken.py b/cps/services/SyncToken.py index b54d8d95..cc67542c 100644 --- a/cps/services/SyncToken.py +++ b/cps/services/SyncToken.py @@ -183,3 +183,12 @@ class SyncToken: }, } return b64encode_json(token) + + def __str__(self): + return "{},{},{},{},{},{},{}".format(self.raw_kobo_store_token, + self.books_last_created, + self.books_last_modified, + self.archive_last_modified, + self.reading_state_last_modified, + self.tags_last_modified, + self.books_last_id) diff --git a/cps/static/css/caliBlur.css b/cps/static/css/caliBlur.css index f479f07f..3b226acb 100644 --- a/cps/static/css/caliBlur.css +++ b/cps/static/css/caliBlur.css @@ -3291,7 +3291,6 @@ div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] .dropd transform-origin: center top; border: 0; left: 0 !important; - max-height: 80%; overflow-y: auto; } diff --git a/cps/static/js/caliBlur.js b/cps/static/js/caliBlur.js index 1a2814fc..ce230730 100644 --- a/cps/static/js/caliBlur.js +++ b/cps/static/js/caliBlur.js @@ -413,7 +413,11 @@ if($("body.advsearch").length > 0) { }); $('#add-to-shelf').height("40px"); function search_dropdownToggle() { - topPos = $("#add-to-shelf").offset().top-20; + if( $("#add-to-shelf").length) { + topPos = $("#add-to-shelf").offset().top - 20; + } else { + topPos = 0 + } if ($('div[aria-label="Add to shelves"]').length > 0) { position = $('div[aria-label="Add to shelves"]').offset().left diff --git a/cps/subproc_wrapper.py b/cps/subproc_wrapper.py index c6c65851..3cc4a070 100644 --- a/cps/subproc_wrapper.py +++ b/cps/subproc_wrapper.py @@ -20,7 +20,7 @@ from __future__ import division, print_function, unicode_literals import sys import os import subprocess - +import re def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subprocess.PIPE, newlines=True): # Linux py2.7 encode as list without quotes no empty element for parameters @@ -44,12 +44,19 @@ def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subpro return subprocess.Popen(exc_command, shell=False, stdout=sout, stderr=serr, universal_newlines=newlines, env=env) # nosec -def process_wait(command, serr=subprocess.PIPE): +def process_wait(command, serr=subprocess.PIPE, pattern=""): # Run command, wait for process to terminate, and return an iterator over lines of its output. newlines = os.name != 'nt' + ret_val = "" p = process_open(command, serr=serr, newlines=newlines) p.wait() for line in p.stdout.readlines(): if isinstance(line, bytes): - line = line.decode('utf-8') - yield line + line = line.decode('utf-8', errors="ignore") + match = re.search(pattern, line, re.IGNORECASE) + if match and ret_val == "": + ret_val = match + break + p.stdout.close() + p.stderr.close() + return ret_val diff --git a/cps/templates/author.html b/cps/templates/author.html index 4e32db80..b011bae8 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -5,7 +5,7 @@ {% if author is not none %}
{%if author.image_url is not none %} - {{author.name|safe}} + {{author.name|safe}} {% endif %} {%if author.about is not none %} @@ -37,14 +37,14 @@
- + {% if entry.id in read_book_ids %}{% endif %}
-

{{entry.title|shortentitle}}

+

{{entry.title|shortentitle}}

{% for author in entry.authors %} @@ -104,11 +104,11 @@

- +
-

{{entry.title|shortentitle}}

+

{{entry.title|shortentitle}}

{% for author in entry.authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index b932f56d..2da7b09b 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -3,7 +3,7 @@ {% if book %}

- {{ book.title }} + {{ book.title }}
{% if g.user.role_delete_books() %}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index e5a67cde..76a5a87d 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -4,7 +4,7 @@
- {{ entry.title }} + {{ entry.title }}
@@ -122,7 +122,7 @@ {% endif %} {% if entry.series|length > 0 %} -

{{_('Book')}} {{entry.series_index}} {{_('of')}} {{entry.series[0].name}}

+

{{_('Book')}} {{entry.series_index|formatfloat(2)}} {{_('of')}} {{entry.series[0].name}}

{% endif %} {% if entry.languages.__len__() > 0 %} diff --git a/cps/templates/discover.html b/cps/templates/discover.html index d57994b4..f6d8207d 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -9,7 +9,7 @@ {% if entry.has_cover is defined %} - {{ entry.title }} + {{ entry.title }} {% if entry.id in read_book_ids %}{% endif %} @@ -17,7 +17,7 @@
-

{{entry.title|shortentitle}}

+

{{entry.title|shortentitle}}

{% for author in entry.authors %} diff --git a/cps/templates/grid.html b/cps/templates/grid.html index 1e79c43d..f8beffc5 100644 --- a/cps/templates/grid.html +++ b/cps/templates/grid.html @@ -29,14 +29,14 @@

- {{ entry[0].name }} + {{ entry[0].name }} {{entry.count}}
-

{{entry[0].series[0].name|shortentitle}}

+

{{entry[0].series[0].name|shortentitle}}
diff --git a/cps/templates/index.html b/cps/templates/index.html index d300fc65..b11500e4 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -9,14 +9,14 @@
- {{ entry.title }} + {{ entry.title }} {% if entry.id in read_book_ids %}{% endif %}
-

{{entry.title|shortentitle}}

+

{{entry.title|shortentitle}}

{% for author in entry.authors %} @@ -86,14 +86,14 @@

- {{ entry.title }} + {{ entry.title }} {% if entry.id in read_book_ids %}{% endif %}
-

{{entry.title|shortentitle}}

+

{{entry.title|shortentitle}}

{% for author in entry.authors %} diff --git a/cps/templates/read.html b/cps/templates/read.html index b38f783c..3d2566e0 100644 --- a/cps/templates/read.html +++ b/cps/templates/read.html @@ -3,7 +3,7 @@ - ePub Reader + {{_('epub Reader')}} | {{title}} diff --git a/cps/templates/readcbr.html b/cps/templates/readcbr.html index bad5f1f6..411e3fdd 100644 --- a/cps/templates/readcbr.html +++ b/cps/templates/readcbr.html @@ -1,10 +1,10 @@ - Comic Reader + {{_('Comic Reader')}} | {{title}} diff --git a/cps/templates/readdjvu.html b/cps/templates/readdjvu.html index c192ffcb..9771b7c9 100644 --- a/cps/templates/readdjvu.html +++ b/cps/templates/readdjvu.html @@ -7,7 +7,7 @@ -Djvu HTML5 browser demo +{{_('DJVU Reader')}} | {{title}} diff --git a/cps/templates/readpdf.html b/cps/templates/readpdf.html index 7af417ea..586625cc 100644 --- a/cps/templates/readpdf.html +++ b/cps/templates/readpdf.html @@ -26,7 +26,7 @@ See https://github.com/adobe-type-tools/cmap-resources - {{_('PDF reader')}} + {{_('PDF Reader')}} | {{title}} diff --git a/cps/templates/readtxt.html b/cps/templates/readtxt.html index da862fb2..ea294948 100644 --- a/cps/templates/readtxt.html +++ b/cps/templates/readtxt.html @@ -3,7 +3,7 @@ - {{_('Basic txt Reader')}} + {{_('txt Reader')}} | {{title}} diff --git a/cps/templates/search.html b/cps/templates/search.html index 81ab2d99..b63819be 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -44,7 +44,7 @@ {% if entry.has_cover is defined %} - {{ entry.title }} + {{ entry.title }} {% if entry.id in read_book_ids %}{% endif %} @@ -52,7 +52,7 @@

-

{{entry.title|shortentitle}}

+

{{entry.title|shortentitle}}

{% for author in entry.authors %} diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index 1ad79dbd..7ee96f7d 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -31,14 +31,14 @@

- {{ entry.title }} + {{ entry.title }} {% if entry.id in read_book_ids %}{% endif %}
-

{{entry.title|shortentitle}}

+

{{entry.title|shortentitle}}

{% for author in entry.authors %} diff --git a/cps/templates/shelf_order.html b/cps/templates/shelf_order.html index 1e49f29a..fc53a69a 100644 --- a/cps/templates/shelf_order.html +++ b/cps/templates/shelf_order.html @@ -9,9 +9,9 @@

diff --git a/cps/templates/shelfdown.html b/cps/templates/shelfdown.html index 77251e02..1d781310 100644 --- a/cps/templates/shelfdown.html +++ b/cps/templates/shelfdown.html @@ -35,7 +35,7 @@
-

{{entry.title|shortentitle}}

+

{{entry.title|shortentitle}}

{% for author in entry.authors %} {{author.name.replace('|',',')}} diff --git a/cps/web.py b/cps/web.py index ba3c5ae3..9dab285a 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1680,28 +1680,33 @@ def read_book(book_id, book_format): ub.Bookmark.format == book_format.upper())).first() if book_format.lower() == "epub": log.debug(u"Start epub reader for %d", book_id) - return render_title_template('read.html', bookid=book_id, title=_(u"Read a Book"), bookmark=bookmark) + return render_title_template('read.html', bookid=book_id, title=book.title, bookmark=bookmark) elif book_format.lower() == "pdf": log.debug(u"Start pdf reader for %d", book_id) - return render_title_template('readpdf.html', pdffile=book_id, title=_(u"Read a Book")) + return render_title_template('readpdf.html', pdffile=book_id, title=book.title) elif book_format.lower() == "txt": log.debug(u"Start txt reader for %d", book_id) - return render_title_template('readtxt.html', txtfile=book_id, title=_(u"Read a Book")) + return render_title_template('readtxt.html', txtfile=book_id, title=book.title) elif book_format.lower() == "djvu": log.debug(u"Start djvu reader for %d", book_id) - return render_title_template('readdjvu.html', djvufile=book_id, title=_(u"Read a Book")) + return render_title_template('readdjvu.html', djvufile=book_id, title=book.title) else: for fileExt in constants.EXTENSIONS_AUDIO: if book_format.lower() == fileExt: entries = calibre_db.get_filtered_book(book_id) log.debug(u"Start mp3 listening for %d", book_id) return render_title_template('listenmp3.html', mp3file=book_id, audioformat=book_format.lower(), - title=_(u"Read a Book"), entry=entries, bookmark=bookmark) + entry=entries, bookmark=bookmark) for fileExt in ["cbr", "cbt", "cbz"]: if book_format.lower() == fileExt: all_name = str(book_id) + title = book.title + if len(book.series): + title = title + " - " + book.series[0].name + if book.series_index: + title = title + " #" + '{0:.2f}'.format(book.series_index).rstrip('0').rstrip('.') log.debug(u"Start comic reader for %d", book_id) - return render_title_template('readcbr.html', comicfile=all_name, title=_(u"Read a Book"), + return render_title_template('readcbr.html', comicfile=all_name, title=title, extension=fileExt) log.debug(u"Oops! Selected book title is unavailable. File does not exist or is not accessible") flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), category="error")