Merge branch 'worker_task' into Develop

cps/db.py

@@ -648,9 +648,10 @@ class CalibreDB():
     # read search results from calibre-database and return it (function is used for feed and simple search
     def get_search_results(self, term, offset=None, order=None, limit=None):
         order = order or [Books.sort]
+        pagination = None
         if offset != None and limit != None:
             offset = int(offset)
-            limit = offset + int(limit)
+            limit_all = offset + int(limit)
         term.strip().lower()
         self.session.connection().connection.connection.create_function("lower", 1, lcase)
         q = list()
@@ -665,7 +666,9 @@ class CalibreDB():
                 func.lower(Books.title).ilike("%" + term + "%")
                 )).order_by(*order).all()
         result_count = len(result)
-        return result[offset:limit], result_count
+        if offset != None and limit != None:
+            pagination = Pagination((offset / (int(limit)) + 1), limit, result_count)
+        return result[offset:limit_all], result_count, pagination
 
     # Creates for all stored languages a translated speaking name in the array for the UI
     def speaking_language(self, languages=None):

cps/helper.py

@@ -74,40 +74,33 @@ log = logger.create()
 def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id, kindle_mail=None):
     book = calibre_db.get_book(book_id)
     data = calibre_db.get_book_format(book.id, old_book_format)
+    file_path = os.path.join(calibrepath, book.path, data.name)
     if not data:
         error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id)
         log.error("convert_book_format: %s", error_message)
         return error_message
     if config.config_use_google_drive:
-        df = gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower())
+        if not gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower()):
-        if df:
-            datafile = os.path.join(calibrepath, book.path, data.name + u"." + old_book_format.lower())
-            if not os.path.exists(os.path.join(calibrepath, book.path)):
-                os.makedirs(os.path.join(calibrepath, book.path))
-            df.GetContentFile(datafile)
-        else:
             error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
                               format=old_book_format, fn=data.name + "." + old_book_format.lower())
             return error_message
-    file_path = os.path.join(calibrepath, book.path, data.name)
-    if os.path.exists(file_path + "." + old_book_format.lower()):
-        # read settings and append converter task to queue
-        if kindle_mail:
-            settings = config.get_mail_settings()
-            settings['subject'] = _('Send to Kindle')  # pretranslate Subject for e-mail
-            settings['body'] = _(u'This e-mail has been sent via Calibre-Web.')
-            # text = _(u"%(format)s: %(book)s", format=new_book_format, book=book.title)
-        else:
-            settings = dict()
-        txt = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title))
-        settings['old_book_format'] = old_book_format
-        settings['new_book_format'] = new_book_format
-        WorkerThread.add(user_id, TaskConvert(file_path, book.id, txt, settings, kindle_mail))
-        return None
     else:
-        error_message = _(u"%(format)s not found: %(fn)s",
+        if not os.path.exists(file_path + "." + old_book_format.lower()):
-                          format=old_book_format, fn=data.name + "." + old_book_format.lower())
+            error_message = _(u"%(format)s not found: %(fn)s",
-        return error_message
+                              format=old_book_format, fn=data.name + "." + old_book_format.lower())
+            return error_message
+    # read settings and append converter task to queue
+    if kindle_mail:
+        settings = config.get_mail_settings()
+        settings['subject'] = _('Send to Kindle')  # pretranslate Subject for e-mail
+        settings['body'] = _(u'This e-mail has been sent via Calibre-Web.')
+    else:
+        settings = dict()
+    txt = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title))
+    settings['old_book_format'] = old_book_format
+    settings['new_book_format'] = new_book_format
+    WorkerThread.add(user_id, TaskConvert(file_path, book.id, txt, settings, kindle_mail, user_id))
+    return None
 
 
 def send_test_mail(kindle_mail, user_name):

cps/jinjia.py

@@ -44,6 +44,8 @@ log = logger.create()
 def url_for_other_page(page):
     args = request.view_args.copy()
     args['page'] = page
+    for get, val in request.args.items():
+        args[get] = val
     return url_for(request.endpoint, **args)
 
 

cps/services/worker.py

@@ -210,7 +210,7 @@ class CalibreTask:
         self._progress = x
 
     def _handleError(self, error_message):
-        log.error(error_message)
+        log.exception(error_message)
         self.stat = STAT_FAIL
         self.progress = 1
         self.error = error_message

cps/static/js/table.js

@@ -109,6 +109,7 @@ $(function() {
     $("#books-table").bootstrapTable({
         sidePagination: "server",
         pagination: true,
+        paginationLoop: false,
         paginationDetailHAlign: " hidden",
         paginationHAlign: "left",
         idField: "id",

cps/tasks/convert.py

@@ -20,34 +20,52 @@ log = logger.create()
 
 
 class TaskConvert(CalibreTask):
-    def __init__(self, file_path, bookid, taskMessage, settings, kindle_mail):
+    def __init__(self, file_path, bookid, taskMessage, settings, kindle_mail, user=None):
         super(TaskConvert, self).__init__(taskMessage)
         self.file_path = file_path
         self.bookid = bookid
         self.settings = settings
         self.kindle_mail = kindle_mail
+        self.user = user
 
         self.results = dict()
 
     def run(self, worker_thread):
         self.worker_thread = worker_thread
+        if config.config_use_google_drive:
+            cur_book = calibre_db.get_book(self.bookid)
+            data = calibre_db.get_book_format(self.bookid, self.settings['old_book_format'])
+            df = gdriveutils.getFileFromEbooksFolder(cur_book.path,
+                                                     data.name + "." + self.settings['old_book_format'].lower())
+            if df:
+                datafile = os.path.join(config.config_calibre_dir,
+                                        cur_book.path,
+                                        data.name + u"." + self.settings['old_book_format'].lower())
+                if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)):
+                    os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path))
+                df.GetContentFile(datafile)
+            else:
+                error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
+                                  format=self.settings['old_book_format'],
+                                  fn=data.name + "." + self.settings['old_book_format'].lower())
+                return error_message
+
         filename = self._convert_ebook_format()
+        if config.config_use_google_drive:
+            os.remove(self.file_path + u'.' + self.settings['old_book_format'].lower())
 
         if filename:
             if config.config_use_google_drive:
+                # Upload files to gdrive
                 gdriveutils.updateGdriveCalibreFromLocal()
+                self._handleSuccess()
             if self.kindle_mail:
                 # if we're sending to kindle after converting, create a one-off task and run it immediately
                 # todo: figure out how to incorporate this into the progress
                 try:
-                    task = TaskEmail(self.settings['subject'], self.results["path"],
+                    worker_thread.add(self.user, TaskEmail(self.settings['subject'], self.results["path"],
                                filename, self.settings, self.kindle_mail,
-                               self.settings['subject'], self.settings['body'], internal=True)
+                               self.settings['subject'], self.settings['body'], internal=True))
-                    task.start(worker_thread)
-
-                    # even though the convert task might be finished, if this task fails, fail the whole thing
-                    if task.stat != STAT_FINISH_SUCCESS:
-                        raise Exception(task.error)
                 except Exception as e:
                     return self._handleError(str(e))
 
@@ -101,9 +119,8 @@ class TaskConvert(CalibreTask):
                     return
                 self.results['path'] = cur_book.path
                 self.results['title'] = cur_book.title
-                if config.config_use_google_drive:
+                if not config.config_use_google_drive:
-                    os.remove(file_path + format_old_ext)
+                    self._handleSuccess()
-                self._handleSuccess()
                 return os.path.basename(file_path + format_new_ext)
             else:
                 error_message = _('%(format)s format not found on disk', format=format_new_ext.upper())
@@ -179,6 +196,8 @@ class TaskConvert(CalibreTask):
             progress = re.search(r"(\d+)%\s.*", nextline)
             if progress:
                 self.progress = int(progress.group(1)) / 100
+                if config.config_use_google_drive:
+                    self.progress *= 0.9
 
         # process returncode
         check = p.returncode

cps/tasks/mail.py

@@ -169,7 +169,7 @@ class TaskEmail(CalibreTask):
         except (MemoryError) as e:
             log.exception(e)
             self._handleError(u'MemoryError sending email: ' + str(e))
-            return None
+            # return None
         except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
             if hasattr(e, "smtp_error"):
                 text = e.smtp_error.decode('utf-8').replace("\n", '. ')
@@ -181,10 +181,11 @@ class TaskEmail(CalibreTask):
                 log.exception(e)
                 text = ''
             self._handleError(u'Smtplib Error sending email: ' + text)
-            return None
+            # return None
         except (socket.error) as e:
             self._handleError(u'Socket Error sending email: ' + e.strerror)
-            return None
+            # return None
+
 
     @property
     def progress(self):

cps/templates/book_edit.html

@@ -12,7 +12,9 @@
     {% if book.data|length > 1 %}
       <div class="text-center more-stuff"><h4>{{_('Delete formats:')}}</h4>
       {% for file in book.data %}
-        <button type="button" class="btn btn-danger" id="delete_format" data-toggle="modal" data-delete-id="{{ book.id }}" data-delete-format="{{ file.format }}" data-target="#deleteModal">{{_('Delete')}} - {{file.format}}</button>
+        <div class="form-group">
+          <button type="button" class="btn btn-danger" id="delete_format" data-toggle="modal" data-delete-id="{{ book.id }}" data-delete-format="{{ file.format }}" data-target="#deleteModal">{{_('Delete')}} - {{file.format}}</button>
+        </div>
       {% endfor %}
       </div>
     {% endif %}

cps/templates/book_table.html

@@ -2,13 +2,13 @@
 {% macro text_table_row(parameter, edit_text, show_text, validate) -%}
 <th data-field="{{ parameter }}" id="{{ parameter }}" data-sortable="true"
     data-visible = "{{visiblility.get(parameter)}}"
-    {% if g.user.role_edit() %}
+    {% if g.user.role_edit() %}
         data-editable-type="text"
         data-editable-url="{{ url_for('editbook.edit_list_book', param=parameter)}}"
         data-editable-title="{{ edit_text }}"
         data-edit="true"
-        {% if validate %}data-edit-validate="{{ _('This Field is Required') }}" {% endif %}
+        {% if validate %}data-edit-validate="{{ _('This Field is Required') }}" {% endif %}
-    {% endif %}
+    {% endif %}
 >{{ show_text }}</th>
 {%- endmacro %}
 
@@ -39,9 +39,9 @@
            data-url="{{url_for('web.list_books')}}">
       <thead>
         <tr>
-          {% if g.user.role_edit() %}
+          {% if g.user.role_edit() %}
             <th data-field="state" data-checkbox="true" data-sortable="true"></th>
-          {% endif %}
+          {% endif %}
             <th data-field="id" id="id" data-visible="false" data-switchable="false"></th>
             {{ text_table_row('title', _('Enter Title'),_('Title'), true) }}
             {{ text_table_row('sort', _('Enter Title Sort'),_('Title Sort'), false) }}
@@ -49,13 +49,13 @@
             {{ text_table_row('authors', _('Enter Authors'),_('Authors'), true) }}
             {{ text_table_row('tags', _('Enter Categories'),_('Categories'), false) }}
             {{ text_table_row('series', _('Enter Series'),_('Series'), false) }}
-            <th data-field="series_index" id="series_index" data-visible="{{visiblility.get('series_index')}}" data-edit-validate="{{ _('This Field is Required') }}" data-sortable="true" {% if g.user.role_edit() %} data-editable-type="number" data-editable-placeholder="1" data-editable-step="0.01" data-editable-min="0" data-editable-url="{{ url_for('editbook.edit_list_book', param='series_index')}}" data-edit="true" data-editable-title="{{_('Enter title')}}"{% endif %}>{{_('Series Index')}}</th>
+            <th data-field="series_index" id="series_index" data-visible="{{visiblility.get('series_index')}}" data-edit-validate="{{ _('This Field is Required') }}" data-sortable="true" {% if g.user.role_edit() %} data-editable-type="number" data-editable-placeholder="1" data-editable-step="0.01" data-editable-min="0" data-editable-url="{{ url_for('editbook.edit_list_book', param='series_index')}}" data-edit="true" data-editable-title="{{_('Enter title')}}"{% endif %}>{{_('Series Index')}}</th>
             {{ text_table_row('languages', _('Enter Languages'),_('Languages'), false) }}
             <!--th data-field="pubdate" data-type="date" data-visible="{{visiblility.get('pubdate')}}" data-viewformat="dd.mm.yyyy" id="pubdate" data-sortable="true">{{_('Publishing Date')}}</th-->
             {{ text_table_row('publishers', _('Enter Publishers'),_('Publishers'), false) }}
-          {% if g.user.role_edit() %}
+          {% if g.user.role_edit() %}
             <th data-align="right" data-formatter="EbookActions" data-switchable="false">{{_('Delete')}}</th>
-          {% endif %}
+          {% endif %}
         </tr>
       </thead>
     </table>

cps/updater.py

@@ -227,6 +227,7 @@ class Updater(threading.Thread):
             os.sep + 'vendor', os.sep + 'calibre-web.log', os.sep + '.git', os.sep + 'client_secrets.json',
             os.sep + 'gdrive_credentials', os.sep + 'settings.yaml', os.sep + 'venv', os.sep + 'virtualenv',
             os.sep + 'access.log', os.sep + 'access.log1', os.sep + 'access.log2',
+            os.sep + '.calibre-web.log.swp'
         )
         additional_path = self.is_venv()
         if additional_path:

cps/web.py

@@ -35,6 +35,7 @@ from babel import Locale as LC
 from babel.core import UnknownLocaleError
 from flask import Blueprint, jsonify
 from flask import render_template, request, redirect, send_from_directory, make_response, g, flash, abort, url_for
+from flask import session as flask_session
 from flask_babel import gettext as _
 from flask_login import login_user, logout_user, login_required, current_user, confirm_login
 from sqlalchemy.exc import IntegrityError, InvalidRequestError, OperationalError
@@ -669,10 +670,11 @@ def render_books_list(data, sort, book_id, page):
     elif data == "search":
         term = (request.args.get('query') or '')
         offset = int(int(config.config_books_per_page) * (page - 1))
-        if '&' not in term:
+        return render_search_results(term, offset, order, config.config_books_per_page)
-            return render_search_results(term, offset, order, config.config_books_per_page)
+    elif data == "advsearch":
-        else:
+        term = json.loads(flask_session['query'])
-            return render_adv_search_results(term, offset, order, config.config_books_per_page)
+        offset = int(int(config.config_books_per_page) * (page - 1))
+        return render_adv_search_results(term, offset, order, config.config_books_per_page)
     else:
         website = data or "newest"
         entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, order)
@@ -953,13 +955,14 @@ def render_prepare_search_form(cc):
 
 
 def render_search_results(term, offset=None, order=None, limit=None):
-    entries, result_count = calibre_db.get_search_results(term, offset, order, limit)
+    entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit)
     ids = list()
     for element in entries:
         ids.append(element.id)
     searched_ids[current_user.id] = ids
     return render_title_template('search.html',
                                  searchterm=term,
+                                 pagination=pagination,
                                  query=term,
                                  adv_searchterm=term,
                                  entries=entries,
@@ -967,6 +970,7 @@ def render_search_results(term, offset=None, order=None, limit=None):
                                  title=_(u"Search"),
                                  page="search")
 
+
 # ################################### View Books list ##################################################################
 
 
@@ -1007,7 +1011,7 @@ def list_books():
     search = request.args.get("search")
     total_count = calibre_db.session.query(db.Books).count()
     if search:
-        entries, filtered_count = calibre_db.get_search_results(search, off, order, limit)
+        entries, filtered_count, pagination = calibre_db.get_search_results(search, off, order, limit)
     else:
         entries, __, __ = calibre_db.fill_indexpage((int(off) / (int(limit)) + 1), limit, db.Books, True, order)
         filtered_count = total_count
@@ -1227,7 +1231,7 @@ def reconnect():
 def search():
     term = request.args.get("query")
     if term:
-        return render_search_results(term)
+        return render_search_results(term, 0, None, config.config_books_per_page)
     else:
         return render_title_template('search.html',
                                      searchterm="",
@@ -1238,9 +1242,19 @@ def search():
 @web.route("/advanced_search", methods=['POST'])
 @login_required_if_no_ano
 def advanced_search():
+    term = request.form
+    return render_adv_search_results(term, 0, None, config.config_books_per_page)
+
+def render_adv_search_results(term, offset=None, order=None, limit=None):
+    order = order or [db.Books.sort]
+    pagination = None
+    if offset != None and limit != None:
+        offset = int(offset)
+        limit_all = offset + int(limit)
+
     cc = get_cc_columns(filter_config_custom_read=True)
     calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase)
-    q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters(True)).order_by(db.Books.sort)
+    q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters(True))
 
     include_tag_inputs = request.form.getlist('include_tag')
     exclude_tag_inputs = request.form.getlist('exclude_tag')
@@ -1251,14 +1265,14 @@ def advanced_search():
     include_extension_inputs = request.form.getlist('include_extension')
     exclude_extension_inputs = request.form.getlist('exclude_extension')
 
-    author_name = request.form.get("author_name")
+    author_name = term.get("author_name")
-    book_title = request.form.get("book_title")
+    book_title = term.get("book_title")
-    publisher = request.form.get("publisher")
+    publisher = term.get("publisher")
-    pub_start = request.form.get("Publishstart")
+    pub_start = term.get("Publishstart")
-    pub_end = request.form.get("Publishend")
+    pub_end = term.get("Publishend")
-    rating_low = request.form.get("ratinghigh")
+    rating_low = term.get("ratinghigh")
-    rating_high = request.form.get("ratinglow")
+    rating_high = term.get("ratinglow")
-    description = request.form.get("comment")
+    description = term.get("comment")
     if author_name:
         author_name = author_name.strip().lower().replace(',', '|')
     if book_title:
@@ -1367,17 +1381,25 @@ def advanced_search():
                 else:
                     q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
                         func.lower(db.cc_classes[c.id].value).ilike("%" + custom_query + "%")))
-        q = q.all()
+        q = q.order_by(*order).all()
+        flask_session['query'] = json.dumps(term)
+        # ToDo: Check saved ids mechanism ?
         ids = list()
         for element in q:
             ids.append(element.id)
         searched_ids[current_user.id] = ids
-        return render_title_template('search.html',
+        # entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit)
-                                     adv_searchterm=searchterm,
+        result_count = len(q)
-                                     query=request.form,
+        if offset != None and limit != None:
-                                     entries=q,
+            pagination = Pagination((offset / (int(limit)) + 1), limit, result_count)
-                                     result_count=len(q),
+    return render_title_template('search.html',
-                                     title=_(u"search"), page="search")
+                                 adv_searchterm=searchterm,
+                                 pagination=pagination,
+                                 # query=request.form,
+                                 entries=q[offset:limit_all],
+                                 result_count=result_count,
+                                 title=_(u"search"), page="advsearch")
+
 
 
 @web.route("/advanced_search", methods=['GET'])