Merge remote-tracking branch 'Convert_all/ebookconvert-any'

# Conflicts:
#	cps/web.py

cps/helper.py

@@ -53,7 +53,7 @@ def make_mobi(book_id, calibrepath, user_id, kindle_mail):
     book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
     data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first()
     if not data:
-        error_message = _(u"epub format not found for book id: %(book)d", book=book_id)
+        error_message = _(u"Epub format not found for book id: %(book)d", book=book_id)
         app.logger.error("make_mobi: " + error_message)
         return error_message
     if ub.config.config_use_google_drive:
@@ -64,16 +64,19 @@ def make_mobi(book_id, calibrepath, user_id, kindle_mail):
                 os.makedirs(os.path.join(calibrepath, book.path))
             df.GetContentFile(datafile)
         else:
-            error_message = (u"make_mobi: epub not found on gdrive: %s.epub" % data.name)
+            error_message = (u"Epub not found on Google Drive: %s.epub" % data.name)
             return error_message
     file_path = os.path.join(calibrepath, book.path, data.name)
     if os.path.exists(file_path + u".epub"):
-        # append converter to queue
-        global_WorkerThread.add_convert(file_path, book.id, user_id, _(u"Convert: %s" % book.title), ub.get_mail_settings(),
+        # read settings and append converter task to queue
+        settings = ub.get_mail_settings()
+        settings['old_book_format'] = u'EPUB'
+        settings['new_book_format'] = u'MOBI'
+        global_WorkerThread.add_convert(file_path, book.id, user_id, _(u"Convert: %s" % book.title), settings,
                                       kindle_mail)
         return None
     else:
-        error_message = (u"make_mobi: epub not found: %s.epub" % file_path)
+        error_message = (u"Epub not found: %s.epub" % file_path)
         return error_message
 
 
@@ -136,6 +139,52 @@ def send_mail(book_id, kindle_mail, calibrepath, user_id):
         return _(u"The requested file could not be read. Maybe wrong permissions?")
 
 
+# Convert existing book entry to new format
+def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id):
+    book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == old_book_format).first()
+    if not data:
+        error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id)
+        app.logger.error("convert_book_format: " + error_message)
+        return error_message
+    if ub.config.config_use_google_drive:
+        df = gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower())
+        if df:
+            datafile = os.path.join(calibrepath, book.path, data.name + "." + old_book_format.lower())
+            if not os.path.exists(os.path.join(calibrepath, book.path)):
+                os.makedirs(os.path.join(calibrepath, book.path))
+            df.GetContentFile(datafile)
+        else:
+            error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
+                              format=old_book_format, fn=data.name + "." + old_book_format.lower())
+            return error_message
+    file_path = os.path.join(calibrepath, book.path, data.name)
+    if os.path.exists(file_path + "." + old_book_format.lower()):
+        # append converter to queue
+        settings = {'old_book_format': old_book_format,
+                    'new_book_format': new_book_format}
+
+        app.logger.debug("Creating task for worker thread:")
+        app.logger.debug("filepath: " + file_path + " " +
+                         "bookid: " + str(book.id) + " " +
+                         "userid: " + str(user_id) + " " +
+                         "taskmsg: " + _(u"Convert to %(format)s: %(book)s",
+                                                       format=new_book_format, book=book.title) + " " +
+                         "settings:old_book_format: " + settings['old_book_format'] + " " +
+                         "settings:new_book_format: " + settings['new_book_format']
+                         )
+
+        global_WorkerThread.add_convert(file_path, book.id,
+                                            user_id, _(u"Convert to %(format)s: %(book)s",
+                                                       format=new_book_format, book=book.title),
+                                            settings)
+        return None
+    else:
+        error_message = _(u"%(format)s not found: %(fn)s",
+                          format=old_book_format, fn=data.name + "." + old_book_format.lower())
+        return error_message
+
+
 def get_valid_filename(value, replace_whitespace=True):
     """
     Returns the given string converted to a string that can be used for a clean
@@ -339,7 +388,7 @@ def save_cover(url, book_path):
         f.write(img.content)
         f.close()
         uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg'), os.path.join(tmpDir, f.name))
-        web.app.logger.info("Cover is saved on gdrive")
+        web.app.logger.info("Cover is saved on Google Drive")
         return True
 
     f = open(os.path.join(ub.config.config_calibre_dir, book_path, "cover.jpg"), "wb")

cps/static/js/edit_books.js

@@ -254,3 +254,4 @@ $("#btn-upload-cover").on("change", function () {
     } // Remove c:\fake at beginning from localhost chrome
     $("#upload-cover").html(filename);
 });
+

cps/templates/admin.html

@@ -5,7 +5,7 @@
   <table class="table table-striped" id="table_user">
     <tr>
         <th>{{_('Nickname')}}</th>
-        <th>{{_('Email')}}</th>
+        <th>{{_('E-mail')}}</th>
         <th>{{_('Kindle')}}</th>
         <th>{{_('DLS')}}</th>
         <th class="hidden-xs">{{_('Admin')}}</th>

cps/templates/book_edit.html

@@ -1,7 +1,7 @@
 {% extends "layout.html" %}
 {% block body %}
 {% if book %}
-<form role="form" action="{{ url_for('edit_book', book_id=book.id) }}" method="post" enctype="multipart/form-data">
+
 
   <div class="col-sm-3 col-lg-3 col-xs-12">
     <div class="cover">
@@ -16,7 +16,7 @@
       <button type="button" class="btn btn-danger" id="delete" data-toggle="modal" data-target="#deleteModal">{{_("Delete Book")}}</button>
     </div>
     {% if book.data|length > 1 %}
-      <div class="text-center more-stuff"><h4> {{_('Delete formats:')}} </h4>
+      <div class="text-center more-stuff"><h4>{{_('Delete formats:')}}</h4>
       {% for file in book.data %}
         <div class="form-group">
           <a href="{{ url_for('delete_book', book_id=book.id, book_format=file.format) }}" class="btn btn-danger" type="button">{{_('Delete')}} - {{file.format}}</a>
@@ -25,7 +25,35 @@
       </div>
     {% endif %}
 {% endif %}
+
+{%  if display_convertbtn and conversion_formats|length > 0 %}
+  <div class="text-center more-stuff"><h4>{{_('Convert book format:')}}</h4>
+      <form class="padded-bottom" action="{{ url_for('convert_bookformat', book_id=book.id) }}" method="post" id="book_convert_frm">
+          <div class="form-group">
+              <div class="text-left">
+                  <label class="control-label" for="book_format_from">{{_('Convert from:')}}</label>
+                  <select class="form-control" name="book_format_from" id="book_format_from">
+                    <option disabled selected value> -- select an option -- </option>
+                    {% for file in book.data %}
+                    <option>{{file.format}} </option>
+                    {% endfor %}
+                  </select>
+                  <label class="control-label" for="book_format_to">{{_('Convert to:')}}</label>
+                  <select class="form-control" name="book_format_to" id="book_format_to">
+                    <option disabled selected value> -- select an option -- </option>
+                    {% for format in conversion_formats %}
+                    <option>{{format|upper}} </option>
+                    {% endfor %}
+                  </select>
+              </div>
+          </div>
+          <button type="submit" class="btn btn-primary" id="btn-book-convert" name="btn-book-convert"><span class="glyphicon glyphicon-duplicate"></span> {{_('Convert book')}}</button>
+      </form>
   </div>
+{% endif %}
+
+  </div>
+<form role="form" action="{{ url_for('edit_book', book_id=book.id) }}" method="post" enctype="multipart/form-data" id="book_edit_frm">
   <div class="col-sm-9 col-xs-12">
     <div class="form-group">
       <label for="book_title">{{_('Book Title')}}</label>
@@ -61,7 +89,7 @@
       <input type="text" class="form-control" name="cover_url" id="cover_url" value="">
     </div>
     <div class="form-group" aria-label="Upload cover from local drive">
-        <label class="btn btn-default btn-file" for="btn-upload-cover">{{ _('Upload Cover from local drive') }}</label>
+        <label class="btn btn-primary btn-file" for="btn-upload-cover">{{ _('Upload Cover from local drive') }}</label>
         <div class="upload-cover-input-text" id="upload-cover"></div>
         <input id="btn-upload-cover" name="btn-upload-cover" type="file">
     </div>
@@ -131,7 +159,7 @@
       {% if g.user.role_upload() or g.user.role_admin()%}
         {% if g.allow_upload %}
           <div role="group" aria-label="Upload new book format">
-            <label class="btn btn-default btn-file" for="btn-upload-format">{{ _('Upload format') }}</label>
+            <label class="btn btn-primary btn-file" for="btn-upload-format">{{ _('Upload format') }}</label>
             <div class="upload-format-input-text" id="upload-format"></div>
             <input id="btn-upload-format" name="btn-upload-format" type="file">
           </div>

cps/templates/register.html

@@ -12,7 +12,7 @@
       <input type="password" class="form-control" id="password" name="password" placeholder="{{_('Choose a password')}}" required>
     </div-->
     <div class="form-group required">
-      <label for="email">{{_('Email address')}}</label>
+      <label for="email">{{_('E-mail address')}}</label>
       <input type="email" class="form-control" id="email" name="email" placeholder="{{_('Your email address')}}" required>
     </div>
     <button type="submit" class="btn btn-primary">{{_('Register')}}</button>

cps/templates/remote_login.html

@@ -1,7 +1,7 @@
 {% extends "layout.html" %}
 {% block body %}
 <div class="well">
-  <h2 style="margin-top: 0">{{_('Remote Login')}}</h2>
+  <h2 style="margin-top: 0">{{_('Remote login')}}</h2>
   <p>
     {{_('Using your another device, visit')}} <a href="{{verify_url}}">{{verify_url}}</a> {{_('and log in')}}.
   </p>

cps/templates/user_edit.html

@@ -10,7 +10,7 @@
     </div>
     {% endif %}
     <div class="form-group">
-      <label for="email">{{_('Email address')}}</label>
+      <label for="email">{{_('E-mail address')}}</label>
       <input type="email" class="form-control" name="email" id="email" value="{{ content.email if content.email != None }}" autocomplete="off">
     </div>
     {% if ( g.user and g.user.role_passwd() or g.user.role_admin() ) and not content.role_anonymous() %}

cps/web.py

@@ -3271,10 +3271,28 @@ def edit_book(book_id):
     for authr in book.authors:
         author_names.append(authr.name.replace('|', ','))
 
+    # Option for showing convertbook button
+    if config.config_ebookconverter == 2:
+        display_convertbtn = True
+    else:
+        display_convertbtn = False
+
+    # Determine what formats don't already exist
+    allowed_conversion_formats = ALLOWED_EXTENSIONS.copy()
+    for file in book.data:
+        try:
+            allowed_conversion_formats.remove(file.format.lower())
+        except Exception:
+            app.logger.warning(file.format.lower() + ' already removed from list.')
+
+    app.logger.debug('Allowed conversion formats:')
+    app.logger.debug(allowed_conversion_formats)
+
     # Show form
     if request.method != 'POST':
         return render_title_template('book_edit.html', book=book, authors=author_names, cc=cc,
-                                     title=_(u"edit metadata"), page="editbook")
+                                     title=_(u"edit metadata"), page="editbook", display_convertbtn=display_convertbtn,
+                                     conversion_formats=allowed_conversion_formats)
 
     # Update book
     edited_books_id = set()
@@ -3700,3 +3718,30 @@ def upload():
                 return render_title_template('detail.html', entry=book, cc=cc,
                                              title=book.title, books_shelfs=book_in_shelfs, page="upload")
     return redirect(url_for("index"))
+
+
+@app.route("/admin/book/convert/<int:book_id>", methods=['POST'])
+@login_required_if_no_ano
+@edit_required
+def convert_bookformat(book_id):
+    # check to see if we have form fields to work with -  if not send user back
+    book_format_from = request.form.get('book_format_from', None)
+    book_format_to = request.form.get('book_format_to', None)
+
+    if (book_format_from is None) or (book_format_to is None):
+        flash(_(u"Source or destination format for conversion missing"), category="error")
+        return redirect(request.environ["HTTP_REFERER"])
+
+    app.logger.debug('converting: book id: ' + str(book_id) +
+                     ' from: ' + request.form['book_format_from'] +
+                     ' to: ' + request.form['book_format_to'])
+    rtn = helper.convert_book_format(book_id, config.config_calibre_dir, book_format_from.upper(),
+                                     book_format_to.upper(), current_user.nickname)
+
+    if rtn is None:
+        flash(_(u"Book successfully queued for converting to %(book_format)s",
+                    book_format=book_format_to),
+                    category="success")
+    else:
+        flash(_(u"There was an error converting this book: %(res)s", res=rtn), category="error")
+    return redirect(request.environ["HTTP_REFERER"])

cps/worker.py

@@ -42,6 +42,7 @@ STAT_FINISH_SUCCESS = 3
 TASK_EMAIL = 1
 TASK_CONVERT = 2
 TASK_UPLOAD = 3
+TASK_CONVERT_ANY = 4
 
 RET_FAIL = 0
 RET_SUCCESS = 1
@@ -171,7 +172,9 @@ class WorkerThread(threading.Thread):
                 if self.queue[self.current]['typ'] == TASK_EMAIL:
                     self.send_raw_email()
                 if self.queue[self.current]['typ'] == TASK_CONVERT:
-                    self.convert_mobi()
+                    self.convert_any_format()
+                if self.queue[self.current]['typ'] == TASK_CONVERT_ANY:
+                    self.convert_any_format()
                 # TASK_UPLOAD is handled implicitly
                 self.current += 1
             else:
@@ -200,61 +203,75 @@ class WorkerThread(threading.Thread):
     def get_taskstatus(self):
         if self.current  < len(self.queue):
             if self.queue[self.current]['status'] == STAT_STARTED:
-                if not self.queue[self.current]['typ'] == TASK_CONVERT:
+                if self.queue[self.current]['typ'] == TASK_EMAIL:
                     self.UIqueue[self.current]['progress'] = self.get_send_status()
                 self.UIqueue[self.current]['runtime'] = self._formatRuntime(
                                                         datetime.now() - self.queue[self.current]['starttime'])
         return self.UIqueue
 
-    def convert_mobi(self):
+    def convert_any_format(self):
         # convert book, and upload in case of google drive
         self.queue[self.current]['status'] = STAT_STARTED
         self.UIqueue[self.current]['status'] = _('Started')
         self.queue[self.current]['starttime'] = datetime.now()
         self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
-        filename=self.convert()
-        if web.ub.config.config_use_google_drive:
-            gd.updateGdriveCalibreFromLocal()
-        if(filename):
-            self.add_email(_(u'Send to Kindle'), self.queue[self.current]['path'], filename,
+        curr_task = self.queue[self.current]['typ']
+        filename = self.convert_ebook_format()
+        if filename:
+            if web.ub.config.config_use_google_drive:
+                gd.updateGdriveCalibreFromLocal()
+            if curr_task == TASK_CONVERT:
+                self.add_email(_(u'Send to Kindle'), self.queue[self.current]['path'], filename,
                        self.queue[self.current]['settings'], self.queue[self.current]['kindle'],
                        self.UIqueue[self.current]['user'], _(u"E-mail: %s" % self.queue[self.current]['title']))
 
-    def convert(self):
+
+    def convert_ebook_format(self):
         error_message = None
         file_path = self.queue[self.current]['file_path']
         bookid = self.queue[self.current]['bookid']
-        # check if converter-excecutable is existing
+        format_old_ext = u'.' + self.queue[self.current]['settings']['old_book_format'].lower()
+        format_new_ext = u'.' + self.queue[self.current]['settings']['new_book_format'].lower()
+        # check if converter-executable is existing
         if not os.path.exists(web.ub.config.config_converterpath):
             self._handleError(_(u"Convertertool %(converter)s not found", converter=web.ub.config.config_converterpath))
             return
         try:
             # check which converter to use kindlegen is "1"
-            if web.ub.config.config_ebookconverter == 1:
-                command = [web.ub.config.config_converterpath, u'"' + file_path + u'.epub"']
-            else:
+            if format_old_ext == '.epub' and format_new_ext == '.mobi':
+                if web.ub.config.config_ebookconverter == 1:
+                    if os.name == 'nt':
+                        command = web.ub.config.config_converterpath + u' "' + file_path + u'.epub"'
+                        if sys.version_info < (3, 0):
+                            command = command.encode(sys.getfilesystemencoding())
+                    else:
+                        command = [web.ub.config.config_converterpath, file_path + u'.epub']
+                        if sys.version_info < (3, 0):
+                            command = [x.encode(sys.getfilesystemencoding()) for x in command]
+            if web.ub.config.config_ebookconverter == 2:
                 # Linux py2.7 encode as list without quotes no empty element for parameters
                 # linux py3.x no encode and as list without quotes no empty element for parameters
-                # windows py2.7 encode as string with qoutes empty element for parameters is okay
-                # windows py 3.x no encode and as string with qoutes empty element for parameters is okay
-                # seperate handling for windows and linux
+                # windows py2.7 encode as string with quotes empty element for parameters is okay
+                # windows py 3.x no encode and as string with quotes empty element for parameters is okay
+                # separate handling for windows and linux
                 if os.name == 'nt':
-                    command = web.ub.config.config_converterpath + u' "' + file_path + u'.epub" "' + \
-                              file_path + u'.mobi" ' + web.ub.config.config_calibre
+                    command = web.ub.config.config_converterpath + u' "' + file_path + format_old_ext + u'" "' + \
+                              file_path + format_new_ext + u'" ' + web.ub.config.config_calibre
                     if sys.version_info < (3, 0):
                         command = command.encode(sys.getfilesystemencoding())
                 else:
-                    command = [web.ub.config.config_converterpath, (file_path + u'.epub'),
-                               (file_path + u'.mobi')]
+                    command = [web.ub.config.config_converterpath, (file_path + format_old_ext),
+                               (file_path + format_new_ext)]
                     if web.ub.config.config_calibre:
                         command.append(web.ub.config.config_calibre)
                     if sys.version_info < (3, 0):
-                        command = [ x.encode(sys.getfilesystemencoding()) for x in command ]
+                        command = [x.encode(sys.getfilesystemencoding()) for x in command]
 
             p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
         except OSError as e:
             self._handleError(_(u"Ebook-converter failed: %s" % e))
             return
+
         if web.ub.config.config_ebookconverter == 1:
             nextline = p.communicate()[0]
             # Format of error message (kindlegen translates its output texts):
@@ -265,7 +282,6 @@ class WorkerThread(threading.Thread):
                 error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
                                   error=conv_error.group(1), message=conv_error.group(2).strip())
             web.app.logger.debug("convert_kindlegen: " + nextline)
-
         else:
             while p.poll() is None:
                 nextline = p.stdout.readline()
@@ -277,30 +293,31 @@ class WorkerThread(threading.Thread):
                 if progress:
                     self.UIqueue[self.current]['progress'] = progress.group(1) + ' %'
 
-        #process returncode
+        # process returncode
         check = p.returncode
 
         # kindlegen returncodes
         # 0 = Info(prcgen):I1036: Mobi file built successfully
         # 1 = Info(prcgen):I1037: Mobi file built with WARNINGS!
         # 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors!
-        if ( check < 2 and web.ub.config.config_ebookconverter == 1) or \
-                (check == 0 and web.ub.config.config_ebookconverter == 2):
+        if (check < 2 and web.ub.config.config_ebookconverter == 1) or \
+            (check == 0 and web.ub.config.config_ebookconverter == 2):
             cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first()
-            new_format = web.db.Data(name=cur_book.data[0].name,book_format="MOBI",
-                                     book=bookid,uncompressed_size=os.path.getsize(file_path + ".mobi"))
+            new_format = web.db.Data(name=cur_book.data[0].name,
+                                     book_format=self.queue[self.current]['settings']['new_book_format'],
+                                     book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext))
             cur_book.data.append(new_format)
             web.db.session.commit()
             self.queue[self.current]['path'] = cur_book.path
             self.queue[self.current]['title'] = cur_book.title
             if web.ub.config.config_use_google_drive:
-                os.remove(file_path + u".epub")
+                os.remove(file_path + format_old_ext)
             self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
             self.UIqueue[self.current]['status'] = _('Finished')
             self.UIqueue[self.current]['progress'] = "100 %"
             self.UIqueue[self.current]['runtime'] = self._formatRuntime(
-                                                    datetime.now() - self.queue[self.current]['starttime'])
-            return file_path + ".mobi"
+                datetime.now() - self.queue[self.current]['starttime'])
+            return file_path + format_new_ext
         else:
             web.app.logger.info("ebook converter failed with error while converting book")
             if not error_message:
@@ -309,18 +326,20 @@ class WorkerThread(threading.Thread):
             return
 
 
-    def add_convert(self, file_path, bookid, user_name, typ, settings, kindle_mail):
+    def add_convert(self, file_path, bookid, user_name, typ, settings, kindle_mail=None):
         addLock = threading.Lock()
         addLock.acquire()
         if self.last >= 20:
             self.delete_completed_tasks()
         # progress, runtime, and status = 0
         self.id += 1
-        self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, 'kindle':kindle_mail,
-                           'status': STAT_WAITING, 'typ': TASK_CONVERT, 'settings':settings})
+        task = TASK_CONVERT_ANY
+        if kindle_mail:
+            task = TASK_CONVERT
+        self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, 'kindle': kindle_mail,
+                           'status': STAT_WAITING, 'typ': task, 'settings':settings})
         self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': typ,
                              'runtime': '0 s', 'status': _('Waiting'),'id': self.id } )
-        self.id += 1
 
         self.last=len(self.queue)
         addLock.release()
@@ -334,12 +353,12 @@ class WorkerThread(threading.Thread):
         if self.last >= 20:
             self.delete_completed_tasks()
         # progress, runtime, and status = 0
+        self.id += 1
         self.queue.append({'subject':subject, 'attachment':attachment, 'filepath':filepath,
                            'settings':settings, 'recipent':recipient, 'starttime': 0,
                            'status': STAT_WAITING, 'typ': TASK_EMAIL, 'text':text})
         self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': typ,
                              'runtime': '0 s', 'status': _('Waiting'),'id': self.id })
-        self.id += 1
         self.last=len(self.queue)
         addLock.release()
 
@@ -350,15 +369,15 @@ class WorkerThread(threading.Thread):
         if self.last >= 20:
             self.delete_completed_tasks()
         # progress=100%, runtime=0, and status finished
+        self.id += 1
         self.queue.append({'starttime': datetime.now(), 'status': STAT_FINISH_SUCCESS, 'typ': TASK_UPLOAD})
         self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': "100 %", 'type': typ,
                              'runtime': '0 s', 'status': _('Finished'),'id': self.id })
         self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
-        self.id += 1
         self.last=len(self.queue)
         addLock.release()
-    
-        
+
+
     def send_raw_email(self):
         self.queue[self.current]['starttime'] = datetime.now()
         self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']