Merge remote-tracking branch 'Convert_all/ebookconvert-any'

# Conflicts:
#	cps/web.py
This commit is contained in:
OzzieIsaacs 2018-08-31 15:00:22 +02:00
commit 4a95404e17
21 changed files with 4217 additions and 2357 deletions

View File

@ -53,7 +53,7 @@ def make_mobi(book_id, calibrepath, user_id, kindle_mail):
book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first()
if not data:
error_message = _(u"epub format not found for book id: %(book)d", book=book_id)
error_message = _(u"Epub format not found for book id: %(book)d", book=book_id)
app.logger.error("make_mobi: " + error_message)
return error_message
if ub.config.config_use_google_drive:
@ -64,16 +64,19 @@ def make_mobi(book_id, calibrepath, user_id, kindle_mail):
os.makedirs(os.path.join(calibrepath, book.path))
df.GetContentFile(datafile)
else:
error_message = (u"make_mobi: epub not found on gdrive: %s.epub" % data.name)
error_message = (u"Epub not found on Google Drive: %s.epub" % data.name)
return error_message
file_path = os.path.join(calibrepath, book.path, data.name)
if os.path.exists(file_path + u".epub"):
# append converter to queue
global_WorkerThread.add_convert(file_path, book.id, user_id, _(u"Convert: %s" % book.title), ub.get_mail_settings(),
# read settings and append converter task to queue
settings = ub.get_mail_settings()
settings['old_book_format'] = u'EPUB'
settings['new_book_format'] = u'MOBI'
global_WorkerThread.add_convert(file_path, book.id, user_id, _(u"Convert: %s" % book.title), settings,
kindle_mail)
return None
else:
error_message = (u"make_mobi: epub not found: %s.epub" % file_path)
error_message = (u"Epub not found: %s.epub" % file_path)
return error_message
@ -136,6 +139,52 @@ def send_mail(book_id, kindle_mail, calibrepath, user_id):
return _(u"The requested file could not be read. Maybe wrong permissions?")
# Convert existing book entry to new format
def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id):
book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == old_book_format).first()
if not data:
error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id)
app.logger.error("convert_book_format: " + error_message)
return error_message
if ub.config.config_use_google_drive:
df = gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower())
if df:
datafile = os.path.join(calibrepath, book.path, data.name + "." + old_book_format.lower())
if not os.path.exists(os.path.join(calibrepath, book.path)):
os.makedirs(os.path.join(calibrepath, book.path))
df.GetContentFile(datafile)
else:
error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
format=old_book_format, fn=data.name + "." + old_book_format.lower())
return error_message
file_path = os.path.join(calibrepath, book.path, data.name)
if os.path.exists(file_path + "." + old_book_format.lower()):
# append converter to queue
settings = {'old_book_format': old_book_format,
'new_book_format': new_book_format}
app.logger.debug("Creating task for worker thread:")
app.logger.debug("filepath: " + file_path + " " +
"bookid: " + str(book.id) + " " +
"userid: " + str(user_id) + " " +
"taskmsg: " + _(u"Convert to %(format)s: %(book)s",
format=new_book_format, book=book.title) + " " +
"settings:old_book_format: " + settings['old_book_format'] + " " +
"settings:new_book_format: " + settings['new_book_format']
)
global_WorkerThread.add_convert(file_path, book.id,
user_id, _(u"Convert to %(format)s: %(book)s",
format=new_book_format, book=book.title),
settings)
return None
else:
error_message = _(u"%(format)s not found: %(fn)s",
format=old_book_format, fn=data.name + "." + old_book_format.lower())
return error_message
def get_valid_filename(value, replace_whitespace=True):
"""
Returns the given string converted to a string that can be used for a clean
@ -339,7 +388,7 @@ def save_cover(url, book_path):
f.write(img.content)
f.close()
uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg'), os.path.join(tmpDir, f.name))
web.app.logger.info("Cover is saved on gdrive")
web.app.logger.info("Cover is saved on Google Drive")
return True
f = open(os.path.join(ub.config.config_calibre_dir, book_path, "cover.jpg"), "wb")

View File

@ -254,3 +254,4 @@ $("#btn-upload-cover").on("change", function () {
} // Remove c:\fake at beginning from localhost chrome
$("#upload-cover").html(filename);
});

View File

@ -5,7 +5,7 @@
<table class="table table-striped" id="table_user">
<tr>
<th>{{_('Nickname')}}</th>
<th>{{_('Email')}}</th>
<th>{{_('E-mail')}}</th>
<th>{{_('Kindle')}}</th>
<th>{{_('DLS')}}</th>
<th class="hidden-xs">{{_('Admin')}}</th>

View File

@ -1,7 +1,7 @@
{% extends "layout.html" %}
{% block body %}
{% if book %}
<form role="form" action="{{ url_for('edit_book', book_id=book.id) }}" method="post" enctype="multipart/form-data">
<div class="col-sm-3 col-lg-3 col-xs-12">
<div class="cover">
@ -16,7 +16,7 @@
<button type="button" class="btn btn-danger" id="delete" data-toggle="modal" data-target="#deleteModal">{{_("Delete Book")}}</button>
</div>
{% if book.data|length > 1 %}
<div class="text-center more-stuff"><h4> {{_('Delete formats:')}} </h4>
<div class="text-center more-stuff"><h4>{{_('Delete formats:')}}</h4>
{% for file in book.data %}
<div class="form-group">
<a href="{{ url_for('delete_book', book_id=book.id, book_format=file.format) }}" class="btn btn-danger" type="button">{{_('Delete')}} - {{file.format}}</a>
@ -25,7 +25,35 @@
</div>
{% endif %}
{% endif %}
{% if display_convertbtn and conversion_formats|length > 0 %}
<div class="text-center more-stuff"><h4>{{_('Convert book format:')}}</h4>
<form class="padded-bottom" action="{{ url_for('convert_bookformat', book_id=book.id) }}" method="post" id="book_convert_frm">
<div class="form-group">
<div class="text-left">
<label class="control-label" for="book_format_from">{{_('Convert from:')}}</label>
<select class="form-control" name="book_format_from" id="book_format_from">
<option disabled selected value> -- select an option -- </option>
{% for file in book.data %}
<option>{{file.format}} </option>
{% endfor %}
</select>
<label class="control-label" for="book_format_to">{{_('Convert to:')}}</label>
<select class="form-control" name="book_format_to" id="book_format_to">
<option disabled selected value> -- select an option -- </option>
{% for format in conversion_formats %}
<option>{{format|upper}} </option>
{% endfor %}
</select>
</div>
</div>
<button type="submit" class="btn btn-primary" id="btn-book-convert" name="btn-book-convert"><span class="glyphicon glyphicon-duplicate"></span> {{_('Convert book')}}</button>
</form>
</div>
{% endif %}
</div>
<form role="form" action="{{ url_for('edit_book', book_id=book.id) }}" method="post" enctype="multipart/form-data" id="book_edit_frm">
<div class="col-sm-9 col-xs-12">
<div class="form-group">
<label for="book_title">{{_('Book Title')}}</label>
@ -61,7 +89,7 @@
<input type="text" class="form-control" name="cover_url" id="cover_url" value="">
</div>
<div class="form-group" aria-label="Upload cover from local drive">
<label class="btn btn-default btn-file" for="btn-upload-cover">{{ _('Upload Cover from local drive') }}</label>
<label class="btn btn-primary btn-file" for="btn-upload-cover">{{ _('Upload Cover from local drive') }}</label>
<div class="upload-cover-input-text" id="upload-cover"></div>
<input id="btn-upload-cover" name="btn-upload-cover" type="file">
</div>
@ -131,7 +159,7 @@
{% if g.user.role_upload() or g.user.role_admin()%}
{% if g.allow_upload %}
<div role="group" aria-label="Upload new book format">
<label class="btn btn-default btn-file" for="btn-upload-format">{{ _('Upload format') }}</label>
<label class="btn btn-primary btn-file" for="btn-upload-format">{{ _('Upload format') }}</label>
<div class="upload-format-input-text" id="upload-format"></div>
<input id="btn-upload-format" name="btn-upload-format" type="file">
</div>

View File

@ -12,7 +12,7 @@
<input type="password" class="form-control" id="password" name="password" placeholder="{{_('Choose a password')}}" required>
</div-->
<div class="form-group required">
<label for="email">{{_('Email address')}}</label>
<label for="email">{{_('E-mail address')}}</label>
<input type="email" class="form-control" id="email" name="email" placeholder="{{_('Your email address')}}" required>
</div>
<button type="submit" class="btn btn-primary">{{_('Register')}}</button>

View File

@ -1,7 +1,7 @@
{% extends "layout.html" %}
{% block body %}
<div class="well">
<h2 style="margin-top: 0">{{_('Remote Login')}}</h2>
<h2 style="margin-top: 0">{{_('Remote login')}}</h2>
<p>
{{_('Using your another device, visit')}} <a href="{{verify_url}}">{{verify_url}}</a> {{_('and log in')}}.
</p>

View File

@ -10,7 +10,7 @@
</div>
{% endif %}
<div class="form-group">
<label for="email">{{_('Email address')}}</label>
<label for="email">{{_('E-mail address')}}</label>
<input type="email" class="form-control" name="email" id="email" value="{{ content.email if content.email != None }}" autocomplete="off">
</div>
{% if ( g.user and g.user.role_passwd() or g.user.role_admin() ) and not content.role_anonymous() %}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -3271,10 +3271,28 @@ def edit_book(book_id):
for authr in book.authors:
author_names.append(authr.name.replace('|', ','))
# Option for showing convertbook button
if config.config_ebookconverter == 2:
display_convertbtn = True
else:
display_convertbtn = False
# Determine what formats don't already exist
allowed_conversion_formats = ALLOWED_EXTENSIONS.copy()
for file in book.data:
try:
allowed_conversion_formats.remove(file.format.lower())
except Exception:
app.logger.warning(file.format.lower() + ' already removed from list.')
app.logger.debug('Allowed conversion formats:')
app.logger.debug(allowed_conversion_formats)
# Show form
if request.method != 'POST':
return render_title_template('book_edit.html', book=book, authors=author_names, cc=cc,
title=_(u"edit metadata"), page="editbook")
title=_(u"edit metadata"), page="editbook", display_convertbtn=display_convertbtn,
conversion_formats=allowed_conversion_formats)
# Update book
edited_books_id = set()
@ -3700,3 +3718,30 @@ def upload():
return render_title_template('detail.html', entry=book, cc=cc,
title=book.title, books_shelfs=book_in_shelfs, page="upload")
return redirect(url_for("index"))
@app.route("/admin/book/convert/<int:book_id>", methods=['POST'])
@login_required_if_no_ano
@edit_required
def convert_bookformat(book_id):
# check to see if we have form fields to work with - if not send user back
book_format_from = request.form.get('book_format_from', None)
book_format_to = request.form.get('book_format_to', None)
if (book_format_from is None) or (book_format_to is None):
flash(_(u"Source or destination format for conversion missing"), category="error")
return redirect(request.environ["HTTP_REFERER"])
app.logger.debug('converting: book id: ' + str(book_id) +
' from: ' + request.form['book_format_from'] +
' to: ' + request.form['book_format_to'])
rtn = helper.convert_book_format(book_id, config.config_calibre_dir, book_format_from.upper(),
book_format_to.upper(), current_user.nickname)
if rtn is None:
flash(_(u"Book successfully queued for converting to %(book_format)s",
book_format=book_format_to),
category="success")
else:
flash(_(u"There was an error converting this book: %(res)s", res=rtn), category="error")
return redirect(request.environ["HTTP_REFERER"])

View File

@ -42,6 +42,7 @@ STAT_FINISH_SUCCESS = 3
TASK_EMAIL = 1
TASK_CONVERT = 2
TASK_UPLOAD = 3
TASK_CONVERT_ANY = 4
RET_FAIL = 0
RET_SUCCESS = 1
@ -171,7 +172,9 @@ class WorkerThread(threading.Thread):
if self.queue[self.current]['typ'] == TASK_EMAIL:
self.send_raw_email()
if self.queue[self.current]['typ'] == TASK_CONVERT:
self.convert_mobi()
self.convert_any_format()
if self.queue[self.current]['typ'] == TASK_CONVERT_ANY:
self.convert_any_format()
# TASK_UPLOAD is handled implicitly
self.current += 1
else:
@ -200,61 +203,75 @@ class WorkerThread(threading.Thread):
def get_taskstatus(self):
if self.current < len(self.queue):
if self.queue[self.current]['status'] == STAT_STARTED:
if not self.queue[self.current]['typ'] == TASK_CONVERT:
if self.queue[self.current]['typ'] == TASK_EMAIL:
self.UIqueue[self.current]['progress'] = self.get_send_status()
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return self.UIqueue
def convert_mobi(self):
def convert_any_format(self):
# convert book, and upload in case of google drive
self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['status'] = _('Started')
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
filename=self.convert()
if web.ub.config.config_use_google_drive:
gd.updateGdriveCalibreFromLocal()
if(filename):
self.add_email(_(u'Send to Kindle'), self.queue[self.current]['path'], filename,
curr_task = self.queue[self.current]['typ']
filename = self.convert_ebook_format()
if filename:
if web.ub.config.config_use_google_drive:
gd.updateGdriveCalibreFromLocal()
if curr_task == TASK_CONVERT:
self.add_email(_(u'Send to Kindle'), self.queue[self.current]['path'], filename,
self.queue[self.current]['settings'], self.queue[self.current]['kindle'],
self.UIqueue[self.current]['user'], _(u"E-mail: %s" % self.queue[self.current]['title']))
def convert(self):
def convert_ebook_format(self):
error_message = None
file_path = self.queue[self.current]['file_path']
bookid = self.queue[self.current]['bookid']
# check if converter-excecutable is existing
format_old_ext = u'.' + self.queue[self.current]['settings']['old_book_format'].lower()
format_new_ext = u'.' + self.queue[self.current]['settings']['new_book_format'].lower()
# check if converter-executable is existing
if not os.path.exists(web.ub.config.config_converterpath):
self._handleError(_(u"Convertertool %(converter)s not found", converter=web.ub.config.config_converterpath))
return
try:
# check which converter to use kindlegen is "1"
if web.ub.config.config_ebookconverter == 1:
command = [web.ub.config.config_converterpath, u'"' + file_path + u'.epub"']
else:
if format_old_ext == '.epub' and format_new_ext == '.mobi':
if web.ub.config.config_ebookconverter == 1:
if os.name == 'nt':
command = web.ub.config.config_converterpath + u' "' + file_path + u'.epub"'
if sys.version_info < (3, 0):
command = command.encode(sys.getfilesystemencoding())
else:
command = [web.ub.config.config_converterpath, file_path + u'.epub']
if sys.version_info < (3, 0):
command = [x.encode(sys.getfilesystemencoding()) for x in command]
if web.ub.config.config_ebookconverter == 2:
# Linux py2.7 encode as list without quotes no empty element for parameters
# linux py3.x no encode and as list without quotes no empty element for parameters
# windows py2.7 encode as string with qoutes empty element for parameters is okay
# windows py 3.x no encode and as string with qoutes empty element for parameters is okay
# seperate handling for windows and linux
# windows py2.7 encode as string with quotes empty element for parameters is okay
# windows py 3.x no encode and as string with quotes empty element for parameters is okay
# separate handling for windows and linux
if os.name == 'nt':
command = web.ub.config.config_converterpath + u' "' + file_path + u'.epub" "' + \
file_path + u'.mobi" ' + web.ub.config.config_calibre
command = web.ub.config.config_converterpath + u' "' + file_path + format_old_ext + u'" "' + \
file_path + format_new_ext + u'" ' + web.ub.config.config_calibre
if sys.version_info < (3, 0):
command = command.encode(sys.getfilesystemencoding())
else:
command = [web.ub.config.config_converterpath, (file_path + u'.epub'),
(file_path + u'.mobi')]
command = [web.ub.config.config_converterpath, (file_path + format_old_ext),
(file_path + format_new_ext)]
if web.ub.config.config_calibre:
command.append(web.ub.config.config_calibre)
if sys.version_info < (3, 0):
command = [ x.encode(sys.getfilesystemencoding()) for x in command ]
command = [x.encode(sys.getfilesystemencoding()) for x in command]
p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
except OSError as e:
self._handleError(_(u"Ebook-converter failed: %s" % e))
return
if web.ub.config.config_ebookconverter == 1:
nextline = p.communicate()[0]
# Format of error message (kindlegen translates its output texts):
@ -265,7 +282,6 @@ class WorkerThread(threading.Thread):
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
error=conv_error.group(1), message=conv_error.group(2).strip())
web.app.logger.debug("convert_kindlegen: " + nextline)
else:
while p.poll() is None:
nextline = p.stdout.readline()
@ -277,30 +293,31 @@ class WorkerThread(threading.Thread):
if progress:
self.UIqueue[self.current]['progress'] = progress.group(1) + ' %'
#process returncode
# process returncode
check = p.returncode
# kindlegen returncodes
# 0 = Info(prcgen):I1036: Mobi file built successfully
# 1 = Info(prcgen):I1037: Mobi file built with WARNINGS!
# 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors!
if ( check < 2 and web.ub.config.config_ebookconverter == 1) or \
(check == 0 and web.ub.config.config_ebookconverter == 2):
if (check < 2 and web.ub.config.config_ebookconverter == 1) or \
(check == 0 and web.ub.config.config_ebookconverter == 2):
cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first()
new_format = web.db.Data(name=cur_book.data[0].name,book_format="MOBI",
book=bookid,uncompressed_size=os.path.getsize(file_path + ".mobi"))
new_format = web.db.Data(name=cur_book.data[0].name,
book_format=self.queue[self.current]['settings']['new_book_format'],
book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext))
cur_book.data.append(new_format)
web.db.session.commit()
self.queue[self.current]['path'] = cur_book.path
self.queue[self.current]['title'] = cur_book.title
if web.ub.config.config_use_google_drive:
os.remove(file_path + u".epub")
os.remove(file_path + format_old_ext)
self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['status'] = _('Finished')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return file_path + ".mobi"
datetime.now() - self.queue[self.current]['starttime'])
return file_path + format_new_ext
else:
web.app.logger.info("ebook converter failed with error while converting book")
if not error_message:
@ -309,18 +326,20 @@ class WorkerThread(threading.Thread):
return
def add_convert(self, file_path, bookid, user_name, typ, settings, kindle_mail):
def add_convert(self, file_path, bookid, user_name, typ, settings, kindle_mail=None):
addLock = threading.Lock()
addLock.acquire()
if self.last >= 20:
self.delete_completed_tasks()
# progress, runtime, and status = 0
self.id += 1
self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, 'kindle':kindle_mail,
'status': STAT_WAITING, 'typ': TASK_CONVERT, 'settings':settings})
task = TASK_CONVERT_ANY
if kindle_mail:
task = TASK_CONVERT
self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, 'kindle': kindle_mail,
'status': STAT_WAITING, 'typ': task, 'settings':settings})
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': typ,
'runtime': '0 s', 'status': _('Waiting'),'id': self.id } )
self.id += 1
self.last=len(self.queue)
addLock.release()
@ -334,12 +353,12 @@ class WorkerThread(threading.Thread):
if self.last >= 20:
self.delete_completed_tasks()
# progress, runtime, and status = 0
self.id += 1
self.queue.append({'subject':subject, 'attachment':attachment, 'filepath':filepath,
'settings':settings, 'recipent':recipient, 'starttime': 0,
'status': STAT_WAITING, 'typ': TASK_EMAIL, 'text':text})
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': typ,
'runtime': '0 s', 'status': _('Waiting'),'id': self.id })
self.id += 1
self.last=len(self.queue)
addLock.release()
@ -350,15 +369,15 @@ class WorkerThread(threading.Thread):
if self.last >= 20:
self.delete_completed_tasks()
# progress=100%, runtime=0, and status finished
self.id += 1
self.queue.append({'starttime': datetime.now(), 'status': STAT_FINISH_SUCCESS, 'typ': TASK_UPLOAD})
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': "100 %", 'type': typ,
'runtime': '0 s', 'status': _('Finished'),'id': self.id })
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
self.id += 1
self.last=len(self.queue)
addLock.release()
def send_raw_email(self):
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']

File diff suppressed because it is too large Load Diff