diff --git a/cps/asyncmail.py b/cps/asyncmail.py index d7707f26..857ca0d1 100644 --- a/cps/asyncmail.py +++ b/cps/asyncmail.py @@ -135,7 +135,7 @@ class EMailThread(threading.Thread): return self.UIqueue - def add_email(self, data, settings, recipient, user_name): + def add_email(self, data, settings, recipient, user_name, type): # if more than 50 entries in the list, clean the list addLock = threading.Lock() addLock.acquire() @@ -144,7 +144,7 @@ class EMailThread(threading.Thread): # progress, runtime, and status = 0 self.queue.append({'data':data, 'settings':settings, 'recipent':recipient, 'starttime': 0, 'status': STAT_WAITING}) - self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': 'E-Mail', + self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': type, 'runtime': '0 s', 'status': _('Waiting') }) # access issue self.last=len(self.queue) diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 17e51f0c..c00543b9 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -9,6 +9,7 @@ import os from ub import config import cli import shutil +from flask import Response, stream_with_context from sqlalchemy import * from sqlalchemy.ext.declarative import declarative_base @@ -281,18 +282,18 @@ def moveGdriveFolderRemote(origin_file, target_folder): # drive.auth.service.files().delete(fileId=previous_parents).execute() -def downloadFile(path, filename, output): - f = getFileFromEbooksFolder(path, filename) - f.GetContentFile(output) +#def downloadFile(path, filename, output): +# f = getFileFromEbooksFolder(path, filename) +# return f.GetContentFile(output) - -def backupCalibreDbAndOptionalDownload(drive, f=None): +# ToDo: Check purpose Parameter f ??, purpose of function ? +def backupCalibreDbAndOptionalDownload(drive): drive = getDrive(drive) metaDataFile = "'%s' in parents and title = 'metadata.db' and trashed = false" % getEbooksFolderId() fileList = drive.ListFile({'q': metaDataFile}).GetList() - databaseFile = fileList[0] - if f: - databaseFile.GetContentFile(f) + #databaseFile = fileList[0] + #if f: + # databaseFile.GetContentFile(f) def copyToDrive(drive, uploadFile, createRoot, replaceFiles, @@ -446,7 +447,7 @@ def deleteDatabaseOnChange(): session.commit() def updateGdriveCalibreFromLocal(): - backupCalibreDbAndOptionalDownload(Gdrive.Instance().drive) + # backupCalibreDbAndOptionalDownload(Gdrive.Instance().drive) copyToDrive(Gdrive.Instance().drive, config.config_calibre_dir, False, True) for x in os.listdir(config.config_calibre_dir): if os.path.isdir(os.path.join(config.config_calibre_dir, x)): @@ -463,3 +464,47 @@ def updateDatabaseOnEdit(ID,newPath): def deleteDatabaseEntry(ID): session.query(GdriveId).filter(GdriveId.gdrive_id == ID).delete() session.commit() + +# Gets cover file from gdrive +def get_cover_via_gdrive(cover_path): + df = getFileFromEbooksFolder(cover_path, 'cover.jpg') + if df: + if not session.query(PermissionAdded).filter(PermissionAdded.gdrive_id == df['id']).first(): + df.GetPermissions() + df.InsertPermission({ + 'type': 'anyone', + 'value': 'anyone', + 'role': 'reader', + 'withLink': True}) + permissionAdded = PermissionAdded() + permissionAdded.gdrive_id = df['id'] + session.add(permissionAdded) + session.commit() + return df.metadata.get('webContentLink') + else: + return None + +# Creates chunks for downloading big files +def partial(total_byte_len, part_size_limit): + s = [] + for p in range(0, total_byte_len, part_size_limit): + last = min(total_byte_len - 1, p + part_size_limit - 1) + s.append([p, last]) + return s + +# downloads files in chunks from gdrive +def do_gdrive_download(df, headers): + total_size = int(df.metadata.get('fileSize')) + download_url = df.metadata.get('downloadUrl') + s = partial(total_size, 1024 * 1024) # I'm downloading BIG files, so 100M chunk size is fine for me + + def stream(): + for byte in s: + headers = {"Range": 'bytes=%s-%s' % (byte[0], byte[1])} + resp, content = df.auth.Get_Http_Object().request(download_url, headers=headers) + if resp.status == 206: + yield content + else: + web.app.logger.info('An error occurred: %s' % resp) + return + return Response(stream_with_context(stream()), headers=headers) diff --git a/cps/helper.py b/cps/helper.py index 0c5cc2d2..a0241a08 100755 --- a/cps/helper.py +++ b/cps/helper.py @@ -14,6 +14,7 @@ import unicodedata from io import BytesIO import converter import asyncmail +import time try: from StringIO import StringIO @@ -29,6 +30,7 @@ except ImportError as e: from email import encoders from email.utils import formatdate from email.utils import make_msgid +from flask import send_from_directory, make_response, redirect, abort from flask_babel import gettext as _ import threading import shutil @@ -86,13 +88,14 @@ def send_test_mail(kindle_mail, user_name): msg['Subject'] = _(u'Calibre-web test email') text = _(u'This email has been sent via calibre web.') msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8')) - global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_name) + global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_name, _('Test E-Mail')) return # send_raw_email(kindle_mail, msg) def send_mail(book_id, kindle_mail, calibrepath, user_id): """Send email with attachments""" # create MIME message + result= None msg = MIMEMultipart() msg['Subject'] = _(u'Send to Kindle') msg['Message-Id'] = make_msgid('calibre-web') @@ -107,48 +110,72 @@ def send_mail(book_id, kindle_mail, calibrepath, user_id): for entry in data: if entry.format == "MOBI": - formats["mobi"] = os.path.join(calibrepath, book.path, entry.name + ".mobi") + formats["mobi"] = entry.name + ".mobi" # os.path.join(calibrepath, book.path, entry.name + ".mobi") if entry.format == "EPUB": - formats["epub"] = os.path.join(calibrepath, book.path, entry.name + ".epub") + formats["epub"] = entry.name + ".epub" # os.path.join(calibrepath, book.path, entry.name + ".epub") if entry.format == "PDF": - formats["pdf"] = os.path.join(calibrepath, book.path, entry.name + ".pdf") + formats["pdf"] = entry.name + ".pdf" # os.path.join(calibrepath, book.path, entry.name + ".pdf") if len(formats) == 0: return _("Could not find any formats suitable for sending by email") if 'mobi' in formats: - msg.attach(get_attachment(formats['mobi'])) + result = get_attachment(calibrepath, book.path, formats['mobi']) + if result: + msg.attach(result) elif 'epub' in formats: data, resultCode = make_mobi(book.id, calibrepath) if resultCode == RET_SUCCESS: - msg.attach(get_attachment(data)) + result = get_attachment(calibrepath, book.path, data) # toDo check data + if result: + msg.attach(result) else: app.logger.error = data return data # _("Could not convert epub to mobi") elif 'pdf' in formats: - msg.attach(get_attachment(formats['pdf'])) + result = get_attachment(calibrepath, book.path, formats['pdf']) + if result: + msg.attach(result) else: return _("Could not find any formats suitable for sending by email") - global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_id) - return None # send_raw_email(kindle_mail, msg) + if result: + global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_id, _(u"E-Mail: %s" % book.title)) + return None # send_raw_email(kindle_mail, msg) + else: + return _('The requested file could not be read. Maybe wrong permissions?') - -def get_attachment(file_path): +def get_attachment(calibrepath, bookpath, filename): """Get file as MIMEBase message""" - try: - file_ = open(file_path, 'rb') - attachment = MIMEBase('application', 'octet-stream') - attachment.set_payload(file_.read()) - file_.close() - encoders.encode_base64(attachment) + if ub.config.config_use_google_drive: + df = gd.getFileFromEbooksFolder(bookpath, filename) + if df: + # tmpDir = gettempdir() + datafile = os.path.join(calibrepath, bookpath, filename) + if not os.path.exists(os.path.join(calibrepath, bookpath)): + os.makedirs(os.path.join(calibrepath, bookpath)) + df.GetContentFile(datafile) + file_ = open(datafile, 'rb') + data = file_.read() + file_.close() + os.remove(datafile) + else: + return None + else: + try: + file_ = open(os.path.join(calibrepath, bookpath, filename), 'rb') + data = file_.read() + file_.close() + except IOError: + traceback.print_exc() + app.logger.error = u'The requested file could not be read. Maybe wrong permissions?' + return None - attachment.add_header('Content-Disposition', 'attachment', - filename=os.path.basename(file_path)) - return attachment - except IOError: - traceback.print_exc() - app.logger.error = u'The requested file could not be read. Maybe wrong permissions?' - return None + attachment = MIMEBase('application', 'octet-stream') + attachment.set_payload(data) + encoders.encode_base64(attachment) + attachment.add_header('Content-Disposition', 'attachment', + filename=filename) + return attachment def get_valid_filename(value, replace_whitespace=True): @@ -309,6 +336,60 @@ def delete_book(book, calibrepath): return delete_book_gdrive(book) else: return delete_book_file(book, calibrepath) + +def get_book_cover(cover_path): + if ub.config.config_use_google_drive: + try: + path=gd.get_cover_via_gdrive(cover_path) + if path: + return redirect(path) + else: + web.app.logger.error(cover_path + '/cover.jpg not found on Google Drive') + return send_from_directory(os.path.join(os.path.dirname(__file__), "static"), "generic_cover.jpg") + except Exception as e: + web.app.logger.error("Error Message: "+e.message) + web.app.logger.exception(e) + # traceback.print_exc() + return send_from_directory(os.path.join(os.path.dirname(__file__), "static"),"generic_cover.jpg") + else: + return send_from_directory(os.path.join(ub.config.config_calibre_dir, cover_path), "cover.jpg") + +# saves book cover to gdrive or locally +def save_cover(url, book_path): + img = requests.get(url) + if img.headers.get('content-type') != 'image/jpeg': + web.app.logger.error("Cover is no jpg file, can't save") + return false + + if ub.config.config_use_google_drive: + tmpDir = gettempdir() + f = open(os.path.join(tmpDir, "uploaded_cover.jpg"), "wb") + f.write(img.content) + f.close() + uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg'), os.path.join(tmpDir, f.name)) + web.app.logger.info("Cover is saved on gdrive") + return true + + f = open(os.path.join(ub.config.config_calibre_dir, book_path, "cover.jpg"), "wb") + f.write(img.content) + f.close() + web.app.logger.info("Cover is saved") + return true + +def do_download_file(book, book_format, data, headers): + if ub.config.config_use_google_drive: + startTime = time.time() + df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) + web.app.logger.debug(time.time() - startTime) + if df: + return gd.do_gdrive_download(df, headers) + else: + abort(404) + else: + response = make_response(send_from_directory(os.path.join(ub.config.config_calibre_dir, book.path), data.name + "." + book_format)) + response.headers = headers + return response + ################################## diff --git a/cps/web.py b/cps/web.py index 0e304b56..7224bad5 100755 --- a/cps/web.py +++ b/cps/web.py @@ -27,7 +27,6 @@ except ImportError: import mimetypes import logging from logging.handlers import RotatingFileHandler -import textwrap from flask import (Flask, render_template, request, Response, redirect, url_for, send_from_directory, make_response, g, flash, abort, Markup, stream_with_context) @@ -70,7 +69,7 @@ import sys import re import db from shutil import move, copyfile -import shutil +# import shutil import gdriveutils import converter import tempfile @@ -842,36 +841,11 @@ def feed_shelf(book_id): response.headers["Content-Type"] = "application/atom+xml; charset=utf-8" return response -def partial(total_byte_len, part_size_limit): - s = [] - for p in range(0, total_byte_len, part_size_limit): - last = min(total_byte_len - 1, p + part_size_limit - 1) - s.append([p, last]) - return s - - -def do_gdrive_download(df, headers): - total_size = int(df.metadata.get('fileSize')) - download_url = df.metadata.get('downloadUrl') - s = partial(total_size, 1024 * 1024) # I'm downloading BIG files, so 100M chunk size is fine for me - - def stream(): - for byte in s: - headers = {"Range": 'bytes=%s-%s' % (byte[0], byte[1])} - resp, content = df.auth.Get_Http_Object().request(download_url, headers=headers) - if resp.status == 206: - yield content - else: - app.logger.info('An error occurred: %s' % resp) - return - return Response(stream_with_context(stream()), headers=headers) - @app.route("/opds/download///") @requires_basic_auth_if_no_ano @download_required def get_opds_download_link(book_id, book_format): - startTime = time.time() book_format = book_format.split(".")[0] book = db.session.query(db.Books).filter(db.Books.id == book_id).first() data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == book_format.upper()).first() @@ -888,16 +862,15 @@ def get_opds_download_link(book_id, book_format): headers["Content-Type"] = mimetypes.types_map['.' + book_format] except KeyError: headers["Content-Type"] = "application/octet-stream" - app.logger.info(time.time() - startTime) - startTime = time.time() - if config.config_use_google_drive: - app.logger.info(time.time() - startTime) - df = gdriveutils.getFileFromEbooksFolder(book.path, data.name + "." + book_format) - return do_gdrive_download(df, headers) - else: - response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + book_format)) - response.headers = headers - return response + return helper.do_download_file(book, book_format, data, headers) + #if config.config_use_google_drive: + # app.logger.info(time.time() - startTime) + # df = gdriveutils.getFileFromEbooksFolder(book.path, data.name + "." + book_format) + # return do_gdrive_download(df, headers) + #else: + # response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + book_format)) + # response.headers = headers + # return response @app.route("/ajax/book/") @@ -1652,7 +1625,7 @@ def on_received_watch_confirmation(): gdriveutils.downloadFile(None, "metadata.db", os.path.join(tmpDir, "tmp_metadata.db")) app.logger.info('Setting up new DB') # prevent error on windows, as os.rename does on exisiting files - shutil.move(os.path.join(tmpDir, "tmp_metadata.db"), dbpath) + move(os.path.join(tmpDir, "tmp_metadata.db"), dbpath) db.setup_db() except Exception as e: app.logger.info(e.message) @@ -1823,44 +1796,11 @@ def advanced_search(): series=series, title=_(u"search"), page="advsearch") -def get_cover_via_gdrive(cover_path): - df = gdriveutils.getFileFromEbooksFolder(cover_path, 'cover.jpg') - if df: - if not gdriveutils.session.query(gdriveutils.PermissionAdded).filter(gdriveutils.PermissionAdded.gdrive_id == df['id']).first(): - df.GetPermissions() - df.InsertPermission({ - 'type': 'anyone', - 'value': 'anyone', - 'role': 'reader', - 'withLink': True}) - permissionAdded = gdriveutils.PermissionAdded() - permissionAdded.gdrive_id = df['id'] - gdriveutils.session.add(permissionAdded) - gdriveutils.session.commit() - return df.metadata.get('webContentLink') - else: - return None - @app.route("/cover/") @login_required_if_no_ano def get_cover(cover_path): - if config.config_use_google_drive: - try: - path=get_cover_via_gdrive(cover_path) - if path: - return redirect(path) - else: - app.logger.error(cover_path + '/cover.jpg not found on Google Drive') - return send_from_directory(os.path.join(os.path.dirname(__file__), "static"), "generic_cover.jpg") - except Exception as e: - app.logger.error("Error Message: "+e.message) - app.logger.exception(e) - # traceback.print_exc() - return send_from_directory(os.path.join(os.path.dirname(__file__), "static"),"generic_cover.jpg") - else: - return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg") - + return helper.get_book_cover(cover_path) @app.route("/show//") @login_required_if_no_ano @@ -1888,10 +1828,7 @@ def serve_book(book_id, book_format): @requires_basic_auth_if_no_ano def feed_get_cover(book_id): book = db.session.query(db.Books).filter(db.Books.id == book_id).first() - if config.config_use_google_drive: - return redirect(get_cover_via_gdrive(book.path)) - else: - return send_from_directory(os.path.join(config.config_calibre_dir, book.path), "cover.jpg") + return helper.get_book_cover(book.path) def render_read_books(page, are_read, as_xml=False): @@ -2019,16 +1956,17 @@ def get_download_link(book_id, book_format): except KeyError: headers["Content-Type"] = "application/octet-stream" headers["Content-Disposition"] = "attachment; filename*=UTF-8''%s.%s" % (quote(file_name.encode('utf-8')), book_format) - if config.config_use_google_drive: - df = gdriveutils.getFileFromEbooksFolder(book.path, '%s.%s' % (data.name, book_format)) - if df: - return do_gdrive_download(df, headers) - else: - abort(404) - else: - response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + book_format)) - response.headers = headers - return response + return helper.do_download_file(book, book_format, data, headers) + #if config.config_use_google_drive: + # df = gdriveutils.getFileFromEbooksFolder(book.path, '%s.%s' % (data.name, book_format)) + # if df: + # return do_gdrive_download(df, headers) + # else: + # abort(404) + #else: + # response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + book_format)) + # response.headers = headers + # return response else: abort(404) @@ -3158,7 +3096,7 @@ def edit_book(book_id): if not error: if to_save["cover_url"]: - if save_cover(to_save["cover_url"], book.path) is true: + if helper.save_cover(to_save["cover_url"], book.path) is true: book.has_cover = 1 else: flash(_(u"Cover is not a jpg file, can't save"), category="error") @@ -3315,28 +3253,6 @@ def edit_book(book_id): return redirect(url_for('show_book', book_id=book.id)) -def save_cover(url, book_path): - img = requests.get(url) - if img.headers.get('content-type') != 'image/jpeg': - app.logger.error("Cover is no jpg file, can't save") - return false - - if config.config_use_google_drive: - tmpDir = tempfile.gettempdir() - f = open(os.path.join(tmpDir, "uploaded_cover.jpg"), "wb") - f.write(img.content) - f.close() - gdriveutils.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg'), os.path.join(tmpDir, f.name)) - app.logger.info("Cover is saved on gdrive") - return true - - f = open(os.path.join(config.config_calibre_dir, book_path, "cover.jpg"), "wb") - f.write(img.content) - f.close() - app.logger.info("Cover is saved") - return true - - @app.route("/upload", methods=["GET", "POST"]) @login_required_if_no_ano @upload_required