Added ability to send emails via gmail (#1905)

Gmail email sending

cps/admin.py

@@ -39,7 +39,7 @@ from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError
 from sqlalchemy.sql.expression import func, or_
 
-from . import constants, logger, helper, services, gmail
+from . import constants, logger, helper, services
 from .cli import filepicker
 from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
 from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash
@@ -58,7 +58,8 @@ feature_support = {
         'ldap': bool(services.ldap),
         'goodreads': bool(services.goodreads_support),
         'kobo':  bool(services.kobo),
-        'updater': constants.UPDATER_AVAILABLE
+        'updater': constants.UPDATER_AVAILABLE,
+        'gmail': bool(services.gmail)
     }
 
 try:
@@ -1311,7 +1312,7 @@ def new_user():
 def edit_mailsettings():
     content = config.get_mail_settings()
     return render_title_template("email_edit.html", content=content, title=_(u"Edit E-mail Server Settings"),
-                                 page="mailset")
+                                 page="mailset", feature_support=feature_support)
 
 
 @admi.route("/admin/mailsettings", methods=["POST"])
@@ -1320,15 +1321,21 @@ def edit_mailsettings():
 def update_mailsettings():
     to_save = request.form.to_dict()
     _config_int(to_save, "mail_server_type")
-    if to_save.get("invalidate_server"):
+    if to_save.get("invalidate"):
         config.mail_gmail_token = {}
         try:
             flag_modified(config, "mail_gmail_token")
         except AttributeError:
             pass
     elif to_save.get("gmail"):
-        config.mail_gmail_token = gmail.setup_gmail(config)
+        try:
+            config.mail_gmail_token = services.gmail.setup_gmail(config.mail_gmail_token)
             flash(_(u"G-Mail Account Verification Successfull"), category="success")
+        except Exception as e:
+            flash(e, category="error")
+            log.error(e)
+            return edit_mailsettings()
+
     else:
         _config_string(to_save, "mail_server")
         _config_int(to_save, "mail_port")

cps/config_sql.py

@@ -249,15 +249,15 @@ class _ConfigSQL(object):
 
     def get_mail_server_configured(self):
         return bool((self.mail_server != constants.DEFAULT_MAIL_SERVER and self.mail_server_type == 0)
-                    or (self.mail_gmail_token != b"" and self.mail_server_type == 1))
+                    or (self.mail_gmail_token != {} and self.mail_server_type == 1))
 
 
     def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None):
-        '''Possibly updates a field of this object.
+        """Possibly updates a field of this object.
         The new value, if present, is grabbed from the given dictionary, and optionally passed through a convertor.
 
         :returns: `True` if the field has changed value
-        '''
+        """
         new_value = dictionary.get(field, default)
         if new_value is None:
             # log.debug("_ConfigSQL set_from_dictionary field '%s' not found", field)
@@ -308,6 +308,9 @@ class _ConfigSQL(object):
                 have_metadata_db = os.path.isfile(db_file)
         self.db_configured = have_metadata_db
         constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip().lower() for x in self.config_upload_formats.split(',')]
+        if os.environ.get('FLASK_DEBUG'):
+            logfile = logger.setup(logger.LOG_TO_STDOUT, logger.logging.DEBUG)
+        else:
             # pylint: disable=access-member-before-definition
             logfile = logger.setup(self.config_logfile, self.config_log_level)
         if logfile != self.config_logfile:

cps/gmail.py

@@ -1,64 +0,0 @@
-from __future__ import print_function
-import os.path
-from googleapiclient.discovery import build
-from google_auth_oauthlib.flow import InstalledAppFlow
-from google.auth.transport.requests import Request
-from google.oauth2.credentials import Credentials
-from .constants import BASE_DIR
-import json
-from datetime import datetime
-
-subject = "Test"
-msg = "Testnachricht"
-sender = "matthias1.knopp@googlemail.com"
-receiver = "matthias.knopp@web.de"
-
-SCOPES = ['https://www.googleapis.com/auth/gmail.send']
-
-def setup_gmail(config):
-    token = config.mail_gmail_token
-    # if config.mail_gmail_token != "{}":
-    # If there are no (valid) credentials available, let the user log in.
-    creds = None
-    if "token" in token:
-        creds = Credentials(
-            token=token['token'],
-            refresh_token=token['refresh_token'],
-            token_uri=token['token_uri'],
-            client_id=token['client_id'],
-            client_secret=token['client_secret'],
-            scopes=token['scopes'],
-        )
-        creds.expiry = datetime.fromisoformat(token['expiry'])
-
-    if not creds or not creds.valid:
-        # don't forget to dump one more time after the refresh
-        # also, some file-locking routines wouldn't be needless
-        if creds and creds.expired and creds.refresh_token:
-            creds.refresh(Request())
-        else:
-            flow = InstalledAppFlow.from_client_secrets_file(
-                os.path.join(BASE_DIR, 'gmail.json'), SCOPES)
-            creds = flow.run_local_server(port=0)
-
-        return {
-            'token': creds.token,
-            'refresh_token': creds.refresh_token,
-            'token_uri': creds.token_uri,
-            'client_id': creds.client_id,
-            'client_secret': creds.client_secret,
-            'scopes': creds.scopes,
-            'expiry': creds.expiry.isoformat(),
-        }
-
-# implement your storage logic here, e.g. just good old json.dump() / json.load()
-
-# service = build('gmail', 'v1', credentials=creds)
-# message = MIMEText(msg)
-# message['to'] = receiver
-# message['from'] = sender
-# message['subject'] = subject
-# raw = base64.urlsafe_b64encode(message.as_bytes())
-# raw = raw.decode()
-# body = {'raw' : raw}
-# message = (service.users().messages().send(userId='me', body=body).execute())

cps/services/__init__.py

@@ -45,3 +45,9 @@ except ImportError as err:
     log.debug("Cannot import SyncToken, syncing books with Kobo Devices will not work: %s", err)
     kobo = None
     SyncToken = None
+
+try:
+    from . import gmail
+except ImportError as err:
+    log.debug("Cannot import Gmail, sending books via G-Mail Accounts will not work: %s", err)
+    gmail = None

cps/services/gmail.py

@@ -0,0 +1,80 @@
+from __future__ import print_function
+import os.path
+from google_auth_oauthlib.flow import InstalledAppFlow
+from google.auth.transport.requests import Request
+from googleapiclient.discovery import build
+from google.oauth2.credentials import Credentials
+
+from datetime import datetime
+import base64
+from flask_babel import gettext as _
+from ..constants import BASE_DIR
+from .. import logger
+
+
+log = logger.create()
+
+SCOPES = ['openid', 'https://www.googleapis.com/auth/gmail.send', 'https://www.googleapis.com/auth/userinfo.email']
+
+def setup_gmail(token):
+    # If there are no (valid) credentials available, let the user log in.
+    creds = None
+    if "token" in token:
+        creds = Credentials(
+            token=token['token'],
+            refresh_token=token['refresh_token'],
+            token_uri=token['token_uri'],
+            client_id=token['client_id'],
+            client_secret=token['client_secret'],
+            scopes=token['scopes'],
+        )
+        creds.expiry = datetime.fromisoformat(token['expiry'])
+
+    if not creds or not creds.valid:
+        # don't forget to dump one more time after the refresh
+        # also, some file-locking routines wouldn't be needless
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            cred_file = os.path.join(BASE_DIR, 'gmail.json')
+            if not os.path.exists(cred_file):
+                raise Exception(_("Found no valid gmail.json file with OAuth information"))
+            flow = InstalledAppFlow.from_client_secrets_file(
+                os.path.join(BASE_DIR, 'gmail.json'), SCOPES)
+            creds = flow.run_local_server(port=0)
+            user_info = get_user_info(creds)
+        return {
+            'token': creds.token,
+            'refresh_token': creds.refresh_token,
+            'token_uri': creds.token_uri,
+            'client_id': creds.client_id,
+            'client_secret': creds.client_secret,
+            'scopes': creds.scopes,
+            'expiry': creds.expiry.isoformat(),
+            'email': user_info
+        }
+
+def get_user_info(credentials):
+    user_info_service = build(serviceName='oauth2', version='v2',credentials=credentials)
+    user_info = user_info_service.userinfo().get().execute()
+    return user_info.get('email', "")
+
+def send_messsage(token, msg):
+    creds = Credentials(
+        token=token['token'],
+        refresh_token=token['refresh_token'],
+        token_uri=token['token_uri'],
+        client_id=token['client_id'],
+        client_secret=token['client_secret'],
+        scopes=token['scopes'],
+    )
+    creds.expiry = datetime.fromisoformat(token['expiry'])
+    if creds and creds.expired and creds.refresh_token:
+        creds.refresh(Request())
+    service = build('gmail', 'v1', credentials=creds)
+    message_as_bytes = msg.as_bytes()  # the message should converted from string to bytes.
+    message_as_base64 = base64.urlsafe_b64encode(message_as_bytes)  # encode in base64 (printable letters coding)
+    raw = message_as_base64.decode()  # convert to something  JSON serializable
+    body = {'raw': raw}
+
+    (service.users().messages().send(userId='me', body=body).execute())

cps/tasks/mail.py

@@ -4,6 +4,8 @@ import os
 import smtplib
 import threading
 import socket
+import mimetypes
+import base64
 
 try:
     from StringIO import StringIO
@@ -16,11 +18,14 @@ except ImportError:
     from email.mime.multipart import MIMEMultipart
     from email.mime.text import MIMEText
 
+
+
 from email import encoders
 from email.utils import formatdate, make_msgid
 from email.generator import Generator
 
 from cps.services.worker import CalibreTask
+from cps.services import gmail
 from cps import logger, config
 
 from cps import gdriveutils
@@ -98,7 +103,7 @@ class EmailSSL(EmailBase, smtplib.SMTP_SSL):
 
 
 class TaskEmail(CalibreTask):
-    def __init__(self, subject, filepath, attachment, settings, recipient, taskMessage, text, internal=False):
+    def __init__(self, subject, filepath, attachment, settings, recipient, taskMessage, text):
         super(TaskEmail, self).__init__(taskMessage)
         self.subject = subject
         self.attachment = attachment
@@ -107,39 +112,59 @@ class TaskEmail(CalibreTask):
         self.recipent = recipient
         self.text = text
         self.asyncSMTP = None
-
         self.results = dict()
 
     def prepare_message(self):
-        msg = MIMEMultipart()
+        message = MIMEMultipart()
-        msg['Subject'] = self.subject
+        message['to'] = self.recipent
-        msg['Message-Id'] = make_msgid('calibre-web')
+        message['from'] = self.settings["mail_from"]
-        msg['Date'] = formatdate(localtime=True)
+        message['subject'] = self.subject
+        message['Message-Id'] = make_msgid('calibre-web')
+        message['Date'] = formatdate(localtime=True)
         text = self.text
-        msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
+        msg = MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8')
+        message.attach(msg)
         if self.attachment:
             result = self._get_attachment(self.filepath, self.attachment)
             if result:
-                msg.attach(result)
+                message.attach(result)
             else:
                 self._handleError(u"Attachment not found")
                 return
-
+        return message
-        msg['From'] = self.settings["mail_from"]
-        msg['To'] = self.recipent
-        # convert MIME message to string
-        fp = StringIO()
-        gen = Generator(fp, mangle_from_=False)
-        gen.flatten(msg)
-        return fp.getvalue()
 
     def run(self, worker_thread):
         # create MIME message
         msg = self.prepare_message()
-
-        use_ssl = int(self.settings.get('mail_use_ssl', 0))
         try:
-            # send email
+            if self.settings['mail_server_type'] == 0:
+                self.send_standard_email(msg)
+            else:
+                self.send_gmail_email(msg)
+        except MemoryError as e:
+            log.debug_or_exception(e)
+            self._handleError(u'MemoryError sending email: {}'.format(str(e)))
+        except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
+            log.debug_or_exception(e)
+            if hasattr(e, "smtp_error"):
+                text = e.smtp_error.decode('utf-8').replace("\n", '. ')
+            elif hasattr(e, "message"):
+                text = e.message
+            elif hasattr(e, "args"):
+                text = '\n'.join(e.args)
+            else:
+                text = ''
+            self._handleError(u'Smtplib Error sending email: {}'.format(text))
+        except socket.error as e:
+            log.debug_or_exception(e)
+            self._handleError(u'Socket Error sending email: {}'.format(e.strerror))
+        except Exception as e:
+            log.debug_or_exception(e)
+            self._handleError(u'Error sending email: {}'.format(e))
+
+
+    def send_standard_email(self, msg):
+        use_ssl = int(self.settings.get('mail_use_ssl', 0))
         timeout = 600  # set timeout to 5mins
 
         # redirect output to logfile on python2 on python3 debugoutput is caught with overwritten
@@ -161,31 +186,22 @@ class TaskEmail(CalibreTask):
             self.asyncSMTP.starttls()
         if self.settings["mail_password"]:
             self.asyncSMTP.login(str(self.settings["mail_login"]), str(self.settings["mail_password"]))
-            self.asyncSMTP.sendmail(self.settings["mail_from"], self.recipent, msg)
+
+        # Convert message to something to send
+        fp = StringIO()
+        gen = Generator(fp, mangle_from_=False)
+        gen.flatten(msg)
+
+        self.asyncSMTP.sendmail(self.settings["mail_from"], self.recipent, fp.getvalue())
         self.asyncSMTP.quit()
         self._handleSuccess()
 
         if sys.version_info < (3, 0):
             smtplib.stderr = org_smtpstderr
 
-        except (MemoryError) as e:
-            log.debug_or_exception(e)
-            self._handleError(u'MemoryError sending email: ' + str(e))
-        except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
-            log.debug_or_exception(e)
-            if hasattr(e, "smtp_error"):
-                text = e.smtp_error.decode('utf-8').replace("\n", '. ')
-            elif hasattr(e, "message"):
-                text = e.message
-            elif hasattr(e, "args"):
-                text = '\n'.join(e.args)
-            else:
-                text = ''
-            self._handleError(u'Smtplib Error sending email: ' + text)
-        except (socket.error) as e:
-            log.debug_or_exception(e)
-            self._handleError(u'Socket Error sending email: ' + e.strerror)
 
+    def send_gmail_email(self, message):
+        return gmail.send_messsage(self.settings.get('mail_gmail_token', None), message)
 
     @property
     def progress(self):
@@ -205,13 +221,13 @@ class TaskEmail(CalibreTask):
     @classmethod
     def _get_attachment(cls, bookpath, filename):
         """Get file as MIMEBase message"""
-        calibrepath = config.config_calibre_dir
+        calibre_path = config.config_calibre_dir
         if config.config_use_google_drive:
             df = gdriveutils.getFileFromEbooksFolder(bookpath, filename)
             if df:
-                datafile = os.path.join(calibrepath, bookpath, filename)
+                datafile = os.path.join(calibre_path, bookpath, filename)
-                if not os.path.exists(os.path.join(calibrepath, bookpath)):
+                if not os.path.exists(os.path.join(calibre_path, bookpath)):
-                    os.makedirs(os.path.join(calibrepath, bookpath))
+                    os.makedirs(os.path.join(calibre_path, bookpath))
                 df.GetContentFile(datafile)
             else:
                 return None
@@ -221,19 +237,22 @@ class TaskEmail(CalibreTask):
             os.remove(datafile)
         else:
             try:
-                file_ = open(os.path.join(calibrepath, bookpath, filename), 'rb')
+                file_ = open(os.path.join(calibre_path, bookpath, filename), 'rb')
                 data = file_.read()
                 file_.close()
             except IOError as e:
                 log.debug_or_exception(e)
                 log.error(u'The requested file could not be read. Maybe wrong permissions?')
                 return None
-
+        # Set mimetype
-        attachment = MIMEBase('application', 'octet-stream')
+        content_type, encoding = mimetypes.guess_type(filename)
+        if content_type is None or encoding is not None:
+            content_type = 'application/octet-stream'
+        main_type, sub_type = content_type.split('/', 1)
+        attachment = MIMEBase(main_type, sub_type)
         attachment.set_payload(data)
         encoders.encode_base64(attachment)
-        attachment.add_header('Content-Disposition', 'attachment',
+        attachment.add_header('Content-Disposition', 'attachment', filename=filename)
-                              filename=filename)
         return attachment
 
     @property

cps/templates/admin.html

@@ -55,6 +55,7 @@
     <div class="col">
       <h2>{{_('E-mail Server Settings')}}</h2>
       {% if config.get_mail_server_configured() %}
+        {% if email.mail_server_type == 0 %}
         <div class="col-xs-12 col-sm-12">
           <div class="row">
             <div class="col-xs-6 col-sm-3">{{_('SMTP Hostname')}}</div>
@@ -77,6 +78,18 @@
             <div class="col-xs-6 col-sm-3">{{email.mail_from}}</div>
           </div>
         </div>
+      {% else %}
+        <div class="col-xs-12 col-sm-12">
+          <div class="row">
+            <div class="col-xs-6 col-sm-3">{{_('E-Mail Service')}}</div>
+            <div class="col-xs-6 col-sm-3">{{_('Gmail via Oauth2')}}</div>
+          </div>
+          <div class="row">
+            <div class="col-xs-6 col-sm-3">{{_('From E-mail')}}</div>
+            <div class="col-xs-6 col-sm-3">{{email.mail_gmail_token['email']}}</div>
+          </div>
+        </div>
+      {% endif %}
     {% endif %}
       <a class="btn btn-default emailconfig" id="admin_edit_email" href="{{url_for('admin.edit_mailsettings')}}">{{_('Edit E-mail Server Settings')}}</a>
     </div>

cps/templates/email_edit.html

@@ -7,6 +7,7 @@
 <div class="discover">
   <h1>{{title}}</h1>
   <form role="form" class="col-md-10 col-lg-6" method="POST">
+    {% if feature_support['gmail'] %}
     <div class="form-group">
       <label for="mail_server_type">{{_('Choose Server Type')}}</label>
       <select name="mail_server_type" id="config_email_type" class="form-control" data-control="email-settings">
@@ -24,6 +25,7 @@
       </div>
     </div>
     <div data-related="email-settings-0">
+   {% endif %}
       <div class="form-group">
         <label for="mail_server">{{_('SMTP Hostname')}}</label>
         <input type="text" class="form-control" name="mail_server" id="mail_server" value="{{content.mail_server}}" required>
@@ -61,8 +63,10 @@
       </div>
       <button type="submit" name="submit" value="submit" class="btn btn-default">{{_('Save')}}</button>
       <button type="submit" name="test" value="test" class="btn btn-default">{{_('Save and Send Test E-mail')}}</button>
+   {% if feature_support['gmail'] %}
     </div>
-      <a href="{{ url_for('admin.admin') }}" id="back" class="btn btn-default">{{_('Cancel')}}</a>
+   {% endif %}
+      <a href="{{ url_for('admin.admin') }}" id="back" class="btn btn-default">{{_('Back')}}</a>
   </form>
     {% if g.allow_registration %}
   <div class="col-md-10 col-lg-6">

gmail.json

@@ -0,0 +1 @@
+{"installed":{"client_id":"686643671665-uglhp9pmlvjhsoq5q0528cttd16krgpj.apps.googleusercontent.com","project_id":"calibre-web-260207","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"hbLugwKAw0xqMctO1KZuhRKy"}}