Code refactoring

cps/admin.py

@@ -1495,6 +1495,51 @@ def get_updater_status():
     return ''
 
 
+def ldap_import_create_user(user, user_data):
+    user_login_field = extract_dynamic_field_from_filter(user, config.config_ldap_user_object)
+
+    username = user_data[user_login_field][0].decode('utf-8')
+    # check for duplicate username
+    if ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == username.lower()).first():
+        # if ub.session.query(ub.User).filter(ub.User.nickname == username).first():
+        log.warning("LDAP User  %s Already in Database", user_data)
+        return 0, None
+
+    kindlemail = ''
+    if 'mail' in user_data:
+        useremail = user_data['mail'][0].decode('utf-8')
+        if len(user_data['mail']) > 1:
+            kindlemail = user_data['mail'][1].decode('utf-8')
+
+    else:
+        log.debug('No Mail Field Found in LDAP Response')
+        useremail = username + '@email.com'
+    # check for duplicate email
+    if ub.session.query(ub.User).filter(func.lower(ub.User.email) == useremail.lower()).first():
+        log.warning("LDAP Email %s Already in Database", user_data)
+        return 0, None
+    content = ub.User()
+    content.nickname = username
+    content.password = ''  # dummy password which will be replaced by ldap one
+    content.email = useremail
+    content.kindle_mail = kindlemail
+    content.role = config.config_default_role
+    content.sidebar_view = config.config_default_show
+    content.allowed_tags = config.config_allowed_tags
+    content.denied_tags = config.config_denied_tags
+    content.allowed_column_value = config.config_allowed_column_value
+    content.denied_column_value = config.config_denied_column_value
+    ub.session.add(content)
+    try:
+        ub.session.commit()
+        return 1, None    # increase no of users
+    except Exception as e:
+        log.warning("Failed to create LDAP user: %s - %s", user, e)
+        ub.session.rollback()
+        message = _(u'Failed to Create at Least One LDAP User')
+        return 0, message
+
+
 @admi.route('/import_ldap_users')
 @login_required
 @admin_required
@@ -1534,47 +1579,11 @@ def import_ldap_users():
             log.debug_or_exception(e)
             continue
         if user_data:
-            user_login_field = extract_dynamic_field_from_filter(user, config.config_ldap_user_object)
+            user_count, message = ldap_import_create_user(user, user_data, showtext)
-
+            if message:
-            username = user_data[user_login_field][0].decode('utf-8')
+                showtext['text'] = message
-            # check for duplicate username
-            if ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == username.lower()).first():
-                # if ub.session.query(ub.User).filter(ub.User.nickname == username).first():
-                log.warning("LDAP User  %s Already in Database", user_data)
-                continue
-
-            kindlemail = ''
-            if 'mail' in user_data:
-                useremail = user_data['mail'][0].decode('utf-8')
-                if len(user_data['mail']) > 1:
-                    kindlemail = user_data['mail'][1].decode('utf-8')
-
             else:
-                log.debug('No Mail Field Found in LDAP Response')
+                imported += user_count
-                useremail = username + '@email.com'
-            # check for duplicate email
-            if ub.session.query(ub.User).filter(func.lower(ub.User.email) == useremail.lower()).first():
-                log.warning("LDAP Email %s Already in Database", user_data)
-                continue
-            content = ub.User()
-            content.nickname = username
-            content.password = ''  # dummy password which will be replaced by ldap one
-            content.email = useremail
-            content.kindle_mail = kindlemail
-            content.role = config.config_default_role
-            content.sidebar_view = config.config_default_show
-            content.allowed_tags = config.config_allowed_tags
-            content.denied_tags = config.config_denied_tags
-            content.allowed_column_value = config.config_allowed_column_value
-            content.denied_column_value = config.config_denied_column_value
-            ub.session.add(content)
-            try:
-                ub.session.commit()
-                imported += 1
-            except Exception as e:
-                log.warning("Failed to create LDAP user: %s - %s", user, e)
-                ub.session.rollback()
-                showtext['text'] = _(u'Failed to Create at Least One LDAP User')
         else:
             log.warning("LDAP User: %s Not Found", user)
             showtext['text'] = _(u'At Least One LDAP User Not Found in Database')

cps/db.py

@@ -442,49 +442,13 @@ class CalibreDB():
 
         self.instances.add(self)
 
-
     def initSession(self, expire_on_commit=True):
         self.session = self.session_factory()
         self.session.expire_on_commit = expire_on_commit
         self.update_title_sort(self.config)
 
     @classmethod
-    def setup_db(cls, config, app_db_path):
+    def setup_db_cc_classes(self, cc):
-        cls.config = config
-        cls.dispose()
-
-        # toDo: if db changed -> delete shelfs, delete download books, delete read boks, kobo sync??
-
-        if not config.config_calibre_dir:
-            config.invalidate()
-            return False
-
-        dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
-        if not os.path.exists(dbpath):
-            config.invalidate()
-            return False
-
-        try:
-            cls.engine = create_engine('sqlite://',
-                                       echo=False,
-                                       isolation_level="SERIALIZABLE",
-                                       connect_args={'check_same_thread': False},
-                                       poolclass=StaticPool)
-            with cls.engine.begin() as connection:
-                connection.execute(text("attach database '{}' as calibre;".format(dbpath)))
-                connection.execute(text("attach database '{}' as app_settings;".format(app_db_path)))
-
-            conn = cls.engine.connect()
-            # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
-        except Exception as e:
-            config.invalidate(e)
-            return False
-
-        config.db_configured = True
-
-        if not cc_classes:
-            cc = conn.execute(text("SELECT id, datatype FROM custom_columns"))
-
         cc_ids = []
         books_custom_column_links = {}
         for row in cc:
@@ -550,6 +514,46 @@ class CalibreDB():
                                      secondary=books_custom_column_links[cc_id[0]],
                                      backref='books'))
 
+        return cc_classes
+
+    @classmethod
+    def setup_db(cls, config, app_db_path):
+        cls.config = config
+        cls.dispose()
+
+        # toDo: if db changed -> delete shelfs, delete download books, delete read boks, kobo sync??
+
+        if not config.config_calibre_dir:
+            config.invalidate()
+            return False
+
+        dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
+        if not os.path.exists(dbpath):
+            config.invalidate()
+            return False
+
+        try:
+            cls.engine = create_engine('sqlite://',
+                                       echo=False,
+                                       isolation_level="SERIALIZABLE",
+                                       connect_args={'check_same_thread': False},
+                                       poolclass=StaticPool)
+            with cls.engine.begin() as connection:
+                connection.execute(text("attach database '{}' as calibre;".format(dbpath)))
+                connection.execute(text("attach database '{}' as app_settings;".format(app_db_path)))
+
+            conn = cls.engine.connect()
+            # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
+        except Exception as e:
+            config.invalidate(e)
+            return False
+
+        config.db_configured = True
+
+        if not cc_classes:
+            cc = conn.execute("SELECT id, datatype FROM custom_columns")
+            cls.setup_db_cc_classes(cc)
+
         cls.session_factory = scoped_session(sessionmaker(autocommit=False,
                                                           autoflush=True,
                                                           bind=cls.engine))

cps/editbooks.py

@@ -883,20 +883,7 @@ def create_book_on_upload(modif_date, meta):
     calibre_db.session.flush()
     return db_book, input_authors, title_dir
 
-@editbook.route("/upload", methods=["GET", "POST"])
+def file_handling_on_upload(requested_file):
-@login_required_if_no_ano
-@upload_required
-def upload():
-    if not config.config_uploading:
-        abort(404)
-    if request.method == 'POST' and 'btn-upload' in request.files:
-        for requested_file in request.files.getlist("btn-upload"):
-            try:
-                modif_date = False
-                # create the function for sorting...
-                calibre_db.update_title_sort(config)
-                calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
-
     # check if file extension is correct
     if '.' in requested_file.filename:
         file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
@@ -904,10 +891,10 @@ def upload():
             flash(
                 _("File extension '%(ext)s' is not allowed to be uploaded to this server",
                   ext=file_ext), category="error")
-                        return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+            return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
     else:
         flash(_('File to be uploaded must have an extension'), category="error")
-                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+        return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
 
     # extract metadata from file
     try:
@@ -916,22 +903,11 @@ def upload():
         log.error("File %s could not saved to temp dir", requested_file.filename)
         flash(_(u"File %(filename)s could not saved to temp dir",
                 filename=requested_file.filename), category="error")
-                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+        return None, Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
+    return meta, None
 
-                db_book, input_authors, title_dir = create_book_on_upload(modif_date, meta)
-
-                # Comments needs book id therfore only possible after flush
-                modif_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
-
-                book_id = db_book.id
-                title = db_book.title
-
-                error = helper.update_dir_structure_file(book_id,
-                                                   config.config_calibre_dir,
-                                                   input_authors[0],
-                                                   meta.file_path,
-                                                   title_dir + meta.extension)
 
+def move_coverfile(meta, db_book):
     # move cover to final directory, including book id
     if meta.cover:
         coverfile = meta.cover
@@ -948,6 +924,43 @@ def upload():
                 error=e),
               category="error")
 
+
+@editbook.route("/upload", methods=["GET", "POST"])
+@login_required_if_no_ano
+@upload_required
+def upload():
+    if not config.config_uploading:
+        abort(404)
+    if request.method == 'POST' and 'btn-upload' in request.files:
+        for requested_file in request.files.getlist("btn-upload"):
+            try:
+                modif_date = False
+                # create the function for sorting...
+                calibre_db.update_title_sort(config)
+                calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
+
+                response, error = file_handling_on_upload(requested_file)
+                if error:
+                    return response
+                else:
+                    meta = response
+
+                db_book, input_authors, title_dir = create_book_on_upload(modif_date, meta)
+
+                # Comments needs book id therefore only possible after flush
+                modif_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
+
+                book_id = db_book.id
+                title = db_book.title
+
+                error = helper.update_dir_structure_file(book_id,
+                                                   config.config_calibre_dir,
+                                                   input_authors[0],
+                                                   meta.file_path,
+                                                   title_dir + meta.extension)
+
+                move_coverfile(meta, db_book)
+
                 # save data to database, reread data
                 calibre_db.session.commit()
 

cps/epub.py

@@ -87,18 +87,29 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
     lang = epub_metadata['language'].split('-', 1)[0].lower()
     epub_metadata['language'] = isoLanguages.get_lang3(lang)
 
-    series = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series']/@content", namespaces=ns)
+    epub_metadata = parse_epbub_series(tree, epub_metadata)
-    if len(series) > 0:
-        epub_metadata['series'] = series[0]
-    else:
-        epub_metadata['series'] = ''
 
-    series_id = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series_index']/@content", namespaces=ns)
+    coverfile = parse_ebpub_cover(ns, tree, epubZip, coverpath, tmp_file_path)
-    if len(series_id) > 0:
-        epub_metadata['series_id'] = series_id[0]
-    else:
-        epub_metadata['series_id'] = '1'
 
+    if not epub_metadata['title']:
+        title = original_file_name
+    else:
+        title = epub_metadata['title']
+
+    return BookMeta(
+        file_path=tmp_file_path,
+        extension=original_file_extension,
+        title=title.encode('utf-8').decode('utf-8'),
+        author=epub_metadata['creator'].encode('utf-8').decode('utf-8'),
+        cover=coverfile,
+        description=epub_metadata['description'],
+        tags=epub_metadata['subject'].encode('utf-8').decode('utf-8'),
+        series=epub_metadata['series'].encode('utf-8').decode('utf-8'),
+        series_id=epub_metadata['series_id'].encode('utf-8').decode('utf-8'),
+        languages=epub_metadata['language'],
+        publisher="")
+
+def parse_ebpub_cover(ns, tree, epubZip, coverpath, tmp_file_path):
     coversection = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns)
     coverfile = None
     if len(coversection) > 0:
@@ -126,21 +137,18 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
                     coverfile = extractCover(epubZip, filename, "", tmp_file_path)
             else:
                 coverfile = extractCover(epubZip, coversection[0], coverpath, tmp_file_path)
+    return coverfile
 
-    if not epub_metadata['title']:
+def parse_epbub_series(tree, epub_metadata):
-        title = original_file_name
+    series = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series']/@content", namespaces=ns)
+    if len(series) > 0:
+        epub_metadata['series'] = series[0]
     else:
-        title = epub_metadata['title']
+        epub_metadata['series'] = ''
 
-    return BookMeta(
+    series_id = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='calibre:series_index']/@content", namespaces=ns)
-        file_path=tmp_file_path,
+    if len(series_id) > 0:
-        extension=original_file_extension,
+        epub_metadata['series_id'] = series_id[0]
-        title=title.encode('utf-8').decode('utf-8'),
+    else:
-        author=epub_metadata['creator'].encode('utf-8').decode('utf-8'),
+        epub_metadata['series_id'] = '1'
-        cover=coverfile,
+    return epub_metadata
-        description=epub_metadata['description'],
-        tags=epub_metadata['subject'].encode('utf-8').decode('utf-8'),
-        series=epub_metadata['series'].encode('utf-8').decode('utf-8'),
-        series_id=epub_metadata['series_id'].encode('utf-8').decode('utf-8'),
-        languages=epub_metadata['language'],
-        publisher="")

cps/oauth_bb.py

@@ -299,6 +299,19 @@ if ub.oauth_support:
         )  # ToDo: Translate
         flash(msg, category="error")
 
+    @oauth_error.connect_via(oauthblueprints[1]['blueprint'])
+    def google_error(blueprint, error, error_description=None, error_uri=None):
+        msg = (
+            u"OAuth error from {name}! "
+            u"error={error} description={description} uri={uri}"
+        ).format(
+            name=blueprint.name,
+            error=error,
+            description=error_description,
+            uri=error_uri,
+        )  # ToDo: Translate
+        flash(msg, category="error")
+
 
 @oauth.route('/link/github')
 @oauth_required
@@ -332,20 +345,6 @@ if ub.oauth_support:
     return redirect(url_for('web.login'))
 
 
-    @oauth_error.connect_via(oauthblueprints[1]['blueprint'])
-    def google_error(blueprint, error, error_description=None, error_uri=None):
-        msg = (
-            u"OAuth error from {name}! "
-            u"error={error} description={description} uri={uri}"
-        ).format(
-            name=blueprint.name,
-            error=error,
-            description=error_description,
-            uri=error_uri,
-        )  # ToDo: Translate
-        flash(msg, category="error")
-
-
 @oauth.route('/unlink/google', methods=["GET"])
 @login_required
 def google_login_unlink():

cps/tasks/mail.py

@@ -110,8 +110,7 @@ class TaskEmail(CalibreTask):
 
         self.results = dict()
 
-    def run(self, worker_thread):
+    def prepare_message(self):
-        # create MIME message
         msg = MIMEMultipart()
         msg['Subject'] = self.subject
         msg['Message-Id'] = make_msgid('calibre-web')
@@ -128,19 +127,22 @@ class TaskEmail(CalibreTask):
 
         msg['From'] = self.settings["mail_from"]
         msg['To'] = self.recipent
-
-        use_ssl = int(self.settings.get('mail_use_ssl', 0))
-        try:
         # convert MIME message to string
         fp = StringIO()
         gen = Generator(fp, mangle_from_=False)
         gen.flatten(msg)
-            msg = fp.getvalue()
+        return fp.getvalue()
 
+    def run(self, worker_thread):
+        # create MIME message
+        msg = self.prepare_message()
+
+        use_ssl = int(self.settings.get('mail_use_ssl', 0))
+        try:
             # send email
             timeout = 600  # set timeout to 5mins
 
-            # redirect output to logfile on python2 pn python3 debugoutput is caught with overwritten
+            # redirect output to logfile on python2 on python3 debugoutput is caught with overwritten
             # _print_debug function
             if sys.version_info < (3, 0):
                 org_smtpstderr = smtplib.stderr
@@ -169,7 +171,6 @@ class TaskEmail(CalibreTask):
         except (MemoryError) as e:
             log.debug_or_exception(e)
             self._handleError(u'MemoryError sending email: ' + str(e))
-            # return None
         except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
             if hasattr(e, "smtp_error"):
                 text = e.smtp_error.decode('utf-8').replace("\n", '. ')
@@ -181,10 +182,8 @@ class TaskEmail(CalibreTask):
                 log.debug_or_exception(e)
                 text = ''
             self._handleError(u'Smtplib Error sending email: ' + text)
-            # return None
         except (socket.error) as e:
             self._handleError(u'Socket Error sending email: ' + e.strerror)
-            # return None
 
 
     @property

cps/updater.py

@@ -403,6 +403,52 @@ class Updater(threading.Thread):
             return json.dumps(status)
         return ''
 
+    def _stable_updater_set_status(self, i, newer, status, parents, commit):
+        if i == -1 and newer == False:
+            status.update({
+                'update': True,
+                'success': True,
+                'message': _(
+                    u'Click on the button below to update to the latest stable version.'),
+                'history': parents
+            })
+            self.updateFile = commit[0]['zipball_url']
+        elif i == -1 and newer == True:
+            status.update({
+                'update': True,
+                'success': True,
+                'message': _(u'A new update is available. Click on the button below to '
+                             u'update to version: %(version)s', version=commit[0]['tag_name']),
+                'history': parents
+            })
+            self.updateFile = commit[0]['zipball_url']
+        return status
+
+    def _stable_updater_parse_major_version(self, commit, i, parents, current_version, status):
+        if int(commit[i + 1]['tag_name'].split('.')[1]) == current_version[1]:
+            parents.append([commit[i]['tag_name'],
+                            commit[i]['body'].replace('\r\n', '<p>').replace('\n', '<p>')])
+            status.update({
+                'update': True,
+                'success': True,
+                'message': _(u'A new update is available. Click on the button below to '
+                             u'update to version: %(version)s', version=commit[i]['tag_name']),
+                'history': parents
+            })
+            self.updateFile = commit[i]['zipball_url']
+        else:
+            parents.append([commit[i + 1]['tag_name'],
+                            commit[i + 1]['body'].replace('\r\n', '<p>').replace('\n', '<p>')])
+            status.update({
+                'update': True,
+                'success': True,
+                'message': _(u'A new update is available. Click on the button below to '
+                             u'update to version: %(version)s', version=commit[i + 1]['tag_name']),
+                'history': parents
+            })
+            self.updateFile = commit[i + 1]['zipball_url']
+        return status, parents
+
     def _stable_available_updates(self, request_method):
         if request_method == "GET":
             parents = []
@@ -464,48 +510,15 @@ class Updater(threading.Thread):
                     # before major update
                     if i == (len(commit) - 1):
                         i -= 1
-                    if int(commit[i+1]['tag_name'].split('.')[1]) == current_version[1]:
+                    status, parents = self._stable_updater_parse_major_version(self,
-                        parents.append([commit[i]['tag_name'],
+                                                                               commit,
-                                        commit[i]['body'].replace('\r\n', '<p>').replace('\n', '<p>')])
+                                                                               i,
-                        status.update({
+                                                                               parents,
-                            'update': True,
+                                                                               current_version,
-                            'success': True,
+                                                                               status)
-                            'message': _(u'A new update is available. Click on the button below to '
-                                         u'update to version: %(version)s', version=commit[i]['tag_name']),
-                            'history': parents
-                        })
-                        self.updateFile = commit[i]['zipball_url']
-                    else:
-                        parents.append([commit[i+1]['tag_name'],
-                                        commit[i+1]['body'].replace('\r\n', '<p>').replace('\n', '<p>')])
-                        status.update({
-                            'update': True,
-                            'success': True,
-                            'message': _(u'A new update is available. Click on the button below to '
-                                         u'update to version: %(version)s', version=commit[i+1]['tag_name']),
-                            'history': parents
-                        })
-                        self.updateFile = commit[i+1]['zipball_url']
                     break
-            if i == -1 and newer == False:
-                status.update({
-                    'update': True,
-                    'success': True,
-                    'message': _(
-                        u'Click on the button below to update to the latest stable version.'),
-                    'history': parents
-                })
-                self.updateFile = commit[0]['zipball_url']
-            elif i == -1 and newer == True:
-                status.update({
-                    'update': True,
-                    'success': True,
-                    'message': _(u'A new update is available. Click on the button below to '
-                                 u'update to version: %(version)s', version=commit[0]['tag_name']),
-                    'history': parents
-                })
-                self.updateFile = commit[0]['zipball_url']
 
+            status = self._stable_updater_set_status(self, i, newer, status, parents, commit)
         return json.dumps(status)
 
     def _get_request_path(self):