Merge remote-tracking branch 'github/config_sql' into Develop

cps/__init__.py

@@ -25,10 +25,6 @@ from __future__ import division, print_function, unicode_literals
 import sys
 import os
 import mimetypes
-try:
-    import cPickle
-except ImportError:
-    import pickle as cPickle
 
 from babel import Locale as LC
 from babel import negotiate_locale
@@ -38,8 +34,7 @@ from flask_login import LoginManager
 from flask_babel import Babel
 from flask_principal import Principal
 
-from . import logger, cache_buster, ub
+from . import logger, cache_buster, cli, config_sql, ub
-from .constants import TRANSLATIONS_DIR as _TRANSLATIONS_DIR
 from .reverseproxy import ReverseProxied
 
 
@@ -68,16 +63,9 @@ lm.login_view = 'web.login'
 lm.anonymous_user = ub.Anonymous
 
 
-ub.init_db()
+ub.init_db(cli.settingspath)
-config = ub.Config()
+config = config_sql.load_configuration(ub.session)
-from . import db
+from . import db, services
-
-try:
-    with open(os.path.join(_TRANSLATIONS_DIR, 'iso639.pickle'), 'rb') as f:
-        language_table = cPickle.load(f)
-except cPickle.UnpicklingError as error:
-    print("Can't read file cps/translations/iso639.pickle: %s" % error)
-    sys.exit(1)
 
 searched_ids = {}
 
@@ -87,10 +75,8 @@ global_WorkerThread = WorkerThread()
 from .server import WebServer
 web_server = WebServer()
 
-from .ldap_login import Ldap
-ldap1 = Ldap()
-
 babel = Babel()
+_BABEL_TRANSLATIONS = set()
 
 log = logger.create()
 
@@ -109,30 +95,45 @@ def create_app():
     Principal(app)
     lm.init_app(app)
     app.secret_key = os.getenv('SECRET_KEY', 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT')
+
     web_server.init_app(app, config)
-    db.setup_db()
+    db.setup_db(config)
+
     babel.init_app(app)
-    ldap1.init_app(app)
+    _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations())
+    _BABEL_TRANSLATIONS.add('en')
+
+    if services.ldap:
+        services.ldap.init_app(app, config)
+    if services.goodreads:
+        services.goodreads.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads)
+
     global_WorkerThread.start()
     return app
 
 @babel.localeselector
-def get_locale():
+def negociate_locale():
     # if a user is logged in, use the locale from the user settings
     user = getattr(g, 'user', None)
     # user = None
     if user is not None and hasattr(user, "locale"):
         if user.nickname != 'Guest':   # if the account is the guest account bypass the config lang settings
             return user.locale
-    translations = [str(item) for item in babel.list_translations()] + ['en']
+
-    preferred = list()
+    preferred = set()
+    if request.accept_languages:
         for x in request.accept_languages.values():
             try:
-            preferred.append(str(LC.parse(x.replace('-', '_'))))
+                preferred.add(str(LC.parse(x.replace('-', '_'))))
             except (UnknownLocaleError, ValueError) as e:
                 log.warning('Could not parse locale "%s": %s', x, e)
-            preferred.append('en')
+                # preferred.append('en')
-    return negotiate_locale(preferred, translations)
+
+    return negotiate_locale(preferred or ['en'], _BABEL_TRANSLATIONS)
+
+
+def get_locale():
+    return request._locale
 
 
 @babel.timezoneselector

cps/about.py

@@ -69,8 +69,8 @@ def stats():
     versions['pytz'] = 'v' + pytzVersion
 
     versions['Requests'] = 'v' + requests.__version__
-    versions['pySqlite'] = 'v' + db.engine.dialect.dbapi.version
+    versions['pySqlite'] = 'v' + db.session.bind.dialect.dbapi.version
-    versions['Sqlite'] = 'v' + db.engine.dialect.dbapi.sqlite_version
+    versions['Sqlite'] = 'v' + db.session.bind.dialect.dbapi.sqlite_version
     versions.update(converter.versioncheck())
     versions.update(serverVersion)
     versions['Python'] = sys.version

cps/admin.py

@@ -23,13 +23,14 @@
 
 from __future__ import division, print_function, unicode_literals
 import os
+import base64
 import json
 import time
 from datetime import datetime, timedelta
-try:
+# try:
-    from imp import reload
+#     from imp import reload
-except ImportError:
+# except ImportError:
-    pass
+#     pass
 
 from babel import Locale as LC
 from babel.dates import format_datetime
@@ -38,23 +39,19 @@ from flask_login import login_required, current_user, logout_user
 from flask_babel import gettext as _
 from sqlalchemy import and_
 from sqlalchemy.exc import IntegrityError
+from sqlalchemy.sql.expression import func
 from werkzeug.security import generate_password_hash
 
-from . import constants, logger, ldap1
+from . import constants, logger, helper, services
 from . import db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
-from .helper import speaking_language, check_valid_domain, check_unrar, send_test_mail, generate_random_password, \
+from .helper import speaking_language, check_valid_domain, send_test_mail, generate_random_password, send_registration_mail
-                    send_registration_mail
+from .gdriveutils import is_gdrive_ready, gdrive_support
-from .gdriveutils import is_gdrive_ready, gdrive_support, downloadFile, deleteDatabaseOnChange, listRootFolders
 from .web import admin_required, render_title_template, before_request, unconfigured, login_required_if_no_ano
 
-feature_support = dict()
+feature_support = {
-feature_support['ldap'] = ldap1.ldap_supported()
+        'ldap': bool(services.ldap),
-
+        'goodreads': bool(services.goodreads)
-try:
+    }
-    from goodreads.client import GoodreadsClient
-    feature_support['goodreads'] = True
-except ImportError:
-    feature_support['goodreads'] = False
 
 # try:
 #     import rarfile
@@ -63,7 +60,7 @@ except ImportError:
 #     feature_support['rar'] = False
 
 try:
-    from oauth_bb import oauth_check
+    from .oauth_bb import oauth_check
     feature_support['oauth'] = True
 except ImportError:
     feature_support['oauth'] = False
@@ -86,12 +83,10 @@ def admin_forbidden():
 @admin_required
 def shutdown():
     task = int(request.args.get("parameter").strip())
-    if task == 1 or task == 0:  # valid commandos received
+    if task in (0, 1):  # valid commandos received
         # close all database connections
-        db.session.close()
+        db.dispose()
-        db.engine.dispose()
+        ub.dispose()
-        ub.session.close()
-        ub.engine.dispose()
 
         showtext = {}
         if task == 0:
@@ -101,12 +96,12 @@ def shutdown():
         # stop gevent/tornado server
         web_server.stop(task == 0)
         return json.dumps(showtext)
-    else:
+
     if task == 2:
-            db.session.close()
+        log.warning("reconnecting to calibre database")
-            db.engine.dispose()
+        db.setup_db(config)
-            db.setup_db()
+        return '{}'
-            return json.dumps({})
+
     abort(404)
 
 
@@ -133,8 +128,8 @@ def admin():
             commit = version['version']
 
     allUser = ub.session.query(ub.User).all()
-    settings = ub.session.query(ub.Settings).first()
+    email_settings = config.get_mail_settings()
-    return render_title_template("admin.html", allUser=allUser, email=settings, config=config, commit=commit,
+    return render_title_template("admin.html", allUser=allUser, email=email_settings, config=config, commit=commit,
                                  title=_(u"Admin page"), page="admin")
 
 
@@ -142,84 +137,60 @@ def admin():
 @login_required
 @admin_required
 def configuration():
-    return configuration_helper(0)
+    if request.method == "POST":
+        return _configuration_update_helper()
+    return _configuration_result()
 
 
-@admi.route("/admin/viewconfig", methods=["GET", "POST"])
+@admi.route("/admin/viewconfig")
 @login_required
 @admin_required
 def view_configuration():
-    reboot_required = False
-    if request.method == "POST":
-        to_save = request.form.to_dict()
-        content = ub.session.query(ub.Settings).first()
-        if "config_calibre_web_title" in to_save:
-            content.config_calibre_web_title = to_save["config_calibre_web_title"]
-        if "config_columns_to_ignore" in to_save:
-            content.config_columns_to_ignore = to_save["config_columns_to_ignore"]
-        if "config_read_column" in to_save:
-            content.config_read_column = int(to_save["config_read_column"])
-        if "config_theme" in to_save:
-            content.config_theme = int(to_save["config_theme"])
-        if "config_title_regex" in to_save:
-            if content.config_title_regex != to_save["config_title_regex"]:
-                content.config_title_regex = to_save["config_title_regex"]
-                reboot_required = True
-        if "config_random_books" in to_save:
-            content.config_random_books = int(to_save["config_random_books"])
-        if "config_books_per_page" in to_save:
-            content.config_books_per_page = int(to_save["config_books_per_page"])
-        # Mature Content configuration
-        if "config_mature_content_tags" in to_save:
-            content.config_mature_content_tags = to_save["config_mature_content_tags"].strip()
-        if "Show_mature_content" in to_save:
-            content.config_default_show |= constants.MATURE_CONTENT
-
-        if "config_authors_max" in to_save:
-            content.config_authors_max = int(to_save["config_authors_max"])
-
-        # Default user configuration
-        content.config_default_role = 0
-        if "admin_role" in to_save:
-            content.config_default_role |= constants.ROLE_ADMIN
-        if "download_role" in to_save:
-            content.config_default_role |= constants.ROLE_DOWNLOAD
-        if "viewer_role" in to_save:
-            content.config_default_role |= constants.ROLE_VIEWER
-        if "upload_role" in to_save:
-            content.config_default_role |= constants.ROLE_UPLOAD
-        if "edit_role" in to_save:
-            content.config_default_role |= constants.ROLE_EDIT
-        if "delete_role" in to_save:
-            content.config_default_role |= constants.ROLE_DELETE_BOOKS
-        if "passwd_role" in to_save:
-            content.config_default_role |= constants.ROLE_PASSWD
-        if "edit_shelf_role" in to_save:
-            content.config_default_role |= constants.ROLE_EDIT_SHELFS
-
-        val = 0
-        for key, __ in to_save.items():
-            if key.startswith('show'):
-                val |= int(key[5:])
-        content.config_default_show = val
-
-        ub.session.commit()
-        flash(_(u"Calibre-Web configuration updated"), category="success")
-        config.loadSettings()
-        before_request()
-        if reboot_required:
-            # db.engine.dispose() # ToDo verify correct
-            # ub.session.close()
-            # ub.engine.dispose()
-            # stop Server
-            web_server.stop(True)
-            log.info('Reboot required, restarting')
     readColumn = db.session.query(db.Custom_Columns)\
             .filter(and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all()
     return render_title_template("config_view_edit.html", conf=config, readColumns=readColumn,
                                  title=_(u"UI Configuration"), page="uiconfig")
 
 
+@admi.route("/admin/viewconfig", methods=["POST"])
+@login_required
+@admin_required
+def update_view_configuration():
+    reboot_required = False
+    to_save = request.form.to_dict()
+
+    _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y)
+    _config_int = lambda x: config.set_from_dictionary(to_save, x, int)
+
+    _config_string("config_calibre_web_title")
+    _config_string("config_columns_to_ignore")
+    _config_string("config_mature_content_tags")
+    reboot_required |= _config_string("config_title_regex")
+
+    _config_int("config_read_column")
+    _config_int("config_theme")
+    _config_int("config_random_books")
+    _config_int("config_books_per_page")
+    _config_int("config_authors_max")
+
+    config.config_default_role = constants.selected_roles(to_save)
+    config.config_default_role &= ~constants.ROLE_ANONYMOUS
+
+    config.config_default_show = sum(int(k[5:]) for k in to_save if k.startswith('show_'))
+    if "Show_mature_content" in to_save:
+        config.config_default_show |= constants.MATURE_CONTENT
+
+    config.save()
+    flash(_(u"Calibre-Web configuration updated"), category="success")
+    before_request()
+    if reboot_required:
+        db.dispose()
+        ub.dispose()
+        web_server.stop(True)
+
+    return view_configuration()
+
+
 @admi.route("/ajax/editdomain", methods=['POST'])
 @login_required
 @admin_required
@@ -280,281 +251,172 @@ def list_domain():
 @unconfigured
 def basic_configuration():
     logout_user()
-    return configuration_helper(1)
-
-
-def configuration_helper(origin):
-    reboot_required = False
-    gdriveError = None
-    db_change = False
-    success = False
-    filedata = None
-    if not feature_support['gdrive']:
-        gdriveError = _('Import of optional Google Drive requirements missing')
-    else:
-        if not os.path.isfile(gdriveutils.CLIENT_SECRETS):
-            gdriveError = _('client_secrets.json is missing or not readable')
-        else:
-            with open(gdriveutils.CLIENT_SECRETS, 'r') as settings:
-                filedata = json.load(settings)
-            if 'web' not in filedata:
-                gdriveError = _('client_secrets.json is not configured for web application')
     if request.method == "POST":
+        return _configuration_update_helper()
+    return _configuration_result()
+
+
+def _configuration_update_helper():
+    reboot_required = False
+    db_change = False
     to_save = request.form.to_dict()
-        content = ub.session.query(ub.Settings).first()  # type: ub.Settings
+
-        if "config_calibre_dir" in to_save:
+    _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y)
-            if content.config_calibre_dir != to_save["config_calibre_dir"]:
+    _config_int = lambda x: config.set_from_dictionary(to_save, x, int)
-                content.config_calibre_dir = to_save["config_calibre_dir"]
+    _config_checkbox = lambda x: config.set_from_dictionary(to_save, x, lambda y: y == "on", False)
-                db_change = True
+    _config_checkbox_int = lambda x: config.set_from_dictionary(to_save, x, lambda y: 1 if (y == "on") else 0, 0)
+
+    db_change |= _config_string("config_calibre_dir")
+
     # Google drive setup
     if not os.path.isfile(gdriveutils.SETTINGS_YAML):
-            content.config_use_google_drive = False
+        config.config_use_google_drive = False
-        if "config_use_google_drive" in to_save and not content.config_use_google_drive and not gdriveError:
+
-            if filedata:
+    gdrive_secrets = {}
-                if filedata['web']['redirect_uris'][0].endswith('/'):
+    gdriveError = gdriveutils.get_error_text(gdrive_secrets)
-                    filedata['web']['redirect_uris'][0] = filedata['web']['redirect_uris'][0][:-1]
+    if "config_use_google_drive" in to_save and not config.config_use_google_drive and not gdriveError:
-                with open(gdriveutils.SETTINGS_YAML, 'w') as f:
+        if not gdrive_secrets:
-                    yaml = "client_config_backend: settings\nclient_config_file: %(client_file)s\n" \
+            return _configuration_result('client_secrets.json is not configured for web application')
-                           "client_config:\n" \
+        gdriveutils.update_settings(
-                           "  client_id: %(client_id)s\n  client_secret: %(client_secret)s\n" \
+                            gdrive_secrets['client_id'],
-                           "  redirect_uri: %(redirect_uri)s\n\nsave_credentials: True\n" \
+                            gdrive_secrets['client_secret'],
-                           "save_credentials_backend: file\nsave_credentials_file: %(credential)s\n\n" \
+                            gdrive_secrets['redirect_uris'][0]
-                           "get_refresh_token: True\n\noauth_scope:\n" \
+                        )
-                           "  - https://www.googleapis.com/auth/drive\n"
+
-                    f.write(yaml % {'client_file': gdriveutils.CLIENT_SECRETS,
-                                    'client_id': filedata['web']['client_id'],
-                                    'client_secret': filedata['web']['client_secret'],
-                                    'redirect_uri': filedata['web']['redirect_uris'][0],
-                                    'credential': gdriveutils.CREDENTIALS})
-            else:
-                flash(_(u'client_secrets.json is not configured for web application'), category="error")
-                return render_title_template("config_edit.html", config=config, origin=origin,
-                                             gdriveError=gdriveError,
-                                             gfeature_support=feature_support, title=_(u"Basic Configuration"),
-                                             page="config")
     # always show google drive settings, but in case of error deny support
-        if "config_use_google_drive" in to_save and not gdriveError:
+    config.config_use_google_drive = (not gdriveError) and ("config_use_google_drive" in to_save)
-            content.config_use_google_drive = "config_use_google_drive" in to_save
+    if _config_string("config_google_drive_folder"):
-        else:
+        gdriveutils.deleteDatabaseOnChange()
-            content.config_use_google_drive = 0
-        if "config_google_drive_folder" in to_save:
-            if content.config_google_drive_folder != to_save["config_google_drive_folder"]:
-                content.config_google_drive_folder = to_save["config_google_drive_folder"]
-                deleteDatabaseOnChange()
 
-        if "config_port" in to_save:
+    reboot_required |= _config_int("config_port")
-            if content.config_port != int(to_save["config_port"]):
-                content.config_port = int(to_save["config_port"])
-                reboot_required = True
-        if "config_keyfile" in to_save:
-            if content.config_keyfile != to_save["config_keyfile"]:
-                if os.path.isfile(to_save["config_keyfile"]) or to_save["config_keyfile"] is u"":
-                    content.config_keyfile = to_save["config_keyfile"]
-                    reboot_required = True
-                else:
-                    ub.session.commit()
-                    flash(_(u'Keyfile location is not valid, please enter correct path'), category="error")
-                    return render_title_template("config_edit.html", config=config, origin=origin,
-                                                 gdriveError=gdriveError,
-                                                 feature_support=feature_support, title=_(u"Basic Configuration"),
-                                                 page="config")
-        if "config_certfile" in to_save:
-            if content.config_certfile != to_save["config_certfile"]:
-                if os.path.isfile(to_save["config_certfile"]) or to_save["config_certfile"] is u"":
-                    content.config_certfile = to_save["config_certfile"]
-                    reboot_required = True
-                else:
-                    ub.session.commit()
-                    flash(_(u'Certfile location is not valid, please enter correct path'), category="error")
-                    return render_title_template("config_edit.html", config=config, origin=origin,
-                                                 gdriveError=gdriveError, feature_support=feature_support,
-                                                 title=_(u"Basic Configuration"), page="config")
-        content.config_uploading = 0
-        content.config_anonbrowse = 0
-        content.config_public_reg = 0
-        if "config_uploading" in to_save and to_save["config_uploading"] == "on":
-            content.config_uploading = 1
-        if "config_anonbrowse" in to_save and to_save["config_anonbrowse"] == "on":
-            content.config_anonbrowse = 1
-        if "config_public_reg" in to_save and to_save["config_public_reg"] == "on":
-            content.config_public_reg = 1
 
-        if "config_converterpath" in to_save:
+    reboot_required |= _config_string("config_keyfile")
-            content.config_converterpath = to_save["config_converterpath"].strip()
+    if config.config_keyfile and not os.path.isfile(config.config_keyfile):
-        if "config_calibre" in to_save:
+        return _configuration_result('Keyfile location is not valid, please enter correct path', gdriveError)
-            content.config_calibre = to_save["config_calibre"].strip()
+
-        if "config_ebookconverter" in to_save:
+    reboot_required |= _config_string("config_certfile")
-            content.config_ebookconverter = int(to_save["config_ebookconverter"])
+    if config.config_certfile and not os.path.isfile(config.config_certfile):
+        return _configuration_result('Certfile location is not valid, please enter correct path', gdriveError)
+
+    _config_checkbox_int("config_uploading")
+    _config_checkbox_int("config_anonbrowse")
+    _config_checkbox_int("config_public_reg")
+
+    _config_int("config_ebookconverter")
+    _config_string("config_calibre")
+    _config_string("config_converterpath")
+
+    if _config_int("config_login_type"):
+        reboot_required |= config.config_login_type != constants.LOGIN_STANDARD
 
     #LDAP configurator,
-        if "config_login_type" in to_save and to_save["config_login_type"] == "1":
+    if config.config_login_type == constants.LOGIN_LDAP:
-            if not to_save["config_ldap_provider_url"] or not to_save["config_ldap_port"] or not to_save["config_ldap_dn"] or not to_save["config_ldap_user_object"]:
+        _config_string("config_ldap_provider_url")
-                ub.session.commit()
+        _config_int("config_ldap_port")
-                flash(_(u'Please enter a LDAP provider, port, DN and user object identifier'), category="error")
+        _config_string("config_ldap_schema")
-                return render_title_template("config_edit.html", content=config, origin=origin,
+        _config_string("config_ldap_dn")
-                                             gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError,
+        _config_string("config_ldap_user_object")
-                                             feature_support=feature_support, title=_(u"Basic Configuration"),
+        if not config.config_ldap_provider_url or not config.config_ldap_port or not config.config_ldap_dn or not config.config_ldap_user_object:
-                                             page="config")
+            return _configuration_result('Please enter a LDAP provider, port, DN and user object identifier', gdriveError)
-            elif not to_save["config_ldap_serv_username"] or not to_save["config_ldap_serv_password"]:
+
-                ub.session.commit()
+        _config_string("config_ldap_serv_username")
-                flash(_(u'Please enter a LDAP service account and password'), category="error")
+        if not config.config_ldap_serv_username or "config_ldap_serv_password" not in to_save:
-                return render_title_template("config_edit.html", content=config, origin=origin,
+            return _configuration_result('Please enter a LDAP service account and password', gdriveError)
-                                             gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError,
+        config.set_from_dictionary(to_save, "config_ldap_serv_password", base64.b64encode)
-                                             feature_support=feature_support, title=_(u"Basic Configuration"),
+
-                                             page="config")
+    _config_checkbox("config_ldap_use_ssl")
-            else:
+    _config_checkbox("config_ldap_use_tls")
-                content.config_use_ldap = 1
+    _config_checkbox("config_ldap_openldap")
-                content.config_ldap_provider_url = to_save["config_ldap_provider_url"]
+    _config_checkbox("config_ldap_require_cert")
-                content.config_ldap_port = to_save["config_ldap_port"]
+    _config_string("config_ldap_cert_path")
-                content.config_ldap_schema = to_save["config_ldap_schema"]
+    if config.config_ldap_cert_path and not os.path.isfile(config.config_ldap_cert_path):
-                content.config_ldap_serv_username = to_save["config_ldap_serv_username"]
+        return _configuration_result('LDAP Certfile location is not valid, please enter correct path', gdriveError)
-                content.config_ldap_serv_password = base64.b64encode(to_save["config_ldap_serv_password"])
-                content.config_ldap_dn = to_save["config_ldap_dn"]
-                content.config_ldap_user_object = to_save["config_ldap_user_object"]
-                reboot_required = True
-        content.config_ldap_use_ssl = 0
-        content.config_ldap_use_tls = 0
-        content.config_ldap_require_cert = 0
-        content.config_ldap_openldap = 0
-        if "config_ldap_use_ssl" in to_save and to_save["config_ldap_use_ssl"] == "on":
-            content.config_ldap_use_ssl = 1
-        if "config_ldap_use_tls" in to_save and to_save["config_ldap_use_tls"] == "on":
-            content.config_ldap_use_tls = 1
-        if "config_ldap_require_cert" in to_save and to_save["config_ldap_require_cert"] == "on":
-            content.config_ldap_require_cert = 1
-        if "config_ldap_openldap" in to_save and to_save["config_ldap_openldap"] == "on":
-            content.config_ldap_openldap = 1
-        if "config_ldap_cert_path " in to_save:
-            if content.config_ldap_cert_path  != to_save["config_ldap_cert_path "]:
-                if os.path.isfile(to_save["config_ldap_cert_path "]) or to_save["config_ldap_cert_path "] is u"":
-                    content.config_certfile = to_save["config_ldap_cert_path "]
-                else:
-                    ub.session.commit()
-                    flash(_(u'Certfile location is not valid, please enter correct path'), category="error")
-                    return render_title_template("config_edit.html", content=config, origin=origin,
-                                        gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError,
-                                        feature_support=feature_support, title=_(u"Basic Configuration"),
-                                        page="config")
 
     # Remote login configuration
-        content.config_remote_login = ("config_remote_login" in to_save and to_save["config_remote_login"] == "on")
+    _config_checkbox("config_remote_login")
-        if not content.config_remote_login:
+    if not config.config_remote_login:
         ub.session.query(ub.RemoteAuthToken).delete()
 
     # Goodreads configuration
-        content.config_use_goodreads = ("config_use_goodreads" in to_save and to_save["config_use_goodreads"] == "on")
+    _config_checkbox("config_use_goodreads")
-        if "config_goodreads_api_key" in to_save:
+    _config_string("config_goodreads_api_key")
-            content.config_goodreads_api_key = to_save["config_goodreads_api_key"]
+    _config_string("config_goodreads_api_secret")
-        if "config_goodreads_api_secret" in to_save:
+    if services.goodreads:
-            content.config_goodreads_api_secret = to_save["config_goodreads_api_secret"]
+        services.goodreads.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads)
-        if "config_updater" in to_save:
+
-            content.config_updatechannel = int(to_save["config_updater"])
+    _config_int("config_updatechannel")
 
     # GitHub OAuth configuration
-        if "config_login_type" in to_save and to_save["config_login_type"] == "2":
+    if config.config_login_type == constants.LOGIN_OAUTH_GITHUB:
-            if to_save["config_github_oauth_client_id"] == u'' or to_save["config_github_oauth_client_secret"] == u'':
+        _config_string("config_github_oauth_client_id")
-                ub.session.commit()
+        _config_string("config_github_oauth_client_secret")
-                flash(_(u'Please enter Github oauth credentials'), category="error")
+        if not config.config_github_oauth_client_id or not config.config_github_oauth_client_secret:
-                return render_title_template("config_edit.html", config=config, origin=origin,
+            return _configuration_result('Please enter Github oauth credentials', gdriveError)
-                                             gdriveError=gdriveError, feature_support=feature_support,
-                                             title=_(u"Basic Configuration"), page="config")
-            else:
-                content.config_login_type = constants.LOGIN_OAUTH_GITHUB
-                content.config_github_oauth_client_id = to_save["config_github_oauth_client_id"]
-                content.config_github_oauth_client_secret = to_save["config_github_oauth_client_secret"]
-                reboot_required = True
 
     # Google OAuth configuration
-        if "config_login_type" in to_save and to_save["config_login_type"] == "3":
+    if config.config_login_type == constants.LOGIN_OAUTH_GOOGLE:
-            if to_save["config_google_oauth_client_id"] == u'' or to_save["config_google_oauth_client_secret"] == u'':
+        _config_string("config_google_oauth_client_id")
-                ub.session.commit()
+        _config_string("config_google_oauth_client_secret")
-                flash(_(u'Please enter Google oauth credentials'), category="error")
+        if not config.config_google_oauth_client_id or not config.config_google_oauth_client_secret:
-                return render_title_template("config_edit.html", config=config, origin=origin,
+            return _configuration_result('Please enter Google oauth credentials', gdriveError)
-                                             gdriveError=gdriveError, feature_support=feature_support,
-                                             title=_(u"Basic Configuration"), page="config")
-            else:
-                content.config_login_type = constants.LOGIN_OAUTH_GOOGLE
-                content.config_google_oauth_client_id = to_save["config_google_oauth_client_id"]
-                content.config_google_oauth_client_secret = to_save["config_google_oauth_client_secret"]
-                reboot_required = True
 
-        if "config_login_type" in to_save and to_save["config_login_type"] == "0":
+    _config_int("config_log_level")
-            content.config_login_type = constants.LOGIN_STANDARD
+    _config_string("config_logfile")
+    if not logger.is_valid_logfile(config.config_logfile):
+        return _configuration_result('Logfile location is not valid, please enter correct path', gdriveError)
 
-        if "config_log_level" in to_save:
+    reboot_required |= _config_checkbox_int("config_access_log")
-            content.config_log_level = int(to_save["config_log_level"])
+    reboot_required |= _config_string("config_access_logfile")
-        if content.config_logfile != to_save["config_logfile"]:
+    if not logger.is_valid_logfile(config.config_access_logfile):
-            # check valid path, only path or file
+        return _configuration_result('Access Logfile location is not valid, please enter correct path', gdriveError)
-            if not logger.is_valid_logfile(to_save["config_logfile"]):
-                    ub.session.commit()
-                    flash(_(u'Logfile location is not valid, please enter correct path'), category="error")
-                    return render_title_template("config_edit.html", config=config, origin=origin,
-                                                 gdriveError=gdriveError, feature_support=feature_support,
-                                                 title=_(u"Basic Configuration"), page="config")
-            content.config_logfile = to_save["config_logfile"]
-
-        content.config_access_log = 0
-        if "config_access_log" in to_save and to_save["config_access_log"] == "on":
-            content.config_access_log = 1
-            reboot_required = True
-        if "config_access_log" not in to_save and config.config_access_log:
-            reboot_required = True
-
-        if content.config_access_logfile != to_save["config_access_logfile"]:
-            # check valid path, only path or file
-            if not logger.is_valid_logfile(to_save["config_access_logfile"]):
-                    ub.session.commit()
-                    flash(_(u'Access Logfile location is not valid, please enter correct path'), category="error")
-                    return render_title_template("config_edit.html", config=config, origin=origin,
-                                                 gdriveError=gdriveError, feature_support=feature_support,
-                                                 title=_(u"Basic Configuration"), page="config")
-            content.config_access_logfile = to_save["config_access_logfile"]
-            reboot_required = True
 
     # Rarfile Content configuration
-        if "config_rarfile_location" in to_save and to_save['config_rarfile_location'] is not u"":
+    _config_string("config_rarfile_location")
-            check = check_unrar(to_save["config_rarfile_location"].strip())
+    unrar_status = helper.check_unrar(config.config_rarfile_location)
-            if not check[0] :
+    if unrar_status:
-                content.config_rarfile_location = to_save["config_rarfile_location"].strip()
+        return _configuration_result(unrar_status, gdriveError)
-            else:
+
-                flash(check[1], category="error")
-                return render_title_template("config_edit.html", config=config, origin=origin,
-                                             feature_support=feature_support, title=_(u"Basic Configuration"))
     try:
-            if content.config_use_google_drive and is_gdrive_ready() and not \
+        metadata_db = os.path.join(config.config_calibre_dir, "metadata.db")
-                    os.path.exists(os.path.join(content.config_calibre_dir, "metadata.db")):
+        if config.config_use_google_drive and is_gdrive_ready() and not os.path.exists(metadata_db):
-                downloadFile(None, "metadata.db", config.config_calibre_dir + "/metadata.db")
+            gdriveutils.downloadFile(None, "metadata.db", metadata_db)
-            if db_change:
+            db_change = True
-                if config.db_configured:
-                    db.session.close()
-                    db.engine.dispose()
-            ub.session.commit()
-            flash(_(u"Calibre-Web configuration updated"), category="success")
-            config.loadSettings()
     except Exception as e:
-            flash(e, category="error")
+        return _configuration_result('%s' % e, gdriveError)
-            return render_title_template("config_edit.html", config=config, origin=origin,
+
-                                         gdriveError=gdriveError, feature_support=feature_support,
-                                         title=_(u"Basic Configuration"), page="config")
     if db_change:
-            reload(db)
+        # reload(db)
-            if not db.setup_db():
+        if not db.setup_db(config):
-                flash(_(u'DB location is not valid, please enter correct path'), category="error")
+            return _configuration_result('DB location is not valid, please enter correct path', gdriveError)
-                return render_title_template("config_edit.html", config=config, origin=origin,
+
-                                             gdriveError=gdriveError, feature_support=feature_support,
+    config.save()
-                                             title=_(u"Basic Configuration"), page="config")
+    flash(_(u"Calibre-Web configuration updated"), category="success")
     if reboot_required:
-            # stop Server
         web_server.stop(True)
-            log.info('Reboot required, restarting')
+
-        if origin:
+    return _configuration_result(None, gdriveError)
-            success = True
+
-    if is_gdrive_ready() and feature_support['gdrive'] is True and config.config_use_google_drive == True:
+
-        gdrivefolders = listRootFolders()
+def _configuration_result(error_flash=None, gdriveError=None):
+    gdrive_authenticate = not is_gdrive_ready()
+    gdrivefolders = []
+    if gdriveError is None:
+        gdriveError = gdriveutils.get_error_text()
+    if gdriveError:
+        gdriveError = _(gdriveError)
     else:
-        gdrivefolders = list()
+        gdrivefolders = gdriveutils.listRootFolders()
-    return render_title_template("config_edit.html", origin=origin, success=success, config=config,
+
-                                 show_authenticate_google_drive=not is_gdrive_ready(),
+    show_back_button = current_user.is_authenticated
+    show_login_button = config.db_configured and not current_user.is_authenticated
+    if error_flash:
+        config.load()
+        flash(_(error_flash), category="error")
+        show_login_button = False
+
+    return render_title_template("config_edit.html", config=config,
+                                 show_back_button=show_back_button, show_login_button=show_login_button,
+                                 show_authenticate_google_drive=gdrive_authenticate,
                                  gdriveError=gdriveError, gdrivefolders=gdrivefolders, feature_support=feature_support,
                                  title=_(u"Basic Configuration"), page="config")
 
@@ -570,34 +432,14 @@ def new_user():
         to_save = request.form.to_dict()
         content.default_language = to_save["default_language"]
         content.mature_content = "Show_mature_content" in to_save
-        if "locale" in to_save:
+        content.locale = to_save.get("locale", content.locale)
-            content.locale = to_save["locale"]
-
-        val = 0
-        for key, __ in to_save.items():
-            if key.startswith('show'):
-                val += int(key[5:])
-        content.sidebar_view = val
-
 
+        content.sidebar_view = sum(int(key[5:]) for key in to_save if key.startswith('show_'))
         if "show_detail_random" in to_save:
             content.sidebar_view |= constants.DETAIL_RANDOM
 
-        content.role = 0
+        content.role = constants.selected_roles(to_save)
-        if "admin_role" in to_save:
+
-            content.role |= constants.ROLE_ADMIN
-        if "download_role" in to_save:
-            content.role |= constants.ROLE_DOWNLOAD
-        if "upload_role" in to_save:
-            content.role |= constants.ROLE_UPLOAD
-        if "edit_role" in to_save:
-            content.role |= constants.ROLE_EDIT
-        if "delete_role" in to_save:
-            content.role |= constants.ROLE_DELETE_BOOKS
-        if "passwd_role" in to_save:
-            content.role |= constants.ROLE_PASSWD
-        if "edit_shelf_role" in to_save:
-            content.role |= constants.ROLE_EDIT_SHELFS
         if not to_save["nickname"] or not to_save["email"] or not to_save["password"]:
             flash(_(u"Please fill out all fields!"), category="error")
             return render_title_template("user_edit.html", new_user=1, content=content, translations=translations,
@@ -637,24 +479,35 @@ def new_user():
                                  registered_oauth=oauth_check)
 
 
-@admi.route("/admin/mailsettings", methods=["GET", "POST"])
+@admi.route("/admin/mailsettings")
 @login_required
 @admin_required
 def edit_mailsettings():
-    content = ub.session.query(ub.Settings).first()
+    content = config.get_mail_settings()
-    if request.method == "POST":
+    # log.debug("edit_mailsettings %r", content)
+    return render_title_template("email_edit.html", content=content, title=_(u"Edit e-mail server settings"),
+                                 page="mailset")
+
+
+@admi.route("/admin/mailsettings", methods=["POST"])
+@login_required
+@admin_required
+def update_mailsettings():
     to_save = request.form.to_dict()
-        content.mail_server = to_save["mail_server"]
+    log.debug("update_mailsettings %r", to_save)
-        content.mail_port = int(to_save["mail_port"])
+
-        content.mail_login = to_save["mail_login"]
+    _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y)
-        content.mail_password = to_save["mail_password"]
+    _config_int = lambda x: config.set_from_dictionary(to_save, x, int)
-        content.mail_from = to_save["mail_from"]
+
-        content.mail_use_ssl = int(to_save["mail_use_ssl"])
+    _config_string("mail_server")
-        try:
+    _config_int("mail_port")
-            ub.session.commit()
+    _config_int("mail_use_ssl")
-        except Exception as e:
+    _config_string("mail_login")
-            flash(e, category="error")
+    _config_string("mail_password")
-        if "test" in to_save and to_save["test"]:
+    _config_string("mail_from")
+    config.save()
+
+    if to_save.get("test"):
         if current_user.kindle_mail:
             result = send_test_mail(current_user.kindle_mail, current_user.nickname)
             if result is None:
@@ -666,8 +519,8 @@ def edit_mailsettings():
             flash(_(u"Please configure your kindle e-mail address first..."), category="error")
     else:
         flash(_(u"E-mail server settings updated"), category="success")
-    return render_title_template("email_edit.html", content=content, title=_(u"Edit e-mail server settings"),
+
-                                 page="mailset")
+    return edit_mailsettings()
 
 
 @admi.route("/admin/user/<int:user_id>", methods=["GET", "POST"])
@@ -703,53 +556,21 @@ def edit_user(user_id):
             if "password" in to_save and to_save["password"]:
                 content.password = generate_password_hash(to_save["password"])
 
-            if "admin_role" in to_save:
+            anonymous = content.is_anonymous
-                content.role |= constants.ROLE_ADMIN
+            content.role = constants.selected_roles(to_save)
+            if anonymous:
+                content.role |= constants.ROLE_ANONYMOUS
             else:
-                content.role &= ~constants.ROLE_ADMIN
+                content.role &= ~constants.ROLE_ANONYMOUS
 
-            if "download_role" in to_save:
+            val = [int(k[5:]) for k in to_save if k.startswith('show_')]
-                content.role |= constants.ROLE_DOWNLOAD
-            else:
-                content.role &= ~constants.ROLE_DOWNLOAD
-
-            if "viewer_role" in to_save:
-                content.role |= constants.ROLE_VIEWER
-            else:
-                content.role &= ~constants.ROLE_VIEWER
-
-            if "upload_role" in to_save:
-                content.role |= constants.ROLE_UPLOAD
-            else:
-                content.role &= ~constants.ROLE_UPLOAD
-
-            if "edit_role" in to_save:
-                content.role |= constants.ROLE_EDIT
-            else:
-                content.role &= ~constants.ROLE_EDIT
-
-            if "delete_role" in to_save:
-                content.role |= constants.ROLE_DELETE_BOOKS
-            else:
-                content.role &= ~constants.ROLE_DELETE_BOOKS
-
-            if "passwd_role" in to_save:
-                content.role |= constants.ROLE_PASSWD
-            else:
-                content.role &= ~constants.ROLE_PASSWD
-
-            if "edit_shelf_role" in to_save:
-                content.role |= constants.ROLE_EDIT_SHELFS
-            else:
-                content.role &= ~constants.ROLE_EDIT_SHELFS
-
-            val = [int(k[5:]) for k, __ in to_save.items() if k.startswith('show_')]
             sidebar = ub.get_sidebar_config()
             for element in sidebar:
-                if element['visibility'] in val and not content.check_visibility(element['visibility']):
+                value = element['visibility']
-                    content.sidebar_view |= element['visibility']
+                if value in val and not content.check_visibility(value):
-                elif not element['visibility'] in val and content.check_visibility(element['visibility']):
+                    content.sidebar_view |= value
-                    content.sidebar_view &= ~element['visibility']
+                elif not value in val and content.check_visibility(value):
+                    content.sidebar_view &= ~value
 
             if "Show_detail_random" in to_save:
                 content.sidebar_view |= constants.DETAIL_RANDOM

cps/config_sql.py

@@ -0,0 +1,287 @@
+# -*- coding: utf-8 -*-
+
+#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#     Copyright (C) 2019 OzzieIsaacs, pwr
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation, either version 3 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+from __future__ import division, print_function, unicode_literals
+import os
+import json
+
+from sqlalchemy import exc, Column, String, Integer, SmallInteger, Boolean
+from sqlalchemy.ext.declarative import declarative_base
+
+from . import constants, cli, logger
+
+
+log = logger.create()
+_Base = declarative_base()
+
+
+# Baseclass for representing settings in app.db with email server settings and Calibre database settings
+# (application settings)
+class _Settings(_Base):
+    __tablename__ = 'settings'
+
+    id = Column(Integer, primary_key=True)
+    mail_server = Column(String, default='mail.example.org')
+    mail_port = Column(Integer, default=25)
+    mail_use_ssl = Column(SmallInteger, default=0)
+    mail_login = Column(String, default='mail@example.com')
+    mail_password = Column(String, default='mypassword')
+    mail_from = Column(String, default='automailer <mail@example.com>')
+    config_calibre_dir = Column(String)
+    config_port = Column(Integer, default=constants.DEFAULT_PORT)
+    config_certfile = Column(String)
+    config_keyfile = Column(String)
+    config_calibre_web_title = Column(String, default=u'Calibre-Web')
+    config_books_per_page = Column(Integer, default=60)
+    config_random_books = Column(Integer, default=4)
+    config_authors_max = Column(Integer, default=0)
+    config_read_column = Column(Integer, default=0)
+    config_title_regex = Column(String, default=u'^(A|The|An|Der|Die|Das|Den|Ein|Eine|Einen|Dem|Des|Einem|Eines)\s+')
+    config_log_level = Column(SmallInteger, default=logger.DEFAULT_LOG_LEVEL)
+    config_access_log = Column(SmallInteger, default=0)
+    config_uploading = Column(SmallInteger, default=0)
+    config_anonbrowse = Column(SmallInteger, default=0)
+    config_public_reg = Column(SmallInteger, default=0)
+    config_default_role = Column(SmallInteger, default=0)
+    config_default_show = Column(SmallInteger, default=6143)
+    config_columns_to_ignore = Column(String)
+    config_use_google_drive = Column(Boolean, default=False)
+    config_google_drive_folder = Column(String)
+    config_google_drive_watch_changes_response = Column(String)
+    config_remote_login = Column(Boolean, default=False)
+    config_use_goodreads = Column(Boolean, default=False)
+    config_goodreads_api_key = Column(String)
+    config_goodreads_api_secret = Column(String)
+    config_login_type = Column(Integer, default=0)
+    # config_use_ldap = Column(Boolean)
+    config_ldap_provider_url = Column(String)
+    config_ldap_dn = Column(String)
+    # config_use_github_oauth = Column(Boolean)
+    config_github_oauth_client_id = Column(String)
+    config_github_oauth_client_secret = Column(String)
+    # config_use_google_oauth = Column(Boolean)
+    config_google_oauth_client_id = Column(String)
+    config_google_oauth_client_secret = Column(String)
+    config_ldap_provider_url = Column(String, default='localhost')
+    config_ldap_port = Column(SmallInteger, default=389)
+    config_ldap_schema = Column(String, default='ldap')
+    config_ldap_serv_username = Column(String)
+    config_ldap_serv_password = Column(String)
+    config_ldap_use_ssl = Column(Boolean, default=False)
+    config_ldap_use_tls = Column(Boolean, default=False)
+    config_ldap_require_cert = Column(Boolean, default=False)
+    config_ldap_cert_path = Column(String)
+    config_ldap_dn = Column(String)
+    config_ldap_user_object = Column(String)
+    config_ldap_openldap = Column(Boolean, default=False)
+    config_mature_content_tags = Column(String, default='')
+    config_logfile = Column(String)
+    config_access_logfile = Column(String)
+    config_ebookconverter = Column(Integer, default=0)
+    config_converterpath = Column(String)
+    config_calibre = Column(String)
+    config_rarfile_location = Column(String)
+    config_theme = Column(Integer, default=0)
+    config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE)
+
+    def __repr__(self):
+        return self.__class__.__name__
+
+
+# Class holds all application specific settings in calibre-web
+class _ConfigSQL(object):
+    # pylint: disable=no-member
+    def __init__(self, session):
+        self._session = session
+        self._settings = None
+        self.db_configured = None
+        self.config_calibre_dir = None
+        self.load()
+
+    def _read_from_storage(self):
+        if self._settings is None:
+            log.debug("_ConfigSQL._read_from_storage")
+            self._settings = self._session.query(_Settings).first()
+        return self._settings
+
+    def get_config_certfile(self):
+        if cli.certfilepath:
+            return cli.certfilepath
+        if cli.certfilepath == "":
+            return None
+        return self.config_certfile
+
+    def get_config_keyfile(self):
+        if cli.keyfilepath:
+            return cli.keyfilepath
+        if cli.certfilepath == "":
+            return None
+        return self.config_keyfile
+
+    def get_config_ipaddress(self):
+        return cli.ipadress or ""
+
+    def get_ipaddress_type(self):
+        return cli.ipv6
+
+    def _has_role(self, role_flag):
+        return constants.has_flag(self.config_default_role, role_flag)
+
+    def role_admin(self):
+        return self._has_role(constants.ROLE_ADMIN)
+
+    def role_download(self):
+        return self._has_role(constants.ROLE_DOWNLOAD)
+
+    def role_viewer(self):
+        return self._has_role(constants.ROLE_VIEWER)
+
+    def role_upload(self):
+        return self._has_role(constants.ROLE_UPLOAD)
+
+    def role_edit(self):
+        return self._has_role(constants.ROLE_EDIT)
+
+    def role_passwd(self):
+        return self._has_role(constants.ROLE_PASSWD)
+
+    def role_edit_shelfs(self):
+        return self._has_role(constants.ROLE_EDIT_SHELFS)
+
+    def role_delete_books(self):
+        return self._has_role(constants.ROLE_DELETE_BOOKS)
+
+    def show_element_new_user(self, value):
+        return constants.has_flag(self.config_default_show, value)
+
+    def show_detail_random(self):
+        return self.show_element_new_user(constants.DETAIL_RANDOM)
+
+    def show_mature_content(self):
+        return self.show_element_new_user(constants.MATURE_CONTENT)
+
+    def mature_content_tags(self):
+        mct = self.config_mature_content_tags.split(",")
+        return [t.strip() for t in mct]
+
+    def get_log_level(self):
+        return logger.get_level_name(self.config_log_level)
+
+    def get_mail_settings(self):
+        return {k:v for k, v in self.__dict__.items() if k.startswith('mail_')}
+
+    def set_from_dictionary(self, dictionary, field, convertor=None, default=None):
+        '''Possibly updates a field of this object.
+        The new value, if present, is grabbed from the given dictionary, and optionally passed through a convertor.
+
+        :returns: `True` if the field has changed value
+        '''
+        new_value = dictionary.get(field, default)
+        if new_value is None:
+            # log.debug("_ConfigSQL set_from_dictionary field '%s' not found", field)
+            return False
+
+        if field not in self.__dict__:
+            log.warning("_ConfigSQL trying to set unknown field '%s' = %r", field, new_value)
+            return False
+
+        if convertor is not None:
+            new_value = convertor(new_value)
+
+        current_value = self.__dict__.get(field)
+        if current_value == new_value:
+            return False
+
+        # log.debug("_ConfigSQL set_from_dictionary '%s' = %r (was %r)", field, new_value, current_value)
+        setattr(self, field, new_value)
+        return True
+
+    def load(self):
+        '''Load all configuration values from the underlying storage.'''
+        s = self._read_from_storage()  # type: _Settings
+        for k, v in s.__dict__.items():
+            if k[0] != '_':
+                if v is None:
+                    # if the storage column has no value, apply the (possible) default
+                    column = s.__class__.__dict__.get(k)
+                    if column.default is not None:
+                        v = column.default.arg
+                setattr(self, k, v)
+
+        if self.config_google_drive_watch_changes_response:
+            self.config_google_drive_watch_changes_response = json.loads(self.config_google_drive_watch_changes_response)
+        self.db_configured = (self.config_calibre_dir and
+                (not self.config_use_google_drive or os.path.exists(self.config_calibre_dir + '/metadata.db')))
+        logger.setup(self.config_logfile, self.config_log_level)
+
+    def save(self):
+        '''Apply all configuration values to the underlying storage.'''
+        s = self._read_from_storage()  # type: _Settings
+
+        for k, v in self.__dict__.items():
+            if k[0] == '_':
+                continue
+            if hasattr(s, k):  # and getattr(s, k, None) != v:
+                # log.debug("_Settings save '%s' = %r", k, v)
+                setattr(s, k, v)
+
+        log.debug("_ConfigSQL updating storage")
+        self._session.merge(s)
+        self._session.commit()
+        self.load()
+
+    def invalidate(self):
+        log.warning("invalidating configuration")
+        self.db_configured = False
+        self.config_calibre_dir = None
+        self.save()
+
+
+def _migrate_table(session, orm_class):
+    changed = False
+
+    for column_name, column in orm_class.__dict__.items():
+        if column_name[0] != '_':
+            try:
+                session.query(column).first()
+            except exc.OperationalError as err:
+                log.debug("%s: %s", column_name, err)
+                column_default = "" if column.default is None else ("DEFAULT %r" % column.default.arg)
+                alter_table = "ALTER TABLE %s ADD COLUMN `%s` %s %s" % (orm_class.__tablename__, column_name, column.type, column_default)
+                session.execute(alter_table)
+                changed = True
+
+    if changed:
+        session.commit()
+
+
+def _migrate_database(session):
+    # make sure the table is created, if it does not exist
+    _Base.metadata.create_all(session.bind)
+    _migrate_table(session, _Settings)
+
+
+def load_configuration(session):
+    _migrate_database(session)
+
+    if not session.query(_Settings).count():
+        session.add(_Settings())
+        session.commit()
+
+    return _ConfigSQL(session)

cps/constants.py

@@ -74,7 +74,7 @@ SIDEBAR_PUBLISHER       = 1 << 12
 SIDEBAR_RATING          = 1 << 13
 SIDEBAR_FORMAT          = 1 << 14
 
-ADMIN_USER_ROLES        = (ROLE_VIEWER << 1) - 1 - (ROLE_ANONYMOUS | ROLE_EDIT_SHELFS)
+ADMIN_USER_ROLES        = sum(r for r in ALL_ROLES.values()) & ~ROLE_EDIT_SHELFS & ~ROLE_ANONYMOUS
 ADMIN_USER_SIDEBAR      = (SIDEBAR_FORMAT << 1) - 1
 
 UPDATE_STABLE       = 0 << 0
@@ -109,6 +109,9 @@ EXTENSIONS_UPLOAD   = {'txt', 'pdf', 'epub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz'
 def has_flag(value, bit_flag):
     return bit_flag == (bit_flag & (value or 0))
 
+def selected_roles(dictionary):
+    return sum(v for k, v in ALL_ROLES.items() if k in dictionary)
+
 
 # :rtype: BookMeta
 BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, '

cps/converter.py

@@ -24,19 +24,14 @@ import re
 from flask_babel import gettext as _
 
 from . import config
-from .subproc_wrapper import process_open
+from .subproc_wrapper import process_wait
 
 
 def versionKindle():
     versions = _(u'not installed')
     if os.path.exists(config.config_converterpath):
         try:
-            p = process_open(config.config_converterpath)
+            for lines in process_wait(config.config_converterpath):
-            # p = subprocess.Popen(ub.config.config_converterpath, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            p.wait()
-            for lines in p.stdout.readlines():
-                if isinstance(lines, bytes):
-                    lines = lines.decode('utf-8')
                 if re.search('Amazon kindlegen\(', lines):
                     versions = lines
         except Exception:
@@ -48,12 +43,7 @@ def versionCalibre():
     versions = _(u'not installed')
     if os.path.exists(config.config_converterpath):
         try:
-            p = process_open([config.config_converterpath, '--version'])
+            for lines in process_wait([config.config_converterpath, '--version']):
-            # p = subprocess.Popen([ub.config.config_converterpath, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            p.wait()
-            for lines in p.stdout.readlines():
-                if isinstance(lines, bytes):
-                    lines = lines.decode('utf-8')
                 if re.search('ebook-convert.*\(calibre', lines):
                     versions = lines
         except Exception:

cps/db.py

@@ -30,24 +30,10 @@ from sqlalchemy import String, Integer, Boolean
 from sqlalchemy.orm import relationship, sessionmaker, scoped_session
 from sqlalchemy.ext.declarative import declarative_base
 
-from . import config, ub
-
 
 session = None
 cc_exceptions = ['datetime', 'comments', 'float', 'composite', 'series']
-cc_classes = None
+cc_classes = {}
-engine = None
-
-
-# user defined sort function for calibre databases (Series, etc.)
-def title_sort(title):
-    # calibre sort stuff
-    title_pat = re.compile(config.config_title_regex, re.IGNORECASE)
-    match = title_pat.search(title)
-    if match:
-        prep = match.group(1)
-        title = title.replace(prep, '') + ', ' + prep
-    return title.strip()
 
 
 Base = declarative_base()
@@ -325,40 +311,45 @@ class Custom_Columns(Base):
         return display_dict
 
 
-def setup_db():
+def update_title_sort(config, conn=None):
-    global engine
+    # user defined sort function for calibre databases (Series, etc.)
-    global session
+    def _title_sort(title):
-    global cc_classes
+        # calibre sort stuff
+        title_pat = re.compile(config.config_title_regex, re.IGNORECASE)
+        match = title_pat.search(title)
+        if match:
+            prep = match.group(1)
+            title = title.replace(prep, '') + ', ' + prep
+        return title.strip()
 
-    if config.config_calibre_dir is None or config.config_calibre_dir == u'':
+    conn = conn or session.connection().connection.connection
-        content = ub.session.query(ub.Settings).first()
+    conn.create_function("title_sort", 1, _title_sort)
-        content.config_calibre_dir = None
+
-        content.db_configured = False
+
-        ub.session.commit()
+def setup_db(config):
-        config.loadSettings()
+    dispose()
+
+    if not config.config_calibre_dir:
+        config.invalidate()
         return False
 
     dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
-    try:
     if not os.path.exists(dbpath):
-            raise
+        config.invalidate()
+        return False
+
+    try:
         engine = create_engine('sqlite:///{0}'.format(dbpath),
                                echo=False,
                                isolation_level="SERIALIZABLE",
                                connect_args={'check_same_thread': False})
         conn = engine.connect()
-    except Exception:
+    except:
-        content = ub.session.query(ub.Settings).first()
+        config.invalidate()
-        content.config_calibre_dir = None
-        content.db_configured = False
-        ub.session.commit()
-        config.loadSettings()
         return False
-    content = ub.session.query(ub.Settings).first()
+
-    content.db_configured = True
+    config.db_configured = True
-    ub.session.commit()
+    update_title_sort(config, conn.connection)
-    config.loadSettings()
-    conn.connection.create_function('title_sort', 1, title_sort)
     # conn.connection.create_function('lower', 1, lcase)
     # conn.connection.create_function('upper', 1, ucase)
 
@@ -367,7 +358,6 @@ def setup_db():
 
         cc_ids = []
         books_custom_column_links = {}
-        cc_classes = {}
         for row in cc:
             if row.datatype not in cc_exceptions:
                 books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata,
@@ -406,8 +396,38 @@ def setup_db():
                                                                            backref='books'))
 
 
+    global session
     Session = scoped_session(sessionmaker(autocommit=False,
                                              autoflush=False,
                                              bind=engine))
     session = Session()
     return True
+
+
+def dispose():
+    global session
+
+    engine = None
+    if session:
+        engine = session.bind
+        try: session.close()
+        except: pass
+        session = None
+
+    if engine:
+        try: engine.dispose()
+        except: pass
+
+    for attr in list(Books.__dict__.keys()):
+        if attr.startswith("custom_column_"):
+            delattr(Books, attr)
+
+    for db_class in cc_classes.values():
+        Base.metadata.remove(db_class.__table__)
+    cc_classes.clear()
+
+    for table in reversed(Base.metadata.sorted_tables):
+        name = table.key
+        if name.startswith("custom_column_") or name.startswith("books_custom_column_"):
+            if table is not None:
+                Base.metadata.remove(table)

cps/editbooks.py

@@ -33,7 +33,7 @@ from flask_babel import gettext as _
 from flask_login import current_user
 
 from . import constants, logger, isoLanguages, gdriveutils, uploader, helper
-from . import config, get_locale, db, ub, global_WorkerThread, language_table
+from . import config, get_locale, db, ub, global_WorkerThread
 from .helper import order_authors, common_filters
 from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required, login_required
 
@@ -206,7 +206,7 @@ def delete_book(book_id, book_format):
 
 
 def render_edit_book(book_id):
-    db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
+    db.update_title_sort(config)
     cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
     book = db.session.query(db.Books)\
         .filter(db.Books.id == book_id).filter(common_filters()).first()
@@ -215,8 +215,8 @@ def render_edit_book(book_id):
         flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
         return redirect(url_for("web.index"))
 
-    for indx in range(0, len(book.languages)):
+    for lang in book.languages:
-        book.languages[indx].language_name = language_table[get_locale()][book.languages[indx].lang_code]
+        lang.language_name = isoLanguages.get_language_name(get_locale(), lang.lang_code)
 
     book = order_authors(book)
 
@@ -354,7 +354,7 @@ def upload_single_file(request, book, book_id):
                 db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
                 db.session.add(db_format)
                 db.session.commit()
-                db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
+                db.update_title_sort(config)
 
             # Queue uploader info
             uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title)
@@ -385,7 +385,7 @@ def edit_book(book_id):
         return render_edit_book(book_id)
 
     # create the function for sorting...
-    db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
+    db.update_title_sort(config)
     book = db.session.query(db.Books)\
         .filter(db.Books.id == book_id).filter(common_filters()).first()
 
@@ -484,17 +484,12 @@ def edit_book(book_id):
 
             # handle book languages
             input_languages = to_save["languages"].split(',')
-            input_languages = [x.strip().lower() for x in input_languages if x != '']
+            unknown_languages = []
-            input_l = []
+            input_l = isoLanguages.get_language_codes(get_locale(), input_languages, unknown_languages)
-            invers_lang_table = [x.lower() for x in language_table[get_locale()].values()]
+            for l in unknown_languages:
-            for lang in input_languages:
+                log.error('%s is not a valid language', l)
-                try:
+                flash(_(u"%(langname)s is not a valid language", langname=l), category="error")
-                    res = list(language_table[get_locale()].keys())[invers_lang_table.index(lang)]
+            modify_database_object(list(input_l), book.languages, db.Languages, db.session, 'languages')
-                    input_l.append(res)
-                except ValueError:
-                    log.error('%s is not a valid language', lang)
-                    flash(_(u"%(langname)s is not a valid language", langname=lang), category="error")
-            modify_database_object(input_l, book.languages, db.Languages, db.session, 'languages')
 
             # handle book ratings
             if to_save["rating"].strip():
@@ -546,7 +541,7 @@ def upload():
     if request.method == 'POST' and 'btn-upload' in request.files:
         for requested_file in request.files.getlist("btn-upload"):
             # create the function for sorting...
-            db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
+            db.update_title_sort(config)
             db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
 
             # check if file extension is correct
@@ -659,7 +654,7 @@ def upload():
 
             # save data to database, reread data
             db.session.commit()
-            db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
+            db.update_title_sort(config)
             book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
 
             # upload book to gdrive if nesseccary and add "(bookid)" to folder name

cps/gdrive.py

@@ -39,7 +39,7 @@ try:
 except ImportError:
     pass
 
-from . import logger, gdriveutils, config, ub, db
+from . import logger, gdriveutils, config, db
 from .web import admin_required
 
 
@@ -94,12 +94,9 @@ def watch_gdrive():
         try:
             result = gdriveutils.watchChange(gdriveutils.Gdrive.Instance().drive, notification_id,
                                'web_hook', address, gdrive_watch_callback_token, current_milli_time() + 604800*1000)
-            settings = ub.session.query(ub.Settings).first()
+            config.config_google_drive_watch_changes_response = json.dumps(result)
-            settings.config_google_drive_watch_changes_response = json.dumps(result)
+            # after save(), config_google_drive_watch_changes_response will be a json object, not string
-            ub.session.merge(settings)
+            config.save()
-            ub.session.commit()
-            settings = ub.session.query(ub.Settings).first()
-            config.loadSettings()
         except HttpError as e:
             reason=json.loads(e.content)['error']['errors'][0]
             if reason['reason'] == u'push.webhookUrlUnauthorized':
@@ -121,11 +118,8 @@ def revoke_watch_gdrive():
                                     last_watch_response['resourceId'])
         except HttpError:
             pass
-        settings = ub.session.query(ub.Settings).first()
+        config.config_google_drive_watch_changes_response = None
-        settings.config_google_drive_watch_changes_response = None
+        config.save()
-        ub.session.merge(settings)
-        ub.session.commit()
-        config.loadSettings()
     return redirect(url_for('admin.configuration'))
 
 
@@ -157,7 +151,7 @@ def on_received_watch_confirmation():
                         log.info('Setting up new DB')
                         # prevent error on windows, as os.rename does on exisiting files
                         move(os.path.join(tmpDir, "tmp_metadata.db"), dbpath)
-                        db.setup_db()
+                        db.setup_db(config)
             except Exception as e:
                 log.exception(e)
         updateMetaData()

cps/gdriveutils.py

@@ -19,6 +19,7 @@
 
 from __future__ import division, print_function, unicode_literals
 import os
+import json
 import shutil
 
 from flask import Response, stream_with_context
@@ -79,6 +80,9 @@ class Singleton:
         except AttributeError:
             self._instance = self._decorated()
             return self._instance
+        except ImportError as e:
+            log.debug(e)
+            return None
 
     def __call__(self):
         raise TypeError('Singletons must be accessed through `Instance()`.')
@@ -534,3 +538,51 @@ def do_gdrive_download(df, headers):
                 log.warning('An error occurred: %s', resp)
                 return
     return Response(stream_with_context(stream()), headers=headers)
+
+
+_SETTINGS_YAML_TEMPLATE = """
+client_config_backend: settings
+client_config_file: %(client_file)s
+client_config:
+  client_id: %(client_id)s
+  client_secret: %(client_secret)s
+  redirect_uri: %(redirect_uri)s
+
+save_credentials: True
+save_credentials_backend: file
+save_credentials_file: %(credential)s
+
+get_refresh_token: True
+
+oauth_scope:
+  - https://www.googleapis.com/auth/drive
+"""
+
+def update_settings(client_id, client_secret, redirect_uri):
+    if redirect_uri.endswith('/'):
+        redirect_uri = redirect_uri[:-1]
+    config_params = {
+                        'client_file': CLIENT_SECRETS,
+                        'client_id': client_id,
+                        'client_secret': client_secret,
+                        'redirect_uri': redirect_uri,
+                        'credential': CREDENTIALS
+                    }
+
+    with open(SETTINGS_YAML, 'w') as f:
+        f.write(_SETTINGS_YAML_TEMPLATE % config_params)
+
+
+def get_error_text(client_secrets=None):
+    if not gdrive_support:
+        return 'Import of optional Google Drive requirements missing'
+
+    if not os.path.isfile(CLIENT_SECRETS):
+        return 'client_secrets.json is missing or not readable'
+
+    with open(CLIENT_SECRETS, 'r') as settings:
+        filedata = json.load(settings)
+    if 'web' not in filedata:
+        return 'client_secrets.json is not configured for web application'
+    if client_secrets:
+        client_secrets.update(filedata['web'])

cps/helper.py

@@ -26,17 +26,16 @@ import json
 import mimetypes
 import random
 import re
-import requests
 import shutil
 import time
 import unicodedata
 from datetime import datetime, timedelta
-from functools import reduce
 from tempfile import gettempdir
 
+import requests
 from babel import Locale as LC
 from babel.core import UnknownLocaleError
-from babel.dates import format_datetime, format_timedelta
+from babel.dates import format_datetime
 from babel.units import format_unit
 from flask import send_from_directory, make_response, redirect, abort
 from flask_babel import gettext as _
@@ -55,12 +54,6 @@ try:
 except ImportError:
     use_unidecode = False
 
-try:
-    import Levenshtein
-    use_levenshtein = True
-except ImportError:
-    use_levenshtein = False
-
 try:
     from PIL import Image
     use_PIL = True
@@ -71,7 +64,7 @@ from . import logger, config, global_WorkerThread, get_locale, db, ub, isoLangua
 from . import gdriveutils as gd
 from .constants import STATIC_DIR as _STATIC_DIR
 from .pagination import Pagination
-from .subproc_wrapper import process_open
+from .subproc_wrapper import process_wait
 from .worker import STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS
 from .worker import TASK_EMAIL, TASK_CONVERT, TASK_UPLOAD, TASK_CONVERT_ANY
 
@@ -110,7 +103,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
     if os.path.exists(file_path + "." + old_book_format.lower()):
         # read settings and append converter task to queue
         if kindle_mail:
-            settings = ub.get_mail_settings()
+            settings = config.get_mail_settings()
             settings['subject'] = _('Send to Kindle') # pretranslate Subject for e-mail
             settings['body'] = _(u'This e-mail has been sent via Calibre-Web.')
             # text = _(u"%(format)s: %(book)s", format=new_book_format, book=book.title)
@@ -128,7 +121,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
 
 
 def send_test_mail(kindle_mail, user_name):
-    global_WorkerThread.add_email(_(u'Calibre-Web test e-mail'),None, None, ub.get_mail_settings(),
+    global_WorkerThread.add_email(_(u'Calibre-Web test e-mail'),None, None, config.get_mail_settings(),
                                   kindle_mail, user_name, _(u"Test e-mail"),
                                   _(u'This e-mail has been sent via Calibre-Web.'))
     return
@@ -145,7 +138,7 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False):
     text += "Don't forget to change your password after first login.\r\n"
     text += "Sincerely\r\n\r\n"
     text += "Your Calibre-Web team"
-    global_WorkerThread.add_email(_(u'Get Started with Calibre-Web'),None, None, ub.get_mail_settings(),
+    global_WorkerThread.add_email(_(u'Get Started with Calibre-Web'),None, None, config.get_mail_settings(),
                                   e_mail, None, _(u"Registration e-mail for user: %(name)s", name=user_name), text)
     return
 
@@ -218,7 +211,7 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
         for entry in iter(book.data):
             if entry.format.upper() == book_format.upper():
                 result = entry.name + '.' + book_format.lower()
-                global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, ub.get_mail_settings(),
+                global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, config.get_mail_settings(),
                                       kindle_mail, user_id, _(u"E-mail: %(book)s", book=book.title),
                                       _(u'This e-mail has been sent via Calibre-Web.'))
                 return
@@ -561,27 +554,23 @@ def do_download_file(book, book_format, data, headers):
 
 
 def check_unrar(unrarLocation):
-    error = False
+    if not unrarLocation:
-    if os.path.exists(unrarLocation):
+        return
+
+    if not os.path.exists(unrarLocation):
+        return 'Unrar binary file not found'
+
     try:
         if sys.version_info < (3, 0):
             unrarLocation = unrarLocation.encode(sys.getfilesystemencoding())
-            p = process_open(unrarLocation)
+        for lines in process_wait(unrarLocation):
-            p.wait()
+            value = re.search('UNRAR (.*) freeware', lines)
-            for lines in p.stdout.readlines():
-                if isinstance(lines, bytes):
-                    lines = lines.decode('utf-8')
-                value=re.search('UNRAR (.*) freeware', lines)
             if value:
                 version = value.group(1)
-        except OSError as e:
+                log.debug("unrar version %s", version)
-            error = True
+    except OSError as err:
-            log.exception(e)
+        log.exception(err)
-            version =_(u'Error excecuting UnRar')
+        return 'Error excecuting UnRar'
-    else:
-        version = _(u'Unrar binary file not found')
-        error=True
-    return (error, version)
 
 
 
@@ -605,7 +594,7 @@ def json_serial(obj):
 def format_runtime(runtime):
     retVal = ""
     if runtime.days:
-        retVal = format_unit(runtime.days, 'duration-day', length="long", locale=web.get_locale()) + ', '
+        retVal = format_unit(runtime.days, 'duration-day', length="long", locale=get_locale()) + ', '
     mins, seconds = divmod(runtime.seconds, 60)
     hours, minutes = divmod(mins, 60)
     # ToDo: locale.number_symbols._data['timeSeparator'] -> localize time separator ?
@@ -630,6 +619,9 @@ def render_task_status(tasklist):
                 if 'starttime' not in task:
                     task['starttime'] = ""
 
+            if 'formRuntime' not in task:
+                task['runtime'] = ""
+            else:
                 task['runtime'] = format_runtime(task['formRuntime'])
 
             # localize the task status
@@ -754,28 +746,6 @@ def get_search_results(term):
             func.lower(db.Books.title).ilike("%" + term + "%")
             )).all()
 
-def get_unique_other_books(library_books, author_books):
-    # Get all identifiers (ISBN, Goodreads, etc) and filter author's books by that list so we show fewer duplicates
-    # Note: Not all images will be shown, even though they're available on Goodreads.com.
-    #       See https://www.goodreads.com/topic/show/18213769-goodreads-book-images
-    identifiers = reduce(lambda acc, book: acc + map(lambda identifier: identifier.val, book.identifiers),
-                         library_books, [])
-    other_books = filter(lambda book: book.isbn not in identifiers and book.gid["#text"] not in identifiers,
-                         author_books)
-
-    # Fuzzy match book titles
-    if use_levenshtein:
-        library_titles = reduce(lambda acc, book: acc + [book.title], library_books, [])
-        other_books = filter(lambda author_book: not filter(
-            lambda library_book:
-            # Remove items in parentheses before comparing
-            Levenshtein.ratio(re.sub(r"\(.*\)", "", author_book.title), library_book) > 0.7,
-            library_titles
-        ), other_books)
-
-    return other_books
-
-
 def get_cc_columns():
     tmpcc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
     if config.config_columns_to_ignore:
@@ -802,10 +772,7 @@ def get_download_link(book_id, book_format):
             file_name = book.authors[0].name + '_' + file_name
         file_name = get_valid_filename(file_name)
         headers = Headers()
-        try:
+        headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream")
-            headers["Content-Type"] = mimetypes.types_map['.' + book_format]
-        except KeyError:
-            headers["Content-Type"] = "application/octet-stream"
         headers["Content-Disposition"] = "attachment; filename*=UTF-8''%s.%s" % (quote(file_name.encode('utf-8')),
                                                                                  book_format)
         return do_download_file(book, book_format, data, headers)

cps/isoLanguages.py

@@ -18,6 +18,14 @@
 #   along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import division, print_function, unicode_literals
+import sys
+import os
+try:
+    import cPickle
+except ImportError:
+    import pickle as cPickle
+
+from .constants import TRANSLATIONS_DIR as _TRANSLATIONS_DIR
 
 
 try:
@@ -33,14 +41,43 @@ except ImportError:
         __version__ = "? (PyCountry)"
 
     def _copy_fields(l):
-        l.part1 = l.alpha_2
+        l.part1 = getattr(l, 'alpha_2', None)
-        l.part3 = l.alpha_3
+        l.part3 = getattr(l, 'alpha_3', None)
         return l
 
     def get(name=None, part1=None, part3=None):
-        if (part3 is not None):
+        if part3 is not None:
             return _copy_fields(pyc_languages.get(alpha_3=part3))
-        if (part1 is not None):
+        if part1 is not None:
             return _copy_fields(pyc_languages.get(alpha_2=part1))
-        if (name is not None):
+        if name is not None:
             return _copy_fields(pyc_languages.get(name=name))
+
+
+try:
+    with open(os.path.join(_TRANSLATIONS_DIR, 'iso639.pickle'), 'rb') as f:
+        _LANGUAGES = cPickle.load(f)
+except cPickle.UnpicklingError as error:
+    print("Can't read file cps/translations/iso639.pickle: %s" % error)
+    sys.exit(1)
+
+
+def get_language_names(locale):
+    return _LANGUAGES.get(locale)
+
+
+def get_language_name(locale, lang_code):
+    return get_language_names(locale)[lang_code]
+
+
+def get_language_codes(locale, language_names, remainder=None):
+    language_names = set(x.strip().lower() for x in language_names if x)
+
+    for k, v in get_language_names(locale).items():
+        v = v.lower()
+        if v in language_names:
+            language_names.remove(v)
+            yield k
+
+    if remainder is not None:
+        remainder.extend(language_names)

cps/jinjia.py

@@ -71,11 +71,7 @@ def shortentitle_filter(s, nchar=20):
 
 @jinjia.app_template_filter('mimetype')
 def mimetype_filter(val):
-    try:
+    return mimetypes.types_map.get('.' + val, 'application/octet-stream')
-        s = mimetypes.types_map['.' + val]
-    except Exception:
-        s = 'application/octet-stream'
-    return s
 
 
 @jinjia.app_template_filter('formatdate')

cps/ldap_login.py

@@ -1,67 +0,0 @@
-# -*- coding: utf-8 -*-
-
-#   This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
-#     Copyright (C) 2019 Krakinou
-#
-#   This program is free software: you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published by
-#   the Free Software Foundation, either version 3 of the License, or
-#   (at your option) any later version.
-#
-#   This program is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import division, print_function, unicode_literals
-import base64
-
-try:
-   from flask_simpleldap import LDAP # , LDAPException
-   ldap_support = True
-except ImportError:
-   ldap_support = False
-
-from . import config, logger
-
-log = logger.create()
-
-class Ldap():
-
-    def __init__(self):
-        self.ldap = None
-        return
-
-    def init_app(self, app):
-        if ldap_support and config.config_login_type == 1:
-            app.config['LDAP_HOST'] = config.config_ldap_provider_url
-            app.config['LDAP_PORT'] = config.config_ldap_port
-            app.config['LDAP_SCHEMA'] = config.config_ldap_schema
-            app.config['LDAP_USERNAME'] = config.config_ldap_user_object.replace('%s', config.config_ldap_serv_username)\
-                                          + ',' + config.config_ldap_dn
-            app.config['LDAP_PASSWORD'] = base64.b64decode(config.config_ldap_serv_password)
-            if config.config_ldap_use_ssl:
-                app.config['LDAP_USE_SSL'] = True
-            if config.config_ldap_use_tls:
-                app.config['LDAP_USE_TLS'] = True
-            app.config['LDAP_REQUIRE_CERT'] = config.config_ldap_require_cert
-            if config.config_ldap_require_cert:
-                app.config['LDAP_CERT_PATH'] = config.config_ldap_cert_path
-            app.config['LDAP_BASE_DN'] = config.config_ldap_dn
-            app.config['LDAP_USER_OBJECT_FILTER'] = config.config_ldap_user_object
-            if config.config_ldap_openldap:
-                app.config['LDAP_OPENLDAP'] = True
-
-        #    app.config['LDAP_BASE_DN'] = 'ou=users,dc=yunohost,dc=org'
-        #    app.config['LDAP_USER_OBJECT_FILTER'] = '(uid=%s)'
-            self.ldap = LDAP(app)
-
-        elif config.config_login_type == 1 and not ldap_support:
-            log.error('Cannot activate ldap support, did you run \'pip install --target vendor -r optional-requirements.txt\'?')
-
-    @classmethod
-    def ldap_supported(cls):
-        return ldap_support

cps/logger.py

@@ -157,3 +157,8 @@ class StderrLogger(object):
                 self.buffer += message
         except Exception:
             self.log.debug("Logging Error")
+
+
+# if debugging, start logging to stderr immediately
+if os.environ.get('FLASK_DEBUG', None):
+    setup(LOG_TO_STDERR, logging.DEBUG)

cps/oauth_bb.py

@@ -24,7 +24,6 @@
 from __future__ import division, print_function, unicode_literals
 import json
 from functools import wraps
-from oauth import OAuthBackend
 
 from flask import session, request, make_response, abort
 from flask import Blueprint, flash, redirect, url_for
@@ -37,6 +36,7 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from . import constants, logger, config, app, ub
 from .web import login_required
+from .oauth import OAuthBackend
 # from .web import github_oauth_required
 
 

cps/opds.py

@@ -31,7 +31,7 @@ from flask_login import current_user
 from sqlalchemy.sql.expression import func, text, or_, and_
 from werkzeug.security import check_password_hash
 
-from . import logger, config, db, ub, ldap1
+from . import constants, logger, config, db, ub, services
 from .helper import fill_indexpage, get_download_link, get_book_cover
 from .pagination import Pagination
 from .web import common_filters, get_search_results, render_read_books, download_required
@@ -40,7 +40,6 @@ from .web import common_filters, get_search_results, render_read_books, download
 opds = Blueprint('opds', __name__)
 
 log = logger.create()
-ldap_support = ldap1.ldap_supported()
 
 
 def requires_basic_auth_if_no_ano(f):
@@ -51,8 +50,8 @@ def requires_basic_auth_if_no_ano(f):
             if not auth or not check_auth(auth.username, auth.password):
                 return authenticate()
         return f(*args, **kwargs)
-    if config.config_login_type == 1 and ldap_support:
+    if config.config_login_type == constants.LOGIN_LDAP and services.ldap:
-        return ldap1.ldap.basic_auth_required(f)
+        return services.ldap.basic_auth_required(f)
     return decorated
 
 

cps/server.py

@@ -197,6 +197,7 @@ class WebServer:
         self.stop()
 
     def stop(self, restart=False):
+        log.info("webserver stop (restart=%s)", restart)
         self.restart = restart
         if self.wsgiserver:
             if _GEVENT:

cps/services/__init__.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2019 pwr
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import division, print_function, unicode_literals
+
+from .. import logger
+
+
+log = logger.create()
+
+
+try: from . import goodreads
+except ImportError as err:
+    log.warning("goodreads: %s", err)
+    goodreads = None
+
+
+try: from . import simpleldap as ldap
+except ImportError as err:
+    log.warning("simpleldap: %s", err)
+    ldap = None

cps/services/goodreads.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2018-2019 OzzieIsaacs, pwr
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import division, print_function, unicode_literals
+import time
+from functools import reduce
+
+from goodreads.client import GoodreadsClient
+
+try: import Levenshtein
+except ImportError: Levenshtein = False
+
+from .. import logger
+
+
+log = logger.create()
+_client = None  # type: GoodreadsClient
+
+# GoodReads TOS allows for 24h caching of data
+_CACHE_TIMEOUT = 23 * 60 * 60  # 23 hours (in seconds)
+_AUTHORS_CACHE = {}
+
+
+def connect(key=None, secret=None, enabled=True):
+    global _client
+
+    if not enabled or not key or not secret:
+        _client = None
+        return
+
+    if _client:
+        # make sure the configuration has not changed since last we used the client
+        if _client.client_key != key or _client.client_secret != secret:
+            _client = None
+
+    if not _client:
+        _client = GoodreadsClient(key, secret)
+
+
+def get_author_info(author_name):
+    now = time.time()
+    author_info = _AUTHORS_CACHE.get(author_name, None)
+    if author_info:
+        if now < author_info._timestamp + _CACHE_TIMEOUT:
+            return author_info
+        # clear expired entries
+        del _AUTHORS_CACHE[author_name]
+
+    if not _client:
+        log.warning("failed to get a Goodreads client")
+        return
+
+    try:
+        author_info = _client.find_author(author_name=author_name)
+    except Exception as ex:
+        # Skip goodreads, if site is down/inaccessible
+        log.warning('Goodreads website is down/inaccessible? %s', ex)
+        return
+
+    if author_info:
+        author_info._timestamp = now
+        _AUTHORS_CACHE[author_name] = author_info
+    return author_info
+
+
+def get_other_books(author_info, library_books=None):
+    # Get all identifiers (ISBN, Goodreads, etc) and filter author's books by that list so we show fewer duplicates
+    # Note: Not all images will be shown, even though they're available on Goodreads.com.
+    #       See https://www.goodreads.com/topic/show/18213769-goodreads-book-images
+
+    if not author_info:
+        return
+
+    identifiers = []
+    library_titles = []
+    if library_books:
+        identifiers = list(reduce(lambda acc, book: acc + [i.val for i in book.identifiers if i.val], library_books, []))
+        library_titles = [book.title for book in library_books]
+
+    for book in author_info.books:
+        if book.isbn in identifiers:
+            continue
+        if book.gid["#text"] in identifiers:
+            continue
+
+        if Levenshtein and library_titles:
+            goodreads_title = book._book_dict['title_without_series']
+            if any(Levenshtein.ratio(goodreads_title, title) > 0.7 for title in library_titles):
+                continue
+
+        yield book

cps/services/simpleldap.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2018-2019 OzzieIsaacs, pwr
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import division, print_function, unicode_literals
+import base64
+
+from flask_simpleldap import LDAP
+from ldap import SERVER_DOWN, INVALID_CREDENTIALS
+
+from .. import constants, logger
+
+
+log = logger.create()
+_ldap = None
+
+
+def init_app(app, config):
+    global _ldap
+
+    if config.config_login_type != constants.LOGIN_LDAP:
+        _ldap = None
+        return
+
+    app.config['LDAP_HOST'] = config.config_ldap_provider_url
+    app.config['LDAP_PORT'] = config.config_ldap_port
+    app.config['LDAP_SCHEMA'] = config.config_ldap_schema
+    app.config['LDAP_USERNAME'] = config.config_ldap_user_object.replace('%s', config.config_ldap_serv_username)\
+                                  + ',' + config.config_ldap_dn
+    app.config['LDAP_PASSWORD'] = base64.b64decode(config.config_ldap_serv_password)
+    app.config['LDAP_REQUIRE_CERT'] = bool(config.config_ldap_require_cert)
+    if config.config_ldap_require_cert:
+        app.config['LDAP_CERT_PATH'] = config.config_ldap_cert_path
+    app.config['LDAP_BASE_DN'] = config.config_ldap_dn
+    app.config['LDAP_USER_OBJECT_FILTER'] = config.config_ldap_user_object
+    app.config['LDAP_USE_SSL'] = bool(config.config_ldap_use_ssl)
+    app.config['LDAP_USE_TLS'] = bool(config.config_ldap_use_tls)
+    app.config['LDAP_OPENLDAP'] = bool(config.config_ldap_openldap)
+
+    # app.config['LDAP_BASE_DN'] = 'ou=users,dc=yunohost,dc=org'
+    # app.config['LDAP_USER_OBJECT_FILTER'] = '(uid=%s)'
+    _ldap = LDAP(app)
+
+
+def basic_auth_required(func):
+    return _ldap.basic_auth_required(func)
+
+
+def bind_user(username, password):
+    '''Attempts a LDAP login.
+
+    :returns: True if login succeeded, False if login failed, None if server unavailable.
+    '''
+    try:
+        result = _ldap.bind_user(username, password)
+        log.debug("LDAP login '%s': %r", username, result)
+        return result is not None
+    except SERVER_DOWN as ex:
+        log.warning('LDAP Server down: %s', ex)
+        return None
+    except INVALID_CREDENTIALS as ex:
+        log.info("LDAP login '%s' failed: %s", username, ex)
+        return False

cps/subproc_wrapper.py

@@ -43,3 +43,13 @@ def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subpro
             exc_command = [x for x in command]
 
     return subprocess.Popen(exc_command, shell=False, stdout=sout, stderr=serr, universal_newlines=True, env=env)
+
+
+def process_wait(command, serr=subprocess.PIPE):
+    '''Run command, wait for process to terminate, and return an iterator over lines of its output.'''
+    p = process_open(command, serr=serr)
+    p.wait()
+    for l in p.stdout.readlines():
+        if isinstance(l, bytes):
+            l = l.decode('utf-8')
+        yield l

cps/templates/admin.html

@@ -69,7 +69,7 @@
         </div>
         <div class="row">
             <div class="col-xs-6 col-sm-6">{{_('Log level')}}</div>
-            <div class="col-xs-6 col-sm-6">{{config.get_Log_Level()}}</div>
+            <div class="col-xs-6 col-sm-6">{{config.get_log_level()}}</div>
         </div>
         <div class="row">
             <div class="col-xs-6 col-sm-6">{{_('Port')}}</div>

cps/templates/config_edit.html

@@ -94,8 +94,8 @@
           <input type="text" class="form-control" name="config_keyfile" id="config_keyfile" value="{% if config.config_keyfile != None %}{{ config.config_keyfile }}{% endif %}" autocomplete="off">
         </div>
         <div class="form-group">
-          <label for="config_updater">{{_('Update channel')}}</label>
+          <label for="config_updatechannel">{{_('Update channel')}}</label>
-            <select name="config_updater" id="config_updater" class="form-control">
+            <select name="config_updatechannel" id="config_updatechannel" class="form-control">
                     <option value="0" {% if config.config_updatechannel == 0 %}selected{% endif %}>{{_('Stable')}}</option>
                     <!--option value="1" {% if config.config_updatechannel == 1 %}selected{% endif %}>{{_('Stable (Automatic)')}}</option-->
                     <option value="2" {% if config.config_updatechannel == 2 %}selected{% endif %}>{{_('Nightly')}}</option>
@@ -327,10 +327,10 @@
 
     <div class="col-sm-12">
     <button type="submit" name="submit" class="btn btn-default">{{_('Submit')}}</button>
-    {% if not origin %}
+    {% if show_back_button %}
       <a href="{{ url_for('admin.admin') }}" class="btn btn-default">{{_('Back')}}</a>
     {% endif %}
-    {% if success %}
+    {% if show_login_button %}
       <a href="{{ url_for('web.login') }}" name="login" class="btn btn-default">{{_('Login')}}</a>
     {% endif %}
     </div>

cps/templates/layout.html

@@ -38,7 +38,7 @@
           </button>
           <a class="navbar-brand" href="{{url_for('web.index')}}">{{instance}}</a>
         </div>
-        {% if g.user.is_authenticated or g.user.is_anonymous %}
+        {% if g.user.is_authenticated or g.allow_anonymous %}
           <form class="navbar-form navbar-left" role="search" action="{{url_for('web.search')}}" method="GET">
             <div class="form-group input-group input-group-sm">
               <label for="query" class="sr-only">{{_('Search')}}</label>
@@ -50,13 +50,13 @@
           </form>
         {% endif %}
         <div class="navbar-collapse collapse">
-          {% if g.user.is_authenticated or g.user.is_anonymous %}
+          {% if g.user.is_authenticated or g.allow_anonymous %}
           <ul class="nav navbar-nav ">
             <li><a href="{{url_for('web.advanced_search')}}"><span class="glyphicon glyphicon-search"></span><span class="hidden-sm"> {{_('Advanced Search')}}</span></a></li>
           </ul>
           {% endif %}
           <ul class="nav navbar-nav navbar-right" id="main-nav">
-            {% if g.user.is_authenticated or g.user.is_anonymous %}
+            {% if g.user.is_authenticated or g.allow_anonymous %}
               {% if g.user.role_upload() or g.user.role_admin()%}
                 {% if g.allow_upload %}
                   <li>
@@ -115,7 +115,7 @@
     {%endif%}
     <div class="container-fluid">
       <div class="row-fluid">
-        {% if g.user.is_authenticated or g.user.is_anonymous %}
+        {% if g.user.is_authenticated or g.allow_anonymous %}
         <div class="col-sm-2">
           <nav class="navigation">
             <ul class="list-unstyled" id="scnd-nav" intent in-standard-append="nav.navigation" in-mobile-after="#main-nav" in-mobile-class="nav navbar-nav">
@@ -128,7 +128,7 @@
                 {% endif %}
               {% endfor %}
 
-              {% if g.user.is_authenticated or g.user.is_anonymous %}
+              {% if g.user.is_authenticated or allow_anonymous %}
                 <li class="nav-head hidden-xs public-shelves">{{_('Public Shelves')}}</li>
                 {% for shelf in g.public_shelfes %}
                   <li><a href="{{url_for('shelf.show_shelf', shelf_id=shelf.id)}}"><span class="glyphicon glyphicon-list public_shelf"></span>{{shelf.name|shortentitle(40)}}</a></li>

cps/ub.py

@@ -19,15 +19,14 @@
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import division, print_function, unicode_literals
-import sys
 import os
 import datetime
-import json
 from binascii import hexlify
 
 from flask import g
 from flask_babel import gettext as _
 from flask_login import AnonymousUserMixin
+from werkzeug.local import LocalProxy
 try:
     from flask_dance.consumer.backend.sqla import OAuthConsumerMixin
     oauth_support = True
@@ -40,22 +39,18 @@ from sqlalchemy.orm import relationship, sessionmaker
 from sqlalchemy.ext.declarative import declarative_base
 from werkzeug.security import generate_password_hash
 
-from . import constants, logger, cli
+from . import constants # , config
 
 
 session = None
-
-engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False)
 Base = declarative_base()
 
 
 def get_sidebar_config(kwargs=None):
     kwargs = kwargs or []
     if 'content' in kwargs:
-        if not isinstance(kwargs['content'], Settings):
+        content = kwargs['content']
-            content = not kwargs['content'].role_anonymous()
+        content = isinstance(content, (User,LocalProxy)) and not content.role_anonymous()
-        else:
-            content = False
     else:
         content = 'conf' in kwargs
     sidebar = list()
@@ -148,7 +143,7 @@ class UserBase:
 
     @property
     def is_anonymous(self):
-        return False
+        return self.role_anonymous()
 
     def get_id(self):
         return str(self.id)
@@ -200,7 +195,6 @@ class Anonymous(AnonymousUserMixin, UserBase):
 
     def loadSettings(self):
         data = session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS).first()  # type: User
-        settings = session.query(Settings).first()
         self.nickname = data.nickname
         self.role = data.role
         self.id=data.id
@@ -208,9 +202,11 @@ class Anonymous(AnonymousUserMixin, UserBase):
         self.default_language = data.default_language
         self.locale = data.locale
         self.mature_content = data.mature_content
-        self.anon_browse = settings.config_anonbrowse
         self.kindle_mail = data.kindle_mail
 
+        # settings = session.query(config).first()
+        # self.anon_browse = settings.config_anonbrowse
+
     def role_admin(self):
         return False
 
@@ -220,7 +216,7 @@ class Anonymous(AnonymousUserMixin, UserBase):
 
     @property
     def is_anonymous(self):
-        return self.anon_browse
+        return True # self.anon_browse
 
     @property
     def is_authenticated(self):
@@ -295,78 +291,6 @@ class Registration(Base):
         return u"<Registration('{0}')>".format(self.domain)
 
 
-# Baseclass for representing settings in app.db with email server settings and Calibre database settings
-# (application settings)
-class Settings(Base):
-    __tablename__ = 'settings'
-
-    id = Column(Integer, primary_key=True)
-    mail_server = Column(String)
-    mail_port = Column(Integer, default=25)
-    mail_use_ssl = Column(SmallInteger, default=0)
-    mail_login = Column(String)
-    mail_password = Column(String)
-    mail_from = Column(String)
-    config_calibre_dir = Column(String)
-    config_port = Column(Integer, default=constants.DEFAULT_PORT)
-    config_certfile = Column(String)
-    config_keyfile = Column(String)
-    config_calibre_web_title = Column(String, default=u'Calibre-Web')
-    config_books_per_page = Column(Integer, default=60)
-    config_random_books = Column(Integer, default=4)
-    config_authors_max = Column(Integer, default=0)
-    config_read_column = Column(Integer, default=0)
-    config_title_regex = Column(String, default=u'^(A|The|An|Der|Die|Das|Den|Ein|Eine|Einen|Dem|Des|Einem|Eines)\s+')
-    config_log_level = Column(SmallInteger, default=logger.DEFAULT_LOG_LEVEL)
-    config_access_log = Column(SmallInteger, default=0)
-    config_uploading = Column(SmallInteger, default=0)
-    config_anonbrowse = Column(SmallInteger, default=0)
-    config_public_reg = Column(SmallInteger, default=0)
-    config_default_role = Column(SmallInteger, default=0)
-    config_default_show = Column(SmallInteger, default=6143)
-    config_columns_to_ignore = Column(String)
-    config_use_google_drive = Column(Boolean)
-    config_google_drive_folder = Column(String)
-    config_google_drive_watch_changes_response = Column(String)
-    config_remote_login = Column(Boolean)
-    config_use_goodreads = Column(Boolean)
-    config_goodreads_api_key = Column(String)
-    config_goodreads_api_secret = Column(String)
-    config_login_type = Column(Integer, default=0)
-    # config_use_ldap = Column(Boolean)
-    config_ldap_provider_url = Column(String)
-    config_ldap_dn = Column(String)
-    # config_use_github_oauth = Column(Boolean)
-    config_github_oauth_client_id = Column(String)
-    config_github_oauth_client_secret = Column(String)
-    # config_use_google_oauth = Column(Boolean)
-    config_google_oauth_client_id = Column(String)
-    config_google_oauth_client_secret = Column(String)
-    config_ldap_provider_url = Column(String, default='localhost')
-    config_ldap_port = Column(SmallInteger, default=389)
-    config_ldap_schema = Column(String, default='ldap')
-    config_ldap_serv_username = Column(String)
-    config_ldap_serv_password = Column(String)
-    config_ldap_use_ssl = Column(Boolean, default=False)
-    config_ldap_use_tls = Column(Boolean, default=False)
-    config_ldap_require_cert = Column(Boolean, default=False)
-    config_ldap_cert_path = Column(String)
-    config_ldap_dn = Column(String)
-    config_ldap_user_object = Column(String)
-    config_ldap_openldap = Column(Boolean)
-    config_mature_content_tags = Column(String)
-    config_logfile = Column(String)
-    config_access_logfile = Column(String)
-    config_ebookconverter = Column(Integer, default=0)
-    config_converterpath = Column(String)
-    config_calibre = Column(String)
-    config_rarfile_location = Column(String)
-    config_theme = Column(Integer, default=0)
-    config_updatechannel = Column(Integer, default=0)
-
-    def __repr__(self):
-        pass
-
 
 class RemoteAuthToken(Base):
     __tablename__ = 'remote_auth_token'
@@ -385,153 +309,11 @@ class RemoteAuthToken(Base):
         return '<Token %r>' % self.id
 
 
-# Class holds all application specific settings in calibre-web
-class Config:
-    def __init__(self):
-        self.db_configured = None
-        self.config_logfile = None
-        self.loadSettings()
-
-    def loadSettings(self):
-        data = session.query(Settings).first()  # type: Settings
-
-        self.config_calibre_dir = data.config_calibre_dir
-        self.config_port = data.config_port
-        self.config_certfile = data.config_certfile
-        self.config_keyfile  = data.config_keyfile
-        self.config_calibre_web_title = data.config_calibre_web_title
-        self.config_books_per_page = data.config_books_per_page
-        self.config_random_books = data.config_random_books
-        self.config_authors_max = data.config_authors_max
-        self.config_title_regex = data.config_title_regex
-        self.config_read_column = data.config_read_column
-        self.config_log_level = data.config_log_level
-        self.config_access_log = data.config_access_log
-        self.config_uploading = data.config_uploading
-        self.config_anonbrowse = data.config_anonbrowse
-        self.config_public_reg = data.config_public_reg
-        self.config_default_role = data.config_default_role
-        self.config_default_show = data.config_default_show
-        self.config_columns_to_ignore = data.config_columns_to_ignore
-        self.config_use_google_drive = data.config_use_google_drive
-        self.config_google_drive_folder = data.config_google_drive_folder
-        self.config_ebookconverter = data.config_ebookconverter
-        self.config_converterpath = data.config_converterpath
-        self.config_calibre = data.config_calibre
-        if data.config_google_drive_watch_changes_response:
-            self.config_google_drive_watch_changes_response = json.loads(data.config_google_drive_watch_changes_response)
-        else:
-            self.config_google_drive_watch_changes_response=None
-        self.config_columns_to_ignore = data.config_columns_to_ignore
-        self.db_configured = bool(self.config_calibre_dir is not None and
-                (not self.config_use_google_drive or os.path.exists(self.config_calibre_dir + '/metadata.db')))
-        self.config_remote_login = data.config_remote_login
-        self.config_use_goodreads = data.config_use_goodreads
-        self.config_goodreads_api_key = data.config_goodreads_api_key
-        self.config_goodreads_api_secret = data.config_goodreads_api_secret
-        self.config_login_type = data.config_login_type
-        # self.config_use_ldap = data.config_use_ldap
-        self.config_ldap_user_object = data.config_ldap_user_object
-        self.config_ldap_openldap = data.config_ldap_openldap
-        self.config_ldap_provider_url = data.config_ldap_provider_url
-        self.config_ldap_port = data.config_ldap_port
-        self.config_ldap_schema = data.config_ldap_schema
-        self.config_ldap_serv_username = data.config_ldap_serv_username
-        self.config_ldap_serv_password = data.config_ldap_serv_password
-        self.config_ldap_use_ssl = data.config_ldap_use_ssl
-        self.config_ldap_use_tls = data.config_ldap_use_ssl
-        self.config_ldap_require_cert = data.config_ldap_require_cert
-        self.config_ldap_cert_path = data.config_ldap_cert_path
-        self.config_ldap_dn = data.config_ldap_dn
-        # self.config_use_github_oauth = data.config_use_github_oauth
-        self.config_github_oauth_client_id = data.config_github_oauth_client_id
-        self.config_github_oauth_client_secret = data.config_github_oauth_client_secret
-        # self.config_use_google_oauth = data.config_use_google_oauth
-        self.config_google_oauth_client_id = data.config_google_oauth_client_id
-        self.config_google_oauth_client_secret = data.config_google_oauth_client_secret
-        self.config_mature_content_tags = data.config_mature_content_tags or u''
-        self.config_logfile = data.config_logfile or u''
-        self.config_access_logfile = data.config_access_logfile or u''
-        self.config_rarfile_location = data.config_rarfile_location
-        self.config_theme = data.config_theme
-        self.config_updatechannel = data.config_updatechannel
-        logger.setup(self.config_logfile, self.config_log_level)
-
-    @property
-    def get_update_channel(self):
-        return self.config_updatechannel
-
-    def get_config_certfile(self):
-        if cli.certfilepath:
-            return cli.certfilepath
-        if cli.certfilepath is "":
-            return None
-        return self.config_certfile
-
-    def get_config_keyfile(self):
-        if cli.keyfilepath:
-            return cli.keyfilepath
-        if cli.certfilepath is "":
-            return None
-        return self.config_keyfile
-
-    def get_config_ipaddress(self):
-        return cli.ipadress or ""
-
-    def get_ipaddress_type(self):
-        return cli.ipv6
-
-    def _has_role(self, role_flag):
-        return constants.has_flag(self.config_default_role, role_flag)
-
-    def role_admin(self):
-        return self._has_role(constants.ROLE_ADMIN)
-
-    def role_download(self):
-        return self._has_role(constants.ROLE_DOWNLOAD)
-
-    def role_viewer(self):
-        return self._has_role(constants.ROLE_VIEWER)
-
-    def role_upload(self):
-        return self._has_role(constants.ROLE_UPLOAD)
-
-    def role_edit(self):
-        return self._has_role(constants.ROLE_EDIT)
-
-    def role_passwd(self):
-        return self._has_role(constants.ROLE_PASSWD)
-
-    def role_edit_shelfs(self):
-        return self._has_role(constants.ROLE_EDIT_SHELFS)
-
-    def role_delete_books(self):
-        return self._has_role(constants.ROLE_DELETE_BOOKS)
-
-    def show_element_new_user(self, value):
-        return constants.has_flag(self.config_default_show, value)
-
-    def show_detail_random(self):
-        return self.show_element_new_user(constants.DETAIL_RANDOM)
-
-    def show_mature_content(self):
-        return self.show_element_new_user(constants.MATURE_CONTENT)
-
-    def mature_content_tags(self):
-        if sys.version_info > (3, 0): # Python3 str, Python2 unicode
-            lstrip = str.lstrip
-        else:
-            lstrip = unicode.lstrip
-        return list(map(lstrip, self.config_mature_content_tags.split(",")))
-
-    def get_Log_Level(self):
-        return logger.get_level_name(self.config_log_level)
-
-
 # Migrate database to current version, has to be updated after every database change. Currently migration from
 # everywhere to curent should work. Migration is done by checking if relevant coloums are existing, and than adding
 # rows with SQL commands
-def migrate_Database():
+def migrate_Database(session):
+    engine = session.bind
     if not engine.dialect.has_table(engine.connect(), "book_read_link"):
         ReadBook.__table__.create(bind=engine)
     if not engine.dialect.has_table(engine.connect(), "bookmark"):
@@ -547,45 +329,12 @@ def migrate_Database():
         conn = engine.connect()
         conn.execute("insert into registration (domain) values('%.%')")
         session.commit()
-    try:
-        session.query(exists().where(Settings.config_use_google_drive)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_use_google_drive` INTEGER DEFAULT 0")
-        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_folder` String DEFAULT ''")
-        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_watch_changes_response` String DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_columns_to_ignore)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_columns_to_ignore` String DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_default_role)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_default_role` SmallInteger DEFAULT 0")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_authors_max)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_authors_max` INTEGER DEFAULT 0")
-        session.commit()
     try:
         session.query(exists().where(BookShelf.order)).scalar()
     except exc.OperationalError:  # Database is not compatible, some rows are missing
         conn = engine.connect()
         conn.execute("ALTER TABLE book_shelf_link ADD column 'order' INTEGER DEFAULT 1")
         session.commit()
-    try:
-        session.query(exists().where(Settings.config_rarfile_location)).scalar()
-        session.commit()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_rarfile_location` String DEFAULT ''")
-        session.commit()
     try:
         create = False
         session.query(exists().where(User.sidebar_view)).scalar()
@@ -617,146 +366,7 @@ def migrate_Database():
         conn.execute("ALTER TABLE user ADD column `mature_content` INTEGER DEFAULT 1")
 
     if session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS).first() is None:
-        create_anonymous_user()
+        create_anonymous_user(session)
-    try:
-        session.query(exists().where(Settings.config_remote_login)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_remote_login` INTEGER DEFAULT 0")
-    try:
-        session.query(exists().where(Settings.config_use_goodreads)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_use_goodreads` INTEGER DEFAULT 0")
-        conn.execute("ALTER TABLE Settings ADD column `config_goodreads_api_key` String DEFAULT ''")
-        conn.execute("ALTER TABLE Settings ADD column `config_goodreads_api_secret` String DEFAULT ''")
-    try:
-        session.query(exists().where(Settings.config_mature_content_tags)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_mature_content_tags` String DEFAULT ''")
-    try:
-        session.query(exists().where(Settings.config_default_show)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_default_show` SmallInteger DEFAULT 2047")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_logfile)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_logfile` String DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_certfile)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_certfile` String DEFAULT ''")
-        conn.execute("ALTER TABLE Settings ADD column `config_keyfile` String DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_read_column)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_read_column` INTEGER DEFAULT 0")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_ebookconverter)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_ebookconverter` INTEGER DEFAULT 0")
-        conn.execute("ALTER TABLE Settings ADD column `config_converterpath` String DEFAULT ''")
-        conn.execute("ALTER TABLE Settings ADD column `config_calibre` String DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_login_type)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_login_type` INTEGER DEFAULT 0")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_ldap_provider_url)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_provider_url` String DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_ldap_port)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_port` INTEGER DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_ldap_schema)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_schema` String DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_ldap_serv_username)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_serv_username` String DEFAULT ''")
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_serv_password` String DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_ldap_use_ssl)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_use_ssl` INTEGER DEFAULT 0")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_ldap_use_tls)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_use_tls` INTEGER DEFAULT 0")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_ldap_require_cert)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_require_cert` INTEGER DEFAULT 0")
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_cert_path` String DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_ldap_dn)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_dn` String DEFAULT ''")
-        conn.execute("ALTER TABLE Settings ADD column `config_github_oauth_client_id` String DEFAULT ''")
-        conn.execute("ALTER TABLE Settings ADD column `config_github_oauth_client_secret` String DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_ldap_user_object)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_user_object` String DEFAULT ''")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_ldap_openldap)).scalar()
-    except exc.OperationalError:
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_ldap_openldap` INTEGER DEFAULT 0")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_theme)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_theme` INTEGER DEFAULT 0")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_updatechannel)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_updatechannel` INTEGER DEFAULT 0")
-        session.commit()
-    try:
-        session.query(exists().where(Settings.config_access_log)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE Settings ADD column `config_access_log` INTEGER DEFAULT 0")
-        conn.execute("ALTER TABLE Settings ADD column `config_access_logfile` String DEFAULT ''")
-        session.commit()
     try:
         # check if one table with autoincrement is existing (should be user table)
         conn = engine.connect()
@@ -792,42 +402,13 @@ def migrate_Database():
     session.commit()
 
 
-def clean_database():
+def clean_database(session):
     # Remove expired remote login tokens
     now = datetime.datetime.now()
     session.query(RemoteAuthToken).filter(now > RemoteAuthToken.expiration).delete()
-
-
-def create_default_config():
-    settings = Settings()
-    settings.mail_server = "mail.example.com"
-    settings.mail_port = 25
-    settings.mail_use_ssl = 0
-    settings.mail_login = "mail@example.com"
-    settings.mail_password = "mypassword"
-    settings.mail_from = "automailer <mail@example.com>"
-
-    session.add(settings)
     session.commit()
 
 
-def get_mail_settings():
-    settings = session.query(Settings).first()
-
-    if not settings:
-        return {}
-
-    data = {
-        'mail_server': settings.mail_server,
-        'mail_port': settings.mail_port,
-        'mail_use_ssl': settings.mail_use_ssl,
-        'mail_login': settings.mail_login,
-        'mail_password': settings.mail_password,
-        'mail_from': settings.mail_from
-    }
-
-    return data
-
 # Save downloaded books per user in calibre-web's own database
 def update_download(book_id, user_id):
     check = session.query(Downloads).filter(Downloads.user_id == user_id).filter(Downloads.book_id ==
@@ -844,7 +425,7 @@ def delete_download(book_id):
     session.commit()
 
 # Generate user Guest (translated text), as anoymous user, no rights
-def create_anonymous_user():
+def create_anonymous_user(session):
     user = User()
     user.nickname = "Guest"
     user.email = 'no@email'
@@ -854,12 +435,12 @@ def create_anonymous_user():
     session.add(user)
     try:
         session.commit()
-    except Exception:
+    except Exception as e:
         session.rollback()
 
 
 # Generate User admin with admin123 password, and access to everything
-def create_admin_user():
+def create_admin_user(session):
     user = User()
     user.nickname = "admin"
     user.role = constants.ADMIN_USER_ROLES
@@ -873,23 +454,37 @@ def create_admin_user():
     except Exception:
         session.rollback()
 
-def init_db():
+
+def init_db(app_db_path):
     # Open session for database connection
     global session
+
+    engine = create_engine(u'sqlite:///{0}'.format(app_db_path), echo=False)
+
     Session = sessionmaker()
     Session.configure(bind=engine)
     session = Session()
 
-
+    if os.path.exists(app_db_path):
-    if not os.path.exists(cli.settingspath):
-        try:
         Base.metadata.create_all(engine)
-            create_default_config()
+        migrate_Database(session)
-            create_admin_user()
+        clean_database(session)
-            create_anonymous_user()
-        except Exception:
-            raise
     else:
         Base.metadata.create_all(engine)
-        migrate_Database()
+        create_admin_user(session)
-        clean_database()
+        create_anonymous_user(session)
+
+
+def dispose():
+    global session
+
+    engine = None
+    if session:
+        engine = session.bind
+        try: session.close()
+        except: pass
+        session = None
+
+    if engine:
+        try: engine.dispose()
+        except: pass

cps/updater.py

@@ -22,7 +22,6 @@ import sys
 import os
 import datetime
 import json
-import requests
 import shutil
 import threading
 import time
@@ -30,6 +29,7 @@ import zipfile
 from io import BytesIO
 from tempfile import gettempdir
 
+import requests
 from babel.dates import format_datetime
 from flask_babel import gettext as _
 
@@ -58,15 +58,13 @@ class Updater(threading.Thread):
         self.updateIndex = None
 
     def get_current_version_info(self):
-        if config.get_update_channel == constants.UPDATE_STABLE:
+        if config.config_updatechannel == constants.UPDATE_STABLE:
             return self._stable_version_info()
-        else:
         return self._nightly_version_info()
 
     def get_available_updates(self, request_method):
-        if config.get_update_channel == constants.UPDATE_STABLE:
+        if config.config_updatechannel == constants.UPDATE_STABLE:
             return self._stable_available_updates(request_method)
-        else:
         return self._nightly_available_updates(request_method)
 
     def run(self):
@@ -430,9 +428,8 @@ class Updater(threading.Thread):
         return json.dumps(status)
 
     def _get_request_path(self):
-        if config.get_update_channel == constants.UPDATE_STABLE:
+        if config.config_updatechannel == constants.UPDATE_STABLE:
             return self.updateFile
-        else:
         return _REPOSITORY_API_URL + '/zipball/master'
 
     def _load_remote_data(self, repository_url):

cps/web.py

@@ -41,18 +41,20 @@ from werkzeug.exceptions import default_exceptions
 from werkzeug.datastructures import Headers
 from werkzeug.security import generate_password_hash, check_password_hash
 
-from . import constants, logger, isoLanguages, ldap1
+from . import constants, logger, isoLanguages, services
-from . import global_WorkerThread, searched_ids, lm, babel, db, ub, config, get_locale, app, language_table
+from . import global_WorkerThread, searched_ids, lm, babel, db, ub, config, negociate_locale, get_locale, app
 from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download
 from .helper import common_filters, get_search_results, fill_indexpage, speaking_language, check_valid_domain, \
-        order_authors, get_typeahead, render_task_status, json_serial, get_unique_other_books, get_cc_columns, \
+        order_authors, get_typeahead, render_task_status, json_serial, get_cc_columns, \
         get_book_cover, get_download_link, send_mail, generate_random_password, send_registration_mail, \
         check_send_to_kindle, check_read_formats, lcase
 from .pagination import Pagination
 from .redirect import redirect_back
 
-feature_support = dict()
+feature_support = {
-feature_support['ldap'] = ldap1.ldap_supported()
+        'ldap': bool(services.ldap),
+        'goodreads': bool(services.goodreads)
+    }
 
 try:
     from .oauth_bb import oauth_check, register_user_with_oauth, logout_oauth_user, get_oauth_status
@@ -61,12 +63,6 @@ except ImportError:
     feature_support['oauth'] = False
     oauth_check = {}
 
-try:
-    from goodreads.client import GoodreadsClient
-    feature_support['goodreads'] = True
-except ImportError:
-    feature_support['goodreads'] = False
-
 try:
     from functools import wraps
 except ImportError:
@@ -230,8 +226,11 @@ def render_title_template(*args, **kwargs):
 
 @web.before_app_request
 def before_request():
+    # log.debug("before_request: %s %s %r", request.method, request.path, getattr(request, 'locale', None))
+    request._locale = negociate_locale()
     g.user = current_user
     g.allow_registration = config.config_public_reg
+    g.allow_anonymous = config.config_anonbrowse
     g.allow_upload = config.config_uploading
     g.current_theme = config.config_theme
     g.config_authors_max = config.config_authors_max
@@ -292,7 +291,7 @@ def toggle_read(book_id):
         ub.session.commit()
     else:
         try:
-            db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
+            db.update_title_sort(config)
             book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
             read_status = getattr(book, 'custom_column_' + str(config.config_read_column))
             if len(read_status):
@@ -396,10 +395,10 @@ def get_series_json():
 def get_languages_json():
     if request.method == "GET":
         query = request.args.get('q').lower()
-        languages = language_table[get_locale()]
+        language_names = isoLanguages.get_language_names(get_locale())
-        entries_start = [s for key, s in languages.items() if s.lower().startswith(query.lower())]
+        entries_start = [s for key, s in language_names.items() if s.lower().startswith(query.lower())]
         if len(entries_start) < 5:
-            entries = [s for key, s in languages.items() if query in s.lower()]
+            entries = [s for key, s in language_names.items() if query in s.lower()]
             entries_start.extend(entries[0:(5-len(entries_start))])
             entries_start = list(set(entries_start))
         json_dumps = json.dumps([dict(name=r) for r in entries_start[0:5]])
@@ -534,29 +533,26 @@ def render_hot_books(page):
         abort(404)
 
 
-def render_author_books(page, book_id, order):
+
-    entries, __, pagination = fill_indexpage(page, db.Books, db.Books.authors.any(db.Authors.id == book_id),
+def render_author_books(page, author_id, order):
+    entries, __, pagination = fill_indexpage(page, db.Books, db.Books.authors.any(db.Authors.id == author_id),
                                              [order[0], db.Series.name, db.Books.series_index],
                                              db.books_series_link, db.Series)
     if entries is None or not len(entries):
         flash(_(u"Error opening eBook. File does not exist or file is not accessible:"), category="error")
         return redirect(url_for("web.index"))
 
-    name = db.session.query(db.Authors).filter(db.Authors.id == book_id).first().name.replace('|', ',')
+    author = db.session.query(db.Authors).get(author_id)
+    author_name = author.name.replace('|', ',')
 
     author_info = None
     other_books = []
-    if feature_support['goodreads'] and config.config_use_goodreads:
+    if services.goodreads and config.config_use_goodreads:
-        try:
+        author_info = services.goodreads.get_author_info(author_name)
-            gc = GoodreadsClient(config.config_goodreads_api_key, config.config_goodreads_api_secret)
+        other_books = services.goodreads.get_other_books(author_info, entries)
-            author_info = gc.find_author(author_name=name)
-            other_books = get_unique_other_books(entries.all(), author_info.books)
-        except Exception:
-            # Skip goodreads, if site is down/inaccessible
-            logger.error('Goodreads website is down/inaccessible')
 
-    return render_title_template('author.html', entries=entries, pagination=pagination, id=book_id,
+    return render_title_template('author.html', entries=entries, pagination=pagination, id=author_id,
-                                 title=_(u"Author: %(name)s", name=name), author=author_info, other_books=other_books,
+                                 title=_(u"Author: %(name)s", name=author_name), author=author_info, other_books=other_books,
                                  page="author")
 
 
@@ -985,10 +981,7 @@ def serve_book(book_id, book_format):
     log.info('Serving book: %s', data.name)
     if config.config_use_google_drive:
         headers = Headers()
-        try:
+        headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream")
-            headers["Content-Type"] = mimetypes.types_map['.' + book_format]
-        except KeyError:
-            headers["Content-Type"] = "application/octet-stream"
         df = getFileFromEbooksFolder(book.path, data.name + "." + book_format)
         return do_gdrive_download(df, headers)
     else:
@@ -1007,7 +1000,7 @@ def download_link(book_id, book_format, anyname):
 @login_required
 @download_required
 def send_to_kindle(book_id, book_format, convert):
-    settings = ub.get_mail_settings()
+    settings = config.get_mail_settings()
     if settings.get("mail_server", "mail.example.com") == "mail.example.com":
         flash(_(u"Please configure the SMTP mail settings first..."), category="error")
     elif current_user.kindle_mail:
@@ -1085,38 +1078,30 @@ def login():
         return redirect(url_for('admin.basic_configuration'))
     if current_user is not None and current_user.is_authenticated:
         return redirect(url_for('web.index'))
-    if config.config_login_type == 1 and not feature_support['ldap']:
+    if config.config_login_type == constants.LOGIN_LDAP and not services.ldap:
         flash(_(u"Cannot activate LDAP authentication"), category="error")
     if request.method == "POST":
         form = request.form.to_dict()
         user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == form['username'].strip().lower())\
             .first()
-        if config.config_login_type == 1 and user and feature_support['ldap']:
+        if config.config_login_type == constants.LOGIN_LDAP and services.ldap and user:
-            try:
+            login_result = services.ldap.bind_user(form['username'], form['password'])
-                if ldap1.ldap.bind_user(form['username'], form['password']) is not None:
+            if login_result:
                 login_user(user, remember=True)
                 flash(_(u"you are now logged in as: '%(nickname)s'", nickname=user.nickname),
                       category="success")
                 return redirect_back(url_for("web.index"))
-            except ldap1.ldap.INVALID_CREDENTIALS as e:
+            if login_result is None:
-                log.error('Login Error: ' + str(e))
+                flash(_(u"Could not login. LDAP server down, please contact your administrator"), category="error")
+            else:
                 ipAdress = request.headers.get('X-Forwarded-For', request.remote_addr)
                 log.info('LDAP Login failed for user "%s" IP-adress: %s', form['username'], ipAdress)
                 flash(_(u"Wrong Username or Password"), category="error")
-            except ldap1.ldap.SERVER_DOWN:
-                log.info('LDAP Login failed, LDAP Server down')
-                flash(_(u"Could not login. LDAP server down, please contact your administrator"), category="error")
-            '''except LDAPException as exception:
-                app.logger.error('Login Error: ' + str(exception))
-                ipAdress = request.headers.get('X-Forwarded-For', request.remote_addr)
-                app.logger.info('LDAP Login failed for user "' + form['username'] + ', IP-address :' + ipAdress)
-                flash(_(u"Wrong Username or Password"), category="error")'''
         else:
-            if user and check_password_hash(user.password, form['password']) and user.nickname is not "Guest":
+            if user and check_password_hash(user.password, form['password']) and user.nickname != "Guest":
                 login_user(user, remember=True)
                 flash(_(u"You are now logged in as: '%(nickname)s'", nickname=user.nickname), category="success")
                 return redirect_back(url_for("web.index"))
-            else:
             ipAdress = request.headers.get('X-Forwarded-For', request.remote_addr)
             log.info('Login failed for user "%s" IP-adress: %s', form['username'], ipAdress)
             flash(_(u"Wrong Username or Password"), category="error")

cps/worker.py

@@ -340,6 +340,8 @@ class WorkerThread(threading.Thread):
         check = p.returncode
         calibre_traceback = p.stderr.readlines()
         for ele in calibre_traceback:
+            if sys.version_info < (3, 0):
+                ele = ele.decode('utf-8')
             log.debug(ele.strip('\n'))
             if not ele.startswith('Traceback') and not ele.startswith('  File'):
                 error_message = "Calibre failed with error: %s" % ele.strip('\n')