diff --git a/.gitignore b/.gitignore index 981158fe..0ce14757 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ build/ *.egg-info/ .installed.cfg *.egg +.pylint.d # calibre-web *.db diff --git a/cps/__init__.py b/cps/__init__.py index fa389a85..d26c3cdc 100755 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) @@ -34,8 +33,9 @@ from flask_login import LoginManager from flask_babel import Babel from flask_principal import Principal -from . import logger, cache_buster, cli, config_sql, ub +from . import logger, cache_buster, cli, config_sql, ub, db, services from .reverseproxy import ReverseProxied +from .server import WebServer mimetypes.init() @@ -64,16 +64,10 @@ lm.anonymous_user = ub.Anonymous ub.init_db(cli.settingspath) +# pylint: disable=no-member config = config_sql.load_configuration(ub.session) -from . import db, services searched_ids = {} -feature_support = [] - -from .worker import WorkerThread -global_WorkerThread = WorkerThread() - -from .server import WebServer web_server = WebServer() babel = Babel() @@ -109,7 +103,6 @@ def create_app(): if services.goodreads: services.goodreads.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads) - global_WorkerThread.start() return app @babel.localeselector @@ -140,8 +133,7 @@ def get_locale(): @babel.timezoneselector def get_timezone(): user = getattr(g, 'user', None) - if user is not None: - return user.timezone + return user.timezone if user else None from .updater import Updater updater_thread = Updater() diff --git a/cps/about.py b/cps/about.py index 42ffe559..451b6411 100644 --- a/cps/about.py +++ b/cps/about.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) @@ -23,56 +22,46 @@ from __future__ import division, print_function, unicode_literals import sys -import requests +import sqlite3 +from collections import OrderedDict -from flask import Blueprint -from flask import __version__ as flaskVersion +import babel, pytz, requests, sqlalchemy +import werkzeug, flask, flask_login, flask_principal, jinja2 from flask_babel import gettext as _ -from flask_principal import __version__ as flask_principalVersion -from flask_login import login_required -try: - from flask_login import __version__ as flask_loginVersion -except ImportError: - from flask_login.__about__ import __version__ as flask_loginVersion -from werkzeug import __version__ as werkzeugVersion -from babel import __version__ as babelVersion -from jinja2 import __version__ as jinja2Version -from pytz import __version__ as pytzVersion -from sqlalchemy import __version__ as sqlalchemyVersion - -from . import db, converter, uploader -from .isoLanguages import __version__ as iso639Version -from .server import VERSION as serverVersion +from . import db, converter, uploader, server, isoLanguages from .web import render_title_template -about = Blueprint('about', __name__) +about = flask.Blueprint('about', __name__) + + +_VERSIONS = OrderedDict( + Python=sys.version, + WebServer=server.VERSION, + Flask=flask.__version__, + Flask_Login=flask_login.__version__, + Flask_Principal=flask_principal.__version__, + Werkzeug=werkzeug.__version__, + Babel=babel.__version__, + Jinja2=jinja2.__version__, + Requests=requests.__version__, + SqlAlchemy=sqlalchemy.__version__, + pySqlite=sqlite3.version, + SQLite=sqlite3.sqlite_version, + iso639=isoLanguages.__version__, + pytz=pytz.__version__, +) +_VERSIONS.update(uploader.get_versions()) @about.route("/stats") -@login_required +@flask_login.login_required def stats(): counter = db.session.query(db.Books).count() authors = db.session.query(db.Authors).count() categorys = db.session.query(db.Tags).count() series = db.session.query(db.Series).count() - versions = uploader.get_versions() - versions['Babel'] = 'v' + babelVersion - versions['Sqlalchemy'] = 'v' + sqlalchemyVersion - versions['Werkzeug'] = 'v' + werkzeugVersion - versions['Jinja2'] = 'v' + jinja2Version - versions['Flask'] = 'v' + flaskVersion - versions['Flask Login'] = 'v' + flask_loginVersion - versions['Flask Principal'] = 'v' + flask_principalVersion - versions['Iso 639'] = 'v' + iso639Version - versions['pytz'] = 'v' + pytzVersion - - versions['Requests'] = 'v' + requests.__version__ - versions['pySqlite'] = 'v' + db.session.bind.dialect.dbapi.version - versions['Sqlite'] = 'v' + db.session.bind.dialect.dbapi.sqlite_version - versions.update(converter.versioncheck()) - versions.update(serverVersion) - versions['Python'] = sys.version - return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=versions, + _VERSIONS['ebook converter'] = _(converter.get_version()) + return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=_VERSIONS, categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat") diff --git a/cps/admin.py b/cps/admin.py index c7645da0..3e82930c 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -27,10 +27,6 @@ import base64 import json import time from datetime import datetime, timedelta -# try: -# from imp import reload -# except ImportError: -# pass from babel import Locale as LC from babel.dates import format_datetime diff --git a/cps/cli.py b/cps/cli.py index de12be5a..e76a12cc 100644 --- a/cps/cli.py +++ b/cps/cli.py @@ -1,7 +1,5 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- - # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) # Copyright (C) 2018 OzzieIsaacs # @@ -22,50 +20,17 @@ from __future__ import division, print_function, unicode_literals import sys import os import argparse +import socket from .constants import CONFIG_DIR as _CONFIG_DIR from .constants import STABLE_VERSION as _STABLE_VERSION from .constants import NIGHTLY_VERSION as _NIGHTLY_VERSION -VALID_CHARACTERS = 'ABCDEFabcdef:0123456789' - -ipv6 = False - def version_info(): if _NIGHTLY_VERSION[1].startswith('$Format'): return "Calibre-Web version: %s - unkown git-clone" % _STABLE_VERSION['version'] - else: - return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'],_NIGHTLY_VERSION[1]) - - -def validate_ip4(address): - address_list = address.split('.') - if len(address_list) != 4: - return False - for val in address_list: - if not val.isdigit(): - return False - i = int(val) - if i < 0 or i > 255: - return False - return True - - -def validate_ip6(address): - address_list = address.split(':') - return ( - len(address_list) == 8 - and all(len(current) <= 4 for current in address_list) - and all(current in VALID_CHARACTERS for current in address) - ) - - -def validate_ip(address): - if validate_ip4(address) or ipv6: - return address - print("IP address is invalid. Exiting") - sys.exit(1) + return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'], _NIGHTLY_VERSION[1]) parser = argparse.ArgumentParser(description='Calibre Web is a web app' @@ -95,8 +60,8 @@ if sys.version_info < (3, 0): args.s = args.s.decode('utf-8') -settingspath = args.p or os.path.join(_CONFIG_DIR, "app.db") -gdpath = args.g or os.path.join(_CONFIG_DIR, "gdrive.db") +settingspath = args.p or os.path.join(_CONFIG_DIR, "app.db") +gdpath = args.g or os.path.join(_CONFIG_DIR, "gdrive.db") # handle and check parameter for ssl encryption certfilepath = None @@ -108,7 +73,7 @@ if args.c: print("Certfilepath is invalid. Exiting...") sys.exit(1) -if args.c is "": +if args.c == "": certfilepath = "" if args.k: @@ -122,15 +87,26 @@ if (args.k and not args.c) or (not args.k and args.c): print("Certfile and Keyfile have to be used together. Exiting...") sys.exit(1) -if args.k is "": +if args.k == "": keyfilepath = "" # handle and check ipadress argument -if args.i: - ipv6 = validate_ip6(args.i) - ipadress = validate_ip(args.i) -else: - ipadress = None +ipadress = args.i or None +if ipadress: + try: + # try to parse the given ip address with socket + if hasattr(socket, 'inet_pton'): + if ':' in ipadress: + socket.inet_pton(socket.AF_INET6, ipadress) + else: + socket.inet_pton(socket.AF_INET, ipadress) + else: + # on windows python < 3.4, inet_pton is not available + # inet_atom only handles IPv4 addresses + socket.inet_aton(ipadress) + except socket.error as err: + print(ipadress, ':', err) + sys.exit(1) # handle and check user password argument -user_password = args.s or None +user_password = args.s or None diff --git a/cps/config_sql.py b/cps/config_sql.py index 5a9aa412..f4a97a35 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -147,9 +147,6 @@ class _ConfigSQL(object): def get_config_ipaddress(self): return cli.ipadress or "" - def get_ipaddress_type(self): - return cli.ipv6 - def _has_role(self, role_flag): return constants.has_flag(self.config_default_role, role_flag) @@ -236,8 +233,14 @@ class _ConfigSQL(object): if self.config_google_drive_watch_changes_response: self.config_google_drive_watch_changes_response = json.loads(self.config_google_drive_watch_changes_response) - self.db_configured = (self.config_calibre_dir and - (not self.config_use_google_drive or os.path.exists(self.config_calibre_dir + '/metadata.db'))) + + have_metadata_db = bool(self.config_calibre_dir) + if have_metadata_db: + if not self.config_use_google_drive: + db_file = os.path.join(self.config_calibre_dir, 'metadata.db') + have_metadata_db = os.path.isfile(db_file) + self.db_configured = have_metadata_db + logger.setup(self.config_logfile, self.config_log_level) def save(self): @@ -274,6 +277,7 @@ def _migrate_table(session, orm_class): log.debug("%s: %s", column_name, err) column_default = "" if column.default is None else ("DEFAULT %r" % column.default.arg) alter_table = "ALTER TABLE %s ADD COLUMN `%s` %s %s" % (orm_class.__tablename__, column_name, column.type, column_default) + log.debug(alter_table) session.execute(alter_table) changed = True diff --git a/cps/constants.py b/cps/constants.py index 79f01c6f..d1b535c6 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) @@ -136,4 +135,3 @@ NIGHTLY_VERSION[1] = '$Format:%cI$' # clean-up the module namespace del sys, os, namedtuple - diff --git a/cps/converter.py b/cps/converter.py index 6dc44383..32bc273f 100644 --- a/cps/converter.py +++ b/cps/converter.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) @@ -21,41 +20,36 @@ from __future__ import division, print_function, unicode_literals import os import re -from flask_babel import gettext as _ - -from . import config +from . import config, logger from .subproc_wrapper import process_wait -def versionKindle(): - versions = _(u'not installed') - if os.path.exists(config.config_converterpath): +log = logger.create() + +_NOT_CONFIGURED = 'not configured' +_NOT_INSTALLED = 'not installed' +_EXECUTION_ERROR = 'Execution permissions missing' + + +def _get_command_version(path, pattern, argument=None): + if os.path.exists(path): + command = [path] + if argument: + command.append(argument) try: - for lines in process_wait(config.config_converterpath): - if re.search('Amazon kindlegen\(', lines): - versions = lines - except Exception: - versions = _(u'Excecution permissions missing') - return {'kindlegen' : versions} + for line in process_wait(command): + if re.search(pattern, line): + return line + except Exception as ex: + log.warning("%s: %s", path, ex) + return _EXECUTION_ERROR + return _NOT_INSTALLED -def versionCalibre(): - versions = _(u'not installed') - if os.path.exists(config.config_converterpath): - try: - for lines in process_wait([config.config_converterpath, '--version']): - if re.search('ebook-convert.*\(calibre', lines): - versions = lines - except Exception: - versions = _(u'Excecution permissions missing') - return {'Calibre converter' : versions} - - -def versioncheck(): +def get_version(): + version = None if config.config_ebookconverter == 1: - return versionKindle() + version = _get_command_version(config.config_converterpath, r'Amazon kindlegen\(') elif config.config_ebookconverter == 2: - return versionCalibre() - else: - return {'ebook_converter':_(u'not configured')} - + version = _get_command_version(config.config_converterpath, r'ebook-convert.*\(calibre', '--version') + return version or _NOT_CONFIGURED diff --git a/cps/db.py b/cps/db.py index edcdef63..b9853896 100755 --- a/cps/db.py +++ b/cps/db.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) @@ -407,20 +406,18 @@ def setup_db(config): def dispose(): global session - engine = None - if session: - engine = session.bind - try: session.close() - except: pass - session = None - - if engine: - try: engine.dispose() + old_session = session + session = None + if old_session: + try: old_session.close() except: pass + if old_session.bind: + try: old_session.bind.dispose() + except: pass for attr in list(Books.__dict__.keys()): if attr.startswith("custom_column_"): - delattr(Books, attr) + setattr(Books, attr, None) for db_class in cc_classes.values(): Base.metadata.remove(db_class.__table__) diff --git a/cps/editbooks.py b/cps/editbooks.py index 7f850254..2da9991f 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) @@ -30,12 +29,12 @@ from uuid import uuid4 from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response from flask_babel import gettext as _ -from flask_login import current_user +from flask_login import current_user, login_required from . import constants, logger, isoLanguages, gdriveutils, uploader, helper -from . import config, get_locale, db, ub, global_WorkerThread +from . import config, get_locale, db, ub, worker from .helper import order_authors, common_filters -from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required, login_required +from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required editbook = Blueprint('editbook', __name__) @@ -301,7 +300,7 @@ def edit_cc_data(book_id, book, to_save): # remove old cc_val del_cc = getattr(book, cc_string)[0] getattr(book, cc_string).remove(del_cc) - if len(del_cc.books) == 0: + if not del_cc.books or len(del_cc.books) == 0: db.session.delete(del_cc) else: input_tags = to_save[cc_string].split(',') @@ -358,7 +357,7 @@ def upload_single_file(request, book, book_id): # Queue uploader info uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title) - global_WorkerThread.add_upload(current_user.nickname, + worker.add_upload(current_user.nickname, "" + uploadText + "") @@ -667,7 +666,7 @@ def upload(): if error: flash(error, category="error") uploadText=_(u"File %(file)s uploaded", file=book.title) - global_WorkerThread.add_upload(current_user.nickname, + worker.add_upload(current_user.nickname, "" + uploadText + "") # create data for displaying display Full language name instead of iso639.part3language diff --git a/cps/helper.py b/cps/helper.py index 1ceeb0b8..c94093db 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) @@ -60,7 +59,7 @@ try: except ImportError: use_PIL = False -from . import logger, config, global_WorkerThread, get_locale, db, ub, isoLanguages +from . import logger, config, get_locale, db, ub, isoLanguages, worker from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR from .pagination import Pagination @@ -112,7 +111,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, text = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title)) settings['old_book_format'] = old_book_format settings['new_book_format'] = new_book_format - global_WorkerThread.add_convert(file_path, book.id, user_id, text, settings, kindle_mail) + worker.add_convert(file_path, book.id, user_id, text, settings, kindle_mail) return None else: error_message = _(u"%(format)s not found: %(fn)s", @@ -121,9 +120,9 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, def send_test_mail(kindle_mail, user_name): - global_WorkerThread.add_email(_(u'Calibre-Web test e-mail'),None, None, config.get_mail_settings(), - kindle_mail, user_name, _(u"Test e-mail"), - _(u'This e-mail has been sent via Calibre-Web.')) + worker.add_email(_(u'Calibre-Web test e-mail'), None, None, + config.get_mail_settings(), kindle_mail, user_name, + _(u"Test e-mail"), _(u'This e-mail has been sent via Calibre-Web.')) return @@ -138,8 +137,9 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False): text += "Don't forget to change your password after first login.\r\n" text += "Sincerely\r\n\r\n" text += "Your Calibre-Web team" - global_WorkerThread.add_email(_(u'Get Started with Calibre-Web'),None, None, config.get_mail_settings(), - e_mail, None, _(u"Registration e-mail for user: %(name)s", name=user_name), text) + worker.add_email(_(u'Get Started with Calibre-Web'), None, None, + config.get_mail_settings(), e_mail, None, + _(u"Registration e-mail for user: %(name)s", name=user_name), text) return @@ -207,15 +207,15 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id): if convert: # returns None if success, otherwise errormessage return convert_book_format(book_id, calibrepath, u'epub', book_format.lower(), user_id, kindle_mail) - else: - for entry in iter(book.data): - if entry.format.upper() == book_format.upper(): - result = entry.name + '.' + book_format.lower() - global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, config.get_mail_settings(), - kindle_mail, user_id, _(u"E-mail: %(book)s", book=book.title), - _(u'This e-mail has been sent via Calibre-Web.')) - return - return _(u"The requested file could not be read. Maybe wrong permissions?") + + for entry in iter(book.data): + if entry.format.upper() == book_format.upper(): + converted_file_name = entry.name + '.' + book_format.lower() + worker.add_email(_(u"Send to Kindle"), book.path, converted_file_name, + config.get_mail_settings(), kindle_mail, user_id, + _(u"E-mail: %(book)s", book=book.title), _(u'This e-mail has been sent via Calibre-Web.')) + return + return _(u"The requested file could not be read. Maybe wrong permissions?") def get_valid_filename(value, replace_whitespace=True): @@ -232,7 +232,7 @@ def get_valid_filename(value, replace_whitespace=True): value = value.replace(u'§', u'SS') value = value.replace(u'ß', u'ss') value = unicodedata.normalize('NFKD', value) - re_slugify = re.compile('[\W\s-]', re.UNICODE) + re_slugify = re.compile(r'[\W\s-]', re.UNICODE) if isinstance(value, str): # Python3 str, Python2 unicode value = re_slugify.sub('', value).strip() else: @@ -254,7 +254,7 @@ def get_valid_filename(value, replace_whitespace=True): def get_sorted_author(value): try: if ',' not in value: - regexes = ["^(JR|SR)\.?$", "^I{1,3}\.?$", "^IV\.?$"] + regexes = [r"^(JR|SR)\.?$", r"^I{1,3}\.?$", r"^IV\.?$"] combined = "(" + ")|(".join(regexes) + ")" value = value.split(" ") if re.match(combined, value[-1].upper()): diff --git a/cps/logger.py b/cps/logger.py index ddadb472..892dd252 100644 --- a/cps/logger.py +++ b/cps/logger.py @@ -102,15 +102,20 @@ def setup(log_file, log_level=None): log_file = _absolute_log_file(log_file, DEFAULT_LOG_FILE) + log_level = log_level or DEFAULT_LOG_LEVEL + logging.getLogger(__package__).setLevel(log_level) + r = logging.root - r.setLevel(log_level or DEFAULT_LOG_LEVEL) + if log_level >= logging.INFO or os.environ.get('FLASK_DEBUG'): + # avoid spamming the log with debug messages from libraries + r.setLevel(log_level) previous_handler = r.handlers[0] if r.handlers else None if previous_handler: # if the log_file has not changed, don't create a new handler if getattr(previous_handler, 'baseFilename', None) == log_file: return - r.debug("logging to %s level %s", log_file, r.level) + logging.debug("logging to %s level %s", log_file, r.level) if log_file == LOG_TO_STDERR: file_handler = StreamHandler() @@ -162,5 +167,3 @@ class StderrLogger(object): self.buffer += message except Exception: self.log.debug("Logging Error") - - diff --git a/cps/server.py b/cps/server.py index 1d564824..e5fe78e4 100644 --- a/cps/server.py +++ b/cps/server.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) @@ -28,23 +27,30 @@ try: from gevent.pywsgi import WSGIServer from gevent.pool import Pool from gevent import __version__ as _version - VERSION = {'Gevent': 'v' + _version} + VERSION = 'Gevent ' + _version _GEVENT = True except ImportError: from tornado.wsgi import WSGIContainer from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop from tornado import version as _version - VERSION = {'Tornado': 'v' + _version} + VERSION = 'Tornado ' + _version _GEVENT = False -from . import logger, global_WorkerThread +from . import logger log = logger.create() -class WebServer: + +def _readable_listen_address(address, port): + if ':' in address: + address = "[" + address + "]" + return '%s:%s' % (address, port) + + +class WebServer(object): def __init__(self): signal.signal(signal.SIGINT, self._killServer) @@ -56,14 +62,12 @@ class WebServer: self.app = None self.listen_address = None self.listen_port = None - self.IPV6 = False self.unix_socket_file = None self.ssl_args = None def init_app(self, application, config): self.app = application self.listen_address = config.get_config_ipaddress() - self.IPV6 = config.get_ipaddress_type() self.listen_port = config.config_port if config.config_access_log: @@ -78,8 +82,7 @@ class WebServer: keyfile_path = config.get_config_keyfile() if certfile_path and keyfile_path: if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path): - self.ssl_args = {"certfile": certfile_path, - "keyfile": keyfile_path} + self.ssl_args = dict(certfile=certfile_path, keyfile=keyfile_path) else: log.warning('The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl.') log.warning('Cert path: %s', certfile_path) @@ -107,32 +110,33 @@ class WebServer: if os.name != 'nt': unix_socket_file = os.environ.get("CALIBRE_UNIX_SOCKET") if unix_socket_file: - output = "socket:" + unix_socket_file + ":" + str(self.listen_port) - return self._make_gevent_unix_socket(unix_socket_file), output + return self._make_gevent_unix_socket(unix_socket_file), "unix:" + unix_socket_file if self.listen_address: - return (self.listen_address, self.listen_port), self._get_readable_listen_address() + return (self.listen_address, self.listen_port), None if os.name == 'nt': self.listen_address = '0.0.0.0' - return (self.listen_address, self.listen_port), self._get_readable_listen_address() + return (self.listen_address, self.listen_port), None - address = ('', self.listen_port) try: + address = ('::', self.listen_port) sock = WSGIServer.get_listener(address, family=socket.AF_INET6) - output = self._get_readable_listen_address(True) except socket.error as ex: log.error('%s', ex) log.warning('Unable to listen on "", trying on IPv4 only...') - output = self._get_readable_listen_address(False) + address = ('', self.listen_port) sock = WSGIServer.get_listener(address, family=socket.AF_INET) - return sock, output + + return sock, _readable_listen_address(*address) def _start_gevent(self): ssl_args = self.ssl_args or {} try: sock, output = self._make_gevent_socket() + if output is None: + output = _readable_listen_address(self.listen_address, self.listen_port) log.info('Starting Gevent server on %s', output) self.wsgiserver = WSGIServer(sock, self.app, log=self.access_logger, spawn=Pool(), **ssl_args) self.wsgiserver.serve_forever() @@ -142,30 +146,18 @@ class WebServer: self.unix_socket_file = None def _start_tornado(self): - log.info('Starting Tornado server on %s', self._get_readable_listen_address()) + log.info('Starting Tornado server on %s', _readable_listen_address(self.listen_address, self.listen_port)) # Max Buffersize set to 200MB ) http_server = HTTPServer(WSGIContainer(self.app), - max_buffer_size = 209700000, - ssl_options=self.ssl_args) + max_buffer_size=209700000, + ssl_options=self.ssl_args) http_server.listen(self.listen_port, self.listen_address) - self.wsgiserver=IOLoop.instance() + self.wsgiserver = IOLoop.instance() self.wsgiserver.start() # wait for stop signal self.wsgiserver.close(True) - def _get_readable_listen_address(self, ipV6=False): - if self.listen_address == "": - listen_string = '""' - else: - ipV6 = self.IPV6 - listen_string = self.listen_address - if ipV6: - adress = "[" + listen_string + "]" - else: - adress = listen_string - return adress + ":" + str(self.listen_port) - def start(self): try: if _GEVENT: @@ -179,7 +171,6 @@ class WebServer: return False finally: self.wsgiserver = None - global_WorkerThread.stop() if not self.restart: log.info("Performing shutdown of Calibre-Web") @@ -193,7 +184,7 @@ class WebServer: os.execv(sys.executable, arguments) return True - def _killServer(self, signum, frame): + def _killServer(self, ignored_signum, ignored_frame): self.stop() def stop(self, restart=False): diff --git a/cps/services/simpleldap.py b/cps/services/simpleldap.py index 8dd082b5..42a9aacd 100644 --- a/cps/services/simpleldap.py +++ b/cps/services/simpleldap.py @@ -29,10 +29,7 @@ _ldap = LDAP() def init_app(app, config): - global _ldap - if config.config_login_type != constants.LOGIN_LDAP: - _ldap = None return app.config['LDAP_HOST'] = config.config_ldap_provider_url diff --git a/cps/ub.py b/cps/ub.py index 5d0b8c40..d87b5a6a 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -478,13 +478,11 @@ def init_db(app_db_path): def dispose(): global session - engine = None - if session: - engine = session.bind - try: session.close() - except: pass - session = None - - if engine: - try: engine.dispose() + old_session = session + session = None + if old_session: + try: old_session.close() except: pass + if old_session.bind: + try: old_session.bind.dispose() + except: pass diff --git a/cps/web.py b/cps/web.py index e169c6af..bab52edb 100644 --- a/cps/web.py +++ b/cps/web.py @@ -41,8 +41,8 @@ from werkzeug.exceptions import default_exceptions from werkzeug.datastructures import Headers from werkzeug.security import generate_password_hash, check_password_hash -from . import constants, logger, isoLanguages, services -from . import global_WorkerThread, searched_ids, lm, babel, db, ub, config, negociate_locale, get_locale, app +from . import constants, logger, isoLanguages, services, worker +from . import searched_ids, lm, babel, db, ub, config, negociate_locale, get_locale, app from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download from .helper import common_filters, get_search_results, fill_indexpage, speaking_language, check_valid_domain, \ order_authors, get_typeahead, render_task_status, json_serial, get_cc_columns, \ @@ -245,7 +245,7 @@ def before_request(): @web.route("/ajax/emailstat") @login_required def get_email_status_json(): - tasks = global_WorkerThread.get_taskstatus() + tasks = worker.get_taskstatus() answer = render_task_status(tasks) js = json.dumps(answer, default=json_serial) response = make_response(js) @@ -760,7 +760,7 @@ def category_list(): @login_required def get_tasks_status(): # if current user admin, show all email, otherwise only own emails - tasks = global_WorkerThread.get_taskstatus() + tasks = worker.get_taskstatus() answer = render_task_status(tasks) return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks") diff --git a/cps/worker.py b/cps/worker.py index 30ee0907..9744b91f 100644 --- a/cps/worker.py +++ b/cps/worker.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) @@ -25,7 +24,7 @@ import smtplib import socket import time import threading -from datetime import datetime, timedelta +from datetime import datetime try: from StringIO import StringIO @@ -66,6 +65,13 @@ RET_FAIL = 0 RET_SUCCESS = 1 +def _get_main_thread(): + for t in threading.enumerate(): + if t.__class__.__name__ == '_MainThread': + return t + raise Exception("main thread not found?!") + + # For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append # it in MIME Base64 encoded to def get_attachment(bookpath, filename): @@ -173,19 +179,19 @@ class email_SSL(emailbase, smtplib.SMTP_SSL): class WorkerThread(threading.Thread): def __init__(self): - self._stopevent = threading.Event() threading.Thread.__init__(self) self.status = 0 self.current = 0 self.last = 0 self.queue = list() self.UIqueue = list() - self.asyncSMTP=None + self.asyncSMTP = None self.id = 0 # Main thread loop starting the different tasks def run(self): - while not self._stopevent.isSet(): + main_thread = _get_main_thread() + while main_thread.is_alive(): doLock = threading.Lock() doLock.acquire() if self.current != self.last: @@ -200,10 +206,8 @@ class WorkerThread(threading.Thread): self.current += 1 else: doLock.release() - time.sleep(1) - - def stop(self): - self._stopevent.set() + if main_thread.is_alive(): + time.sleep(1) def get_send_status(self): if self.asyncSMTP: @@ -317,7 +321,7 @@ class WorkerThread(threading.Thread): nextline = p.communicate()[0] # Format of error message (kindlegen translates its output texts): # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting. - conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE) + conv_error = re.search(r".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE) # If error occoures, store error message for logfile if conv_error: error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s", @@ -332,7 +336,7 @@ class WorkerThread(threading.Thread): nextline = nextline.decode('utf-8') log.debug(nextline.strip('\r\n')) # parse progress string from calibre-converter - progress = re.search("(\d+)%\s.*", nextline) + progress = re.search(r"(\d+)%\s.*", nextline) if progress: self.UIqueue[self.current]['progress'] = progress.group(1) + ' %' @@ -511,3 +515,23 @@ class WorkerThread(threading.Thread): self.UIqueue[self.current]['stat'] = STAT_FINISH_SUCCESS self.UIqueue[self.current]['progress'] = "100 %" self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime'] + + +_worker = WorkerThread() +_worker.start() + + +def get_taskstatus(): + return _worker.get_taskstatus() + + +def add_email(subject, filepath, attachment, settings, recipient, user_name, taskMessage, text): + return _worker.add_email(subject, filepath, attachment, settings, recipient, user_name, taskMessage, text) + + +def add_upload(user_name, taskMessage): + return _worker.add_upload(user_name, taskMessage) + + +def add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail=None): + return _worker.add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail) diff --git a/optional-requirements.txt b/optional-requirements.txt index d71636a8..db96fbfb 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -11,13 +11,20 @@ PyDrive==1.3.1 PyYAML==3.12 rsa==3.4.2 six==1.10.0 + # goodreads goodreads>=0.3.2 python-Levenshtein>=0.12.0 -#extracting metadata + +# ldap login +python_ldap>=3.0.0 +flask-simpleldap + +# extracting metadata lxml>=3.8.0 Pillow>=4.0.0 rarfile>=2.7 + # other natsort>=2.2.0 git+https://github.com/OzzieIsaacs/comicapi.git@5346716578b2843f54d522f44d01bc8d25001d24#egg=comicapi