From baffe1f5377cd2d6bf1a53b59eef1f7aadda0f9a Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Mon, 28 Mar 2022 13:58:41 +0200 Subject: [PATCH] Plus ("+" vs. "%2B") encoded search strings for opds search feeds are now working in request string (fix for #2175) --- cps/gevent_wsgi.py | 29 + cps/opds.py | 2 +- cps/server.py | 10 +- cps/tornado_wsgi.py | 94 +++ test/Calibre-Web TestSummary_Linux.html | 938 ++++++++---------------- 5 files changed, 445 insertions(+), 628 deletions(-) create mode 100644 cps/gevent_wsgi.py create mode 100644 cps/tornado_wsgi.py diff --git a/cps/gevent_wsgi.py b/cps/gevent_wsgi.py new file mode 100644 index 00000000..b044f31b --- /dev/null +++ b/cps/gevent_wsgi.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2022 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from gevent.pywsgi import WSGIHandler + +class MyWSGIHandler(WSGIHandler): + def get_environ(self): + env = super().get_environ() + path, __ = self.path.split('?', 1) if '?' in self.path else (self.path, '') + env['RAW_URI'] = path + return env + + diff --git a/cps/opds.py b/cps/opds.py index 8907f628..cb8f397e 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -85,7 +85,7 @@ def feed_osd(): @requires_basic_auth_if_no_ano def feed_cc_search(query): # Handle strange query from Libera Reader with + instead of spaces - plus_query = unquote_plus(request.base_url.split('/opds/search/')[1]).strip() + plus_query = unquote_plus(request.environ['RAW_URI'].split('/opds/search/')[1]).strip() return feed_search(plus_query) diff --git a/cps/server.py b/cps/server.py index e261c50a..0ffdbd18 100644 --- a/cps/server.py +++ b/cps/server.py @@ -25,6 +25,7 @@ import subprocess # nosec try: from gevent.pywsgi import WSGIServer + from .gevent_wsgi import MyWSGIHandler from gevent.pool import Pool from gevent import __version__ as _version from greenlet import GreenletExit @@ -32,7 +33,7 @@ try: VERSION = 'Gevent ' + _version _GEVENT = True except ImportError: - from tornado.wsgi import WSGIContainer + from .tornado_wsgi import MyWSGIContainer from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop from tornado import version as _version @@ -202,7 +203,8 @@ class WebServer(object): if output is None: output = _readable_listen_address(self.listen_address, self.listen_port) log.info('Starting Gevent server on %s', output) - self.wsgiserver = WSGIServer(sock, self.app, log=self.access_logger, spawn=Pool(), **ssl_args) + self.wsgiserver = WSGIServer(sock, self.app, log=self.access_logger, handler_class=MyWSGIHandler, + spawn=Pool(), **ssl_args) if ssl_args: wrap_socket = self.wsgiserver.wrap_socket def my_wrap_socket(*args, **kwargs): @@ -225,8 +227,8 @@ class WebServer(object): asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) log.info('Starting Tornado server on %s', _readable_listen_address(self.listen_address, self.listen_port)) - # Max Buffersize set to 200MB ) - http_server = HTTPServer(WSGIContainer(self.app), + # Max Buffersize set to 200MB + http_server = HTTPServer(MyWSGIContainer(self.app), max_buffer_size=209700000, ssl_options=self.ssl_args) http_server.listen(self.listen_port, self.listen_address) diff --git a/cps/tornado_wsgi.py b/cps/tornado_wsgi.py new file mode 100644 index 00000000..af93219c --- /dev/null +++ b/cps/tornado_wsgi.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2022 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from tornado.wsgi import WSGIContainer +import tornado + +from tornado import escape +from tornado import httputil + +from typing import List, Tuple, Optional, Callable, Any, Dict, Text +from types import TracebackType +import typing + +if typing.TYPE_CHECKING: + from typing import Type # noqa: F401 + from wsgiref.types import WSGIApplication as WSGIAppType # noqa: F4 + +class MyWSGIContainer(WSGIContainer): + + def __call__(self, request: httputil.HTTPServerRequest) -> None: + data = {} # type: Dict[str, Any] + response = [] # type: List[bytes] + + def start_response( + status: str, + headers: List[Tuple[str, str]], + exc_info: Optional[ + Tuple[ + "Optional[Type[BaseException]]", + Optional[BaseException], + Optional[TracebackType], + ] + ] = None, + ) -> Callable[[bytes], Any]: + data["status"] = status + data["headers"] = headers + return response.append + + app_response = self.wsgi_application( + MyWSGIContainer.environ(request), start_response + ) + try: + response.extend(app_response) + body = b"".join(response) + finally: + if hasattr(app_response, "close"): + app_response.close() # type: ignore + if not data: + raise Exception("WSGI app did not call start_response") + + status_code_str, reason = data["status"].split(" ", 1) + status_code = int(status_code_str) + headers = data["headers"] # type: List[Tuple[str, str]] + header_set = set(k.lower() for (k, v) in headers) + body = escape.utf8(body) + if status_code != 304: + if "content-length" not in header_set: + headers.append(("Content-Length", str(len(body)))) + if "content-type" not in header_set: + headers.append(("Content-Type", "text/html; charset=UTF-8")) + if "server" not in header_set: + headers.append(("Server", "TornadoServer/%s" % tornado.version)) + + start_line = httputil.ResponseStartLine("HTTP/1.1", status_code, reason) + header_obj = httputil.HTTPHeaders() + for key, value in headers: + header_obj.add(key, value) + assert request.connection is not None + request.connection.write_headers(start_line, header_obj, chunk=body) + request.connection.finish() + self._log(status_code, request) + + @staticmethod + def environ(request: httputil.HTTPServerRequest) -> Dict[Text, Any]: + environ = WSGIContainer.environ(request) + environ['RAW_URI'] = request.path + return environ + diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 4427548a..54fda2b1 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2022-03-26 21:40:01

+

Start Time: 2022-03-28 06:40:49

-

Stop Time: 2022-03-27 04:18:33

+

Stop Time: 2022-03-28 12:18:13

-

Duration: 4h 50 min

+

Duration: 4h 46 min

@@ -891,11 +891,11 @@ - + TestEditBooks 36 - 34 - 1 + 35 + 0 0 1 @@ -1237,31 +1237,11 @@ - +
TestEditBooks - test_upload_cover_hdd
- -
- FAIL -
- - - - + PASS @@ -1606,15 +1586,15 @@ AssertionError: <selenium.webdriver.remote.webelement.WebElement (session= - + TestEditBooksOnGdrive - 20 18 - 2 + 18 + 0 0 0 - Detail + Detail @@ -1764,38 +1744,7 @@ AssertionError: <selenium.webdriver.remote.webelement.WebElement (session= - - -
TestEditBooksOnGdrive - test_upload_book_epub
- - -
- FAIL -
- - - - - - - - - +
TestEditBooksOnGdrive - test_upload_book_lit
@@ -1804,36 +1753,7 @@ AssertionError: '8936' != '1103' - - -
TestEditBooksOnGdrive - test_upload_cover_hdd
- - -
- FAIL -
- - - - - - - - - +
TestEditBooksOnGdrive - test_watch_metadata
@@ -2011,11 +1931,11 @@ AssertionError: 0.0 not greater than 0.02 - + TestErrorReadColumn 2 - 1 - 1 + 2 + 0 0 0 @@ -2034,87 +1954,11 @@ AssertionError: 0.0 not greater than 0.02 - +
TestErrorReadColumn - test_invalid_custom_read_column
- -
- FAIL -
- - - - - - - - - - - _ErrorHolder - 1 - 0 - 0 - 1 - 0 - - Detail - - - - - - - -
tearDownClass (test_error_read_column)
- - -
- ERROR -
- - - - + PASS @@ -2128,13 +1972,13 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:160 1 - Detail + Detail - +
TestFilePicker - test_filepicker_limited_file
@@ -2143,19 +1987,19 @@ element.find/</<@chrome://remote/content/marionette/element.js:300:16 +
TestFilePicker - test_filepicker_new_file
- SKIP + SKIP
-