From d1e6a858037988816de9817f3d2bdec1549e88c3 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 29 Aug 2021 14:36:05 +0200 Subject: [PATCH] Parallel requests of metadata provider --- README.md | 2 +- cps/metadata_provider/comicvine.py | 2 +- cps/metadata_provider/google.py | 2 +- cps/metadata_provider/scholar.py | 5 ++--- cps/search_metadata.py | 23 +++++++++++++++++++---- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index bddc4257..0fc6447d 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Please note that running the above install command can fail on some versions of ## Requirements -python 3.x+ +python 3.5+ Optionally, to enable on-the-fly conversion from one ebook format to another when using the send-to-kindle feature, or during editing of ebooks metadata: diff --git a/cps/metadata_provider/comicvine.py b/cps/metadata_provider/comicvine.py index 2309fc81..8f496608 100644 --- a/cps/metadata_provider/comicvine.py +++ b/cps/metadata_provider/comicvine.py @@ -26,7 +26,7 @@ class ComicVine(Metadata): __name__ = "ComicVine" __id__ = "comicvine" - def search(self, query): + def search(self, query, __): val = list() apikey = "57558043c53943d5d1e96a9ad425b0eb85532ee6" if self.active: diff --git a/cps/metadata_provider/google.py b/cps/metadata_provider/google.py index 167ab8a3..f3d02d8e 100644 --- a/cps/metadata_provider/google.py +++ b/cps/metadata_provider/google.py @@ -26,7 +26,7 @@ class Google(Metadata): __name__ = "Google" __id__ = "google" - def search(self, query): + def search(self, query, __): if self.active: val = list() result = requests.get("https://www.googleapis.com/books/v1/volumes?q="+query.replace(" ","+")) diff --git a/cps/metadata_provider/scholar.py b/cps/metadata_provider/scholar.py index 9302806c..6e13c768 100644 --- a/cps/metadata_provider/scholar.py +++ b/cps/metadata_provider/scholar.py @@ -17,7 +17,6 @@ # along with this program. If not, see . from scholarly import scholarly -from flask import url_for from cps.services.Metadata import Metadata @@ -26,7 +25,7 @@ class scholar(Metadata): __name__ = "Google Scholar" __id__ = "googlescholar" - def search(self, query): + def search(self, query, generic_cover=""): val = list() if self.active: scholar_gen = scholarly.search_pubs(' '.join(query.split('+'))) @@ -45,7 +44,7 @@ class scholar(Metadata): v['tags'] = "" v['ratings'] = 0 v['series'] = "" - v['cover'] = url_for('static', filename='generic_cover.jpg') + v['cover'] = generic_cover v['url'] = publication.get('pub_url') or publication.get('eprint_url') or "", v['source'] = { "id": self.__id__, diff --git a/cps/search_metadata.py b/cps/search_metadata.py index 4d648b1c..65a33bed 100644 --- a/cps/search_metadata.py +++ b/cps/search_metadata.py @@ -22,8 +22,10 @@ import json import importlib import sys import inspect +import datetime +import concurrent.futures -from flask import Blueprint, request, Response +from flask import Blueprint, request, Response, url_for from flask_login import current_user from flask_login import login_required from sqlalchemy.orm.attributes import flag_modified @@ -32,6 +34,7 @@ from sqlalchemy.exc import OperationalError, InvalidRequestError from . import constants, logger, ub from cps.services.Metadata import Metadata + meta = Blueprint('metadata', __name__) log = logger.create() @@ -101,7 +104,19 @@ def metadata_search(): data = list() active = current_user.view_settings.get('metadata', {}) if query: - for c in cl: - if active.get(c.__id__, True): - data.extend(c.search(query)) + a = datetime.datetime.now() + static_cover = url_for('static', filename='generic_cover.jpg') + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + meta = {executor.submit(c.search, query, static_cover): c for c in cl if active.get(c.__id__, True)} + for future in concurrent.futures.as_completed(meta): + data.extend(future.result()) + b = datetime.datetime.now() + c = b - a + log.info(c.total_seconds()) return Response(json.dumps(data), mimetype='application/json') + + + + + +