Parallel requests of metadata provider

README.md

@@ -62,7 +62,7 @@ Please note that running the above install command can fail on some versions of
 
 ## Requirements
 
-python 3.x+
+python 3.5+
 
 Optionally, to enable on-the-fly conversion from one ebook format to another when using the send-to-kindle feature, or during editing of ebooks metadata:
 

cps/metadata_provider/comicvine.py

@@ -26,7 +26,7 @@ class ComicVine(Metadata):
     __name__ = "ComicVine"
     __id__ = "comicvine"
 
-    def search(self, query):
+    def search(self, query, __):
         val = list()
         apikey = "57558043c53943d5d1e96a9ad425b0eb85532ee6"
         if self.active:

cps/metadata_provider/google.py

@@ -26,7 +26,7 @@ class Google(Metadata):
     __name__ = "Google"
     __id__ = "google"
 
-    def search(self, query):
+    def search(self, query, __):
         if self.active:
             val = list()
             result = requests.get("https://www.googleapis.com/books/v1/volumes?q="+query.replace(" ","+"))

cps/metadata_provider/scholar.py

@@ -17,7 +17,6 @@
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 from scholarly import scholarly
-from flask import url_for
 
 from cps.services.Metadata import Metadata
 
@@ -26,7 +25,7 @@ class scholar(Metadata):
     __name__ = "Google Scholar"
     __id__ = "googlescholar"
 
-    def search(self, query):
+    def search(self, query, generic_cover=""):
         val = list()
         if self.active:
             scholar_gen = scholarly.search_pubs(' '.join(query.split('+')))
@@ -45,7 +44,7 @@ class scholar(Metadata):
                 v['tags'] = ""
                 v['ratings'] = 0
                 v['series'] = ""
-                v['cover'] = url_for('static', filename='generic_cover.jpg')
+                v['cover'] = generic_cover
                 v['url'] = publication.get('pub_url') or publication.get('eprint_url') or "",
                 v['source'] = {
                     "id": self.__id__,

cps/search_metadata.py

@@ -22,8 +22,10 @@ import json
 import importlib
 import sys
 import inspect
+import datetime
+import concurrent.futures
 
-from flask import Blueprint, request, Response
+from flask import Blueprint, request, Response, url_for
 from flask_login import current_user
 from flask_login import login_required
 from sqlalchemy.orm.attributes import flag_modified
@@ -32,6 +34,7 @@ from sqlalchemy.exc import OperationalError, InvalidRequestError
 from . import constants, logger, ub
 from cps.services.Metadata import Metadata
 
+
 meta = Blueprint('metadata', __name__)
 
 log = logger.create()
@@ -101,7 +104,19 @@ def metadata_search():
     data = list()
     active = current_user.view_settings.get('metadata', {})
     if query:
-        for c in cl:
-            if active.get(c.__id__, True):
-                data.extend(c.search(query))
+        a = datetime.datetime.now()
+        static_cover = url_for('static', filename='generic_cover.jpg')
+        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+            meta = {executor.submit(c.search, query, static_cover): c for c in cl if active.get(c.__id__, True)}
+            for future in concurrent.futures.as_completed(meta):
+                data.extend(future.result())
+        b = datetime.datetime.now()
+        c = b - a
+        log.info(c.total_seconds())
     return Response(json.dumps(data), mimetype='application/json')
+
+
+
+
+
+