Integrate with the official Kobo store endpoint so that no

functionanility is lost by overriding the api_endpoint setting.

Requests are either:
 * Redirected to the Kobo Store
 * Proxied to the Kobo Store
 * Proxied to the Kobo Store and merged with results from CalibreWeb.

cps/kobo.py

@@ -25,16 +25,27 @@ from datetime import datetime
 from time import gmtime, strftime
 
 from jsonschema import validate, exceptions
-from flask import Blueprint, request, make_response, jsonify, json, current_app, url_for
+from flask import (
-
+    Blueprint,
+    request,
+    make_response,
+    jsonify,
+    json,
+    current_app,
+    url_for,
+    redirect,
+)
 from flask_login import login_required
+from werkzeug.datastructures import Headers
 from sqlalchemy import func
+import requests
 
 from . import config, logger, kobo_auth, db, helper
 from .web import download_required
 
-#TODO: Test more formats :) .
+# TODO: Test more formats :) .
 KOBO_SUPPORTED_FORMATS = {"KEPUB"}
+KOBO_STOREAPI_URL = "https://storeapi.kobo.com"
 
 kobo = Blueprint("kobo", __name__, url_prefix="/kobo/<auth_token>")
 kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo)
@@ -55,6 +66,47 @@ def to_epoch_timestamp(datetime_object):
     return (datetime_object - datetime(1970, 1, 1)).total_seconds()
 
 
+def get_store_url_for_current_request():
+    # Programmatically modify the current url to point to the official Kobo store
+    base, sep, request_path_with_auth_token = request.full_path.rpartition("/kobo/")
+    auth_token, sep, request_path = request_path_with_auth_token.rstrip("?").partition(
+        "/"
+    )
+    return KOBO_STOREAPI_URL + "/" + request_path
+
+
+CONNECTION_SPECIFIC_HEADERS = [
+    "connection",
+    "content-encoding",
+    "content-length",
+    "transfer-encoding",
+]
+
+
+def redirect_or_proxy_request():
+    if request.method == "GET":
+        return redirect(get_store_url_for_current_request(), 307)
+    else:
+        # The Kobo device turns other request types into GET requests on redirects, so we instead proxy to the Kobo store ourselves.
+        outgoing_headers = Headers(request.headers)
+        outgoing_headers.remove("Host")
+        store_response = requests.request(
+            method=request.method,
+            url=get_store_url_for_current_request(),
+            headers=outgoing_headers,
+            data=request.get_data(),
+            allow_redirects=False,
+        )
+
+        response_headers = store_response.headers
+        for header_key in CONNECTION_SPECIFIC_HEADERS:
+            response_headers.pop(header_key, default=None)
+
+        return make_response(
+            store_response.content, store_response.status_code, response_headers.items()
+        )
+
+
 class SyncToken:
     """ The SyncToken is used to persist state accross requests.
     When serialized over the response headers, the Kobo device will propagate the token onto following requests to the service.
@@ -138,6 +190,14 @@ class SyncToken:
             books_last_modified=books_last_modified,
         )
 
+    def set_kobo_store_header(self, store_headers):
+        store_headers.set(SyncToken.SYNC_TOKEN_HEADER, self.raw_kobo_store_token)
+
+    def merge_from_store_response(self, store_response):
+        self.raw_kobo_store_token = store_response.headers.get(
+            SyncToken.SYNC_TOKEN_HEADER, ""
+        )
+
     def to_headers(self, headers):
         headers[SyncToken.SYNC_TOKEN_HEADER] = self.build_sync_token()
 
@@ -198,13 +258,40 @@ def HandleSyncRequest():
 
     # Missing feature: Detect server-side book deletions.
 
-    # Missing feature: Join the response with results from the official Kobo store so that users can still buy and access books from the device store (particularly while on-the-road).
+    return generate_sync_response(request, sync_token, entitlements)
+
+
+def generate_sync_response(request, sync_token, entitlements):
+    # We first merge in sync results from the official Kobo store.
+    outgoing_headers = Headers(request.headers)
+    outgoing_headers.remove("Host")
+    sync_token.set_kobo_store_header(outgoing_headers)
+    store_response = requests.request(
+        method=request.method,
+        url=get_store_url_for_current_request(),
+        headers=outgoing_headers,
+        data=request.get_data(),
+    )
+
+    store_entitlements = store_response.json()
+    entitlements += store_entitlements
+    sync_token.merge_from_store_response(store_response)
 
     response = make_response(jsonify(entitlements))
 
     sync_token.to_headers(response.headers)
-    response.headers["x-kobo-sync-mode"] = "delta"
+    try:
-    response.headers["x-kobo-apitoken"] = "e30="
+        # These headers could probably use some more investigation.
+        response.headers["x-kobo-sync"] = store_response.headers["x-kobo-sync"]
+        response.headers["x-kobo-sync-mode"] = store_response.headers[
+            "x-kobo-sync-mode"
+        ]
+        response.headers["x-kobo-recent-reads"] = store_response.headers[
+            "x-kobo-recent-reads"
+        ]
+    except KeyError:
+        pass
+
     return response
 
 
@@ -216,7 +303,7 @@ def HandleMetadataRequest(book_uuid):
     book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
     if not book or not book.data:
         log.info(u"Book %s not found in database", book_uuid)
-        return make_response("Book not found in database.", 404)
+        return redirect_or_proxy_request()
 
     metadata = get_metadata(book)
     return jsonify([metadata])
@@ -356,7 +443,7 @@ def HandleCoverImageRequest(book_uuid, horizontal, vertical, jpeg_quality, monoc
         book_uuid, use_generic_cover_on_failure=False
     )
     if not book_cover:
-        return make_response()
+        return redirect(get_store_url_for_current_request(), 307)
     return book_cover
 
 
@@ -365,173 +452,41 @@ def TopLevelEndpoint():
     return make_response(jsonify({}))
 
 
-@kobo.route("/v1/user/profile")
+# TODO: Implement the following routes
-@kobo.route("/v1/user/loyalty/benefits")
+@kobo.route("/v1/library/<dummy>", methods=["DELETE", "GET"])
-@kobo.route("/v1/analytics/gettests/", methods=["GET", "POST"])
+@kobo.route("/v1/library/<book_uuid>/state", methods=["PUT"])
-@kobo.route("/v1/user/wishlist")
+@kobo.route("/v1/library/tags", methods=["POST"])
-@kobo.route("/v1/user/<dummy>")
+@kobo.route("/v1/library/tags/<shelf_name>", methods=["POST"])
-@kobo.route("/v1/user/recommendations")
+@kobo.route("/v1/library/tags/<tag_id>", methods=["DELETE"])
-@kobo.route("/v1/products/<dummy>")
+def HandleUnimplementedRequest(book_uuid=None, shelf_name=None, tag_id=None):
-@kobo.route("/v1/products/<dummy>/nextread")
+    return redirect_or_proxy_request()
-@kobo.route("/v1/products/featured/<dummy>")
-@kobo.route("/v1/products/featured/")
-@kobo.route("/v1/library/<dummy>", methods=["DELETE", "GET"])  # TODO: implement
-def HandleDummyRequest(dummy=None):
-    return make_response(jsonify({}))
 
 
-@kobo.route("/v1/auth/device", methods=["POST"])
+@kobo.app_errorhandler(404)
-def HandleAuthRequest():
+def handle_404(err):
-    # This AuthRequest isn't used for most of our usecases.
+    # This handler acts as a catch-all for endpoints that we don't have an interest in
-    response = make_response(
+    # implementing (e.g: v1/analytics/gettests, v1/user/recommendations, etc)
-        jsonify(
+    return redirect_or_proxy_request()
-            {
-                "AccessToken": "abcde",
-                "RefreshToken": "abcde",
-                "TokenType": "Bearer",
-                "TrackingId": "abcde",
-                "UserKey": "abcdefgeh",
-            }
-        )
-    )
-    return response
 
 
 @kobo.route("/v1/initialization")
 def HandleInitRequest():
-    resources = NATIVE_KOBO_RESOURCES(
+    outgoing_headers = Headers(request.headers)
-        calibre_web_url=url_for("web.index", _external=True).strip("/")
+    outgoing_headers.remove("Host")
+    store_response = requests.request(
+        method=request.method,
+        url=get_store_url_for_current_request(),
+        headers=outgoing_headers,
+        data=request.get_data(),
     )
-    response = make_response(jsonify({"Resources": resources}))
-    response.headers["x-kobo-apitoken"] = "e30="
-    return response
 
+    store_response_json = store_response.json()
+    if "Resources" in store_response_json:
+        kobo_resources = store_response_json["Resources"]
 
-def NATIVE_KOBO_RESOURCES(calibre_web_url):
+        calibre_web_url=url_for("web.index", _external=True).strip("/")
-    return {
+        kobo_resources["image_host"] = calibre_web_url
-        "account_page": "https://secure.kobobooks.com/profile",
+        kobo_resources["image_url_quality_template"] = calibre_web_url + "/{ImageId}/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg"
-        "account_page_rakuten": "https://my.rakuten.co.jp/",
+        kobo_resources["image_url_template"] = calibre_web_url + "/{ImageId}/{Width}/{Height}/false/image.jpg"
-        "add_entitlement": "https://storeapi.kobo.com/v1/library/{RevisionIds}",
+
-        "affiliaterequest": "https://storeapi.kobo.com/v1/affiliate",
+    return make_response(store_response_json, store_response.status_code)
-        "audiobook_subscription_orange_deal_inclusion_url": "https://authorize.kobo.com/inclusion",
-        "authorproduct_recommendations": "https://storeapi.kobo.com/v1/products/books/authors/recommendations",
-        "autocomplete": "https://storeapi.kobo.com/v1/products/autocomplete",
-        "blackstone_header": {"key": "x-amz-request-payer", "value": "requester"},
-        "book": "https://storeapi.kobo.com/v1/products/books/{ProductId}",
-        "book_detail_page": "https://store.kobobooks.com/{culture}/ebook/{slug}",
-        "book_detail_page_rakuten": "http://books.rakuten.co.jp/rk/{crossrevisionid}",
-        "book_landing_page": "https://store.kobobooks.com/ebooks",
-        "book_subscription": "https://storeapi.kobo.com/v1/products/books/subscriptions",
-        "categories": "https://storeapi.kobo.com/v1/categories",
-        "categories_page": "https://store.kobobooks.com/ebooks/categories",
-        "category": "https://storeapi.kobo.com/v1/categories/{CategoryId}",
-        "category_featured_lists": "https://storeapi.kobo.com/v1/categories/{CategoryId}/featured",
-        "category_products": "https://storeapi.kobo.com/v1/categories/{CategoryId}/products",
-        "checkout_borrowed_book": "https://storeapi.kobo.com/v1/library/borrow",
-        "configuration_data": "https://storeapi.kobo.com/v1/configuration",
-        "content_access_book": "https://storeapi.kobo.com/v1/products/books/{ProductId}/access",
-        "customer_care_live_chat": "https://v2.zopim.com/widget/livechat.html?key=Y6gwUmnu4OATxN3Tli4Av9bYN319BTdO",
-        "daily_deal": "https://storeapi.kobo.com/v1/products/dailydeal",
-        "deals": "https://storeapi.kobo.com/v1/deals",
-        "delete_entitlement": "https://storeapi.kobo.com/v1/library/{Ids}",
-        "delete_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}",
-        "delete_tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/items/delete",
-        "device_auth": "https://storeapi.kobo.com/v1/auth/device",
-        "device_refresh": "https://storeapi.kobo.com/v1/auth/refresh",
-        "dictionary_host": "https://kbdownload1-a.akamaihd.net",
-        "discovery_host": "https://discovery.kobobooks.com",
-        "eula_page": "https://www.kobo.com/termsofuse?style=onestore",
-        "exchange_auth": "https://storeapi.kobo.com/v1/auth/exchange",
-        "external_book": "https://storeapi.kobo.com/v1/products/books/external/{Ids}",
-        "facebook_sso_page": "https://authorize.kobo.com/signin/provider/Facebook/login?returnUrl=http://store.kobobooks.com/",
-        "featured_list": "https://storeapi.kobo.com/v1/products/featured/{FeaturedListId}",
-        "featured_lists": "https://storeapi.kobo.com/v1/products/featured",
-        "free_books_page": {
-            "EN": "https://www.kobo.com/{region}/{language}/p/free-ebooks",
-            "FR": "https://www.kobo.com/{region}/{language}/p/livres-gratuits",
-            "IT": "https://www.kobo.com/{region}/{language}/p/libri-gratuiti",
-            "NL": "https://www.kobo.com/{region}/{language}/List/bekijk-het-overzicht-van-gratis-ebooks/QpkkVWnUw8sxmgjSlCbJRg",
-            "PT": "https://www.kobo.com/{region}/{language}/p/livros-gratis",
-        },
-        "fte_feedback": "https://storeapi.kobo.com/v1/products/ftefeedback",
-        "get_tests_request": "https://storeapi.kobo.com/v1/analytics/gettests",
-        "giftcard_epd_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem-ereader",
-        "giftcard_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem",
-        "help_page": "http://www.kobo.com/help",
-        "image_host": calibre_web_url,
-        "image_url_quality_template": calibre_web_url
-        + "/{ImageId}/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg",
-        "image_url_template": calibre_web_url
-        + "/{ImageId}/{Width}/{Height}/false/image.jpg",
-        "kobo_audiobooks_enabled": "False",
-        "kobo_audiobooks_orange_deal_enabled": "False",
-        "kobo_audiobooks_subscriptions_enabled": "False",
-        "kobo_nativeborrow_enabled": "True",
-        "kobo_onestorelibrary_enabled": "False",
-        "kobo_redeem_enabled": "True",
-        "kobo_shelfie_enabled": "False",
-        "kobo_subscriptions_enabled": "False",
-        "kobo_superpoints_enabled": "False",
-        "kobo_wishlist_enabled": "True",
-        "library_book": "https://storeapi.kobo.com/v1/user/library/books/{LibraryItemId}",
-        "library_items": "https://storeapi.kobo.com/v1/user/library",
-        "library_metadata": "https://storeapi.kobo.com/v1/library/{Ids}/metadata",
-        "library_prices": "https://storeapi.kobo.com/v1/user/library/previews/prices",
-        "library_stack": "https://storeapi.kobo.com/v1/user/library/stacks/{LibraryItemId}",
-        "library_sync": "https://storeapi.kobo.com/v1/library/sync",
-        "love_dashboard_page": "https://store.kobobooks.com/{culture}/kobosuperpoints",
-        "love_points_redemption_page": "https://store.kobobooks.com/{culture}/KoboSuperPointsRedemption?productId={ProductId}",
-        "magazine_landing_page": "https://store.kobobooks.com/emagazines",
-        "notifications_registration_issue": "https://storeapi.kobo.com/v1/notifications/registration",
-        "oauth_host": "https://oauth.kobo.com",
-        "overdrive_account": "https://auth.overdrive.com/account",
-        "overdrive_library": "https://{libraryKey}.auth.overdrive.com/library",
-        "overdrive_library_finder_host": "https://libraryfinder.api.overdrive.com",
-        "overdrive_thunder_host": "https://thunder.api.overdrive.com",
-        "password_retrieval_page": "https://www.kobobooks.com/passwordretrieval.html",
-        "post_analytics_event": "https://storeapi.kobo.com/v1/analytics/event",
-        "privacy_page": "https://www.kobo.com/privacypolicy?style=onestore",
-        "product_nextread": "https://storeapi.kobo.com/v1/products/{ProductIds}/nextread",
-        "product_prices": "https://storeapi.kobo.com/v1/products/{ProductIds}/prices",
-        "product_recommendations": "https://storeapi.kobo.com/v1/products/{ProductId}/recommendations",
-        "product_reviews": "https://storeapi.kobo.com/v1/products/{ProductIds}/reviews",
-        "products": "https://storeapi.kobo.com/v1/products",
-        "provider_external_sign_in_page": "https://authorize.kobo.com/ExternalSignIn/{providerName}?returnUrl=http://store.kobobooks.com/",
-        "purchase_buy": "https://www.kobo.com/checkout/createpurchase/",
-        "purchase_buy_templated": "https://www.kobo.com/{culture}/checkout/createpurchase/{ProductId}",
-        "quickbuy_checkout": "https://storeapi.kobo.com/v1/store/quickbuy/{PurchaseId}/checkout",
-        "quickbuy_create": "https://storeapi.kobo.com/v1/store/quickbuy/purchase",
-        "rating": "https://storeapi.kobo.com/v1/products/{ProductId}/rating/{Rating}",
-        "reading_state": "https://storeapi.kobo.com/v1/library/{Ids}/state",
-        "redeem_interstitial_page": "https://store.kobobooks.com",
-        "registration_page": "https://authorize.kobo.com/signup?returnUrl=http://store.kobobooks.com/",
-        "related_items": "https://storeapi.kobo.com/v1/products/{Id}/related",
-        "remaining_book_series": "https://storeapi.kobo.com/v1/products/books/series/{SeriesId}",
-        "rename_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}",
-        "review": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}",
-        "review_sentiment": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}/sentiment/{Sentiment}",
-        "shelfie_recommendations": "https://storeapi.kobo.com/v1/user/recommendations/shelfie",
-        "sign_in_page": "https://authorize.kobo.com/signin?returnUrl=http://store.kobobooks.com/",
-        "social_authorization_host": "https://social.kobobooks.com:8443",
-        "social_host": "https://social.kobobooks.com",
-        "stacks_host_productId": "https://store.kobobooks.com/collections/byproductid/",
-        "store_home": "www.kobo.com/{region}/{language}",
-        "store_host": "store.kobobooks.com",
-        "store_newreleases": "https://store.kobobooks.com/{culture}/List/new-releases/961XUjtsU0qxkFItWOutGA",
-        "store_search": "https://store.kobobooks.com/{culture}/Search?Query={query}",
-        "store_top50": "https://store.kobobooks.com/{culture}/ebooks/Top",
-        "tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/Items",
-        "tags": "https://storeapi.kobo.com/v1/library/tags",
-        "taste_profile": "https://storeapi.kobo.com/v1/products/tasteprofile",
-        "update_accessibility_to_preview": "https://storeapi.kobo.com/v1/library/{EntitlementIds}/preview",
-        "use_one_store": "False",
-        "user_loyalty_benefits": "https://storeapi.kobo.com/v1/user/loyalty/benefits",
-        "user_platform": "https://storeapi.kobo.com/v1/user/platform",
-        "user_profile": "https://storeapi.kobo.com/v1/user/profile",
-        "user_ratings": "https://storeapi.kobo.com/v1/user/ratings",
-        "user_recommendations": "https://storeapi.kobo.com/v1/user/recommendations",
-        "user_reviews": "https://storeapi.kobo.com/v1/user/reviews",
-        "user_wishlist": "https://storeapi.kobo.com/v1/user/wishlist",
-        "userguide_host": "https://kbdownload1-a.akamaihd.net",
-        "wishlist_page": "https://store.kobobooks.com/{region}/{language}/account/wishlist",
-    }