Inital Kobo

cps.py

@@ -41,6 +41,8 @@ from cps.shelf import shelf
 from cps.admin import admi
 from cps.gdrive import gdrive
 from cps.editbooks import editbook
+from cps.kobo import kobo
+
 try:
     from cps.oauth_bb import oauth
     oauth_available = True
@@ -58,6 +60,7 @@ def main():
     app.register_blueprint(admi)
     app.register_blueprint(gdrive)
     app.register_blueprint(editbook)
+    app.register_blueprint(kobo)
     if oauth_available:
         app.register_blueprint(oauth)
     success = web_server.start()

cps/admin.py

@@ -300,6 +300,8 @@ def _configuration_update_helper():
     if config.config_certfile and not os.path.isfile(config.config_certfile):
         return _configuration_result('Certfile location is not valid, please enter correct path', gdriveError)
         
+    _config_string("config_server_url")
+
     _config_checkbox_int("config_uploading")
     _config_checkbox_int("config_anonbrowse")
     _config_checkbox_int("config_public_reg")
@@ -597,6 +599,18 @@ def edit_user(user_id):
                 content.default_language = to_save["default_language"]
             if "locale" in to_save and to_save["locale"]:
                 content.locale = to_save["locale"]
+
+            if "kobo_user_key" in to_save and to_save["kobo_user_key"]:
+                kobo_user_key_hash = generate_password_hash(to_save["kobo_user_key"])
+                if kobo_user_key_hash != content.kobo_user_key_hash:
+                    existing_kobo_user_key = ub.session.query(ub.User).filter(ub.User.kobo_user_key_hash == kobo_user_key_hash).first()
+                    if not existing_kobo_user_key:
+                        content.kobo_user_key_hash = kobo_user_key_hash
+                    else:
+                        flash(_(u"Found an existing account for this Kobo UserKey."), category="error")
+                        return render_title_template("user_edit.html", translations=translations, languages=languages,
+                                                     new_user=0, content=content, downloads=downloads, registered_oauth=oauth_check,
+                                                     title=_(u"Edit User %(nick)s", nick=content.nickname), page="edituser")
             if to_save["email"] and to_save["email"] != content.email:
                 existing_email = ub.session.query(ub.User).filter(ub.User.email == to_save["email"].lower()) \
                     .first()

cps/config_sql.py

@@ -49,6 +49,7 @@ class _Settings(_Base):
     config_port = Column(Integer, default=constants.DEFAULT_PORT)
     config_certfile = Column(String)
     config_keyfile = Column(String)
+    config_server_url = Column(String, default='')
 
     config_calibre_web_title = Column(String, default=u'Calibre-Web')
     config_books_per_page = Column(Integer, default=60)

cps/db.py

@@ -25,7 +25,7 @@ import ast
 
 from sqlalchemy import create_engine
 from sqlalchemy import Table, Column, ForeignKey
-from sqlalchemy import String, Integer, Boolean
+from sqlalchemy import String, Integer, Boolean, TIMESTAMP
 from sqlalchemy.orm import relationship, sessionmaker, scoped_session
 from sqlalchemy.ext.declarative import declarative_base
 
@@ -251,10 +251,10 @@ class Books(Base):
     title = Column(String)
     sort = Column(String)
     author_sort = Column(String)
-    timestamp = Column(String)
+    timestamp = Column(TIMESTAMP)
     pubdate = Column(String)
     series_index = Column(String)
-    last_modified = Column(String)
+    last_modified = Column(TIMESTAMP)
     path = Column(String)
     has_cover = Column(Integer)
     uuid = Column(String)

cps/helper.py

@@ -446,32 +446,46 @@ def delete_book(book, calibrepath, book_format):
         return delete_book_file(book, calibrepath, book_format)
 
 
+def get_cover_on_failure(use_generic_cover):
+    if use_generic_cover:
+        return send_from_directory(_STATIC_DIR, "generic_cover.jpg")
+    else:
+        return None
+
 def get_book_cover(book_id):
     book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
-    if book.has_cover:
+    return get_book_cover_internal(book, use_generic_cover_on_failure=True)
 
+def get_book_cover_with_uuid(book_uuid,
+                   use_generic_cover_on_failure=True):
+    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    return get_book_cover_internal(book, use_generic_cover_on_failure)
+
+def get_book_cover_internal(book,
+                   use_generic_cover_on_failure):
+    if book and book.has_cover:
         if config.config_use_google_drive:
             try:
                 if not gd.is_gdrive_ready():
-                    return send_from_directory(_STATIC_DIR, "generic_cover.jpg")
+                    return get_cover_on_failure(use_generic_cover_on_failure)
                 path=gd.get_cover_via_gdrive(book.path)
                 if path:
                     return redirect(path)
                 else:
                     log.error('%s/cover.jpg not found on Google Drive', book.path)
-                    return send_from_directory(_STATIC_DIR, "generic_cover.jpg")
+                    return get_cover_on_failure(use_generic_cover_on_failure)
             except Exception as e:
                 log.exception(e)
                 # traceback.print_exc()
-                return send_from_directory(_STATIC_DIR,"generic_cover.jpg")
+                return get_cover_on_failure(use_generic_cover_on_failure)
         else:
             cover_file_path = os.path.join(config.config_calibre_dir, book.path)
             if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")):
                 return send_from_directory(cover_file_path, "cover.jpg")
             else:
-                return send_from_directory(_STATIC_DIR,"generic_cover.jpg")
+                return get_cover_on_failure(use_generic_cover_on_failure)
     else:
-        return send_from_directory(_STATIC_DIR,"generic_cover.jpg")
+        return get_cover_on_failure(use_generic_cover_on_failure)
 
 
 # saves book cover from url

cps/kobo.py

@@ -0,0 +1,523 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2018-2019 shavitmichael, OzzieIsaacs
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import uuid
+from base64 import b64decode, b64encode
+from datetime import datetime
+from time import gmtime, strftime
+
+from jsonschema import validate, exceptions
+from flask import Blueprint, request, make_response, jsonify, json
+from flask_login import login_required
+from sqlalchemy import func
+
+from . import config, logger, kobo_auth, db, helper
+from .web import download_required
+
+kobo = Blueprint("kobo", __name__)
+kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo)
+
+log = logger.create()
+
+
+def b64encode_json(json_data):
+    if sys.version_info < (3, 0):
+        return b64encode(json.dumps(json_data))
+    else:
+        return b64encode(json.dumps(json_data).encode())
+
+
+# Python3 has a timestamp() method we could be calling, however it's not avaiable in python2.
+def to_epoch_timestamp(datetime_object):
+    return (datetime_object - datetime(1970, 1, 1)).total_seconds()
+
+
+class SyncToken:
+    """ The SyncToken is used to persist state accross requests.
+    When serialized over the response headers, the Kobo device will propagate the token onto following requests to the service.
+    As an example use-case, the SyncToken is used to detect books that have been added to the library since the last time the device synced to the server.
+
+    Attributes:
+        books_last_created: Datetime representing the newest book that the device knows about.
+        books_last_modified: Datetime representing the last modified book that the device knows about.
+    """
+
+    SYNC_TOKEN_HEADER = "x-kobo-synctoken"
+    VERSION = "1-0-0"
+    MIN_VERSION = "1-0-0"
+
+    token_schema = {
+        "type": "object",
+        "properties": {"version": {"type": "string"}, "data": {"type": "object"},},
+    }
+    # This Schema doesn't contain enough information to detect and propagate book deletions from Calibre to the device.
+    # A potential solution might be to keep a list of all known book uuids in the token, and look for any missing from the db.
+    data_schema_v1 = {
+        "type": "object",
+        "properties": {
+            "raw_kobo_store_token": {"type": "string"},
+            "books_last_modified": {"type": "string"},
+            "books_last_created": {"type": "string"},
+        },
+    }
+
+    def __init__(
+        self,
+        raw_kobo_store_token="",
+        books_last_created=datetime.min,
+        books_last_modified=datetime.min,
+    ):
+        self.raw_kobo_store_token = raw_kobo_store_token
+        self.books_last_created = books_last_created
+        self.books_last_modified = books_last_modified
+
+    @staticmethod
+    def from_headers(headers):
+        sync_token_header = headers.get(SyncToken.SYNC_TOKEN_HEADER, "")
+        if sync_token_header == "":
+            return SyncToken()
+
+        # On the first sync from a Kobo device, we may receive the SyncToken
+        # from the official Kobo store. Without digging too deep into it, that
+        # token is of the form [b64encoded blob].[b64encoded blob 2]
+        if "." in sync_token_header:
+            return SyncToken(raw_kobo_store_token=sync_token_header)
+
+        try:
+            sync_token_json = json.loads(
+                b64decode(sync_token_header + "=" * (-len(sync_token_header) % 4))
+            )
+            validate(sync_token_json, SyncToken.token_schema)
+            if sync_token_json["version"] < SyncToken.MIN_VERSION:
+                raise ValueError
+
+            data_json = sync_token_json["data"]
+            validate(sync_token_json, SyncToken.data_schema_v1)
+        except (exceptions.ValidationError, ValueError) as e:
+            log.error("Sync token contents do not follow the expected json schema.")
+            return SyncToken()
+
+        raw_kobo_store_token = data_json["raw_kobo_store_token"]
+        try:
+            books_last_modified = datetime.utcfromtimestamp(
+                data_json["books_last_modified"]
+            )
+            books_last_created = datetime.utcfromtimestamp(
+                data_json["books_last_created"]
+            )
+        except TypeError:
+            log.error("SyncToken timestamps don't parse to a datetime.")
+            return SyncToken(raw_kobo_store_token=raw_kobo_store_token)
+
+        return SyncToken(
+            raw_kobo_store_token=raw_kobo_store_token,
+            books_last_created=books_last_created,
+            books_last_modified=books_last_modified,
+        )
+
+    def to_headers(self, headers):
+        headers[SyncToken.SYNC_TOKEN_HEADER] = self.build_sync_token()
+
+    def build_sync_token(self):
+        token = {
+            "version": SyncToken.VERSION,
+            "data": {
+                "raw_kobo_store_token": self.raw_kobo_store_token,
+                "books_last_modified": to_epoch_timestamp(self.books_last_modified),
+                "books_last_created": to_epoch_timestamp(self.books_last_created),
+            },
+        }
+        return b64encode_json(token)
+
+
+@kobo.route("/v1/library/sync")
+@login_required
+@download_required
+def HandleSyncRequest():
+    sync_token = SyncToken.from_headers(request.headers)
+    log.info("Kobo library sync request received.")
+
+    # TODO: Limit the number of books return per sync call, and rely on the sync-continuatation header
+    # instead so that the device triggers another sync.
+
+    new_books_last_modified = sync_token.books_last_modified
+    new_books_last_created = sync_token.books_last_created
+    entitlements = []
+
+    # sqlite gives unexpected results when performing the last_modified comparison without the datetime cast.
+    # It looks like it's treating the db.Books.last_modified field as a string and may fail
+    # the comparison because of the +00:00 suffix.
+    changed_entries = (
+        db.session.query(db.Books)
+        .filter(func.datetime(db.Books.last_modified) != sync_token.books_last_modified)
+        .all()
+    )
+    for book in changed_entries:
+        entitlement = {
+            "BookEntitlement": create_book_entitlement(book),
+            "BookMetadata": get_metadata(book),
+            "ReadingState": reading_state(book),
+        }
+        if book.timestamp > sync_token.books_last_created:
+            entitlements.append({"NewEntitlement": entitlement})
+        else:
+            entitlements.append({"ChangedEntitlement": entitlement})
+
+        new_books_last_modified = max(
+            book.last_modified, sync_token.books_last_modified
+        )
+        new_books_last_created = max(book.timestamp, sync_token.books_last_modified)
+
+    sync_token.books_last_created = new_books_last_created
+    sync_token.books_last_modified = new_books_last_modified
+
+    # Missing feature: Detect server-side book deletions.
+
+    # Missing feature: Join the response with results from the official Kobo store so that users can still buy and access books from the device store (particularly while on-the-road).
+
+    response = make_response(jsonify(entitlements))
+
+    sync_token.to_headers(response.headers)
+    response.headers["x-kobo-sync-mode"] = "delta"
+    response.headers["x-kobo-apitoken"] = "e30="
+    return response
+
+
+@kobo.route("/v1/library/<book_uuid>/metadata")
+@login_required
+@download_required
+def HandleMetadataRequest(book_uuid):
+    log.info("Kobo library metadata request received for book %s" % book_uuid)
+    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    if not book or not book.data:
+        log.info(u"Book %s not found in database", book_uuid)
+        return make_response("Book not found in database.", 404)
+
+    metadata = get_metadata(book)
+    return jsonify([metadata])
+
+
+def get_download_url_for_book(book, book_format):
+    return "{url_base}/download/{book_id}/{book_format}".format(
+        url_base=request.environ['werkzeug.request'].base_url,
+        book_id=book.id,
+        book_format=book_format.lower(),
+    )
+
+
+def create_book_entitlement(book):
+    book_uuid = book.uuid
+    return {
+        "Accessibility": "Full",
+        "ActivePeriod": {"From": current_time(),},
+        "Created": book.timestamp,
+        "CrossRevisionId": book_uuid,
+        "Id": book_uuid,
+        "IsHiddenFromArchive": False,
+        "IsLocked": False,
+        # Setting this to true removes from the device.
+        "IsRemoved": False,
+        "LastModified": book.last_modified,
+        "OriginCategory": "Imported",
+        "RevisionId": book_uuid,
+        "Status": "Active",
+    }
+
+
+def current_time():
+    return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
+
+
+def get_description(book):
+    if not book.comments:
+        return None
+    return book.comments[0].text
+
+
+# TODO handle multiple authors
+def get_author(book):
+    if not book.authors:
+        return None
+    return book.authors[0].name
+
+
+def get_publisher(book):
+    if not book.publishers:
+        return None
+    return book.publishers[0].name
+
+
+def get_series(book):
+    if not book.series:
+        return None
+    return book.series[0].name
+
+
+def get_metadata(book):
+    ALLOWED_FORMATS = {"KEPUB"}
+    download_urls = []
+
+    for book_data in book.data:
+        if book_data.format in ALLOWED_FORMATS:
+            download_urls.append(
+                {
+                    "Format": book_data.format,
+                    "Size": book_data.uncompressed_size,
+                    "Url": get_download_url_for_book(book, book_data.format),
+                    # "DrmType": "None", # Not required
+                    "Platform": "Android",  # Required field.
+                }
+            )
+
+    book_uuid = book.uuid
+    metadata = {
+        "Categories": ["00000000-0000-0000-0000-000000000001",],
+        "Contributors": get_author(book),
+        "CoverImageId": book_uuid,
+        "CrossRevisionId": book_uuid,
+        "CurrentDisplayPrice": {"CurrencyCode": "USD", "TotalAmount": 0},
+        "CurrentLoveDisplayPrice": {"TotalAmount": 0},
+        "Description": get_description(book),
+        "DownloadUrls": download_urls,
+        "EntitlementId": book_uuid,
+        "ExternalIds": [],
+        "Genre": "00000000-0000-0000-0000-000000000001",
+        "IsEligibleForKoboLove": False,
+        "IsInternetArchive": False,
+        "IsPreOrder": False,
+        "IsSocialEnabled": True,
+        "Language": "en",
+        "PhoneticPronunciations": {},
+        "PublicationDate": "2019-02-03T00:25:03.0000000Z",  # current_time(),
+        "Publisher": {"Imprint": "", "Name": get_publisher(book),},
+        "RevisionId": book_uuid,
+        "Title": book.title,
+        "WorkId": book_uuid,
+    }
+
+    if get_series(book):
+        if sys.version_info < (3, 0):
+            name = get_series(book).encode("utf-8")
+        else:
+            name = get_series(book)
+        metadata["Series"] = {
+            "Name": get_series(book),
+            "Number": book.series_index,
+            "NumberFloat": float(book.series_index),
+            # Get a deterministic id based on the series name.
+            "Id": uuid.uuid3(uuid.NAMESPACE_DNS, name),
+        }
+
+    return metadata
+
+
+def reading_state(book):
+    # TODO: Implement
+    reading_state = {
+        # "StatusInfo": {
+        #     "LastModified": get_single_cc_value(book, "lastreadtimestamp"),
+        #     "Status": get_single_cc_value(book, "reading_status"),
+        # }
+        # TODO: CurrentBookmark, Location
+    }
+    return reading_state
+
+
+@kobo.route(
+    "/<book_uuid>/<horizontal>/<vertical>/<jpeg_quality>/<monochrome>/image.jpg"
+)
+def HandleCoverImageRequest(book_uuid, horizontal, vertical, jpeg_quality, monochrome):
+    book_cover = helper.get_book_cover_with_uuid(
+        book_uuid, use_generic_cover_on_failure=False
+    )
+    if not book_cover:
+        return make_response()
+    return book_cover
+
+
+@kobo.route("/v1/user/profile")
+@kobo.route("/v1/user/loyalty/benefits")
+@kobo.route("/v1/analytics/gettests", methods=["GET", "POST"])
+@kobo.route("/v1/user/wishlist")
+@kobo.route("/v1/user/<dummy>")
+@kobo.route("/v1/user/recommendations")
+@kobo.route("/v1/products/<dummy>")
+@kobo.route("/v1/products/<dummy>/nextread")
+@kobo.route("/v1/products/featured/<dummy>")
+@kobo.route("/v1/products/featured/")
+@kobo.route("/v1/library/<dummy>", methods=["DELETE", "GET"])  # TODO: implement
+def HandleDummyRequest(dummy=None):
+    return make_response(jsonify({}))
+
+
+@kobo.route("/v1/auth/device", methods=["POST"])
+def HandleAuthRequest():
+    # This AuthRequest isn't used for most of our usecases.
+    response = make_response(
+        jsonify(
+            {
+                "AccessToken": "abcde",
+                "RefreshToken": "abcde",
+                "TokenType": "Bearer",
+                "TrackingId": "abcde",
+                "UserKey": "abcdefgeh",
+            }
+        )
+    )
+    return response
+
+
+@kobo.route("/v1/initialization")
+def HandleInitRequest():
+    resources = NATIVE_KOBO_RESOURCES(calibre_web_url=config.config_server_url)
+    response = make_response(jsonify({"Resources": resources}))
+    response.headers["x-kobo-apitoken"] = "e30="
+    return response
+
+
+def NATIVE_KOBO_RESOURCES(calibre_web_url):
+    return {
+        "account_page": "https://secure.kobobooks.com/profile",
+        "account_page_rakuten": "https://my.rakuten.co.jp/",
+        "add_entitlement": "https://storeapi.kobo.com/v1/library/{RevisionIds}",
+        "affiliaterequest": "https://storeapi.kobo.com/v1/affiliate",
+        "audiobook_subscription_orange_deal_inclusion_url": "https://authorize.kobo.com/inclusion",
+        "authorproduct_recommendations": "https://storeapi.kobo.com/v1/products/books/authors/recommendations",
+        "autocomplete": "https://storeapi.kobo.com/v1/products/autocomplete",
+        "blackstone_header": {"key": "x-amz-request-payer", "value": "requester"},
+        "book": "https://storeapi.kobo.com/v1/products/books/{ProductId}",
+        "book_detail_page": "https://store.kobobooks.com/{culture}/ebook/{slug}",
+        "book_detail_page_rakuten": "http://books.rakuten.co.jp/rk/{crossrevisionid}",
+        "book_landing_page": "https://store.kobobooks.com/ebooks",
+        "book_subscription": "https://storeapi.kobo.com/v1/products/books/subscriptions",
+        "categories": "https://storeapi.kobo.com/v1/categories",
+        "categories_page": "https://store.kobobooks.com/ebooks/categories",
+        "category": "https://storeapi.kobo.com/v1/categories/{CategoryId}",
+        "category_featured_lists": "https://storeapi.kobo.com/v1/categories/{CategoryId}/featured",
+        "category_products": "https://storeapi.kobo.com/v1/categories/{CategoryId}/products",
+        "checkout_borrowed_book": "https://storeapi.kobo.com/v1/library/borrow",
+        "configuration_data": "https://storeapi.kobo.com/v1/configuration",
+        "content_access_book": "https://storeapi.kobo.com/v1/products/books/{ProductId}/access",
+        "customer_care_live_chat": "https://v2.zopim.com/widget/livechat.html?key=Y6gwUmnu4OATxN3Tli4Av9bYN319BTdO",
+        "daily_deal": "https://storeapi.kobo.com/v1/products/dailydeal",
+        "deals": "https://storeapi.kobo.com/v1/deals",
+        "delete_entitlement": "https://storeapi.kobo.com/v1/library/{Ids}",
+        "delete_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}",
+        "delete_tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/items/delete",
+        "device_auth": "https://storeapi.kobo.com/v1/auth/device",
+        "device_refresh": "https://storeapi.kobo.com/v1/auth/refresh",
+        "dictionary_host": "https://kbdownload1-a.akamaihd.net",
+        "discovery_host": "https://discovery.kobobooks.com",
+        "eula_page": "https://www.kobo.com/termsofuse?style=onestore",
+        "exchange_auth": "https://storeapi.kobo.com/v1/auth/exchange",
+        "external_book": "https://storeapi.kobo.com/v1/products/books/external/{Ids}",
+        "facebook_sso_page": "https://authorize.kobo.com/signin/provider/Facebook/login?returnUrl=http://store.kobobooks.com/",
+        "featured_list": "https://storeapi.kobo.com/v1/products/featured/{FeaturedListId}",
+        "featured_lists": "https://storeapi.kobo.com/v1/products/featured",
+        "free_books_page": {
+            "EN": "https://www.kobo.com/{region}/{language}/p/free-ebooks",
+            "FR": "https://www.kobo.com/{region}/{language}/p/livres-gratuits",
+            "IT": "https://www.kobo.com/{region}/{language}/p/libri-gratuiti",
+            "NL": "https://www.kobo.com/{region}/{language}/List/bekijk-het-overzicht-van-gratis-ebooks/QpkkVWnUw8sxmgjSlCbJRg",
+            "PT": "https://www.kobo.com/{region}/{language}/p/livros-gratis",
+        },
+        "fte_feedback": "https://storeapi.kobo.com/v1/products/ftefeedback",
+        "get_tests_request": "https://storeapi.kobo.com/v1/analytics/gettests",
+        "giftcard_epd_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem-ereader",
+        "giftcard_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem",
+        "help_page": "http://www.kobo.com/help",
+        "image_host": calibre_web_url,
+        "image_url_quality_template": calibre_web_url
+        + "/{ImageId}/{Width}/{Height}/{Quality}/{IsGreyscale}/image.jpg",
+        "image_url_template": calibre_web_url
+        + "/{ImageId}/{Width}/{Height}/false/image.jpg",
+        "kobo_audiobooks_enabled": "False",
+        "kobo_audiobooks_orange_deal_enabled": "False",
+        "kobo_audiobooks_subscriptions_enabled": "False",
+        "kobo_nativeborrow_enabled": "True",
+        "kobo_onestorelibrary_enabled": "False",
+        "kobo_redeem_enabled": "True",
+        "kobo_shelfie_enabled": "False",
+        "kobo_subscriptions_enabled": "False",
+        "kobo_superpoints_enabled": "False",
+        "kobo_wishlist_enabled": "True",
+        "library_book": "https://storeapi.kobo.com/v1/user/library/books/{LibraryItemId}",
+        "library_items": "https://storeapi.kobo.com/v1/user/library",
+        "library_metadata": "https://storeapi.kobo.com/v1/library/{Ids}/metadata",
+        "library_prices": "https://storeapi.kobo.com/v1/user/library/previews/prices",
+        "library_stack": "https://storeapi.kobo.com/v1/user/library/stacks/{LibraryItemId}",
+        "library_sync": "https://storeapi.kobo.com/v1/library/sync",
+        "love_dashboard_page": "https://store.kobobooks.com/{culture}/kobosuperpoints",
+        "love_points_redemption_page": "https://store.kobobooks.com/{culture}/KoboSuperPointsRedemption?productId={ProductId}",
+        "magazine_landing_page": "https://store.kobobooks.com/emagazines",
+        "notifications_registration_issue": "https://storeapi.kobo.com/v1/notifications/registration",
+        "oauth_host": "https://oauth.kobo.com",
+        "overdrive_account": "https://auth.overdrive.com/account",
+        "overdrive_library": "https://{libraryKey}.auth.overdrive.com/library",
+        "overdrive_library_finder_host": "https://libraryfinder.api.overdrive.com",
+        "overdrive_thunder_host": "https://thunder.api.overdrive.com",
+        "password_retrieval_page": "https://www.kobobooks.com/passwordretrieval.html",
+        "post_analytics_event": "https://storeapi.kobo.com/v1/analytics/event",
+        "privacy_page": "https://www.kobo.com/privacypolicy?style=onestore",
+        "product_nextread": "https://storeapi.kobo.com/v1/products/{ProductIds}/nextread",
+        "product_prices": "https://storeapi.kobo.com/v1/products/{ProductIds}/prices",
+        "product_recommendations": "https://storeapi.kobo.com/v1/products/{ProductId}/recommendations",
+        "product_reviews": "https://storeapi.kobo.com/v1/products/{ProductIds}/reviews",
+        "products": "https://storeapi.kobo.com/v1/products",
+        "provider_external_sign_in_page": "https://authorize.kobo.com/ExternalSignIn/{providerName}?returnUrl=http://store.kobobooks.com/",
+        "purchase_buy": "https://www.kobo.com/checkout/createpurchase/",
+        "purchase_buy_templated": "https://www.kobo.com/{culture}/checkout/createpurchase/{ProductId}",
+        "quickbuy_checkout": "https://storeapi.kobo.com/v1/store/quickbuy/{PurchaseId}/checkout",
+        "quickbuy_create": "https://storeapi.kobo.com/v1/store/quickbuy/purchase",
+        "rating": "https://storeapi.kobo.com/v1/products/{ProductId}/rating/{Rating}",
+        "reading_state": "https://storeapi.kobo.com/v1/library/{Ids}/state",
+        "redeem_interstitial_page": "https://store.kobobooks.com",
+        "registration_page": "https://authorize.kobo.com/signup?returnUrl=http://store.kobobooks.com/",
+        "related_items": "https://storeapi.kobo.com/v1/products/{Id}/related",
+        "remaining_book_series": "https://storeapi.kobo.com/v1/products/books/series/{SeriesId}",
+        "rename_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}",
+        "review": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}",
+        "review_sentiment": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}/sentiment/{Sentiment}",
+        "shelfie_recommendations": "https://storeapi.kobo.com/v1/user/recommendations/shelfie",
+        "sign_in_page": "https://authorize.kobo.com/signin?returnUrl=http://store.kobobooks.com/",
+        "social_authorization_host": "https://social.kobobooks.com:8443",
+        "social_host": "https://social.kobobooks.com",
+        "stacks_host_productId": "https://store.kobobooks.com/collections/byproductid/",
+        "store_home": "www.kobo.com/{region}/{language}",
+        "store_host": "store.kobobooks.com",
+        "store_newreleases": "https://store.kobobooks.com/{culture}/List/new-releases/961XUjtsU0qxkFItWOutGA",
+        "store_search": "https://store.kobobooks.com/{culture}/Search?Query={query}",
+        "store_top50": "https://store.kobobooks.com/{culture}/ebooks/Top",
+        "tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/Items",
+        "tags": "https://storeapi.kobo.com/v1/library/tags",
+        "taste_profile": "https://storeapi.kobo.com/v1/products/tasteprofile",
+        "update_accessibility_to_preview": "https://storeapi.kobo.com/v1/library/{EntitlementIds}/preview",
+        "use_one_store": "False",
+        "user_loyalty_benefits": "https://storeapi.kobo.com/v1/user/loyalty/benefits",
+        "user_platform": "https://storeapi.kobo.com/v1/user/platform",
+        "user_profile": "https://storeapi.kobo.com/v1/user/profile",
+        "user_ratings": "https://storeapi.kobo.com/v1/user/ratings",
+        "user_recommendations": "https://storeapi.kobo.com/v1/user/recommendations",
+        "user_reviews": "https://storeapi.kobo.com/v1/user/reviews",
+        "user_wishlist": "https://storeapi.kobo.com/v1/user/wishlist",
+        "userguide_host": "https://kbdownload1-a.akamaihd.net",
+        "wishlist_page": "https://store.kobobooks.com/{region}/{language}/account/wishlist",
+    }

cps/kobo_auth.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
+#    Copyright (C) 2018-2019 shavitmichael, OzzieIsaacs
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""This module is used to control authentication/authorization of Kobo sync requests.
+This module also includes research notes into the auth protocol used by Kobo devices.
+
+Log-in:
+When first booting a Kobo device the user must sign into a Kobo (or affiliate) account.
+Upon successful sign-in, the user is redirected to 
+    https://auth.kobobooks.com/CrossDomainSignIn?id=<some id>
+which serves the following response:
+    <script type='text/javascript'>location.href='kobo://UserAuthenticated?userId=<redacted>&userKey<redacted>&email=<redacted>&returnUrl=https%3a%2f%2fwww.kobo.com';</script>.
+And triggers the insertion of a userKey into the device's User table.
+
+IMPORTANT SECURITY CAUTION:
+Together, the device's DeviceId and UserKey act as an *irrevocable* authentication 
+token to most (if not all) Kobo APIs. In fact, in most cases only the UserKey is
+required to authorize the API call.
+
+Changing Kobo password *does not* invalidate user keys! This is apparently a known
+issue for a few years now https://www.mobileread.com/forums/showpost.php?p=3476851&postcount=13
+(although this poster hypothesised that Kobo could blacklist a DeviceId, many endpoints
+will still grant access given the userkey.)
+
+Official Kobo Store Api authorization:
+* For most of the endpoints we care about (sync, metadata, tags, etc), the userKey is
+passed in the x-kobo-userkey header, and is sufficient to authorize the API call.
+* Some endpoints (e.g: AnnotationService) instead make use of Bearer tokens pass through
+an authorization header. To get a BearerToken, the device makes a POST request to the
+v1/auth/device endpoint with the secret UserKey and the device's DeviceId.
+* The book download endpoint passes an auth token as a URL param instead of a header.
+
+Our implementation:
+For now, we rely on the official Kobo store's UserKey for authentication.
+Once authenticated, we set the login cookie on the response that will be sent back for
+the duration of the session to authorize subsequent API calls.
+Ideally we'd only perform UserKey-based authentication for the v1/initialization or the
+v1/device/auth call, however sessions don't always start with those calls.
+
+Because of the irrevocable power granted by the key, we only ever store and compare a
+hash of the key. To obtain their UserKey, a user can query the user table from the
+.kobo/KoboReader.sqlite database found on their device.
+This isn't exactly user friendly however.
+
+Some possible alternatives that require more research:
+ * Instead of having users query the device database to find out their UserKey, we could
+ provide a list of recent Kobo sync attempts in the calibre-web UI for users to
+ authenticate sync attempts (e.g: 'this was me' button).
+ * We may be able to craft a sign-in flow with a redirect back to the CalibreWeb
+ server containing the KoboStore's UserKey (if the same as the devices?). 
+ * Can we create our own UserKey instead of relying on the real store's userkey?
+  (Maybe using something like location.href=kobo://UserAuthenticated?userId=...?)
+"""
+
+from functools import wraps
+from flask import request, make_response
+from flask_login import login_user
+from werkzeug.security import check_password_hash
+
+from . import logger, ub, lm
+
+USER_KEY_HEADER = "x-kobo-userkey"
+USER_KEY_URL_PARAM = "kobo_userkey"
+
+log = logger.create()
+
+
+def disable_failed_auth_redirect_for_blueprint(bp):
+    lm.blueprint_login_views[bp.name] = None
+
+
+@lm.request_loader
+def load_user_from_kobo_request(request):
+    user_key = request.headers.get(USER_KEY_HEADER)
+    if user_key:
+        for user in (
+            ub.session.query(ub.User).filter(ub.User.kobo_user_key_hash != "").all()
+        ):
+            if check_password_hash(str(user.kobo_user_key_hash), user_key):
+                # The Kobo device won't preserve the cookie accross sessions, even if we
+                # were to set remember_me=true.
+                login_user(user)
+                return user
+    log.info("Received Kobo request without a recognizable UserKey.")
+    return None

cps/templates/config_edit.html

@@ -104,6 +104,10 @@
                     <!--option-- value="3" {% if config.config_updatechannel == 3 %}selected{% endif %}>{{_('Nightly (Automatic)')}}</option-->
             </select>
         </div>
+        <div class="form-group">
+          <label for="config_server_url">{{_('Server Url. This is only used for the (experimental) Kobo device library sync')}}</label>
+          <input type="text" class="form-control" name="config_server_url" id="config_server_url" value="{% if config.config_server_url != None %}{{ config.config_server_url }}{% endif %}" autocomplete="off">
+        </div>
       </div>
     </div>
   </div>

cps/templates/user_edit.html

@@ -28,6 +28,10 @@
       <input type="email" class="form-control" name="kindle_mail" id="kindle_mail" value="{{ content.kindle_mail if content.kindle_mail != None }}">
     </div>
     <div class="form-group">
+    <div class="form-group">
+        <label for="kobo_user_key">{{_('KoboStore UserKey')}}</label>
+        <input type="password" class="form-control" name="kobo_user_key" id="kobo_user_key" value="" autocomplete="off">
+    </div>
     <label for="locale">{{_('Language')}}</label>
         <select name="locale" id="locale" class="form-control">
             {%  for translation in translations %}

cps/ub.py

@@ -173,6 +173,7 @@ class User(UserBase, Base):
     role = Column(SmallInteger, default=constants.ROLE_USER)
     password = Column(String)
     kindle_mail = Column(String(120), default="")
+    kobo_user_key_hash = Column(String, unique=True, default="")
     shelf = relationship('Shelf', backref='user', lazy='dynamic', order_by='Shelf.name')
     downloads = relationship('Downloads', backref='user', lazy='dynamic')
     locale = Column(String(2), default="en")
@@ -375,7 +376,12 @@ def migrate_Database(session):
     except exc.OperationalError:
         conn = engine.connect()
         conn.execute("ALTER TABLE user ADD column `mature_content` INTEGER DEFAULT 1")
-
+    try:
+        session.query(exists().where(User.kobo_user_key_hash)).scalar()
+    except exc.OperationalError:  # Database is not compatible, some rows are missing
+        conn = engine.connect()
+        conn.execute("ALTER TABLE user ADD column `kobo_user_key_hash` VARCHAR")
+        session.commit()
     if session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS).first() is None:
         create_anonymous_user(session)
     try:
@@ -391,6 +397,7 @@ def migrate_Database(session):
                             "role SMALLINT,"
                             "password VARCHAR,"
                             "kindle_mail VARCHAR(120),"
+                            "kobo_user_key_hash VARCHAR,"
                             "locale VARCHAR(2),"
                             "sidebar_view INTEGER,"
                             "default_language VARCHAR(3),"
@@ -398,9 +405,9 @@ def migrate_Database(session):
                             "UNIQUE (nickname),"
                             "UNIQUE (email),"
                             "CHECK (mature_content IN (0, 1)))")
-        conn.execute("INSERT INTO user_id(id, nickname, email, role, password, kindle_mail,locale,"
+        conn.execute("INSERT INTO user_id(id, nickname, email, role, password, kindle_mail,kobo_user_key_hash, locale,"
                         "sidebar_view, default_language, mature_content) "
-                     "SELECT id, nickname, email, role, password, kindle_mail, locale,"
+                     "SELECT id, nickname, email, role, password, kindle_mail, kobo_user_key_hash, locale,"
                         "sidebar_view, default_language, mature_content FROM user")
         # delete old user table and rename new user_id table to user:
         conn.execute("DROP TABLE user")

cps/web.py

@@ -1288,6 +1288,8 @@ def profile():
                 current_user.password = generate_password_hash(to_save["password"])
         if "kindle_mail" in to_save and to_save["kindle_mail"] != current_user.kindle_mail:
             current_user.kindle_mail = to_save["kindle_mail"]
+        if "kobo_user_key" in to_save and to_save["kobo_user_key"]:
+                current_user.kobo_user_key_hash = generate_password_hash(to_save["kobo_user_key"])
         if to_save["email"] and to_save["email"] != current_user.email:
             if config.config_public_reg and not check_valid_domain(to_save["email"]):
                 flash(_(u"E-mail is not from valid domain"), category="error")
@@ -1331,7 +1333,7 @@ def profile():
             ub.session.commit()
         except IntegrityError:
             ub.session.rollback()
-            flash(_(u"Found an existing account for this e-mail address."), category="error")
+            flash(_(u"Found an existing account for this e-mail address or Kobo UserKey."), category="error")
             return render_title_template("user_edit.html", content=current_user, downloads=downloads,
                                          translations=translations,
                                          title=_(u"%(name)s's profile", name=current_user.nickname), page="me",

requirements.txt

@@ -13,3 +13,4 @@ SQLAlchemy>=1.1.0
 tornado>=4.1
 Wand>=0.4.4
 unidecode>=0.04.19
+jsonschema>=3.2.0