Add support for syncing Kobo reading state.

cps/kobo.py

@@ -17,11 +17,11 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.
 
+import datetime
 import sys
 import base64
 import os
 import uuid
-from datetime import datetime
 from time import gmtime, strftime
 try:
     from urllib import unquote
@@ -38,12 +38,13 @@ from flask import (
     redirect,
     abort
 )
-from flask_login import login_required
+from flask_login import current_user, login_required
 from werkzeug.datastructures import Headers
 from sqlalchemy import func
+from sqlalchemy.sql.expression import and_
 import requests
 
-from . import config, logger, kobo_auth, db, helper
+from . import config, logger, kobo_auth, db, helper, ub
 from .services import SyncToken as SyncToken
 from .web import download_required
 from .kobo_auth import requires_kobo_auth
@@ -116,6 +117,9 @@ def redirect_or_proxy_request():
         return make_response(jsonify({}))
 
 
+def convert_to_kobo_timestamp_string(timestamp):
+    return timestamp.strftime("%Y-%m-%dT%H:%M:%SZ")
+
 @kobo.route("/v1/library/sync")
 @requires_kobo_auth
 @download_required
@@ -130,7 +134,8 @@ def HandleSyncRequest():
 
     new_books_last_modified = sync_token.books_last_modified
     new_books_last_created = sync_token.books_last_created
-    entitlements = []
+    new_reading_state_last_modified = sync_token.reading_state_last_modified
+    sync_results = []
 
     # We reload the book database so that the user get's a fresh view of the library
     # in case of external changes (e.g: adding a book through Calibre).
@@ -147,41 +152,63 @@ def HandleSyncRequest():
         .all()
     )
 
+    reading_states_in_new_entitlements = []
     for book in changed_entries:
+        kobo_reading_state = get_or_create_reading_state(book.id)
         entitlement = {
             "BookEntitlement": create_book_entitlement(book),
             "BookMetadata": get_metadata(book),
-            "ReadingState": reading_state(book),
         }
+
+        if kobo_reading_state.last_modified > sync_token.reading_state_last_modified:
+            entitlement["ReadingState"] = get_kobo_reading_state_response(book, kobo_reading_state)
+            new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified)
+            reading_states_in_new_entitlements.append(book.id)
+
         if book.timestamp > sync_token.books_last_created:
-            entitlements.append({"NewEntitlement": entitlement})
+            sync_results.append({"NewEntitlement": entitlement})
         else:
-            entitlements.append({"ChangedEntitlement": entitlement})
+            sync_results.append({"ChangedEntitlement": entitlement})
 
         new_books_last_modified = max(
             book.last_modified, sync_token.books_last_modified
         )
         new_books_last_created = max(book.timestamp, sync_token.books_last_created)
 
+    changed_reading_states = (
+        ub.session.query(ub.KoboReadingState)
+        .filter(and_(func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified,
+                     ub.KoboReadingState.user_id == current_user.id,
+                     ub.KoboReadingState.book_id.notin_(reading_states_in_new_entitlements))))
+    for kobo_reading_state in changed_reading_states.all():
+        book = db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one()
+        sync_results.append({
+            "ChangedReadingState": {
+                "ReadingState": get_kobo_reading_state_response(book, kobo_reading_state)
+            }
+        })
+        new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified)
+
     sync_token.books_last_created = new_books_last_created
     sync_token.books_last_modified = new_books_last_modified
+    sync_token.reading_state_last_modified = new_reading_state_last_modified
 
     if config.config_kobo_proxy:
-        return generate_sync_response(request, sync_token, entitlements)
+        return generate_sync_response(request, sync_token, sync_results)
 
-    return make_response(jsonify(entitlements))
+    return make_response(jsonify(sync_results))
     # Missing feature: Detect server-side book deletions.
 
 
-def generate_sync_response(request, sync_token, entitlements):
+def generate_sync_response(request, sync_token, sync_results):
     extra_headers = {}
     if config.config_kobo_proxy:
         # Merge in sync results from the official Kobo store.
         try:
             store_response = make_request_to_kobo_store(sync_token)
 
-            store_entitlements = store_response.json()
+            store_sync_results = store_response.json()
-            entitlements += store_entitlements
+            sync_results += store_sync_results
             sync_token.merge_from_store_response(store_response)
             extra_headers["x-kobo-sync"] = store_response.headers.get("x-kobo-sync")
             extra_headers["x-kobo-sync-mode"] = store_response.headers.get("x-kobo-sync-mode")
@@ -191,7 +218,7 @@ def generate_sync_response(request, sync_token, entitlements):
             log.error("Failed to receive or parse response from Kobo's sync endpoint: " + str(e))
     sync_token.to_headers(extra_headers)
 
-    response = make_response(jsonify(entitlements), extra_headers)
+    response = make_response(jsonify(sync_results), extra_headers)
 
     return response
 
@@ -243,25 +270,21 @@ def create_book_entitlement(book):
     book_uuid = book.uuid
     return {
         "Accessibility": "Full",
-        "ActivePeriod": {"From": current_time(),},
+        "ActivePeriod": {"From": convert_to_kobo_timestamp_string(datetime.datetime.now())},
-        "Created": book.timestamp.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "Created": convert_to_kobo_timestamp_string(book.timestamp),
         "CrossRevisionId": book_uuid,
         "Id": book_uuid,
         "IsHiddenFromArchive": False,
         "IsLocked": False,
         # Setting this to true removes from the device.
         "IsRemoved": False,
-        "LastModified": book.last_modified.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "LastModified": convert_to_kobo_timestamp_string(book.last_modified),
         "OriginCategory": "Imported",
         "RevisionId": book_uuid,
         "Status": "Active",
     }
 
 
-def current_time():
-    return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
-
-
 def get_description(book):
     if not book.comments:
         return None
@@ -324,6 +347,8 @@ def get_metadata(book):
         "IsSocialEnabled": True,
         "Language": "en",
         "PhoneticPronunciations": {},
+        # TODO: Fix book.pubdate to return a datetime object so that we can easily
+        # convert it to the format Kobo devices expect.
         "PublicationDate": book.pubdate,
         "Publisher": {"Imprint": "", "Name": get_publisher(book),},
         "RevisionId": book_uuid,
@@ -347,16 +372,148 @@ def get_metadata(book):
     return metadata
 
 
-def reading_state(book):
+@kobo.route("/v1/library/<book_uuid>/state", methods=["GET", "PUT"])
-    # TODO: Implement
+@login_required
-    reading_state = {
+def HandleStateRequest(book_uuid):
-        # "StatusInfo": {
+    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
-        #     "LastModified": get_single_cc_value(book, "lastreadtimestamp"),
+    if not book or not book.data:
-        #     "Status": get_single_cc_value(book, "reading_status"),
+        log.info(u"Book %s not found in database", book_uuid)
-        # }
+        return redirect_or_proxy_request()
-        # TODO: CurrentBookmark, Location
+
+    kobo_reading_state = get_or_create_reading_state(book.id)
+
+    if request.method == "GET":
+        return jsonify([get_kobo_reading_state_response(book, kobo_reading_state)])
+    else:
+        update_results_response = {"EntitlementId": book_uuid}
+
+        request_data = request.json
+        if "ReadingStates" not in request_data:
+            abort(400, description="Malformed request data is missing 'ReadingStates' key")
+        request_reading_state = request_data["ReadingStates"][0]
+
+        request_bookmark = request_reading_state.get("CurrentBookmark")
+        if request_bookmark:
+            current_bookmark = kobo_reading_state.current_bookmark
+            current_bookmark.progress_percent = request_bookmark.get("ProgressPercent")
+            current_bookmark.content_source_progress_percent = request_bookmark.get("ContentSourceProgressPercent")
+            location = request_bookmark.get("Location")
+            if location:
+                current_bookmark.location_value = location.get("Value")
+                current_bookmark.location_type = location.get("Type")
+                current_bookmark.location_source = location.get("Source")
+            update_results_response["CurrentBookmarkResult"] = {"Result": "Success"}
+
+        request_statistics = request_reading_state.get("Statistics")
+        if request_statistics:
+            statistics = kobo_reading_state.statistics
+            statistics.spent_reading_minutes = request_statistics.get("SpentReadingMinutes")
+            statistics.remaining_time_minutes = request_statistics.get("RemainingTimeMinutes")
+            update_results_response["StatisticsResult"] = {"Result": "Success"}
+
+        request_status_info = request_reading_state.get("StatusInfo")
+        if request_status_info:
+            book_read = kobo_reading_state.book_read_link
+            new_book_read_status = get_ub_read_status(request_status_info.get("Status"))
+            if new_book_read_status == ub.ReadBook.STATUS_IN_PROGRESS and new_book_read_status != book_read.read_status:
+                book_read.times_started_reading += 1
+                book_read.last_time_started_reading = datetime.datetime.utcnow()
+            book_read.read_status = new_book_read_status
+            update_results_response["StatusInfoResult"] = {"Result": "Success"}
+
+        ub.session.merge(kobo_reading_state)
+        ub.session.commit()
+        return jsonify({
+            "RequestResult": "Success",
+            "UpdateResults": [update_results_response],
+        })
+
+
+def get_read_status_for_kobo(ub_book_read):
+    enum_to_string_map = {
+        None: "ReadyToRead",
+        ub.ReadBook.STATUS_UNREAD: "ReadyToRead",
+        ub.ReadBook.STATUS_FINISHED: "Finished",
+        ub.ReadBook.STATUS_IN_PROGRESS: "Reading",
     }
-    return reading_state
+    return enum_to_string_map[ub_book_read.read_status]
+
+
+def get_ub_read_status(kobo_read_status):
+    string_to_enum_map = {
+        None: None,
+        "ReadyToRead": ub.ReadBook.STATUS_UNREAD,
+        "Finished": ub.ReadBook.STATUS_FINISHED,
+        "Reading": ub.ReadBook.STATUS_IN_PROGRESS,
+    }
+    return string_to_enum_map[kobo_read_status]
+
+
+def get_or_create_reading_state(book_id):
+    book_read = ub.session.query(ub.ReadBook).filter(and_(ub.ReadBook.book_id == book_id,
+                                                          ub.ReadBook.user_id == current_user.id)).one_or_none()
+    if not book_read:
+        book_read = ub.ReadBook(user_id=current_user.id, book_id=book_id)
+    if not book_read.kobo_reading_state:
+        kobo_reading_state = ub.KoboReadingState(user_id=book_read.user_id, book_id=book_id)
+        kobo_reading_state.current_bookmark = ub.KoboBookmark()
+        kobo_reading_state.statistics = ub.KoboStatistics()
+        book_read.kobo_reading_state = kobo_reading_state
+    ub.session.add(book_read)
+    ub.session.commit()
+    return book_read.kobo_reading_state
+
+
+def get_kobo_reading_state_response(book, kobo_reading_state):
+    return {
+        "EntitlementId": book.uuid,
+        "Created": convert_to_kobo_timestamp_string(book.timestamp),
+        "LastModified": convert_to_kobo_timestamp_string(kobo_reading_state.last_modified),
+        # AFAICT PriorityTimestamp is always equal to LastModified.
+        "PriorityTimestamp": convert_to_kobo_timestamp_string(kobo_reading_state.priority_timestamp),
+        "StatusInfo": get_status_info_response(kobo_reading_state.book_read_link),
+        "Statistics": get_statistics_response(kobo_reading_state.statistics),
+        "CurrentBookmark": get_current_bookmark_response(kobo_reading_state.current_bookmark),
+    }
+
+
+def get_status_info_response(book_read):
+    resp = {
+        "LastModified": convert_to_kobo_timestamp_string(book_read.last_modified),
+        "Status": get_read_status_for_kobo(book_read),
+        "TimesStartedReading": book_read.times_started_reading,
+    }
+    if book_read.last_time_started_reading:
+        resp["LastTimeStartedReading"] = convert_to_kobo_timestamp_string(book_read.last_time_started_reading)
+    return resp
+
+
+def get_statistics_response(statistics):
+    resp = {
+        "LastModified": convert_to_kobo_timestamp_string(statistics.last_modified),
+    }
+    if statistics.spent_reading_minutes:
+        resp["SpentReadingMinutes"] = statistics.spent_reading_minutes
+    if statistics.remaining_time_minutes:
+        resp["RemainingTimeMinutes"] = statistics.remaining_time_minutes
+    return resp
+
+
+def get_current_bookmark_response(current_bookmark):
+    resp = {
+        "LastModified": convert_to_kobo_timestamp_string(current_bookmark.last_modified),
+    }
+    if current_bookmark.progress_percent:
+        resp["ProgressPercent"] = current_bookmark.progress_percent
+    if current_bookmark.content_source_progress_percent:
+        resp["ContentSourceProgressPercent"] = current_bookmark.content_source_progress_percent
+    if current_bookmark.location_value:
+        resp["Location"] = {
+            "Value": current_bookmark.location_value,
+            "Type": current_bookmark.location_type,
+            "Source": current_bookmark.location_source,
+        }
+    return resp
 
 
 @kobo.route("/<book_uuid>/image.jpg")
@@ -381,7 +538,6 @@ def TopLevelEndpoint():
 
 # TODO: Implement the following routes
 @kobo.route("/v1/library/<dummy>", methods=["DELETE", "GET"])
-@kobo.route("/v1/library/<book_uuid>/state", methods=["PUT"])
 @kobo.route("/v1/library/tags", methods=["POST"])
 @kobo.route("/v1/library/tags/<shelf_name>", methods=["POST"])
 @kobo.route("/v1/library/tags/<tag_id>", methods=["DELETE"])

cps/services/SyncToken.py

@@ -42,6 +42,13 @@ def to_epoch_timestamp(datetime_object):
     return (datetime_object - datetime(1970, 1, 1)).total_seconds()
 
 
+def get_datetime_from_json(json_object, field_name):
+    try:
+        return datetime.utcfromtimestamp(json_object[field_name])
+    except KeyError:
+        return datetime.min
+
+
 class SyncToken():
     """ The SyncToken is used to persist state accross requests.
     When serialized over the response headers, the Kobo device will propagate the token onto following requests to the service.
@@ -53,7 +60,8 @@ class SyncToken():
     """
 
     SYNC_TOKEN_HEADER = "x-kobo-synctoken"
-    VERSION = "1-0-0"
+    VERSION = "1-1-0"
+    LAST_MODIFIED_ADDED_VERSION = "1-1-0"
     MIN_VERSION = "1-0-0"
 
     token_schema = {
@@ -68,6 +76,7 @@ class SyncToken():
             "raw_kobo_store_token": {"type": "string"},
             "books_last_modified": {"type": "string"},
             "books_last_created": {"type": "string"},
+            "reading_state_last_modified": {"type": "string"},
         },
     }
 
@@ -76,10 +85,13 @@ class SyncToken():
         raw_kobo_store_token="",
         books_last_created=datetime.min,
         books_last_modified=datetime.min,
+        reading_state_last_modified=datetime.min,
     ):
         self.raw_kobo_store_token = raw_kobo_store_token
         self.books_last_created = books_last_created
         self.books_last_modified = books_last_modified
+        self.reading_state_last_modified = reading_state_last_modified
+
 
     @staticmethod
     def from_headers(headers):
@@ -109,12 +121,9 @@ class SyncToken():
 
         raw_kobo_store_token = data_json["raw_kobo_store_token"]
         try:
-            books_last_modified = datetime.utcfromtimestamp(
+            books_last_modified = get_datetime_from_json(data_json, "books_last_modified")
-                data_json["books_last_modified"]
+            books_last_created = get_datetime_from_json(data_json, "books_last_created")
-            )
+            reading_state_last_modified = get_datetime_from_json(data_json, "reading_state_last_modified")
-            books_last_created = datetime.utcfromtimestamp(
-                data_json["books_last_created"]
-            )
         except TypeError:
             log.error("SyncToken timestamps don't parse to a datetime.")
             return SyncToken(raw_kobo_store_token=raw_kobo_store_token)
@@ -123,6 +132,7 @@ class SyncToken():
             raw_kobo_store_token=raw_kobo_store_token,
             books_last_created=books_last_created,
             books_last_modified=books_last_modified,
+            reading_state_last_modified=reading_state_last_modified
         )
 
     def set_kobo_store_header(self, store_headers):
@@ -143,6 +153,7 @@ class SyncToken():
                 "raw_kobo_store_token": self.raw_kobo_store_token,
                 "books_last_modified": to_epoch_timestamp(self.books_last_modified),
                 "books_last_created": to_epoch_timestamp(self.books_last_created),
+                "reading_state_last_modified": to_epoch_timestamp(self.reading_state_last_modified)
             },
         }
         return b64encode_json(token)

cps/ub.py

@@ -20,6 +20,7 @@
 from __future__ import division, print_function, unicode_literals
 import os
 import datetime
+import itertools
 from binascii import hexlify
 
 from flask import g
@@ -31,10 +32,10 @@ try:
     oauth_support = True
 except ImportError:
     oauth_support = False
-from sqlalchemy import create_engine, exc, exists
+from sqlalchemy import create_engine, exc, exists, event
 from sqlalchemy import Column, ForeignKey
-from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime
+from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float
-from sqlalchemy.orm import foreign, relationship, remote, sessionmaker
+from sqlalchemy.orm import backref, foreign, relationship, remote, sessionmaker, Session
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.sql.expression import and_
 from werkzeug.security import generate_password_hash
@@ -292,8 +293,12 @@ class ReadBook(Base):
     id = Column(Integer, primary_key=True)
     book_id = Column(Integer, unique=False)
     user_id = Column(Integer, ForeignKey('user.id'), unique=False)
-    is_read = Column(Boolean, unique=False)
+    read_status = Column(Integer, unique=False, default=STATUS_UNREAD, nullable=False)
-    read_status = Column(Integer, unique=False, default=STATUS_UNREAD)
+    kobo_reading_state = relationship("KoboReadingState", uselist=False, primaryjoin="and_(ReadBook.user_id == foreign(KoboReadingState.user_id), "
+                                "ReadBook.book_id == foreign(KoboReadingState.book_id))", cascade="all", backref=backref("book_read_link", uselist=False))
+    last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
+    last_time_started_reading = Column(DateTime, nullable=True)
+    times_started_reading = Column(Integer, default=0, nullable=False)
 
 
 class Bookmark(Base):
@@ -306,6 +311,54 @@ class Bookmark(Base):
     bookmark_key = Column(String)
 
 
+# The Kobo ReadingState API keeps track of 4 timestamped entities:
+#   ReadingState, StatusInfo, Statistics, CurrentBookmark
+# Which we map to the following 4 tables:
+#   KoboReadingState, ReadBook, KoboStatistics and KoboBookmark
+class KoboReadingState(Base):
+    __tablename__ = 'kobo_reading_state'
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    book_id = Column(Integer)
+    last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
+    priority_timestamp = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
+    current_bookmark = relationship("KoboBookmark", uselist=False, backref="kobo_reading_state", cascade="all")
+    statistics = relationship("KoboStatistics", uselist=False, backref="kobo_reading_state", cascade="all")
+
+
+class KoboBookmark(Base):
+    __tablename__ = 'kobo_bookmark'
+
+    id = Column(Integer, primary_key=True)
+    kobo_reading_state_id = Column(Integer, ForeignKey('kobo_reading_state.id'))
+    last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
+    location_source = Column(String)
+    location_type = Column(String)
+    location_value = Column(String)
+    progress_percent = Column(Float)
+    content_source_progress_percent = Column(Float)
+
+
+class KoboStatistics(Base):
+    __tablename__ = 'kobo_statistics'
+
+    id = Column(Integer, primary_key=True)
+    kobo_reading_state_id = Column(Integer, ForeignKey('kobo_reading_state.id'))
+    last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
+    remaining_time_minutes = Column(Integer)
+    spent_reading_minutes = Column(Integer)
+
+
+# Updates the last_modified timestamp in the KoboReadingState table if any of its children tables are modified.
+@event.listens_for(Session, 'before_flush')
+def receive_before_flush(session, flush_context, instances):
+    for change in itertools.chain(session.new, session.dirty):
+        if isinstance(change, (ReadBook, KoboStatistics, KoboBookmark)):
+            if change.kobo_reading_state:
+                change.kobo_reading_state.last_modified = datetime.datetime.utcnow()
+
+
 # Baseclass representing Downloads from calibre-web in app.db
 class Downloads(Base):
     __tablename__ = 'downloads'
@@ -358,6 +411,12 @@ def migrate_Database(session):
         ReadBook.__table__.create(bind=engine)
     if not engine.dialect.has_table(engine.connect(), "bookmark"):
         Bookmark.__table__.create(bind=engine)
+    if not engine.dialect.has_table(engine.connect(), "kobo_reading_state"):
+        KoboReadingState.__table__.create(bind=engine)
+    if not engine.dialect.has_table(engine.connect(), "kobo_bookmark"):
+        KoboBookmark.__table__.create(bind=engine)
+    if not engine.dialect.has_table(engine.connect(), "kobo_statistics"):
+        KoboStatistics.__table__.create(bind=engine)
     if not engine.dialect.has_table(engine.connect(), "registration"):
         ReadBook.__table__.create(bind=engine)
         conn = engine.connect()
@@ -381,10 +440,13 @@ def migrate_Database(session):
         session.commit()
     try:
         session.query(exists().where(ReadBook.read_status)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some columns are missing
+    except exc.OperationalError:
         conn = engine.connect()
         conn.execute("ALTER TABLE book_read_link ADD column 'read_status' INTEGER DEFAULT 0")
         conn.execute("UPDATE book_read_link SET 'read_status' = 1 WHERE is_read")
+        conn.execute("ALTER TABLE book_read_link ADD column 'last_modified' DATETIME")
+        conn.execute("ALTER TABLE book_read_link ADD column 'last_time_started_reading' DATETIME")
+        conn.execute("ALTER TABLE book_read_link ADD column 'times_started_reading' INTEGER DEFAULT 0")    
         session.commit()
 
     # Handle table exists, but no content

cps/web.py

@@ -319,11 +319,14 @@ def toggle_read(book_id):
             else:
                 book.read_status = ub.ReadBook.STATUS_FINISHED
         else:
-            readBook = ub.ReadBook()
+            readBook = ub.ReadBook(user_id=current_user.id, book_id = book_id)
-            readBook.user_id = int(current_user.id)
-            readBook.book_id = book_id
             readBook.read_status = ub.ReadBook.STATUS_FINISHED
             book = readBook
+        if not book.kobo_reading_state:
+            kobo_reading_state = ub.KoboReadingState(user_id=current_user.id, book_id=book_id)
+            kobo_reading_state.current_bookmark = ub.KoboBookmark()
+            kobo_reading_state.statistics = ub.KoboStatistics()
+            book.kobo_reading_state = kobo_reading_state
         ub.session.merge(book)
         ub.session.commit()
     else: