Merge remote-tracking branch 'koko_sync/feature/kobo-shelf' into Develop

cps/kobo.py

@@ -42,7 +42,7 @@ from flask import (
 from flask_login import current_user
 from werkzeug.datastructures import Headers
 from sqlalchemy import func
-from sqlalchemy.sql.expression import and_
+from sqlalchemy.sql.expression import and_, or_
 from sqlalchemy.exc import StatementError
 import requests
 
@@ -81,6 +81,7 @@ CONNECTION_SPECIFIC_HEADERS = [
     "transfer-encoding",
 ]
 
+
 def get_kobo_activated():
     return config.config_kobo_sync
 
@@ -151,30 +152,45 @@ def HandleSyncRequest():
     # in case of external changes (e.g: adding a book through Calibre).
     calibre_db.reconnect_db(config, ub.app_DB_path)
 
-    if sync_token.books_last_id > -1:
+    only_kobo_shelves = (
+                            calibre_db.session.query(ub.Shelf)
+                                .filter(ub.Shelf.user_id == current_user.id)
+                                .filter(ub.Shelf.kobo_sync)
+                                .count()
+                        ) > 0
+
+    if only_kobo_shelves:
         changed_entries = (
-            calibre_db.session.query(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived)
+            calibre_db.session.query(db.Books,
-            .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id)
+                                     ub.ArchivedBook.last_modified,
-            .filter(db.Books.last_modified >= sync_token.books_last_modified)
+                                     ub.BookShelf.date_added,
-            .filter(db.Books.id>sync_token.books_last_id)
+                                     ub.ArchivedBook.is_archived)
-            .filter(db.Data.format.in_(KOBO_FORMATS))
+                .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id)
-            .order_by(db.Books.last_modified)
+                .filter(or_(db.Books.last_modified > sync_token.books_last_modified,
-            .order_by(db.Books.id)
+                            ub.BookShelf.date_added > sync_token.books_last_modified))
-            .limit(SYNC_ITEM_LIMIT)
+                .filter(db.Data.format.in_(KOBO_FORMATS))
+                .order_by(db.Books.id)
+                .order_by(ub.ArchivedBook.last_modified)
+                .join(ub.BookShelf, db.Books.id == ub.BookShelf.book_id)
+                .join(ub.Shelf)
+                .filter(ub.Shelf.kobo_sync)
+                .distinct()
         )
     else:
         changed_entries = (
             calibre_db.session.query(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived)
-            .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id)
+                .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id)
-            .filter(db.Books.last_modified > sync_token.books_last_modified)
+                .filter(db.Books.last_modified > sync_token.books_last_modified)
-            .filter(db.Data.format.in_(KOBO_FORMATS))
+                .filter(db.Data.format.in_(KOBO_FORMATS))
-            .order_by(db.Books.last_modified)
+                .order_by(db.Books.last_modified)
-            .order_by(db.Books.id)
+                .order_by(db.Books.id)
-            .limit(SYNC_ITEM_LIMIT)
         )
 
+    if sync_token.books_last_id > -1:
+        changed_entries = changed_entries.filter(db.Books.id > sync_token.books_last_id)
+
     reading_states_in_new_entitlements = []
-    for book in changed_entries:
+    for book in changed_entries.limit(SYNC_ITEM_LIMIT):
         formats = [data.format for data in book.Books.data]
         if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats:
             helper.convert_book_format(book.Books.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)
@@ -190,7 +206,14 @@ def HandleSyncRequest():
             new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified)
             reading_states_in_new_entitlements.append(book.Books.id)
 
-        if book.Books.timestamp > sync_token.books_last_created:
+        ts_created = book.Books.timestamp
+
+        try:
+            ts_created = max(ts_created, book.date_added)
+        except AttributeError:
+            pass
+
+        if ts_created > sync_token.books_last_created:
             sync_results.append({"NewEntitlement": entitlement})
         else:
             sync_results.append({"ChangedEntitlement": entitlement})
@@ -198,7 +221,14 @@ def HandleSyncRequest():
         new_books_last_modified = max(
             book.Books.last_modified, new_books_last_modified
         )
-        new_books_last_created = max(book.Books.timestamp, new_books_last_created)
+        try:
+            new_books_last_modified = max(
+                new_books_last_modified, book.date_added
+            )
+        except AttributeError:
+            pass
+
+        new_books_last_created = max(ts_created, new_books_last_created)
 
     max_change = (changed_entries
         .from_self()
@@ -222,11 +252,36 @@ def HandleSyncRequest():
         books_last_id = -1
 
     # generate reading state data
+    changed_reading_states = ub.session.query(ub.KoboReadingState)
+
+    if only_kobo_shelves:
+        changed_reading_states = (
+            changed_reading_states.join(ub.BookShelf, ub.KoboReadingState.book_id == ub.BookShelf.book_id)
+            .join(ub.Shelf)
+            .filter(
+                ub.Shelf.kobo_sync,
+                or_(
+                    func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified,
+                    ub.BookShelf.date_added > sync_token.books_last_modified
+                )
+            )
+        ).distinct()
+
+    else:
+        changed_reading_states = (
+            changed_reading_states.filter(
+                func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified
+            )
+        )
     changed_reading_states = (
-        ub.session.query(ub.KoboReadingState)
+        changed_reading_states.filter(
-        .filter(and_(func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified,
+            and_(
-                     ub.KoboReadingState.user_id == current_user.id,
+                ub.KoboReadingState.user_id == current_user.id,
-                     ub.KoboReadingState.book_id.notin_(reading_states_in_new_entitlements))))
+                ub.KoboReadingState.book_id.notin_(reading_states_in_new_entitlements)
+            )
+        )
+    )
+
     for kobo_reading_state in changed_reading_states.all():
         book = calibre_db.session.query(db.Books).filter(db.Books.id == kobo_reading_state.book_id).one_or_none()
         if book:
@@ -237,7 +292,7 @@ def HandleSyncRequest():
             })
             new_reading_state_last_modified = max(new_reading_state_last_modified, kobo_reading_state.last_modified)
 
-    sync_shelves(sync_token, sync_results)
+    sync_shelves(sync_token, sync_results, only_kobo_shelves=only_kobo_shelves)
 
     sync_token.books_last_created = new_books_last_created
     sync_token.books_last_modified = new_books_last_modified
@@ -392,7 +447,7 @@ def get_metadata(book):
 
     book_uuid = book.uuid
     metadata = {
-        "Categories": ["00000000-0000-0000-0000-000000000001",],
+        "Categories": ["00000000-0000-0000-0000-000000000001", ],
         # "Contributors": get_author(book),
         "CoverImageId": book_uuid,
         "CrossRevisionId": book_uuid,
@@ -599,13 +654,14 @@ def HandleTagRemoveItem(tag_id):
 
 # Add new, changed, or deleted shelves to the sync_results.
 # Note: Public shelves that aren't owned by the user aren't supported.
-def sync_shelves(sync_token, sync_results):
+def sync_shelves(sync_token, sync_results, only_kobo_shelves=False):
     new_tags_last_modified = sync_token.tags_last_modified
 
-    for shelf in ub.session.query(ub.ShelfArchive).filter(func.datetime(ub.ShelfArchive.last_modified) > sync_token.tags_last_modified,
+    for shelf in ub.session.query(ub.ShelfArchive).filter(
-                                                          ub.ShelfArchive.user_id == current_user.id):
+        func.datetime(ub.ShelfArchive.last_modified) > sync_token.tags_last_modified,
+        ub.ShelfArchive.user_id == current_user.id
+    ):
         new_tags_last_modified = max(shelf.last_modified, new_tags_last_modified)
-
         sync_results.append({
             "DeletedTag": {
                 "Tag": {
@@ -615,8 +671,29 @@ def sync_shelves(sync_token, sync_results):
             }
         })
 
-    for shelf in ub.session.query(ub.Shelf).filter(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified,
+    extra_filters = []
-                                                   ub.Shelf.user_id == current_user.id):
+    if only_kobo_shelves:
+        for shelf in ub.session.query(ub.Shelf).filter(
+            func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified,
+            ub.Shelf.user_id == current_user.id,
+            not ub.Shelf.kobo_sync
+        ):
+            sync_results.append({
+                "DeletedTag": {
+                    "Tag": {
+                        "Id": shelf.uuid,
+                        "LastModified": convert_to_kobo_timestamp_string(shelf.last_modified)
+                    }
+                }
+            })
+        extra_filters.append(ub.Shelf.kobo_sync)
+
+    for shelf in ub.session.query(ub.Shelf).outerjoin(ub.BookShelf).filter(
+        or_(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified,
+            ub.BookShelf.date_added > sync_token.tags_last_modified),
+        ub.Shelf.user_id == current_user.id,
+        *extra_filters
+    ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc()):
         if not shelf_lib.check_shelf_view_permissions(shelf):
             continue
 

cps/shelf.py

@@ -21,20 +21,20 @@
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import division, print_function, unicode_literals
-from datetime import datetime
+
 import sys
+from datetime import datetime
 
-from flask import Blueprint, request, flash, redirect, url_for
+from flask import Blueprint, flash, redirect, request, url_for
 from flask_babel import gettext as _
-from flask_login import login_required, current_user
+from flask_login import current_user, login_required
+from sqlalchemy.exc import InvalidRequestError, OperationalError
 from sqlalchemy.sql.expression import func, true
-from sqlalchemy.exc import OperationalError, InvalidRequestError
 
-from . import logger, ub, calibre_db, db
+from . import calibre_db, config, db, logger, ub
 from .render_template import render_title_template
 from .usermanagement import login_required_if_no_ano
 
-
 shelf = Blueprint('shelf', __name__)
 log = logger.create()
 
@@ -246,6 +246,13 @@ def create_edit_shelf(shelf, title, page, shelf_id=False):
             shelf.is_public = 1
         else:
             shelf.is_public = 0
+
+        if config.config_kobo_sync:
+            if "kobo_sync" in to_save:
+                shelf.kobo_sync = True
+            else:
+                shelf.kobo_sync = False
+
         if check_shelf_is_unique(shelf, to_save, shelf_id):
             shelf.name = to_save["title"]
             # shelf.last_modified = datetime.utcnow()
@@ -271,7 +278,11 @@ def create_edit_shelf(shelf, title, page, shelf_id=False):
                 ub.session.rollback()
                 log.debug_or_exception(ex)
                 flash(_(u"There was an error"), category="error")
-    return render_title_template('shelf_edit.html', shelf=shelf, title=title, page=page)
+    return render_title_template('shelf_edit.html',
+                                 shelf=shelf,
+                                 title=title,
+                                 page=page,
+                                 kobo_sync_enabled=config.config_kobo_sync)
 
 
 def check_shelf_is_unique(shelf, to_save, shelf_id=False):
@@ -362,8 +373,8 @@ def order_shelf(shelf_id):
     shelf = ub.session.query(ub.Shelf).filter(ub.Shelf.id == shelf_id).first()
     result = list()
     if shelf and check_shelf_view_permissions(shelf):
-        result = calibre_db.session.query(db.Books)\
+        result = calibre_db.session.query(db.Books) \
-            .join(ub.BookShelf,ub.BookShelf.book_id == db.Books.id , isouter=True) \
+            .join(ub.BookShelf, ub.BookShelf.book_id == db.Books.id, isouter=True) \
             .add_columns(calibre_db.common_filters().label("visible")) \
             .filter(ub.BookShelf.shelf == shelf_id).order_by(ub.BookShelf.order.asc()).all()
     return render_title_template('shelf_order.html', entries=result,
@@ -372,7 +383,7 @@ def order_shelf(shelf_id):
 
 
 def change_shelf_order(shelf_id, order):
-    result = calibre_db.session.query(db.Books).join(ub.BookShelf,ub.BookShelf.book_id == db.Books.id)\
+    result = calibre_db.session.query(db.Books).join(ub.BookShelf, ub.BookShelf.book_id == db.Books.id) \
         .filter(ub.BookShelf.shelf == shelf_id).order_by(*order).all()
     for index, entry in enumerate(result):
         book = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id) \
@@ -412,13 +423,13 @@ def render_show_shelf(shelf_type, shelf_id, page_no, sort_param):
             page = 'shelfdown.html'
 
         result, __, pagination = calibre_db.fill_indexpage(page_no, pagesize,
-                                                            db.Books,
+                                                           db.Books,
-                                                            ub.BookShelf.shelf == shelf_id,
+                                                           ub.BookShelf.shelf == shelf_id,
-                                                            [ub.BookShelf.order.asc()],
+                                                           [ub.BookShelf.order.asc()],
-                                                            ub.BookShelf,ub.BookShelf.book_id == db.Books.id)
+                                                           ub.BookShelf, ub.BookShelf.book_id == db.Books.id)
         # delete chelf entries where book is not existent anymore, can happen if book is deleted outside calibre-web
-        wrong_entries = calibre_db.session.query(ub.BookShelf)\
+        wrong_entries = calibre_db.session.query(ub.BookShelf) \
-            .join(db.Books, ub.BookShelf.book_id == db.Books.id, isouter=True)\
+            .join(db.Books, ub.BookShelf.book_id == db.Books.id, isouter=True) \
             .filter(db.Books.id == None).all()
         for entry in wrong_entries:
             log.info('Not existing book {} in {} deleted'.format(entry.book_id, shelf))

cps/templates/shelf_edit.html

@@ -14,6 +14,14 @@
         </label>
       </div>
     {% endif %}
+    {% if kobo_sync_enabled %}
+      <div class="checkbox">
+          <label>
+              <input type="checkbox" name="kobo_sync"
+                     {% if shelf.kobo_sync == 1 %}checked{% endif %}> {{ _('Synchronize with Kobo device') }}
+          </label>
+      </div>
+    {% endif %}
     <button type="submit" class="btn btn-default" id="submit">{{_('Save')}}</button>
     {% if shelf.id != None %}
       <a href="{{ url_for('shelf.show_shelf', shelf_id=shelf.id) }}" class="btn btn-default">{{_('Cancel')}}</a>

cps/ub.py

@@ -270,6 +270,7 @@ class Shelf(Base):
     name = Column(String)
     is_public = Column(Integer, default=0)
     user_id = Column(Integer, ForeignKey('user.id'))
+    kobo_sync = Column(Boolean, default=False)
     books = relationship("BookShelf", backref="ub_shelf", cascade="all, delete-orphan", lazy="dynamic")
     created = Column(DateTime, default=datetime.datetime.utcnow)
     last_modified = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow)
@@ -502,6 +503,7 @@ def migrate_shelfs(engine, session):
             conn.execute("ALTER TABLE shelf ADD column 'created' DATETIME")
             conn.execute("ALTER TABLE shelf ADD column 'last_modified' DATETIME")
             conn.execute("ALTER TABLE book_shelf_link ADD column 'date_added' DATETIME")
+            conn.execute("ALTER TABLE shelf ADD column 'kobo_sync' BOOLEAN DEFAULT false")
         for shelf in session.query(Shelf).all():
             shelf.uuid = str(uuid.uuid4())
             shelf.created = datetime.datetime.now()
@@ -509,6 +511,15 @@ def migrate_shelfs(engine, session):
         for book_shelf in session.query(BookShelf).all():
             book_shelf.date_added = datetime.datetime.now()
         session.commit()
+
+    try:
+        session.query(exists().where(Shelf.kobo_sync)).scalar()
+    except exc.OperationalError:
+        with engine.connect() as conn:
+
+            conn.execute("ALTER TABLE shelf ADD column 'kobo_sync' BOOLEAN DEFAULT false")
+        session.commit()
+
     try:
         session.query(exists().where(BookShelf.order)).scalar()
     except exc.OperationalError:  # Database is not compatible, some columns are missing