Merge branch 'master' into Develop

README.md

@@ -40,7 +40,7 @@ Calibre-Web is a web app providing a clean interface for browsing, reading and d
 ## Quick start
 
 #### Install via pip
-1. Install calibre web via pip with the command `pip install calibreweb`.
+1. Install calibre web via pip with the command `pip install calibreweb` (Depending on your OS and or distro the command could also be `pip3`). 
 2. Optional features can also be installed via pip, please refer to [this page](https://github.com/janeczku/calibre-web/wiki/Dependencies-in-Calibre-Web-Linux-Windows) for details 
 3. Calibre-Web can be started afterwards by typing `cps` or `python3 -m cps` 
 

cps/admin.py

@@ -40,7 +40,6 @@ from sqlalchemy.exc import IntegrityError, OperationalError, InvalidRequestError
 from sqlalchemy.sql.expression import func, or_, text
 
 from . import constants, logger, helper, services
-# from .cli import filepicker
 from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
 from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \
     valid_email, check_username
@@ -1351,7 +1350,8 @@ def _handle_new_user(to_save, content, languages, translations, kobo_support):
         content.denied_tags = config.config_denied_tags
         content.allowed_column_value = config.config_allowed_column_value
         content.denied_column_value = config.config_denied_column_value
-        content.kobo_only_shelves_sync = 0  # No default value for kobo sync shelf setting
+        # No default value for kobo sync shelf setting
+        content.kobo_only_shelves_sync = to_save.get("kobo_only_shelves_sync", 0) == "on"
         ub.session.add(content)
         ub.session.commit()
         flash(_(u"User '%(user)s' created", user=content.name), category="success")
@@ -1370,6 +1370,13 @@ def _delete_user(content):
     if ub.session.query(ub.User).filter(ub.User.role.op('&')(constants.ROLE_ADMIN) == constants.ROLE_ADMIN,
                                         ub.User.id != content.id).count():
         if content.name != "Guest":
+            # Delete all books in shelfs belonging to user, all shelfs of user, downloadstat of user, read status
+            # and user itself
+            ub.session.query(ub.ReadBook).filter(ub.User.id == ub.ReadBook.user_id).delete()
+            ub.session.query(ub.Downloads).filter(ub.User.id == ub.Downloads.user_id).delete()
+            for us in ub.session.query(ub.Shelf).filter(ub.User.id == ub.Shelf.user_id):
+                ub.session.query(ub.BookShelf).filter(us.id == ub.BookShelf.shelf).delete()
+            ub.session.query(ub.Shelf).filter(ub.User.id == ub.Shelf.user_id).delete()
             ub.session.query(ub.User).filter(ub.User.id == content.id).delete()
             ub.session_commit()
             log.info(u"User {} deleted".format(content.name))

cps/helper.py

@@ -329,9 +329,10 @@ def delete_book_file(book, calibrepath, book_format=None):
                     except (IOError, OSError) as e:
                         log.error("Deleting authorpath for book %s failed: %s", book.id, e)
                 return True, None
-            else:
+
-                log.error("Deleting book %s failed, book path not valid: %s", book.id, book.path)
+    log.error("Deleting book %s from database only, book path in database not valid: %s",
-                return True, _("Deleting book %(id)s, book path not valid: %(path)s",
+              book.id, book.path)
+    return True, _("Deleting book %(id)s from database only, book path in database not valid: %(path)s",
                    id=book.id,
                    path=book.path)
 
@@ -382,7 +383,7 @@ def update_dir_structure_file(book_id, calibrepath, first_author, orignal_filepa
                             # os.unlink(os.path.normcase(os.path.join(dir_name, file)))
             # change location in database to new author/title path
             localbook.path = os.path.join(new_authordir, new_titledir).replace('\\','/')
-        except OSError as ex:
+        except (OSError) as ex:
             log.error("Rename title from: %s to %s: %s", path, new_path, ex)
             log.debug(ex, exc_info=True)
             return _("Rename title from: '%(src)s' to '%(dest)s' failed with error: %(error)s",
@@ -397,7 +398,7 @@ def update_dir_structure_file(book_id, calibrepath, first_author, orignal_filepa
                 file_format.name = new_name
             if not orignal_filepath and len(os.listdir(os.path.dirname(path))) == 0:
                 shutil.rmtree(os.path.dirname(path))
-        except OSError as ex:
+        except (OSError) as ex:
             log.error("Rename file in path %s to %s: %s", new_path, new_name, ex)
             log.debug(ex, exc_info=True)
             return _("Rename file in path '%(src)s' to '%(dest)s' failed with error: %(error)s",

cps/kobo.py

@@ -44,6 +44,8 @@ from werkzeug.datastructures import Headers
 from sqlalchemy import func
 from sqlalchemy.sql.expression import and_, or_
 from sqlalchemy.exc import StatementError
+from sqlalchemy import __version__ as sql_version
+from sqlalchemy.sql import select
 import requests
 
 from . import config, logger, kobo_auth, db, calibre_db, helper, shelf as shelf_lib, ub
@@ -64,6 +66,7 @@ kobo_auth.register_url_value_preprocessor(kobo)
 
 log = logger.create()
 
+sql2 = ([int(x) for x in sql_version.split('.')] >= [2,0,0])
 
 def get_store_url_for_current_request():
     # Programmatically modify the current url to point to the official Kobo store
@@ -153,14 +156,19 @@ def HandleSyncRequest():
     calibre_db.reconnect_db(config, ub.app_DB_path)
 
     only_kobo_shelves = current_user.kobo_only_shelves_sync
-    # calibre_db.session.query(ub.Shelf).filter(ub.Shelf.user_id == current_user.id).filter(ub.Shelf.kobo_sync).count() > 0
 
     if only_kobo_shelves:
-        changed_entries = (
+        if sql2:
-            calibre_db.session.query(db.Books,
+            changed_entries = select(db.Books,
                                      ub.ArchivedBook.last_modified,
                                      ub.BookShelf.date_added,
                                      ub.ArchivedBook.is_archived)
+        else:
+            changed_entries = calibre_db.session.query(db.Books,
+                                                       ub.ArchivedBook.last_modified,
+                                                       ub.BookShelf.date_added,
+                                                       ub.ArchivedBook.is_archived)
+        changed_entries = (changed_entries
                 .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id)
                 .filter(or_(db.Books.last_modified > sync_token.books_last_modified,
                             ub.BookShelf.date_added > sync_token.books_last_modified))
@@ -169,12 +177,18 @@ def HandleSyncRequest():
                 .order_by(ub.ArchivedBook.last_modified)
                 .join(ub.BookShelf, db.Books.id == ub.BookShelf.book_id)
                 .join(ub.Shelf)
+                .filter(ub.Shelf.user_id == current_user.id)
                 .filter(ub.Shelf.kobo_sync)
                 .distinct()
         )
     else:
-        changed_entries = (
+        if sql2:
-            calibre_db.session.query(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived)
+            changed_entries = select(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived)
+        else:
+            changed_entries = calibre_db.session.query(db.Books,
+                                                       ub.ArchivedBook.last_modified,
+                                                       ub.ArchivedBook.is_archived)
+        changed_entries = (changed_entries
                 .join(db.Data).outerjoin(ub.ArchivedBook, db.Books.id == ub.ArchivedBook.book_id)
                 .filter(db.Books.last_modified > sync_token.books_last_modified)
                 .filter(calibre_db.common_filters())
@@ -187,7 +201,11 @@ def HandleSyncRequest():
         changed_entries = changed_entries.filter(db.Books.id > sync_token.books_last_id)
 
     reading_states_in_new_entitlements = []
-    for book in changed_entries.limit(SYNC_ITEM_LIMIT):
+    if sql2:
+        books = calibre_db.session.execute(changed_entries.limit(SYNC_ITEM_LIMIT))
+    else:
+        books = changed_entries.limit(SYNC_ITEM_LIMIT)
+    for book in books:
         formats = [data.format for data in book.Books.data]
         if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats:
             helper.convert_book_format(book.Books.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name)
@@ -227,6 +245,12 @@ def HandleSyncRequest():
 
         new_books_last_created = max(ts_created, new_books_last_created)
 
+    if sql2:
+        max_change = calibre_db.session.execute(changed_entries
+                                                .filter(ub.ArchivedBook.is_archived)
+                                                .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()))\
+            .columns(db.Books).first()
+    else:
         max_change = changed_entries.from_self().filter(ub.ArchivedBook.is_archived) \
             .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()).first()
 
@@ -235,10 +259,14 @@ def HandleSyncRequest():
     new_archived_last_modified = max(new_archived_last_modified, max_change)
 
     # no. of books returned
+    if sql2:
+        entries = calibre_db.session.execute(changed_entries).all()
+        book_count = len(entries)
+    else:
+        entries = changed_entries.all()
         book_count = changed_entries.count()
-
     # last entry:
-    books_last_id = changed_entries.all()[-1].Books.id or -1 if book_count else -1
+    books_last_id = entries[-1].Books.id or -1 if book_count else -1
 
     # generate reading state data
     changed_reading_states = ub.session.query(ub.KoboReadingState)
@@ -247,10 +275,11 @@ def HandleSyncRequest():
         changed_reading_states = changed_reading_states.join(ub.BookShelf,
                                                              ub.KoboReadingState.book_id == ub.BookShelf.book_id)\
             .join(ub.Shelf)\
+            .filter(current_user.id == ub.Shelf.user_id)\
             .filter(ub.Shelf.kobo_sync,
                     or_(
                         func.datetime(ub.KoboReadingState.last_modified) > sync_token.reading_state_last_modified,
-                        ub.BookShelf.date_added > sync_token.books_last_modified
+                        func.datetime(ub.BookShelf.date_added) > sync_token.books_last_modified
                     )).distinct()
     else:
         changed_reading_states = changed_reading_states.filter(
@@ -666,12 +695,23 @@ def sync_shelves(sync_token, sync_results, only_kobo_shelves=False):
             })
         extra_filters.append(ub.Shelf.kobo_sync)
 
-    for shelf in ub.session.query(ub.Shelf).outerjoin(ub.BookShelf).filter(
+    if sql2:
+        shelflist = ub.session.execute(select(ub.Shelf).outerjoin(ub.BookShelf).filter(
             or_(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified,
-            ub.BookShelf.date_added > sync_token.tags_last_modified),
+                func.datetime(ub.BookShelf.date_added) > sync_token.tags_last_modified),
             ub.Shelf.user_id == current_user.id,
             *extra_filters
-    ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc()):
+        ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc())).columns(ub.Shelf)
+    else:
+        shelflist = ub.session.query(ub.Shelf).outerjoin(ub.BookShelf).filter(
+            or_(func.datetime(ub.Shelf.last_modified) > sync_token.tags_last_modified,
+                func.datetime(ub.BookShelf.date_added) > sync_token.tags_last_modified),
+            ub.Shelf.user_id == current_user.id,
+            *extra_filters
+        ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc())
+
+
+    for shelf in shelflist:
         if not shelf_lib.check_shelf_view_permissions(shelf):
             continue