Initial gdrive commit

Work on watching metadata

More efficient storing folder keys to database

Nearly completed. Need to do final touches to callback for when metadata.db updated on real server, as cannot test locally

Changed callback for file changes from being hard coded to mine

used url_for in template as apposed to hard coded links

Fix to drive template

First attempt at redownload metadata.db

Fixed incorrect call to downloadFile

Added logging

Fixed call to copy file

Added exception logging to gdriveutils + fixed string long concat

Fix file download

Fix backup metadata

Added slashes to paths

Removed threading temporarily

Fix for reloading database

Fix reinitialising of variables

Fix check to see if custom column already setup

Update to showing authenticate google drive callback + fix for reinitialising database

Fixed logic for showing authenticate with google drive

cps/db.py

@@ -12,9 +12,9 @@ import ub
 
 session = None
 cc_exceptions = None
-cc_classes = None
-cc_ids = None
-books_custom_column_links = None
+cc_classes = {}
+cc_ids = []
+books_custom_column_links = {}
 engine = None
 
 
@@ -293,41 +293,40 @@ def setup_db():
 
     cc = conn.execute("SELECT id, datatype FROM custom_columns")
 
-    cc_ids = []
     cc_exceptions = ['datetime', 'int', 'comments', 'float', 'composite', 'series']
-    books_custom_column_links = {}
-    cc_classes = {}
     for row in cc:
         if row.datatype not in cc_exceptions:
-            books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata,
+            if row.id not in books_custom_column_links:
+                books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata,
                                                       Column('book', Integer, ForeignKey('books.id'),
                                                              primary_key=True),
                                                       Column('value', Integer,
                                                              ForeignKey('custom_column_' + str(row.id) + '.id'),
                                                              primary_key=True)
                                                       )
-            cc_ids.append([row.id, row.datatype])
-            if row.datatype == 'bool':
-                ccdict = {'__tablename__': 'custom_column_' + str(row.id),
-                          'id': Column(Integer, primary_key=True),
-                          'book': Column(Integer, ForeignKey('books.id')),
-                          'value': Column(Boolean)}
-            else:
-                ccdict = {'__tablename__': 'custom_column_' + str(row.id),
-                          'id': Column(Integer, primary_key=True),
-                          'value': Column(String)}
-            cc_classes[row.id] = type('Custom_Column_' + str(row.id), (Base,), ccdict)
+                cc_ids.append([row.id, row.datatype])
+                if row.datatype == 'bool':
+                    ccdict = {'__tablename__': 'custom_column_' + str(row.id),
+                              'id': Column(Integer, primary_key=True),
+                              'book': Column(Integer, ForeignKey('books.id')),
+                              'value': Column(Boolean)}
+                else:
+                    ccdict = {'__tablename__': 'custom_column_' + str(row.id),
+                              'id': Column(Integer, primary_key=True),
+                              'value': Column(String)}
+                cc_classes[row.id] = type('Custom_Column_' + str(row.id), (Base,), ccdict)
 
     for id in cc_ids:
-        if id[1] == 'bool':
-            setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]],
-                                                                       primaryjoin=(
-                                                                       Books.id == cc_classes[id[0]].book),
-                                                                       backref='books'))
-        else:
-            setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]],
-                                                                       secondary=books_custom_column_links[id[0]],
-                                                                       backref='books'))
+        if not hasattr(Books, 'custom_column_' + str(id[0])):
+            if id[1] == 'bool':
+                setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]],
+                                                                           primaryjoin=(
+                                                                           Books.id == cc_classes[id[0]].book),
+                                                                           backref='books'))
+            else:
+                setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]],
+                                                                           secondary=books_custom_column_links[id[0]],
+                                                                           backref='books'))
 
     # Base.metadata.create_all(engine)
     Session = sessionmaker()

cps/gdriveutils.py

@@ -0,0 +1,313 @@
+from pydrive.auth import GoogleAuth
+from pydrive.drive import GoogleDrive
+import os, time
+
+from ub import config
+
+from sqlalchemy import *
+from sqlalchemy import exc
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import *
+
+from apiclient import errors
+
+import web
+
+
+dbpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep), "gdrive.db")
+engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False)
+Base = declarative_base()
+
+# Open session for database connection
+Session = sessionmaker()
+Session.configure(bind=engine)
+session = Session()
+
+class GdriveId(Base):
+    __tablename__='gdrive_ids'
+
+    id = Column(Integer, primary_key=True)
+    gdrive_id = Column(Integer, unique=True)
+    path = Column(String)
+
+    def __repr__(self):
+        return str(self.path)
+
+if not os.path.exists(dbpath):
+    try:
+        Base.metadata.create_all(engine)
+    except Exception:
+        raise
+
+def getDrive(gauth=None):
+    if not gauth:
+        gauth=GoogleAuth(settings_file='settings.yaml')
+    # Try to load saved client credentials
+    gauth.LoadCredentialsFile("gdrive_credentials")
+    if gauth.access_token_expired:
+        # Refresh them if expired
+        gauth.Refresh()
+    else:
+        # Initialize the saved creds
+        gauth.Authorize()
+    # Save the current credentials to a file
+    return GoogleDrive(gauth)
+
+def getEbooksFolder(drive=None):
+    if not drive:
+        drive = getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    ebooksFolder= "title = '%s' and 'root' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false" % config.config_google_drive_folder
+
+    fileList = drive.ListFile({'q': ebooksFolder}).GetList()
+    return fileList[0]
+
+def getEbooksFolderId(drive=None):
+    storedPathName=session.query(GdriveId).filter(GdriveId.path == '/').first()
+    if storedPathName:
+        return storedPathName.gdrive_id
+    else:
+        gDriveId=GdriveId()
+        gDriveId.gdrive_id=getEbooksFolder(drive)['id']
+        gDriveId.path='/'
+        session.merge(gDriveId)
+        session.commit()
+        return 
+
+def getFolderInFolder(parentId, folderName, drive=None):
+    if not drive:
+        drive = getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    folder= "title = '%s' and '%s' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false"  % (folderName.replace("'", "\\'"), parentId)
+    fileList = drive.ListFile({'q': folder}).GetList()
+    return fileList[0]    
+
+def getFile(pathId, fileName, drive=None):
+    if not drive:
+        drive = getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    metaDataFile="'%s' in parents and trashed = false and title = '%s'" % (pathId, fileName.replace("'", "\\'"))
+
+    fileList = drive.ListFile({'q': metaDataFile}).GetList()
+    return fileList[0]
+
+def getFolderId(path, drive=None):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    currentFolderId=getEbooksFolderId(drive)
+    sqlCheckPath=path if path[-1] =='/' else path + '/'
+    storedPathName=session.query(GdriveId).filter(GdriveId.path == sqlCheckPath).first()
+
+    if not storedPathName:
+        dbChange=False
+        s=path.split('/')
+        for i, x in enumerate(s):
+            if len(x) > 0:
+                currentPath="/".join(s[:i+1])
+                if currentPath[-1] != '/':
+                    currentPath = currentPath + '/'
+                storedPathName=session.query(GdriveId).filter(GdriveId.path == currentPath).first()
+                if storedPathName:
+                    currentFolderId=storedPathName.gdrive_id
+                else:
+                    currentFolderId=getFolderInFolder(currentFolderId, x, drive)['id']
+                    gDriveId=GdriveId()
+                    gDriveId.gdrive_id=currentFolderId
+                    gDriveId.path=currentPath
+                    session.merge(gDriveId)
+                    dbChange=True
+        if dbChange:
+            session.commit()
+    else:
+        currentFolderId=storedPathName.gdrive_id
+    return currentFolderId
+
+
+def getFileFromEbooksFolder(drive, path, fileName):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    if path:
+        sqlCheckPath=path if path[-1] =='/' else path + '/'
+        folderId=getFolderId(path, drive)
+    else:
+        folderId=getEbooksFolderId(drive)
+    
+    return getFile(folderId, fileName, drive)
+
+def copyDriveFileRemote(drive, origin_file_id, copy_title):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    copied_file = {'title': copy_title}
+    try:
+        file_data = drive.auth.service.files().copy(
+        fileId=origin_file_id, body=copied_file).execute()
+        return drive.CreateFile({'id': file_data['id']})
+    except errors.HttpError as error:
+        print ('An error occurred: %s' % error)
+    return None
+
+def downloadFile(drive, path, filename, output):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    f=getFileFromEbooksFolder(drive, path, filename)
+    f.GetContentFile(output)
+
+def backupCalibreDbAndOptionalDownload(drive, f=None):
+    pass
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    metaDataFile="'%s' in parents and title = 'metadata.db' and trashed = false" % getEbooksFolderId()
+
+    fileList = drive.ListFile({'q': metaDataFile}).GetList()
+ 
+    databaseFile=fileList[0]
+
+    if f:
+        databaseFile.GetContentFile(f)
+
+def copyToDrive(drive, uploadFile, createRoot, replaceFiles,
+    ignoreFiles=[],
+    parent=None, prevDir=''):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    isInitial=not bool(parent)
+    if not parent:
+        parent=getEbooksFolder(drive)
+    if os.path.isdir(os.path.join(prevDir,uploadFile)):
+        existingFolder=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList()
+        if len(existingFolder) == 0 and (not isInitial or createRoot):
+            parent = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], 
+                "mimeType": "application/vnd.google-apps.folder" })
+            parent.Upload()
+        else:
+            if (not isInitial or createRoot) and len(existingFolder) > 0:
+                parent=existingFolder[0]
+        for f in os.listdir(os.path.join(prevDir,uploadFile)):
+            if f not in ignoreFiles:
+                copyToDrive(drive, f, True, replaceFiles, ignoreFiles, parent, os.path.join(prevDir,uploadFile))
+    else:
+        if os.path.basename(uploadFile) not in ignoreFiles:
+            existingFiles=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList()
+            if len(existingFiles) > 0:
+                driveFile = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], })
+            else:
+                driveFile=existingFiles[0]
+            driveFile.SetContentFile(os.path.join(prevDir,uploadFile))
+            driveFile.Upload()
+
+def watchChange(drive, channel_id, channel_type, channel_address,
+              channel_token=None, expiration=None):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    """Watch for all changes to a user's Drive.
+    Args:
+    service: Drive API service instance.
+    channel_id: Unique string that identifies this channel.
+    channel_type: Type of delivery mechanism used for this channel.
+    channel_address: Address where notifications are delivered.
+    channel_token: An arbitrary string delivered to the target address with
+                   each notification delivered over this channel. Optional.
+    channel_address: Address where notifications are delivered. Optional.
+    Returns:
+    The created channel if successful
+    Raises:
+    apiclient.errors.HttpError: if http request to create channel fails.
+    """
+    body = {
+    'id': channel_id,
+    'type': channel_type,
+    'address': channel_address
+    }
+    if channel_token:
+        body['token'] = channel_token
+    if expiration:
+        body['expiration'] = expiration
+    return  drive.auth.service.changes().watch(body=body).execute()
+
+def watchFile(drive, file_id, channel_id, channel_type, channel_address,
+              channel_token=None, expiration=None):
+    """Watch for any changes to a specific file.
+    Args:
+    service: Drive API service instance.
+    file_id: ID of the file to watch.
+    channel_id: Unique string that identifies this channel.
+    channel_type: Type of delivery mechanism used for this channel.
+    channel_address: Address where notifications are delivered.
+    channel_token: An arbitrary string delivered to the target address with
+                   each notification delivered over this channel. Optional.
+    channel_address: Address where notifications are delivered. Optional.
+    Returns:
+    The created channel if successful
+    Raises:
+    apiclient.errors.HttpError: if http request to create channel fails.
+    """
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+
+    body = {
+        'id': channel_id,
+        'type': channel_type,
+        'address': channel_address
+    }
+    if channel_token:
+        body['token'] = channel_token
+    if expiration:
+        body['expiration'] = expiration
+    return drive.auth.service.files().watch(fileId=file_id, body=body).execute()
+
+def stopChannel(drive, channel_id, resource_id):
+    """Stop watching to a specific channel.
+    Args:
+    service: Drive API service instance.
+    channel_id: ID of the channel to stop.
+    resource_id: Resource ID of the channel to stop.
+    Raises:
+    apiclient.errors.HttpError: if http request to create channel fails.
+    """
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh() 
+    service=drive.auth.service
+    body = {
+    'id': channel_id,
+    'resourceId': resource_id
+    }
+    return drive.auth.service.channels().stop(body=body).execute()
+
+def getChangeById (drive, change_id):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh() 
+    """Print a single Change resource information.
+
+    Args:
+    service: Drive API service instance.
+    change_id: ID of the Change resource to retrieve.
+    """
+    try:
+        change = drive.auth.service.changes().get(changeId=change_id).execute()
+        return change
+    except errors.HttpError, error:
+        web.app.logger.exception(error) 
+        return None

cps/templates/config_edit.html

@@ -7,6 +7,45 @@
       <label for="config_calibre_dir">{{_('Location of Calibre database')}}</label>
       <input type="text" class="form-control" name="config_calibre_dir" id="config_calibre_dir" value="{% if content.config_calibre_dir != None %}{{ content.config_calibre_dir }}{% endif %}" autocomplete="off">
     </div>
+    <div class="form-group required">
+      <label for="config_use_google_drive">{{_('Use google drive?')}}</label>
+      <input type="checkbox" id="config_use_google_drive" class="form-control" name="config_use_google_drive" id="config_use_google_drive" {% if content.config_use_google_drive %}checked{% endif %} >
+    </div>
+    <div id="gdrive_settings">
+      <div class="form-group required">
+        <label for="config_google_drive_client_id">{{_('Client id')}}</label>
+        <input type="text" class="form-control" name="config_google_drive_client_id" id="config_google_client_id" value="{% if content.config_google_drive_client_id %}{{content.config_google_drive_client_id}}{% endif %}" autocomplete="off">
+      </div>
+      <div class="form-group required">
+        <label for="config_google_drive_client_secret">{{_('Client secret')}}</label>
+        <input type="text" class="form-control" name="config_google_drive_client_secret" id="config_google_drive_client_secret" value="{% if content.config_google_drive_client_secret %}{{content.config_google_drive_client_secret}}{% endif %}" autocomplete="off">
+      </div>
+      <div class="form-group required">
+        <label for="config_google_drive_calibre_url_base">{{_('Calibre Base URL')}}</label>
+        <input type="text" class="form-control" name="config_google_drive_calibre_url_base" id="config_google_drive_calibre_url_base" value="{% if content.config_google_drive_calibre_url_base %}{{content.config_google_drive_calibre_url_base}}{% endif %}" autocomplete="off">
+      </div>
+      <div class="form-group required">
+        <label for="config_google_drive_folder">{{_('Google drive Calibre folder')}}</label>
+        <input type="text" class="form-control" name="config_google_drive_folder" id="config_google_drive_folder" value="{% if content.config_google_drive_folder %}{{content.config_google_drive_folder}}{% endif %}" autocomplete="off" required>
+      </div>
+      {% if show_authenticate_google_drive %}
+        <div class="form-group required">
+          <a href="{{ url_for('authenticate_google_drive') }}" class="btn btn-primary">Authenticate Google Drive</a>
+        </div>
+      {% else %}
+        {% if content.config_google_drive_watch_changes_response %}
+        <label for="config_google_drive_watch_changes_response">{{_('Metadata Watch Channel ID')}}</label>
+        <div class="form-group input-group required">
+          <input type="text" class="form-control" name="config_google_drive_watch_changes_response" id="config_google_drive_watch_changes_response" value="{{ content.config_google_drive_watch_changes_response['id'] }} expires on {{ content.config_google_drive_watch_changes_response['expiration'] |  strftime }}" autocomplete="off" disabled="">
+          <span class="input-group-btn">
+          <a href="{{ url_for('revoke_watch_gdrive') }}" class="btn btn-primary">Revoke</a>
+          </span>
+          {% else %}
+          <a href="{{ url_for('watch_gdrive') }}" class="btn btn-primary">Enable watch of metadata.db</a>
+          {% endif %}
+        </div>
+        {% endif %}
+    </div>
     <div class="form-group">
       <label for="config_port">{{_('Server Port')}}</label>
       <input type="number" min="1" max="65535" class="form-control" name="config_port" id="config_port" value="{% if content.config_port != None %}{{ content.config_port }}{% endif %}" autocomplete="off" required>
@@ -80,3 +119,22 @@
   </form>
 </div>
 {% endblock %}
+{% block js %}
+<script>
+$(document).ready(function() {
+  $('#config_use_google_drive').trigger("change");
+});
+$('#config_use_google_drive').change(function(){
+    formInputs=$("#gdrive_settings :input");
+    isChecked=this.checked;
+    formInputs.each(function(formInput) {
+      $(this).prop('required',isChecked);
+    });
+    if (this.checked) {
+      $('#gdrive_settings').show();
+    } else {
+      $('#gdrive_settings').hide();
+    }
+});
+</script>
+{% endblock %}

cps/ub.py

@@ -11,6 +11,7 @@ import traceback
 import logging
 from werkzeug.security import generate_password_hash
 from flask_babel import gettext as _
+import json
 
 dbpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep), "app.db")
 engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False)
@@ -269,6 +270,12 @@ class Settings(Base):
     config_anonbrowse = Column(SmallInteger, default=0)
     config_public_reg = Column(SmallInteger, default=0)
     config_default_role = Column(SmallInteger, default=0)
+    config_use_google_drive = Column(Boolean)
+    config_google_drive_client_id = Column(String)
+    config_google_drive_client_secret = Column(String)
+    config_google_drive_folder = Column(String)
+    config_google_drive_calibre_url_base = Column(String)
+    config_google_drive_watch_changes_response = Column(String)
 
     def __repr__(self):
         pass
@@ -295,6 +302,16 @@ class Config:
         self.config_anonbrowse = data.config_anonbrowse
         self.config_public_reg = data.config_public_reg
         self.config_default_role = data.config_default_role
+        self.config_use_google_drive = data.config_use_google_drive
+        self.config_google_drive_client_id = data.config_google_drive_client_id
+        self.config_google_drive_client_secret = data.config_google_drive_client_secret
+        self.config_google_drive_calibre_url_base = data.config_google_drive_calibre_url_base
+        self.config_google_drive_folder = data.config_google_drive_folder
+        if data.config_google_drive_watch_changes_response:
+            self.config_google_drive_watch_changes_response = json.loads(data.config_google_drive_watch_changes_response)
+        else:
+            self.config_google_drive_watch_changes_response=None
+
         if self.config_calibre_dir is not None:
             self.db_configured = True
         else:
@@ -379,6 +396,17 @@ def migrate_Database():
         conn.execute("ALTER TABLE Settings ADD column `config_anonbrowse` SmallInteger DEFAULT 0")
         conn.execute("ALTER TABLE Settings ADD column `config_public_reg` SmallInteger DEFAULT 0")
         session.commit()
+
+    try:
+        session.query(exists().where(Settings.config_use_google_drive)).scalar()
+    except exc.OperationalError:
+        conn = engine.connect()
+        conn.execute("ALTER TABLE Settings ADD column `config_use_google_drive` INTEGER DEFAULT 0")
+        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_id` String DEFAULT ''")
+        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_secret` String DEFAULT ''")
+        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_calibre_url_base` INTEGER DEFAULT 0")
+        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_folder` String DEFAULT ''")
+        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_watch_changes_response` String DEFAULT ''")
     try:
         session.query(exists().where(Settings.config_default_role)).scalar()
         session.commit()

cps/web.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+from pydrive.auth import GoogleAuth
+
 import mimetypes
 import logging
 from logging.handlers import RotatingFileHandler
 from tempfile import gettempdir
 import textwrap
 from flask import Flask, render_template, session, request, Response, redirect, url_for, send_from_directory, \
-    make_response, g, flash, abort
+        make_response, g, flash, abort, send_file
 import ub
 from ub import config
 import helper
@@ -42,6 +44,15 @@ import db
 from shutil import move, copyfile
 from tornado.ioloop import IOLoop
 import StringIO
+from shutil import move
+import gdriveutils
+import io
+import hashlib
+import threading
+
+import time
+
+current_milli_time = lambda: int(round(time.time() * 1000))
 
 try:
     from wand.image import Image
@@ -52,13 +63,67 @@ except ImportError, e:
 from cgi import escape
 
 # Global variables
+gdrive_watch_callback_token='target=calibreweb-watch_files'
 global_task = None
 
+def md5(fname):
+    hash_md5 = hashlib.md5()
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+class Singleton:
+    """
+    A non-thread-safe helper class to ease implementing singletons.
+    This should be used as a decorator -- not a metaclass -- to the
+    class that should be a singleton.
+
+    The decorated class can define one `__init__` function that
+    takes only the `self` argument. Also, the decorated class cannot be
+    inherited from. Other than that, there are no restrictions that apply
+    to the decorated class.
+
+    To get the singleton instance, use the `Instance` method. Trying
+    to use `__call__` will result in a `TypeError` being raised.
+
+    """
+
+    def __init__(self, decorated):
+        self._decorated = decorated
+
+    def Instance(self):
+        """
+        Returns the singleton instance. Upon its first call, it creates a
+        new instance of the decorated class and calls its `__init__` method.
+        On all subsequent calls, the already created instance is returned.
+
+        """
+        try:
+            return self._instance
+        except AttributeError:
+            self._instance = self._decorated()
+            return self._instance
+
+    def __call__(self):
+        raise TypeError('Singletons must be accessed through `Instance()`.')
+
+    def __instancecheck__(self, inst):
+        return isinstance(inst, self._decorated)
+
+@Singleton
+class Gauth:
+    def __init__(self):
+        self.auth=GoogleAuth(settings_file='settings.yaml')
+
+@Singleton
+class Gdrive:
+    def __init__(self):
+        self.drive=gdriveutils.getDrive(Gauth.Instance().auth)
 
-# Proxy Helper class
 class ReverseProxied(object):
     """Wrap the application in this middleware and configure the
-    front-end server to add these headers, to let you quietly bind 
+    front-end server to add these headers, to let you quietly bind
     this to a URL other than / and to an HTTP scheme that is 
     different than what is used locally.
 
@@ -187,6 +252,12 @@ def authenticate():
         'You have to login with proper credentials', 401,
         {'WWW-Authenticate': 'Basic realm="Login Required"'})
 
+def updateGdriveCalibreFromLocal():
+    gdriveutils.backupCalibreDbAndOptionalDownload(Gdrive.Instance().drive)
+    gdriveutils.copyToDrive(Gdrive.Instance().drive, config.config_calibre_dir, False, True)
+    for x in os.listdir(config.config_calibre_dir):
+        if os.path.isdir(os.path.join(config.config_calibre_dir,x)):
+            shutil.rmtree(os.path.join(config.config_calibre_dir,x))
 
 def requires_basic_auth_if_no_ano(f):
     @wraps(f)
@@ -286,6 +357,17 @@ def formatdate(val):
     formatdate = datetime.datetime.strptime(conformed_timestamp[:15], "%Y%m%d %H%M%S")
     return format_date(formatdate, format='medium',locale=get_locale())
 
+@app.template_filter('strftime')
+def timestamptodate(date, fmt=None):
+    date=datetime.datetime.fromtimestamp(
+        int(date)/1000
+    )
+    native = date.replace(tzinfo=None)
+    if fmt:
+        format=fmt
+    else:
+        format='%d %m %Y - %H:%S'
+    return native.strftime(format)
 
 def admin_required(f):
     """
@@ -668,8 +750,15 @@ def get_opds_download_link(book_id, format):
     file_name = book.title
     if len(book.authors) > 0:
         file_name = book.authors[0].name + '-' + file_name
-    file_name = helper.get_valid_filename(file_name)
-    response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
+
+    if config.config_use_google_drive:
+        df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, '%s.%s' % (data.name, format))
+        download_url = df.metadata.get('downloadUrl')
+        resp, content = df.auth.Get_Http_Object().request(download_url)
+        response=send_file(io.BytesIO(content))
+    else:
+        file_name = helper.get_valid_filename(file_name)
+        response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
     response.headers["Content-Disposition"] = "attachment; filename=\"%s.%s\"" % (data.name, format)
     return response
 
@@ -802,7 +891,9 @@ def hot_books(page):
     hot_books = all_books.offset(off).limit(config.config_books_per_page)
     entries = list()
     for book in hot_books:
-        entries.append(db.session.query(db.Books).filter(filter).filter(db.Books.id == book.Downloads.book_id).first())
+        entry=db.session.query(db.Books).filter(filter).filter(db.Books.id == book.Downloads.book_id).first()
+        if entry:
+            entries.append(entry)
     numBooks = entries.__len__()
     pagination = Pagination(page, config.config_books_per_page, numBooks)
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
@@ -1037,6 +1128,99 @@ def stats():
                                  categorycounter=categorys, seriecounter=series, title=_(u"Statistics"))
 
 
+#@app.route("/load_gdrive")
+#@login_required
+#@admin_required
+#def load_all_gdrive_folder_ids():
+#    books=db.session.query(db.Books).all()
+#    for book in books:
+#        gdriveutils.getFolderId(book.path, Gdrive.Instance().drive)
+#    return 
+
+@app.route("/gdrive/authenticate")
+@login_required
+@admin_required
+def authenticate_google_drive():
+    authUrl=Gauth.Instance().auth.GetAuthUrl()
+    return redirect(authUrl)
+
+@app.route("/gdrive/callback")
+def google_drive_callback():
+    auth_code = request.args.get('code')
+    credentials = Gauth.Instance().auth.flow.step2_exchange(auth_code)
+    with open('gdrive_credentials' ,'w') as f:
+        f.write(credentials.to_json())
+    return redirect(url_for('configuration'))
+
+@app.route("/gdrive/watch/subscribe")
+@login_required
+@admin_required
+def watch_gdrive():
+    if not config.config_google_drive_watch_changes_response:
+        address = '%scalibre-web/gdrive/watch/callback' % config.config_google_drive_calibre_url_base
+        notification_id=str(uuid4())
+        result = gdriveutils.watchChange(Gdrive.Instance().drive, notification_id,
+                               'web_hook', address, gdrive_watch_callback_token, current_milli_time() + 604800*1000)
+        print (result)
+        settings = ub.session.query(ub.Settings).first()
+        settings.config_google_drive_watch_changes_response=json.dumps(result)
+        ub.session.merge(settings)
+        ub.session.commit()
+        settings = ub.session.query(ub.Settings).first()
+        config.loadSettings()
+
+        print (settings.config_google_drive_watch_changes_response)
+
+    return redirect(url_for('configuration'))
+
+@app.route("/gdrive/watch/revoke")
+@login_required
+@admin_required
+def revoke_watch_gdrive():
+    last_watch_response=config.config_google_drive_watch_changes_response
+    if last_watch_response:
+        response=gdriveutils.stopChannel(Gdrive.Instance().drive, last_watch_response['id'], last_watch_response['resourceId'])
+        settings = ub.session.query(ub.Settings).first()
+        settings.config_google_drive_watch_changes_response=None
+        ub.session.merge(settings)
+        ub.session.commit()
+        config.loadSettings()
+    return redirect(url_for('configuration'))
+
+@app.route("/gdrive/watch/callback", methods=['GET', 'POST'])
+def on_received_watch_confirmation():
+    app.logger.info (request.headers)
+    if request.headers.get('X-Goog-Channel-Token') == gdrive_watch_callback_token \
+        and request.headers.get('X-Goog-Resource-State') == 'change' \
+        and request.data:
+
+        data=request.data
+
+        def updateMetaData():
+            app.logger.info ('Change received from gdrive')
+            app.logger.info (data) 
+            try:
+                j=json.loads(data)
+                app.logger.info ('Getting change details') 
+                response=gdriveutils.getChangeById(Gdrive.Instance().drive, j['id'])
+                app.logger.info (response)
+                if response:
+                    dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
+                    if not response['deleted'] and response['file']['title'] == 'metadata.db' and response['file']['md5Checksum'] != md5(dbpath):
+                        app.logger.info ('Database file updated')
+                        copyfile (dbpath, config.config_calibre_dir + "/metadata.db_" + str(current_milli_time()))
+                        app.logger.info ('Backing up existing and downloading updated metadata.db')
+                        gdriveutils.downloadFile(Gdrive.Instance().drive, None, "metadata.db", config.config_calibre_dir + "/tmp_metadata.db")
+                        app.logger.info ('Setting up new DB')
+                        os.rename(config.config_calibre_dir + "/tmp_metadata.db", dbpath)
+                        db.setup_db()
+            except Exception, e:
+                app.logger.exception(e)
+
+        updateMetaData()
+    return ''
+
+
 @app.route("/shutdown")
 @login_required
 @admin_required
@@ -1173,8 +1357,15 @@ def advanced_search():
 @app.route("/cover/<path:cover_path>")
 @login_required_if_no_ano
 def get_cover(cover_path):
-    return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg")
+    if config.config_use_google_drive:
+        df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, cover_path, 'cover.jpg')
+        download_url = df.metadata.get('webContentLink')
+        return redirect(download_url)
+    else:
+        return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg")
+    resp.headers['Content-Type']='image/jpeg'
 
+    return resp
 
 @app.route("/opds/thumb_240_240/<path:book_id>")
 @app.route("/opds/cover_240_240/<path:book_id>")
@@ -1183,7 +1374,12 @@ def get_cover(cover_path):
 @requires_basic_auth_if_no_ano
 def feed_get_cover(book_id):
     book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
-    return send_from_directory(os.path.join(config.config_calibre_dir, book.path), "cover.jpg")
+    if config.config_use_google_drive:
+        df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, cover_path, 'cover.jpg')
+        download_url = df.metadata.get('webContentLink')
+        return redirect(download_url)
+    else:
+        return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg")
 
 def render_read_books(page, are_read, as_xml=False):
     readBooks=ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id)).filter(ub.ReadBook.is_read == True).all()
@@ -1308,8 +1504,13 @@ def get_download_link(book_id, format):
         if len(book.authors) > 0:
             file_name = book.authors[0].name + '-' + file_name
         file_name = helper.get_valid_filename(file_name)
-        response = make_response(
-            send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
+        if config.config_use_google_drive:
+            df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, '%s.%s' % (data.name, format))
+            download_url = df.metadata.get('downloadUrl')
+            resp, content = df.auth.Get_Http_Object().request(download_url)
+            response=send_file(io.BytesIO(content))
+        else:
+            response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
         try:
             response.headers["Content-Type"] = mimetypes.types_map['.' + format]
         except:
@@ -1682,6 +1883,38 @@ def configuration_helper(origin):
             if content.config_calibre_dir != to_save["config_calibre_dir"]:
                 content.config_calibre_dir = to_save["config_calibre_dir"]
                 db_change = True
+        ##Google drive setup
+        create_new_yaml=False
+        if "config_google_drive_client_id" in to_save:
+            if content.config_google_drive_client_id != to_save["config_google_drive_client_id"]:
+                content.config_google_drive_client_id = to_save["config_google_drive_client_id"]
+                create_new_yaml=True
+                db_change = True
+        if "config_google_drive_client_secret" in to_save:
+            if content.config_google_drive_client_secret != to_save["config_google_drive_client_secret"]:
+                content.config_google_drive_client_secret = to_save["config_google_drive_client_secret"]
+                create_new_yaml=True
+                db_change = True
+        if "config_google_drive_calibre_url_base" in to_save:
+            if content.config_google_drive_calibre_url_base != to_save["config_google_drive_calibre_url_base"]:
+                content.config_google_drive_calibre_url_base = to_save["config_google_drive_calibre_url_base"]
+                create_new_yaml=True
+                db_change = True
+        if ("config_use_google_drive" in to_save and not content.config_use_google_drive) or ("config_use_google_drive" not in to_save and content.config_use_google_drive):
+            content.config_use_google_drive = "config_use_google_drive" in to_save
+            db_change = True
+            if not content.config_use_google_drive:
+                create_new_yaml=False
+        if create_new_yaml:
+            with open('settings.yaml', 'w') as f:
+                with open('gdrive_template.yaml' ,'r') as t:
+                    f.write(t.read() % {'client_id' : content.config_google_drive_client_id, 'client_secret' : content.config_google_drive_client_secret,
+                     "redirect_uri" : content.config_google_drive_calibre_url_base + 'gdrive/callback'})
+        if "config_google_drive_folder" in to_save:
+            if content.config_google_drive_folder != to_save["config_google_drive_folder"]:
+                content.config_google_drive_folder = to_save["config_google_drive_folder"]
+                db_change = True
+        ##
         if "config_port" in to_save:
             if content.config_port != int(to_save["config_port"]):
                 content.config_port = int(to_save["config_port"])
@@ -1751,6 +1984,7 @@ def configuration_helper(origin):
         if origin:
             success = True
     return render_title_template("config_edit.html", origin=origin, success=success, content=config,
+                                 show_authenticate_google_drive=not os.path.exists('settings.yaml') or not os.path.exists('gdrive_credentials'),
                                  title=_(u"Basic Configuration"))
 
 
@@ -1999,7 +2233,7 @@ def edit_book(book_id):
             modify_database_object(input_authors, book.authors, db.Authors, db.session, 'author')
             if author0_before_edit != book.authors[0].name:
                 edited_books_id.add(book.id)
-                book.author_sort=helper.get_sorted_author(input_authors[0]) 
+                book.author_sort=helper.get_sorted_author(input_authors[0])
 
             if to_save["cover_url"] and os.path.splitext(to_save["cover_url"])[1].lower() == ".jpg":
                 img = requests.get(to_save["cover_url"])
@@ -2163,6 +2397,8 @@ def edit_book(book_id):
                 author_names.append(author.name)
             for b in edited_books_id:
                 helper.update_dir_stucture(b, config.config_calibre_dir)
+            if config.config_use_google_drive:
+                updateGdriveCalibreFromLocal()
             if "detail_view" in to_save:
                 return redirect(url_for('show_book', id=book.id))
             else:
@@ -2227,7 +2463,7 @@ def upload():
         if is_author:
             db_author = is_author
         else:
-            db_author = db.Authors(author, helper.get_sorted_author(author), "") 
+            db_author = db.Authors(author, helper.get_sorted_author(author), "")
             db.session.add(db_author)
         # combine path and normalize path from windows systems
         path = os.path.join(author_dir, title_dir).replace('\\','/')
@@ -2242,6 +2478,9 @@ def upload():
         author_names = []
         for author in db_book.authors:
             author_names.append(author.name)
+        if config.config_use_google_drive:
+            if not current_user.role_edit() and not current_user.role_admin():
+                updateGdriveCalibreFromLocal()
     cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
     if current_user.role_edit() or current_user.role_admin():
         return render_title_template('book_edit.html', book=db_book, authors=author_names, cc=cc,

gdrive_template.yaml

@@ -0,0 +1,14 @@
+client_config_backend: settings
+client_config:
+  client_id: %(client_id)s
+  client_secret: %(client_secret)s
+  redirect_uri: %(redirect_uri)s
+  
+save_credentials: True
+save_credentials_backend: file
+save_credentials_file: gdrive_credentials
+
+get_refresh_token: True
+
+oauth_scope:
+  - https://www.googleapis.com/auth/drive