calibre-web/cps/updater.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
#    Copyright (C) 2018-2019 OzzieIsaacs
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program. If not, see <http://www.gnu.org/licenses/>.


from cps import config, get_locale, Server, app
import threading
import zipfile
import requests
import time
from io import BytesIO
import os
import sys
import shutil
from ub import UPDATE_STABLE
from tempfile import gettempdir
import datetime
import json
from flask_babel import gettext as _
from babel.dates import format_datetime


def is_sha1(sha1):
    if len(sha1) != 40:
        return False
    try:
        int(sha1, 16)
    except ValueError:
        return False
    return True


class Updater(threading.Thread):

    def __init__(self):
        threading.Thread.__init__(self)
        self.status = 0
        self.updateIndex = None

    def get_current_version_info(self):
        if config.get_update_channel == UPDATE_STABLE:
            return self._stable_version_info()
        else:
            return self._nightly_version_info()

    def get_available_updates(self, request_method):
        if config.get_update_channel == UPDATE_STABLE:
            return self._stable_available_updates(request_method)
        else:
            return self._nightly_available_updates(request_method)

    def run(self):
        try:
            self.status = 1
            app.logger.debug(u'Download update file')
            headers = {'Accept': 'application/vnd.github.v3+json'}
            r = requests.get(self._get_request_path(), stream=True, headers=headers)
            r.raise_for_status()

            self.status = 2
            app.logger.debug(u'Opening zipfile')
            z = zipfile.ZipFile(BytesIO(r.content))
            self.status = 3
            app.logger.debug(u'Extracting zipfile')
            tmp_dir = gettempdir()
            z.extractall(tmp_dir)
            foldername = os.path.join(tmp_dir, z.namelist()[0])[:-1]
            if not os.path.isdir(foldername):
                self.status = 11
                app.logger.info(u'Extracted contents of zipfile not found in temp folder')
                return
            self.status = 4
            app.logger.debug(u'Replacing files')
            self.update_source(foldername, config.get_main_dir)
            self.status = 6
            app.logger.debug(u'Preparing restart of server')
            time.sleep(2)
            Server.setRestartTyp(True)
            Server.stopServer()
            self.status = 7
            time.sleep(2)
        except requests.exceptions.HTTPError as ex:
            app.logger.info( u'HTTP Error' + ' ' + str(ex))
            self.status = 8
        except requests.exceptions.ConnectionError:
            app.logger.info(u'Connection error')
            self.status = 9
        except requests.exceptions.Timeout:
            app.logger.info(u'Timeout while establishing connection')
            self.status = 10
        except requests.exceptions.RequestException:
            self.status = 11
            app.logger.info(u'General error')

    def get_update_status(self):
        return self.status

    @classmethod
    def file_to_list(self, filelist):
        return [x.strip() for x in open(filelist, 'r') if not x.startswith('#EXT')]

    @classmethod
    def one_minus_two(self, one, two):
        return [x for x in one if x not in set(two)]

    @classmethod
    def reduce_dirs(self, delete_files, new_list):
        new_delete = []
        for filename in delete_files:
            parts = filename.split(os.sep)
            sub = ''
            for part in parts:
                sub = os.path.join(sub, part)
                if sub == '':
                    sub = os.sep
                count = 0
                for song in new_list:
                    if song.startswith(sub):
                        count += 1
                        break
                if count == 0:
                    if sub != '\\':
                        new_delete.append(sub)
                    break
        return list(set(new_delete))

    @classmethod
    def reduce_files(self, remove_items, exclude_items):
        rf = []
        for item in remove_items:
            if not item.startswith(exclude_items):
                rf.append(item)
        return rf

    @classmethod
    def moveallfiles(self, root_src_dir, root_dst_dir):
        change_permissions = True
        if sys.platform == "win32" or sys.platform == "darwin":
            change_permissions = False
        else:
            app.logger.debug('Update on OS-System : ' + sys.platform)
            new_permissions = os.stat(root_dst_dir)
            # print new_permissions
        for src_dir, __, files in os.walk(root_src_dir):
            dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1)
            if not os.path.exists(dst_dir):
                os.makedirs(dst_dir)
                app.logger.debug('Create-Dir: '+dst_dir)
                if change_permissions:
                    # print('Permissions: User '+str(new_permissions.st_uid)+' Group '+str(new_permissions.st_uid))
                    os.chown(dst_dir, new_permissions.st_uid, new_permissions.st_gid)
            for file_ in files:
                src_file = os.path.join(src_dir, file_)
                dst_file = os.path.join(dst_dir, file_)
                if os.path.exists(dst_file):
                    if change_permissions:
                        permission = os.stat(dst_file)
                    app.logger.debug('Remove file before copy: '+dst_file)
                    os.remove(dst_file)
                else:
                    if change_permissions:
                        permission = new_permissions
                shutil.move(src_file, dst_dir)
                app.logger.debug('Move File '+src_file+' to '+dst_dir)
                if change_permissions:
                    try:
                        os.chown(dst_file, permission.st_uid, permission.st_gid)
                    except (Exception) as e:
                        # ex = sys.exc_info()
                        old_permissions = os.stat(dst_file)
                        app.logger.debug('Fail change permissions of ' + str(dst_file) + '. Before: '
                            + str(old_permissions.st_uid) + ':' + str(old_permissions.st_gid) + ' After: '
                            + str(permission.st_uid) + ':' + str(permission.st_gid) + ' error: '+str(e))
        return

    def update_source(self, source, destination):
        # destination files
        old_list = list()
        exclude = (
            os.sep + 'app.db', os.sep + 'calibre-web.log1', os.sep + 'calibre-web.log2', os.sep + 'gdrive.db',
            os.sep + 'vendor', os.sep + 'calibre-web.log', os.sep + '.git', os.sep +'client_secrets.json',
            os.sep + 'gdrive_credentials', os.sep + 'settings.yaml')
        for root, dirs, files in os.walk(destination, topdown=True):
            for name in files:
                old_list.append(os.path.join(root, name).replace(destination, ''))
            for name in dirs:
                old_list.append(os.path.join(root, name).replace(destination, ''))
        # source files
        new_list = list()
        for root, dirs, files in os.walk(source, topdown=True):
            for name in files:
                new_list.append(os.path.join(root, name).replace(source, ''))
            for name in dirs:
                new_list.append(os.path.join(root, name).replace(source, ''))

        delete_files = self.one_minus_two(old_list, new_list)

        rf = self.reduce_files(delete_files, exclude)

        remove_items = self.reduce_dirs(rf, new_list)

        self.moveallfiles(source, destination)

        for item in remove_items:
            item_path = os.path.join(destination, item[1:])
            if os.path.isdir(item_path):
                app.logger.debug("Delete dir " + item_path)
                shutil.rmtree(item_path, ignore_errors=True)
            else:
                try:
                    app.logger.debug("Delete file " + item_path)
                    # log_from_thread("Delete file " + item_path)
                    os.remove(item_path)
                except Exception:
                    app.logger.debug("Could not remove:" + item_path)
        shutil.rmtree(source, ignore_errors=True)

    def _nightly_version_info(self):
        content = {}
        content[0] = '$Format:%H$'
        content[1] = '$Format:%cI$'
        # content[0] = 'bb7d2c6273ae4560e83950d36d64533343623a57'
        # content[1] = '2018-09-09T10:13:08+02:00'
        if is_sha1(content[0]) and len(content[1]) > 0:
            return {'version': content[0], 'datetime': content[1]}
        return False

    def _stable_version_info(self):
        return {'version': '0.6.1'} # Current version

    def _nightly_available_updates(self, request_method):
        tz = datetime.timedelta(seconds=time.timezone if (time.localtime().tm_isdst == 0) else time.altzone)
        if request_method == "GET":
            repository_url = 'https://api.github.com/repos/janeczku/calibre-web'
            status, commit = self._load_remote_data(repository_url +'/git/refs/heads/master')
            parents = []
            if status['message'] != '':
                return json.dumps(status)
            if 'object' not in commit:
                status['message'] = _(u'Unexpected data while reading update information')
                return json.dumps(status)

            if commit['object']['sha'] == status['current_commit_hash']:
                status.update({
                    'update': False,
                    'success': True,
                    'message': _(u'No update available. You already have the latest version installed')
                })
                return json.dumps(status)

            # a new update is available
            status['update'] = True

            try:
                headers = {'Accept': 'application/vnd.github.v3+json'}
                r = requests.get(repository_url + '/git/commits/' + commit['object']['sha'], headers=headers)
                r.raise_for_status()
                update_data = r.json()
            except requests.exceptions.HTTPError as e:
                status['error'] = _(u'HTTP Error') + ' ' + str(e)
            except requests.exceptions.ConnectionError:
                status['error'] = _(u'Connection error')
            except requests.exceptions.Timeout:
                status['error'] = _(u'Timeout while establishing connection')
            except requests.exceptions.RequestException:
                status['error'] = _(u'General error')

            if status['message'] != '':
                return json.dumps(status)

            if 'committer' in update_data and 'message' in update_data:
                status['success'] = True
                status['message'] = _(
                    u'A new update is available. Click on the button below to update to the latest version.')

                new_commit_date = datetime.datetime.strptime(
                    update_data['committer']['date'], '%Y-%m-%dT%H:%M:%SZ') - tz
                parents.append(
                    [
                        format_datetime(new_commit_date, format='short', locale=get_locale()),
                        update_data['message'],
                        update_data['sha']
                    ]
                )

                # it only makes sense to analyze the parents if we know the current commit hash
                if status['current_commit_hash'] != '':
                    try:
                        parent_commit = update_data['parents'][0]
                        # limit the maximum search depth
                        remaining_parents_cnt = 10
                    except IndexError:
                        remaining_parents_cnt = None

                    if remaining_parents_cnt is not None:
                        while True:
                            if remaining_parents_cnt == 0:
                                break

                            # check if we are more than one update behind if so, go up the tree
                            if parent_commit['sha'] != status['current_commit_hash']:
                                try:
                                    headers = {'Accept': 'application/vnd.github.v3+json'}
                                    r = requests.get(parent_commit['url'], headers=headers)
                                    r.raise_for_status()
                                    parent_data = r.json()

                                    parent_commit_date = datetime.datetime.strptime(
                                        parent_data['committer']['date'], '%Y-%m-%dT%H:%M:%SZ') - tz
                                    parent_commit_date = format_datetime(
                                        parent_commit_date, format='short', locale=get_locale())

                                    parents.append([parent_commit_date,
                                                    parent_data['message'].replace('\r\n','<p>').replace('\n','<p>')])
                                    parent_commit = parent_data['parents'][0]
                                    remaining_parents_cnt -= 1
                                except Exception:
                                    # it isn't crucial if we can't get information about the parent
                                    break
                            else:
                                # parent is our current version
                                break

            else:
                status['success'] = False
                status['message'] = _(u'Could not fetch update information')

            # a new update is available
            status['update'] = True
            if 'body' in commit:
                status['success'] = True
                status['message'] = _(
                    u'A new update is available. Click on the button below to update to the latest version.')

                new_commit_date = datetime.datetime.strptime(
                    commit['committer']['date'], '%Y-%m-%dT%H:%M:%SZ') - tz
                parents.append(
                    [
                        format_datetime(new_commit_date, format='short', locale=get_locale()),
                        commit['message'],
                        commit['sha']
                    ]
                )

                # it only makes sense to analyze the parents if we know the current commit hash
                if status['current_commit_hash'] != '':
                    try:
                        parent_commit = commit['parents'][0]
                        # limit the maximum search depth
                        remaining_parents_cnt = 10
                    except IndexError:
                        remaining_parents_cnt = None

                    if remaining_parents_cnt is not None:
                        while True:
                            if remaining_parents_cnt == 0:
                                break

                            # check if we are more than one update behind if so, go up the tree
                            if commit['sha'] != status['current_commit_hash']:
                                try:
                                    headers = {'Accept': 'application/vnd.github.v3+json'}
                                    r = requests.get(parent_commit['url'], headers=headers)
                                    r.raise_for_status()
                                    parent_data = r.json()

                                    parent_commit_date = datetime.datetime.strptime(
                                        parent_data['committer']['date'], '%Y-%m-%dT%H:%M:%SZ') - tz
                                    parent_commit_date = format_datetime(
                                        parent_commit_date, format='short', locale=get_locale())

                                    parents.append([parent_commit_date, parent_data['message'], parent_data['sha']])
                                    parent_commit = parent_data['parents'][0]
                                    remaining_parents_cnt -= 1
                                except Exception:
                                    # it isn't crucial if we can't get information about the parent
                                    break
                            else:
                                # parent is our current version
                                break
            status['history'] = parents[::-1]
            return json.dumps(status)
        return ''

    def _stable_available_updates(self, request_method):
        if request_method == "GET":
            parents = []
            # repository_url = 'https://api.github.com/repos/flatpak/flatpak/releases'  # test URL
            repository_url = 'https://api.github.com/repos/janeczku/calibre-web/releases'
            status, commit = self._load_remote_data(repository_url)
            if status['message'] != '':
                return json.dumps(status)
            if not commit:
                status['success'] = True
                status['message'] = _(u'No release information available')
                return json.dumps(status)
            version = status['current_commit_hash']
            current_version = status['current_commit_hash'].split('.')

            # we are already on newest version, no update available
            if 'tag_name' not in commit[0]:
                status['message'] = _(u'Unexpected data while reading update information')
                return json.dumps(status)
            if commit[0]['tag_name'] == version:
                status.update({
                    'update': False,
                    'success': True,
                    'message': _(u'No update available. You already have the latest version installed')
                })
                return json.dumps(status)

            i = len(commit) - 1
            while i >= 0:
                if 'tag_name' not in commit[i] or 'body' not in commit[i]:
                    status['message'] = _(u'Unexpected data while reading update information')
                    return json.dumps(status)
                major_version_update = int(commit[i]['tag_name'].split('.')[0])
                minor_version_update = int(commit[i]['tag_name'].split('.')[1])
                patch_version_update = int(commit[i]['tag_name'].split('.')[2])

                # Check if major versions are identical search for newest nonenqual commit and update to this one
                if major_version_update == int(current_version[0]):
                    if (minor_version_update == int(current_version[1]) and
                            patch_version_update > int(current_version[2])) or \
                            minor_version_update > int(current_version[1]):
                        parents.append([commit[i]['tag_name'],commit[i]['body'].replace('\r\n', '<p>')])
                    i -= 1
                    continue
                if major_version_update < int(current_version[0]):
                    i -= 1
                    continue
                if major_version_update > int(current_version[0]):
                    # found update update to last version before major update, unless current version is on last version
                    # before major update
                    if commit[i+1]['tag_name'].split('.')[1] == current_version[1]:
                        parents.append([commit[i]['tag_name'],
                                        commit[i]['body'].replace('\r\n', '<p>').replace('\n', '<p>')])
                        status.update({
                            'update': True,
                            'success': True,
                            'message': _(u'A new update is available. Click on the button below to '
                                         u'update to version: %(version)s', version=commit[i]['tag_name']),
                            'history': parents
                        })
                        self.updateFile = commit[i]['zipball_url']
                    else:
                        status.update({
                            'update': True,
                            'success': True,
                            'message': _(u'A new update is available. Click on the button below to '
                                         u'update to version: %(version)s', version=commit[i]['tag_name']),
                            'history': parents
                        })
                        self.updateFile = commit[i +1]['zipball_url']
                    break
            if i == -1:
                status.update({
                    'update': True,
                    'success': True,
                    'message': _(
                        u'A new update is available. Click on the button below to update to the latest version.'),
                    'history': parents
                })
                self.updateFile = commit[0]['zipball_url']
        return json.dumps(status)

    def _get_request_path(self):
        if config.get_update_channel == UPDATE_STABLE:
            return self.updateFile
        else:
            return 'https://api.github.com/repos/janeczku/calibre-web/zipball/master'

    def _load_remote_data(self, repository_url):
        status = {
            'update': False,
            'success': False,
            'message': '',
            'current_commit_hash': ''
        }
        commit = None
        version = self.get_current_version_info()
        if version is False:
            status['current_commit_hash'] = _(u'Unknown')
        else:
            status['current_commit_hash'] = version['version']
        try:
            headers = {'Accept': 'application/vnd.github.v3+json'}
            r = requests.get(repository_url, headers=headers)
            commit = r.json()
            r.raise_for_status()
        except requests.exceptions.HTTPError as e:
            if commit:
                if 'message' in commit:
                    status['message'] = _(u'HTTP Error') + ': ' + commit['message']
            else:
                status['message'] = _(u'HTTP Error') + ': ' + str(e)
        except requests.exceptions.ConnectionError:
            status['message'] = _(u'Connection error')
        except requests.exceptions.Timeout:
            status['message'] = _(u'Timeout while establishing connection')
        except requests.exceptions.RequestException:
            status['message'] = _(u'General error')

        return status, commit