import sys
import os
import re
import logging
import subprocess
import json
from argparse import ArgumentParser, SUPPRESS
import time
import shutil
import signal

WORK_DIR = '/tmp'

MOUNTED_POINTS_EXCLUDED_PATTERN = '^((/sys)|(/proc)|(/dev)|(/boot)|(/run)|(/var/lib/Acronis/mount)|(/home/cagefs-skeleton)|(/home/virtfs))'
MOUNTED_POINTS_EXCLUDED_FS = 'sysfs,cgroup,proc,devtmpfs,devpts,pstore,debugfs,hugetlbfs,mqueue,configfs,tmpfs,securityfs,selinuxfs,efivarfs,rpc_pipefs,autofs,binfmt_misc'
MOUNT_DIR = "/var/lib/Acronis/mount"

MYSQL_FREEZE_LOCKFILE = os.path.join(WORK_DIR, 'freeze_mysql.lock')
MYSQL_OUTPUT_FILE = os.path.join(WORK_DIR, 'freeze_mysql.out')
MYSQL_FREEZE = True
MYSQL_FREEZE_TIMEOUT = 120  # in seconds.
MYSQL_FREEZE_SNAPSHOT_TIMEOUT = 10  # in seconds.
MYSQL_FREEZE_ONLY_MYISAM = False

LOGFILE = os.path.join(WORK_DIR, 'prepostdatacapture.log')

TARGET_METADATA_PATH = '/home/metadata.json'

logger = logging.getLogger()


class CommandError(Exception):
    def __init__(self, command, error):
        super().__init__("CPanel command ({0}) finished with error: {1}".format(command, error))
        self.error = error


class FreezeError(Exception):
    def __init__(self, error):
        super().__init__("mysql freeze error: {0}".format(error))
        self.error = error


class MountingError(Exception):
    pass


def run_shell(cmd, raise_e=True, log=True):
    try:
        return subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate()[0]
    except subprocess.CalledProcessError as e:
        if log:
            logger.error(
                "Command \"{0}\" has failed: {1}\"".format(
                    cmd,
                    '' if e.stderr is None else e.stderr.decode('utf-8')
                )
            )
        if raise_e:
            raise


def quote(str_to_quote):
    _find_unsafe = re.compile(r'[^\w@%+=:,./-]').search
    if not str_to_quote:
        return "''"
    if _find_unsafe(str_to_quote) is None:
        return str_to_quote
    return "'" + str_to_quote.replace("'", "'\"'\"'") + "'"


class CPanelClient:
    WHM_FEATURE_FOLDER = '/usr/local/cpanel/whostmgr/addonfeatures'
    CPANEL_USERS_FOLDER = '/var/cpanel/users'
    DISABLED_FEATURES = '/var/cpanel/features/disabled'
    DISABLED_FEATURE_LIST = 'disabled'
    RESELLER_FEATURE = 'acronisbackup_reseller'
    RETENTION_FEATURES_PREFIX = "acronisbackup_"
    RETENTION_FEATURES_UNLIMITED = "acronisbackup_0"

    @classmethod
    def _run_shell(cls, cmd):
        cmd = '{0} --output=json'.format(cmd)
        output = run_shell(cmd)
        try:
            return json.loads(output.decode('utf-8', 'ignore'))
        except json.JSONDecodeError:
            logger.error("Cannot parse response from command \"{0}\"".format(cmd))
            raise

    def _call_whmapi1(self, cmd):
        whmapi1_cmd = 'whmapi1 {0}'.format(cmd)
        response = self._run_shell(whmapi1_cmd)
        metadata = response['metadata']
        if metadata['result'] == 0:
            raise CommandError(whmapi1_cmd, metadata['reason'])
        return response.get('data', {})

    def _call_cpapi2(self, cmd):
        cpapi2_cmd = 'cpapi2 {0}'.format(cmd)
        response = self._run_shell(cpapi2_cmd)
        result = response['cpanelresult']
        if result.get('error'):
            raise CommandError(cpapi2_cmd, result['error'])
        return result['data']

    def _call_uapi(self, cmd):
        uapi_cmd = 'uapi {0}'.format(cmd)
        try:
            response = self._run_shell(uapi_cmd)
        except subprocess.CalledProcessError as e:
            raise CommandError(uapi_cmd, e.stderr)
        result = response['result']
        if result.get('errors'):
            raise CommandError(uapi_cmd, '; '.join(result['errors']))
        return result['data']

    def fetch_version(self):
        return self._call_whmapi1('version')['version']

    def fetch_accounts(self):
        return self._call_whmapi1('listaccts').get('acct', [])

    def fetch_databases(self):
        return self._call_whmapi1('list_databases').get('payload', [])

    def fetch_account_domains_data(self, account_name):
        domains = self._call_uapi(
            '--user={0} DomainInfo domains_data'.format(quote(account_name))
        )
        return (
            domains.get('main_domain'),
            domains.get('sub_domains', []),
            domains.get('addon_domains', []),
        )

    def fetch_addon_domains_data(self, account_name):
        return self._call_cpapi2(
            '--user={0} AddonDomain listaddondomains'.format(quote(account_name))
        )

    def fetch_account_domains(self, account_name):
        domains = self._call_uapi(
            '--user={0} DomainInfo list_domains'.format(quote(account_name))
        )
        return (
            domains.get('main_domain'),
            domains.get('sub_domains', []),
            domains.get('addon_domains', []),
        )

    def fetch_account_emails(self, account_name):
        return self._call_uapi('--user={0} Email list_pops'.format(quote(account_name)))

    def fetch_email_quota(self, account_name, user, domain):
        return self._call_uapi('--user={0} Email get_pop_quota email={1} domain={2} as_bytes=1'.format(
            quote(account_name),
            quote(user),
            quote(domain),
        ))

    def fetch_mysql_servers(self):
        return self._call_whmapi1('remote_mysql_read_profiles')

    def fetch_local_mysql_version(self):
        return self._call_whmapi1('current_mysql_version')['version']

    def fetch_user_main_domain(self, account_name):
        return self._call_cpapi2('--user={0} DomainLookup getmaindomain'.format(
            quote(account_name))
        )

    def fetch_subdomains_data(self, account_name):
        return self._call_cpapi2('--user={0} SubDomain listsubdomains'.format(
            quote(account_name)
        ))


class CPanelMetaCollector:
    DBMS_TYPE_MYSQL = 'mysql'
    RPM_PACKAGE = 'acronis-backup-cpanel'

    def __init__(self, cpanel_client):
        self.cpanel_client = cpanel_client
        self.meta = None

    def _get_ent(self, type_, key, index):
        getent_cmd = 'getent {0} {1} | cut -d: -f{2}'.format(type_, quote(key), index)
        process = subprocess.Popen(getent_cmd, shell=True, stdout=subprocess.PIPE)
        out, _ = process.communicate()
        result = out.decode().strip()
        return result if process.returncode == 0 and result else None

    def _get_dbs(self):
        dbs = dict()
        for db in self.cpanel_client.fetch_databases():
            if db['engine'] != self.DBMS_TYPE_MYSQL:
                continue
            if db['cpuser'] not in dbs:
                dbs[db['cpuser']] = list()
            dbs[db['cpuser']].append({
                'name': db['name'],
                'type': db['engine'],
            })
        return dbs

    def _get_account_emails_and_filters(
        self,
        account_name,
        home_dir
    ):
        emails = []
        for res_item in self.cpanel_client.fetch_account_emails(account_name):
            if '@' in res_item['email']:
                login, domain = res_item['email'].split('@', 1)
            else:
                login = res_item['email']
                domain = self.cpanel_client.fetch_user_main_domain(account_name)[0]['main_domain']
            quota = self.cpanel_client.fetch_email_quota(account_name, login, domain)
            quota = quota.strip() if isinstance(quota, str) else quota
            # Quota maybe '' or 'unlimited' in different cPanel versions
            quota = 0 if not quota or quota == 'unlimited' else int(quota)
            emails.append({
                'login': res_item['login'],
                'email': res_item['email'],
                'quota': quota,
            })

        if home_dir is None:
            return emails, []

        filters = []
        for email in emails:
            if '@' in email['email']:
                login, domain = email['email'].split('@', 1)
                filter_cache_path = os.path.join(home_dir, 'etc', domain, login, 'filter.cache')
            else:
                filter_cache_path = os.path.join(home_dir, 'etc', 'filter.cache')

            if not os.path.exists(filter_cache_path):
                continue

            try:
                with open(filter_cache_path, 'r') as f:
                    json.load(f)
            except json.JSONDecodeError:
                logger.warning(
                    "Mail filter \"{0}\" contains invalid json.".format(filter_cache_path)
                )
                continue
            except OSError:
                logger.warning(
                    "Cannot read mail filter \"{0}\".".format(filter_cache_path)
                )
                continue
            except Exception as e:
                logger.warning(
                    "Cannot parse mail filter \"{0}\". Error: {1}.".format(
                        filter_cache_path,
                        str(e)
                    )
                )
                continue

            filters.append({
                'email': email['email'],
                'path': filter_cache_path,
            })

        return emails, filters

    def collect_accounts(self, collect_dbs=True):
        self.meta['accounts'] = {}

        accounts = self.cpanel_client.fetch_accounts()
        logger.info("Accounts number: {0}".format(len(accounts)))

        if collect_dbs:
            dbs = self._get_dbs()
            dbs_number = sum(map(len, dbs.values()))
            logger.info("Databases number: {0}".format(dbs_number))
        else:
            logger.warning("Skipping databases collection.")
            dbs = {}

        def _make_metadata_for_account(account):
            def _update_domain_data(domains, domains_document_root):
                for domain in domains:
                    if not isinstance(domain, dict):
                        continue
                    try:
                        domains_document_root[domain['domain']] = {
                            'documentroot': domain['documentroot'],
                            'type': domain.get('type'),
                            'owner': domain.get('owner'),
                            'user': domain.get('user'),
                        }
                    except KeyError:
                        logger.warning("Unable to save document root for domain {0}.".format(domain))

            def get_gid_by_group_name(group_name):
                gid = self._get_ent('group', group_name, 3)
                return int(gid) if gid is not None else None

            account_name = account['user']
            owner = account['owner']
            domains_data = {}
            processed_subdomains, processed_addon_domains = [], []

            main_domain, sub_domains, addon_domains =\
                self.cpanel_client.fetch_account_domains(account_name)
            main_domain_data, sub_domains_data, addon_domains_data =\
                self.cpanel_client.fetch_account_domains_data(account_name)

            if main_domain:
                home_dir = main_domain_data.get('homedir')
                _update_domain_data([main_domain_data], domains_data)
            else:
                home_dir = None

            if sub_domains:
                _update_domain_data(sub_domains_data, domains_data)
                # Can only be fetched from old API.
                try:
                    sub_domain_data = self.cpanel_client.fetch_subdomains_data(account_name)
                    for sub_domain in sub_domain_data:
                        if not isinstance(sub_domain, dict):
                            continue
                        if domains_data.get(sub_domain['domain']):
                            domains_data[sub_domain['domain']]['rootdomain'] = \
                                sub_domain.get('rootdomain')
                        processed_subdomains.append(sub_domain['domain'])
                except Exception:
                    logger.exception("Failed to collect subdomains for account: {0}".format(
                        account_name))
                    processed_subdomains = []

            if addon_domains:
                _update_domain_data(addon_domains_data, domains_data)
                # Can only be fetched from old API.
                try:
                    addon_domain_data = self.cpanel_client.fetch_addon_domains_data(account_name)
                    for addon_domain in addon_domain_data:
                        if not isinstance(addon_domain, dict):
                            continue
                        if domains_data.get(addon_domain['domain']):
                            domains_data[addon_domain['domain']]['subdomain'] =\
                                addon_domain.get('subdomain')
                        processed_addon_domains.append(addon_domain['domain'])
                except Exception:
                    logger.exception("Failed to collect addon domains for {0}".format(account_name))
                    processed_addon_domains = []

            try:
                emails, filters = self._get_account_emails_and_filters(account_name, home_dir)
            except Exception:
                logger.exception("Failed to collect emails and filters for {0}".format(account_name))
                emails, filters = [], []

            gid = get_gid_by_group_name(account_name)

            return {
                'id': account['uid'],
                'gid': gid,
                'name': account_name,
                'owner': owner,
                'domain': main_domain,
                'subDomains': processed_subdomains,
                'addonDomains': processed_addon_domains,
                'homeDir': home_dir,
                'dbs': dbs.get(account_name, []),
                'emails': emails,
                'filters': filters,
                'domainsData': domains_data,
            }

        for account in accounts:
            try:
                self.meta['accounts'][account['uid']] = _make_metadata_for_account(account)
            except Exception:
                # DO NOT RAISE
                logger.exception(
                    "Skip account \"{0}\", cannot make metadata.".format(account['user'])
                )

    def collect_db_servers(self):
        self.meta['dbServers'] = {}

        mysql_servers = self.cpanel_client.fetch_mysql_servers()
        local_mysql_server = mysql_servers.get('localhost')
        if not local_mysql_server or not local_mysql_server['active']:
            logger.warning("There is no active local mysql server.")
            return False

        self.meta['dbServers']['localhost'] = {
            'host': local_mysql_server['mysql_host'],
            'port': local_mysql_server['mysql_port'],
            'type': self.DBMS_TYPE_MYSQL,
            'dbmsOptions': {
                'myCnfPath': '/root/.my.cnf',
                'datadir': '/var/lib/mysql/',
                'version': self.cpanel_client.fetch_local_mysql_version(),
            },
        }
        logger.info(
            "Local mysql server: {0}:{1}".format(
                local_mysql_server['mysql_host'],
                local_mysql_server['mysql_port'],
            )
        )
        return True

    def collect_unmapped_dbs(self):

        def get_home_dir_by_uid(uid):
            return self._get_ent('passwd', str(uid), 6)

        self.meta['unmappedDBs'] = []

        skip_lines = set(['Database', 'mysql', 'information_schema', 'performance_schema', 'sys', ''])
        for account in self.meta['accounts'].values():
            for db in account['dbs']:
                skip_lines.add(db['name'])

        server = self.meta['dbServers'].get('localhost')
        if not server:
            logger.warning("Skipped collecting unmapped DBs, there's no localhost MySQL instance.")
            return

        change_home_cmd = 'export HOME={0}'.format(
            get_home_dir_by_uid(os.getuid())
        )
        cmd = "mysql --host={0} --port={1} --execute='SHOW DATABASES;'".format(
            server['host'],
            server['port'],
        )
        result = run_shell('{0} && {1}'.format(change_home_cmd, cmd))

        output_list = result.decode().split('\n')
        for db in output_list:
            if db not in skip_lines:
                self.meta['unmappedDBs'].append({'name': db, 'type': server['type']})

    def collect_cpanel_info(self):
        cpanel_version = self.cpanel_client.fetch_version()
        self.meta['cpanel'] = {
            'version': cpanel_version,
        }
        logger.info("CPanel version: {0}".format(cpanel_version))

    def collect(self):
        self.meta = {
            'timestamp': int(time.time()),
        }
        self.collect_cpanel_info()
        if self.collect_db_servers():
            self.collect_accounts()
            self.collect_unmapped_dbs()
        else:
            self.collect_accounts(False)


def collect_meta(cpanel_client):
    logger.info("Generating metadata...")
    try:
        cpanel_meta_collector = CPanelMetaCollector(cpanel_client)
        cpanel_meta_collector.collect()
        # For security reasons - first create an empty file, set permissions and then fill it in
        tmp_meta_path = os.path.join(WORK_DIR, 'metadata.json')
        with open(tmp_meta_path, 'w') as f:
            json.dump({}, f)
        os.chmod(tmp_meta_path, 0o600)
        with open(tmp_meta_path, 'w') as f:
            json.dump(cpanel_meta_collector.meta, f)
        shutil.move(tmp_meta_path, TARGET_METADATA_PATH)
    except Exception:
        logger.exception("Generating metadata has failed")
        raise
    logger.info('Collecting metadata succeeded.')


def freeze_mysql():
    run_shell("mysqladmin ping > /dev/null 2>&1")   # raise exception if ping fails

    if os.path.isfile(MYSQL_FREEZE_LOCKFILE):
        if os.stat(MYSQL_FREEZE_LOCKFILE).st_size > 0:
            output = run_shell("ps -p \"$(/bin/cat \"{0}\")\" -o pid=".format(MYSQL_FREEZE_LOCKFILE), raise_e=False, log=False)
            if output and int(output) > 0:
                raise FreezeError('Can not start freezing because process already exists')

        logger.info('Deleting freeze lock file...')
        try:
            os.remove(MYSQL_FREEZE_LOCKFILE)
        except Exception:
            raise FreezeError("Unable to remove lockfile: {0}".format(MYSQL_FREEZE_LOCKFILE))

    # Prepare freeze query
    if MYSQL_FREEZE_ONLY_MYISAM:
        sql_myisam_tables = "SET SESSION group_concat_max_len = 4294967295; \
            SELECT IFNULL(GROUP_CONCAT(t.db_table SEPARATOR ','), '') as res \
            FROM (\
                SELECT CONCAT('\`', TABLE_SCHEMA, '\`.\`', TABLE_NAME, '\`') as db_table \
                FROM information_schema.TABLES \
                WHERE ENGINE='MyISAM' AND TABLE_SCHEMA NOT IN('mysql','information_schema','performance_schema'\
                )\
            ) t;"
        cmd = "SQL_MYISAM=\"{0}\"".format(sql_myisam_tables) + " ; mysql --defaults-file=\"/root/.my.cnf\" --execute=\"$SQL_MYISAM\" | tail -n +2 | awk '{$1=$1;print}'"
        myisam_tables = run_shell(cmd)
        if myisam_tables is None:
            logger.info('There are no MyISAM tables to lock.')
            return
        myisam_tables = myisam_tables.decode()
        logger.info("Trying to lock {0} MyISAM tables.".format(myisam_tables.count(',')))

        freeze_sql = "\
            SELECT CONNECTION_ID();\
            FLUSH TABLES {0};\
            FLUSH TABLES {0};\
            FLUSH TABLES {0} WITH READ LOCK;\
            SYSTEM touch \"{1}\";\
            SYSTEM sleep {2};".format(myisam_tables.replace('`', '\\`'), MYSQL_FREEZE_LOCKFILE, MYSQL_FREEZE_SNAPSHOT_TIMEOUT)
    else:
        logger.info('Trying to lock all tables.')
        freeze_sql = "\
            SELECT CONNECTION_ID();\
            FLUSH TABLES;\
            FLUSH TABLES;\
            FLUSH TABLES WITH READ LOCK;\
            SYSTEM touch \"{0}\";\
            SYSTEM sleep {1};".format(MYSQL_FREEZE_LOCKFILE, MYSQL_FREEZE_SNAPSHOT_TIMEOUT)
    freeze_sql = freeze_sql.replace('"', '\\"')

    # Run freeze command
    cmd = "mysql --defaults-file=\"/root/.my.cnf\" --unbuffered -N -s --execute=\"{0}\" 1> \"{1}\" 2>> \"{2}\"".format(freeze_sql, MYSQL_OUTPUT_FILE, LOGFILE)
    logger.debug(cmd)
    proc = subprocess.Popen(cmd, shell=True)
    freeze_check_interval_sec = 1
    seconds_elapsed = 0
    freeze_session_pid = proc.pid
    while not os.path.isfile(MYSQL_FREEZE_LOCKFILE):
        proc.poll()
        if proc.returncode is None:
            logger.info('Waiting for mysql to freeze tables...')
            time.sleep(freeze_check_interval_sec)
            seconds_elapsed += freeze_check_interval_sec

        output = run_shell("ps -p {0} -o pid=".format(freeze_session_pid))
        if output is None:
            raise FreezeError('Seems like MySQL freeze statement failed. Aborted.')
        if seconds_elapsed > MYSQL_FREEZE_TIMEOUT:
            logger.error('MySQL cannot freeze in suitable time. Aborting...')
            try:
                os.kill(freeze_session_pid, signal.SIGKILL)
            except Exception:
                raise FreezeError("Unable to kill {0}".format(freeze_session_pid))

            with open(MYSQL_OUTPUT_FILE, 'r') as f:
                freeze_thread_id = f.readline()
                if freeze_thread_id and int(freeze_thread_id) > 0:
                    logger.error("Killing MySQL thread. ID is {0}".format(freeze_thread_id))
                    output = run_shell("mysqladmin --defaults-file=\"/root/.my.cnf\" kill \"{0}\"".format(freeze_thread_id))
                    raise FreezeError("{0} thread was killed.".format(freeze_thread_id))
                raise FreezeError('Temp file does not contain thread ID.')

    with open(MYSQL_FREEZE_LOCKFILE, 'w') as f:
        f.write(str(freeze_session_pid))
    logger.info('Freeze successful')


def is_dir_mounted(abs_path):
    mounted = run_shell("mount |grep 'on {0} type'".format(abs_path), False, False)
    return True if mounted else False


def purge():
    if not os.path.isdir(MOUNT_DIR):
        logger.debug("{0} directory not present, skipping purging".format(MOUNT_DIR))
        return

    logger.debug("Purging mountpoints")
    for partition_dir in os.listdir(MOUNT_DIR):
        logger.debug("Processing mount dir {0}".format(partition_dir))
        tries = 5
        partition_abs_dir = os.path.join(MOUNT_DIR, partition_dir)
        while is_dir_mounted(partition_abs_dir) and tries > 0:
            run_shell("umount -l {0}".format(partition_abs_dir))
            tries -= 1
            try:
                logger.debug("Removing directory {0}".format(partition_abs_dir))
                os.rmdir(partition_abs_dir)
            except OSError:
                logger.debug("Directory {0} could not be removed".format(partition_abs_dir))
        if tries <= 0:
            raise MountingError


def pre(cpanel_client):
    logger.info('Pre-capture data script started.')

    purge()
    collect_meta(cpanel_client)

    logger.info('Placing files with mountpoints...')
    output = run_shell(
        "findmnt -n -l -it \"{0}\" -o TARGET | egrep -v \"{1}\"".format(MOUNTED_POINTS_EXCLUDED_FS, MOUNTED_POINTS_EXCLUDED_PATTERN)
        )
    for mnt_point in output.decode().split('\n'):
        mp_file = os.path.join(mnt_point, '.mp')
        if os.path.isfile(mp_file):
            os.remove(mp_file)
        with open(mp_file, 'w') as f:
            f.write(mnt_point)

    if MYSQL_FREEZE:
        freeze_mysql()


def post(cpanel_client):
    logger.info('Post-capture data script started')

    # Cleanup mysql freeze
    if os.path.exists(MYSQL_FREEZE_LOCKFILE) and os.path.isfile(MYSQL_FREEZE_LOCKFILE):
        freeze_session_pid = ''
        with open(MYSQL_FREEZE_LOCKFILE) as pidfile:
            freeze_session_pid = pidfile.readline().strip()

        if freeze_session_pid:
            logger.info('Terminating freeze session. PID is {0}.'.format(freeze_session_pid))
            try:
                os.kill(freeze_session_pid, signal.SIGKILL)
                logger.info('{0} was killed.'.format(freeze_session_pid))
            except Exception:
                logger.info('Unable to kill {0}.'.format(freeze_session_pid))
        else:
            logger.info('Lock file does not contain pid of mysql freeze session.')
        logger.info('Deleting freeze lock file...')
        os.remove(MYSQL_FREEZE_LOCKFILE)

    # Remove metadata
    try:
        os.remove(TARGET_METADATA_PATH)
        logger.info("Metadata removed.")
    except Exception:
        logger.exception("Failed to remove file with metadata.")

    logger.info('Post-capture data script finished.')


if __name__ == "__main__":
    parser = ArgumentParser(description="Mount attached devices inside VM")
    parser.add_argument('action', choices=['pre', 'post'])
    parser.add_argument("-v", "--verbose", action="store_true", help="enable script debug output")
    parser.add_argument("-tf", "--temp-folder", default=SUPPRESS, help="temporary folder for script actions")

    options = parser.parse_args()

    stdout_handler = logging.StreamHandler(sys.stdout)
    file_handler = logging.FileHandler(LOGFILE)
    if options.verbose:
        logger.setLevel(logging.DEBUG)
        stdout_handler.setLevel(logging.DEBUG)
        file_handler.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)
        stdout_handler.setLevel(logging.INFO)
        file_handler.setLevel(logging.INFO)
    logger.addHandler(stdout_handler)
    logger.addHandler(file_handler)

    if 'temp_folder' in options:
        WORK_DIR = os.path.normpath(options.temp_folder)

    cpanel_client = CPanelClient()

    if options.action == 'pre':
        pre(cpanel_client)
    elif options.action == 'post':
        post(cpanel_client)
    # else should not be possible due to argparse choices
