#!/skynet/python/bin/python -sBtt
# -*- coding: utf-8 -*-

"""
sky return codes:
0 - success
1 - some error occurred
2 - resolved hosts list is empty
"""
import warnings
import errno
import time
import sys
import os

from pkg_resources import require
from collections import defaultdict

from skycmd import Cmd
from kernel.util.sys.user import getUserName
from library.sky.out import printOutputMessage, StderrRedirect

from library.sky.cmds import pbModule
from library.sky.cmds.skybone import CmdCopier

from api.logger import SkynetLoggingHandler
from kernel.util import logging
from api.skycore import ServiceManager

# Copier cmds
from library.sky.cmds.skybone.get import CmdGet      # noqa
from library.sky.cmds.skybone.share import CmdShare  # noqa
from library.sky.cmds.skybone.files import CmdFiles  # noqa

log = logging.getLogger('ya.skynet.sky')
logging.initialize(log, logging.INFO, handler=SkynetLoggingHandler('sky', 'sky.log'), formatter=None)
logging.getLogger().addHandler(logging.NullHandler())  # mute any messages about nonexistent loggers

debug_log = logging.getLogger('ya.skynet.debug')
debug_log.addHandler(logging.StreamHandler(sys.stderr))
debug_log.setLevel(logging.WARNING)

try:
    # load local config to override global one
    try:
        import yaml
        override_cfg = yaml.load(open(os.path.expanduser('~/.skynet/etc/ya.skynet.sky.yaml'), 'rb'))['Common']
    except Exception:
        override_cfg = {}
    from library.config import query
    cfg = query('skynet.tools.sky', 'config', overrides=override_cfg)
except KeyError:
    # old config, there is no sky tool config yet -> use default one
    from kernel.util.config import Config
    cfg = Config('ya.skynet.sky', """
Common:
  # Вид транспорта для тех операций, которые это поддерживают
  CQueueImplementation: cqudp
  # Выбор синтаксиса для описания множества хостов (A — автоопределение, O — старый YR-формат, N — калькулятор Блинова)
  HostSyntax: A
  # Выбор режима выполнения для sky run (h — похостовый, s — пошардовый, i — поинстансовый)
  Mode: h
  # Формат вывода для тех команд, которые это поддерживают
  # (ls — человекочитаемый формат, json или msgpack — соответствующая сериализация)
  ListFormat: ls
  # Домены, которые нужно отрезать при включенной опции CutDomain
  DomainsToCut:
    - 'search.yandex.net'
    - 'yandex.ru'
    - 'yandex.net'
  # Подробный вывод
  Verbose: 0
  # Аутентифицироваться от имени текущего пользователя
  CurrentUserAuth: True
  # Пробрасывать на удалённую сторону ssh-agent
  ForwardUserAuthAgent: False
  # Не пытаться проверять и резолвить в FQDN переданные имена хостов
  DontCheckHostNames: False
  # Использовать короткие имена хостов. Если включено, доменный суффикс будет отрезан (host.yandex.ru → host)
  CutDomain: False
  # Использовать новый резолвер (через HQ)
  NewResolver: True
  # Отображать прогресс выполнения задачи. Имеет значение для тех операций, которые выполняются сразу на
  # нескольких машинах
  ProgressBar: False
  # Использовать облачный резолвер
  CloudResolver: False
  # Адрес облачного резолвера
  CloudResolverUrl: 'http://resolver.clusterstate.yandex-team.ru'
  # Таймаут для резолвинга
  CloudResolverTimeout: 15
  UseNannyServiceResolvePolicy: False
  # Выводить в sky run в случае ошибки запускавшуюся команду
  ShowCommandOnFailure: False
  PrintNumberOfHostsTwice: False
  # Выводять поток stderr с хостов на stdout вместо stderr
  MergeStderrIntoStdout: False
  YpApiAuth: empty
  YpUseSDResolver: False
  HostsNumThreshold: 1000
  KnownDCs:
    - 'sas'
    - 'man'
    - 'vla'
    - 'iva'
    - 'myt'
""").Common


def check_vulnerable_keys(signer):
    vulnerable_keys = []
    for key in signer:
        if (
            key.has_private()
            and (key.type() == 'ssh-dss' or key.type() == 'ssh-rsa' and key.size() < 2047)
        ):
            vulnerable_keys.append(key)
    if vulnerable_keys:
        message = '\n'.join("   {} ({}) {} {}".format(
            key.type(),
            key.size(),
            ':'.join(x.encode('hex') for x in key.fingerprint()),
            key.description(),
        ) for key in vulnerable_keys)
        printOutputMessage(
            "WARNING! You have vulnerable potentially vulnerable keys.\nYou are encouraged to replace them:",
            message, error=True
        )


def check_cqueue_usage(options):
    if options.cqueueImplementation == 'cqueue':
        printOutputMessage("WARNING: cqueue transport is deprecated and will be disabled soon!\n"
                           "Please consider using cqudp, which is default one now. (Don't use -C or --cqueue)",
                           error=True)


def fix_ipv6_addresses(hosts):
    return ['[{}]'.format(host) if host.count(':') > 1 and not host.startswith('[') else host for host in hosts]


class TaskRunnerWrapper(object):
    def __init__(self, task):
        self.task = task

    def __str__(self):
        return str(self.task)

    def run(self, hosts, options, iterate=False):
        check_cqueue_usage(options)

        hosts = fix_ipv6_addresses(hosts)
        debug_log.info("task will run on hosts: %s", hosts)

        if options.user is not None:
            from library.tasks.user import UserTask
            self.task = UserTask(self.task, options.user, options.forwardAgent)
            debug_log.debug("wrapping with UserTask(user=%r, forwardAgent=%r)", options.user, options.forwardAgent)

        timeout = getattr(options, 'timeout', None)
        if timeout:
            from library.tasks.timeout import TimeOutTask
            timeout = float(timeout)
            self.task = TimeOutTask(self.task, timeout, True)
            debug_log.debug("wrapping with TimeoutTask(timeout=%r)", timeout)

        if getattr(options, 'smooth', None):
            from library.tasks.smooth import SmoothTask
            self.task = SmoothTask(self.task, float(options.smooth))
            debug_log.debug("wrapping with SmoothTask(smooth=%r)", float(options.smooth))

        client_opts = dict(p.split(':', 1) for p in getattr(options, 'client_opts', []))
        debug_log.info("client options: %s", client_opts)

        from api.cqueue import Client, CQWarning

        # we display own warning
        warnings.simplefilter('ignore', category=CQWarning)
        client = Client(options.cqueueImplementation, **client_opts)

        if options.identityFile:
            try:
                keys = list(client.signer.loadKeys(open(options.identityFile, 'rb'), options.identityFile))
            except EnvironmentError:
                import traceback
                printOutputMessage("Can't load keys from '{0}'".format(options.identityFile), traceback.format_exc(),
                                   error=True)
                raise SystemExit(1)

            for key in keys:
                client.signer.addKey(key)
                debug_log.info("Loaded key %s", ':'.join(s.encode('hex') for s in key.fingerprint()))

        if not list(client.signer):
            printOutputMessage("No private keys available. Cannot run the task.", error=True)
            raise SystemExit(1)

        check_vulnerable_keys(client.signer)

        if iterate:
            return client.iter(hosts, self.task)
        else:
            return client.run(hosts, self.task)


class ShellCommandWrapper(object):
    def __init__(self, command, extra_env=None):
        self.command = command
        self.extra_env = extra_env

    def __str__(self):
        return str(self.command)

    def run(self, hosts, options, iterate=False):
        check_cqueue_usage(options)

        debug_log.info("shell command will run on hosts: %s", hosts)
        client_opts = dict(p.split(':', 1) for p in options.client_opts)
        debug_log.info("client options: %s", client_opts)

        from api.cqueue import Client, CQWarning

        # we display own warning
        warnings.simplefilter('ignore', category=CQWarning)
        client = Client(options.cqueueImplementation, **client_opts)

        extra_opts = {}
        if options.identityFile:
            try:
                keys = list(client.signer.loadKeys(open(options.identityFile, 'rb'), options.identityFile))
            except EnvironmentError:
                import traceback
                printOutputMessage("Can't load keys from '{0}'".format(options.identityFile), traceback.format_exc(),
                                   error=True)
                raise SystemExit(1)

            for key in keys:
                client.signer.addKey(key)
                debug_log.info("Loaded key %s", ':'.join(s.encode('hex') for s in key.fingerprint()))

        if options.port_range:
            extra_opts['port_range'] = options.port_range

        if options.forwardAgent:
            extra_opts['forward_agent'] = True

        if options.data_limit:
            extra_opts['data_limit'] = options.data_limit

        if self.extra_env:
            extra_opts['extra_env'] = self.extra_env

        if not list(client.signer):
            printOutputMessage("No private keys available. Cannot run the task.", error=True)
            raise SystemExit(1)

        check_vulnerable_keys(client.signer)

        timeout = float(options.timeout) if options.timeout is not None else None
        return self._start(client, hosts, options, timeout, extra_opts)

    def _start(self, client, hosts, options, timeout, extra_opts):
        return client.run_shell(hosts, self.command, options.user, timeout, **extra_opts)


class PortoshellCommandWrapper(ShellCommandWrapper):
    def _start(self, client, hosts, options, timeout, extra_opts):
        kwargs = {
            'streaming': options.stream,
        }
        if options.smooth:
            kwargs['smooth'] = float(options.smooth)

        extra_env = extra_opts.get('extra_env')
        if extra_env:
            kwargs['extra_env'] = extra_env

        return client.run_in_portoshell(hosts, self.command, options.user, **kwargs)


class CQTransportOptions(object):
    supported_implementations = ['cqudp', 'cqueue']
    available_implementations = []

    @classmethod
    def add_to_parser(cls, parser):
        if not cls.available_implementations:
            try:
                skycore_services = ServiceManager().list_services('skynet')
            except Exception:
                skycore_services = []
            for impl in cls.supported_implementations:
                if impl in skycore_services:
                    cls.available_implementations.append(impl)
                    if impl == 'cqudp':
                        parser.add_option(
                            '--%s' % impl, dest='cqueueImplementation', action='store_const', const=impl,
                            help='Use %s transport' % impl)
                    else:
                        parser.add_option(
                            '-%s' % impl[0].capitalize(),
                            '--%s' % impl, dest='cqueueImplementation', action='store_const', const=impl,
                            help='Use %s transport' % impl)

        if cfg.CQueueImplementation not in cls.available_implementations:
            printOutputMessage(
                'WARNING! default service "%s" is not installed, service "%s" will be used as default instead' %
                (cfg.CQueueImplementation, cls.available_implementations[0]), error=True)
            cfg.CQueueImplementation = cls.available_implementations[0]

        parser.add_option(
            '--cqueue_implementation', dest='cqueueImplementation', default=cfg.CQueueImplementation,
            choices=cls.available_implementations,
            help='Chosen cqueue implementation: %s' % cls.available_implementations)

        parser.add_option(
            '-u', '--user', dest='user', default=None,
            help='User to execute command from')

        parser.add_option(
            '-U', action='store_true', dest='currentUser', default=cfg.CurrentUserAuth,
            help='[set by default] Execute command from current user')

        parser.add_option(
            '-A', action='store_true', dest='forwardAgent', default=cfg.ForwardUserAuthAgent,
            help='Enables forwarding of the authentication agent connection')

        parser.add_option(
            '-i', dest='identityFile', default=None, metavar='IDENTITY_FILE',
            help='Selects a file from which the identity (private key) for public key authentication is read')

        parser.add_option(
            '--client-opt', action='append', dest='client_opts', type=str, default=[],
            help='For testing purposes only! --client-opt=key:value passes the parameter key=value to the Client')

    @classmethod
    def process_options(cls, options):
        if not options.user and options.currentUser:
            options.user = getUserName()

        if options.forwardAgent and options.user is None and options.cqueueImplementation != 'cqudp':
            printOutputMessage('SSH agent cannot be forwarded, when user is not set', error=True)
            raise SystemExit(1)


class MagicHostResolver(Cmd):
    def __init__(self):
        super(MagicHostResolver, self).__init__()

    def _resolve_hosts_magic(self, clause):
        from library.sky.hostresolver.errors import HrHostCheckFailedError, HrSyntaxError

        resolved_hosts = set()
        err_code = 0

        hosts = getattr(self.options, 'hosts', None)
        syntax = getattr(self.options, 'host_syntax', str(cfg.HostSyntax))
        resolve_mode = getattr(self.options, 'mode', str(cfg.Mode))
        check_hostnames = not getattr(self.options, 'dont_check_hosts', cfg.DontCheckHostNames)
        use_service_resolve_policy = getattr(self.options, 'use_service_resolve_policy', cfg.UseNannyServiceResolvePolicy)
        no_tag = getattr(self.options, 'no_tag', False)
        yp_use_sd_resolver = getattr(self.options, 'yp_use_sd_resolver', cfg.YpUseSDResolver)
        start_percents = end_percents = None

        # remove comments
        import re
        for i, c in enumerate(clause):
            clause[i] = re.sub(r'#.*', '', c)

        try:
            if hosts is not None:
                # file with hosts specified
                if not os.path.exists(hosts) or not os.path.isfile(hosts):
                    print >> sys.stderr, 'Cannot read hosts file from: {}'.format(hosts)
                    raise SystemExit(1)

                # read clause from the file
                with open(hosts, 'r') as f:
                    clause = [i for i in [i.strip() for i in f] if i]

            if resolve_mode not in ['h', 'i', 's', 'f']:
                printOutputMessage(
                    'Unknown mode "{0}"\n"h" (hosts), "s" (shards),'
                    '"i" (instances) and "f" (families) are only possible'.format(resolve_mode),
                    error=True
                )
                raise SystemExit(1)

            if syntax == 'A':
                from library.sky.hosts import detectResolver
                resolver = detectResolver(clause)
                if resolver == 'blinov':
                    syntax = 'N'
                elif resolver == 'yr':
                    syntax = 'O'
                else:
                    printOutputMessage('Failed to automatically detect resolver, using Blinov calc')
                    syntax = 'N'

            if syntax == 'O':
                # printOutputMessage('Old syntax is deprecated, please use new one', error=True)
                # convert to new format
                clause = set(map(lambda x: x.replace('+', ''), clause))
                extract_clause = set(map(lambda x: x.replace('-', ''), filter(lambda x: x.startswith('-'), clause)))
                clause -= extract_clause

            from library.sky.hostresolver.resolver import Resolver
            from library.sky.hosts import braceExpansion
            from api.hq import HQServiceNotFoundError

            command = braceExpansion(clause, True)

            # handle percents first and remove it from command
            if command.count('%') > 1:
                printOutputMessage('Error: Only one `percents` tag allowed', error=True)
                raise SystemExit(1)

            if command.count('%') == 1:
                subcommands = command.split()
                for subcommand in subcommands:
                    if subcommand.startswith('%'):
                        try:
                            items = subcommand[1:].split(':')
                            if len(items) == 1:
                                start_percents = 0
                                end_percents = int(items[0])
                            elif len(items) == 2:
                                start_percents = int(items[0]) if items[0] else 0
                                end_percents = int(items[1]) if items[1] else 100
                            if start_percents > 100 or start_percents < 0 or end_percents > 100 or end_percents < 0 or start_percents > end_percents:
                                raise ValueError()
                        except ValueError:
                            printOutputMessage('Error: Invalid value for `percents` tag', error=True)
                            raise SystemExit(1)

                        subcommands.remove(subcommand)
                        break
                command = ' '.join(subcommands)

            resolver = Resolver(check_hostnames, no_tag, yp_use_sd_resolver=yp_use_sd_resolver, use_service_resolve_policy=use_service_resolve_policy)

            meth = {
                'h': resolver.resolveHosts,
                's': resolver.resolveShards,
                'i': resolver.resolveInstances,
                'f': resolver.resolveSlots
            }.get(resolve_mode, None)

            if meth is None:
                raise Exception('Unknown mode %r (choose from hosts, shards or instances)' % (resolve_mode,))

            resolved_hosts = meth(command)

        except HrSyntaxError as ex:
            msg = 'Syntax error: {0}'.format(ex, )
            printOutputMessage(msg, error=True)
            err_code = 1
        except HrHostCheckFailedError as ex:
            msg = 'Failed to resolve some hosts:\n{0}'.format(ex.format())
            printOutputMessage(msg, error=True)
            err_code = 1
        except Exception as ex:
            msg = 'Failed to resolve hosts: {0:s}'.format(ex)
            printOutputMessage(msg, error=True)
            err_code = 1

        if resolved_hosts and (start_percents or end_percents):
            start_index = len(resolved_hosts) * start_percents / 100
            end_index = len(resolved_hosts) * end_percents / 100
            resolved_hosts = set(sorted(resolved_hosts)[start_index:end_index])

        if not resolved_hosts:
            printOutputMessage(
                'Warning: no hosts resolved',
                error=True
            )
            if err_code == 0:
                # empty hosts list has been resolved
                err_code = 2

        return err_code, resolved_hosts


class CmdStreamOutput(object):
    def stream_runner(self):
        raise NotImplementedError

    def do_run_stream(self, hosts):
        # stream mode is a special case, so we handle it here
        remaining_hosts = dict.fromkeys(hosts)
        for key in remaining_hosts.iterkeys():
            remaining_hosts[key] = key

        max_len = len(max(remaining_hosts.itervalues(), key=len))
        try:
            with self.stream_runner().run(hosts, self.options, iterate=True) as it:
                debug_log.info("session id: %s", it.id)
                timeout = float(self.options.timeout) if self.options.timeout is not None else None
                debug_log.debug("timeout is %r", timeout)
                for (host, result, error) in it.wait(timeout=timeout):
                    if error:
                        if not isinstance(error, StopIteration):
                            self.process_task_result(host, None, error)
                        else:
                            self.host_done(host)
                            debug_log.debug("host %r StopIteration received", host)

                        remaining_hosts.pop(host, None)
                    else:
                        from library.tasks.command import IterCommandRunner

                        if len(result) == 3:
                            mark, result = (
                                (IterCommandRunner.STDOUT, result[0])
                                if result[0] else
                                (IterCommandRunner.STDERR, result[1])
                            )
                        else:
                            mark, result = result
                        if not result:
                            continue
                        out = sys.stdout if mark == IterCommandRunner.STDOUT else sys.stderr
                        fmt = '<{host: ^{max_len}}> {line}' if self.options.verbose >= 0 else '{line}'
                        out.write(
                            fmt.format(
                                host=remaining_hosts.get(host, host),
                                line=result if result[-1] == '\n' else result + '\n',
                                max_len=max_len
                            )
                        )
                        out.flush()

        except KeyboardInterrupt:
            err = KeyboardInterrupt('Keyboard interrupt')
            for host in remaining_hosts.iterkeys():
                self.process_task_result(host, None, err)
            raise

        err = Exception('Timeout exceeded')
        for host in remaining_hosts.iterkeys():
            self.process_task_result(host, None, err)


class CmdResolveSingleHost(MagicHostResolver):
    def __init__(self):
        super(CmdResolveSingleHost, self).__init__()

    def resolve_single_host(self, clause):
        code, resolved_hosts = self._resolve_hosts_magic(clause)

        resolved_host = None
        if resolved_hosts:
            if len(resolved_hosts) > 1:
                printOutputMessage(
                    'Error: command works only with 1 host, we have %d' % (len(resolved_hosts),),
                    error=True
                )
                code = 1
            else:
                resolved_host = resolved_hosts.pop()

        return code, resolved_host


class CmdResolveHosts(MagicHostResolver):
    def __init__(self):
        super(CmdResolveHosts, self).__init__()

        self.parser.add_option(
            '-N', '--new_syntax', dest='host_syntax', action='store_const', const='N', default=cfg.HostSyntax,
            help='[set by default] Forcibly use new host calculator syntax (e.g. S@RusTier0 . C@HEAD)')
        self.parser.add_option(
            '--old_syntax', dest='host_syntax', action='store_const', const='O', default=cfg.HostSyntax,
            help='Old syntax is DEPRECATED, please do not use this option.'
                 'Forcibly use old host calculator syntax (e.g. +host -host)')
        self.parser.add_option(
            '--hosts', dest='hosts',
            help='Use list of hosts from the specified file')
        self.parser.add_option(
            '--dont_check_hosts', dest='dont_check_hosts', action='store_true', default=cfg.DontCheckHostNames,
            help='Do not check host names. Host names have to be verified by user')

        self.parser.add_option(
            '--yp-use-sd-resolver', dest='yp_use_sd_resolver', action='store_true', default=cfg.YpUseSDResolver,
            help='Use new SD resolver for resolving YP pods and boxes')
        self.parser.add_option(
            '--no-yp-use-sd-resolver', dest='yp_use_sd_resolver', action='store_false', default=cfg.YpUseSDResolver,
            help='Use old SD resolver for resolving YP pods and boxes')

        self.parser.add_option(
            '--use-nanny-service-resolve-policy', dest='use_service_resolve_policy', action='store_true',
            default=cfg.UseNannyServiceResolvePolicy, help='Use nanny service resolve policy from nanny api')
        self.parser.add_option(
            '--no-use-nanny-service-resolve-policy', dest='use_service_resolve_policy', action='store_false',
            default=cfg.UseNannyServiceResolvePolicy, help='Use nanny service resolve policy from nanny api')
        
        self.parser.add_option(
            '--cut-domain', dest='cut_domain', action='store_true', default=cfg.CutDomain,
            help='Cut known domains ({}) from hostname'.format(cfg.DomainsToCut))

        # for backward compatibility only. will be removed in Q3'15
        self.parser.add_option(
            '-l', '--long_names', dest='long_names', action='store_true', default=True,
            help='[set by default] Display host names with domain')

    def resolve_hosts(self, clause):
        return self._resolve_hosts_magic(clause)

    @staticmethod
    def format_hosts(hosts):
        from library.format import formatHosts
        try:
            return formatHosts(hosts, addDomain=True)
        except BaseException:
            # failed to format -> return same value
            return hosts

    @staticmethod
    def cut_domain(hostname):
        for domain in cfg.DomainsToCut:
            if hostname.endswith(domain):
                return hostname[:-(len(domain) + 1)]

        return hostname


class CmdRemote(CmdResolveHosts):
    hasProgressBar = True
    related_services = 'cqueue|cqudp'

    def __init__(self):
        super(CmdRemote, self).__init__()

        self.parser.add_option(
            '-v', '--verbose', dest='verbose', action='count', default=cfg.Verbose, help='Verbose output')
        self.parser.add_option(
            '-q', '--quiet', dest='quiet', action='count', default=0, help='Quiet output')
        self.parser.add_option(
            '-s', '--smooth', dest='smooth', default=None,
            help='Smooth task start time (random delay in range (0, SMOOTH) seconds before start)')
        self.parser.add_option(
            '-t', '--timeout', dest='timeout', default=None,
            help='Operation timeout in seconds (can have float value)')
        self.parser.add_option(
            '--step', dest='step', type=int, default=0,
            help='Number of hosts for single run of task (default = 0 [all])')
        self.parser.add_option(
            '--retry', dest='retry', type=int, default=0,
            help='Number of retries for hosts with errors')
        self.parser.add_option(
            '--stop-on-error', dest='stop_on_error', action='store_true', default=False,
            help='Stop execution after first error result')
        self.parser.add_option(
            '-p', '--progress', dest='progress', action='store_true',
            default=cfg.ProgressBar and self.hasProgressBar,
            help='Use progress bar')
        self.parser.add_option(
            '--log_failed_hosts', dest='log_failed_hosts',
            help='Log all failed hosts to the specified file')

        CQTransportOptions.add_to_parser(self.parser)

        self.progressBar = None
        self.progressBarLet = None
        self.errors = []
        self.hosts = set()
        self.remained = set()
        self.results = []
        self.extra_env = {}

    def parse(self, argv):
        super(CmdRemote, self).parse(argv)

        CQTransportOptions.process_options(self.options)

        self.options.verbose -= self.options.quiet
        if self.options.verbose > 2:
            debug_log.setLevel(logging.DEBUG)
        elif self.options.verbose > 1:
            debug_log.setLevel(logging.INFO)

        try:
            extra_env = self.options.env
            self.extra_env = dict(val.split('=', 1) for val in extra_env) if extra_env else None
        except AttributeError:
            # if subcommand doesn't support --env
            self.extra_env = None

        if not self.hasProgressBar and self.options.progress:
            print >> sys.stderr, 'Progress bar is not allowed in {0} mode'.format(self.mode)
            raise SystemExit(1)

    def runner(self):
        return None

    def process_task_result(self, host, result, error, errors_only=False):
        self.host_done(host)

        if self.options.cut_domain:
            host = self.cut_domain(host)

        if error is not None:
            self.log_failed_hosts(host)
            self.errors.append((host, error))
        else:
            if not errors_only:
                self.results.append((host, result))

    def finish(self):
        process_results_return = self.process_results()
        self.process_errors()

        if self.errors or process_results_return:
            return 1
        else:
            return 0

    def do_run(self, hosts):
        try:
            with self.runner().run(hosts, self.options) as session:
                debug_log.info("session id: %s", session.id)
                for (host, result, error) in session.wait():
                    self.process_task_result(host, result, error)

        except KeyboardInterrupt:
            err = KeyboardInterrupt('Keyboard interrupt')
            for host in self.remained.copy():
                self.process_task_result(host, None, err)
            raise

    def run(self):
        code, hosts = self.resolve_hosts(self.argv)
        if code != 0:
            return code

        hosts = fix_ipv6_addresses(hosts)

        if self.options.log_failed_hosts is not None:
            try:
                with open(self.options.log_failed_hosts, 'w'):
                    pass
            except IOError:
                print >> sys.stderr, 'Failed to create log file: "{}"'.format(self.options.log_failed_hosts)
                return 1

        try_count = 0
        ret = 0
        total_errors = []

        while try_count <= self.options.retry:
            if try_count > 0:
                # retry error hosts only
                tmp_error_hosts = set([error[0] for error in total_errors])
                hosts = tmp_error_hosts if not isinstance(hosts, dict) else {
                    k: v for k, v in hosts.iteritems() if k in tmp_error_hosts
                }
                total_errors = []

            if self.options.retry:
                printOutputMessage('Try #{} of {} ...'.format(try_count+1, self.options.retry+1), error=True)

            host_pointer = 0
            hosts_list = list(set(hosts) if isinstance(hosts, dict) else hosts)

            while host_pointer < len(hosts_list):
                if self.options.step:
                    selected_hosts = hosts_list[host_pointer:host_pointer+self.options.step] \
                        if not isinstance(hosts, dict) else {
                            k: v for k, v in hosts.iteritems() if k in hosts_list[host_pointer:host_pointer+self.options.step]
                        }
                    host_pointer += self.options.step
                else:
                    selected_hosts = hosts
                    host_pointer = len(hosts_list)

                self.hosts = set(selected_hosts)
                self.remained = self.hosts.copy()
                self.results = []
                self.errors = []

                self.start_progress_bar()
                try:
                    self.do_run(selected_hosts)
                finally:
                    self.finish_progress_bar()
                    ret = self.finish()

                    if ret and self.options.stop_on_error:
                        return ret

                    total_errors.extend(self.errors)

            if len(total_errors) == 0:
                # no need to retry in case of success
                break

            try_count += 1

        return ret

    def start_progress_bar(self):
        if self.options.progress:
            pb = pbModule()
            self.progressBar = pb.ProgressBar(len(self.remained), [
                pb.SimpleProgress(sep=' / '), ' ', pb.Bar(), ' ', pb.Timer()
            ])
            self.progressBar.start()

            import threading
            self.progressBarLet = threading.Thread(target=self.update_progress_bar_periodically)
            self.progressBarLet.daemon = True
            self.progressBarLet.start()

    def update_progress_bar_periodically(self):
        while self.progressBar:
            self.update_progress_bar()
            time.sleep(1)

    def update_progress_bar(self):
        if self.progressBar:
            self.progressBar.update(len(self.hosts) - len(self.remained))

    def finish_progress_bar(self):
        if self.progressBar:
            self.progressBar.finish()
            self.progressBar = None

    def host_done(self, host):
        self.remained.discard(host)
        self.update_progress_bar()

    def just(self, s, width, ljust=True):
        s = str(s)
        if len(s) <= width or self.options.verbose > 0:
            if ljust:
                return s.ljust(width)
            else:
                return s.rjust(width)
        else:
            return s[:width - 4] + '... '

    @staticmethod
    def aggregate(results):
        aggregated = {}
        res = {}
        for host, result in results:
            resrepr = str(result)
            aggregated.setdefault(resrepr, []).append(host)
            res.setdefault(resrepr, result)

        return zip(res.itervalues(), aggregated.itervalues())

    def process_results(self):
        for result, hosts in self.aggregate(self.results):
            printOutputMessage(
                'Success on {0:d} host(s): {1:s}'.format(
                    len(hosts),
                    self.format_hosts(hosts)
                ),
                result,
                'Success on {0:d} host(s)'.format(len(hosts)) if cfg.PrintNumberOfHostsTwice else None
            )

    def process_errors(self):
        deprecated_user = False
        for error, hosts in self.aggregate(self.errors):
            if isinstance(error, Exception) and 'CQueueError' in [c.__name__ for c in error.__class__.__mro__]:
                msg = "Internal CQueue error on {cnt:d} host(s)"
                if not deprecated_user:
                    from api.cqueue.exceptions import CQueueAuthError, CQueueRuntimeError
                    if (
                        self.options.user == 'skynet' and (
                            isinstance(error, CQueueAuthError) or (
                                isinstance(error, CQueueRuntimeError) and error.message == 'Authentication failure'
                            )
                        ) or self.options.user is None and (
                            # in case of cqueue default user is skynet
                            isinstance(error, CQueueRuntimeError) and error.message == 'Authentication failure'
                        )
                    ):
                        deprecated_user = True
            else:
                msg = "Failure on {cnt:d} host(s)"

            printOutputMessage(
                (msg + ': {hosts:s}').format(
                    cnt=len(hosts),
                    hosts=self.format_hosts(hosts)
                ),
                str(error),
                msg.format(cnt=len(hosts)) if cfg.PrintNumberOfHostsTwice else None,
                error=True
            )

        if deprecated_user:
            printOutputMessage(
                "WARNING: user 'skynet' is deprecated.\nPlease use -u <user name> or -U for the current user."
            )

    def log_failed_hosts(self, host):
        if self.options.log_failed_hosts is not None:
            with open(self.options.log_failed_hosts, 'a') as f:
                f.write(host)
                f.write('\n')


class CmdUpload(CmdRemote, CmdCopier):
    related_services = 'cqueue|cqudp copier|skybone'
    usage = '\n'.join([
        '<path> <dest_path> <Blinov`s calc expression>',
        'Examples:',
        '   sky upload /etc/hosts /var/tmp I@itag_skynet_experiment',
        '   sky upload /var/tmp/folder /var/tmp h@ws1-100',
        '   sky upload -d /etc hosts /var/tmp I@itag_skynet_experiment',
        '   sky upload --hosts=hosts.txt /etc/hosts /var/tmp',
    ])
    mode = 'upload'
    description = 'upload data to the specified path on target servers'

    def __init__(self):
        super(CmdUpload, self).__init__()

        self.parser.add_option(
            '-d', '--dir', dest='dir', action='store', default=None,
            help='Target directory for upload path. By default cwd is used.'
        )
        self.parser.conflict_handler = 'resolve'
        self.parser.add_option(
            '-N', '--network', dest='network', action='store', default=None,
            choices=('Auto', 'Backbone', 'Fastbone'),
            help='Choose preferred network to use (Auto, Fastbone or Backbone)'
        )
        self._upload_runner = None

    def parse(self, argv):
        return CmdRemote.parse(self, argv)

    def runner(self):
        return TaskRunnerWrapper(self._upload_runner)

    def run(self):
        import api.copier

        want_args = (2 if self.options.hosts else 3)
        if len(self.argv) < want_args:
            print >> sys.stderr, 'Invalid usage (want {0:d} arguments, got {1:d})\n'.format(want_args, len(self.argv))
            self.help()
            return 1

        src_path = self.argv.pop(0)
        tgt_path = self.argv.pop(0)

        try:
            # we have to import new 'requests' module before Copier().create
            # otherwise old 'requests' module will be loaded and it breaks resolver
            import api.hq
            handle = api.copier.Copier().create([src_path], cwd=self.options.dir)
            resid = handle.resid()
        except Exception as err:
            code = self.analyze_error(err)
            return self.print_error(err, code)

        from library.tasks.upload import UploadRunner
        self._upload_runner = UploadRunner(resid, tgt_path, network=self.options.network)

        return CmdRemote.run(self)

    def process_results(self):
        if self.options.verbose <= 0:
            return

        successful_hosts = []
        for host, result in self.results:
            successful_hosts.append(host)
        if successful_hosts:
            printOutputMessage(
                'Upload successful on {0:d} host(s): {1}'.format(
                    len(successful_hosts),
                    self.format_hosts(successful_hosts)
                )
            )


class CmdDownload(CmdRemote, CmdCopier):
    related_services = 'cqueue|cqudp copier|skybone'
    usage = '\n'.join([
        '<remote_path> <dest_path> <Blinov`s calc expression>',
        'Examples:',
        '   sky download /etc/hosts ./ I@itag_skynet_experiment',
        '   sky download /var/tmp/folder /var/tmp I@itag_skynet_experiment',
        '   sky download -t 60 -v /etc/hosts /var/tmp I@itag_skynet_experiment',
        '   sky download --hosts=hosts.txt /etc/hosts ./',
    ])
    mode = 'download'
    description = 'download data from target servers to the localhost'

    def __init__(self):
        super(CmdDownload, self).__init__()
        self._download_runner = None

    def runner(self):
        return TaskRunnerWrapper(self._download_runner)

    def run(self):
        import api.copier

        want_args = (2 if self.options.hosts else 3)
        if len(self.argv) < want_args:
            print >> sys.stderr, 'Invalid usage (want {0:d} arguments, got {1:d})\n'.format(want_args, len(self.argv))
            self.help()
            return 1

        remote_src_path = self.argv.pop(0)
        local_dest_path = self.argv.pop(0)

        from library.tasks.download import DownloadRunner
        self._download_runner = DownloadRunner(remote_src_path)

        CmdRemote.run(self)

        if not self.results:
            return 1

        c = api.copier.Copier()
        handles = []

        for host, resid in self.results:
            try:
                path = os.path.join(local_dest_path, host)
                if not os.path.exists(path):
                    oldUmask = os.umask(0)
                    try:
                        os.makedirs(path)
                    finally:
                        os.umask(oldUmask)

                handles.append(c.handle(resid).get(path))
            except Exception as err:
                if self.options.traceback:
                    from kernel.util.errors import formatException
                    print >> sys.stderr, formatException()
                else:
                    print >> sys.stderr, 'Error: {0}'.format(err)

        if not handles:
            return 1

        all_ok = True

        for handle in handles:
            try:
                handle.wait()
            except Exception as err:
                all_ok = False
                if self.options.traceback:
                    from kernel.util.errors import formatException
                    print >> sys.stderr, formatException()
                else:
                    print >> sys.stderr, 'Error: {0}'.format(err)

        return 0 if all_ok else 1

    def process_results(self):
        if self.options.verbose <= 0:
            return

        successful_hosts = []
        for host, result in self.results:
            successful_hosts.append(host)

        if self.results:
            printOutputMessage('Create resource successful on {0:d} host(s): {1:>s}'.format(
                len(self.results), self.format_hosts([host for host, result in self.results])
            ))


class CmdListConf(Cmd):
    usage = '<service name> (optional)'
    example = 'production_sas_mmeta'
    mode = 'listconf'
    description = 'print list of available configurations: no service == CMS, service specified == HQ'

    def __init__(self):
        super(CmdListConf, self).__init__(argvreqd=False)

        self.parser.add_option(
            '--use-nanny-service-resolve-policy', dest='use_service_resolve_policy', action='store_true',
            default=cfg.UseNannyServiceResolvePolicy, help='Use nanny service resolve policy from nanny api')
        self.parser.add_option(
            '--no-use-nanny-service-resolve-policy', dest='use_service_resolve_policy', action='store_false',
            default=cfg.UseNannyServiceResolvePolicy, help='Use nanny service resolve policy from nanny api')

    def run(self):
        if self.argv:
            use_service_resolve_policy = getattr(self.options, 'use_service_resolve_policy', cfg.UseNannyServiceResolvePolicy)
            from api.hq import HQResolver, HQResolverError
            try:
                result = ''
                for service in self.argv:
                    try:
                        hq_resolver = HQResolver(use_service_resolve_policy=use_service_resolve_policy)
                    except TypeError:  # no support for use_service_resolve_policy
                        hq_resolver = HQResolver()
                    configs = hq_resolver.get_configurations(service)
                    for k, v in configs.iteritems():
                        active = v.get('Active', 0)
                        installed = v.get('Installed', 0)
                        total = v.get('Total', 0)
                        state = 'UNKNOWN'
                        if total:
                            if active == total:
                                state = 'ACTIVE'
                            elif active:
                                state = 'PARTIALLY ACTIVE'
                            else:
                                state = 'INSTALLED'

                        result += '{}{:20} ({:^5}/{:^5}/{:^5}) {}'.format(
                            '\n' if result else '',
                            state,
                            active,
                            installed,
                            total,
                            k)

                print('-' * 80)
                print('       State         (  A  /  I  /  T  )   Configuration name')
                print('-' * 80)
                print(result)
            except HQResolverError as ex:
                printOutputMessage('ERROR: {}'.format(ex), error=True)
        else:
            from api.cms import Registry

            for conf in Registry.listConf():
                print(conf)


class CmdList(CmdResolveHosts):
    usage = '<Blinov`s calc expression>'
    example = 'H@ASEARCH - I@SEARCH2'
    mode = 'list'
    description = 'print list of hosts by group names or host names'

    def __init__(self):
        super(CmdList, self).__init__()
        self.parser.add_option('-f', '--format', dest='format', default=cfg.ListFormat,
                               choices=('ls', 'compact', 'msgpack', 'json'),
                               help='Output format (ls, compact, json, msgpack)')

        self.errors = []
        self.hosts = set()
        self.results = []

    def run(self):
        code, resolved_hosts = self.resolve_hosts(self.argv)
        if code != 0:
            return code

        try:
            if self.options.cut_domain:
                resolved_hosts = [self.cut_domain(h) for h in resolved_hosts]

            if self.options.format == 'compact':
                print(self.format_hosts(resolved_hosts))
                return 0

            res = sorted(resolved_hosts)

            if self.options.format == 'ls':
                for i in res:
                    print(i)
            elif self.options.format == 'msgpack':
                import msgpack
                print(msgpack.dumps(res))
            elif self.options.format == 'json':
                try:
                    import simplejson as json
                except ImportError:
                    json = None

                if json is None:
                    import json
                print(json.dumps(res))

        except EnvironmentError as err:
            if err.errno != errno.EPIPE:
                raise

        return 0


class CmdListInstances(CmdResolveHosts):
    usage = '<Blinov`s calc expression>'
    example = 'gencfg@GROUP1 - gencfg@GROUP2:TAG'
    mode = 'listinstances'
    description = 'print list of instances (only new syntax supported)'

    def __init__(self):
        super(CmdListInstances, self).__init__()
        self.parser.add_option(
            '-f', '--format', dest='format', default=cfg.ListFormat,
            choices=('ls', 'msgpack', 'json'),
            help='Output format (ls, json, msgpack). Default: %s' % cfg.ListFormat)
        self.parser.add_option(
            '--no_tag', dest='no_tag', action='store_true', default=False,
            help='Cut tag from the instance name.'
        )
        self.parser.remove_option('--new_syntax')
        self.parser.remove_option('--old_syntax')

    def run(self):
        self.options.mode = 'i'
        self.options.host_syntax = 'N'
        code, resolved_hosts = self.resolve_hosts(self.argv)
        if code != 0:
            return code

        resolved_instances = []
        for host, instances in resolved_hosts.items():
            for instance in instances:
                if self.options.no_tag:
                    resolved_instances.append(instance[1].split('@')[0])
                else:
                    resolved_instances.append(instance[1])

        try:
            res = sorted(resolved_instances)

            if self.options.format == 'ls':
                for i in res:
                    print(i)
            elif self.options.format == 'msgpack':
                import msgpack
                print(msgpack.dumps(res))
            elif self.options.format == 'json':
                try:
                    import simplejson as json
                except ImportError:
                    json = None

                if json is None:
                    import json
                print(json.dumps(res))

        except EnvironmentError as err:
            import errno
            if err.errno != errno.EPIPE:
                raise

        return 0


class CmdPing(CmdRemote):
    usage = '<Blinov`s calc expression>'
    example = 'I@ASEARCH - K@SEARCH2'
    mode = 'ping'
    description = 'ping remote machine(s)'

    def __init__(self):
        super(CmdPing, self).__init__()
        self.parser.add_option(
            '--extended', dest='extended', action='store_true', default=False,
            help='Use extended ping that thoroughly checks CQueue on host and does not require authorization'
        )
        self.parser.add_option(
            '--check-writable', dest='check_writable', action='store_true', default=False,
            help='Perform additional check if the cqueue eggs directory is writable'
        )

    def runner(self):
        from library.tasks.ping import PingRunner
        return TaskRunnerWrapper(PingRunner())

    def do_run(self, hosts):
        if not self.options.extended:
            return super(CmdPing, self).do_run(hosts)

        check_cqueue_usage(self.options)

        from api.cqueue import Client, CQWarning

        # we display own warning
        warnings.simplefilter('ignore', category=CQWarning)

        try:
            with Client(self.options.cqueueImplementation).ping(
                    hosts, checkWritable=self.options.check_writable
            ) as session:
                debug_log.debug("starting ping(checkWritable=%r)", self.options.check_writable)
                debug_log.info("session id: %s", session.id)
                timeout = float(self.options.timeout) if self.options.timeout is not None else None
                for host, result, error in session.wait(timeout=timeout):
                    self.process_task_result(host, result, error)

                err = IOError(errno.ETIMEDOUT, "Ping timed out")
                for host in self.remained.copy():
                    self.process_task_result(host, None, err)
        except KeyboardInterrupt:
            err = KeyboardInterrupt('Keyboard interrupt')
            for host in self.remained.copy():
                self.process_task_result(host, None, err)
            raise

    def process_results(self):
        if self.options.verbose <= 0:
            if not self.errors:
                printOutputMessage('All Ok')
            return
        successful_hosts = []
        for host, result in self.results:
            successful_hosts.append(host)
        printOutputMessage(
            'Ping Ok on {0:d} host(s): {1}'.format(
                len(successful_hosts),
                self.format_hosts(successful_hosts)
            )
        )


class CmdRun(CmdRemote, CmdStreamOutput):
    usage = '<command> <Blinov`s calc expression>'
    mode = 'run'
    example = '"ls -la /tmp" H@ASEARCH - I@SEARCH2'
    description = 'run shell command on remote machine(s)'

    def __init__(self):
        super(CmdRun, self).__init__()

        self.parser.add_option(
            '--mode', dest='mode', type=str, default=cfg.Mode,
            help='Set mode "h" (hosts [set by default]), "s" (shards), "i" (instances)')
        self.parser.add_option(
            '--no_wait', action='store_true', dest='noWait',
            default=False, help='Do not wait for command execution end')
        self.parser.add_option(
            '--separate', dest='separateOutput', action='store_true',
            default=False, help='Separate each output per host')
        self.parser.add_option(
            '--show-cmd', dest='showCmd', action='store_true',
            default=cfg.ShowCommandOnFailure, help='show executed command upon failure',
        )
        self.parser.add_option(
            '--stream', dest='stream', action='store_true', default=False,
            help='Interactively show remote host(s) intermediate output')
        self.parser.add_option(
            '--data-limit', dest='data_limit', type='int', default=1 << 20,
            help='Limit per-host output size')
        self.parser.add_option(
            '--small-port-range', dest='port_range', action='store_true', default=None,
            help='Run task on small port range (specified in cqudp config)'
        )
        self.parser.add_option(
            '--env', action='append', dest='env', type=str, metavar='KEY=VALUE', default=[],
            help='Add custom environment variables. No additional value escape is needed',
        )
        self.parser.add_option(
            '--sort', dest='sort_mode', default=None, choices=['text', 'number'],
            help='Sort results. Available sorting modes: %s' % ['text', 'number']
        )
        self.parser.add_option(
            '-F', '--file', dest='file', default=None,
            help='Read command from file, dont read arguments')

        self.extra_env = {}
        self.entities_by_hosts = {}
        self.command = None
        self.argv = None

    def parse(self, argv):
        super(CmdRun, self).parse(argv)

        extra_env = self.options.env
        self.extra_env = dict(val.split('=', 1) for val in extra_env) if extra_env else None

        if self.options.file is not None:
            try:
                with open(self.options.file) as f:
                    self.command = f.read().strip()
            except EnvironmentError as ex:
                printOutputMessage(
                    'Failed to read commands from {0!r}: {1}'.format(self.options.file, str(ex)),
                    error=True
                )
                raise SystemExit(1)
        else:
            if self.argv:
                self.command = self.argv[0]
            else:
                printOutputMessage(
                    'Warning: no command to execute',
                    error=True
                )
                raise SystemExit(1)

            self.argv = self.argv[1:]

        if self.options.stream:
            for opt in [('progress', None), ('detach', None), ('noWait', 'no_wait')]:
                if getattr(self.options, opt[0], None):
                    printOutputMessage(
                        '"stream" mode is incompatible with "{0}" mode.'.format(opt[1] if opt[1] else opt[0]),
                        error=True
                    )
                    raise SystemExit(1)

    def resolve_hosts(self, clause):
        code, hosts = super(CmdRun, self).resolve_hosts(clause)

        # check cross-dc in case of successful resolving only
        if code == 0:
            self._check_crossdc_dangerous_cmd(hosts)

        return code, hosts

    def do_run(self, hosts):
        self.entities_by_hosts = hosts

        if not self.options.stream:
            return super(CmdRun, self).do_run(hosts)

        return self.do_run_stream(hosts)

    def stream_runner(self):
        from library.tasks.command import IterCommandRunner
        return TaskRunnerWrapper(IterCommandRunner(self.command, env=self.extra_env))

    def runner(self):
        from library.tasks.command import CommandRunner, ShardCommandRunner, InstanceCommandRunner

        if self.options.mode == 's':
            return TaskRunnerWrapper(ShardCommandRunner(
                self.entities_by_hosts,
                self.command,
                self.options.noWait,
                env=self.extra_env,
            ))
        if self.options.mode == 'i':
            return TaskRunnerWrapper(InstanceCommandRunner(
                self.entities_by_hosts,
                self.command,
                self.options.noWait,
                env=self.extra_env,
            ))

        # cqudp can handle shell commands in a more efficient way but ...
        # for 'smooth' task we will use old method and run command as a task
        if self.options.cqueueImplementation == 'cqudp' and not self.options.smooth:
            return ShellCommandWrapper(self.command, extra_env=self.extra_env)

        return TaskRunnerWrapper(CommandRunner(
            self.command,
            self.options.noWait,
            env=self.extra_env,
        ))

    def process_results(self):
        if self.options.mode == 's':
            return self.__process_results('shards')
        if self.options.mode == 'i':
            return self.__process_results('instances')

        return_code = None
        if not self.options.separateOutput:
            results_out = {}
            results_err = {}

            for host, result in self.results:
                if isinstance(result, tuple) and len(result) == 3:
                    # valid result
                    (stdout, stderr, ret_code) = result
                else:
                    # result invalid, inform user somehow
                    stdout = ''
                    stderr = '[{}] - result from unsupported version of skynet. Please update it!'.format(
                        str(result).strip())
                    ret_code = 0

                if stdout or self.options.verbose > 0:
                    results_out.setdefault((stdout.strip(), ret_code), list()).append(host)
                if stderr:
                    results_err.setdefault((stderr.strip(), ret_code), list()).append(host)
                if ret_code != 0 or stderr:
                    return_code = 1

            if self.options.sort_mode:
                tmp_results = sorted(results_out) if self.options.sort_mode == 'text' else self._human_sort(
                    results_out.keys())
            else:
                tmp_results = results_out.keys()

            for (stdout, ret_code) in tmp_results:
                hosts = results_out[(stdout, ret_code)]
                if stdout and self.options.verbose < 0:
                    sys.stdout.write("{0}\n".format(stdout))
                    sys.stdout.flush()
                elif ret_code == 0:
                    printOutputMessage(
                        'Success on {0:d} host(s): {1}'.format(len(hosts), self.format_hosts(hosts)),
                        stdout,
                        'Success on {0:d} host(s)'.format(len(hosts)) if cfg.PrintNumberOfHostsTwice else None
                    )
                else:
                    printOutputMessage(
                        'Error on {0:d} host(s) [out]: {1}'.format(len(hosts), self.format_hosts(hosts)),
                        stdout,
                        ('Error on {0:d} host(s)\n'.format(len(hosts)) if cfg.PrintNumberOfHostsTwice else '') +
                        ('The executed command was: {0}\n'.format(self.command) if self.options.showCmd else '') +
                        'Return code: {0:d}'.format(ret_code)
                    )

            if self.options.sort_mode:
                tmp_results = sorted(results_err) if self.options.sort_mode == 'text' else self._human_sort(
                    results_err.keys())
            else:
                tmp_results = results_err.keys()

            for (stderr, ret_code) in tmp_results:
                hosts = results_err[(stderr, ret_code)]
                if stderr and self.options.verbose < 0:
                    sys.stderr.write("{0}\n".format(stderr))
                    sys.stderr.flush()
                elif ret_code == 0:
                    printOutputMessage(
                        'Error on {0:d} host(s): {1}'.format(len(hosts), self.format_hosts(hosts)),
                        stderr,
                        'Error on {0:d} host(s)'.format(len(hosts)) if cfg.PrintNumberOfHostsTwice else None,
                        error=True
                    )
                else:
                    printOutputMessage(
                        'Error on {0:d} host(s) [err]: {1}'.format(len(hosts), self.format_hosts(hosts)),
                        stderr,
                        ('Error on {0:d} host(s)\n'.format(len(hosts)) if cfg.PrintNumberOfHostsTwice else '') +
                        ('The executed command was: {0}\n'.format(self.command) if self.options.showCmd else '') +
                        'Return code: {0:d}'.format(ret_code),
                        error=True
                    )
        else:
            for host, result in self.results:
                if isinstance(result, tuple) and len(result) == 3:
                    # valid result
                    (stdout, stderr, ret_code) = result
                else:
                    # result invalid, inform user somehow
                    stdout = ''
                    stderr = '[{}] - result from unsupported version of skynet. Please update it!'.format(
                        str(result).strip())
                    ret_code = 0

                if self.options.verbose < 0:
                    if stdout:
                        sys.stdout.write("{0}\n".format(stdout))
                        sys.stdout.flush()
                    if stderr:
                        sys.stderr.write("{0}\n".format(stderr))
                        sys.stderr.flush()
                        return_code = 1
                elif ret_code == 0:
                    if stdout or self.options.verbose > 0:
                        printOutputMessage('Success on host: {0}'.format(host), stdout)
                    if stderr:
                        printOutputMessage('Error on host: {0}'.format(host), stderr, error=True)
                        return_code = 1
                else:
                    return_code = 1
                    if stdout or self.options.verbose > 0:
                        printOutputMessage(
                            'Error on host [out]: {0}'.format(host), stdout,
                            ('The executed command was: {0}\n'.format(self.command) if self.options.showCmd else '') +
                            'Return code: {0:d}'.format(ret_code)
                        )
                    if stderr:
                        printOutputMessage(
                            'Error on host [err]: {0}'.format(host), stderr,
                            ('The executed command was: {0}\n'.format(self.command) if self.options.showCmd else '') +
                            'Return code: {0:d}'.format(ret_code), error=True
                        )
        return return_code

    def __process_results(self, name):
        return_code = None
        if not self.options.separateOutput:
            results_out = {}
            results_err = {}
            entities_by_hosts = {}

            for host, host_results in self.results:
                entities_by_hosts[host] = set(host_results.keys())

            for host, host_results in self.results:
                for instance, (stdout, stderr, retCode) in host_results.iteritems():
                    if stdout or self.options.verbose > 0:
                        results_out.setdefault(
                            (stdout.strip(), retCode), dict()
                        ).setdefault(
                            host, set()
                        ).add(instance)
                    if stderr:
                        results_err.setdefault(
                            (stderr.strip(), retCode), dict()
                        ).setdefault(
                            host, set()
                        ).add(instance)
                    if retCode != 0 or stderr:
                        return_code = 1

            def parse_result_on_entities_by_hosts(hosts):
                same_result_on_hosts = []
                diff_result_host_entities = []
                for host, entities in hosts.iteritems():
                    if entities == entities_by_hosts[host]:
                        same_result_on_hosts.append(host)
                    else:
                        diff_result_host_entities.append('{0} ({1})'.format(host, ' ,'.join(entities)))

                return same_result_on_hosts, diff_result_host_entities

            for (stdout, retCode), hosts in results_out.iteritems():
                same_result_on_hosts, diff_result_host_entities = parse_result_on_entities_by_hosts(hosts)

                if stdout and self.options.verbose < 0:
                    sys.stdout.write("{0}\n".format(stdout))
                    sys.stdout.flush()
                elif retCode == 0:
                    message = ''
                    if same_result_on_hosts:
                        message += 'Success on {:d} host(s) (on all {} same result): {}'.format(
                            len(same_result_on_hosts), name, self.format_hosts(same_result_on_hosts)
                        )
                    if same_result_on_hosts and diff_result_host_entities:
                        message += '\nAnd\n'
                    if diff_result_host_entities:
                        message += 'Success on {:d} host(s) (by {}): {}'.format(
                            len(diff_result_host_entities), name,
                            ' '.join(diff_result_host_entities)
                        )
                    printOutputMessage(message, stdout)
                else:
                    message = ''
                    if same_result_on_hosts:
                        message += 'Error on {:d} host(s) (on all {} same result) [out]: {}'.format(
                            len(same_result_on_hosts), name, self.format_hosts(same_result_on_hosts)
                        )
                    if same_result_on_hosts and diff_result_host_entities:
                        message += '\nAnd\n'
                    if diff_result_host_entities:
                        message += 'Error on {:d} host(s) (by {}) [out]: {}'.format(
                            len(diff_result_host_entities), name, ' '.join(diff_result_host_entities)
                        )
                    printOutputMessage(message, stdout, 'Return code: {0:d}'.format(retCode))

            for (stderr, retCode), hosts in results_err.iteritems():
                same_result_on_hosts, diff_result_host_entities = parse_result_on_entities_by_hosts(hosts)

                if stderr and self.options.verbose < 0:
                    sys.stderr.write("{0}\n".format(stderr))
                    sys.stderr.flush()
                elif retCode == 0:
                    message = ''
                    if same_result_on_hosts:
                        message += 'Error on {:d} host(s) (on all {} same result): {}'.format(
                            len(same_result_on_hosts), name, self.format_hosts(same_result_on_hosts)
                        )
                    if same_result_on_hosts and diff_result_host_entities:
                        message += '\nAnd\n'
                    if diff_result_host_entities:
                        message += 'Error on {:d} host(s) (by {}): {}'.format(
                            len(diff_result_host_entities), name,
                            ' '.join(diff_result_host_entities)
                        )
                    printOutputMessage(message, stderr, error=True)
                else:
                    message = ''
                    if same_result_on_hosts:
                        message += 'Error on {:d} host(s) (on all {} same result) [err]: {}'.format(
                            len(same_result_on_hosts), name, self.format_hosts(same_result_on_hosts)
                        )
                    if same_result_on_hosts and diff_result_host_entities:
                        message += '\nAnd\n'
                    if diff_result_host_entities:
                        message += 'Error on {:d} host(s) (by {}) [err]: {}'.format(
                            len(diff_result_host_entities), name, ' '.join(diff_result_host_entities)
                        )
                    printOutputMessage(message, stderr, 'Return code: {0:d}'.format(retCode), error=True)
        else:
            for host, host_results in self.results:
                for instance, (stdout, stderr, retCode) in host_results.iteritems():
                    if self.options.verbose < 0:
                        if stdout:
                            sys.stdout.write("{0}\n".format(stdout))
                            sys.stdout.flush()
                        if stderr:
                            sys.stderr.write("{0}\n".format(stderr))
                            sys.stderr.flush()
                            return_code = 1
                    elif retCode == 0:
                        if stdout or self.options.verbose > 0:
                            printOutputMessage(
                                'Success on host "{}" on {} "{}"'.format(host, name, instance), stdout
                            )
                        if stderr:
                            printOutputMessage(
                                'Error on host "{}" on {} "{}"'.format(host, name, instance), stderr, error=True
                            )
                            return_code = 1
                    else:
                        return_code = 1
                        if stdout or self.options.verbose > 0:
                            printOutputMessage(
                                'Error on host [out] "{}" on {} "{}"'.format(host, name, instance), stdout,
                                'Return code: {0:d}'.format(retCode)
                            )
                        if stderr:
                            printOutputMessage(
                                'Error on host [err] "{}" on {} "{}"'.format(host, name, instance), stderr,
                                'Return code: {0:d}'.format(retCode), error=True
                            )
        return return_code

    def process_errors(self):
        if not self.errors:
            return

        if self.options.separateOutput:
            for host, error in self.errors:
                printOutputMessage(
                    'Internal CQueue error on host: {0}'.format(host), str(error)
                )
        else:
            super(CmdRun, self).process_errors()

    def _is_dangerous_cmd(self, hosts):
        if len(hosts) <= cfg.HostsNumThreshold or (self.options.step and self.options.step <= cfg.HostsNumThreshold):
            return None

        user = self.options.user or getUserName()
        client_opts = dict(p.split(':', 1) for p in getattr(self.options, 'client_opts', []))

        if user == 'root':
            return 'command as root'

        if 'sudo ' in self.command:
            return 'sudo command'

        if client_opts.get('no_porto', 'false').lower() == 'true':
            return 'not isolated command'

        return None

    def _check_crossdc_dangerous_cmd(self, hosts):
        dangerous_cmd = self._is_dangerous_cmd(hosts)
        if dangerous_cmd and self._is_crossdc(hosts):
            message = "You're going to run " + dangerous_cmd + " in more than one DC."

            printOutputMessage(
                "WARNING! {msg}\n"
                "Please run this command per DC, for instance, using d@ prefix:\n"
                "   sky run \"{cmd}\" [{selector}] . d@sas\n"
                " or using --step option:\n"
                "   sky run --step=100 \"{cmd}\" [{selector}]\n\n"
                "Available DCs are: {dcs}".format(msg=message, cmd=self.command, selector=' '.join(self.argv), dcs=', '.join(cfg.KnownDCs)),
                error=True
            )
            sys.exit(1)

    @staticmethod
    def _is_crossdc(hosts):
        from library.sky.hostresolver.resolver import Resolver
        dc_count = 0
        for dc in cfg.KnownDCs:
            try:
                dc_hosts = Resolver().resolveHosts('d@' + dc)
                if len(hosts & dc_hosts):
                    dc_count += 1
            except BaseException:
                # failed to resolve DC, use heuristics
                if any(dc in host for host in hosts):
                    dc_count += 1

            if dc_count > 1:
                return True

        return False

    @staticmethod
    def _human_sort(l):
        import re
        convert = lambda text: int(text) if text.isdigit() else text
        alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key[0])]
        return sorted(l, key=alphanum_key)


class CmdLoadAvg(CmdRemote):
    usage = '<Blinov`s calc expression>'
    mode = 'load'
    example = 'H@ASEARCH - I@SEARCH2'
    description = 'prints current load average on target hosts'

    def __init__(self):
        super(CmdLoadAvg, self).__init__()
        self.parser.add_option(
            '-e',
            '--extended',
            action='store_true',
            dest='extended',
            default=False,
            help='Show extended information with CPU count and load percent'
        )

    def runner(self):
        from library.tasks.loadavg import LoadAvgRunner
        return TaskRunnerWrapper(LoadAvgRunner(self.options.extended))

    def process_results(self):
        maxhostlen = (max(len(host) for host, _ in self.results) + 5) if self.results else 20

        if self.options.extended:
            self.results.sort(lambda a, b: cmp(
                (a[1][2], a[1][0], a[1][1], a[0]), (b[1][2], b[1][0], b[1][1], b[0])))
            for host, load_avg in self.results:
                load_average1, cpu_count, load_per_cpu = load_avg
                print(self.just(host, maxhostlen)
                      + self.just('{0:0.4f}'.format(load_average1), 15, False)
                      + self.just('{0}'.format(cpu_count), 10, False)
                      + self.just('{0:0.2f}'.format(load_per_cpu), 15, False))
        else:
            self.results.sort(lambda a, b: cmp((a[1], a[0]), (b[1], b[0])))
            for host, load_avg in self.results:
                print(self.just(host, maxhostlen) + self.just('{0:0.4f}'.format(load_avg), 10, False))


class CmdPortoRun(CmdRemote, CmdStreamOutput):
    usage = '<command> <Blinov`s calc expression>'
    mode = 'portorun'
    example = '"ps aufxwww" f@mail_search_prod'
    description = 'run shell command in remote service container(s)'
    related_services = 'cqudp'

    def __init__(self):
        cfg.CQueueImplementation = 'cqudp'  # FIXME (torkve) sooo dirty
        CQTransportOptions.supported_implementations = ['cqudp']  # FIXME (torkve) shame on me!

        super(CmdPortoRun, self).__init__()
        self.command = None
        self.argv = None
        self.extra_env = {}

        self.parser.add_option(
            '--stream', dest='stream', action='store_true', default=False,
            help='Interactively show remote host(s) intermediate output')
        self.parser.add_option(
            '--data-limit', dest='data_limit', type='int', default=1 << 20,
            help='Limit per-host output size')
        self.parser.add_option(
            '--small-port-range', dest='port_range', action='store_true', default=None,
            help='Run task on small port range (specified in cqudp config)'
        )
        self.parser.add_option(
            '--env', action='append', dest='env', type=str, metavar='KEY=VALUE', default=[],
            help='Add custom environment variables. No additional value escape is needed',
        )

        self.parser.defaults['currentUser'] = False  # FIXME
        self.parser.get_option('-U').action = 'store_false'

    def parse(self, argv):
        super(CmdPortoRun, self).parse(argv)
        self.command = self.argv[0]
        self.argv = self.argv[1:]
        # force proper mode ('f' means family, special mode which is not visible for user)
        self.options.mode = 'f'
        self.options.host_syntax = 'N'

        extra_env = self.options.env
        self.extra_env = dict(val.split('=', 1) for val in extra_env) if extra_env else None

    def runner(self):
        return PortoshellCommandWrapper(self.command, extra_env=self.extra_env)

    def stream_runner(self):
        return PortoshellCommandWrapper(self.command, extra_env=self.extra_env)

    def do_run(self, hosts):
        if not self.options.stream:
            return super(CmdPortoRun, self).do_run(hosts)

        return self.do_run_stream(hosts)

    def process_results(self):
        results_out = defaultdict(set)
        results_err = defaultdict(set)
        for host, (mark, out) in self.results:

            if out:
                if mark == 'stdout':
                    results_out[out].add(str(host))
                elif mark == 'stderr':
                    results_err[out].add(str(host))

        for stdout, hosts in results_out.iteritems():
            printOutputMessage('Success on slots(s) %s' % ','.join(hosts), stdout)

        for stderr, hosts in results_err.iteritems():
            printOutputMessage('Error on slots(s) %s' % ','.join(hosts), stderr)

        return 1 if len(results_err) else 0


class CmdTop(CmdRemote):
    usage = '<Blinov`s calc expression>'
    mode = 'top'
    example = 'H@SEARCH - I@SEARCH2'
    description = 'print top-like statistics'

    def __init__(self):
        super(CmdTop, self).__init__()

        self.parser.add_option('', '--sortby', dest='sortby', action='store', default='IDLE', help='Sort field')

    def runner(self):
        from library.tasks.top import StatCollector
        return TaskRunnerWrapper(StatCollector())

    def process_results(self):
        from library.tasks.top import printTop
        printTop(self.results, self.options.sortby)


class CmdUptime(CmdRemote):
    usage = '<Blinov`s calc expression>'
    mode = 'uptime'
    example = 'H@SEARCH - I@SEARCH2'
    description = 'prints current uptime on target hosts'

    def __init__(self):
        super(CmdUptime, self).__init__()

        self.parser.add_option(
            '--text',
            dest='text',
            action='store_true',
            default=False,
            help='Display uptime in text format [by default in seconds]')

    def runner(self):
        from library.tasks.uptime import UptimeRunner
        return TaskRunnerWrapper(UptimeRunner(self.options.text))

    def process_results(self):
        maxhostlen = (max(len(host) for host, _ in self.results) + 5) if self.results else 20

        if not self.options.text:
            # sort result only if uptime in seconds
            self.results.sort(lambda a, b: -cmp((a[1], a[0]), (b[1], b[0])))
        for host, uptime in self.results:
            print(self.just(host, maxhostlen) + uptime if self.options.text else self.just('{0:0.0f}'.format(uptime), 20, False))


class CmdVersion(CmdRemote):
    usage = '<Blinov`s calc expression>'
    mode = 'version'
    example = 'H@SEARCH - I@SEARCH2'
    description = 'get skynet version info'

    def __init__(self):
        super(CmdVersion, self).__init__()

        self.parser.add_option(
            '--svn', action='store_true', default=False,
            help='Show svn url and revision instead of version'
        )

    def runner(self):
        from library.tasks.version import VersionRunner
        return TaskRunnerWrapper(VersionRunner())

    def process_task_result(self, host, result, error, errors_only=False):
        if isinstance(result, dict):
            if not self.options.svn:
                result = result.get('version', None)
            else:
                result = '%s@%s' % (result.get('url', ''), result.get('revision', ''))
        return super(CmdVersion, self).process_task_result(host, result, error, errors_only=errors_only)


class CmdLaunch(CmdRemote):
    """[remoteWorkingDirectory] [localBinPath] ([binParams, ..])"""

    example = '/hol/search/database dbg_meta -p 11211'
    mode = 'launch'
    description = 'running binary on remote hosts'

    def __init__(self):
        super(CmdLaunch, self).__init__()
        self.parser.add_option(
            '', '--no_wait', action='store_true', dest='noWait', default=False,
            help='Do not wait for command execution end'
        )
        self.parser.set_defaults(hosts='./hosts.txt')
        self.remote_wd = ''
        self.local_bin = ''
        self.execution_params = None

    def parse(self, argv):
        super(CmdLaunch, self).parse(argv)
        self.remote_wd = self.argv[0]
        self.local_bin = self.argv[1]
        self.execution_params = self.argv[2:]
        self.argv = []

    def runner(self):
        from library.tasks.launch import LaunchRunner
        return TaskRunnerWrapper(
            LaunchRunner(
                self.remote_wd, self.local_bin, self.execution_params, self.options.noWait
            )
        )

    def process_results(self):
        for host, result in self.results:
            if result[0] or self.options.verbose > 0:
                printOutputMessage(
                    'Success on host: {0}'.format(host),
                    result[0], 'Return code: {0:d}'.format(result[2])
                )
            if result[1]:
                printOutputMessage(
                    'Error on host: {0}'.format(host),
                    result[1], error=True
                )


class CmdShell(CmdResolveSingleHost):
    """host [command [args]]"""
    related_services = 'cqueue|cqudp'
    mode = 'shell'
    description = 'access remote host using shell'

    def __init__(self):
        super(CmdShell, self).__init__()

        self.parser.add_option(
            '-v', '--verbose', dest='verbose', action='count', default=cfg.Verbose, help='Verbose output')
        self.parser.add_option(
            '-q', '--quiet', dest='quiet', action='count', default=0, help='Quiet output')

        CQTransportOptions.add_to_parser(self.parser)

    def parse(self, argv):
        super(CmdShell, self).parse(argv)
        CQTransportOptions.process_options(self.options)

        if self.options.verbose > 2:
            debug_log.setLevel(logging.DEBUG)
        elif self.options.verbose > 1:
            debug_log.setLevel(logging.INFO)

    def run(self):
        from library.tasks.shell import RunShell, RunCommand
        if not self.argv:
            self.help()
            return 1
        commands = self.argv[1:]
        if len(commands):
            task = RunCommand(commands)
        else:
            task = RunShell()

        code, host = self.resolve_single_host(self.argv[:1])
        if code != 0:
            return code

        self.run_remote_shell(host, task)

    def run_remote_shell(self, host, task):
        from library.tasks.shell import ShellRunner

        job = TaskRunnerWrapper(
            ShellRunner(
                term_type=os.environ.get('TERM', 'vt100'), command=task, iterate=True
            )
        ).run(
            [host],
            self.options,
            iterate=True
        )

        ok = False
        results = job.poll()

        for host, result, failure in results:
            if failure is not None and not isinstance(failure, StopIteration):
                print('{0}: {1}'.format(host, failure))

            if result is not None:
                try:
                    telnet_port = int(result)
                    ok = True
                except:
                    print('{0}: {1}'.format(host, result))

        if not ok:
            sys.exit(1)

        self.run_telnet(host, telnet_port)

    @staticmethod
    def run_telnet(host, port):
        import time
        from library.tasks.shell import TelnetClient
        attempts = 0
        while True:
            try:
                client = TelnetClient()
                client.connect(host, port)
            except KeyboardInterrupt:
                raise
            except Exception:
                attempts += 1
                time.sleep(0.1 * attempts)
                if attempts >= 10:
                    sys.stderr.write('connection failed\n')
                    break
                continue
            client.interact()
            break


class CmdPortoshell(CmdResolveSingleHost):
    """service@host [COMMAND]"""
    mode = "portoshell"
    description = "access container on remote host using shell"

    def __init__(self):
        super(CmdPortoshell, self).__init__()

        self.parser.add_option(
            '-i', dest='identityFile', default=None, metavar='IDENTITY_FILE',
            help='Selects a file from which the identity (private key) for public key authentication is read')
        self.parser.add_option(
            '-p', '--port', dest='port', type=int, default=10045,
            help='Specify custom server port'
        )
        self.parser.add_option(
            '-u', '--user', dest='user', type=str, default='',
            help='Specify custom user to authenticate as',
        )
        self.parser.add_option(
            '--env', action='append', dest='env', type=str, metavar='KEY=VALUE', default=[],
            help='Add custom environment variables. No additional value escape is needed',
        )
        self.parser.add_option(
            '--unset-env', action='append', dest='unset_env', type=str, metavar='KEY', default=[],
            help='Unset custom env variables',
        )
        self.parser.add_option(
            '-I', '--interactive', action='store_true', default=False,
            help='Run in interactive mode with tty',
        )
        self.parser.add_option(
            '-C', '--configuration-id', dest='configuration_id', type=str, default='',
            help='Use specified configuration id. If none specified, the ACTIVE one will be used',
        )
        self.parser.add_option(
            '-H', '--host', dest='hostname', type=str, default=None,
            help="Remote host with the required slot to use. If none specified, it's assumed that slot has format SERVICE@HOST",
        )
        self.parser.add_option(
            '-t', '--timeout', type=int, default=3600,
            help='Period of terminal inactivity, after this period session will be disconnected',
        )

    def run(self):
        extra_env = self.options.env
        extra_env = dict(val.split('=', 1) for val in extra_env)
        extra_env.setdefault('TERM', os.getenv('TERM', 'vt100'))
        extra_env = list(extra_env.iteritems())

        from library.tasks.portoshell import portoshell
        portoshell(slot=self.argv[0] if len(self.argv) else '',
                   server_port=self.options.port,
                   user=self.options.user,
                   identity_file=self.options.identityFile,
                   extra_env=extra_env,
                   unset_env=self.options.unset_env,
                   configuration_id=self.options.configuration_id,
                   hostname=self.options.hostname,
                   command=self.argv[1] if len(self.argv) > 1 else None,
                   interactive_cmd=self.options.interactive,
                   inactivity_timeout=self.options.timeout,
                   )


class CmdPortoCopy(Cmd):
    """[-C configuration_id] [slot]:file1 ... [slot]:file2"""
    mode = "portocopy"
    description = "copy files to or from remote container"

    def __init__(self):
        super(CmdPortoCopy, self).__init__()

        self.parser.add_option(
            '-u', '--user', dest='user', type=str, default='',
            help='Specify custom user to authenticate as',
        )
        self.parser.add_option(
            '-p', '--port', dest='port', type=int, default=10046,
            help='Specify custom SSH server port'
        )
        self.parser.add_option(
            '-C', '--configuration-id', dest='configuration_id', type=str, default='',
            help='Use specified configuration id. If none specified, the ACTIVE one will be used',
        )
        self.parser.add_option(
            '-H', '--host', dest='hostname', type=str, default=None,
            help="Remote host with the required slot to use. If none specified, it's assumed that slot has format SERVICE@HOST",
        )
        self.parser.add_option(
            '-r', dest='recursive', action='store_true', default=False,
            help='Recursively copy entire directories. Note that scp follows symbolic links encountered in the tree traversal',
        )

    def parse(self, argv):
        super(CmdPortoCopy, self).parse(argv)

        if len(self.argv) < 2:
            printOutputMessage("Missing source and/or destination", error=True)
            raise SystemExit(1)

        sources = self.argv[:-1]
        destination = self.argv[-1]
        remote_destination = ':' in destination
        if remote_destination and any(':' in source for source in sources):
            printOutputMessage("Cannot copy from remote location to remote location", error=True)
            raise SystemExit(1)
        elif not remote_destination:
            slot = sources[0].split(':', 1)[0]
            if not all(source.startswith(slot + ':') for source in sources):
                printOutputMessage("There can by only one remote location", error=True)
                raise SystemExit(1)
            sources = [source[len(slot) + 1:] for source in sources]
        else:
            slot, destination = destination.split(':', 1)

        if self.options.hostname:
            self.hostname = self.options.hostname
        elif '@' in slot:
            self.hostname = slot.split('@', 1)[1]
        else:
            printOutputMessage(
                'Cannot deduce hostname from the slot name %r. Use option -H to specify remote hostname.' % (slot,),
                error=True)
            raise SystemExit(1)

        self.remote_destination = remote_destination
        self.slot = slot
        self.sources = sources
        self.destination = destination

    def run(self):
        auth_command = {
            'user': getUserName(),
            'slot': self.slot,
        }
        if self.options.configuration_id:
            auth_command['configuration_id'] = self.options.configuration_id
        if self.options.user:
            auth_command['login_as'] = self.options.user

        auth_command = 'User=//' + '//'.join('%s:%s' % (k, v) for k, v in auth_command.items())
        command = ['scp', '-P', str(self.options.port), '-o', auth_command]
        if self.options.recursive:
            command.append('-r')
        if self.remote_destination:
            command += self.sources
            command.append('%s:%s' % (self.hostname, self.destination))
        else:
            command += ['%s:%s' % (self.hostname, source) for source in self.sources]
            command.append(self.destination)
        os.execvp('scp', command)
        printOutputMessage("'scp' exec failed", error=True)
        raise SystemExit(1)


def safe_repr(args):
    out_args = []
    arg_started = False
    for arg in args:
        if arg_started:
            out_args.append('HIDDEN')
            arg_started = False
        elif arg == '--env':
            out_args.append(arg)
            arg_started = True
        elif arg.startswith('--env='):
            out_args.append('--env=HIDDEN')
        else:
            out_args.append(arg)

    return repr(out_args)


if __name__ == '__main__':
    logged_args = safe_repr(sys.argv)
    log.info('BGN: {0} {1}'.format(getUserName(), logged_args))
    try:
        with StderrRedirect(sys.stdout if cfg.MergeStderrIntoStdout else sys.stderr):
            class VersionOpt(object):
                @staticmethod
                def add_opt(parser):
                    parser.add_option(
                        '-V', '--version', dest='version', action='store_true', default=False,
                        help='Show skynet version'
                    )

            class CalcHelpOpt(object):
                @staticmethod
                def add_opt(parser):
                    parser.add_option(
                        '--help-calc', dest='help_calc', action='store_true', default=False,
                        help='Show Blinov`s Calc help'
                    )

            cmd = Cmd(
                options=[VersionOpt(), CalcHelpOpt()]
            )

            if '-V' in sys.argv or '--version' in sys.argv:
                sky = os.path.realpath(__file__)
                base = os.path.dirname(os.path.dirname(sky))
                infofile = os.path.join(base, '.info')
                if os.path.exists(infofile):
                    require('pyyaml')
                    import yaml
                    try:
                        info = yaml.load(open(infofile, 'rb'))
                    except yaml.constructor.ConstructorError:
                        class Path(str):
                            strpath = str
                        info = yaml.load(open(infofile, 'rb'))
                    print('Version: {0}'.format(info['version'] or 'unknown'))
                else:
                    print('Version: unknown (%s not exists)' % (infofile,))
                raise SystemExit(0)

            if '--help_calc' in sys.argv or '--help-calc' in sys.argv:
                title = '| Full Prefix name |    Alias(es)     | Short |     Description            |'
                title_len = len(title)
                print('\n'.join(
                    [
                        '=' * title_len,
                        title,
                        '|                  |                  |  name |                            |',
                        '=' * title_len,
                        '| host             | -                | h     | Single host                |',
                        '-' * title_len,
                        '| conf             | CONF             | C     | Configuration (HQ)         |',
                        '-' * title_len,
                        '| conductorgroup   | CONDUCTORGROUP   | K     | Conductor group            |',
                        '-' * title_len,
                        '| conductortag     | CONDUCTORTAG     | k     | Conductor tag              |',
                        '-' * title_len,
                        '| conductorproject | CONDUCTORPROJECT | P     | Conductor project          |',
                        '-' * title_len,
                        '| conductordc      | CONDUCTORDC      | D     | Conductor data-center      |',
                        '-' * title_len,
                        '| gencfg           | GENCFG           | G     | GenCfg group trunk/tag     |',
                        '|                  |                  |       |  G@group[:tag]             |',
                        '-' * title_len,
                        '| walle            | WALLE            | W     | Wall-e project:            |',
                        '|                  |                  |       |  W@project[:ips]           |',
                        '-' * title_len,
                        '| walletag         | WALLETAG         | w     | Wall-e tag:                |',
                        '|                  |                  |       |  w@tag[:ips]               |',
                        '-' * title_len,
                        '| tor              | TOR              | t     | Hosts in rack:             |',
                        '|                  |                  |       |  t@switch[:host_num]       |',
                        '-' * title_len,
                        '| qloud            | QLOUD            | Q     | Qloud component [DNS]      |',
                        '-' * title_len,
                        '| qloudhw          | QLOUDHW          | q     | Qloud hardware:            |',
                        '|                  |                  |       |  q@sys[.hwseg][:state][?dc]|',
                        '|                  |                  |       |  sys: test, main or ext    |',
                        '-' * title_len,
                        '| mtn              | MTN              | M     | MTN containers:            |',
                        '|                  |                  |       |  M@group[:tag][?itags] or  |',
                        '|                  |                  |       |  M@service[:confid][?itags]|',
                        '-' * title_len,
                        '| slavemtn         | SLAVEMTN         | m     | Slave MTN containers:      |',
                        '|                  |                  |       |  M@master_grp[:tag][?itags]|',
                        '-' * title_len,
                        '| family           | FAMILY           | f     | HQ service or conf_id:     |',
                        '|                  | HQ               |       |  f@service[:confid][?itags]|',
                        '-' * title_len,
                        '| dc               | -                | d     | Hosts in data-center:      |',
                        '|                  |                  |       |  d@dc[:all|hw|virtual]     |',
                        '-' * title_len,
                        '| line             | -                | l     | Hosts in DC`s line:        |',
                        '|                  |                  |       |  l@line[:all|hw|virtual]   |',
                        '-' * title_len,
                        '| samogon          | SAMOGON          | sg    | Samogon service:           |',
                        '|                  | SG               |       |  sg@service[:servant][?dc] |',
                        '-' * title_len,
                        '| yp               | YP               | Y     | YP pods:                   |',
                        '|                  |                  |       |  Y@podset[:transient|      |',
                        '|                  |                  |       |            persistent|node]|',
                        '-' * title_len,
                        '| pods             | PODS             | p     | Y.Deploy pods:             |',
                        '|                  |                  |       |  p@prj[.stg[.du]]          |',
                        '|                  |                  |       |[:transient|persistent|node]|',
                        '|                  |                  |       |   [?cluster|               |',
                        '-' * title_len,
                        '| boxes            | BOXES            | b     | Y.Deploy boxes:            |',
                        '|                  |                  |       |  b@prj[.stg[.du[.box]]]    |',
                        '|                  |                  |       |   [?cluster|               |',
                        '=' * title_len,
                        '|     D E P R E C A T E D    P R E F I X E S (resolved in empty set always)|',
                        '=' * title_len,
                        '| group            | hostgroup        | H     | YR host group              |',
                        '|                  | HOSTGROUP        |       |                            |',
                        '|                  | GROUP            |       |                            |',
                        '-' * title_len,
                        '| shard            | -                | s     | single shard               |',
                        '-' * title_len,
                        '| shardtag         | SHARDTAG         | S     | CMS shard tag              |',
                        '|                  | stag             |       |                            |',
                        '|                  | STAG             |       |                            |',
                        '-' * title_len,
                        '| instancetag      | INSTANCETAG      | I     | CMS instance tag           |',
                        '|                  | itag             |       |                            |',
                        '|                  | ITAG             |       |                            |',
                        '=' * title_len
                    ]
                ))
                raise SystemExit(0)

            code = cmd.Run()
            raise SystemExit(code)
    except KeyboardInterrupt:
        import signal

        signal.signal(signal.SIGINT, signal.SIG_DFL)
        os.kill(os.getpid(), signal.SIGINT)
    finally:
        log.info('END: {0} {1}'.format(getUserName(), logged_args))

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
