#!/usr/bin/env python2
# coding=utf-8
"""
log parser for porto 1.13

+ worked with porto 1.17
+ volumes/layers support - https://st.yandex-team.ru/ISS-4705

"""
import argparse
import os
import re
import sys
import time

DEBUG = False

TIMESTAMP_RE = re.compile(r'^(\d{4}-\d{2}-\d{2} \d\d:\d\d:\d\d) (.+?)$')
SOURCE_RE = re.compile(r'^([^\s:]+:)\s+(.*)$')
ACTION_RE = re.compile(r'^([A-Z]{3})\s(.*)$')

PORTOD_LOG_LOCAL = './portod.log'
PORTOD_LOG_SYSTEM = '/var/log/portod.log'

HALF_A_BANNER = '-' * 35


def comment(msg):
    print "#", msg


def debug(msg):
    if DEBUG:
        comment("DEBUG: " + msg)


def error(msg):
    comment("ERROR: " + msg)


def banner(msg):
    comment(HALF_A_BANNER + msg + HALF_A_BANNER)


def error_command(command, message):
    banner(message)
    for c in command:
        print c.raw
    print


def _print_dict_dict_list(ddl, label):
    print "*** %s: %d" % (label, len(ddl))
    for k1, v1 in ddl.iteritems():
        print "== %s ==" % (k1,)
        for k2, v2 in v1.iteritems():
            print "    %s --> %s" % (k2, len(v2))
    print


class AbstractNLinesBufferedFileIterator(object):
    def __init__(self, fh, buffer_size):
        self.fh = fh
        self.buffer_size = buffer_size
        self.buffer = []

    def __iter__(self):
        return iter(self.generate_lines())

    def filter_buffer(self):
        raise NotImplementedError

    def _is_buffer_full(self):
        return len(self.buffer) >= self.buffer_size

    def _fill_buffer(self):
        if self._is_buffer_full():
            return

        for line in self.fh:
            self.buffer.append(line)
            if self._is_buffer_full():
                break

        if not self.buffer:
            raise StopIteration

    def generate_lines(self):
        while True:
            self._fill_buffer()
            self.filter_buffer()

            if self.buffer:
                line = self.buffer.pop(0)
                yield line


class VolumeSpamRemoverIterator(AbstractNLinesBufferedFileIterator):
    """
    2017-07-08 08:20:06 portod-worker31[549826]: REQ volumeAPI: create /place/db/iss3/jobs/mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqcaDK space_limit=2199023255552 backend=native from 5:java(1037667)
    2017-07-08 08:20:06 portod-worker31[549826]: RSP Error: VolumeAlreadyExists (Volume already exists) to 5:java(1037667) (request took 0ms)
    2017-07-08 08:20:06 portod-worker23[549818]: REQ volumeAPI: link /place/db/iss3/jobs/mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqcaDK to ISS-AGENT--mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqc
    2017-07-08 08:20:06 portod-worker23[549818]: RSP Error: VolumeAlreadyLinked (Already linked) to 5:java(1037667) (request took 0ms)

    T0D0^W: make new version with buffer of about 1000 lines to group lines from REQ to RESP for same worker and avoid such:
2017-07-08 08:08:05 portod-worker24[549819]: REQ importLayer { layer: "ISS-AGENT_5Cjv3DcG1nM" tarball: "/place/db/iss3/resources/nirvana-base-layer-3690931f253a913753da05aed35ca219-3690931f253a913753da05aed35ca2
2017-07-08 08:08:05 portod-worker10[549805]: REQ pset self/NIRVANA__env-anaconda-install__7 ulimit core: unlimited unlimited from 12:java(495948) loadbase:loadbase from ISS-AGENT--n1RAd1MmTnanYnzo6bxLtA_n1RAd1Mm
2017-07-08 08:08:05 portod-worker10[549805]: RSP Ok to 12:java(495948) (request took 0ms)
2017-07-08 08:08:05 portod-worker24[549819]: RSP Error: LayerAlreadyExists (Layer already exists) to 5:java(1037667) (request took 0ms)

    porto creates its own workers so its impossible to assemble commands in common:
2017-07-08 08:17:02 portod-spawn-c[1]: ACT bind mount /proc/kcore /dev/null
2017-07-08 08:17:02 portod-spawn-c[1]: ACT remount /proc/kcore ro,remount,bind
2017-07-08 08:17:02 portod-worker4[549799]: ACT ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U: change state starting -> meta
2017-07-08 08:17:02 portod-worker4[549799]: ACT Set memory:/porto%ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U memory.soft_limit_in_bytes = 1048576
2017-07-08 08:17:02 portod-worker4[549799]: ACT Start ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U/iss_hook_start
2017-07-08 08:17:02 portod-worker4[549799]: ACT ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U/iss_hook_start: change state stopped -> starting
2017-07-08 08:17:02 portod-worker4[549799]: ACT Create cgroup freezer:/porto/ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U/iss_hook_start
2017-07-08 08:17:02 portod-worker4[549799]: ACT Create cgroup memory:/porto%ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U/iss_hook_start
2017-07-08 08:17:02 portod-worker4[549799]: ACT Create cgroup cpuacct:/porto%ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U/iss_hook_start
2017-07-08 08:17:02 portod-worker4[549799]: ACT Set memory:/porto%ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U/iss_hook_start cgroup.event_control = 98 12
2017-07-08 08:17:02 portod-worker4[549799]: ACT Set memory:/porto%ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U/iss_hook_start memory.anon.limit = 4966055936
2017-07-08 08:17:02 portod-worker4[549799]: ACT Apply ulimits
2017-07-08 08:17:02 portod-spawn-p[511319]: ACT Attach process 511319 to freezer:/porto/ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U/iss_hook_start
2017-07-08 08:17:02 portod-spawn-p[511319]: ACT Attach process 511319 to memory:/porto%ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U/iss_hook_start
2017-07-08 08:17:02 portod-spawn-p[511319]: ACT Attach process 511319 to cpu:/porto%ISS-AGENT--bAvDo57fTDSN8OzMjhUACQ_bAvDo57fTDSN8OzMjhUACQ_YkHEFfImb1U
    """
    WORKER_RE = re.compile(r"^\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d (portod-worker\d+\[\d+\]):")

    PAIRS = (
        ("REQ volumeAPI: create", "RSP Error: VolumeAlreadyExists"),
        ("REQ volumeAPI: link", "RSP Error: VolumeAlreadyLinked"),
        ("REQ importLayer", "RSP Error: LayerAlreadyExists"),
    )

    def get_worker(self, line):
        match = VolumeSpamRemoverIterator.WORKER_RE.match(line)
        if match:
            return match.group(1)
        else:
            return None

    def __init__(self, fh):
        super(VolumeSpamRemoverIterator, self).__init__(fh, 2)
        self.spam_lines_count = 0
        self.spam_lines_size = 0

    def filter_buffer(self):
        if len(self.buffer) < 2:
            return

        line1, line2 = self.buffer[:2]

        for part1, part2 in self.PAIRS:
            if (part1 in line1) and (part2 in line2):
                worker1 = self.get_worker(line1)
                if not worker1:
                    continue
                worker2 = self.get_worker(line2)
                if worker1 == worker2:
                    self.spam_lines_count += 2
                    self.spam_lines_size += len(self.buffer[0])
                    self.spam_lines_size += len(self.buffer[1])

                    self.buffer[0:2] = []  # remove two matched spam lines
                    break

    def get_spam_lines_count(self):
        return self.spam_lines_count

    def get_spam_lines_size(self):
        return self.spam_lines_size


class PortoLogLine(object):
    DUMMY_LINE = "0000-00-00 00:00:00 GroupedActionsGenerator: RSP this is dummy command for assembling last REQ"
    HOOK_NAME_RE = re.compile(r"(?<=/)(iss_hook_[a-z]+)")

    def __init__(self, raw, line_index):
        self.line_index = line_index

        self.raw = None
        self.empty = None
        self.parse_error = None
        self.timestamp = None
        self.source = None
        self.raw_message = None
        self.action = None
        self.message = None
        self.first_word_lo = None
        self.parse_ok = False

        self._parse(raw)

    @staticmethod
    def get_dummy_record():
        return PortoLogLine(PortoLogLine.DUMMY_LINE, sys.maxint - 1)

    def _parse(self, raw):
        self.raw = raw

        if not raw.strip():
            self.empty = True
            return

        match = TIMESTAMP_RE.match(raw)
        if not match:
            self.parse_error = "cant get timestamp from line: %s" % (raw,)
            return

        self.timestamp, remainder = match.group(1, 2)

        match = SOURCE_RE.match(remainder.strip())
        if not match:
            self.parse_error = "cant get source from line: %s" % (raw,)
            return

        self.source, self.raw_message = match.group(1, 2)

        # try to extract action
        match = ACTION_RE.match(self.raw_message)
        if match:
            self.action, self.message = match.group(1, 2)
            self.first_word_lo = self.message.split()[0].lower()
        else:
            self.message = self.raw_message

        self.parse_ok = True

    def get_iss_hook_name(self):
        """
        returns hook name form first line, if present or empty string
        """
        match = self.HOOK_NAME_RE.search(self.raw)
        if match:
            return match.group(1)
        else:
            return ""


class PortoLogProcessorBase(object):
    """
    parse each portod.log line into PortoLogLine object and call process_line() on each
    """

    def __init__(self, filename, leave_volume_spam, no_spam_log):
        self.filename = filename
        self.lines_count = None
        self.lines_size = None
        self.pending_stop = False
        self.leave_volume_spam = leave_volume_spam
        self.no_spam_log = no_spam_log
        self.stats = []

    def stop(self):
        self.pending_stop = True

    def process_line(self, log_record):
        pass

    def add_stats_line(self, line):
        self.stats.append(line)

    def print_stats(self):
        for line in self.stats:
            comment(line)

    def run(self):
        self.lines_count = 0
        self.lines_size = 0

        fh = open(self.filename)
        no_spam_fh = None

        if self.no_spam_log:
            no_spam_fh = open(self.no_spam_log, "wb")

        if not self.leave_volume_spam:
            fh = VolumeSpamRemoverIterator(fh)

        for raw_line in fh:
            self.lines_count += 1
            self.lines_size += len(raw_line)

            if self.no_spam_log:
                no_spam_fh.write(raw_line)

            line = raw_line.splitlines()[0]
            record = PortoLogLine(line, self.lines_count)
            self.process_line(record)

            if self.pending_stop:
                comment("stop processing")
                break

        if self.no_spam_log:
            no_spam_fh.close()

        message = "done, %d lines / %d mb" % (self.lines_count, self.lines_size >> 20)
        if not self.leave_volume_spam:
            message += " (volume spam: %d lines / %d mb)" % (fh.get_spam_lines_count(), fh.get_spam_lines_size() >> 20)

        self.add_stats_line(message)


class LogStatAccumulator(PortoLogProcessorBase):
    """
    accumulates (action, command_code) pairs from log

    result on some log:

        ('ACT', '/:')
        ('ACT', '/porto:')
        ('ACT', 'Create')
        ('ACT', 'Destroy')
        ('ACT', 'Kill')
        ('ACT', 'Remove')
        ('ACT', 'Set')
        ('ACT', 'Start')
        ('ACT', 'Stop')
        ('ACT', 'Touch')
        ('ACT', 'Unlink')
        ('ACT', 'iss-agent-container-5617192517817094794:')
        ('ACT', 'kill')
        ('ACT', 'mkdir')
        ('ACT', 'mount')
        ('EVT', 'Exit')
        ('REQ', 'create')
        ('REQ', 'destroy')
        ('REQ', 'pset')
        ('REQ', 'start')
        ('RSP', 'Error:')
        ('RSP', 'Ok')
        ('SYS', '--------------------------------------------------------------------------------')
        ('SYS', 'Started')
        ('SYS', 'network')
    """

    def __init__(self, filename):
        super(LogStatAccumulator, self).__init__(filename)
        PortoLogProcessorBase.__init__(self, filename)
        self.action_args = set()

    def process_line(self, record):
        if not record.parse_ok:
            return

        if record.action:
            first_word = record.message.split()[0]

            if not first_word.startswith('ISS-AGENT--'):
                self.action_args.add((record.action, first_word))

    def run(self):
        super(LogStatAccumulator, self).run()

        print "found (action, command) pairs:"
        for i in sorted(self.action_args):
            print i


class GroupedActionsGenerator(PortoLogProcessorBase):
    """
    generate stream of full commands:
      REQ [ACT]* RSP
      or
       EVT [ACT]*

    then call process_command() on each
    """

    def __init__(self, filename, leave_volume_spam, no_spam_log):
        super(GroupedActionsGenerator, self).__init__(filename, leave_volume_spam, no_spam_log)
        self.command_count = 0
        self.records = []

    def process_line(self, log_record):
        if not log_record.parse_ok:
            return

        if not log_record.action:
            return

        self.records.append(log_record)

        self._extract_full_commands()

    def process_command(self, command):
        pass

    def _extract_full_commands(self):
        while self._extract_one_full_command():
            pass

    def _extract_one_full_command(self):
        # skip trash
        while self.records and self.records[0].action not in ['EVT', 'REQ']:
            self.records.pop(0)

        if not self.records:
            return False

        first_action = self.records[0].action

        # find first non-ACT
        different_pos = None
        for idx, record in enumerate(self.records[1:], 1):
            if record.action != 'ACT':
                different_pos = idx
                break

        if different_pos is None:
            return False

        if first_action == 'EVT':
            # strip records before
            command_size = different_pos
        elif first_action == 'REQ':
            command_size = different_pos + 1
        else:
            raise ValueError("wrong first line action: %s" % (first_action,))

        self._strip_and_handle_record(command_size)

        return True

    def _strip_and_handle_record(self, command_size):
        if command_size > len(self.records):
            raise ValueError("request %d of %d records batch" % (command_size, len(self.records)))

        command, self.records = self.records[:command_size], self.records[command_size:]

        self.command_count += 1

        self.process_command(command)

    def run(self):
        super(GroupedActionsGenerator, self).run()

        self.records.append(PortoLogLine.get_dummy_record())
        self._extract_full_commands()

        self.add_stats_line("%d full commands" % self.command_count)


class GroupHistoryFilter(GroupedActionsGenerator):
    def __init__(self, filename, name_part, replace_with, leave_volume_spam=False, no_spam_log=None, indent_responses=True):
        super(GroupHistoryFilter, self).__init__(filename, leave_volume_spam, no_spam_log)
        self.name_part = name_part
        self.replace_with = replace_with
        self.indent_responses = indent_responses
        self.prev_hook_name = ""
        self.line_patcher = self._get_line_patcher()
        self.hook_entry_count = {}
        self.commands_to_print = []

    def _command_has_part(self, command):
        for c in command:
            if self.name_part in c.raw:
                return True
        return False

    def _string_has_part(self, s):
        return self.name_part in s

    def _get_line_patcher(self):

        def patch_every_line(line):
            pass1 = line.replace(self.name_part, self.replace_with)

            if self.indent_responses:
                parts = pass1.split(" ", 5)

                if len(parts) == 6:
                    indent = False

                    if parts[3] not in ('REQ', 'EVT'):
                        indent = True
                    elif parts[3] == 'REQ' and parts[4] == 'pset':
                        indent = True

                    if indent:
                        parts[3] = '    ' + parts[3]
                        pass1 = ' '.join(parts)
            return pass1

        def pass_line(line):
            return line

        if self.replace_with:
            return patch_every_line
        else:
            return pass_line

    def save_one_command_to_print_queue(self, command):
        self.commands_to_print.append(command)

    def save_commands_to_print_queue(self, commands):
        for command in commands:
            self.save_one_command_to_print_queue(command)

    def print_one_command(self, command):
        for c in command:
            print self.line_patcher(c.raw)
        print

    def print_all_commands(self):
        processed = set()

        try:
            self.commands_to_print.sort(key=lambda cmd: cmd[0].line_index)
        except AttributeError:
            print "1"

        for command in self.commands_to_print:
            first_line_index = command[0].line_index
            if first_line_index in processed:
                continue
            else:
                processed.add(first_line_index)

            hook_name = command[0].get_iss_hook_name()

            if hook_name != self.prev_hook_name:
                banner(hook_name)
                self.prev_hook_name = hook_name

                if hook_name:
                    self.hook_entry_count[hook_name] = 1 + self.hook_entry_count.get(hook_name, 0)

            self.print_one_command(command)

    def process_command(self, command):
        if self._command_has_part(command):
            self.save_one_command_to_print_queue(command)

    def link_commands(self):
        """add links to commands that do not contain configuration name"""
        pass

    def run(self):
        start_time = time.time()

        super(GroupHistoryFilter, self).run()

        self.link_commands()

        self.print_all_commands()

        if self.replace_with:
            self.add_stats_line("name part [%s] replaced with [%s]" % (self.name_part, self.replace_with))

        if self.hook_entry_count:
            self.add_stats_line("=== hook run count")
            for hook in sorted(self.hook_entry_count.keys()):
                self.add_stats_line("%s: %d times" % (hook, self.hook_entry_count[hook]))
            self.add_stats_line("")

        self.print_stats()
        comment("processing time: %d secs" % (int(time.time() - start_time),))


class HistoryWithVolumes(GroupHistoryFilter):
    def __init__(self, filename, name_part, replace_with, leave_volume_spam=False, no_spam_log=None, indent_responses=True):
        super(HistoryWithVolumes, self).__init__(filename, name_part, replace_with, leave_volume_spam, no_spam_log, indent_responses)
        self.uniq_cmds = set()
        self.all_import_layer_commands = {}
        self.all_remove_layer_commands = {}
        self.volume2layers = {}
        self.dget_containers = {}
        self.containers_weak = {}
        self.created_volumes = {}
        self.container2volume = {}
        self.container2proc = {}
        self.proc_exit = {}

        self.unsupported_commands = set()
        self.unknown_pset_comands = set()
        self.unknown_command_codes = set()
        self.unknown_layer_commands = set()
        self.unknown_volume_api_commands = set()

    def _put_ddl(self, adict, key1, key2, value):
        """
        put command into "dict of dict of list"
        """
        adict.setdefault(key1, dict()) \
            .setdefault(key2, list()) \
            .append(value)

    def process_command(self, command):
        code = command[0].first_word_lo
        # debug
        if code not in self.uniq_cmds:
            self.uniq_cmds.add(code)

        if code in ("importlayer", "removelayer"):
            self._register_layer_command(code, command)
        elif code == "volumeapi:":
            self._register_volumeapi(command)
        elif code in ("create", "destroy"):
            self._register_create_destroy_container(command)
        elif code == "dget":
            self._register_dget(command)
        elif code in ("start", "exit"):
            self._register_start_exit(command)
        elif code == "pset":
            self._register_pset(command)
        elif code in ("wait",):
            pass  # ignore
        else:
            if code not in self.unknown_command_codes:
                error_command(command, "UNKNOWN: " + code)
                self.unknown_command_codes.add(code)

        super(HistoryWithVolumes, self).process_command(command)

    # PARENT_CONTAINER_RE = re.compile(r" from (?:\d+:\S+) (?:\S+?:\S+?) from (\S+)")

    def _get_parent_container_name(self, line):
        """
        extract name of parent container from command
          or return None

        sample command:
    REQ destroy self/NIRVANA__job-command from 25:java(567694) loadbase:loadbase from ISS-AGENT--HaSIVju8SSmKmJs4SkbRvA_HaSIVju8SSmKmJs4SkbRvA_53XngOwfEML/iss_hook_start

    bad sample:
2017-07-08 08:01:31 portod-worker0[549795]: REQ volumeAPI: unlink /place/db/iss3/volumes/ISS-AGENT_u70hMlDDZFE from iss-agent-container-5617192517817094794 from 5:java(1037667)
        """
        parts = line.split(" from ")
        if len(parts) >= 3:
            middle_parts = parts[-2].split()
            if len(middle_parts) == 2:
                if ":" in middle_parts[0] and ":" in middle_parts[1]:
                    # middle_parts = "25:java(567694) loadbase:loadbase"
                    parent_container = parts[-1].split()[0]
                    return parent_container

        return None

    def _should_skip_because_of_unknown_parent(self, line):
        parent_container = self._get_parent_container_name(line)
        if parent_container:
            if not self._string_has_part(parent_container):
                return True
        return False

    KNOWN_HOST_VOLUMES = (
        "/opt/supervisor",
        "/place/db/iss3",
        "/place/db/iss3/resources",
    )

    def _register_pset(self, command):
        """
2017-07-08 08:02:28 portod-worker7[549802]: REQ pset ISS-AGENT--S9UBZ2M3SG-LT7i-YpfvbA_S9UBZ2M3SG-LT7i-YpfvbA_O3ZCV7vf0BB bind /place/db/iss3 /place/db/iss3; /place/db/iss3/jobs/S9UBZ2M3SG-LT7i-YpfvbA_S9UBZ2M3SG-LT7i-YpfvbA_O3ZCV7vf0BB /place/db/iss3/jobs/S9UBZ2M3SG-LT7i-YpfvbA_S9UBZ2M3SG-LT7i-YpfvbA_O3ZCV7vf0BB;/place/db/iss3/volumes/ISS-AGENT_gAFnmok4LUO /place/db/iss3/volumes/ISS-AGENT_gAFnmok4LUO;/place/db/iss3/jobs/S9UBZ2M3SG-LT7i-YpfvbA_S9UBZ2M3SG-LT7i-YpfvbA_O3ZCV7vf0BB /place/db/iss3/volumes/ISS-AGENT_gAFnmok4LUO/place/db/iss3/jobs/S9UBZ2M3SG-LT7i-YpfvbA_S9UBZ2M3SG-LT7i-YpfvbA_O3ZCV7vf0BB; /place/db/iss3/resources /place/db/iss3/resources; /place/berkanavt/supervisor /opt/supervisor ro from 5:java(1037667)
2017-07-08 08:02:28 portod-worker7[549802]: RSP Ok to 5:java(1037667) (request took 0ms)

# -----------------------------------unknown pset command -> command-----------------------------------
2017-07-08 08:00:46 portod-worker23[549818]: REQ pset ISS-AGENT--kWrbNrsNTWi4T3Rlh2H8eg_kWrbNrsNTWi4T3Rlh2H8eg_32EiexSZ4DQ/iss_hook_start command /place/db/iss3/jobs/kWrbNrsNTWi4T3Rlh2H8eg_kWrbNrsNTWi4T3Rlh2H8eg_32EiexSZ4DQ/iss_hook_start from 5:java(1037667)
2017-07-08 08:00:46 portod-worker23[549818]: RSP Ok to 5:java(1037667) (request took 0ms)

        """
        if self._should_skip_because_of_unknown_parent(command[0].message):
            return

        parts = command[0].message.split()
        parts.pop(0)
        container_name = parts.pop(0)
        code = parts.pop(0)

        if code == "bind":
            # Bind mounts: <source> <target> [ro|rw];...
            raw = " ".join(parts)
            match = re.match(r"^(.+?)\sfrom\s\d", raw)
            if not match:
                error("bind: cant strip 'from' on line " + command[0].message)
                return
            bind_list = match.group(1)
            bind_records = bind_list.split(";")
            # debug("bind: container " + container_name)
            for rec in bind_records:
                bb = rec.split()
                if len(bb) < 2:
                    error("wrong bind record: %r" % (rec,))
                    continue
                volume_name = bb[1]
                if volume_name in self.KNOWN_HOST_VOLUMES:
                    continue
                # debug("    src: " + volume_name)
                self._put_ddl(self.container2volume, container_name, volume_name, command)
        elif code == "command":
            proc_name = parts.pop(0)
            self._put_ddl(self.container2proc, container_name, proc_name, command)
        elif code == "root":
            volume_name = parts.pop(0)
            self._put_ddl(self.container2volume, container_name, volume_name, command)
        elif code in ("respawn", "cwd", "private", "ulimit", "anon_limit", "isolate", "user", "env", "weak",
                      "enable_porto", "cpu_guarantee", "memory_limit", "memory_guarantee", "porto_namespace",
                      "stderr_path"):
            # will go into output if known container name
            pass
        else:
            self.unknown_pset_comands.add(code)
            # error_command(command, "unknown pset command -> " + code)

    def _register_start_exit(self, command):
        """
2017-07-08 08:15:08 portod-worker31[549826]: REQ start ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E/iss_hook_stop from 5:java(1037667)
2017-07-08 08:15:08 portod-worker31[549826]: ACT Start ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E/iss_hook_stop
2017-07-08 08:15:08 portod-worker31[549826]: ACT ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E/iss_hook_stop: change state stopped -> starting
2017-07-08 08:15:08 portod-worker31[549826]: ACT Create cgroup freezer:/porto/ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E/iss_hook_stop
2017-07-08 08:15:08 portod-worker31[549826]: ACT Create cgroup cpuacct:/porto%ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E/iss_hook_stop
2017-07-08 08:15:08 portod-worker31[549826]: ACT Apply ulimits
2017-07-08 08:15:08 portod-spawn-p[510205]: ACT Attach process 510205 to freezer:/porto/ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E/iss_hook_stop
2017-07-08 08:15:08 portod-spawn-p[510205]: ACT Attach process 510205 to memory:/porto%ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E
2017-07-08 08:15:08 portod-spawn-p[510205]: ACT Attach process 510205 to cpu:/porto%ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E
2017-07-08 08:15:08 portod-spawn-p[510205]: ACT Attach process 510205 to cpuacct:/porto%ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E/iss_hook_stop
2017-07-08 08:15:08 portod-spawn-p[510205]: ACT Attach process 510205 to cpuset:/
2017-07-08 08:15:08 portod-spawn-p[510205]: ACT Attach process 510205 to net_cls:/porto%ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E
2017-07-08 08:15:08 portod-spawn-p[510205]: ACT Attach process 510205 to blkio:/porto%ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E
2017-07-08 08:15:08 portod-spawn-p[510205]: ACT Attach process 510205 to devices:/porto%ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E
2017-07-08 08:15:08 portod-worker31[549826]: ACT ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E/iss_hook_stop: change state starting -> running
2017-07-08 08:15:08 portod-worker31[549826]: ACT Set memory:/porto%ISS-AGENT--VO_FtTvGT9CTt67irpRYCA_VO_FtTvGT9CTt67irpRYCA_9zqnD7m6Q2E memory.soft_limit_in_bytes = 18446744073709551615
2017-07-08 08:15:08 portod-worker31[549826]: RSP Ok to 5:java(1037667) (request took 66ms)

2017-07-08 08:00:59 portod-event0[549827]: EVT Exit ISS-AGENT--qRlBnWJSSf-nl6_aCqx5IQ_qRlBnWJSSf-nl6_aCqx5IQ_MulAN8UPjYD/iss_hook_start/NIRVANA__job-command exit code: 0
2017-07-08 08:00:59 portod-event0[549827]: ACT Terminate tasks in ISS-AGENT--qRlBnWJSSf-nl6_aCqx5IQ_qRlBnWJSSf-nl6_aCqx5IQ_MulAN8UPjYD/iss_hook_start/NIRVANA__job-command
2017-07-08 08:00:59 portod-event0[549827]: ACT ISS-AGENT--qRlBnWJSSf-nl6_aCqx5IQ_qRlBnWJSSf-nl6_aCqx5IQ_MulAN8UPjYD/iss_hook_start/NIRVANA__job-command: change state running -> dead
        """
        if self._should_skip_because_of_unknown_parent(command[0].message):
            return

        # TODO: command names starts with container names
        parts = command[0].message.split()
        proc_name = parts[1]
        self.proc_exit.setdefault(proc_name, list()).append(command)

    def _register_dget(self, command):
        """
2017-07-08 08:06:54 portod-worker17[549812]: REQ dget ISS-AGENT--n1RAd1MmTnanYnzo6bxLtA_n1RAd1MmTnanYnzo6bxLtA_n76c3UxGQ0V/iss_hook_start state from 5:java(1037667)
2017-07-08 08:06:54 portod-worker17[549812]: RSP Error: ContainerDoesNotExist (container ISS-AGENT--n1RAd1MmTnanYnzo6bxLtA_n1RAd1MmTnanYnzo6bxLtA_n76c3UxGQ0V/iss_hook_start not found) to 5:java(1037667) (request took 0ms)
        """
        parts = command[0].message.split()
        container_name = parts[1]

        self.dget_containers.setdefault(container_name, list()).append(command)

    def _register_create_destroy_container(self, command):
        """
2017-07-08 08:15:12 portod-worker26[549821]: REQ create ISS-AGENT--mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqcaDK from 5:java(1037667)
2017-07-08 08:15:12 portod-worker26[549821]: ACT Create ISS-AGENT--mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqcaDK
2017-07-08 08:15:12 portod-worker26[549821]: RSP Ok to 5:java(1037667) (request took 1ms)

2017-07-08 08:19:03 portod-worker21[549816]: REQ create weak self/NIRVANA__job-command from 98:java(513546) loadbase:loadbase from ISS-AGENT--mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqcaDK/iss_hook_start

        create weak self/NIRVANA__env-anaconda-install__5 from 12:java(493039) loadbase:loadbase from ISS-AGENT--_cJwCkC7Rx29pUQ61hmYtQ__cJwCkC7Rx29pUQ61hmYtQ_uTtDpTgvIKL/iss_hook_start
        """
        if self._should_skip_because_of_unknown_parent(command[0].message):
            return

        parts = command[0].message.split()

        if parts[1] == "weak":
            container_name = parts[2]
            parent_container = parts[7]
            self._put_ddl(self.containers_weak, parent_container, container_name, command)
        else:
            container_name = parts[1]

            if self._string_has_part(container_name):  # filter container names here to avoid OOM
                self.dget_containers.setdefault(container_name, list()).append(command)

    def _register_volumeapi(self, command):
        """
-- create volume
2017-07-08 08:15:12 portod-worker13[549808]: REQ volumeAPI: create /place/db/iss3/jobs/mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqcaDK space_limit=2199023255552 backend=native from 5:java(1037667)
2017-07-08 08:15:12 portod-worker13[549808]: ACT Build volume: /place/db/iss3/jobs/mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqcaDK backend: native
2017-07-08 08:15:12 portod-worker13[549808]: ACT Creating project quota: /place/porto_volumes/78912968/native bytes: 2199023255552 inodes: 0
2017-07-08 08:15:12 portod-worker13[549808]: ACT bind mount /place/porto_volumes/78912968/volume /place/porto_volumes/78912968/native
2017-07-08 08:15:12 portod-worker13[549808]: ACT remount /place/porto_volumes/78912968/volume nosuid,nodev,remount,bind
2017-07-08 08:15:12 portod-worker13[549808]: ACT bind mount /proc/self/fd/94 /place/porto_volumes/78912968/volume
2017-07-08 08:15:12 portod-worker13[549808]: RSP Ok to 5:java(1037667) (request took 38ms)

-- link volume to iss container
2017-07-08 08:15:12 portod-worker14[549809]: REQ volumeAPI: link /place/db/iss3/jobs/mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqcaDK to ISS-AGENT--mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqcaDK from 5:java(1037667)
2017-07-08 08:15:12 portod-worker14[549809]: RSP Ok to 5:java(1037667) (request took 0ms)

-- unlink volume from container
2017-07-08 08:02:10 portod-worker31[549826]: REQ volumeAPI: unlink /place/db/iss3/jobs/TPyJ2DIGQ1u4ZVginmnPcA_TPyJ2DIGQ1u4ZVginmnPcA_uxLT1ruoeYP from ISS-AGENT--TPyJ2DIGQ1u4ZVginmnPcA_TPyJ2DIGQ1u4ZVginmnPcA_uxLT1ruoeYP from 5:java(1037667)
2017-07-08 08:02:10 portod-worker31[549826]: ACT Destroy volume: /place/db/iss3/jobs/TPyJ2DIGQ1u4ZVginmnPcA_TPyJ2DIGQ1u4ZVginmnPcA_uxLT1ruoeYP backend: native
2017-07-08 08:02:10 portod-worker31[549826]: ACT umount nested /place/db/iss3/jobs/TPyJ2DIGQ1u4ZVginmnPcA_TPyJ2DIGQ1u4ZVginmnPcA_uxLT1ruoeYP
2017-07-08 08:02:10 portod-worker31[549826]: ACT umount all /place/db/iss3/jobs/TPyJ2DIGQ1u4ZVginmnPcA_TPyJ2DIGQ1u4ZVginmnPcA_uxLT1ruoeYP
2017-07-08 08:02:10 portod-worker31[549826]: ACT umount all /place/porto_volumes/78912859/volume
2017-07-08 08:02:10 portod-worker31[549826]: ACT Destroying project quota: /place/porto_volumes/78912859/native
2017-07-08 08:02:10 portod-worker31[549826]: ACT umount nested /place/porto_volumes/78912859
2017-07-08 08:02:10 portod-worker31[549826]: ACT Call helper: find . -xdev -mindepth 1 -delete  in /place/porto_volumes/78912859/native
2017-07-08 08:02:10 portod-worker31[488517]: ACT Attach process 488517 to memory:/portod-helpers
2017-07-08 08:02:10 portod-worker31[549826]: RSP Ok to 5:java(1037667) (request took 76ms)

        """
        parts = command[0].message.split()

        if len(parts) < 3:
            error_command(command, "volumeAPI: create")
            return

        if parts[0] != "volumeAPI:":
            error_command(command, "cant parse")
            return

        if parts[1] == "create":
            parts = parts[2:]
            volume_name = parts.pop(0)

            layers_raw = [s for s in parts if s.startswith("layers=")]

            # debug("volumeAPI create: " + volume_name)
            self.created_volumes.setdefault(volume_name, list()).append(command)

            if layers_raw:
                # debug("layers: " + layers_raw[0])
                layers = layers_raw[0].split("=", 1)[1].split(";")
                self.volume2layers.setdefault(volume_name, set()).update(layers)

        elif parts[1] == "link":
            # volumeAPI: link /place/db/iss3/jobs/mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqcaDK to ISS-AGENT--mSWg_bzaR32tBE4o8xUFaQ_mSWg_bzaR32tBE4o8xUFaQ_CgeNqwqcaDK from 5:java(1037667)
            # 0          1    2                                                                             3  4
            volume_name = parts[2]
            container_name = parts[4]
            self._put_ddl(self.container2volume, container_name, volume_name, command)
            # debug("link volume(%s) --> container(%s)" % (volume_name, container_name))
        elif parts[1] == "unlink":
            """
volumeAPI: unlink /place/db/iss3/volumes/ISS-AGENT_UkwKOEchgBL from ISS-AGENT--GCauoT0QEeeJpgAlkJQnzA_GCauoT0QEeeJpgAlkJQnzA_kKnisE5glbE from 9:java(364374) []
            """
            volume_name = parts[2]
            container_name = parts[4]
            self._put_ddl(self.container2volume, container_name, volume_name, command)
        else:
            if parts[1] not in self.unknown_volume_api_commands:
                error_command(command, "unknown volume api command: " + parts[1])
                self.unknown_volume_api_commands.add(parts[1])
            return

    def _get_layer_name(self, line):
        parts = line.split()
        while parts:
            p = parts.pop(0)
            if p == "layer:":
                if parts:
                    layer_name = parts.pop(0).strip('"')  # TODO: parse json
                    return layer_name
        return ""

    def _has_known_container_in_import_layer(self, command):
        """
        all container names have full format and name from the root

ACT Create ISS-AGENT--Y3z96OoST-6jYfu9cPlpEw_Y3z96OoST-6jYfu9cPlpEw_rTktW2mAS5C/iss_hook_stop
        """
        if len(command) < 3:
            return False

        for cmd in command[1:-1]:
            if cmd.action == "ACT" and cmd.first_word_lo == "create":
                if self._string_has_part(cmd.message):
                    return True  # not first occurrence - can be more than one string in command

        return False

    def _register_layer_command(self, command_name, command):
        """
        'importLayer { layer: "ISS-AGENT_8gC8VRCJYUN" tarball: "/place/db/iss3/resources/nirvana-base-layer-b08505b77d638c0c1e3c21826a4034ac-b08505b77d638c0c1e3c21826a4034ac_YrUexk4GdvT/ubuntu-trusty-skynet.tar.xz" merge: false } from 5:java(1037667)'
        'removeLayer { layer: "ISS-AGENT_QwSF4Ocj5CR" } from 5:java(1037667)'
        """
        layer_name = self._get_layer_name(command[0].message)

        if layer_name:
            if command_name == "importlayer":
                self.all_import_layer_commands.setdefault(layer_name, list()).append(command)
            elif command_name == "removelayer":
                self.all_remove_layer_commands.setdefault(layer_name, list()).append(command)
            else:
                if command_name not in self.unknown_layer_commands:
                    error("UNKNOWN layer command: %s" % (command_name,))
                    self.unknown_layer_commands.add(command_name)
        else:
            error_command(command, "cant parse " + command_name)

    def add_comment_to_command(self, command, message):
        header = PortoLogLine("", command[0].line_index)
        header.raw = "# (reporto) WARNING: %s" % (message,)
        return [header, ] + command

    def add_layer_commands(self, known_layers):
        """
        for each layer from known_layers print one import command before known commands
        and one destroy commands after known commands
        """
        if not known_layers:
            return

        if not self.commands_to_print:
            return

        first_good_line = min(cmd[0].line_index for cmd in self.commands_to_print)
        last_good_line = max(cmd[0].line_index for cmd in self.commands_to_print)

        for layer_name in known_layers:
            import_found = False
            remove_found = False

            if layer_name in self.all_import_layer_commands:
                # assert that commands are ordered
                commands = self.all_import_layer_commands[layer_name]
                if commands:
                    import_found = True
                    before = [x for x in commands if x[0].line_index < first_good_line]
                    if before:
                        cmd = before[-1]
                    else:
                        cmd = commands[0]
                    commented_command = self.add_comment_to_command(cmd,
                                                                    "this command is for reference only - does not directly link to configuration")
                    self.save_one_command_to_print_queue(commented_command)

            if layer_name in self.all_remove_layer_commands:
                # assert that commands are ordered
                commands = self.all_remove_layer_commands[layer_name]
                if commands:
                    remove_found = True
                    after = [x for x in commands if x[0].line_index > last_good_line]
                    if after:
                        cmd = after[0]
                    else:
                        cmd = commands[-1]
                    commented_command = self.add_comment_to_command(cmd,
                                                                    "this command is for reference only - does not directly link to configuration")
                    self.save_one_command_to_print_queue(commented_command)

            if not import_found:
                comment("no importLayer command for layer %s" % (layer_name,))

            if not remove_found:
                comment("no removeLayer command for layer %s" % (layer_name,))

    def _report_stats(self):

        def report_dict2list(dict2list, label):
            max_value_items = 0
            total_items = 0
            for k, v in dict2list.iteritems():
                max_value_items = max(max_value_items, len(v))
                total_items += len(v)
            debug("DictList: %s: %d pairs, total %d items, max %d items on value" % (
            label, len(dict2list), total_items, max_value_items))

        def report_ddl(ddl, label):
            max_value_items = 0
            total_items = 0
            for k1, v1 in ddl.iteritems():
                for k2, v2 in v1.iteritems():
                    max_value_items = max(max_value_items, len(v2))
                    total_items += len(v2)
            debug("DictDictList: %s: %d records, total %d items, max %d items on value" % (
            label, len(ddl), total_items, max_value_items))

        report_dict2list(self.all_import_layer_commands, "all_import_layer_commands")
        report_dict2list(self.all_remove_layer_commands, "all_remove_layer_commands")
        report_dict2list(self.volume2layers, "volume2layers")
        report_dict2list(self.dget_containers, "dget_containers")
        report_dict2list(self.created_volumes, "created_volumes")
        report_dict2list(self.proc_exit, "proc_exit")

        report_ddl(self.containers_weak, "containers_weak")
        report_ddl(self.container2volume, "container2volume")
        report_ddl(self.container2proc, "container2proc")

    def link_commands(self):
        super(HistoryWithVolumes, self).link_commands()

        if DEBUG:
            self._report_stats()

        known_containers = set()

        source_and_label = (
            (self.container2volume, "container2volume"),
            (self.container2proc, "container2proc"),
            (self.dget_containers, "dget_containers"),
        )

        if self.unsupported_commands:
            comment("unknown porto commands: %s" % (self.unsupported_commands,))

        if self.unknown_command_codes:
            comment("unknown commands: %s" % (self.unknown_command_codes,))

        if self.unknown_pset_comands:
            comment("unknown pset subcommands: %s" % (self.unknown_pset_comands,))

        if self.unknown_layer_commands:
            comment("unknown layer commands: %s" % (self.unknown_layer_commands,))

        if self.unknown_volume_api_commands:
            comment("unknown volume api commands: %s" % (self.unknown_volume_api_commands,))

        for ddl, label in source_and_label:
            for k in ddl.keys():
                if self._string_has_part(k):
                    known_containers.add(k)
                    debug("add container %s - has conf name - %s" % (k, label))

        # TODO: self.created_volumes
        # container2volume

        # expand weak containers
        # TODO: make extra loop for tree of weak containers
        start_known_containers = list(known_containers)
        # self.containers_weak, parent, container_name, command
        for parent_container, weak_container_dict in self.containers_weak.iteritems():
            if parent_container in start_known_containers:
                for weak_name, weak_commands in weak_container_dict.iteritems():
                    if weak_name not in known_containers:
                        # containers.add(weak_name)
                        debug("add weak container %s - parent %s - containers_weak" % (weak_name, parent_container))
                    self.save_commands_to_print_queue(weak_commands)

        # containers from [dget]
        for container_name, commands in self.dget_containers.iteritems():
            if container_name in start_known_containers:
                self.save_commands_to_print_queue(commands)

        # collect procs
        known_procs = set()
        for container_name, procs_dict in self.container2proc.iteritems():
            if container_name in known_containers:
                for proc_name, proc_commands in procs_dict.iteritems():
                    if proc_name not in known_procs:
                        debug("add proc %s from container %s - container2proc" % (proc_name, container_name))
                    self.save_commands_to_print_queue(proc_commands)

        # report procs exit
        for proc_name, commands in self.proc_exit.iteritems():
            if proc_name in known_procs:
                self.save_commands_to_print_queue(commands)

        # collect volumes
        known_volumes = set()
        for container_name, volumes_dict in self.container2volume.iteritems():
            if container_name in known_containers:
                debug("link volumes(%s) --> container(%s)" % (volumes_dict.keys(), container_name))
                known_volumes |= set(volumes_dict.keys())
                for commands in volumes_dict.itervalues():
                    self.save_commands_to_print_queue(commands)

        known_layers = set()
        # self.volume2layers.setdefault(path, set())
        for volume, volume_layers in self.volume2layers.iteritems():
            if volume in known_volumes:
                debug("add layers(%s) to volume(%s)" % (volume_layers, volume))
                known_layers |= set(volume_layers)

        self.add_layer_commands(known_layers)

        # create volume commands
        for volume_name, commands in self.created_volumes.iteritems():
            if volume_name in known_volumes:
                debug("add volume create command for %s" % (volume_name,))
                self.save_commands_to_print_queue(commands)


class FindParallelContainers(GroupedActionsGenerator):
    def __init__(self, filename, target_container_name_part):
        super(FindParallelContainers, self).__init__(filename)
        self.target_container_name_part = target_container_name_part
        self.running_containers = dict()
        self.create_count = {}

    def process_command(self, command):
        destroy = False
        create = False
        container_name = ""

        first = command[0]
        last = command[-1]

        if first.action == "REQ":
            if first.first_word_lo in ("destroy", "create") \
                    and len(command) > 1 \
                    and last.action == "RSP" and last.first_word_lo == "ok":
                container_name = first.message.split()[1]
                if first.first_word_lo == "destroy":
                    destroy = True
                else:
                    create = True
        elif first.action == "EVT":
            if first.first_word_lo == "exit":
                container_name = first.message.split()[1]
                destroy = True
            else:
                """
                TODO:
                # unknown EVT: 29824 killed by OOM
                # unknown EVT: Updating
                """
                comment("unknown EVT: %s" % (first.message,))

        if container_name:
            container_name = container_name.rstrip(":")

        if destroy:
            if container_name in self.running_containers:
                del self.running_containers[container_name]
                # print "-", container_name
                self.create_count[container_name] = 1 + self.create_count.get(container_name, 0)
            else:
                # print "-", container_name, "(NOT EXISTS)"
                pass
        if create:
            if container_name in self.running_containers:
                # print "+", container_name, "(DUPLICATE)"
                pass
            else:
                self.running_containers[container_name] = command
                # print "+", container_name
                self.check_gotcha(container_name)

    def check_gotcha(self, container_name):
        if self.target_container_name_part in container_name:
            print "=== just created target %s ===" % (container_name,)
            print "running containers:"
            for i, name in enumerate(sorted(self.running_containers.iterkeys()), 1):
                print "[%d] %s" % (i, name)

            print

            print "last events:"
            for cmd in self.running_containers[container_name]:
                print cmd.raw
            print

            # print stats
            print "container with many creation count (> 10):"
            for k, v in sorted(self.create_count.iteritems(), key=lambda x: x[1], reverse=True):
                if v <= 10:
                    continue
                print "%s - %d times" % (k, v)

                # self.stop()


def parse_args():
    """
    filename
    container-name-part
    string to replace container name with
    """
    parser = argparse.ArgumentParser()
    parser.add_argument("name_part",
                        help="part of container name, might be group name or full container name")
    parser.add_argument("--replace", dest="replace_with", type=str, default="",
                        help="string to replace 'name_part' substring with (if specified)")
    parser.add_argument("-f", "--file", dest="filename", type=str, default=None,
                        help="porto log file to parse (search order if not specified: %s, then %s)" %
                             (PORTOD_LOG_LOCAL, PORTOD_LOG_SYSTEM))
    parser.add_argument("--leave-volume-spam", action="store_true",
                        help="turn off agent's volume spam eliminator feature")
    parser.add_argument("--no-spam-log", dest="no_spam_log", type=str, default=None,
                        help="file to store stripped source file, works without --leave-volume-spam only.")
    args = parser.parse_args()

    if args.filename is None:
        if os.path.isfile(PORTOD_LOG_LOCAL):
            args.filename = PORTOD_LOG_LOCAL
        elif os.path.isfile(PORTOD_LOG_SYSTEM):
            args.filename = PORTOD_LOG_SYSTEM
        else:
            print "ERROR: default log files not found and no -f option specified"
            sys.exit(1)

    comment("input file: %s" % (args.filename,))
    if not os.path.isfile(args.filename):
        print "ERROR: file not found: %s" % (args.filename,)
        sys.exit(1)

    comment("name part: %s" % (args.name_part,))
    if args.replace_with:
        comment("will replace [%s] with [%s]" % (args.name_part, args.replace_with))

    if args.no_spam_log is not None and args.leave_volume_spam:
        print "ERROR: both options specified: --leave-volume-spam / --no-spam-log"
        sys.exit(1)

    return args


def main():
    args = parse_args()

    parser = HistoryWithVolumes(args.filename, args.name_part, args.replace_with, args.leave_volume_spam, args.no_spam_log)
    parser.run()

    # failed_container = "ISS-AGENT--CUtlsXjkEeWO1QAlkJQnzA_master_CUtlsXjkEeWO1QAlkJQnzA_DnNQ05P1nb/iss_hook_start"
    # parser = FindParallelContainers("/Users/abcdenis/tmp/portod.log", failed_container)
    # parser.run()


if __name__ == '__main__':
    main()
