#!/usr/bin/env python2
# coding=utf-8
"""
log parser for porto 1.13

+ worked with porto 1.17
"""
import argparse
import os
import re
import sys


TIMESTAMP_RE = re.compile(r'^(\d{4}-\d{2}-\d{2} \d\d:\d\d:\d\d) (.+?)$')
SOURCE_RE = re.compile(r'^([^\s:]+:)\s+(.*)$')
ACTION_RE = re.compile(r'^([A-Z]{3})\s(.*)$')

PORTOD_LOG_LOCAL = './portod.log'
PORTOD_LOG_SYSTEM = '/var/log/portod.log'


def comment(msg):
    print "#", msg


class PortoLogLine(object):

    DUMMY_LINE = "0000-00-00 00:00:00 GroupedActionsGenerator: RSP this is dummy command for assembling last REQ"
    HOOK_NAME_RE = re.compile(r"(?<=/)(iss_hook_[a-z]+)")

    def __init__(self, raw):
        self.raw = None
        self.empty = None
        self.parse_error = None
        self.timestamp = None
        self.source = None
        self.raw_message = None
        self.action = None
        self.message = None
        self.first_word_lo = None
        self.parse_ok = False

        self._parse(raw)

    @staticmethod
    def get_dummy_record():
        return PortoLogLine(PortoLogLine.DUMMY_LINE)

    def _parse(self, raw):
        self.raw = raw

        if not raw.strip():
            self.empty = True
            return

        match = TIMESTAMP_RE.match(raw)
        if not match:
            self.parse_error = "cant get timestamp from line: %s" % (raw, )
            return

        self.timestamp, remainder = match.group(1, 2)

        match = SOURCE_RE.match(remainder.strip())
        if not match:
            self.parse_error = "cant get source from line: %s" % (raw, )
            return

        self.source, self.raw_message = match.group(1, 2)

        # try to extract action
        match = ACTION_RE.match(self.raw_message)
        if match:
            self.action, self.message = match.group(1, 2)
            self.first_word_lo = self.message.split()[0].lower()
        else:
            self.message = self.raw_message

        self.parse_ok = True

    def get_iss_hook_name(self):
        """
        returns hook name form first line, if present or empty string
        """
        match = self.HOOK_NAME_RE.search(self.raw)
        if match:
            return match.group(1)
        else:
            return ""


class PortoLogProcessorBase(object):
    """
    parses each portod.log line into PortoLogLine object and call process_line() on each
    """

    def __init__(self, filename):
        self.filename = filename
        self.lines_count = None
        self.pending_stop = False

    def stop(self):
        self.pending_stop = True

    def process_line(self, log_record):
        pass

    def run(self):
        self.lines_count = 0

        for raw_line in open(self.filename):
            self.lines_count += 1

            line = raw_line.splitlines()[0]
            record = PortoLogLine(line)
            self.process_line(record)

            if self.pending_stop:
                comment("stop processing")
                break

        comment("done, %d lines" % (self.lines_count, ))


class LogStatAccumulator(PortoLogProcessorBase):
    """
    accumulates (action, command_code) pairs from log

    result on some log:

        ('ACT', '/:')
        ('ACT', '/porto:')
        ('ACT', 'Create')
        ('ACT', 'Destroy')
        ('ACT', 'Kill')
        ('ACT', 'Remove')
        ('ACT', 'Set')
        ('ACT', 'Start')
        ('ACT', 'Stop')
        ('ACT', 'Touch')
        ('ACT', 'Unlink')
        ('ACT', 'iss-agent-container-5617192517817094794:')
        ('ACT', 'kill')
        ('ACT', 'mkdir')
        ('ACT', 'mount')
        ('EVT', 'Exit')
        ('REQ', 'create')
        ('REQ', 'destroy')
        ('REQ', 'pset')
        ('REQ', 'start')
        ('RSP', 'Error:')
        ('RSP', 'Ok')
        ('SYS', '--------------------------------------------------------------------------------')
        ('SYS', 'Started')
        ('SYS', 'network')

    2nd pass:
        ('ACT', 'Attach')
        ('ACT', 'ClearDirectory')
        ('ACT', 'Create')
        ('ACT', 'Destroy')
        ('ACT', 'Kill')
        ('ACT', 'KillAll')
        ('ACT', 'Remove')
        ('ACT', 'Send')
        ('ACT', 'Set')
        ('ACT', 'Start')
        ('ACT', 'Stop')
        ('ACT', 'iss-agent-container-5617192517817094794:')
        ('REQ', 'create')
        ('REQ', 'destroy')
        ('REQ', 'importLayer')
        ('REQ', 'pset')
        ('REQ', 'removeLayer')
        ('REQ', 'start')
        ('REQ', 'volumeAPI:')
        ('RSP', 'Wait')
    """

    def __init__(self, filename):
        super(LogStatAccumulator, self).__init__(filename)
        PortoLogProcessorBase.__init__(self, filename)
        self.action_args = set()

    def process_line(self, record):
        if not record.parse_ok:
            return

        if record.action:
            first_word = record.message.split()[0]

            if not first_word.startswith('ISS-AGENT--'):
                self.action_args.add((record.action, first_word))

    def run(self):
        super(LogStatAccumulator, self).run()

        print "found (action, command) pairs:"
        for i in sorted(self.action_args):
            print i


class GroupedActionsGenerator(PortoLogProcessorBase):
    """
    generate stream of full commands:
        REQ
            [ACT]*
        RSP
    --or--
        EVT
            [ACT]*

    then call process_command() on each
    """

    def __init__(self, filename):
        super(GroupedActionsGenerator, self).__init__(filename)
        self.command_count = 0
        self.records = []

    def process_line(self, log_record):
        if not log_record.parse_ok:
            return

        if not log_record.action:
            return

        self.records.append(log_record)

        self._extract_full_commands()

    def process_command(self, command):
        pass

    def _extract_full_commands(self):
        while self._extract_one_full_command():
            pass

    def _extract_one_full_command(self):
        # skip trash
        while self.records and self.records[0].action not in ['EVT', 'REQ']:
            print "# trash:", self.records.pop(0).raw

        if not self.records:
            return False

        first_action = self.records[0].action

        # find first non-ACT
        different_pos = None
        for idx, record in enumerate(self.records[1:], 1):
            if record.action != 'ACT':
                different_pos = idx
                break

        if different_pos is None:
            return False

        if first_action == 'EVT':
            # strip records before
            command_size = different_pos
        elif first_action == 'REQ':
            command_size = different_pos + 1
        else:
            raise ValueError("wrong first line action: %s" % (first_action, ))

        self._strip_and_handle_record(command_size)

        return True

    def _strip_and_handle_record(self, command_size):
        if command_size > len(self.records):
            raise ValueError("request %d of %d records batch" % (command_size, len(self.records)))

        command, self.records = self.records[:command_size], self.records[command_size:]

        self.command_count += 1

        self.process_command(command)

    def run(self):
        super(GroupedActionsGenerator, self).run()

        self.records.append(PortoLogLine.get_dummy_record())
        self._extract_full_commands()

        comment("%d full commands" % self.command_count)


class GroupHistoryFilter(GroupedActionsGenerator):

    def __init__(self, filename, name_part, replace_with, indent_responses=True):
        super(GroupHistoryFilter, self).__init__(filename)
        self.name_part = name_part
        self.replace_with = replace_with
        self.indent_responses = indent_responses
        self.prev_hook_name = ""
        self.line_patcher = self._get_line_patcher()
        self.hook_entry_count = {}

    def _has_part(self, command):
        for c in command:
            if self.name_part in c.raw:
                return True
        return False

    def _get_line_patcher(self):

        def patch_every_line(line):
            pass1 = line.replace(self.name_part, self.replace_with)

            if self.indent_responses:
                parts = pass1.split(" ", 5)

                if len(parts) == 6:
                    indent = False

                    if parts[3] not in ('REQ', 'EVT'):
                        indent = True
                    elif parts[3] == 'REQ' and parts[4] == 'pset':
                        indent = True

                    if indent:
                        parts[3] = '    ' + parts[3]
                        pass1 = ' '.join(parts)
            return pass1

        def pass_line(line):
            return line

        if self.replace_with:
            return patch_every_line
        else:
            return pass_line

    def print_command(self, command):
        for c in command:
            print self.line_patcher(c.raw)

    def process_command(self, command):
        if self._has_part(command):
            hook_name = command[0].get_iss_hook_name()

            if hook_name != self.prev_hook_name:
                comment('-' * 35 + hook_name + '-' * 35)
                self.prev_hook_name = hook_name

                if hook_name:
                    self.hook_entry_count[hook_name] = 1 + self.hook_entry_count.get(hook_name, 0)

            self.print_command(command)

    def run(self):
        super(GroupHistoryFilter, self).run()

        if self.replace_with:
            comment("name part [%s] replaced with [%s]" % (self.name_part, self.replace_with))

        if len(self.hook_entry_count):
            comment("=== hook run count")
            for hook in sorted(self.hook_entry_count.keys()):
                comment("%s: %d times" % (hook, self.hook_entry_count[hook]))
            comment('')


class FindParallelContainers(GroupedActionsGenerator):

    def __init__(self, filename, target_container_name_part):
        super(FindParallelContainers, self).__init__(filename)
        self.target_container_name_part = target_container_name_part
        self.running_containers = dict()
        self.create_count = {}

    def process_command(self, command):
        destroy = False
        create = False
        container_name = ''

        first = command[0]
        last = command[-1]

        if first.action == 'REQ':
            if first.first_word_lo in ('destroy', 'create') \
                    and len(command) > 1 \
                    and last.action == 'RSP' and last.first_word_lo == 'ok':
                container_name = first.message.split()[1]
                if first.first_word_lo == 'destroy':
                    destroy = True
                else:
                    create = True
        elif first.action == 'EVT':
            if first.first_word_lo == 'exit':
                container_name = first.message.split()[1]
                destroy = True
            else:
                """
                TODO:
                # unknown EVT: 29824 killed by OOM
                # unknown EVT: Updating
                """
                comment("unknown EVT: %s" % (first.message, ))

        if container_name:
            container_name = container_name.rstrip(":")

        if destroy:
            if container_name in self.running_containers:
                del self.running_containers[container_name]
                # print "-", container_name
                self.create_count[container_name] = 1 + self.create_count.get(container_name, 0)
            else:
                # print "-", container_name, "(NOT EXISTS)"
                pass
        if create:
            if container_name in self.running_containers:
                # print "+", container_name, "(DUPLICATE)"
                pass
            else:
                self.running_containers[container_name] = command
                # print "+", container_name
                self.check_gotcha(container_name)

    def check_gotcha(self, container_name):
        if self.target_container_name_part in container_name:
            print "=== just created target %s ===" % (container_name, )
            print "running containers:"
            for i, name in enumerate(sorted(self.running_containers.iterkeys()), 1):
                print "[%d] %s" % (i, name)

            print

            print "last events:"
            for cmd in self.running_containers[container_name]:
                print cmd.raw
            print

            # print stats
            print "container with many creation count (> 10):"
            for k, v in sorted(self.create_count.iteritems(), key=lambda x: x[1], reverse=True):
                if v <= 10:
                    continue
                print "%s - %d times" % (k, v)

            # self.stop()


def parse_args():
    """
    filename
    container-name-part
    string to replace container name with
    """
    parser = argparse.ArgumentParser()
    parser.add_argument("name_part",
                        help="part of container name, might be group name or full container name")
    parser.add_argument("--replace", dest="replace_with", type=str, default="",
                        help="string to replace 'name_part' substring with (if specified)")
    parser.add_argument("-f", "--file", dest="filename", type=str, default=None,
                        help="porto log file to parse (search order if not specified: %s, then %s)" %
                             (PORTOD_LOG_LOCAL, PORTOD_LOG_SYSTEM))
    args = parser.parse_args()

    if args.filename is None:
        if os.path.isfile(PORTOD_LOG_LOCAL):
            args.filename = PORTOD_LOG_LOCAL
        elif os.path.isfile(PORTOD_LOG_SYSTEM):
            args.filename = PORTOD_LOG_SYSTEM
        else:
            print "ERROR: default log files not found and no -f option specified"
            sys.exit(1)

    comment("input file: %s" % (args.filename, ))
    if not os.path.isfile(args.filename):
        print "ERROR: file not found: %s" % (args.filename, )
        sys.exit(1)

    comment("name part: %s" % (args.name_part, ))
    if args.replace_with:
        comment("will replace [%s] with [%s]" % (args.name_part, args.replace_with))

    return args


def main():
    args = parse_args()



    parser = GroupHistoryFilter(args.filename, args.name_part, args.replace_with)
    parser.run()


    """
    instance_name:
        i-367e211a171d_qloud-cocaine_cocaine_test_api_test_2XmARtoJNCC

    start timestamp:
        2016-07-25 06:49:32 portod-worker3[504726]:


    """

    # failed_container = "ISS-AGENT--CUtlsXjkEeWO1QAlkJQnzA_master_CUtlsXjkEeWO1QAlkJQnzA_DnNQ05P1nb/iss_hook_start"
    # parser = FindParallelContainers("/Users/abcdenis/tmp/portod.log", failed_container)
    # parser.run()

if __name__ == '__main__':
    main()
