#!/usr/bin/python
# -*- coding: utf8 -*-

import sys
import os

sys.path.insert(0, '/opt/direct-py/startrek-python-client-sni-fix')
from startrek_client import Startrek

import logging
import yaml
import json
import re
import subprocess
import argparse
from enum import Enum
import pygraphviz as pgv
import tempfile
import time
import datetime
import dateutil.parser
from dateutil import tz
from retry.api import retry_call
from collections import defaultdict

logging.getLogger('startrek_client').setLevel(logging.CRITICAL)

pathlist = ["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin", "/usr/games", "/usr/local/games"]
os.environ["PATH"] = os.pathsep.join(pathlist)

SCRIPT_NAME = os.path.basename(__file__)
HOSTNAME = subprocess.check_output(['hostname', '-f']).strip()
SIGN = u"----\nСкрипт %s с машины %s" % (SCRIPT_NAME, HOSTNAME)

STARTREK_TOKEN_FILE = '/etc/direct-tokens/startrek'
startrek_token = open(STARTREK_TOKEN_FILE).readline().strip()
startrek_client = Startrek(token=startrek_token, useragent=SCRIPT_NAME)

DEPS_COMPONENT = u'"Зависимости между релизами"'
ST_QUERY = u'Queue: DIRECT Components: %s Status: !Closed "Sort by": key desc' % DEPS_COMPONENT

APPS_CONF_FILE = "/etc/yandex-direct/direct-apps.conf.yaml"

opts = {}
apps_dict = {}
component2app = {}


html_escape_table = {
    "&": "&amp;",
    '"': "&quot;",
    "'": "&apos;",
    ">": "&gt;",
    "<": "&lt;",
}


def html_escape(string, limit=None):
    pos_limit = limit
    if not pos_limit:
        pos_limit = len(string)
    return "".join([html_escape_table[c] if c in html_escape_table else c for c in string[:pos_limit]])


class DirEdge(Enum):
    TO = 0
    FROM = 1


class Graph:
    def __init__(self, graph_dict=None):
        if graph_dict == None:
            graph_dict = {}
        self.__graph_dict = graph_dict
        self.clusters = []
        self.rclusters = {}
        self.ver2rcluster = {}
        self.rcluster2ver = {}
        self.build_message = u""

    def build_graph(self, tickets):
        global startrek_client, opts

        message = u""

        for ticket in tickets:
            try:
                cur_message = u""
                desc = ticket.description.split("\n")

                if u"43460" not in [component.id for component in ticket.components]:
                    cur_message += u"FATAL: dependency ticket %s doesn't have right component\n" % ticket.key

                if not re.search(ur"^Зависимость релизов:.+", ticket.summary, re.I | re.U):
                    cur_message += u"WARNING: unexpected ticket summary format\n"

                in_section = False
                desc.append(" ")

                cluster = set()
                edges = []

                for line in desc:
                    if re.search(ur'^(?:\*\*)?Порядок', line, re.I | re.U):
                        in_section = True
                        prev_ver = None
                        cur_ver = None
                        continue

                    if in_section:
                        match_result = re.match(ur'^([^\/]*)\/(.*)$', line, re.I | re.U)

                        if not match_result:
                            if not cur_ver:
                                cur_message += u"WARNING: empty section\n"
                            elif not prev_ver:
                                cur_message += u"WARNING: section consists of only one ticket\n"
                            in_section = False
                            continue

                        prev_ver = cur_ver

                        cur_ver = {'ticket': match_result.group(2).strip(), 'app': match_result.group(1).strip()}

                        if cur_ver['app'] not in apps_dict:
                            cur_message += u"FATAL: unknown app name '%s'\n" % cur_ver['app']
                        elif cur_ver['app'] != 'direct' and 'tracker-deployed-tag' not in apps_dict[cur_ver['app']]:
                            cur_message += u"FATAL: unsupported app '%s'\n" % cur_ver['app']

                        try:
                            startrek_client.issues[cur_ver['ticket']]
                        except:
                            cur_message += u"FATAL: unknown ticket '%s'\n" % cur_ver['ticket']
                        else:
                            if not cur_ver['ticket'].upper().startswith(u'DIRECT-'):
                                cur_message += u"FATAL: ticket should be in 'DIRECT' queue\n"

                        if prev_ver and cur_ver:
                            edges.append((prev_ver, cur_ver))

                if not re.search(ur"^FATAL:", cur_message, re.I | re.M):
                    for edge in edges:
                        self.add_edge(edge[0], edge[1], ticket.key)
                        for ver in edge:
                            if ver:
                                cluster.add((ver['ticket'], ver['app']))

                    if cluster:
                        self.add_cluster(ticket.key, list(cluster))

                message += cur_message

            except:
                pass

        self.build_message = message
        return self


    def get_release_clusters(self):
        if not self.__graph_dict:
            return

        releases = startrek_client.issues.find(u'Queue: DIRECT Type: Release Status: !closed "Sort by": key asc')
        #releases = startrek_client.issues.find(u'Queue: DIRECT Type: Release Created: > today() -"8w 1d" "Sort by": key desc')

        for release in releases:
            app = get_release_app_name(release)
            tickets_to_check = [release.key]
            tickets_to_check.extend(get_tickets_from_release(release.key))

            release_cluster = set()

            for ticket in tickets_to_check:
                if self.has_vertex(ticket, app):
                    release_cluster.add((ticket, app))

            if release_cluster:
                for ver in release_cluster:
                    if ver[0] not in self.ver2rcluster:
                        self.ver2rcluster[ver[0]] = {}
                    if ver[1] not in self.ver2rcluster[ver[0]]:
                        self.ver2rcluster[ver[0]][ver[1]] = release.key

        self.rclusters = {}
        for ticket in self.ver2rcluster:
            for app in self.ver2rcluster[ticket]:
                if self.ver2rcluster[ticket][app] not in self.rclusters:
                    self.rclusters[self.ver2rcluster[ticket][app]] = []
                self.rclusters[self.ver2rcluster[ticket][app]].append((ticket, app))

        return


    def update_rcluster2ver(self):
        """
        Строим обратное отображение, используя ver2rcluster:
        (релиз -> его тикеты, участвующие в зависимостях)
        """

        self.rcluster2ver = {}

        for ticket in self.ver2rcluster:
            for app in self.ver2rcluster[ticket]:
                if self.ver2rcluster[ticket][app] not in self.rcluster2ver:
                    self.rcluster2ver[self.ver2rcluster[ticket][app]] = []

                self.rcluster2ver[self.ver2rcluster[ticket][app]].append((ticket, app))

        return


    def find_related_dep_tickets(self, release_key):
        """
        Получаем для данного релиза список связанных тикетов-зависимостей
        (т. е. в которых участвуют тикеты из данного релиза)
        """

        if not release_key in self.rcluster2ver:
            return []

        return list(set([to[3] for ver in self.rcluster2ver[release_key] for to in self.__graph_dict[ver[0]][ver[1]]]))


    def find_undeployed_related_releases(self, release_key):
        """
        Находим релизы, которые должны быть выложены раньше чем, данный релиз
        """

        if not release_key in self.rcluster2ver:
            return []

        result = []

        for ver in self.rcluster2ver[release_key]:
            for to in self.__graph_dict[ver[0]][ver[1]]:
                if to[2] == DirEdge.FROM and \
                   (to[0] in self.ver2rcluster and to[1] in self.ver2rcluster[to[0]]) and \
                   not self.__is_ticket_deployed(to[0], to[1]):
                    result.append((self.ver2rcluster[to[0]][to[1]], to[3]))

        return result


    def find_deps_without_release(self, release_key):
        """
        Находим тикеты, от которых зависит данный релиз, но не попавшие в релизы
        """

        if not release_key in self.rcluster2ver:
            return []

        result = []

        for ver in self.rcluster2ver[release_key]:
            for to in self.__graph_dict[ver[0]][ver[1]]:
                if to[2] == DirEdge.FROM and \
                   (to[0] not in self.ver2rcluster or to[1] not in self.ver2rcluster[to[0]]) and \
                   not self.__is_ticket_deployed(to[0], to[1]):
                    result.append((to[0], to[3]))

        return result

    
    def vertices(self):
        return [(ticket, app) for ticket in self.__graph_dict for app in self.__graph_dict[ticket]]

    def edges(self):
        return [
            ((ticket, app), (edge[0], edge[1]))
            for ticket in self.__graph_dict
            for app in self.__graph_dict[ticket]
            for edge in self.__graph_dict[ticket][app] if edge[2] == DirEdge.TO
        ]

    def has_vertex(self, ticket, app):
        return ticket in self.__graph_dict and app in self.__graph_dict[ticket]

    def clear(self):
        self.__graph_dict = {}

    def add_edge(self, fr, to, ticket):
        if fr['ticket'] not in self.__graph_dict:
            self.__graph_dict[fr['ticket']] = {}

        if fr['app'] not in self.__graph_dict[fr['ticket']]:
            self.__graph_dict[fr['ticket']][fr['app']] = set()

        if to['ticket'] not in self.__graph_dict:
            self.__graph_dict[to['ticket']] = {}

        if to['app'] not in self.__graph_dict[to['ticket']]:
            self.__graph_dict[to['ticket']][to['app']] = set()

        self.__graph_dict[fr['ticket']][fr['app']].add((to['ticket'], to['app'], DirEdge.TO, ticket))
        self.__graph_dict[to['ticket']][to['app']].add((fr['ticket'], fr['app'], DirEdge.FROM, ticket))


    def has_rcycle(self):
        used = {}
        parents = {}
        for cluster in self.rclusters:
            used[cluster] = 0

        for release in self.rclusters:
            if used[release] == 0:
                result = self.__find_rcycle(release, used, parents)
                if not result:
                    continue

                ver = parents[self.ver2rcluster[result[0]][result[1]]]
                cycle = []
                cycle.append(((ver[0][0], ver[0][1], self.ver2rcluster[ver[0][0]][ver[0][1]]), (ver[1][0], ver[1][1], self.ver2rcluster[ver[1][0]][ver[1][1]]), ver[2]))

                while self.ver2rcluster[ver[0][0]][ver[0][1]] != self.ver2rcluster[result[0]][result[1]]:
                    ver = parents[self.ver2rcluster[ver[0][0]][ver[0][1]]]
                    cycle.append(((ver[0][0], ver[0][1], self.ver2rcluster[ver[0][0]][ver[0][1]]), (ver[1][0], ver[1][1], self.ver2rcluster[ver[1][0]][ver[1][1]]), ver[2]))

                cycle = list(reversed(cycle))

                return cycle

        return None


    def __find_rcycle(self, release, used, parents):
        used[release] = 1

        for ver in self.rclusters[release]:
            for to in self.__graph_dict[ver[0]][ver[1]]:
                if to[2] != DirEdge.TO or to[0] not in self.ver2rcluster or to[1] not in self.ver2rcluster[to[0]]:
                    continue

                parents[self.ver2rcluster[to[0]][to[1]]] = (ver, (to[0], to[1]), to[3])

                if used[self.ver2rcluster[to[0]][to[1]]] == 1:
                    return to[0], to[1]
                elif used[self.ver2rcluster[to[0]][to[1]]] == 0:
                    result = self.__find_rcycle(self.ver2rcluster[to[0]][to[1]], used, parents)
                    if result:
                        return result

        used[release] = 2
        return None


    def has_cycle(self):
        used = {}
        for ticket in self.__graph_dict:
            used[ticket] = {}
            for app in self.__graph_dict[ticket]:
                used[ticket][app] = 0
        parents = {}

        for ticket in self.__graph_dict:
            for app in self.__graph_dict[ticket]:
                if used[ticket][app] == 0:
                    result = self.__find_cycle(ticket, app, used, parents)
                    if not result:
                        continue

                    ver = result[1]
                    cycle = []
                    
                    while ver != result[0]:
                        cycle.append(ver)
                        ver = parents[ver]

                    cycle.append(ver)
                    cycle = list(reversed(cycle))

                    return cycle
                       
        return None


    def __find_cycle(self, ticket, app, used, parents):
        used[ticket][app] = 1

        for to in self.__graph_dict[ticket][app]:
            if to[2] == DirEdge.TO:
                if used[to[0]][to[1]] == 1:
                    return ((to[0], to[1]), (ticket, app))
                elif used[to[0]][to[1]] == 0:
                    parents[(to[0],to[1])] = (ticket, app)
                    result = self.__find_cycle(to[0], to[1], used, parents)
                    if result:
                        return result
        
        used[ticket][app] = 2
        return None

    
    def check_rollback(self, ticket, app):
        result = []
        if ticket not in self.__graph_dict or app not in self.__graph_dict[ticket]:
            return result

        for to in self.__graph_dict[ticket][app]:
            if to[2] == DirEdge.TO and self.__is_ticket_deployed(to[0], to[1]):
                result.append(((ticket, app), (to[0], to[1]), to[3]))

        return result


    def find_unsatisfied_dependencies(self, tickets, app):
        result = {} 

        for ticket in tickets:
            if ticket not in self.__graph_dict or app not in self.__graph_dict[ticket]:
                continue

            errors = []
            self.__check_ticket_for_satisfaction(ticket, app, errors)

            if errors:
                result[ticket] = errors

        return result

            
    def __check_ticket_for_satisfaction(self, ticket, app, errors):
        global startrek_client, apps_dict

        for to in self.__graph_dict[ticket][app]:
            if to[2] == DirEdge.FROM:
                self.__check_ticket_for_satisfaction(to[0], to[1], errors)
                if not self.__is_ticket_deployed(to[0], to[1]):
                    errors.append(((to[0], to[1]), (ticket, app), to[3]))

        return


    def __is_ticket_deployed(self, ticket, app):
        return apps_dict[app]['tracker-deployed-tag'] in startrek_client.issues[ticket].tags


    def are_all_dependencies_satisfied(self):
        global startrek_client, apps_dict

        for ticket in self.__graph_dict:
            for app in self.__graph_dict[ticket]:
                for to in self.__graph_dict[ticket][app]:
                    if to[2] == DirEdge.FROM and not self.__is_ticket_deployed(to[0], to[1]):
                        return False

                if not self.__is_ticket_deployed(ticket, app):
                    return False

        return True


    def __format_name(self, ticket, app):
        global startrek_client

        return "%s/%s" % (ticket, app)


    def add_cluster(self, name, vertices):
        self.clusters.append({'name': name, 'vertices': vertices})

        return

 
    def visualize(self, filename, with_releases=False):
        global startrek_client

        graph_image = pgv.AGraph(strict=False, directed=True, rankdir="LR", dpi="300")
        graph_image.node_attr.update(style='filled')
        used = {}

        if with_releases:
            for ticket in self.__graph_dict:
                for app in self.__graph_dict[ticket]:
                    for to in self.__graph_dict[ticket][app]:
                        if to[2] == DirEdge.TO:
                            if ((ticket in self.ver2rcluster and app in self.ver2rcluster[ticket]) or
                                    (to[0] in self.ver2rcluster and to[1] in self.ver2rcluster[to[0]])):
                                graph_image.add_edge(
                                    self.__format_name(ticket, app),
                                    self.__format_name(to[0], to[1]),
                                    to[3],
                                    label="<<U>%s</U>>" % to[3],
                                    labelhref='https://st.yandex-team.ru/' + to[3],
                                    labeltarget='_blank',
                                    labeltooltip='Open in Startrek',
                                    fontsize=9.0
                                )

            for ticket in self.__graph_dict:
                for app in self.__graph_dict[ticket]:
                    try:
                        node = graph_image.get_node(self.__format_name(ticket, app))
                        node.attr['fillcolor'] = '#5ad65a' if self.__is_ticket_deployed(ticket, app) else '#ff3333'
                        node.attr['label'] = "<<TABLE BORDER='0' CELLPADDING='0' CELLSPACING='0'>%s%s%s</TABLE>>" % (
                            "<TR><TD TARGET='_blank' HREF='%s' TITLE='Open in Startrek'><U>%s</U></TD></TR>" % (
                                "https://st.yandex-team.ru/" + ticket, ticket
                            ),
                            "<TR><TD>%s</TD></TR>" % app,
                            "<TR><TD><FONT POINT-SIZE='10'>'%s...'</FONT></TD></TR>" % (
                                html_escape(startrek_client.issues[ticket].summary, 30)
                            )
                        )
                    except:
                        pass

            # выделяем цикл цветом
            cycle = self.has_rcycle()
            if cycle:
                for pair in cycle:
                    try:
                        edge = graph_image.get_edge(
                            self.__format_name(pair[0][0], pair[0][1]),
                            self.__format_name(pair[1][0], pair[1][1])
                        )
                        edge.attr['color'] = 'purple'
                        edge.attr['penwidth'] = 7
                    except:
                        pass

            for index, cluster in enumerate(self.rclusters, 1):
                graph_image.add_subgraph(
                    [self.__format_name(*ver) for ver in self.rclusters[cluster]],
                    name='cluster_%d' % index,
                    label="<<U>" + cluster + u" / " + get_release_app_name(startrek_client.issues[cluster]) + "</U>>",
                    href='https://st.yandex-team.ru/' + cluster,
                    target='_blank',
                    tooltip='Open in Startrek',
                    fontsize=15.0,
                    rank='same'
                )
        else:
            for ticket in self.__graph_dict:
                for app in self.__graph_dict[ticket]:
                    for to in self.__graph_dict[ticket][app]:
                        if to[2] == DirEdge.TO:
                            graph_image.add_edge(self.__format_name(ticket, app), self.__format_name(to[0], to[1]), to[3])

            for ticket in self.__graph_dict:
                for app in self.__graph_dict[ticket]:
                    node = graph_image.get_node(self.__format_name(ticket, app))
                    node.attr['fillcolor'] = '#5ad65a' if self.__is_ticket_deployed(ticket, app) else '#ff3333'
                    node.attr['label'] = "<<TABLE BORDER='0' CELLPADDING='0' CELLSPACING='0'>%s%s%s</TABLE>>" % (
                        "<TR><TD TARGET='_blank' HREF='%s' TITLE='Open in Startrek'><U>%s</U></TD></TR>" % (
                            "https://st.yandex-team.ru/" + ticket, ticket
                        ),
                        "<TR><TD>%s</TD></TR>" % app,
                        "<TR><TD><FONT POINT-SIZE='10'>'%s...'</FONT></TD></TR>" % (
                            html_escape(startrek_client.issues[ticket].summary, 30)
                        )
                    )
            # выделяем цикл цветом
            cycle = self.has_cycle()
            if cycle:
                cycle.append(cycle[0])
                for i in xrange(len(cycle) - 1):
                    edge = graph_image.get_edge(
                        self.__format_name(cycle[i][0], cycle[i][1]),
                        self.__format_name(cycle[i + 1][0], cycle[i + 1][1])
                    )
                    edge.attr['color'] = 'purple'
                    edge.attr['penwidth'] = 7

            for index, cluster in enumerate(self.clusters):
                graph_image.add_subgraph(
                    [self.__format_name(*ver) for ver in cluster['vertices']],
                    name='cluster_%d' % index,
                    label="<<U>%s</U>>" % cluster['name'],
                    href='https://st.yandex-team.ru/' + cluster['name'],
                    target='_blank',
                    tooltip='Open in Startrek',
                    rank='same'
                )

        if not graph_image:
            return False

        graph_image.layout(prog='dot')
        graph_image.draw(filename, format='svg', prog='dot')
        return True


    def __str__(self):
        return str(self.__graph_dict)

    def __nonzero__(self):
        return bool(self.__graph_dict)


def read_apps_dict(filename):
    return yaml.load(open(APPS_CONF_FILE))['apps']
 

def get_dependencies_tickets():
    global startrek_client, opts

    if opts.dependency:
        return [startrek_client.issues[dependency] for dependency in opts.dependency]
    else:
        return startrek_client.issues.find(ST_QUERY)


def str_cycle(_vertices):
    vertices = list(_vertices)
    if vertices[-1] != vertices[0]:
         vertices.append(vertices[0])

    return " -> ".join(["%s/%s" % (ver[0], ver[1]) for ver in vertices])


def str_rcycle(rcycle, tracker=None):
    if tracker:
        return u"\n".join([
u"""%d. **релизы: %s/((https://st.yandex-team.ru/%s %s)) --> %s/((https://st.yandex-team.ru/%s %s))**
тикеты в релизах: ((https://st.yandex-team.ru/%s %s)) --> ((https://st.yandex-team.ru/%s %s)) (тикет-зависимость: ((https://st.yandex-team.ru/%s %s)))""" \
% (index, pair[0][1], pair[0][2], pair[0][2], pair[1][1], pair[1][2], pair[1][2], pair[0][0], pair[0][0], pair[1][0], pair[1][0], pair[2], pair[2]) 
for index, pair in enumerate(rcycle, 1)])

    return u"\n".join([u"%d. releases: %s/%s --> %s/%s\ntickets in releases: %s -> %s (dep ticket: %s)" \
                       % (index, pair[0][1], pair[0][2], pair[1][1], pair[1][2], pair[0][0], pair[1][0], pair[2])
                       for index, pair in enumerate(rcycle, 1)])


def check_release():
    global startrek_client, opts

    release_ticket = opts.check_release

    print u"Check for release dependencies (%s):\n" % release_ticket

    try:
        issue = startrek_client.issues[release_ticket]
    except:
        print u"FAIL: ticket %s not found" % release_ticket
        exit(1)

    if not check_open_deps(for_check_release=True):
        print u"FAIL: wrong format in dependency tickets (run script with --check-open-deps if you want to see full report about format errors or you can choose only needed tickets for release check with --dependency)"
        exit(1)

    tickets_to_check = re.findall(ur'DIRECT-[0-9]+', issue.description)

    comments = startrek_client.issues[issue.key].comments.get_all()
    for comment in comments:
        tickets_to_check.extend(re.findall(ur'DIRECT-[0-9]+', comment.text))

    dep_tickets = get_dependencies_tickets()

    graph = Graph().build_graph(dep_tickets)

    cycle = graph.has_cycle()

    if cycle:
        print U"FAIL: dependency graph contains a cycle (perhaps more than one dependency ticket is involved in cycle), cycle: %s" % (str_cycle(cycle))
        exit(1)

    unsatisfied_dependencies = graph.find_unsatisfied_dependencies(tickets_to_check, get_release_app_name(issue))

    if not unsatisfied_dependencies:
        app = get_release_app_name(issue)
        for ticket_to_check in tickets_to_check:
            if (ticket_to_check, app) in set(graph.vertices()):
                print u"OK: All dependencies are satisfied"
                return

        print u"OK: no dependencies found"
    else:
        for ticket in unsatisfied_dependencies:
            print u"Dependencies for ticket %s are not satisfied:\n%s\n" % (ticket, "\n".join(["%s/%s -> %s/%s   (%s) FAIL" % (dep[0][0], dep[0][1], dep[1][0], dep[1][1], dep[2]) for dep in unsatisfied_dependencies[ticket]]))

        print u"FAIL: There are some unsatisfied dependencies!!"
        exit(1)
    
    return


def get_tickets_from_release(release):
    result = []

    description = startrek_client.issues[release].description
    if description:
        result.extend(re.findall(ur'DIRECT-[0-9]+', description))

    comments = startrek_client.issues[release].comments.get_all()
    for comment in comments:
        result.extend(re.findall(ur'DIRECT-[0-9]+', comment.text))

    return list(set(result))


def notify():
    global startrek_client

    comment_text = u"Информация о зависимостях:"
    release_viz_comment = u"Визуализацию зависимостей, связанных с релизами, можно смотреть ((https://observatorium.common.yandex.ru/release_deps/ticket_viz?releases=1 здесь))"

    tag = u'deps_notification'

    issues = startrek_client.issues.find(u'Queue: DIRECT Type: Release Status: !closed Tags: !"%s" Status: !new "Sort by": key desc' % tag)

    full_graph = Graph().build_graph(get_dependencies_tickets())
    if not full_graph:
        return

    full_graph.get_release_clusters()
    full_graph.update_rcluster2ver()

    for issue in issues:
        startrek_client.issues[issue.key].update(tags=startrek_client.issues[issue.key].tags + [tag])

        related_deps = full_graph.find_related_dep_tickets(issue.key)

        if not related_deps:
            comment_no_deps = u"Зависимостей, связанных с этим релизом, не нашлось"
            comment_no_deps += "\n%s" % release_viz_comment
            startrek_client.issues[issue.key].comments.create(text=u"%s\n%s" % (comment_no_deps, SIGN))
            continue

        # это будет список "просто зависимостей, без доп. проблем": сначала записываем все, потом выкидываем зависимости с особыми проблемами
        nonblocking_deps = { d: 1 for d in related_deps }

        comment = comment_text
        deps_without_release = full_graph.find_deps_without_release(issue.key)
        if deps_without_release:
            for ticket in deps_without_release: 
                comment += u"\n1. {[ПРОБЛЕМА: зависимость от задачи, еще не собранной в релиз st:%s\nзависимость: %s]}" % (ticket[0], ticket[1])
                if ticket[1] in nonblocking_deps:
                    del nonblocking_deps[ ticket[1] ]

        undeployed_related_releases = full_graph.find_undeployed_related_releases(issue.key)
        if undeployed_related_releases:
            for release in undeployed_related_releases:
                comment += u"\n1. {[будет выкладываться ПОСЛЕ релиза %s\nзависимость: %s]}" % (release[0], release[1]) 
                if release[1] in nonblocking_deps:
                    del nonblocking_deps[ release[1] ]

        for d in nonblocking_deps.keys():
            comment += u"\n1. неблокирующая зависимость: %s" % d

        comment += "\n%s" % release_viz_comment

        startrek_client.issues[issue.key].comments.create(text=u"%s\n%s" % (comment, SIGN))

    return


def close():
    global startrek_client, opts

    dep_tickets = get_dependencies_tickets()

    print u"Ready to close tickets:"

    tickets_to_close = []

    for ticket in dep_tickets:
        graph = Graph().build_graph([ticket])
        if graph and graph.are_all_dependencies_satisfied():
            tickets_to_close.append(ticket.key)
    
    print u", ".join(tickets_to_close) if tickets_to_close else u"nothing"
   
    if opts.tracker:
        for ticket in tickets_to_close:
            startrek_client.issues[ticket].transitions['close'].execute(comment=u"По данному тикету все зависимости выполнены\n%s" % SIGN, resolution='fixed')
             
    return


def search_for_deps():
    global startrek_client, opts

    tickets = startrek_client.issues.find(u'Queue: DIRECT Summary: "зависимость релизов" Updated: > today() - "2d" Components: !"Зависимости между релизами"')

    print u"Possible dependency tickets:\n%s" % (','.join([ticket.key for ticket in tickets]) if tickets else u"таких нет")

    return


def utc_to_local(utc_dt):
    #return utc_dt.replace(tzinfo=tz.gettz('UTC')).astimezone(tz.tzlocal())
    return utc_dt


def is_commented(ticket, text):
    for comment in startrek_client.issues[ticket].comments.get_all():
        if comment.createdBy.id == u'ppc' and comment.text.startswith(text):
            return True

    return False


def create_visualization(dep_tickets, directory):
    global opts

    graph = Graph().build_graph(dep_tickets)
    if graph:
        with tempfile.NamedTemporaryFile(dir=directory, suffix='.svg', delete=False) as tmpfile:
            print u"Processing tickets %s." % (
                ', '.join([ticket.key for ticket in dep_tickets]), 
            )
            if opts.release_visualize:
                graph.get_release_clusters()
                image_created = graph.visualize(filename=tmpfile.name, with_releases=True)
            else:
                image_created = graph.visualize(filename=tmpfile.name)

            if image_created:
                print u"Image of dependency graph for tickets %s:\nFile: %s" % (
                    ', '.join([ticket.key for ticket in dep_tickets]),
                    tmpfile.name
                )
            else:
                print u"Empty image of dependency graph for tickets %s" % (
                    ', '.join([ticket.key for ticket in dep_tickets])
                )
    else:
        print "Can't build graph for tickets %s, build message:\n###\n%s\n###" % (
            ', '.join([ticket.key for ticket in dep_tickets]),
            graph.build_message.strip()
        )


def visualize():
    global startrek_client, opts

    dep_tickets = get_dependencies_tickets()

    if opts.one_image:
        create_visualization(dep_tickets, opts.dir)
    else:
        for ticket in dep_tickets:
            create_visualization([ticket], opts.dir)

            if opts.tracker and opts.visualize:
                visualize_prefix = u'Визуализация этой зависимости доступна'
                VISUALIZE_COMMENT = visualize_prefix + u' ((https://observatorium.common.yandex.ru/release_deps/ticket_viz?ticket=%s здесь))\n%s' % (ticket.key, SIGN)
                if not is_commented(ticket.key, visualize_prefix):
                    startrek_client.issues[ticket.key].comments.create(text=VISUALIZE_COMMENT)

    return


def get_release_app_name(issue):
    global component2app

    for component in issue.components:
        if component.name in component2app:
            return component2app[component.name]

    return ''


def check_open_deps(for_check_release=None):
    global startrek_client, opts

    print u"Checking format for dependency tickets:\n"

    dep_tickets = get_dependencies_tickets()

    result = True

    whole_graph = Graph().build_graph(dep_tickets)
    whole_graph_cycle = whole_graph.has_cycle() if whole_graph else None

    for ticket in dep_tickets:
        graph = Graph().build_graph([ticket])
        message = graph.build_message

        cycle = graph.has_cycle() if graph else None

        if not message:
            if cycle:
                message += u"FATAL: graph contains a cycle (%s)\n" % (str_cycle(cycle))
            elif whole_graph_cycle and set(graph.vertices()) & set(whole_graph_cycle):
                message += u"FATAL: whole graph contains a cycle (%s), check it on https://observatorium.common.yandex.ru/release_deps/ticket_viz?ticket=all\n" % (str_cycle(whole_graph_cycle))
 
        print u"Found dependency ticket %s:\n%s\n" % (ticket.key, u"ok" if not message else message if not for_check_release else u"wrong format")

        if message and re.search(ur"^FATAL:", message, re.I | re.M):
            result = False
 
        if opts.tracker and not for_check_release:
            if message and ticket.status.key != 'needInfo':
                comment = (u"Неправильный формат тикета ((https://st.yandex-team.ru/createTicket?template=1177&queue=DIRECT (смотри пример))):\n"
                           u"Вывод скрипта:\n%%%%%s%%%%\n"
                           u"Проверить тикет после исправления можно командой: %%%%dt-deps-manager --check-open-deps --dependency %s%%%%\n"
                           u"%s") % (message, ticket.key, SIGN)
                trans = 'need_info'

            elif not message and ticket.status.key == 'needInfo':
                comment = u"Сейчас у тикета правильный формат, меняю статус на 'Открыт'\n%s" % SIGN
                trans = 'open'
            
            else:
                continue

            startrek_client.issues[ticket.key].comments.create(text=comment)
            for transition in startrek_client.issues[ticket.key].transitions.get_all():
                if transition.id == trans:
                    startrek_client.issues[ticket.key].transitions[trans].execute()
                    break


    if not result and not for_check_release:
        exit(2)

    return result


def release_deadlock():
    global startrek_client, opts

    tag = u'deps_deadlock_checked'
    issues = []

    if opts.tracker:
        issues = startrek_client.issues.find(u'Queue: DIRECT Type: Release Status: !closed Tags: !"%s" Status: !new "Sort by": key desc' % tag)
        if not issues:
            return

    dep_tickets = get_dependencies_tickets()
    graph = Graph().build_graph(dep_tickets)

    if not graph:
        return

    graph.get_release_clusters()
    rcycle = graph.has_rcycle()

    if rcycle:
        print u"Found deadlock between releases!\n%s" % str_rcycle(rcycle)

        if opts.tracker:
            if any(tag not in startrek_client.issues[release].tags for release in set([ver[i][2] for ver in rcycle for i in xrange(2)])):
                message = u"Найден дедлок между релизами!\n%s" % str_rcycle(rcycle, tracker=True)
                new_issue = startrek_client.issues.create(
                    queue='DIRECT',
                    summary=u'Дедлок релизов: %s' % u", ".join(set([ver[i][1] for ver in rcycle for i in xrange(2)])),
                    type={'name': 'Task'},
                    description=u"%s\nВизуализацию по релизам можно посмотреть ((https://observatorium.common.yandex.ru/release_deps/ticket_viz?releases=1 здесь))\n%s" % (message, SIGN),
                    tags=['release_deadlock'],
                )

                for release in set([ver[i][2] for ver in rcycle for i in xrange(2)]):
                    startrek_client.issues[release].comments.create(text=u"Найдена взаимная блокировка релизов, подробности: %s\n%s" % (new_issue.key, SIGN))

                print u"\nNew ticket created: %s" % new_issue.key
    else:
        print "No deadlocks found"

    if opts.tracker:
        for issue in issues:
            retry_call(
                startrek_client.issues[issue.key].update,
                fkwargs={'tags': startrek_client.issues[issue.key].tags + [tag]},
                tries=3,
                delay=1
            )

    if rcycle:
        exit(3)

    return


def check_rollback():
    global startrek_client, opts

    release_ticket = opts.check_rollback
    release_issue = startrek_client.issues[release_ticket]
    release_create_date = dateutil.parser.parse(release_issue.createdAt).strftime("%Y-%m-%d")
    release_app = get_release_app_name(release_issue)

    # либо тикет-зависимость еще открыт, либо он был закрыт после выкладки данного релиза (т.е. обновлен)
    dep_tickets = startrek_client.issues.find(u'Queue: DIRECT Components: "Зависимости между релизами" (Status: !Closed or Updated: >= %s) "Sort by": key desc' % release_create_date)

    graph = Graph().build_graph(dep_tickets)
    if not graph:
        die("Can't build graph, check dependency tickets format")

    if graph.build_message:
        print "!!!ATTENTION!!!\nSome errors were occured during graph building:%s\n\n" % graph.build_message

    r_tickets = get_tickets_from_release(release_ticket)

    result = []
    for r_ticket in r_tickets:
        result.extend(graph.check_rollback(r_ticket, release_app))
           
    if not result:
        print u"No problems found"
    else:
        print u"This release should not be rolled back!!\n%s" % u"\n".join([u"%s/%s --> %s/%s (dep ticket: %s)" % (el[0][0], el[0][1], el[1][0], el[1][1], el[2]) for el in result])
        exit(4)

    return


def statistics():
    global startrek_client, opts

    date_format = '%Y-%m-%d'
    if opts.stats_days_number:
        date_until = (
            datetime.datetime.strptime(opts.stats_date_from, date_format) +
            datetime.timedelta(days=opts.stats_days_number)
        ).strftime(date_format)
    else:
        date_until = datetime.datetime.today().strftime(date_format)

    dep_tickets = list(
        startrek_client.issues.find(u'Queue: DIRECT Components: %s Created: %s .. %s "Sort by": key desc' % (
            DEPS_COMPONENT, opts.stats_date_from, date_until
        ))
    )
    print "Found %d dependency tickets between %s and %s" % (len(dep_tickets), opts.stats_date_from, date_until)
    chunk_size_for_log = 100

    counts_vertices = defaultdict(int)
    counts_vertices_edges = defaultdict(lambda: defaultdict(int))
    counts_edges_apps = defaultdict(lambda: defaultdict(int))

    for idx, ticket in enumerate(dep_tickets, 1):
        if idx % chunk_size_for_log == 1:
            print "Processing tickets %d - %d ..." % (idx, min(idx + chunk_size_for_log - 1, len(dep_tickets)))
        graph = Graph().build_graph([ticket])

        vertices_count = len(graph.vertices())
        edges_count = len(graph.edges())
        counts_vertices[vertices_count] += 1
        counts_vertices_edges[vertices_count][edges_count] += 1

        for edge in graph.edges():
            counts_edges_apps[edge[0][1]][edge[1][1]] += 1

    print "-" * 40
    print "Tasks in dependency tickets:"
    print "\n".join("%d tasks: in %d tickets" % (key, value) for key, value in sorted(counts_vertices.items()))

    print "-" * 40
    print "Tasks with relations in dependency tickets:"
    print "\n\n".join(
        "\n".join(
            "%d tasks with %d dependencies: in %d tickets" % (v, e, counts_vertices_edges[v][e])
            for e in sorted(counts_vertices_edges[v])
        ) for v in sorted(counts_vertices_edges.keys())
    )

    rows_to_display = 10
    print "-" * 40
    print "Top %d dependencies between apps:" % rows_to_display
    print "\n".join("%s -> %s: %d" % (fr, to, value) for value, fr, to in sorted(
        [(counts_edges_apps[fr][to], fr, to) for fr in counts_edges_apps for to in counts_edges_apps[fr]],
        reverse=True)[:rows_to_display + 1]
    )
 

def die(message=''):
    sys.stderr.write("%s\n" % message)
    exit(1)


def parse_options():
    global opts

    parser = argparse.ArgumentParser(description="Работа с зависимостями между релизами")

    # команды
    cmd_group = parser.add_argument_group('Команды', 'Команды, доступные для запуска')
    cmd_group.add_argument("-cr", "--check-release", dest="check_release", help="проверить, удовлетворены ли зависимости к указанному релизу", type=str)
    cmd_group.add_argument("-cod", "--check-open-deps", dest="check_open_deps", help="проверить формат в открытых тикетах-зависимостях", action='store_true')
    cmd_group.add_argument("-sfd", "--search-for-deps", dest="search_for_deps", help="поискать среди недавно модифицированных тикетов похожие на зависимости", action='store_true')
    cmd_group.add_argument("-cl", "--close", dest="close", help="искать среди открытых зависимостей те, в которых уже все задачи выехали", action='store_true')
    cmd_group.add_argument("-n", "--notify", dest="notify", help="смотрит на релизы и пишет комментарий к тем, которые участвуют в зависимостях", action='store_true')
    cmd_group.add_argument("-rd", "--release-deadlock", dest="release_deadlock", help="проверить, есть ли дедлок между релизами", action='store_true')
    cmd_group.add_argument("-crb", "--check-rollback", dest="check_rollback", help="проверить, безопасно ли откатывать данный релиз", type=str)
    cmd_group.add_argument("-s", "--stats", dest="statistics", help="подсчитать статистику по зависимостям", action='store_true')

    # запрещаем одновременно выполнять разные типы визуализации
    vis_group = cmd_group.add_mutually_exclusive_group()
    vis_group.add_argument("-rv", "--release-visualize", dest="release_visualize", help="сгенерировать визуализации для открытых зависимостей c привязкой к релизам", action='store_true')
    vis_group.add_argument("-v", "--visualize", dest="visualize", help="сгенерировать визуализации для открытых зависимостей", action='store_true')

    # дополнительные агрументы
    args_group = parser.add_argument_group('Дополнительные аргументы', 'Используются вместе с командами')
    args_group.add_argument("-d", "--dependency", dest="dependency", help="название тикета-зависимости, который должен обрабатываться вместо всех открытых (можно использовать несколько раз для списка нужных тикетов)", action='append')
    args_group.add_argument("-t", "--tracker", dest="tracker", help="если выставлен, то отсылать результат в трекер", action='store_true')
    args_group.add_argument("-oi", "--one-image", dest="one_image", help="при параметре visualize или release-visualize создает одно общее изображение", action='store_true')
    args_group.add_argument("--dir", dest="dir", help="директория, где сохранить визуализации", type=str, default='/tmp/temp-ttl/ttl_7d')
    args_group.add_argument(
        "--stats-date-from",
        dest="stats_date_from",
        type=str,
        default="2018-06-27",
        help="(для статистики) дата в формате YYYY-MM-DD, после которой брать тикеты для статистики (по умолчанию: 2018-06-27)"
    )
    args_group.add_argument(
        "--stats-days-number",
        dest="stats_days_number",
        type=int,
        help="(для статистики) количество дней, за которое считать статистику (по умолчанию: до текущего дня)",
    )
    opts, extra = parser.parse_known_args()

    if len(extra) > 0:
        die("There are unknown parameters")

    return opts


cmds = {
    'check_release': check_release,
    'check_open_deps': check_open_deps,
    'close': close,
    'visualize': visualize,
    'search_for_deps': search_for_deps,
    'notify': notify,
    'release_deadlock': release_deadlock,
    'release_visualize': visualize,
    'check_rollback': check_rollback,
    'statistics': statistics,
}


def run():
    global opts, apps_dict, component2app

    opts = parse_options()
    apps_dict = read_apps_dict(APPS_CONF_FILE)
    component2app = {apps_dict[app]['tracker-component'] : app for app in apps_dict if 'tracker-component' in apps_dict[app]}
    
    for arg in vars(opts):
        if getattr(opts, arg) and arg in cmds:
            cmds[arg]()
            print "-" * 40


if __name__ == '__main__':
    run()
