#!/bin/bash 

#TODO: full unattended repair (detach from terminal, stdit/err to logfile)
#TODO: repair all "bad" instances by default (if replication failed/instance down, but standby is alive and synced with master)

set -x

usage()
{
cat <<EOF
usage: $0 database_name [repair|...(see options)] [master]

This script makes repair database easier.

OPTIONS:
    - repair    repair database from backup (default)
    - copy      repair database and copy configs 
    - xtra      repair database using percona-xtrabackup from standby host
    - pxc       check cluster info file
    - show      view backup and standby host for database

Example:
  repairdb.sh ppcdata4 repair 
  repairdb.sh ppcdata4 copy [ppcstandby07i.yandex.ru]  
  repairdb.sh ppcdata4 xtra [ppcstandby07i.yandex.ru] 
  repairdb.sh ppcdata4 show 

EOF
}

if [ -z $1 ];
    then 
        usage
        exit 1;
fi

db=$1
operation=${2:-repair}
logfile='/var/log/repairdb.log'
localdb=$(ls /etc/init.d | grep mysql. | sed s/mysql.//g)
datadir="/opt/mysql.""$db"
#if [ -e "/etc/mysql/$db.cnf" ]; 
#        then datadir="/"$(cat /etc/mysql/"$db".cnf | grep datadir | cut -d '/' -f2-);
#fi
alldbfile='/etc/mysql-create-backup.conf.d/*.conf'
hostname=$(hostname -f)
group=$(curl -s c.yandex-team.ru/api/hosts/$hostname | grep group | sed s/\ \*\<[\/]*group\>//g | sed s/_nonstandby//)
backupsh='ppcbackup04i.yandex.ru::mysqlbackup-rw'

function log() {

  echo "`date +%s`:`date` $0: $@"

}

function checkdbstatus() {

    status=$(lm $db status | awk '$1 ~ /Slave/ { print $3 }' )
    if [[ "$status" =~ 'Yes:Yes' ]]; 
        then echo 'Base is OK'
             if [ "$operation" != "show" ]; then exit 1; fi
        else echo 'Base is corrupted';
    fi

}

function getbackuphosts() {

    hostsingroup=$(curl -s c.yandex-team.ru/api/groups2hosts/$group)
    standbyhost=$(echo "$hostsingroup" | grep standby)
    while [ -z $standbyhost ]; do
            echo "Standby host not found, type manually:"
            read
            standbyhost=$REPLY;
    done
    getbackupsh=$(ssh $standbyhost "cat $alldbfile" | grep -w -A 4 $db | grep -A 3 backup_dest | cut -d '"' -f 2 -s | cut -d '-' -f 1 | sed s/fastbone.//g | sort -u);
    if [ ! -z $getbackupsh ]; then 
            backupsh=$getbackupsh;
    else
            echo "Backup share not found, type manually:"
            read
            backupsh=$REPLY;
    fi
    backuphost=$(echo $backupsh | cut -d ':' -f1)
    
}

function stopbase() {

    lm $db server-stop --force
    sleep 10
    log "Moving current base to tmp folder"
    rm -r $datadir || log "Directory not exist"
    
}

function copybase() {

    log "Create new base directory"
    if [ ! -d $datadir ]; 
        then
            mkdir $datadir && log "Directory created" || log "Can't create directory $datadir";
    fi
    log "Start rsync base $db from host $1"
    rsync -av --progress --delete --whole-file --exclude='*-bin*' --exclude='*-relay*' --exclude='auto.cnf' --sockopts=SO_RCVBUF=2000000,SO_SNDBUF=2000000 $1/$db/0000/ $datadir
    if [[ $? -eq 0 ]]; 
        then 
            log "Rsync ended."
            chown mysql: -R $datadir;
        else 
            log "Rsync fail"
            exit 1;
    fi
    rm $datadir/.rsnap_prot0
}

function startbase() {
    real_datadir="$datadir"
    if [ -e "/etc/mysql/$db.cnf" ]; then
        real_datadir=$(grep datadir /etc/mysql/"$db".cnf | grep "$db" | cut -d= -f2 | perl -lpe 's/^\s+|\s+$//g')
    fi

    log "Generating auto.cnf file"
    perl -e 'chomp($host = qx(hostname -f)); @md5 = split //, qx(echo "$host" | md5sum); printf "[auto]\nserver-uuid=%s\n", join "-", map { join("", @md5[@{$_}]) } ([0..7], [8..11], [12..15], [16..19], [20..31])' > $real_datadir/auto.cnf
    chown mysql: $real_datadir/auto.cnf

    posfile=$(ls -1 "$datadir"/$db*_mysql.pos | tail -n 1)
    gtid_exec=$(perl -le '$/ = undef; $_ = <>; ($gs) = (/Master:Executed_Gtid_Set=([\S\n]+)\nSlave/); $gs =~ s/\n//g; print $gs' < "$posfile")

    log "Try to start server"
    lm $db server-start --force --skip-slave-start
    if [[ $? -eq 0 ]]; then 
        if [ -n "$gtid_exec" ]; then
            echo "reset master; set global gtid_purged='$gtid_exec'" | lm $db
            lm $db status-gtid
            sleep 10 
        fi
        log "Start slave"
        lm $db slave-start;
    fi
}

function copyetc() {

    scp $1:/etc/init.d/mysql.$db /etc/init.d/
    scp $1:/etc/mysql/$db.cnf /etc/mysql/
    if [ ! -d /etc/mysql/$db.conf.d ]; 
        then
            mkdir /etc/mysql/$db.conf.d;
    fi
    rsync -av $1":/etc/mysql/"$db".conf.d/" /etc/mysql/$db".conf.d/"   
    
}

function xtrabackup() {

##    apt-cache policy percona-xtrabackup | grep Installed | tr -d \  | cut -d: -f2
    
    lversion=$(dpkg -l | awk '($1 ~ "ii" && $2 ~ "percona-xtrabackup") {print $3}' | tr -d [a-z])
    rversion=$(ssh "$1" "dpkg -l | awk '(\$1 ~ \"ii\" && \$2 ~ \"percona-xtrabackup\") {print \$3}'" | tr -d [a-z])
    datadir=$(grep datadir /etc/mysql/$db.cnf | rev | cut -d ' ' -f 1 | rev)
    tmpdir="/tmp"
    mkdir $datadir

    if [ "$lversion" = "$rversion" ]; 
        then
            log "Start innobackupex on $1"
            ssh $1 screen -d -m bash -c \'"scp $datadir/master.info $hostname:/tmp/$1_master.info"\;sleep 30\;"innobackupex --defaults-file=/etc/mysql/$db.cnf --user=root --stream=xbstream --compress --compress-threads=16 --parallel=2 --socket=/var/run/mysqld.$db/mysqld.sock $tmpdir/ | pv -L 40m | nc -6 $hostname 9999"\'
            cd $datadir
            rm -rf *
            apt-get install qpress
            log "Start listen data on $hostname"
            nc -6 -l 9999 | xbstream -x -v
            log "Start innobackupex decompress"
            innobackupex --decompress $datadir
            log "Start innobackupex apply-log"
            innobackupex --apply-log --use-memory=40G $datadir
            tmpmasterinfo="/tmp/$1_master.info"
            masterinfo="$datadir"/master.info
            masterhost=sed -n 4p $tmpmasterinfo
            rplcaname=sed -n 5p $tmpmasterinfo
            rplcapass=sed -n 6p $tmpmasterinfo
            sed -i "4s/.*/$masterhost/" $masterinfo
            sed -i "5s/.*/$rplcaname/" $masterinfo
            sed -i "6s/.*/$rplcapass/" $masterinfo
            log "Restore backup ended";
	        chown mysql: -R $datadir
        else
            log "Different version of percona-xtrabackup on $hostname and $1";
    fi

}

function pxc() {

    if [ ! -e '/etc/mysql/'$db'.conf.d/wsrep.cnf' ];
        then
            log "Cannot find cluster info: /etc/mysql/'$db'.conf.d/wsrep.cnf not exist";
        else 
            lm $db server-start --force
            log "Start base \n View process log on /var/log/mysql.$db/mysql.err" 
            echo "Check wsrep_cluster_address /etc/mysql/ppcdict.conf.d/wsrep.cnf on hosts in group $group";
    fi

}

function checkfreespace() {
    if [ "$operation" = "copy" ];
        then
            needspace=$(ssh $backuphost "du -c /\$(cat /etc/rsyncd.conf | grep path.*mysql |cut -d / -f2- | sort -u)/$db/0000 | grep total | cut -f1");
        elif [ ! -z $standbyhost ];
            then
                echo "$datadir"
                needspace=$(ssh $standbyhost 'du -c $datadir | grep total | cut -f1');
            else echo 'No standby host defined'
    fi
    freespace=$(df $datadir | tail -n+2 | awk '{print $4}')
    if [[ $needspace -gt $freespace ]];
                then
                    log "No needed free space on $datadir"
                    exit 1;
            fi
}

if checkdbstatus; then
    getbackuphosts
    if [ ! -z $3 ]; then standbyhost="$3"; fi
    if [ "$standbyhost" = "$hostname" ] && [ "$2" = 'xtra' ]; 
        then 
            echo -e "This machine is standby_host \nUse other host from group $group: \n **** \n$hostsingroup \n **** \nplease type another host: \n"
            read standbyhost;
    fi
    checkfreespace
    case $operation in
        repair)
            stopbase 
            copybase $backupsh
            startbase $db
        ;;
        copy)
            copybase $backupsh
            copyetc $standbyhost 
            startbase $db   
        ;;
        xtra)
            stopbase
            xtrabackup $standbyhost
#	        copyetc $standbyhost
            startbase $db 
        ;;
        pxc)
           stopbase
           pxc 
        ;;
        show)
            echo "Standby host is $standbyhost for $db"
            echo "Backup host is $backuphost for $db"
    esac
fi
