#!/usr/bin/perl
#  $Id:$
# -----------------------------------------------------------------------------
#  Backup master configuration 
#
#  Note: master_backup.root script will be called from here with sudo
# -----------------------------------------------------------------------------
#  Author: Andrey Fomenko
#  Modified by: Alexey Tsibulnik
#  QA by:  
#  Copyright: videoNEXT Network solutions, Inc.
# -----------------------------------------------------------------------------

use strict;
use warnings;
use Getopt::Std;
use SKM::DB "DBLocal";
use XML::Simple;
use Socket;
use Master::Conf;
use Node::Conf "UNI";
use LWP::Simple qw($ua get getstore);
use Log::Log4perl "get_logger";

# -------------------- standard environment -----------------------------------
my $APL = $ENV{APL};
open (ECF,"$APL/base/etc/env.conf") || die("Cannot read base/etc/env.conf");
map {$ENV{$1}=$2 if /^(\w+)=(\S+)/} grep {/^\w+=\S+/} <ECF>;
close ECF;
#------------------------------------------------------------------------------
require "$APL/common/bin/logger.patrol";
my $procname = (split(/\//,$0))[-1];
my $APL_USR = $ENV{APL_USR} || 'apl';
my $APL_HTTPD_GRP = $ENV{APL_HTTPD_GRP} || 'apache';
my $PG_DUMP = $^O =~ /darwin/i ? "$APL/imp/bin/pg_dump" : "/usr/bin/pg_dump";
my $DBEXEC = "$APL/db/bin/db_exec";
my $SMLIST = "$APL/sm/bin/sm_list";
my $IP_ADDR= "$APL/base/bin/ip_addr_show";
my $ACC_CFG= "$APL/var/advantor_asn/asn.5.cfg";
my $ACC_ACT= "$APL/var/advantor_asn/ACTIVATED";
my $NODE_BACKUP = "$APL/conf/bin/node_backup";
my $VCTL = "$APL/vpatch/bin/vctl";
my $UNI = UNI;
my %Opts = ();
getopts('m:c:r:t', \%Opts);
my $bckRoot = "$APL/var/backup/master";
my $bckCache = "$APL/var/backup/node";
my $bckDir;
my $globalErr = 0;
my $Logger = get_logger('NEXTCAM::CONF::MASTERBACKUP');
my $nlist = NodeList;
my $tStart = time;
my $Mode = $Opts{m} ? lc($Opts{m}) : "user";
my $LogXML = {TIME => $tStart, TYPE => $Mode, TARGET => []};	# structured information about backup process
my $pid_name = "$APL/var/backup/master_backup.pid";
my $restore_pid_name = "$APL/var/backup/master_restore.pid";
my $dbl;

$SIG{INT}  = sub { die "Interrupted\n"  };
$SIG{TERM} = sub { die "Terminated\n" };

sub CheckMaster { # -----------------------------------------------------------
    return 1 if -f "$APL/var/conf/master/s_master";
    my $iaddr = gethostbyname('s_master');
    return undef unless $iaddr;
    my $s_master;
    my $ipaddr = inet_ntoa($iaddr);
    open(IF,"$IP_ADDR|") or $Logger->logdie('cannot check ifconfig');
    while(<IF>) {
	chomp;
	$s_master=$ipaddr, last if $ipaddr eq $_;
    }
    close IF;
    return $s_master?1:0;
} # sub CheckMaster

sub PrepareBackup { # ---------------------------------------------------------
	die "Invalid 'Mode' value specified: $Mode\n" unless $Mode =~ /auto|user|pre-restored/;
	die "Not a master\n" unless CheckMaster;
	$ua->timeout(5);
        $ua->agent('skm agent/2.5');
	$bckDir = GetReadableTime($tStart);
	mkdir("$bckRoot/$bckDir") or die "Error creating backup directory: $!\n";
	system("date > $bckRoot/$bckDir/backup.log") == 0 or die "Error storing backup date\n";
	# Write comment if any
	if (defined($Opts{c}) && open(COMMENT, ">$bckRoot/$bckDir/comment")) {
		print COMMENT $Opts{c};
		close COMMENT;
	}
	Log("Backup identification sequence: [$bckDir]");
        Log("Backup area: $bckRoot/$bckDir");
        Log("Backup mode: $Mode");
} # sub PrepareBackup

sub Log { # -------------------------------------------------------------------
	my $str = shift;
	if(defined $bckDir) {
	    open(LOG, ">>$bckRoot/$bckDir/backup.log");
	    print LOG $str."\n";
	    close LOG;
	}
	$Logger->info($str);
	print STDOUT ">>>> $str\n";
} # sub Log

sub RestoreCheck { # ----------------------------------------------------------
    # First check if recovery is in progress
    if (-f $restore_pid_name) {{
	last if -z $restore_pid_name;
	open(RPID, $restore_pid_name) or last;
	my ($pid) = <RPID> =~ /^(\d+)/;
	close RPID;
	chomp $pid if defined $pid;
	last unless $pid;
	kill(0,$pid) or last;
	# Check procname (Linux only)
	if ($^O =~ /linux/i) {
	    open PS, "ps -p $pid -o comm=|";
	    my $proc = <PS>;
	    close PS;
	    chomp $proc if defined $proc;
	    last if not defined $proc or $proc ne 'master_restore';
	}
	Log("Master configuration recovery is in progress. Application will be restarted in few seconds");
	exit 2;
    } unlink $restore_pid_name or Log("Cannot unlink $restore_pid_name: $!"); }
    # Then check if recovery was just finished and application is going to restart
    eval {
	$dbl = DBLocal({PrintError=>0,RaiseError => 1});
        my $arr = $dbl->selectall_arrayref("SELECT rtime FROM _objs WHERE name=? AND otype='D' AND subtype='N'", 
		undef, $UNI);
        if(not defined $arr->[0][0]) {
	    Log("Master configuration was just recovered. Application will be restarted in few seconds");
	    exit 2;
	}
    };
    Log("WARNING: Couldn't check master rtime: $@") if $@;
} # sub RestoreCheck

sub PidCheck { # --------------------------------------------------------------
    my $old_pid = PidRead();
    if($old_pid > 1) {
	open PS, "ps -p $old_pid -o comm=|";
	my $proc = <PS>;
	close PS;
	chomp $proc if defined $proc;
	if (defined $proc && $proc eq $procname) {
	    Log("Concurrent run of master_backup detected. Exiting");
    	    exit 1;
	} 
    }
    PidWrite() or exit 3;
} # sub PidCheck

sub PidRead { # ---------------------------------------------------------------
   my $pid = 0;
   if(open PID,"$pid_name") {
     $pid = <PID>;
     close PID;
     if (defined $pid) {
        chomp $pid;
     } else {
        $pid = 0;
     }
   } 
   return $pid;
} # sub PidRead

sub PidWrite { # --------------------------------------------------------------
   my $ok = 1;
   if(open PID,">$pid_name") {
     print PID "$$\n";
     close PID or do { 
        Log("Handle close failed. Cannot write PID file: $!");
        unlink $pid_name;
        $ok = 0;
     };
   } else {
      Log("Cannot create a PID file $pid_name: $!");
      $ok = 0;
   }
   return $ok;
} # sub PidWrite

sub PidRemove { # -------------------------------------------------------------
    unlink $pid_name if PidRead()==$$;
} # sub PidRemove

sub GetReadableTime {# --------------------------------------------------------
    my $time = shift;
    my ($tsec,$tmin,$thour,$tday,$tmon,$tyear)=(gmtime($time))[0..5];
    return sprintf("%02s%02s%02s_%02s%02s%02s",$tyear-100,$tmon+1,$tday,$thour,$tmin,$tsec);
} # sub GetReadableTime

sub SM_Cmd {
    my $cmd = shift;
    my $cmd_ok = 0;
    for (my $i = 1; $i <= 3; $i++) {
	eval {
	    $dbl->do("update SM_NODES set CMD='$cmd'");
	};
	if ($@) {
	    Log("WARNING: Storage Manager $cmd failed (try=$i)");
	    sleep 3;
	} else { 
	    $cmd_ok = 1; 
	    last; 
	}
    }
    Log($cmd_ok ? "Storage Manager $cmd : OK" : "Storage Manager $cmd : ERROR");
    $globalErr++ unless $cmd_ok;
}

sub NodeBackup { # ------------------------------------------------------------
	my $nodeID = shift;
	if(not exists $nlist->{$nodeID}) {
		Log("Can not backup node: $nodeID - node does not exist");
		#return (0, 0);
		return 0;
	};
	my $out = NodeCmd($nlist->{$nodeID}, "$NODE_BACKUP");
	if ($?) {
	    Log("WARNING: backup collecting failed for node '$nlist->{$nodeID}{IP}'");
	    return 0;
	}
	else {
	    Log("Backup node '$nodeID' : OK");
	    return 1;
	}
} # sub NodeBackup

sub BackupMasterDB { # --------------------------------------------------------
	my $target_index = push(@{$LogXML->{TARGET}}, {NAME=>"MasterDB"}) - 1;
	my ($status, $target_status) = ('OK', 'OK');
	mkdir "$bckRoot/$bckDir/db";
	
	# Backup confdb schema + data
	my $ok = 1;
	$ok &&= system("sudo $APL/db/bin/db_backup confdb $bckRoot/$bckDir/db/confdb.db &>/dev/null") == 0;
	
	# Backup apl chema
	$ok &&= system("sudo $APL/db/bin/db_backup apl $bckRoot/$bckDir/db/apl.db &>/dev/null") == 0;
	
	# Store db version
	my @vc = split(/\|/,`$DBEXEC -F '|' -Atc "select version,iteration from version_confdb" skm_master`);
	$ok &&= $? == 0;
	my @vt = split(/\|/,`$DBEXEC -F '|' -Atc "select version,iteration from version_transdb" skm_master`);
	$ok &&= $? == 0;
	$ok &&= open(FH, ">$bckRoot/$bckDir/db/version_confdb");
	print FH "VERSION=$vc[0]\nITERATION=$vc[1]\n";
	close FH;
	$ok &&= open(FH, ">$bckRoot/$bckDir/db/version_transdb");
	print FH "VERSION=$vt[0]\nITERATION=$vt[1]\n";
	close FH;
	
	if (!$ok) {
	    $status = $target_status = 'ERROR';
	    $globalErr++;
	}	
	
	Log("Backup confdb: $status");
	
	if ($Opts{t}) { # Backup TransDB
	    system("sudo $APL/db/bin/db_backup transdb $bckRoot/$bckDir/db/transdb.db &>/dev/null");
	    if ($?) {
		$status = $target_status = 'ERROR';
	        $globalErr++;
	    }
	    Log("Backup transdb: $status");
	}
	
	$LogXML->{TARGET}[$target_index]{STATUS} = $target_status;
	
} # sub BackupMasterDB


sub BackupConfMaster { # ------------------------------------------------------
	system("cp -r $APL/var/conf/master $bckRoot/$bckDir/ 2>&1");
	Log( "Backup MASTER conf-files: ".($?? "ERROR" : " OK"));
	$globalErr++ if $?;
	push(@{$LogXML->{TARGET}}, {NAME=>"MasterConf",STATUS=>$??'ERROR':'OK'});
} # sub BackupConfMaster

sub BackupMap2d { #------------------------------------------------------------
    system("cp -rp $APL/var/map2d $bckRoot/$bckDir/ 2>&1");
    Log( "Backup Map2d: ".($?? "ERROR" : " OK"));
    $globalErr++ if $?;
    push(@{$LogXML->{TARGET}}, {NAME=>"Map2d",STATUS=>$??'ERROR':'OK'});
} # sub BackupMap2d

sub BackupAdvantorCfg { #---------------------------------------------------------
    mkdir "$bckRoot/$bckDir/acc";
    system("cp -p $ACC_CFG $bckRoot/$bckDir/acc 2>&1");
    if ($? == 0) {
	Log( "Backup AdvantorCfg: OK");
	push(@{$LogXML->{TARGET}}, {NAME=>"AdvantorCfg",STATUS=>$??'ERROR':'OK'});
    }
    else {
	Log( "WARNING: Cannot copy ASN5 configuration file");
    }
} # sub BackupAdvantor

sub BackupNodes { #------------------------------------------------------------
    my $errNodes = 0;
    my $target_status = 'OK';
    mkdir "$bckRoot/$bckDir/domain";
    my $xml = {};
    my $target_index = push(@{$LogXML->{TARGET}}, {NAME=>"Domain"}) - 1;
    foreach my $node (keys %$nlist) {
	my ($res, $time) = NodeBackup($node);
	push(@{$LogXML->{TARGET}[$target_index]{NODE}}, 
	     {ID=>$node,STATUS=>$res?'OK':'ERROR',NAME=>$nlist->{$node}{FQDN},TIME=>$time?$time:'0'});
	unless ($res) { $errNodes++; $target_status = 'ERROR'; }
    }
    if($errNodes) { $globalErr++ }
    $LogXML->{TARGET}[$target_index]{STATUS} = $target_status;
    Log("NODES BACKUP RESULT: ".($errNodes? "$errNodes nodes FAILED" : 'OK'));
} # sub BackupNodes

sub BackupLicense { #---------------------------------------------------------- 	 
    mkdir "$bckRoot/$bckDir/license";
    if (-f "$APL/var/license/license.key") {
	system("cp -p $APL/var/license/* $bckRoot/$bckDir/license 2>&1");
    }
    else {
	Log("No license file in the system");
    }
    Log( "Backup License: ".($?? "ERROR" : " OK"));
    push(@{$LogXML->{TARGET}}, {NAME=>"License",STATUS=>$??'ERROR':'OK'});
} # sub BackupLicense

sub GetVCTL { # ---------------------------------------------------------------
    eval {
	my $info = `$VCTL info 2>/dev/null`;
	die "Error getting VCTL information (exit code:".($?>>8).")\n" if $?;
	open(FH, ">$bckRoot/$bckDir/verinfo") or die "Cannot write verinfo: $!\n";
	print FH $info;
	close FH;
    };
    if ($@) {
	my $errmsg = $@; chomp $errmsg;
	Log($errmsg);
	die "$errmsg\n";
    }
    else {
	Log("VCTL information retrieved : OK");
    }
} # sub GetVCTL 

sub ReplicateBackup {
    if (open(LIST, "$SMLIST 2>/dev/null|")) {
        my @list = grep {$_ ne '/vasm/store'} map {(split(/\s+/,$_))[1]} <LIST>;
        close LIST;
        
        if (@list) {
	    my $lst = join(' ', map {"'$_'"} @list);
    	    # Prepare location for backups
	    `sudo $APL/conf/bin/config_auto_backup.root -b $lst &>/dev/null`;
	    foreach my $mnt (@list) {
    		my $loc = "$mnt/backup/master";
    	        next unless -d $loc;
    	        unless( -d "$loc/$bckDir") {;
    	    	    system("cp -pr $bckRoot/$bckDir $loc");
    		    Log("[Replicate] Successfully replicated backup to $loc");
    		}
    	    }
        }
    }
    else {
	Log("[Replicate] Cannot list volumes: $!");
    }
}

sub RemoveBackup {
    my $bck = $Opts{r};
    if ($bck =~ /^\d{6}_\d{6}$/ && -d "$bckRoot/$bck") {
	return system("rm -rf $bckRoot/$bck") != 0;
    }
    return 1;
}

sub Finalize {
    $LogXML->{STATUS} = $globalErr ? 'ERROR' : 'OK';
    $LogXML->{ERRNUM} = $globalErr;
    $LogXML->{UNI} = $UNI;
    # Unfreeze SM
    SM_Cmd('UNFREEZE');
    # Write stat
    open(OUT, ">$bckRoot/$bckDir/stat.xml");
    print OUT XMLout($LogXML, RootName => 'BACKUP', XMLDecl => 1);
    close OUT;
    # Write origin
    open(OUT, ">$bckRoot/$bckDir/origin");
    print OUT "origin=native\n";
    close OUT;
    system("chown -R $APL_USR:$APL_HTTPD_GRP $bckRoot/$bckDir");
    if($?) {
	Log("Unable to set ownership and permissions on backup directory");
	$globalErr++;
    }
    # Trt to replicate backup to all available Storage Volumes
    ReplicateBackup;
    Log("FINAL: ".($globalErr?"ERROR[s]: $globalErr" : 'OK'));
    exit $globalErr;
}

###############################################################################
############# MAIN ############################################################
###############################################################################

sub main {
    if ($Opts{r}) {
	exit RemoveBackup;
    }
    PidCheck;
    RestoreCheck;
    eval {
	PrepareBackup;	
	GetVCTL;
    };
    if($@) {	# Critical error occured - exit immediately
	warn $@;
	system("rm -rf $bckRoot/$bckDir 1>/dev/null 2>&1") if defined $bckDir;
	exit 1;
    }
    SM_Cmd('FREEZE');
    #BackupLicense;
    BackupNodes;
    BackupMasterDB;
    #BackupConfMaster;
    #BackupMap2d;
    BackupAdvantorCfg if -f $ACC_ACT;
    Finalize;
}

main();

### END ########################################################################

END {
    PidRemove();
    eval { $dbl->disconnect } if $dbl;
}
