#!/usr/bin/perl
#
#  $Id$
# -----------------------------------------------------------------------------
#  SYSTEM_STATUS - checks master components/nodes/storage and calculate
#                  a combine status for the system
# -----------------------------------------------------------------------------
#  Author: Alex Titov
#  QA by:
#  Copyright: videoNEXT NS
# -----------------------------------------------------------------------------
#
# for REQ see: Rally S990: System Status
# notes:
#   1. command line tool
#   2. start as daemon from conf_setup
#
# usage:
#  system_status
#  system_status details
#  system_status daemon
#

use strict;
use SKM::DB;
use Data::Dumper;
use IO::Socket::INET;
use JSON;
use Encode;
use NextCAM::Conf "GetCfgs";
use SKM::Common "ArrayTable";
use Master::Conf;
use Node::Conf ":all";

# CONS
my $APL = $ENV{APL};
my $APL_VAR=$ENV{APL_VAR};
my $STATUSFILE     ="$APL_VAR/wd/status";
my $STATUS         ="$APL/var/sdi/system_status";
my $STATUS_JSON    ="$APL/var/sdi/system_status.json";
my $INSTALL_RESULT ="$APL_VAR/vctl/result";
my $EVENT_INTEGRITY="$APL/var/elog/event_integrity";
my $STATUS_LOG     ="$APL/var/log/system_status.log";
my $SMXCTL         ="sudo $APL/smix/bin/smxctl";
my $AV_STATUS      ="$APL/av/bin/avatar_status";
my $LOADAVG_ALERT  = 4;
my $LOG_LIMIT      =4*1024*1024;
my $UNI=UNI;
my $SLEEP          =30;


# VARS -------------------------------------------------------------------
my $step = 0;
my $dbh;
my $system_status='undefined';
my $alert_notes;
my $critical_notes;
my $silent=($ARGV[0] eq 'details')?0:1;
my $nlist;
my $last_system_status='none';
my $last_notes='none';
my %dbs;				# database statements
my $need_update=0;
my %DBVer = ();
my %DBVerBld = %{ &DB_Version };

sub announce
{
	my $section = shift;
	$step++;
	return if $silent;
	print "\n" if $step > 1;
	print '=' x 5 . " ${step}. ";
	print "$section ";
	print '=' x (20 - length($section));
	print "\n";
}

sub notice {
    return if $silent;
    print "@_\n";
}

sub db_master
{
	eval { $dbh->disconnect if $dbh };
	$dbh = DBMaster({ PrintError=>0, RaiseError=>1 });
    	$dbh->{FetchHashKeyName} = 'NAME_uc';
        $dbs{INSERT}=$dbh->prepare('insert into audit(category,parameters) values(32,?)');
}

sub read_ver {
    %DBVer = ();
    # Read Master DB version
    eval {
	db_master unless $dbh;
	for my $db ('confdb', 'transdb') {
	    my $arr = $dbh->selectrow_arrayref("select version,iteration from version_$db");
	    $DBVer{$db}{ver}  = $arr->[0];
	    $DBVer{$db}{iter} = $arr->[1];
	}
    };

    for my $db ('confdb', 'transdb') {
	my $vercmp = $DBVerBld{$db}{ver} cmp $DBVer{$db}{ver};
	my $itercmp = $DBVerBld{$db}{iter} <=> $DBVer{$db}{iter};
	last if $vercmp == 0 and $itercmp == 0 or $vercmp > 0;
	if ($vercmp < 0)  { $need_update = 1; last; }
	if ($itercmp < 0) { $need_update = 1; last; }
    }
}

sub summary {
    my $notes=$alert_notes;
    $notes=$critical_notes if $critical_notes;# critical overrides !
    print "===========SUMMARY============\n$system_status\n$notes" if ! $silent;
    #-----------------TEST
    open(STATUS,">$STATUS");
     binmode(STATUS, ":utf8");
     print STATUS "$system_status\n$notes";
    close STATUS;
    #-----------------LOG
    if(-f $STATUS_LOG and -s $STATUS_LOG > $LOG_LIMIT) { # rotate the log
      unlink "$STATUS_LOG.1" if -f "$STATUS_LOG.1";
      rename $STATUS_LOG,"$STATUS_LOG.1";
    }
    open(STATUS_LOG,">>$STATUS_LOG");
     binmode(STATUS_LOG, ":utf8");
     print STATUS_LOG scalar(localtime)."\n$system_status\n$notes";
    close STATUS_LOG;
    #-----------------JSON
    my @reasons=split(/\n/,$notes);
    @reasons=('') if not @reasons;     # empty reason to suspend NULL
    my %summary=(status=>$system_status,reasons=>\@reasons);
    my $jsons = encode_json \%summary;
    if(open   STATUS_JSON,">$STATUS_JSON.tmp") {
       print  STATUS_JSON "$jsons\n";
       close  STATUS_JSON;
       rename"$STATUS_JSON.tmp",$STATUS_JSON;
    }
    #--------------- AUDIT
    if($system_status ne $last_system_status or $notes ne $last_notes) { # only audit the change
      if($dbh) {       # database works
        eval {
          $_=$notes; chomp;
          s/\n/|/g; s/"/'/g; # make it readable for one string"
          $dbs{INSERT}->execute(qq([["$system_status","$_"]]));
        }
      }
    }
    $last_system_status=$system_status;
    $last_notes=$notes;
    #--------------- CLEAN
    $system_status='undefined';
    $alert_notes='';
    $critical_notes='';
    $dbh->disconnect if $dbh;
    $dbh='';
}

sub master_status #TBD: check status directly from /var/opt/sarch/wd/status
{
	announce 'Master status';

	my $status = 'unknown';
        if(open STATUS,$STATUSFILE) {
		$status=<STATUS>;
  		chomp $status;
		close STATUS;
	}
	if ($status eq 'run') {
                $system_status='ONLINE';
		notice "ONLINE";
	}elsif($status eq 'stop') {
                $system_status='OFFLINE';
                notice "OFFLINE";
	}else {
                $system_status=uc($status);
		notice uc($status);
	}
}

#-------------------------------------------------
# check installation result for master
# set CRITICAL is installation result != SUCCESS
#-------------------------------------------------
sub install_result {
    announce 'Installation result';
    my ($iresult,$uresult);
    if(open(RESULT, $INSTALL_RESULT)) {
      my %result=map{/(^\w+)=(.+)/} grep {/^\w+=.+/} <RESULT>;
      $iresult=$result{ACTIVATE};
      $uresult=$result{UPDATE};
      close RESULT;
    }
    if(!$iresult) {
      $system_status='CRITICAL';
      my $note="Software installation status: 'UNKNOWN'";
      $critical_notes.="$note\n";
      notice $note;
    } elsif($iresult ne 'SUCCESS') {
      $system_status='CRITICAL';
      my $note="Software installation status: '$iresult'";
      $critical_notes.="$note\n";
      notice $note;
    } else {
      notice 'SUCCESS';
    }
    announce 'Update-result';
    if(!$uresult) {
      $system_status='CRITICAL';
      my $note="Software-update status: 'UNKNOWN'";
      $critical_notes.="$note\n";
      notice $note;
    } elsif($uresult ne 'SUCCESS') {
      $system_status='CRITICAL';
      my $note="Software-update status: '$uresult'";
      $critical_notes.="$note\n";
      notice $note;
    }else {
      notice 'SUCCESS';
    }
}


sub db_status
{
	announce 'DB status';
        my $ret;

	# Try to connect to db
	eval {
		db_master unless $dbh;
		$dbh->selectall_arrayref("SELECT COUNT(1) FROM _objs");
	};
        if($@) {
          $ret='DOWN';
          notice "DOWN";
          $system_status='BROKEN';
          $critical_notes.="Database is DOWN\n";
        }else {
          $ret='ONLINE';
          notice "ONLINE";
        }

        read_ver;
        if($need_update) {
	  $alert_notes.="System database version is higher than software version. ".
    			"Previous version configuration backup recovery is recommended\n";
    	  $system_status='ALERT' if $system_status ne 'CRITICAL' and $system_status ne 'BROKEN';
    	}
  return $ret;
}

sub events_count
{
	announce 'Events';

	my $cnt;
	eval {
		db_master unless $dbh;
		my $arr = $dbh->selectrow_arrayref(
		    "SELECT COUNT(eventid) FROM event"
		);
		$cnt = $arr->[0];
	} if $dbh;
	notice  "Total: ".(defined $cnt ? $cnt : 'UNKNOWN');
}

sub node_status
{
	announce 'Node status';
        $nlist = NodeList; # preload Node list
	my @nodes = ();
        my $verid = $nlist->{$UNI}->{VERID};   # own VERID
	foreach my $uni (keys %$nlist) {
	    my $node = $nlist->{$uni};
	    my $status = $node->{ALIVE} ? 'ALIVE' :
		$node->{DEAD} ? 'DEAD' :
		'UNKNOWN';
	    push @nodes, [$node->{IP}, $node->{VERID}, $status];# TBD: install_result
            if(not $node->{ALIVE}) {
              $critical_notes.="Node $node->{HOST} is OFFLINE\n";
              $system_status='CRITICAL';
            }
            if($node->{INSTALL_RESULT} ne 'SUCCESS') {
              if($uni!=$UNI) {            # do not report master second time
                 $system_status='CRITICAL';
                 $critical_notes.="Installation status for node $node->{HOST} is FAIL\n";
              }
            }
            if($node->{VERID} ne $verid) {
              $alert_notes.="Node $node->{HOST} has mismatch software $node->{VERID}\n";
              $system_status='ALERT' if $system_status ne 'CRITICAL';
            }
	}

	# Check node statistics
	my $ra;
	eval {
		db_master unless $dbh;
		$ra = $dbh->selectall_arrayref(
		    "select o.name as uni, a.val as loadavg from
		    _objs o inner join _obj_attr a on o.obj=a.obj
		    where otype='D' and subtype='N' and deleted=0
		    and a.attr='STAT_LOAD_AVERAGE_CORE'"
		);
	};
	warn("ERROR: Failed to read node statistics: $@") if $@;
	if ($ra and @$ra) {
	    foreach my $row (@$ra) {
		my $node = $nlist->{ $row->[0] };
		next unless $node;
		my @loadavg = split(/\s/, $row->[1]);
		# Look at 5-min load average
		my $avg5 = $loadavg[1];
		if ($avg5 >= $LOADAVG_ALERT) {
		    $alert_notes.="CPU overload is detected on node $node->{HOST}\n";
		    $system_status='ALERT' if $system_status ne 'CRITICAL';
		}
	    }
	}

	ArrayTable(['Node IP', 'SKM Version', 'Status'], \@nodes) if not $silent;
}

sub volume_status # keep as is for screen information only
{
	announce 'Volume status';

	my $ra;
	eval {
		db_master unless $dbh;
		$ra = $dbh->selectall_arrayref(
		    "select sw.id,sw.name,oa.val,so.ost from sm_wheels sw inner join sm_ost so on sw.id=so.id
		    inner join _objs o on sw.nodeid=o.name
		    inner join _obj_attr oa on o.obj=oa.obj
		    where o.otype='D' and o.subtype='N' and o.deleted=0 and oa.attr='IP'", {Slice=>[]}
		);
	};
	warn("ERROR: Failed to read volume information: $@") if $@;
        if(not $silent) {
      	   ArrayTable(['Volume ID', 'Volume name', 'Node IP', 'Volume Status'], $ra) if $ra;
        }
}


sub sm_status
{
   announce 'Storage Status';
   my $ra;
   my $note;
   eval {
      db_master unless $dbh;
      $ra=$dbh->selectall_hashref("select w.id,w.nodeid,w.name,o.ost,c.cst from sm_wheels w, sm_ost o,sm_cst c where o.id=w.id and c.id=o.id order by 2,1",
             'ID',{Slice=>[]});
   };
   #--------------------------------- Check storage for cst!=ost
   foreach my $id (keys %$ra) {
      my $uni=$ra->{$id}->{NODEID};
      if( $ra->{$id}->{CST} ne uc($ra->{$id}->{OST}) and $ra->{$id}->{OST} ne 'Full') {
         $system_status='ALERT' if $system_status ne 'CRITICAL';
         my $node=' on node '. $nlist->{$uni}->{HOST};
         $note = "Volume $ra->{$id}->{NAME} $node has status $ra->{$id}->{OST} (expected $ra->{$id}->{CST})";
         $alert_notes.= "$note\n";
         notice $note;
      }
      $nlist->{$uni}->{ONLINE_WHEELS}++ if $ra->{$id}->{OST}=~/(Online|Degraded)/; # Degraded also 'Online'
   }
   #--------------------------------- check nodes for ONLINE storage presents
   foreach my $uni (keys %$nlist) {
     next if $nlist->{$uni}->{DEAD};       # skip dead nodes
     if(not exists $nlist->{$uni}->{ONLINE_WHEELS} or $nlist->{$uni}->{ONLINE_WHEELS}==0) {
        $system_status='CRITICAL';
        $note = "Node $nlist->{$uni}->{HOST} does not have any ONLINE storage volume";
        $critical_notes.="$note\n";
        notice $note;
     }
   }

   notice "OK" unless $note;
}

sub cache_status
{
   announce 'Cache Status';
   my $cache;
   my $note;
   eval {
      db_master unless $dbh;
      $cache=$dbh->selectall_hashref(
          "select nodeid,size,usage,stime,chunk_loss_ts,"
         ."EXTRACT(EPOCH from now() at time zone 'UTC' - chunk_loss_ts) as sinceloss from sm_cache",
        'NODEID',{Slice=>{}}
      );
   };
   foreach my $uni (keys %$nlist) {
     my $node=$nlist->{$uni};
     next if $node->{DEAD};       # skip the dead
     # check if record is present in cache and set CRITICAL | ALERT with corresponded explanation
     if (not exists $cache->{$uni}) {
        $system_status='ALERT' if  $system_status ne 'CRITICAL';
        $note = "Cache status for node $node->{HOST} is UNKNOWN";
        $alert_notes.="$note\n";
        notice $note;
        next;
     }
     my $ncache=$cache->{$uni};
     if ($ncache->{SIZE} == 0) { # Cache is turned off
        $system_status='CRITICAL';
        $note = "Cache status for node $node->{HOST} is OFFLINE";
        $critical_notes.="$note\n";
        notice $note;
        next;
     }
     #------------------------------------ Analyze cache Usage
     my $pct = int($ncache->{USAGE} / $ncache->{SIZE}) * 100;
     if ($pct > 90) {
        $system_status='CRITICAL';
        $note = "Cache status for node $node->{HOST} is FULL (${pct}%)";
        $critical_notes.="$note\n";
        notice $note;
     }
     elsif ($pct > 75) {
        $system_status='ALERT';
        $note = "Cache status for node $node->{HOST} is DEGRADED (${pct}%)";
        $alert_notes.= "$note\n";
        notice $note;
     }
     #------------------------------------ Analyze chunk loss
     next unless defined $ncache->{CHUNK_LOSS_TS};
     my $sinceloss=int($ncache->{SINCELOSS});
     next if not defined $sinceloss or $sinceloss < 0;
     $sinceloss = int($sinceloss/60) || 1; # In minutes
     if ($sinceloss < 30) {
        $note="Video Archive loss detected $sinceloss minutes ago\n";
        if($sinceloss<5) {
          $system_status='CRITICAL';
          $critical_notes.=$note;
        }else {
          $system_status='ALERT' if  $system_status ne 'CRITICAL';
          $alert_notes.=$note;
        }
        notice $note;
     }
   }

   notice "OK" unless $note;
}


sub event_integrity {
   announce 'Event Integrity';
   my $ei='UNKNOWN';
   if(open EI, $EVENT_INTEGRITY) {
      my %result=map{/(^\w+)=(.+)/} grep {/^\w+=.+/} <EI>;
      $ei=$result{EVENT_INTEGRITY};
      close EI;
   }
   if($ei ne 'CONSISTENT') {
     $system_status='ALERT' if $system_status ne 'CRITICAL';
     $alert_notes.= "Event Database has '$ei' status\n";
   }
   notice $ei;
}


sub device_status
{
	announce 'Device status';

	my ($cams_total, $sens_total, $rel_total) = (0, 0, 0);
	my $raCams;
	eval {
		db_master unless $dbh;
		$raCams = $dbh->selectall_arrayref(
		    "select oa.val, count(o.obj)
		    from _objs o inner join _obj_attr oa on o.obj=oa.obj
		    where o.otype='D' and o.subtype='C'
			and o.deleted=0 and oa.attr='STATUS'
		    group by oa.val",
		    { Slice => [] }
		);
		$cams_total += $_->[1] foreach (@$raCams);

		my $raSens = $dbh->selectrow_arrayref(
		    "select count(*) from _objs where otype='D' and subtype='S' and deleted=0",
		    { Slice => [] }
		);
		my $raRel = $dbh->selectrow_arrayref(
		    "select count(*) from _objs where otype='D' and subtype='R' and deleted=0",
		    { Slice => [] }
		);
		$sens_total += $raSens->[0];
		$rel_total += $raRel->[0];
	};
	warn("ERROR: Failed to read device information: $@") if $@;
	notice "Cameras: $cams_total total";
	if (!$silent) {
        	ArrayTable(['Camera status', 'Count'], $raCams) if $raCams;
        }
	notice "Sensors: $sens_total total\nRelays: $rel_total total";
}


sub camera_status {
   announce 'Cameras status';

   my $ra;
   eval {
	db_master unless $dbh;
	$ra = $dbh->selectall_hashref(
 	   "select o.obj,o.name,o.node_ip as uni,a1.val as state, a2.val as status "
	  ."from _objs o, _obj_attr a1, _obj_attr a2 "
	  ." where otype='D' and subtype='C' and  o.obj=a1.obj and a1.attr='ARCHSTATE'"
	  ."  and  o.obj=a2.obj and a2.attr='STATUS' and deleted=0",'OBJ',{Slice=>[]}
        );
   };
   if($@) {
     $system_status='CRITICAL';
     $critical_notes="Couldn't read configuration database\n"; # should never happen
   }
   my $broken=0;
   foreach my $obj( keys %$ra) {
     $broken++ if $ra->{$obj}->{STATUS}=~/^(BROKEN|DOWN|RESET\-FAILURE)$/;
   }
   if(! $broken) {
     notice 'OK';
   }elsif($broken<4) { # show all broken cameras
     $system_status='ALERT' if $system_status ne 'CRITICAL';
     foreach  my $obj( keys %$ra) {
        my $dev=$ra->{$obj};
        next unless $dev->{STATUS}=~/^(BROKEN|DOWN|RESET\-FAILURE)$/;
        my $note=Encode::encode("utf8", "Camera #$obj ").
    		    $dev->{NAME}.
    		    Encode::encode("utf8", " is BROKEN");
    	$note = Encode::decode("utf8", $note);
        notice $note;
        $alert_notes.="$note\n";
     }
   }else {
     $system_status='ALERT' if $system_status ne 'CRITICAL';
     my $note="System has $broken broken cameras";
     notice $note;
     $alert_notes.="$note\n";
   }
}


sub analytics_status {
   announce 'Analytics status';

   my $ra;
   eval {
	db_master unless $dbh;
	$ra = $dbh->selectall_hashref(
 	   "select o.obj,o.name,o.node_ip as uni,a1.val as state, a2.val as vae, a3.val as vae_stat"
	  ." from _objs o, _obj_attr a1, _obj_attr a2, _obj_attr a3 "
	  ." where otype='D' and subtype='C' and deleted=0 and  o.obj=a1.obj and a1.attr='ARCHSTATE'"
	  ."  and  o.obj=a2.obj and a2.attr='VAE_ACTIVE' and o.obj=a3.obj and a3.attr='STAT_VAE'",'OBJ',{Slice=>[]}
        );
   };
   if($@) {
     $system_status='CRITICAL';
     $critical_notes="Couldn't read configuration database\n"; # should never happen
   }

   foreach my $obj (keys %$ra) {
     next unless $ra->{$obj}{STATE} eq 'on' and $ra->{$obj}{VAE} eq 'yes';

     my $stat = $ra->{$obj}{VAE_STAT};
     next unless $stat;
     $stat = eval { decode_json $stat };
     next if $@ or ref($stat) ne 'HASH';
     foreach my $name (keys %$stat) {
        my $trk = $stat->{$name};
        my ($state,$act,$min,$max,$cur)=($trk->{state},$trk->{active},$trk->{min_fps},$trk->{max_fps},$trk->{cur_fps});
        next if not defined $state or not defined $min or not defined $max or not defined $cur;
        my $avg = $min + int( ($max-$min) / 2);
        $act = 'no' unless $act;
        if ($act eq 'yes') {
    	    if ($state eq 'off') {
    		my $note = "Analyzer '$name' for camera [$obj] '$ra->{$obj}{NAME}' was stopped";
    		notice $note;
    		$system_status='CRITICAL';
    		$critical_notes.="$note\n";
    		last;
    	    }
    	    elsif ($state eq 'on' and $cur < $avg) {
    		my $note = "Analyzer '$name' for camera [$obj] '$ra->{$obj}{NAME}' lacks performance";
    		notice $note;
    		$system_status='ALERT' if $system_status ne 'CRITICAL';
    	        $alert_notes.="$note\n";
    	        last;
    	    }
        }
     }
   }
}

sub ds_status
{
   announce 'DS status';

   my $ra;
   eval {
	db_master unless $dbh;
	$ra = $dbh->selectall_hashref(
 	   "select o.obj,o.name,a1.val as state, a2.val as status "
	  ."from _objs o, _obj_attr a1, _obj_attr a2 "
	  ." where otype='D' and subtype='V' and  o.obj=a1.obj and a1.attr='STATE'"
	  ."  and  o.obj=a2.obj and a2.attr='STATUS' and deleted=0",'OBJ',{Slice=>[]}
        );
   };
   if($@) {
     $system_status='CRITICAL';
     $critical_notes="Couldn't read configuration database\n"; # should never happen
   }
   my $disconnected=0;
   foreach my $obj( keys %$ra) {
     $disconnected++ if $ra->{$obj}->{STATUS} eq 'disconnected';
   }
   if(! $disconnected) {
     notice 'OK';
   }elsif($disconnected<5) {
     $system_status='ALERT' if $system_status ne 'CRITICAL';
     foreach  my $obj( keys %$ra) {
        my $dev=$ra->{$obj};
        next unless $dev->{STATUS} eq 'disconnected';
        my $note=Encode::encode("utf8", "vMX Monitor #$obj ").
    		    $dev->{NAME}.
    		    Encode::encode("utf8", " is DISCONNECTED");
    	$note = Encode::decode("utf8", $note);
        notice $note;
        $alert_notes.="$note\n";
     }
   }else {
     $system_status='ALERT' if $system_status ne 'CRITICAL';
     my $note="System has $disconnected disconnected vMX Monitors";
     notice $note;
     $alert_notes.="$note\n";
   }
}

sub osgi_status
{
    announce 'OSGi Framework status';

    my %status;
    if (open STAT, "$SMXCTL extstatus 2>/dev/null |") {
	%status = map {/^(\w+)=(.*)$/} grep {/^\w+=/} <STAT>;
    }
    foreach my $k (keys %status) {
      notice "$k=$status{$k}";
    }

    if (not defined $status{STATUS}) {
    	$system_status='ALERT' if $system_status ne 'CRITICAL';
	$alert_notes .= "OSGi Framework status unknown\n";
    }
    elsif ($status{STATUS} ne 'run') {
    	$system_status='ALERT' if $system_status ne 'CRITICAL';
	$alert_notes .= "OSGi Framework is not running\n";
    }
    else {
	my $bundle_status = $status{BUNDLE_STATUS};
	last unless defined $bundle_status;

	if (defined $bundle_status and $bundle_status ne 'ok') {
	    $system_status='ALERT' if $system_status ne 'CRITICAL';
	    my $msg = $status{MESSAGE} || "OSGi bundles in error state";
	    chomp $msg;
	    $alert_notes .= "$msg\n";
	}
    }
}

sub cloud_status
{
   my %cloudconf;
   eval {
	db_master unless $dbh;
	my $c = $dbh->selectall_arrayref( qq {
		select attr,val from _obj_attr
		where obj=53 and attr in
		('CLOUD_STORAGE','CLOUD_STORAGE_ENABLED','CLOUD_KEY_ID', 'CLOUD_SECRET_KEY')
	});
	foreach my $av (@$c) {
		$cloudconf{$av->[0]} = $av->[1];
	}
   };
   if($@) {
     $system_status='CRITICAL';
     $critical_notes="Couldn't read configuration database\n"; # should never happen
   }

   if ($cloudconf{CLOUD_STORAGE} ne 'none' && $cloudconf{CLOUD_STORAGE_ENABLED} eq 'yes') {
   	eval {
		no strict 'subs';
		push @INC, "$APL/sm/lib";
		require SM::Cloud;
		SM::Cloud->import($cloudconf{CLOUD_STORAGE});
		Cloud_Init(
			$cloudconf{CLOUD_KEY_ID},
		        $cloudconf{CLOUD_SECRET_KEY}
		);

		my @buckets = Cloud_ListBuckets();
		die "Empty bucket list" if not @buckets;
	};
	if ($@) {
		$system_status='CRITICAL';
		$critical_notes="Cloud connection failure\n";
	}
   }
}

sub ABRT_status
{
    # check do we have directories in /var/tmp/abrt
    my $crashes = `find /var/tmp/abrt -name "ccpp-*" 2>/dev/null`;

    if ($crashes ne "")
    {
        $system_status='ALERT' if $system_status ne 'CRITICAL';
        $alert_notes.="Crash reports were detected. Please send support files to system administrator\n";
    }
}

sub avatar_status
{
    announce 'Avatar status';

    my $json = `$AV_STATUS DETAILS=1 2>/dev/null`;
    if ($json) {
        my $status = eval { decode_json $json };
        if ($@) {
            $system_status='ALERT' if $system_status ne 'CRITICAL';
            $alert_notes="Cannot get avatar status\n";
        }
        else {
            my $offline=0;
            foreach my $obj (keys %$status) {
                $offline++ if $status->{$obj}{status} eq 'OFFLINE';
            }
            if(! $offline) {
                notice 'OK';
            } elsif($offline<5) {
                $system_status='ALERT' if $system_status ne 'CRITICAL';
                foreach my $obj (keys %$status) {
                    next if $status->{$obj}{status} ne 'OFFLINE';
                    my $note = "Avatar #$obj '$status->{$obj}{name}' is OFFLINE";
                    notice $note;
                    $alert_notes.="$note\n";
                }
            } else {
                $system_status='ALERT' if $system_status ne 'CRITICAL';
                my $note="System has $offline offline Avatars";
                notice $note;
                $alert_notes.="$note\n";
            }
        }
    }
}

sub run_once
{

  for(;;) {
#	db_master unless $dbh;
	master_status;
	last if $system_status ne 'ONLINE';
	install_result;

	last if (db_status eq 'DOWN');

	events_count if ! $silent;

    event_integrity;

	osgi_status;

	node_status;

	volume_status;

	sm_status;

	cache_status;

	device_status;

        camera_status;

        analytics_status;

	ds_status;

	cloud_status;

	avatar_status;

        ABRT_status;

        last;
  }
  summary;
}


# MAIN ==================================

for(;;) {
  run_once;
  exit if $ARGV[0] ne 'daemon';
  sleep $SLEEP;
  system('chgrp apache /tmp/vnmq_broker >/dev/null 2>/dev/null ; chmod g+w /tmp/vnmq_broker >/dev/null 2>/dev/null');
}
