#!/usr/bin/perl
#  $Id: peed 33603 2016-02-16 18:46:11Z atsybulnik $
# -----------------------------------------------------------------------------
#  PEED - Policy Enforcment, Event Disposal 
# -----------------------------------------------------------------------------
#  Author: Alex Titov
#  QA by:  Christopher C Gettings
#  Copyright: videoNEXT LLC
# -----------------------------------------------------------------------------

use strict;
use Time::HiRes;
use SKM::DB;
use Data::Dumper;
use XML::Simple qw(:strict);
use Log::Log4perl "get_logger";
require "$ENV{APL}/common/bin/logger.patrol";

# CONS ========================================================================
my $packetSize = 5000;
my $gismoDataSize = 100000;
my $timeout = 9;

my $startPeed = [Time::HiRes::gettimeofday()];
my $maxEventLifespan = 1000000; # Infinite max lifespan by default
my $cloudEnabled = 0;
my $log=get_logger('NEXTCAM::SDI::PEED');
my $dbh=DBMaster({PrintError=>1, 'RaiseError' => 0}) || $log->logdie($DBI::errstr);
$dbh->{FetchHashKeyName} = 'NAME_uc';
$dbh->{RaiseError}=1;
$dbh->{ShowErrorStatement}=1;

# MAIN =======================================================================
$log->info('starting "Event Disposal"');

my @preStmtSchedule = (
	{ desc => "Creating temp tables",
	  stmt => "
CREATE TEMPORARY TABLE tmp_peed_objexp (objid integer, priority integer, lifespan integer, exp timestamp without time zone);
CREATE UNIQUE INDEX tmp_peed_objexp_pkey ON tmp_peed_objexp (objid, priority);
CREATE TEMPORARY TABLE tmp_peed_eventexp (eventid integer);" },

	{ desc => "Making device/priority expiration schedule",
	  stmt => "INSERT INTO tmp_peed_objexp
SELECT a.obj, s.eventpriority, 
       CASE WHEN o.deleted=1 THEN 1
            WHEN s.preserverhours < 0 THEN 0
            ELSE 1 END,
       CASE WHEN o.deleted=1 THEN '9999-12-31 23:59:59'::timestamp
            WHEN s.preserverhours < 0 THEN '9999-12-31 23:59:59'::timestamp
            ELSE date_trunc('hour', now() at time zone 'UTC' - (s.preserverhours || 'hour')::interval) END
  FROM _objs o
 INNER JOIN _obj_attr a ON o.obj = a.obj AND a.attr='EVENT_POLICY'
 INNER JOIN eventpolicystorage s ON s.eventpolicy = cast(a.val as int);" },
);


my @stmtSchedule = (
	{ desc => "Looking for expired events",
	  stmt => "
INSERT INTO tmp_peed_eventexp
 SELECT e.eventid
   FROM tmp_peed_objexp x 
  INNER JOIN event e ON e.objid = x.objid AND e.priority = x.priority
  WHERE (e.utc_from < x.exp AND e.lifespan <= x.lifespan) OR ((now() at time zone 'UTC' - e.utc_from)::interval > '%d days'::interval)
  LIMIT $packetSize;" },

	{ desc => "Starting transaction",
	  stmt => "START TRANSACTION;" },
	  
#	{ desc => "Creating tasks for wipe",
#	  stmt => "INSERT INTO wipe_tasks
#		    (SELECT array_agg_nodup(w.objid),
#			date_trunc('hour',generate_series(e.utc_from, e.utc_to, '1 hour')) AS utc_when_hour
#		    FROM eventwitness w INNER JOIN event e ON e.eventid=w.eventid
#		    WHERE e.eventid IN (SELECT eventid FROM tmp_peed_eventexp)
#		    GROUP BY 2);" },

	{ desc => "Deleting event properties",
	  stmt => "DELETE FROM eventproperty WHERE eventproperty.eventid IN (SELECT eventid FROM tmp_peed_eventexp);" },

	{ desc => "Deleting event bookmarks",
	  stmt => "DELETE FROM eventbookmark WHERE eventbookmark.eventid IN (SELECT eventid FROM tmp_peed_eventexp);" },

	{ desc => "Deleting event witnesses",
	  stmt => "DELETE FROM eventwitness WHERE eventwitness.eventid IN (SELECT eventid FROM tmp_peed_eventexp);" },

	{ desc => "Deleting gismo records",
	  stmt => "DELETE FROM gismo_data WHERE gismo_data.targetElEventId IN (SELECT eventid FROM tmp_peed_eventexp);" },

	{ desc => "Deleting events",
	  stmt => "DELETE FROM event WHERE event.eventid IN (SELECT eventid FROM tmp_peed_eventexp);" },

	{ desc => "Commiting changes",
	  stmt => "COMMIT;" },

	{ desc => "Clean up temporary table",
	  stmt => "TRUNCATE tmp_peed_eventexp;" },
);

my @postStmtSchedule = (
	{ desc => "Deleting workflow",
	  stmt => "
START TRANSACTION;
DELETE FROM workflow
 WHERE deleted=true
   AND NOT EXISTS (SELECT 1 FROM event e WHERE e.workflow=workflow.workflow);
COMMIT;" },

	{ desc => "Deleting old gismo data",
	  stmt =>"
START TRANSACTION;
DELETE FROM gismo_data 
WHERE stime < (SELECT MIN(stime) FROM  
 (SELECT stime FROM gismo_data ORDER BY stime DESC 
 LIMIT $gismoDataSize) a);
COMMIT;" },
	{ desc => "Deleting old content delivery history",
	  stmt => "
START TRANSACTION;
DELETE FROM content_delivery_history
WHERE tstamp < date_trunc('hour', now() at time zone 'UTC' - '1 hour'::interval);
COMMIT; " },
        { desc => "Deleting aod requests for removed devices",
          stmt => "
START TRANSACTION;
DELETE FROM av_archive_requests
WHERE objid IN (SELECT obj FROM _objs WHERE deleted=1);
COMMIT;" }
);


sub execBatch {
	my ($batchName, $refStmtList, $retIndex) = @_;
	my $retValue = 0;
	my $startTs = [Time::HiRes::gettimeofday()];

	foreach my $i (0 .. scalar(@$refStmtList)-1) {
		$log->debug("Stmt=$i:$$refStmtList[$i]{desc} SQL=$$refStmtList[$i]{stmt}");

		my $stmtTS = [Time::HiRes::gettimeofday()];
		eval {
			my $result = $dbh->do($$refStmtList[$i]{stmt});
			if ($retIndex == $i) {
				$retValue = ($result eq '0E0') ? 0 : $result;
			}
			
			if ($log->is_debug()) {
				my $elapsed = Time::HiRes::tv_interval($stmtTS);
				my $msg = sprintf "Stmt=%d completed. Return: %s, elapsed time: %.3f sec",
					$i, ($result eq '0E0') ? 'OK' : $result ,$elapsed;
				$log->debug($msg);
			}
		};
		if ($@) {
			$log->error($@);
			$retValue = -1;
			last;
		}
	}

	my $elapsed = Time::HiRes::tv_interval($startTs);
	my $msg = sprintf "%s completed. Return: %s, elapsed time: %.3f sec", $batchName, $retValue ,$elapsed;
	$log->info($msg);

	return $retValue;
}

sub loadAttr {
	my $rh;
	eval {
		$rh = $dbh->selectall_hashref(
			"SELECT attr,val FROM _obj_attr 
			WHERE obj=53 AND attr in ('CLOUD_STORAGE','ELOG_MAX_EVENT_LIFESPAN')",
			'ATTR', {Slice=>{}}
		);
	};
	if (!$@ && $rh) {
		$cloudEnabled = 1 if $rh->{CLOUD_STORAGE}{VAL} ne 'none';
		$maxEventLifespan = $rh->{ELOG_MAX_EVENT_LIFESPAN}{VAL} || 1000000;
		$maxEventLifespan = 1000000  if $maxEventLifespan < 0;
		
		# Insert value into SQL statement
		$stmtSchedule[0]{stmt} = sprintf($stmtSchedule[0]{stmt}, $maxEventLifespan);
	}
	if ($cloudEnabled) {
		splice @stmtSchedule, 2, 0, 
		    { 
			desc => "Creating tasks for cloud_wipe",
			stmt => "INSERT INTO cloud_wipe_tasks
				(SELECT eventid, now() at time zone 'UTC' 
				 FROM tmp_peed_eventexp)" }
	}
}


die "Peed failed" if execBatch("Create temporary tables", \@preStmtSchedule, -1) == -1;

loadAttr;

my ($eventCount, $eventDeleted, $iteration) = (0, 0, 0);

#
# loop
#
do {
	$iteration++;
	sleep($timeout) if ($iteration > 1);
	$eventDeleted = execBatch("Delete event (iteration $iteration)", \@stmtSchedule, 0);
	($eventDeleted >= 0) || die "Failed to delete event";
	$eventCount += $eventDeleted;
} while ($eventDeleted >= $packetSize);


die "Peed failed" if execBatch('Delete workflow and gismo data', \@postStmtSchedule, -1) == -1;

my $elapsed = Time::HiRes::tv_interval($startPeed);
my $msg = sprintf "Event Disposal completed (%d events, %d iterations). Elapsed time: %.3f sec",
	$eventCount, $iteration, $elapsed;
$log->info($msg);
