#!/usr/bin/perl
use strict;
use warnings;

use Data::Dumper;
use Time::HiRes "gettimeofday";
use NextCAM::Conf;
use SKM::Common;
use Node::Conf;
use SKM::DB;
use File::Basename qw(dirname);
use lib dirname(__FILE__).'/../lib';              # find  SM::Config here
use SM::Config ":all";

# CONS
my $APL = $ENV{APL};
my $APL_VAR = "$ENV{APL_VAR}";
my $CLOUD_PREFIX = "store";
my $BUCKET = UNI;
my $IDENTITY_OBJ = 53;
my $ACTIVATED = (stat("$APL_VAR/vctl/activated"))[9];
my $CHLOG = "$APL_VAR/log/sm/cloud/vacuum.log";
my $CHLOG_MAX_SIZE = 5 * 1024 * 1024; # 5MB

# VAR
my $dbm;
my $sigterm;
my %CloudConf;
my %Devs;
my %Cache;
my $CacheObjid = 0;
my %Stat = (
        SIZE => 0,
        CHUNKS => 0
);

# SIG
$SIG{TERM} = sub { $sigterm = 1 }; # SIGTERM terminates mover

# SUB
sub chlog {
        my $msg = shift;
        
        if (-f $CHLOG) {
                my $sz = (stat $CHLOG)[7];
                rename($CHLOG, "$CHLOG.1") if $sz > $CHLOG_MAX_SIZE;
        }
        open LOG, ">>$CHLOG" or return;
        chomp $msg;
        print LOG "$msg\n";
        close LOG;
}

sub db_master
{
        eval { 
                $dbm->disconnect() if $dbm; 
                $dbm=''; 
        }; # disconnect if defined
        eval {
                $dbm = DBMaster({PrintError=>1,RaiseError => 1});
                # fetch cameras and audio devices from DB
                my $ra = $dbm->selectall_arrayref( qq {
                        select obj,subtype from _objs
                        where otype='D' and subtype in ('C','A') and deleted=0
                });
                foreach my $r (@$ra) {
                        my ($obj, $type) = @$r;
                        my $dev = ($type eq 'C' ? "" : "a") . $obj;
                        $Devs{$obj} = $dev;
                }
        };
        if ($@) {
                SM_error('',  "Cannot connect to master:$@");
        }
        
        $dbm->{FetchHashKeyName} = 'NAME_uc';
        $dbm->{ShowErrorStatement} = 1;
        $dbm->{AutoCommit} = 1;
}


sub check_cloud
{
        eval {
                my $c = $dbm->selectall_arrayref( qq {
                        select attr,val from _obj_attr 
                        where obj=$IDENTITY_OBJ and attr in
                        ('CLOUD_STORAGE','CLOUD_STORAGE_ENABLED',
                        'CLOUD_KEY_ID','CLOUD_SECRET_KEY','CLOUD_BUCKET')
                });
                foreach my $av (@$c) {
                        $CloudConf{$av->[0]} = $av->[1];
                }
        };
        if ($@) {
                SM_LOG->logdie("DB_MASTER: $@");
        }
        return ($CloudConf{CLOUD_STORAGE} ne 'none' and $CloudConf{CLOUD_STORAGE_ENABLED} eq 'yes') ? 1 : 0;
}


sub cloud_init
{
        eval {
                no strict 'subs';
                require SM::Cloud;
                SM::Cloud->import($CloudConf{CLOUD_STORAGE});
                Cloud_Init(
                        $CloudConf{CLOUD_KEY_ID},
                        $CloudConf{CLOUD_SECRET_KEY}
                );

                # If CLOUD_BUCKET system setting is specified 
                # it overrides bucket value which defaults to node UNI
                #
                $BUCKET = $CloudConf{CLOUD_BUCKET} if $CloudConf{CLOUD_BUCKET};

                my @buckets = Cloud_ListBuckets();
                my $found = 0;
                foreach my $bucket (@buckets) {
                        if ($bucket eq $BUCKET) {
                                $found = 1;
                                last;
                        }
                }
                die "Bucket $BUCKET is missing\n" if not $found;
        };
        if ($@) {
                SM_LOG->logdie("Cloud init failed: $@");
        }
}

sub delete_chunk 
{
        my ($key) = @_;

        eval {
                # Remove chunk
                Cloud_DeleteFile($BUCKET, $key->{key});
                # Remove index
                Cloud_DeleteFile($BUCKET, $key->{key}.".idx");
                # Update stat
                my $sz = int($key->{size} / 1024); # KB
                $Stat{SIZE} += $sz;
                $Stat{CHUNKS}++;
                chlog("$key->{key}\t$sz");
        };
        if ($@) {
                $sigterm = 1 if $@ =~ /SIGTERM/;
                SM_LOG->warn("chunk $key->{key} delete failed: $@");
                return 0;
        }
        return 1;
}

sub cache_db_chunks
{
        my ($objid) = @_;
        
        my $t_start = gettimeofday;
        my $cnt = 0;
        SM_LOG->debug("Cache cloud chunk names from DB for OBJID=$objid");
        eval {
                my $ra = $dbm->selectall_arrayref( qq{
                        select chunkid from cloud_chunks
                        where objid=$objid
                });
                %Cache = (); # Clear cache
                $cnt = @$ra;
                foreach my $r (@$ra) {
                        $Cache{$r->[0]} = 1;
                }
        };
        die "cache_db_chunks($objid) failed: $@" if $@;
        $CacheObjid = $objid;
        
        # Write stat
        #
        my $elapsed = gettimeofday - $t_start;
        if ($elapsed > 5) {
                chlog("cache_db_chunks($objid) => $cnt chunks, ${elapsed} s");
        }
}

sub vacuum
{
        chlog("VACUUM started at ".`date`);
        # Scan bucket content
        #
        my $t_start = time;
        my $trunc = 0;
        my $next;
        do {
                my $cfg = {
                        bucket => $CloudConf{CLOUD_BUCKET} 
                };

                if ($trunc && $next) {
                        $cfg->{marker} = $next;
                }
                else {
                        delete $cfg->{marker};
                }
                my $response = eval { Cloud_ListKeys($cfg) };
                
                $trunc = $response->{is_truncated};
                $next = $response->{next_marker} || undef;
                for my $key (@{ $response->{keys} }) {
                        next if $key->{key} =~ /\.idx$/; # Indexes are removed along with chunks
                        my ($dev, $streamnum, $chunk) = $key->{key} =~ m{^$CLOUD_PREFIX/(a?\d+)/(\d+)/(.+)$};
                        next if not $dev or not $streamnum or not $chunk;
                        my $objid = $1 if $dev=~/^a?(\d+)$/;
                        
                        # Remove chunk if corresponding device doesn't exist in DB
                        #
                        if (not $Devs{$objid}) {
                                delete_chunk($key);
                        }
                        else {
                                # Cache chunk names from cloud_chunks for this objid
                                #
                                cache_db_chunks($objid) if $CacheObjid != $objid;
                                delete_chunk($key) if not $Cache{"$streamnum-$chunk"};
                        }
                        die "SIGTERM" if $sigterm;
                }
        } while ($trunc);
}

sub main 
{
        SM_LOG->info("Cloud Vacuum started");

        # Pid control
        #
        die "Concurrent run detected!" if CheckPid;
        WritePid;

        # Do not start vaacuuming in less than a day after system activation
        #
        SM_LOG->info("Too early to do vacuuming. Wait a bit"), exit(0) 
                if time - $ACTIVATED < 86400;

        # Connect to Master DB
        # 
        db_master;
        
        # Read Cloud configuration from DB
        #
        die "No cloud configured!\n" unless check_cloud;

        # Connect to Cloud storage and validate its state
        #
        cloud_init;
        
        eval { 
                vacuum;
        };
        if ($@) {
                if ($@ =~ /^SIGTERM/) {
                        SM_LOG->warn("TERMINATED by request. The task is not completed");
                        exit 1;
                } elsif ($@ =~ /^DB_MASTER/) {            # problems with master DB
                        SM_LOG->logdie("TERMINATED. $@");
                } else {
                        SM_LOG->logdie("TERMINATED. Unknown Error: $@");
                }
        }
}

# MAIN
main;

END {
        RemovePid;
        my $msg = "Total chunks deleted: $Stat{CHUNKS}. Total size: $Stat{SIZE}K";
        SM_LOG->info($msg);
        chlog($msg);
        print "$msg\n";
        SM_LOG->info("Cloud Vacuum finished");
}
