#!/usr/bin/perl
#
#  $Id$
# -----------------------------------------------------------------------------
#  SM_DB - Database intergation and replication
# -----------------------------------------------------------------------------
#  Author: Alex Titov
#  QA by:
#  Copyright: videoNEXT LLC
# -----------------------------------------------------------------------------
#  
#
# for REQ see: Rally S623:Storage Manager TA1112: database replication
# 
# 1. started by sm_engine
# 2. SM_DB will die if problem with DB. It will be restarted by sm_engine
# 3. Synchronize flat-file configuration with master DB
# 4. Check node level command (sm_nodes[CMD])
#      SCAN  -search new disk resources locally and acquire if FREE
#      ADOPT -search marked but unknown disk resources and ADOPT them
#      FREEZE | UNFREEZE | RESTORE - for sync with backup/restore operations
#      mark command completion by setting sm_nodes[CMD_END]
# 5. Check command for unassigned devices (sm_unused [CMD])
#      ASSIGN -assign unused Volume to Wheels. Create Wheel structures
#      REMOVE -remove unused Volume (can be found agains by node(SCAN))
#      in result of command the record will be removed from sm_unused
# 6. Replicate Wheels STAT and Operational State (OST) to sm_stat and sm_ost
# 7. Check desirable state (set by GUI in sm_cst) and manipulate wheels
#      ONLINE|OFFLINE|SUSPEND|DEMOTE
# 8. Synchronise new/remove records in wheels configuration to DB
# 9. TBD: Synhcronise update of attibutes in sm_wheels/sm_unused to flat files
#10. Freeze any DB synhronizations if FREEZE is set in sm_nodes
#    It happens when backup or restore in progress
#11. TBD: Complete restore configuration files from DB when sm_nodes[CMD]=RESTORE
#
# TBD: Transactional DB update! (commit rollback etc)


use warnings;
use strict;
use SKM::DB;
use Data::Dumper;
use File::Basename qw(dirname);
use lib dirname(__FILE__).'/../lib';            # find  SM::Config here
use SM::Config ':all';
use Node::Conf;                                 # defines UNI (unique node id)

# CONS ------------------------------------------------------------------------
my $SMDB     =(split(/\//,$0))[-1];	        # the actual name of the prog
my $APL       =$ENV{APL};
my $SLEEP     =5;                               # 3 (secounds) is recommended
my $TOOL_NAME='sm_tool';
my $TOOL=dirname(__FILE__)."/$TOOL_NAME";       # name with path.
my $UPGRADE_NAME='sm_upgrade';
my $UPGRADE=dirname(__FILE__)."/$UPGRADE_NAME"; # name with path.
my $UMOUNT="sudo $APL/sm/sbin/sm_umount '<all>'";
my $nodeid=UNI;                                 # char22 nodeid in domain
my $HOST=`hostname`; chomp $HOST;               # hostname
my $SMCONF="$ENV{APL_VAR}/conf/sm";

# VARS ------------------------------------------------------------------------
my $dbm;                                        # DB handler
my %dbs;
my $dbfreeze=0;                                 # disable DB sync

# SUBS ------------------------------------------------------------------------

sub db_master{
 eval { $dbm->disconnect() if $dbm; $dbm=''; }; # disconnect if defined
 for (my $i=1;$i<6;$i++) {
   eval {
     $dbm=DBMaster({PrintError=>1,'RaiseError' => 1});
     # ex: $dbs{INSERT}=$dbm->prepare('insert into audit(objid,category,parameters) values(?,?,?)');
     $dbs{SET_ALIVE}=$dbm->prepare("update SM_NODES set ALIVE = now() at time zone 'UTC' where NODEID='$nodeid'");

  };
   if($@) {
     SM_LOG->logdie("Attempt $i (final). Cannot connect to master: $@") if $i>=5;
     SM_LOG->error("Attempt $i. Cannot connect to master: $@");
     SM_LOG->error('Sleep '. $i*30 . ' before next attempt');
     sleep($i*30);
   } else {
     last;                                   # exit cycle if OK
   }
 }
 $dbm->{FetchHashKeyName} = 'NAME_uc';
 $dbm->{ShowErrorStatement}=1;
 SM_LOG->debug("Connected to master db");
}


#------------------------------------------------------------------------------
# Add new wheel in DB: sm_wheel table
#------------------------------------------------------------------------------
sub add_wheel {
  my $vol=shift;                                   # usage $vol->{ost}
  SM_LOG->info("Adding vol=$vol->{ID} to DB:sm_wheels");

  $dbm->do("insert into SM_WHEELS(ID,NODEID,NAME,LIMIT_WRITE,TARGET,MOUNT_OPTIONS) values(?,?,?,?,?,?)",
      undef,($vol->{ID},$nodeid,$vol->{NAME},$vol->{LIMIT_WRITE},$vol->{TARGET},$vol->{MOUNT_OPTIONS}));
  my ($cst,$ost)=($vol->{cst},$vol->{ost});
  $cst='OFFLINE' if not $cst;
  $ost='Offline' if not $ost;
  $cst='ONLINE'  if $cst eq 'undef'; 
  $dbm->do("insert into SM_CST(ID,CST) values(?,?)",undef,($vol->{ID},$cst));
  $dbm->do("insert into SM_OST(ID,OST) values(?,?)",undef,($vol->{ID},$ost));
  my $size=$vol->{stat_SIZE}; $size=0  if not $size;
  my $free=$vol->{stat_FREE}; $free=-1 if not $free;
  my $used=$vol->{stat_USED}; $used=-1 if not $used;
  my $write=$vol->{stat_WRITE}; $write=0 if not $write;
  $write=int($write); 
  $write=1 if !$write && $vol->{stat_WRITE}>0;
  $dbm->do("insert into SM_STAT(ID,SIZE,FREE,USED,LIMIT_WRITE,WRITE) values(?,?,?,?,?,?)",
     undef,($vol->{ID},$size,$free,$used,$vol->{LIMIT_WRITE},$write));
}
#------------------------------------------------------------------------------
# load informaton about wheels from db
# compare and save if different
#------------------------------------------------------------------------------
sub sync_db2file {
   my $vol=shift;   
   my $wheels=$dbm->selectall_hashref(qq{
                   select v.ID,v.NAME,v.LIMIT_WRITE,v.TARGET,v.MOUNT_OPTIONS
                     from SM_WHEELS v
                    where NODEID='$nodeid'
                   }, 'ID',{Slice => {}}
  );
  my %dconf=%{$wheels->{$vol->{ID}}};
  my $count=0;                           # count if differnce
#  print Dumper $vol;
#  print Dumper \%dconf;
  foreach my $attr (keys %dconf) {
     if(not exists $vol->{$attr} or $vol->{$attr} ne $dconf{$attr}) {
        $vol->{$attr}=$dconf{$attr};      #update
        $count++;
     }
  }
  if($count) {                           # difference found. update flatfile
    my $confname="$SMCONF/wheels/$vol->{ID}";
    if (open(CONF,$confname)) {
      my %conf=map{/(^\w+)=(.+)/} grep {/^\w+=.+/} <CONF>;
      close CONF;
      foreach my $attr (keys %conf) {
        $conf{$attr}=$dconf{$attr} if exists $dconf{$attr};
      }
      if(open(CONF,">$confname")) {
        foreach my $attr ( sort keys %conf) {
           print CONF "$attr=$conf{$attr}\n" 
        }
        close CONF;
      }else {
                # message cannot write conf
      }
    }else{
                # message cannot read conf
    }
  }  
}
#------------------------------------------------------------------------------
# initiate tables from flat files 
# this is procedure for initial implementation
#------------------------------------------------------------------------------
sub init_replica {
  my $vols=shift;
  eval {
   my $node=$dbm->selectall_arrayref(qq{
                   select NODEID from SM_NODES where NODEID='$nodeid'
                   }, {Slice => {}}
   );
   #print Dumper($node);
   if(not @$node) {                              #  node is not defined in DB
     SM_LOG->info("Node record is absent, creating record for node '$nodeid'");
     $dbm->do("insert into SM_NODES(NODEID,NAME) values('$nodeid','$HOST')");
     SM_LOG->info("Force unmount all volumes for initial statistic calculation");
     system($UMOUNT);
     my $volumes=$dbm->selectall_hashref(qq{
                   select v.ID,c.CST,o.OST 
                     from SM_WHEELS v, SM_CST c, SM_OST o
                    where NODEID='$nodeid' and v.ID=c.ID and v.ID=o.ID
                   }, 'ID',{Slice => {}}
     );
     #print Dumper($volumes);
     #---------------------------------------------- # include volume if absent
     foreach my $uuid (sort keys %$vols) {           # compare flat files with DB
       add_wheel($vols->{$uuid}) if not exists $volumes->{$uuid};
     }
     #---------------------------------------------- # remove extra vol from db
     foreach my $uuid (sort keys %$volumes) {        # compare DB with flat files
       if (not exists $vols->{$uuid}) {              # volume is absent, remove it
         SM_LOG->info("Volume $uuid is absent on node. Record will be removed");
         $dbm->do("delete from SM_WHEELS where ID='$uuid'"); # cascade delete
       }
     }
   }
  };
  if($@) {
     SM_LOG->error("Cannot initialize replication: $@");
  }
}
#------------------------------------------------------------------------------
sub demote_vol { # TBD: transaction and error handling
   my $vol=shift; 
   eval {
    `$TOOL demote $vol->{ID}`;  
     $dbm->do("insert into SM_UNUSED(NODEID,ID,NAME,LIMIT_WRITE,MOUNT_OPTIONS,TARGET,SIZE,FREE) "
             ."select w.NODEID,w.ID,w.NAME,w.LIMIT_WRITE,w.MOUNT_OPTIONS,w.TARGET,s.SIZE,s.FREE "
             ."from SM_WHEELS w, SM_STAT s where w.ID=s.ID and w.ID=?",undef,($vol->{ID}));
     $dbm->do('delete from SM_WHEELS where ID=?',undef,($vol->{ID})); # ATTN:cascade
     rename(SM_CONF."/wheels/$vol->{ID}",SM_CONF."/unassigned/$vol->{ID}");
   };
   #TBD error handling
}
#------------------------------------------------------------------------------
sub  update_stat {
  my $vol=shift; 
  $dbm->do('update SM_STAT set SIZE=?,FREE=?,USED=?,LIMIT_WRITE=?,WRITE=? where ID=?',
      undef,($vol->{stat_SIZE},  $vol->{stat_FREE}, $vol->{stat_USED},
             $vol->{LIMIT_WRITE},int($vol->{stat_WRITE}+0.5),$vol->{ID}));
}
#------------------------------------------------------------------------------
sub  update_ost {
  my $vol=shift; 
  $dbm->do('update SM_OST set OST=? where ID=?',undef,($vol->{ost},$vol->{ID}));
}
#------------------------------------------------------------------------------
# 1. set cst=ONLINE in DB (!! it is not clean logic!)
# 2. start sm_upgrade($cmd) in background
#
sub upgrade_wheel {
  my ($vol,$cst)=@_; 
  $dbm->do('update SM_CST set CST=? where ID=?',undef,('ONLINE',$vol->{ID})); 
  SM_LOG->info("upgrading($cst) $vol->{ID}");
  SM_LOG->info("$UPGRADE $cst $vol->{ID} ");
  `$UPGRADE $cst $vol->{ID} >/dev/null 2>&1 &`;
}

#------------------------------------------------------------------------------
sub  update_cst {
  my ($vol,$cst)=@_; 
  my $id=$vol->{ID};
  my ($name,$temp)=(SM_STAT."/$id.cst",SM_STAT."/$id.tmp");
  if($cst eq 'DEMOTE')      {       # special case for removing vol from wheels
    if($vol->{ost} ne 'Offline') {
       $cst='OFFLINE';              # put volume offline fist if it not offline
    }else {
       demote_vol($vol);            # move vol from wheels to unassign
       return;
    }
  }
  if($cst=~/^(CONVERT|CLEAN)$/)  {  # special case for migraation
    upgrade_wheel($vol,$cst);
    return;
  }

  if (not $cst=~/^(ONLINE|OFFLINE|SUSPEND|EMPTY)$/) {
    SM_LOG->warn("Cannot set unknown status '$cst' for $id. Command ignored");
    SM_warn($id, "Cannot set unknown status '$cst'. Command ignored");
    return;
  }
  if (open(CST,">$temp")) {
    print CST "$cst\n";
    close CST;
    unlink($name) if -f $name;
    rename($temp,$name);
  } else {
    SM_LOG->error("Cannot open $name for writing\n");
    SM_error($vol,"Cannot $cst volume. IO error"); 
  } 
}
#------------------------------------------------------------------------------
sub wheels_sync {			# TBD: error handling ! remove volume
  my ($vols,$prev)=@_;
  #------------------------------------ get data from database (~ 5-100 rows)
  my $ary_ref = $dbm->selectcol_arrayref("select ID, CST from SM_CST",{Columns=>[1,2]});
  my %cst=@$ary_ref;
  #------------------------------------ 
  foreach(keys %$vols){
    if(not exists $prev->{$_}) {
       add_wheel($vols->{$_});
    }else{                              # TBD: $cst=DEMOTE
       die('DB out of sync is detected') if not defined $cst{$_};
       my($vol,$old)=($vols->{$_},$prev->{$_});
       update_stat($vol)          if $vol->{stat_mtime}!=$old->{stat_mtime};
       update_ost ($vol)          if $vol->{ost} ne $old->{ost};
       sync_db2file($vol)         if $vol->{cst} ne $cst{$_} and $cst{$_} eq 'ONLINE';
       update_cst ($vol,$cst{$_}) if $vol->{cst} ne $cst{$_}
    }
  }
}
#------------------------------------------------------------------------------
# check command for unused (unassigned) volumes for commands ASSIGN,REMOVE
#------------------------------------------------------------------------------
sub unused_cmd {
  my $ary_ref = $dbm->selectcol_arrayref(
     "select ID, CMD from SM_UNUSED where NODEID='$nodeid' and CMD != ''",
     {Columns=>[1,2]}); 
  return if not $ary_ref;           #empty, do nothing
  my %act=@$ary_ref;                #convert into hash
  foreach(keys %act) {
    $dbm->do("update SM_UNUSED set CMD='' where NODEID=? and ID=?",
          undef,($nodeid,$_)); 
    if($act{$_} eq 'ASSIGN') {
      `$TOOL assign $_ wheels `;  
    }elsif($act{$_} eq 'REMOVE') {
      `$TOOL remove $_`;  
    }
    if(not -f SM_CONF."/unassigned/$_") { # if file is removed then rm from DB
       $dbm->do('delete from SM_UNUSED where NODEID=? and ID=?',undef,($nodeid,$_));
    }
  }
}
#------------------------------------------------------------------------------
# create file structire for wheel and set cst=OFFLINE
#------------------------------------------------------------------------------
sub mk_wheel {
  my $vol=shift;
  my %attrs=(UMOUNT=>'default',MOUNT=>'default',TEST=>'default',%$vol);
  # set LABEL if missing
  $attrs{LABEL}= $attrs{NAME} if not defined $attrs{LABEL} and defined $attrs{NAME};
  my $confname="$SMCONF/wheels/$attrs{ID}";
  my ($name,$temp)=(SM_STAT."/$attrs{ID}.cst",SM_STAT."/$attrs{ID}.tmp");
  my $do_restoremark=(-f "$confname")?0:1;   # restore mark if conf not exists
  if (open(WHEEL,">$confname")) {
    foreach my $attr (sort keys %attrs) { 
      print WHEEL "$attr=$attrs{$attr}\n";
    }
    close WHEEL;
    `$TOOL restoremark $attrs{ID}` if $do_restoremark; # ignore errors
    if (open(CST,">$temp")) {
       print CST "OFFLINE\n";
       close CST;
       unlink($name) if -f $name;
       rename($temp,$name);
     } else {
       SM_LOG->error("Cannot open $name for writing\n");
       SM_error($vol,"Cannot set OFFLINE. IO error");
     }
  } else {
     SM_LOG->error("Cannot open $confname for for writing\n");
     SM_error('',"Cannot open $confname for for writing");
  }
}
#------------------------------------------------------------------------------
# restore logic:
# 1. Restore all DB:wheels -> FILE:vols, set FILE:cst=OFFLINE (even if DB:cst==ONLINE)
# 2. Keep FILE:vol if DB:wheel is missing
# 3. remove FILE:unassign if present in DB:wheels
# 4. remove DB:unassign if present in FILE:vols
#------------------------------------------------------------------------------

sub restore {
  mkdir("$SMCONF",           0750)  if ! -d "$SMCONF";
  mkdir("$SMCONF/unassigned",0750)  if ! -d "$SMCONF/unassigned";
  mkdir("$SMCONF/wheels",    0750)  if ! -d "$SMCONF/wheels";
  mkdir("$SMCONF/declared",  0750)  if ! -d "$SMCONF/declared";
  mkdir("$SMCONF/rejected",  0750)  if ! -d "$SMCONF/rejected";
  my $vols=SM_Wheels();                    # from files on disk
  my $wheels=$dbm->selectall_hashref(qq{
                   select v.ID,v.NAME,v.LIMIT_WRITE,v.TARGET,v.MOUNT_OPTIONS
                     from SM_WHEELS v
                    where NODEID='$nodeid'
                   }, 'ID',{Slice => {}}
  );
  foreach my $uuid (sort keys %$wheels) {
       mk_wheel($wheels->{$uuid});
  }  
  sleep 10; # let wheels go OFFLINE;
}
#------------------------------------------------------------------------------
# check command [request] for node scan
#------------------------------------------------------------------------------
sub node_cmd {
   my $ref = $dbm->selectall_arrayref(
      "select TARGET, CMD from SM_NODES \n"
     ."where NODEID='$nodeid'  and STIME>CMD_END ", {Slice=>{}});
   return if not @$ref;                # empty array -> no request
   my ($cmd,$target)=($ref->[0]->{CMD},$ref->[0]->{TARGET});
   return if $dbfreeze and $cmd eq 'FREEZE';
   $dbfreeze=0;                        # presumably any cmd unfreeze DB SYNC
   SM_LOG->info("Node-level command for Storage Manager:$cmd");
   if($cmd=~/^(SCAN|ADOPT)$/) {#-----------------------------------SCAN & ADOPT
     $target='' if not defined $target;# protect from undef
     if($target ne '') {                
       SM_LOG->info("Target is defined '$target'. Skipping scan");
       # TBD: prepare structures for iSCSI, NAS, NFS
     }else{                            #empty target-> local scan
       SM_LOG->info("Scan local disk resources");
       $cmd=lc($cmd);                  #lower case scan | adopt 
       `$TOOL $cmd`;
     } 
     SM_LOG->info("Start acquire disk resoures");
     `$TOOL acquire`;
   }elsif($cmd eq 'FREEZE')   {#-----------------------------------STOP DB SYNC
     $dbfreeze=1;
     SM_LOG->info("DB sync is frozen till UNFREEZE or RESTORE");
     SM_info ('', "DB sync is frozen till UNFREEZE or RESTORE");
   }elsif($cmd eq 'UNFREEZE') {#----------------------------------START DB SYNC
     SM_LOG->info("DB sync is active");  # dummy command for message only
     SM_info ('', "DB sync is active"); 
   }elsif($cmd eq 'RESTORE')  {#-----------------------------------RESTORE
     SM_LOG->info("restore Storage Manager configuration from DB");
     SM_info ('', "restore Storage Manager configuration from DB"); 
     restore();
   }else {                     #-----------------------------------UNKNOWN
     SM_LOG->warn("Command '$cmd' is ignored since unknown");
     SM_warn ('', "Command '$cmd' is ignored since unknown");
   }
   $dbm->do("update SM_NODES set CMD='',CMD_END = now() at time zone 'UTC' where NODEID=?",undef,($nodeid)) if $cmd ne 'FREEZE';
   if($cmd eq 'RESTORE') {
     SM_LOG->info("restore completed, restarting $SMDB");
     SM_info('',  "restore completed, restarting $SMDB");
     die('RESTART');
   }
}
#------------------------------------------------------------------------------
# sysn unused/unassigned
# read db_unused and unassigned
# insert into db_unused if present in unassigned but not exists in db_unused
#------------------------------------------------------------------------------
sub unused_sync {
   #-------------------------------------- get unused from DB
   my $ref = $dbm->selectall_arrayref(
      "select ID,NAME,LIMIT_WRITE,TARGET,MOUNT_OPTIONS from SM_UNUSED \n"
     ."where NODEID='$nodeid'", {Slice=>{}});
   my %unused;
   $unused{$_->{ID}}=($_) foreach (@$ref);
   #-------------------------------------- get unassigned from Files
   my $unassigned=SM_Unassigned();
   #print Dumper($unassigned);
   foreach my $id (keys %$unassigned) {
     if(not exists $unused{$id}) {          # no records in DB
       my %un=%{$unassigned->{$id}};
       #print Dumper(\%un);
       SM_LOG->info("create record volume=$id in DB");
       SM_info ($id,"create record volume=$id in DB");
       $dbm->do("insert into SM_UNUSED(NODEID,ID,NAME,LIMIT_WRITE,TARGET,MOUNT_OPTIONS,SIZE,FREE)\n"
               ."values(?,?,?,?,?,?,?,?)",undef,$nodeid,$un{ID},$un{NAME},
               $un{LIMIT_WRITE},$un{TARGET},$un{MOUNT_OPTIONS},$un{stat_SIZE},$un{stat_FREE});
     }
   }
}

# MAIN =========================================================================
 SM_WritePid($SMDB);                                # write pid for a first time
 if (! $nodeid=~/^\w{22}$/) {
   sleep 15;                                        # sleep prevents fast restart
   SM_LOG->logdie("wrong UNI=$nodeid, $SMDB cannot operate");
 }
 SM_LOG->info("$SMDB starts");
 eval {
  db_master;                                        # connect to database
  my $vols=SM_Wheels();                             # load volumes initially
#  print Dumper ($vols);
  init_replica($vols);
  my $pvols=$vols;                                  # ?? $pvols can be updated in restore
  my $last_time=0;
  for(my $cycle=0;;) { # ----------------------------------- this is main loop
   SM_WritePid($SMDB);                              # indicate healthy life
   sleep $SLEEP;                                    # 5 seconds
   node_cmd;                                        # check for node level commands(scan,restore)
   next if $dbfreeze;                               # dbfreeze is set/cleared in node_cmd;
   $vols=SM_Wheels();                               # re-read volume configuration
                                                    # TBD localy load conf improve performance
   wheels_sync($vols,$pvols);                       # compare & update 
   unused_cmd;                                      # check if cmd ASSIGN,REMOVE
   unused_sync;                                     # sync Files->DB, TBD DB->Files
   $pvols=$vols;                                    # remember previous vols version
   if(time-$last_time>50) {                         # every 50 seconds (will not udpate if dbfreeze)
     $dbs{SET_ALIVE}->execute();                    # set sm_nodes(ALIVE=now)
     $last_time=time;
   }
  }
 }; # end_eval
 
 SM_LOG->error("SM_DB problem: $@") if $@ ne 'RESTART'; 

