#!/usr/bin/perl -w

###########################################################################
#
# Main polling program for czartool. Grabs stuff from various sources every few minutes.
#
###########################################################################

use warnings;
use strict;
use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
use POSIX qw/strftime/;
use List::MoreUtils qw(uniq);
use DateTime;

# local classes
use czartool::Config;
use czartool::CzarDb;
use czartool::Gpc1Db;
use czartool::Pantasks;
use czartool::Nebulous;
use czartool::Plotter;
use czartool::DayMetrics;
use czartool::MetricsIndex;

my $period = 60;
my $save_temps = 0;

GetOptions (
        "period|p=s" => \$period,
        );

my $config = new czartool::Config();
my $czarDb = $config->getCzarDbInstance();
my $gpc1Db = $config->getGpc1Instance;
my $gpc2Db = $config->getGpc2Instance;
my $nebulous = new czartool::Nebulous($czarDb);
my $pantasks = new czartool::Pantasks();

# We have a top level directory that gets general plots and then one for each database
my $plotter0 = czartool::Plotter->new_file($config, undef, $config->getGnuplotPath(), $save_temps);

# for now hardcode two databases. Later I'll look into handling a variable number
unless (-d $config->getGnuplotPath() . "/gpc1") {
    unless (mkdir($config->getGnuplotPath() . "/gpc1")) {
	die "Cannot create output directory " . $config->getGnuplotPath() . "/gpc1\n";
    }
}
unless (-d $config->getGnuplotPath() . "/gpc2") {
    unless (mkdir($config->getGnuplotPath() . "/gpc2")) {
	die "Cannot create output directory " . $config->getGnuplotPath() . "/gpc2\n";
    }
}
my $plotter1 = czartool::Plotter->new_file($config, $gpc1Db, $config->getGnuplotPath() . "/gpc1", $save_temps);
my $plotter2 = czartool::Plotter->new_file($config, $gpc2Db, $config->getGnuplotPath() . "/gpc2", $save_temps);

my @stages = (
        "chip",
        "cam",
        "fake",
        "warp",
        "stack",
        "staticsky",
        "diff",
        "fullforce",
        "dist",
        "pub",
        "summitExp",
        "downloadExp",
        "newExp",
        "rawExp");



my @ippToPspsStages = (
        "processed",
        "loaded_to_datastore",
        "loaded_to_ODM",
        "merge_worthy",
        "deleted_datastore",
        "deleted_dxlayer",
        "merged",
        "deleted_local");

# The ippToPsps pages are no longer operational
@ippToPspsStages = ();

my @allStages = (@stages, @ippToPspsStages);

timePoll($period);

###########################################################################
#
# Updates the dates from pantasks for all interested servers
#
###########################################################################
sub updateDates {

    print "* Updating dates\n";
    my @servers = ("stdscience", "registration");

    my $server = undef;
    $czarDb->emptyServerDates();
    foreach $server (@servers) {

        my @dates = @{$pantasks->getDates($server,'gpc1')};
        if (@dates) {
            $czarDb->updateServerDates($server, \@dates,'gpc1');
        }
        else {
            print "WARNING: No dates to update for '$server' in gpc1\n";
        }

        my @dates2 = @{$pantasks->getDates($server,'gpc2')};
        if (@dates2) {
            $czarDb->updateServerDates($server, \@dates2,'gpc2');
        }
        else {
            print "WARNING: No dates to update for '$server' in gpc2\n";
        }
    }
}

###########################################################################
#
# Updates the labels from pantasks for all interested servers
#
###########################################################################
sub updateLabels {
    my ($dbupdate) = @_;

    # this used to have a concept of labels grouped by pantasks server, but we 
    # don't really care about that.  Just get the labels from the pantasks listed below, 
    # and update them all into the 'stdscience' block:

    print "* Updating labels\n";
    my @servers = ("stdscience", "distribution");

    my @allLabels = ();
    my @allLabels2 = ();
    my @fields = ();

    #first grab the labels fom the relevant pantasks
    my $server = undef;
    foreach $server (@servers) {
        my @labels = @{$pantasks->getLabels($server,'gpc1')};
	push (@allLabels, @labels);

        my @labels2 = @{$pantasks->getLabels($server,'gpc2')};
	push (@allLabels2, @labels2);
    }
    #The way gpc2 pantasks are set up is different from gpc1. Therefore, the same label can appear in both stdscience and distribution. 
    @allLabels = uniq @allLabels;
    @allLabels2 = uniq @allLabels2;

    ## adding extra labels from external file
    my $filename = 'extra_labels.dat';
    if (open(my $fh, '<:encoding(UTF-8)', $filename)) {
      my $header = <$fh>;     
      $header = <$fh>;     
      while (my $row = <$fh>) {
          chomp $row;
          @fields = split(' ', $row);
          if ($fields[1] eq 'gpc1') {
	      push (@allLabels, $fields[0]);
          }
          if ($fields[1] eq 'gpc2') {
	      push (@allLabels2, $fields[0]);
          }
      }
    } else {
      warn "No extra labels found in '$filename' $!";
    }

    # Now, grab the priorities for each label
    my @priorities = ();
    my $row = undef;
    my $label = undef;
    my $priority = undef;
    foreach $label ( @allLabels ) {
        $priority = $gpc1Db->getPriority($label);
        push(@priorities, $priority);
    }

    #also grab gpc2 priorities from the gpc2 Label table
    my @priorities2 = ();
    my $priority2 = undef;
    foreach $label ( @allLabels2 ) {
        $priority2 = $gpc2Db->getPriority($label);
        push(@priorities2, $priority2);
    }

    #sort labels by priority
    my @idx = sort {$priorities[$a] <=> $priorities[$b]} 0 ..$#priorities;
    @allLabels = @allLabels[@idx];
    @priorities = @priorities[@idx]; 

    my @idx2 = sort {$priorities2[$a] <=> $priorities2[$b]} 0 ..$#priorities2;
    @allLabels2 = @allLabels2[@idx2];
    @priorities2 = @priorities2[@idx2]; 

    # store them all in stdscience:
    if($dbupdate == 1) {
        $server = "stdscience";
        if (@allLabels) {
	    $czarDb->updateCurrentLabels($server, \@allLabels,'gpc1', \@priorities);
        } else {
	    print "WARNING: No labels to update for gpc1 for '$server'\n";
        }

        if (@allLabels2) {
	    $czarDb->updateCurrentLabels($server, \@allLabels2,'gpc2', \@priorities2);
        } else {
	    print "WARNING: No labels to update for gpc2 for '$server'\n";
        }
    }
    return(\@allLabels,\@allLabels2);
}

###########################################################################
#
# Updates pantasks server status TODO should really get info for all servers at once
#
###########################################################################
sub updateServerStatus {
    print "* Checking all pantasks servers\n";

    my $servers = $pantasks->getServerList();

    my $server = undef;
    my $alive = undef;
    my $running = undef;
    foreach $server (@{$servers}) {
        $pantasks->getServerStatus($server,'gpc1', \$alive, \$running);
        $czarDb->updateServerStatus($server,'gpc1', $alive, $running);
        $pantasks->getServerStatus($server,'gpc2', \$alive, \$running);
        $czarDb->updateServerStatus($server,'gpc2', $alive, $running);
   }
}

###########################################################################
#
# Updates pantasks server status TODO should really get info for all servers at once
#
###########################################################################
sub updateServerStatusLive {
    print "* Checking all pantasks servers\n";

    my $servers = $pantasks->getServerList();

    my $server = undef;
    my $alive = undef;
    my $running = undef;
    foreach $server (@{$servers}) {
        $pantasks->getServerStatus($server,'gpc1', \$alive, \$running);
        $czarDb->updateServerStatusLive($server,'gpc1', $alive, $running);
        $pantasks->getServerStatus($server,'gpc2', \$alive, \$running);
        $czarDb->updateServerStatusLive($server,'gpc2', $alive, $running);
   }
}

###########################################################################
#
# Polls with provided period (seconds)
#
###########################################################################
sub timePoll {
    my ($period) = @_;

    my $dbupdate;
    my $polldate;
    my $datenow;
    my $timediff;
    my $label;
    my $new;
    my $full;
    my $faults;
    my $stage;
    my $query = undef;
    my $str = undef;
    my $labels = undef;
    my $labels2 = undef;
    my $updateLabels = undef;
    my $row = undef;
    my $begin = undef;
    my $end = undef;
    my $priority = undef;
    my $priority2 = undef;
    my $newState = undef;
    my $nsStatus = undef;
    my $today = undef;
    my $yesterday = undef;
    my $newDayTime = $config->getMetricsStartTime();
    my $lastDayDailyTasks = "2010-01-01";
    my $checkneb = 1; # 1 for nebulous, 0 for localhost test
    # main polling loop
    while (1) {

        my $iterationStartTime = time();
 
        #get the time of last czarpoll update
        $polldate = $czarDb->getPollDate();
        $polldate=~/^(\d{4})\-(\d{2})\-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/;
        my $date=DateTime->new(year=>$1,month=>$2,day=>$3,hour=>$4,minute=>$5,second=>$6,time_zone=>"HST");
        $date->set_time_zone("UTC");

        $datenow=DateTime->now;
        $timediff = ($datenow->epoch) - ($date->epoch); 
        if($timediff > 120.) { 
            print "There is no recent Czarpoll iteration\n";
            print "* Updating database entries*\n";
            $dbupdate = 1;
        } else {
            print "There is a recent Czarpoll iteration already made ($timediff sec ago)\n";
            print "* Skipping database queries and entries\n";
            $dbupdate = 0;
        }


        # sort out times
        $today = strftime('%Y-%m-%d', localtime);
        $yesterday = $czarDb->subtractInterval($today, "1 DAY");
        $end = $czarDb->getNowTimestamp();

        if ($czarDb->isBefore($end, "$today $newDayTime")) {
            # if before $newDayTime today (configured to 18:00), then start plots from $newDayTime yesterday
            $begin = "$yesterday $newDayTime";
        } else {
            # if after $newDayTime today (configured to 18:00), then perform some daily tasks and start plots from $newDayTime  today

            $begin = "$today $newDayTime";

            # check whether yesterday was cleaned. if not, cleanup tables and optimize
            if ($lastDayDailyTasks ne $yesterday) {

                print "* performing daily tasks (clean-up, metrics)\n";

                # create metrics for last 24 hours
                print "* Creating metrics for last 24 hours\n";
                my $dayMetrics = new czartool::DayMetrics($config, $gpc1Db, 1, 0, $today);
                $dayMetrics->writeHTML();

                # now update metrics index page
                my $metricsIndex = new czartool::MetricsIndex($config, $gpc1Db, 1, 0);
                $metricsIndex->writeHTML();

                # now cleanup tables from yesterday and optimize
                #TdB20190703: The ipp113 and ipp117 instances of czarpoll use the same stage DBs in czarDB. 
                #Therefore, it makes no sense to run the cleanup+optimising twice
                print "* Performing database cleanup\n";
                $czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval());
                #print "* Optimizing the database\n";
                #$czarDb->optimize();
                $lastDayDailyTasks = $yesterday;
            }
        }

        # check nightly science status
        if($dbupdate == 1) {
            print "* Checking nightly science status\n";
            if (!$pantasks->getNightlyScienceStatus(\$nsStatus,'gpc1')) {$nsStatus = "Unknown";}
            $czarDb->updateNightlyScience($nsStatus,'gpc1');
            if (!$pantasks->getNightlyScienceStatus(\$nsStatus,'gpc2')) {$nsStatus = "Unknown";}
            $czarDb->updateNightlyScience($nsStatus,'gpc2');
        }

        # check nebulous
        if ($checkneb == 1) {
            print "* Checking Nebulous\n";
            if($dbupdate == 1) {$nebulous->updateClusterSpaceInfo();}
            $plotter0->plotStorageTimeSeries($czarDb->subtractInterval($begin, "1 WEEK") , $end);
            $plotter0->plotDiskUsageHistogram();
            $plotter0->plotDiskUsageHistogramLarge();
        }
        else {
            print "* Locahost, so no checking nebulous\n";
        }

        # check pantasks status
        if($dbupdate == 1) {updateServerStatus();}
        if($dbupdate == 1) {updateServerStatusLive();}

        # check pantasks dates
        if($dbupdate == 1) {updateDates();}

        # check labels
        my ($labels, $labels2) = updateLabels($dbupdate);
        my @labels = @$labels;
        my @labels2 = @$labels2;

        # servers to check
        # my @serversToCheck = ("stdscience", "update");
        my @serversToCheck = ("stdscience");

        my $thisServer = undef;
        foreach $thisServer (@serversToCheck) {

            if ($thisServer eq "update") {$newState = "update";}
            else {$newState = "new";}

            # deal with stdscience labels
            my $size = @{$labels};
            if($size > 0) {
                if($dbupdate == 1) {updateAllStages($thisServer, $newState, $labels,$labels2, $begin, $end);}
                if ($thisServer eq "stdscience") {
                    createPlots($thisServer, $plotter1, $labels, $begin, $end);
                    createPlots($thisServer, $plotter2, $labels2, $begin, $end);
                }
            }
            else { print "* WARNING: no $thisServer labels found in Db\n";}
        }

	my @trashFiles = </tmp/czarplot_gnuplot_*>;
	if ($#trashFiles != -1) { 
	    print "* WARNING: Cleaning " . ($#trashFiles + 1) . " files that were not auto cleaned.\n";
	    foreach my $tFile (@trashFiles) {
		unlink($tFile);
	    }
	}

        my $iterationTime = time() - $iterationStartTime;

        print "done with czarpoll iteration after $iterationTime seconds\n";

        print "--------------------------------------------------------------------------\n";
        if ($iterationTime < $period) {
            print "* Going to sleep\n";
            sleep($period - $iterationTime);
            print "* Waking up\n";
        }
    };
}

###########################################################################
#
# Loops through labels and creates time series and histogram plots
#
###########################################################################
sub createPlots {
    my ($server, $plotter, $rows, $begin, $end) = @_;

    my $stage = undef;
    my $row = undef;

    my $project = $plotter->getProject();
    print "* Generating plots for $project\n";

    # XXX The ippToPsps stuff is not project specific so it should be done somewhere
    # else
    if (scalar @ippToPspsStages) {
        # get ippToPsps labels
        my $ippToPspsLabels = undef;
        $czarDb->getCurrentIppToPspsLabels($begin, $end, \$ippToPspsLabels);
        $plotter->setIppToPspsMode();

        # create ippToPsps labels
        foreach $stage (@ippToPspsStages) {
            foreach $row ( @{$ippToPspsLabels} ) {
                my ($label) = @{$row};

                chomp($label);
                $plotter->createTimeSeries($label, $stage, $begin, $end, 1, 0, 1);
            }
        }

        # create plots for each label for all stages for ippToPsps
        foreach $row ( @{$ippToPspsLabels} ) {
            my ($label) = @{$row};

            $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1);
        }
    }

    # create plots for each label for each stage
    $plotter->setStandardMode();

    #TdB 20190411: We are turning off these plots for the time being. They take up a lot of time and are rarely looked at. If need be, can be generated on the fly using czartool.
    #foreach $stage (@stages) {
    #    foreach $row ( @{$rows} ) {
    #        my ($label) = @{$row};

    #        # skip update labels for now. The plots are not relevant.
    #        next if $label =~ 'ps_ud';

    #        chomp($label);
    #        $plotter->createTimeSeries($label, $stage, $begin, $end, 1, 0, 1);
    #    }
    #}

    my $allServerLabels = undef;

    # create plots for each label for all stages
    #TdB 20190411: We are turning off some these plots for the time being.
    #foreach $row ( @{$rows} ) {
    #    my ($label) = @{$row};

    #  #  # skip update labels for now. The plots are not relevant.
    #  #  next if $label =~ 'ps_ud';
    #  #  # Skip individual label tables for raw-like stages.
    #  # if ($stage =~ /Exp/) { next; } 
    #  #  $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1);

    #    if ($label =~ m/nightlyscience/) {
    #      $plotter->createTimeSeries($label, undef, $begin, $end, 1, 0, 1);
    #    }
    #}

    $allServerLabels = $server."_labels";

    $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 1, 1);

    foreach $stage (@stages) {
        $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 0, 0); # TODO must be a neater way...
    }
}

###########################################################################
#
# Loops through some labels and updates processed/pending/faults in the Db
#
###########################################################################
sub updateAllStages {
    my ($labelServer, $newState, $rows, $rows2, $begin, $end) = @_;

    print "* Updating stage data\n";
    my $totalNew1 = 0;
    my $totalFaults1 = 0;
    my $totalFull1 = 0;
    my $totalNew2 = 0;
    my $totalFaults2 = 0;
    my $totalFull2 = 0;
    my $stage = undef;
    my $reverting = 0;
    my $row = undef;
    my $new = undef;
    my $full = undef;
    my $faults = undef;
    my $server = undef;
    my $state = undef;
    my $today = strftime('%Y-%m-%d', localtime);

    foreach $stage (@stages) {
        $server = $pantasks->getServerForThisStage($stage);
        $pantasks->getRevertStatus($stage,'gpc1', \$reverting);
        $czarDb->updateRevertStatus($stage,'gpc1', $reverting);

        $pantasks->getRevertStatus($stage,'gpc2', \$reverting);
        $czarDb->updateRevertStatus($stage,'gpc2', $reverting);

        print "* Checking labels for $stage stage\n";

	# These have no labels to check, so just get them out of the way fast
	if (($stage eq 'summitExp')||($stage eq 'downloadExp')||($stage eq 'newExp')||($stage eq 'rawExp')) { 
	    ($new,$full,$faults) = $gpc1Db->countRawExposures($today,$stage);
	    $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc1', $new, $full, $faults);

	    ($new,$full,$faults) = $gpc2Db->countRawExposures($today,$stage);
	    $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc2', $new, $full, $faults);

	    next;
	}

        #count the number of exposures under each label, and their faults
        $totalNew1=$totalFaults1=$totalFull1=0;
        foreach $row ( @{$rows} ) {
            my ($label) = $row;

            chomp($label);

            # Move this to a function and pass in the database
            $new = $gpc1Db->countExposures($label, $stage, $newState);
            $full = $gpc1Db->countExposures($label, $stage, "full");
            $faults = $gpc1Db->countFaults($label, $stage, $newState);
            $totalNew1 += $new;
            $totalFull1 += $full;
            $totalFaults1 += $faults;

            $czarDb->insertNewTimeData($stage, $label, 'gpc1', $new, $full, $faults);
        }

        $totalNew2=$totalFaults2=$totalFull2=0;
        foreach $row ( @{$rows2} ) {
            my ($label) = $row;

            chomp($label);

            # Move this to a function and pass in the database
            $new = $gpc2Db->countExposures($label, $stage, $newState);
            $full = $gpc2Db->countExposures($label, $stage, "full");
            $faults = $gpc2Db->countFaults($label, $stage, $newState);
            $totalNew2 += $new;
            $totalFull2 += $full;
            $totalFaults2 += $faults;

            $czarDb->insertNewTimeData($stage, $label, 'gpc2', $new, $full, $faults);
        }

        $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc1', $totalNew1, $totalFull1, $totalFaults1);
        $czarDb->insertNewTimeData($stage, $labelServer."_labels", 'gpc2', $totalNew2, $totalFull2, $totalFaults2);
    }
}

