Changeset 29279 for trunk/tools/roboczar.pl

Timestamp:

Sep 30, 2010, 9:20:25 AM (16 years ago)

Author:

rhenders

Message:

New roboczar tool to inform users when processing goes awry

File:

• trunk/tools/roboczar.pl (modified) (2 diffs)

trunk/tools/roboczar.pl

@@ -14 +14 @@
 use czartool::Burntool;
 
-my $period = 60;
 my $czarDbName = "czardb"; # TODO variables for other Db stuff, host etc
 my $save_temps = 0;
 
 GetOptions (
-        "period|p=s" => \$period, # TODO more Db args
         "dbname|d=s" => \$czarDbName,
         );
 
 my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg
-my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
-my $nebulous = new czartool::Nebulous($czarDb);
-my $pantasks = new czartool::Pantasks();
-my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
-my $burntool = new czartool::Burntool();
 
 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
 
 my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
+my @serversWeCareAbout = ("stdscience", "distribution", "summitcopy", "registration");
 
 
-timePoll($period);
+while(1) {
 
-###########################################################################
-#
-# Updates the labels from pantasks for all interested servers 
-#
-###########################################################################
-sub updateLabels {
-
-    print "* Updating labels\n";
-    my @servers = ("stdscience", "distribution", "publishing", "update");
-
-    my $server = undef;
-    foreach $server (@servers) {
-
-        my @labels = @{$pantasks->getLabels($server)};
-        if (@labels) { 
-        
-            $czarDb->updateCurrentLabels($server, \@labels);
-        }
-        else {
-        
-             print "WARNING: No labels to update for '$server'\n";
-        }
-    }
+    checkServers("20 MINUTE");
+    sleep(1200);
 }
 
 ###########################################################################
 #
-# Updates pantasks server status TODO should really get info for all servers at once 
+# Checks tha the important servers are running 
 #
 ###########################################################################
-sub updateServerStatus {
-    print "* Checking all pantasks servers\n";
+sub checkServers {
+    my ($interval) = @_;
 
-    my $servers = $pantasks->getServerList();
+    my $server;
+    foreach $server (@serversWeCareAbout) {
 
-    my $server = undef;
-    my $alive = undef;
-    my $running = undef;
-    foreach $server (@{$servers}) {
+        if ($czarDb->isServerDown($server, $interval)) {
 
-        $pantasks->getServerStatus($server, \$alive, \$running);
-        $czarDb->updateServerStatus($server, $alive, $running);
+            print "$server has been down for the last $interval\n";
+            sendEmail(
+                    "roydhenderson\@gmail.com", 
+                    "roboczar\@ipp.com", 
+                    "Roboczar update", 
+                    "\n\n* '$server' server has been down for the last $interval\n\n");
+        }
+        else {
+            #print "$server has been running for some of the last $interval\n";
+        }
     }
-}
+} 
 
 ###########################################################################
 #
-# Polls with provided period (seconds) 
+#  Checks the status of a given label
 #
 ###########################################################################
-sub timePoll {
-    my ($period) = @_;
-
-    my $label;
-    my $new;
-    my $full;
-    my $faults;
-    my $stage;
-    my $query = undef;
-    my $str = undef;
-    my $labels = undef;
-    my $updateLabels = undef;
-    my $row = undef;
-    my $begin = undef;
-    my $end = undef;
-    my $priority = undef;
-    my $newState = undef;
-    my $nsStatus = undef;
-
-    while (1) {
-
-        # sort out times
-        $begin =  strftime('%Y-%m-%d 06:35',localtime);
-        $end = $czarDb->getNowTimestamp();
-
-        if ($czarDb->isBefore($end, $begin)) {
-
-            $begin = $czarDb->subtractInterval($begin, "1 DAY");
-        }
-
-        # check nightly science status
-        print "* Checking nightly science status\n";
-        if (!$pantasks->getNightlyScienceStatus(\$nsStatus)) {$nsStatus = "Unknown";}
-        $czarDb->updateNightlyScience($nsStatus);
-
-        # check nebulous
-        print "* Checking Nebulous\n";
-        $nebulous->updateClusterSpaceInfo();
-        $plotter->plotDiskUsageHistogram();
-        updateServerStatus();
-
-        # check labels
-        updateLabels();
-
-        # servers to check
-        my @serversToCheck = ("stdscience", "update");
-
-        my $thisServer = undef;
-        foreach $thisServer (@serversToCheck) {
-
-            if ($thisServer eq "update") {$newState = "update";}
-            else {$newState = "new";}
-
-            # deal with stdscience labels
-            if (!$czarDb->getCurrentLabels($thisServer, \$labels)) {next;}
-            my $size = @{$labels};
-            if($size > 0) {
-
-                # get priority
-                foreach $row ( @{$labels} ) {
-                    my ($label) = @{$row};
-                    $priority = $gpc1Db->getPriority($label);
-                    $czarDb->setLabelPriority($label, $priority);
-                }
-
-                updateAllStages($thisServer, $newState, $labels, $begin, $end);
-                createPlots($thisServer, $labels, $begin, $end);
-            }
-            else { print "* WARNING: no $thisServer labels found in Db\n";}
-        }
-
-        print "--------------------------------------------------------------------------\n";
-        print "* Going to sleep\n";
-        sleep($period);
-        print "* Waking up\n";
-
-        #sendEmail("roydhenderson\@gmail.com", "roboczar\@ipp.com", "Roboczar update", "Some content");
-    };
-}
-
-###########################################################################
-#
-# Loops through labels and creates time series and histogram plots
-#
-###########################################################################
-sub createPlots {
-    my ($server, $rows, $begin, $end) = @_;
-
-    my $stage = undef;
-    my $row = undef;
-
-    print "* Generating plots\n";
-
-    # create plots for each label for each stage
-    foreach $stage (@stages) {
-        foreach $row ( @{$rows} ) {
-            my ($label) = @{$row};
-
-            chomp($label);
-            $plotter->createLogAndLinearTimeSeries($label,  $stage, $begin, $end);
-        }
-    }
-
-    # create plots for each label for all stages
-    foreach $row ( @{$rows} ) {
-        my ($label) = @{$row};
-
-        $plotter->createLogAndLinearTimeSeries($label, undef, $begin, $end);
-        $plotter->createHistogram($label, $begin, $end);
-
-        #routineChecks($label, "1 HOUR");
-    }
-    $plotter->createLogAndLinearTimeSeries("all_".$server."_labels", undef, $begin, $end);
-    $plotter->createHistogram("all_".$server."_labels", $begin, $end);
-    foreach $stage (@stages) {
-
-        $plotter->createLogAndLinearTimeSeries("all_".$server."_labels",  $stage, $begin, $end); # TODO must be a neater way...
-    }
-}
-
-###########################################################################
-#
-# Loops through some labels and updates processed/pending/faults in the Db
-#
-###########################################################################
-sub updateAllStages {
-    my ($labelServer, $newState, $rows, $begin, $end) = @_;
-
-    print "* Updating stage data\n";
-    my $totalNew = undef;
-    my $totalFaults = undef;
-    my $totalFull = undef;
-    my $stage = undef;
-    my $reverting = 0;
-    my $row = undef;
-    my $new = undef;
-    my $full = undef;
-    my $faults = undef;
-    my $server = undef;
-    my $state = undef;
-    
-    foreach $stage (@stages) {
-
-        $server = $pantasks->getServerForThisStage($stage);
-        $pantasks->getRevertStatus($stage, \$reverting);
-        $czarDb->updateRevertStatus($stage, $reverting);
-
-        print "* Checking labels for $stage stage\n";
-
-        $totalNew=$totalFaults=$totalFull=0;
-        foreach $row ( @{$rows} ) {
-            my ($label) = @{$row};
-
-            chomp($label);
-
-            if ($stage eq "burntool") {
-
-                if ($labelServer eq "stdscience") {
-
-                    $burntool->getPendingAndProcessed($label, \$new, \$full);
-                    $faults = 0;
-                }
-                else { $new = $full = $faults = 0;}
-            }
-            else {
-
-                $new = $gpc1Db->countExposures($label, $stage, $newState);
-                $full = $gpc1Db->countExposures($label, $stage, "full");
-                $faults = $gpc1Db->countFaults($label, $stage, $newState);
-            }
-            #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
-            $totalNew += $new;
-            $totalFull += $full;
-            $totalFaults += $faults;
-
-            $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults);
-        }
-
-        $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", $totalNew, $totalFull, $totalFaults);
-    }
-}
-
-###########################################################################
-#
-# Performs some routine checks on processing status and sends alerts if it needs to 
-#
-###########################################################################
-sub routineChecks {
+sub checkLabel {
     my ($label, $interval) = @_;
 
@@ -305 +97 @@
         }
     }
+    #sendEmail("roydhenderson\@gmail.com", "roboczar\@ipp.com", "Roboczar update", "Some content");
 }