Changeset 29686

Timestamp:

Nov 5, 2010, 3:54:50 PM (16 years ago)

Author:

rhenders

Message:

major changes to support generaionof IPP metrics

Location:

trunk/tools

Files:

• czarmetrics.pl (added)
• czarplot.pl (modified) (5 diffs)
• czartool/CzarDb.pm (modified) (11 diffs)
• czartool/Gpc1Db.pm (modified) (2 diffs)
• czartool/MySQLDb.pm (modified) (4 diffs)
• czartool/Plotter.pm (modified) (3 diffs)
• czartool/StageMetrics.pm (added)

trunk/tools/czarplot.pl

@@ -9 +9 @@
 use czartool::CzarDb;
 use czartool::Plotter;
+use czartool::StageMetrics;
 
 my $czarDbName = "czardb";
@@ -57 +58 @@
     print "* UNKNKOWN: option                          @ARGV\n";
 }
+if ($analysis && !$stage) {
+    $quit = 1;
+    print "* REQUIRED: choose a stage for analsis      -s <chip|cam|warp|etc>      (default=none)\n";}
 if (!$histogram) {
     print "* OPTIONAL: plot histogram                  -h                          (default=off)\n";}
@@ -98 +102 @@
 
 # default values
-if (!$nebulous && !$histogram && !$timeSeries) {$timeSeries = 1; $histogram = 0;}
+if (!$nebulous && !$histogram && !$analysis && !$timeSeries) {$timeSeries = 1; $histogram = 0;}
 if (!$verbose) {$verbose = 0;}
 if (!$save_temps) {$save_temps = 0;}
@@ -121 +125 @@
     $begin =  "$day 06:35";
     $end = "$day 23:59";
+        print "JKJKJK '$begin' '$end' \n";
+
 }
 else {
@@ -147 +153 @@
 if($analysis) {
 
-    my ($started, $finished, $stuck, $processed, $pending, $faults, $totalTime);
-    $czarDb->runAnalysis($label, $stage, $begin, $end, \$started, \$finished, \$stuck, \$processed, \$pending, \$faults, \$totalTime);
-
-    if (defined $started) {print "* Processing started at $started\n";}
-    if (defined $finished) {print "* Processing finished at $finished and took $totalTime\n";}
-    else {print "* Processing has not finished\n";}
-    if (defined $stuck) {print "* Processing has been stuck since $stuck\n";}
-    print "* $processed exposures have been processed, with $pending pending and $faults faults\n";
-
-    print "*******************************************************************************\n";
+    my $stageMetrics = new czartool::StageMetrics($stage, $label, $begin, $end);
+    if ($czarDb->runAnalysis($stageMetrics)) {$stageMetrics->printMe();}
 }

trunk/tools/czartool/CzarDb.pm

@@ -10 +10 @@
 use base 'czartool::MySQLDb';
 our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
+
+use czartool::StageMetrics;
 
 ###########################################################################
@@ -49 +51 @@
     return 1;
 }
+
+###########################################################################
+#
+# Gets stdscience lables active in the provided time period
+# TODO ugly hack to avoind getting update labels
+#
+###########################################################################
+sub getStdscienceLabelsInTimePeriod {
+    my ($self, $begin, $end, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT DISTINCT label
+        FROM chip 
+        WHERE timestamp >= '$begin' 
+        AND timestamp <= '$end' 
+        AND label NOT LIKE '%_ud%'
+        AND label NOT LIKE 'update%'
+        AND label NOT LIKE 'all_%_labels'
+SQL
+
+    if (!$query->execute) {return 0;}
+
+    ${$labels} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+
 
 ###########################################################################
@@ -264 +295 @@
 }
 
+###########################################################################
+#
+# Returns the start and end times during one day for the provided label and start/end stages
+#
+###########################################################################
+sub getDayTimings {
+    my ($self, $label, $startStage, $endStage, $begin, $started, $finished, $timeTaken) = @_;
+
+    my $end = $self->addInterval($begin, "1 DAY");
+
+    my $startStageMetrics = new czartool::StageMetrics($startStage, $label, $begin, $end);
+    my $endStageMetrics = new czartool::StageMetrics($endStage, $label, $begin, $end);
+
+    if (!$self->runAnalysis($startStageMetrics)) {return 0;}
+    if (!$self->runAnalysis($endStageMetrics)) {return 0;}
+
+    ${$started} = $startStageMetrics->getStarted();
+    ${$finished} = $endStageMetrics->getFinished100();
+
+    if (!${$started} || !${$finished}) {${$timeTaken} = undef;}
+    else {${$timeTaken} = $self->diffTimes(${$finished}, ${$started});}
+
+    return 1;
+}
 
 ###########################################################################
@@ -271 +326 @@
 ###########################################################################
 sub runAnalysis {
-    my ($self, 
-            $label, 
-            $stage, 
-            $fromTime, 
-            $toTime, 
-            $_started, 
-            $_finished, 
-            $_stuck,
-            $_processed,
-            $_pending,
-            $_faults,
-            $_totalTime) = @_;
+    my ($self, $stageMetrics) = @_;
+
+    if (!$stageMetrics) {return 0;}
+
+    my $stage = $stageMetrics->getStage();
+    my $label = $stageMetrics->getLabel();
+    my $fromTime = $stageMetrics->getStartTime();
+    my $toTime = $stageMetrics->getEndTime();
+
+    $toTime = $self->addInterval($toTime, "20 MINUTE"); # TODO dodgy
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -293 +346 @@
 SQL
 
-    $query->execute;
-
     my $lastProcessed = -1;
     my $lastPending = -1;
@@ -300 +351 @@
     my $lastTimestamp = undef;
     my ($linearProcessed, $linearPending, $linearFaults);
+
+    # first, count toatl processed in time window so that we can find time of 50%, 75%, 90% etc
+    $query->execute;
+    my $totalProcessed = 0;
+    my $firstRow = 1;
+    while (my @row = $query->fetchrow_array()) {
+        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
+
+        # get linear values
+        if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
+            
+            $totalProcessed = $totalProcessed + ($thisProcessed - $lastProcessed);
+        }
+
+        $lastProcessed = $thisProcessed;
+        if ($firstRow) {$stageMetrics->setInitialPending($thisPending); $firstRow = 0;}
+    }
+
+    # nothing to do here
+    if (!$totalProcessed) {return 1;}
+
+    my $percent25 = $totalProcessed * 0.25;
+    my $percent50 = $totalProcessed * 0.5;
+    my $percent75 = $totalProcessed * 0.75;
+    my $percent90 = $totalProcessed * 0.9;
+    my $percent95 = $totalProcessed * 0.95;
+
     my ($derivProcessed, $derivPending, $derivFaults);
-    my $started = undef;
-    my $finished = undef;
-    
     my $notProcessing = undef;
     my $howLongNotProcessing = undef;
     my $notProcessingLongerThanInterval = 0;
-
     my $stuck = undef;
     my $howLongStuck = undef;
     my $stuckLongerThanInterval = 0;
-
     my $stuffToDo = undef;
-    my $interval = "00:20:00";
-
+    my $interval = "00:20:00"; # TODO dodgy - see above
+    my $started = undef;
+    my $finished = undef;
+   
+    my ($have25, $have50, $have75, $have90, $have95, $have100) = 0;
+
+    # now, loop again, this time recording more data
+    $query->execute;
     $linearProcessed = 0;
+    $lastProcessed = -1;
     while (my @row = $query->fetchrow_array()) {
         my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
@@ -326 +406 @@
         $linearPending = $thisPending;
         $linearFaults = $thisFaults;
+
+        if (!$have25 && $linearProcessed >= $percent25) {$stageMetrics->setFinished25($thisTimestamp); $have25 = 1;}
+        if (!$have50 && $linearProcessed >= $percent50) {$stageMetrics->setFinished50($thisTimestamp); $have50 = 1;}
+        if (!$have75 && $linearProcessed >= $percent75) {$stageMetrics->setFinished75($thisTimestamp); $have75 = 1;}
+        if (!$have90 && $linearProcessed >= $percent90) {$stageMetrics->setFinished90($thisTimestamp); $have90 = 1;}
+        if (!$have95 && $linearProcessed >= $percent95) {$stageMetrics->setFinished95($thisTimestamp); $have95 = 1;}
 
         # get first derivative values
@@ -363 +449 @@
                 if (defined $notProcessing) {
                     
-                    if ($stuckLongerThanInterval) {
-
-#                        print "* $stage procesing was stuck for  $howLongStuck, from $stuck until $lastTimestamp\n";
-                    }
-                    if ($notProcessingLongerThanInterval) {
-
-            #            print " * $stage not procesing for  $howLongNotProcessing, from $notProcessing until $thisTimestamp\n";
-                    }
                     $notProcessing = undef; 
                     $stuck = undef; 
                 }
                 if (!defined $started) {
+
                     $started = $lastTimestamp;
                     $finished = undef; 
-#                    print "* $stage processing started at $started\n";
                 }
             }
@@ -418 +496 @@
     }
 
-    ${$_processed} = $linearProcessed;
-    ${$_pending} = $stuffToDo;
-    ${$_faults} = $lastFaults;
+    $stageMetrics->setProcessed($linearProcessed);
+    $stageMetrics->setFinalPending($stuffToDo);
+    $stageMetrics->setFaults($lastFaults);
+
+
+    if (!defined $finished) {
+
+        $stageMetrics->setFinished25(undef);
+        $stageMetrics->setFinished50(undef);
+        $stageMetrics->setFinished75(undef);
+        $stageMetrics->setFinished90(undef);
+        $stageMetrics->setFinished95(undef);
+        $stageMetrics->setFinished100(undef);
+        $stageMetrics->setTotalTime($self->diffTimes($lastTimestamp, $started));
+    }
 
     if (defined $finished) {
-        ${$_finished} = $finished;
-        ${$_totalTime} = $self->diffTimes($finished, $started);
-    }
-
-    elsif(defined $started) {
-    
-        ${$_finished} = undef;
-    }
-    
+
+        $stageMetrics->setFinished100($finished);
+        $stageMetrics->setTotalTime($self->diffTimes($finished, $started));
+    }
+
+
     if (!defined $started){
     
-        ${$_started} = undef;
+        $stageMetrics->setStarted(undef);
     }
     else {
     
-        ${$_started} = $started;
+        my $rate = ($stageMetrics->getProcessed()/$self->getTimeInSecs($stageMetrics->getTotalTime) ) * 3600.0;
+        $stageMetrics->setRate($rate);
+        $stageMetrics->setStarted($started);
     }
     if ($stuck && $stuckLongerThanInterval && !defined $finished) {
 
-        ${$_stuck} = $stuck;
+        $stageMetrics->setStuck($stuck);
     }
     else {
     
-        ${$_stuck} = undef;
+        $stageMetrics->setStuck(undef);
     }
 
@@ -517 +606 @@
             my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
                 
-            if ($thisProcessed >= $lastProcessed){
+            if ($thisProcessed >= $lastProcessed && $timeDiff){
                 $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
             }
@@ -1512 +1601 @@
     $self->setRevision(12);
 }
-
 1;
 

trunk/tools/czartool/Gpc1Db.pm

@@ -56 +56 @@
 
     return "ERROR";
+}
+
+###########################################################################
+#
+# Returns the number of science exposures taken 'last night' for a given day
+# 
+# Notes:
+#
+# - 03:00 HST = 17:00 last night UTC
+# - 17:00 HST = 07:00 this morning UTC
+# - 'OBJECT' = science exposures
+#
+###########################################################################
+sub countScienceExposuresFromLastNight {
+    my ($self, $day) = @_;
+
+    my $begin = "$day 03"; 
+    my $end = "$day 17"; 
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*)
+        FROM summitExp 
+        WHERE dateobs > '$begin' 
+        AND dateobs < '$end' AND
+        exp_type = 'OBJECT';
+SQL
+
+    $query->execute;
+return scalar $query->fetchrow_array();
 }
 
@@ -158 +187 @@
     }
 
-
     $query->execute;
     return scalar $query->fetchrow_array();

trunk/tools/czartool/MySQLDb.pm

@@ -98 +98 @@
 ###########################################################################
 #
+# Subtracts the provided interval from the provided time
+#
+###########################################################################
+sub subtractInterval {
+    my ($self, $time, $interval) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT '$time' - INTERVAL $interval; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
 # Finds the difference of two times
 #
@@ -104 +120 @@
     my ($self, $time1, $time2) = @_;
 
-      my $query = $self->{_db}->prepare(<<SQL);
+    if (!$time1 || !$time2) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
          SELECT TIMEDIFF('$time1','$time2'); 
+SQL
+
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns the provided time in seconds
+#
+###########################################################################
+sub getTimeInSecs {
+    my ($self, $time) = @_;
+
+    if (!$time) {return 0;}
+
+      my $query = $self->{_db}->prepare(<<SQL);
+         SELECT TIME_TO_SEC('$time'); 
 SQL
 
@@ -120 +156 @@
     my ($self, $time1, $time2) = @_;
 
-      my $query = $self->{_db}->prepare(<<SQL);
+    if (!$time1 || !$time2) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
          SELECT TIME_TO_SEC(TIMEDIFF('$time1','$time2')); 
 SQL
@@ -138 +176 @@
       my $query = $self->{_db}->prepare(<<SQL);
           SELECT INTERVAL('$interval1', '$interval2'); 
-SQL
-    $query->execute;
-
-return scalar $query->fetchrow_array();
-}
-
-###########################################################################
-#
-# Subtracts the provided interval from the provided time
-#
-###########################################################################
-sub subtractInterval {
-    my ($self, $time, $interval) = @_;
-
-      my $query = $self->{_db}->prepare(<<SQL);
-          SELECT '$time' - INTERVAL $interval; 
 SQL
     $query->execute;

trunk/tools/czartool/Plotter.pm

@@ -67 +67 @@
     my $stages = undef;                 
 
-    if (!$selectedStage) {$stages = \@allStages;}
+    if (!$selectedStage) {
+        $stages = \@allStages;
+        $selectedStage = 'all_stages';
+    }
     else {$stages = ["$selectedStage"]};        
 
-    $self->{_czarDb}->getTimeMinMaxDiff($label, $selectedStage, $beginTime, $endTime, \$minX, \$maxX, \$timeDiff);
+    $self->{_czarDb}->getTimeMinMaxDiff($label, "chip", $beginTime, $endTime, \$minX, \$maxX, \$timeDiff);
 
     my $divX;
@@ -95 +98 @@
 #        "set xrange [\"$minX\":\"$maxX\"];" .
         "set boxwidth;" .
+        "set xtic rotate by -90 scale 0;" .
         "set style data histogram;" .
         "set style histogram rowstacked;" .
@@ -124 +128 @@
     print GP ";\n";
     close GP;
-}                                                    
+}
+
 ###########################################################################
 #