Changeset 29834

Timestamp:

Nov 26, 2010, 10:33:29 AM (15 years ago)

Author:

eugene

Message:

merge updates from trunk

Location:

branches/eam_branches/ipp-20101103/tools

Files:

• checkfits (copied) (copied from trunk/tools/checkfits )
• czarmetrics.pl (copied) (copied from trunk/tools/czarmetrics.pl )
• czarplot.pl (modified) (15 diffs)
• czarpoll.pl (modified) (7 diffs)
• czartool/CzarDb.pm (modified) (22 diffs)
• czartool/DayMetrics.pm (copied) (copied from trunk/tools/czartool/DayMetrics.pm )
• czartool/Gpc1Db.pm (modified) (2 diffs)
• czartool/Metrics.pm (copied) (copied from trunk/tools/czartool/Metrics.pm )
• czartool/MySQLDb.pm (modified) (7 diffs)
• czartool/OCMetrics.pm (copied) (copied from trunk/tools/czartool/OCMetrics.pm )
• czartool/Plotter.pm (modified) (20 diffs)
• czartool/StageMetrics.pm (copied) (copied from trunk/tools/czartool/StageMetrics.pm )
• heather (copied) (copied from trunk/tools/heather )
• heather/reprocess.OSS.133.pl (copied) (copied from trunk/tools/heather/reprocess.OSS.133.pl )
• heather/reprocess.OSS.pl (copied) (copied from trunk/tools/heather/reprocess.OSS.pl )
• roboczar.pl (modified) (3 diffs)
• runchipimfile.pl (modified) (4 diffs)

branches/eam_branches/ipp-20101103/tools/czarplot.pl

@@ -8 +8 @@
 
 use czartool::CzarDb;
+use czartool::Gpc1Db;
 use czartool::Plotter;
+use czartool::StageMetrics;
 
 my $czarDbName = "czardb";
@@ -14 +16 @@
 my $stage = undef;
 my $save_temps = undef;
-my $interval = undef;
+my $timeinpast = undef;
 my $begin = undef;
 my $end = undef;
@@ -23 +25 @@
 my $timeSeries = undef;
 my $rate = undef;
+my $magicMask = undef;
+my $rateInterval = undef;
 my $deriv = undef;
 my $showCleanup = undef;
@@ -28 +32 @@
 my $savingToFile = undef;
 my $analysis = undef;
+my $exposureId = undef;
 my $log = undef;
 
@@ -34 +39 @@
         "label|l=s" => \$label,
         "stage|s=s" => \$stage,
-        "interval|i=s" => \$interval,
+        "timeinpast|p=s" => \$timeinpast,
+        "rateinterval|i=s" => \$rateInterval,
+        "exposureid|x=s" => \$exposureId,
+        "mask|m" => \$magicMask,
         "begin|b=s" => \$begin,
         "end|e=s" => \$end,
@@ -57 +65 @@
     print "* UNKNKOWN: option                          @ARGV\n";
 }
+if ($analysis && !$stage) {
+    $quit = 1;
+    print "* REQUIRED: choose a stage for analsis      -s <chip|cam|warp|etc>      (default=none)\n";}
 if (!$histogram) {
     print "* OPTIONAL: plot histogram                  -h                          (default=off)\n";}
 if (!$timeSeries) {
     print "* OPTIONAL: plot timeseries                 -t                          (default=on)\n";} 
+if (!$magicMask) {
+    print "* OPTIONAL: plot magic mask for these times -m                          (default=off)\n";} 
+if (!$exposureId) {
+    print "* OPTIONAL: set exposure ID for magic mask  -x                          (default=none)\n";} 
 if (!$rate) {
-    print "* OPTIONAL: plot timeseries of rate         -r                          (default=off)\n";} 
+    print "* OPTIONAL: plot histogram of rate          -r                          (default=off)\n";} 
 if (!$deriv) {
     $deriv = 0;
@@ -74 +89 @@
 if (!$log) {
     $log = 0;
-    print "* OPTIONAL: use log plots                   -g                          (default=$log\n";}
+    print "* OPTIONAL: use log plots                   -g                          (default=$log)\n";}
 if (!$nebulous) {
     print "* OPTIONAL: plot nebulous disk space        -n                          (default=off)\n";} 
@@ -82 +97 @@
 if (!$stage) {
     print "* OPTIONAL: choose a stage                  -s <chip|cam|warp|etc>      (default=none)\n";}
-if (!$interval) {
-    print "* OPTIONAL: choose time interval in past    -i <'1 hour'|'1 day'|etc>   (default=none\n";} 
+if (!$timeinpast) {
+    print "* OPTIONAL: choose time interval in past    -p <'1 hour'|'1 day'|etc>   (default=none\n";} 
+if (!$rateInterval) {
+    print "* OPTIONAL: time interval for rate plot     -i <'1 hour'|'1 day'|etc>   (default=depends on time frame)\n";} 
 if (!$begin) {
     print "* OPTIONAL: choose a begin time             -b <datetime>               (default=6:35am this morning)\n";} 
@@ -98 +115 @@
 
 # default values
-if (!$nebulous && !$histogram && !$timeSeries) {$timeSeries = 1; $histogram = 0;}
+if (!$rate && !$magicMask && !$nebulous && !$histogram && !$analysis && !$timeSeries) {$timeSeries = 1;}
 if (!$verbose) {$verbose = 0;}
 if (!$save_temps) {$save_temps = 0;}
@@ -104 +121 @@
 else {$savingToFile = 1;}
 
+my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
 my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", $verbose, $save_temps);
 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
@@ -109 +127 @@
 
 my $plotter = new czartool::Plotter(
+        $gpc1Db, 
         $czarDb, 
         "%Y%m%d-%H%M%S", 
@@ -121 +140 @@
     $begin =  "$day 06:35";
     $end = "$day 23:59";
+        print "JKJKJK '$begin' '$end' \n";
+
 }
 else {
@@ -127 +148 @@
     if (!$begin) {
 
-        if ($interval) {$begin = $czarDb->subtractInterval($end, $interval);}
+        if ($timeinpast) {$begin = $czarDb->subtractInterval($end, $timeinpast);}
         else {$begin =  strftime('%Y-%m-%d 06:35',localtime);}
     }
@@ -134 +155 @@
 
 if ($rate) {
-
-    if (!$interval) {$interval = "1 HOUR";}
-    $plotter->createRateTimeSeries($label, $stage, $begin, $end, $interval, $log);
+    $plotter->createRateTimeSeries($label, $stage, $begin, $end, $rateInterval, $log);
     exit;
 }
@@ -143 +162 @@
 if ($nebulous && $timeSeries) {$plotter->plotStorageTimeSeries($begin, $end);}
 elsif ($nebulous) {$plotter->plotDiskUsageHistogram();}
+if ($magicMask) {
+    
+    if ($exposureId) {$plotter->plotMagicMaskFractionForThisExposure($exposureId);}
+    else {$plotter->plotMagicMaskFractionHistogram($begin, $end);}
 
+}
 
 if($analysis) {
 
-    my ($started, $finished, $stuck, $processed, $pending, $faults, $totalTime);
-    $czarDb->runAnalysis($label, $stage, $begin, $end, \$started, \$finished, \$stuck, \$processed, \$pending, \$faults, \$totalTime);
-
-    if (defined $started) {print "* Processing started at $started\n";}
-    if (defined $finished) {print "* Processing finished at $finished and took $totalTime\n";}
-    else {print "* Processing has not finished\n";}
-    if (defined $stuck) {print "* Processing has been stuck since $stuck\n";}
-    print "* $processed exposures have been processed, with $pending pending and $faults faults\n";
-
-    print "*******************************************************************************\n";
+    my $stageMetrics = new czartool::StageMetrics($stage, $label, $begin, $end);
+    if ($czarDb->runAnalysis($stageMetrics)) {$stageMetrics->printMe();}
 }

branches/eam_branches/ipp-20101103/tools/czarpoll.pl

@@ -13 +13 @@
 use czartool::Plotter;
 use czartool::Burntool;
+use czartool::DayMetrics;
+
 
 my $period = 60;
@@ -27 +29 @@
 my $nebulous = new czartool::Nebulous($czarDb);
 my $pantasks = new czartool::Pantasks();
-my $plotter = new czartool::Plotter($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
+my $plotter = new czartool::Plotter($gpc1Db, $czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
 my $burntool = new czartool::Burntool();
+
 
 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
@@ -107 +110 @@
     my $lastDay = strftime('%Y-%m-%d', localtime);
     my $today = undef;
+    my $doneMetricsToday = 1;
 
     while (1) {
@@ -118 +122 @@
                 $czarDb->optimize();
                 $lastDay = $today;
+                $doneMetricsToday = 0;
         }
 
@@ -124 +129 @@
         $end = $czarDb->getNowTimestamp();
 
+        # if time now is after 06:35am, then create metrics for past 24 hours
+        if (!$doneMetricsToday && $czarDb->isBefore($begin, $end)) {
+        
+                print "* Creating metrics for last 24 hours\n";
+                # TODO hardcopded path needs to be in config
+                my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $today); 
+                $dayMetrics->writeHTML();
+
+                $doneMetricsToday = 1;
+        }
+
+        # if time now is before 06:35am, include data from previous day
         if ($czarDb->isBefore($end, $begin)) {
 
@@ -197 +214 @@
             chomp($label);
             $plotter->createLogAndLinearTimeSeries($label,  $stage, $begin, $end);
-        }
-    }
+            $plotter->createRateTimeSeries($label, $stage, $begin, $end, undef, 0);
+        }
+    }
+
+    my $allServerLabels = undef;
 
     # create plots for each label for all stages
@@ -205 +225 @@
 
         $plotter->createLogAndLinearTimeSeries($label, undef, $begin, $end);
+        $plotter->createRateTimeSeries($label, undef, $begin, $end, undef, 0);
         $plotter->createHistogram($label, $begin, $end);
-
-        #routineChecks($label, "1 HOUR");
-    }
-    $plotter->createLogAndLinearTimeSeries("all_".$server."_labels", undef, $begin, $end);
-    $plotter->createHistogram("all_".$server."_labels", $begin, $end);
+    }
+
+    $allServerLabels = "all_".$server."_labels";
+
+    $plotter->createLogAndLinearTimeSeries($allServerLabels, undef, $begin, $end);
+    $plotter->createRateTimeSeries($allServerLabels, undef, $begin, $end, undef, 0);
+    $plotter->createHistogram($allServerLabels, $begin, $end);
     foreach $stage (@stages) {
 
-        $plotter->createLogAndLinearTimeSeries("all_".$server."_labels",  $stage, $begin, $end); # TODO must be a neater way...
+        $plotter->createLogAndLinearTimeSeries($allServerLabels,  $stage, $begin, $end); # TODO must be a neater way...
+        $plotter->createRateTimeSeries($allServerLabels, $stage, $begin, $end, undef, 0);
     }
 }

branches/eam_branches/ipp-20101103/tools/czartool/CzarDb.pm

@@ -6 +6 @@
 use strict;
 
+use File::Temp;
+
 my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO put elsewhere
 
 use base 'czartool::MySQLDb';
 our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
+
+use czartool::StageMetrics;
 
 ###########################################################################
@@ -49 +53 @@
     return 1;
 }
+
+###########################################################################
+#
+# Gets stdscience lables active in the provided time period
+# TODO ugly hack to avoind getting update labels
+#
+###########################################################################
+sub getStdscienceLabelsInTimePeriod {
+    my ($self, $begin, $end, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT DISTINCT label
+        FROM chip 
+        WHERE timestamp >= '$begin' 
+        AND timestamp <= '$end' 
+        AND label NOT LIKE '%_ud%'
+        AND label NOT LIKE 'update%'
+        AND label NOT LIKE 'all_%_labels'
+SQL
+
+    if (!$query->execute) {return 0;}
+
+    ${$labels} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+
 
 ###########################################################################
@@ -264 +297 @@
 }
 
+###########################################################################
+#
+# Returns the start and end times during one day for the provided label and start/end stages
+#
+###########################################################################
+sub getDayTimings {
+    my ($self, $label, $startStage, $endStage, $begin, $started, $finished, $timeTaken) = @_;
+
+    my $end = $self->addInterval($begin, "1 DAY");
+
+    my $startStageMetrics = new czartool::StageMetrics($startStage, $label, $begin, $end);
+    my $endStageMetrics = new czartool::StageMetrics($endStage, $label, $begin, $end);
+
+    if (!$self->runAnalysis($startStageMetrics)) {return 0;}
+    if (!$self->runAnalysis($endStageMetrics)) {return 0;}
+
+    ${$started} = $startStageMetrics->getStarted();
+    ${$finished} = $endStageMetrics->getFinished100();
+
+    if (!${$started} || !${$finished}) {${$timeTaken} = undef;}
+    else {${$timeTaken} = $self->diffTimes(${$finished}, ${$started});}
+
+    return 1;
+}
 
 ###########################################################################
@@ -271 +328 @@
 ###########################################################################
 sub runAnalysis {
-    my ($self, 
-            $label, 
-            $stage, 
-            $fromTime, 
-            $toTime, 
-            $_started, 
-            $_finished, 
-            $_stuck,
-            $_processed,
-            $_pending,
-            $_faults,
-            $_totalTime) = @_;
+    my ($self, $stageMetrics) = @_;
+
+    if (!$stageMetrics) {return 0;}
+
+    my $stage = $stageMetrics->getStage();
+    my $label = $stageMetrics->getLabel();
+    my $fromTime = $stageMetrics->getStartTime();
+    my $toTime = $stageMetrics->getEndTime();
+
+    $toTime = $self->addInterval($toTime, "20 MINUTE"); # TODO dodgy
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -293 +348 @@
 SQL
 
-    $query->execute;
-
     my $lastProcessed = -1;
     my $lastPending = -1;
@@ -300 +353 @@
     my $lastTimestamp = undef;
     my ($linearProcessed, $linearPending, $linearFaults);
+
+    # first, count toatl processed in time window so that we can find time of 50%, 75%, 90% etc
+    $query->execute;
+    my $totalProcessed = 0;
+    my $firstRow = 1;
+    while (my @row = $query->fetchrow_array()) {
+        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
+
+        # get linear values
+        if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
+            
+            $totalProcessed = $totalProcessed + ($thisProcessed - $lastProcessed);
+        }
+
+        $lastProcessed = $thisProcessed;
+        if ($firstRow) {$stageMetrics->setInitialPending($thisPending); $firstRow = 0;}
+    }
+
+    # nothing to do here
+    if (!$totalProcessed) {return 1;}
+
+    my $percent25 = $totalProcessed * 0.25;
+    my $percent50 = $totalProcessed * 0.5;
+    my $percent75 = $totalProcessed * 0.75;
+    my $percent90 = $totalProcessed * 0.9;
+    my $percent95 = $totalProcessed * 0.95;
+
     my ($derivProcessed, $derivPending, $derivFaults);
-    my $started = undef;
-    my $finished = undef;
-    
     my $notProcessing = undef;
     my $howLongNotProcessing = undef;
     my $notProcessingLongerThanInterval = 0;
-
     my $stuck = undef;
     my $howLongStuck = undef;
     my $stuckLongerThanInterval = 0;
-
     my $stuffToDo = undef;
-    my $interval = "00:20:00";
-
+    my $interval = "00:20:00"; # TODO dodgy - see above
+    my $started = undef;
+    my $finished = undef;
+   
+    my ($have25, $have50, $have75, $have90, $have95, $have100) = 0;
+
+    # now, loop again, this time recording more data
+    $query->execute;
     $linearProcessed = 0;
+    $lastProcessed = -1;
     while (my @row = $query->fetchrow_array()) {
         my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
@@ -326 +408 @@
         $linearPending = $thisPending;
         $linearFaults = $thisFaults;
+
+        if (!$have25 && $linearProcessed >= $percent25) {$stageMetrics->setFinished25($thisTimestamp); $have25 = 1;}
+        if (!$have50 && $linearProcessed >= $percent50) {$stageMetrics->setFinished50($thisTimestamp); $have50 = 1;}
+        if (!$have75 && $linearProcessed >= $percent75) {$stageMetrics->setFinished75($thisTimestamp); $have75 = 1;}
+        if (!$have90 && $linearProcessed >= $percent90) {$stageMetrics->setFinished90($thisTimestamp); $have90 = 1;}
+        if (!$have95 && $linearProcessed >= $percent95) {$stageMetrics->setFinished95($thisTimestamp); $have95 = 1;}
 
         # get first derivative values
@@ -363 +451 @@
                 if (defined $notProcessing) {
                     
-                    if ($stuckLongerThanInterval) {
-
-#                        print "* $stage procesing was stuck for  $howLongStuck, from $stuck until $lastTimestamp\n";
-                    }
-                    if ($notProcessingLongerThanInterval) {
-
-            #            print " * $stage not procesing for  $howLongNotProcessing, from $notProcessing until $thisTimestamp\n";
-                    }
                     $notProcessing = undef; 
                     $stuck = undef; 
                 }
                 if (!defined $started) {
+
                     $started = $lastTimestamp;
                     $finished = undef; 
-#                    print "* $stage processing started at $started\n";
                 }
             }
@@ -418 +498 @@
     }
 
-    ${$_processed} = $linearProcessed;
-    ${$_pending} = $stuffToDo;
-    ${$_faults} = $lastFaults;
+    $stageMetrics->setProcessed($linearProcessed);
+    $stageMetrics->setFinalPending($stuffToDo);
+    $stageMetrics->setFaults($lastFaults);
+
+
+    if (!defined $finished) {
+
+        $stageMetrics->setFinished25(undef);
+        $stageMetrics->setFinished50(undef);
+        $stageMetrics->setFinished75(undef);
+        $stageMetrics->setFinished90(undef);
+        $stageMetrics->setFinished95(undef);
+        $stageMetrics->setFinished100(undef);
+        $stageMetrics->setTotalTime($self->diffTimes($lastTimestamp, $started));
+    }
 
     if (defined $finished) {
-        ${$_finished} = $finished;
-        ${$_totalTime} = $self->diffTimes($finished, $started);
-    }
-
-    elsif(defined $started) {
-    
-        ${$_finished} = undef;
-    }
-    
+
+        $stageMetrics->setFinished100($finished);
+        $stageMetrics->setTotalTime($self->diffTimes($finished, $started));
+    }
+
+
     if (!defined $started){
     
-        ${$_started} = undef;
+        $stageMetrics->setStarted(undef);
     }
     else {
     
-        ${$_started} = $started;
+        my $rate = ($stageMetrics->getProcessed()/$self->getTimeInSecs($stageMetrics->getTotalTime) ) * 3600.0;
+        $stageMetrics->setRate($rate);
+        $stageMetrics->setStarted($started);
     }
     if ($stuck && $stuckLongerThanInterval && !defined $finished) {
 
-        ${$_stuck} = $stuck;
+        $stageMetrics->setStuck($stuck);
     }
     else {
     
-        ${$_stuck} = undef;
+        $stageMetrics->setStuck(undef);
     }
 
@@ -484 +575 @@
     $query->execute;
 
-    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$dataFile} = $tmpFile->filename;
     open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
 
@@ -497 +590 @@
     
     $linearProcessed = 0;
+    my $someData = 0;
     while (my @row = $query->fetchrow_array()) {
         my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
@@ -517 +611 @@
             my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
                 
-            if ($thisProcessed >= $lastProcessed){
+            if ($thisProcessed >= $lastProcessed && $timeDiff){
                 $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
             }
@@ -547 +641 @@
         else {print GNUDAT "$timestamp $linearPending $linearFaults $linearProcessed\n";}
 
+        if ($linearPending > 0 || $linearFaults > 0 || $linearProcessed > 0) {$someData = 1;}
+
         $lastProcessed = $thisProcessed;
         $lastPending = $thisPending;
@@ -555 +651 @@
     close(GNUDAT) or print "* Problem closing gnuplot data file for plot for '$label' '$stage'\n";
 
-    return 1;
+    #return $someData;
 }
 
@@ -637 +733 @@
 ###########################################################################
 sub createHostsData {
-    my ($self, $limit) = @_;
-
-    my $dataFile = "/tmp/czarplot_hosts_space.dat";
-    open (GNUDAT, ">$dataFile");
+    my ($self, $limit, $tmpFile) = @_;
+
+    open (GNUDAT, ">".$tmpFile->filename);
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -686 +781 @@
         print GNUDAT "\n";
     }
+
     close(GNUDAT);
-
-    return $dataFile;
 }
 
@@ -714 +808 @@
 ###########################################################################
 sub createStorageTimeSeriesData {
-    my ($self, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
-
-    my $dataFile = "/tmp/czarplot_gnuplot_storage_timeseries.dat";
-    open (GNUDAT, ">$dataFile");
+    my ($self, $tmpFile, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
+
+    open (GNUDAT, ">".$tmpFile->filename);
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -753 +846 @@
 
     close(GNUDAT);
-
-    return $dataFile;
 }
 
@@ -763 +854 @@
 ###########################################################################
 sub createProcessingRateData {
-    my ($self, $stage, $label, $startDay, $endDay, $interval, $dataFile, $isLog) = @_;
-
-    my $startTime = $startDay;
+    my ($self, $stage, $label, $begin, $end, $interval, $dataFile, $isLog) = @_;
+
+    my $startTime = $begin;
     my $endTime;
     my $quit = 0;
@@ -772 +863 @@
     my $faults;
     my $timestamp;
-    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_r.dat";
+
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_r.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$dataFile} = $tmpFile->filename;
+
     open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
     my $cleanupCarry = 0;
+    my $someData = 0;
     while(1) {
 
-        if (!$self->isBefore($startTime, $endDay)) {last;}
+
+        #print "NNN $startTime, $end\n";
+        if (!$self->isBefore($startTime, $end)) {last;}
         $endTime = $self->addInterval($startTime, $interval);
         $self->countProcessedPendingAndFaults($label, $stage, $startTime, $endTime, \$processed, \$pending, \$faults);
@@ -783 +881 @@
         print GNUDAT "$timestamp $processed 0 0\n";
 
+        if ($processed > 0 ) {$someData = 1;}
+
         $startTime = $endTime;
     }
 
-    close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n"
+    close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n";
+
+    #return $someData;
 }
 
@@ -1512 +1614 @@
     $self->setRevision(12);
 }
-
 1;
 

branches/eam_branches/ipp-20101103/tools/czartool/Gpc1Db.pm

@@ -60 +60 @@
 ###########################################################################
 #
+# Returns the number of science exposures taken 'last night' for a given day
+# 
+# Notes:
+#
+# - 03:00 HST = 17:00 last night UTC
+# - 17:00 HST = 07:00 this morning UTC
+# - 'OBJECT' = science exposures
+#
+###########################################################################
+sub countScienceExposuresFromLastNight {
+    my ($self, $day) = @_;
+
+    my $begin = "$day 03"; 
+    my $end = "$day 17"; 
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*)
+        FROM summitExp 
+        WHERE dateobs > '$begin' 
+        AND dateobs < '$end' AND
+        exp_type = 'OBJECT';
+SQL
+
+    $query->execute;
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns the number of exposures between two dates that have been registered with chip
+# 
+# Notes:
+#
+# necessary to convert times as rawExp.dateobs is in UTC
+#
+###########################################################################
+sub getProcessedExposures {
+    my ($self, $beginDate, $endDate, $expIds) = @_;
+
+    $beginDate = "$beginDate 03";
+    $endDate = "$endDate 17";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+        SELECT rawExp.exp_id 
+        FROM chipRun, rawExp 
+        WHERE chipRun.exp_id = rawExp.exp_id 
+        AND rawExp.dateobs >= '$beginDate' 
+        AND rawExp.dateobs <= '$endDate';
+SQL
+
+    $query->execute;
+    ${$expIds} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+###########################################################################
+#
 # Returns count of faults for this stage and label
 #
@@ -158 +216 @@
     }
 
-
     $query->execute;
     return scalar $query->fetchrow_array();
 }
+
+###########################################################################
+#
+# Returns magic mask fraction across all chips for a particular exposure
+#
+###########################################################################
+sub getMagicMaskFraction {
+        my ($self, $exp_id, $fracs) = @_;
+
+            my $query = $self->{_db}->prepare(<<SQL);
+            SELECT component, streak_frac 
+                FROM magicDSFile  
+                JOIN magicDSRun USING(magic_ds_id) 
+                JOIN camRun USING(cam_id) 
+                JOIN chipRun USING(chip_id) 
+                WHERE exp_id = $exp_id 
+                AND camRun.label LIKE "%nightlyscience"
+                AND component LIKE "XY%";
+SQL
+    $query->execute;
+    ${$fracs} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+###########################################################################
+#
+# Returns average magic mask fraction across all chips for a particular exposure
+#
+###########################################################################
+sub getAverageMagicMaskFraction {
+        my ($self, $exp_id) = @_;
+
+            my $query = $self->{_db}->prepare(<<SQL);
+            SELECT AVG(streak_frac) 
+                FROM magicDSFile  
+                JOIN magicDSRun USING(magic_ds_id) 
+                JOIN camRun USING(cam_id) 
+                JOIN chipRun USING(chip_id) 
+                WHERE exp_id = $exp_id 
+                AND camRun.label LIKE "%nightlyscience"
+                AND component LIKE "XY%";
+SQL
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
 1;

branches/eam_branches/ipp-20101103/tools/czartool/MySQLDb.pm

@@ -64 +64 @@
     return $self->{_dbHost};                                                 
 }                                                                               
-###########################################################################
-#
-# Adds the provided interval to the provided time
+
+###########################################################################
+#
+# Returns a date formatted wit hthe current formatting
 #
 ###########################################################################
@@ -82 +83 @@
 ###########################################################################
 #
+# Returns a full MySQL formatted date
+#
+###########################################################################
+sub getFullDate {
+    my ($self, $time) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT DATE_FORMAT('$time','%Y-%m-%d %H:%i:%s'); 
+SQL
+    $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
 # Adds the provided interval to the provided time
 #
@@ -98 +115 @@
 ###########################################################################
 #
+# Subtracts the provided interval from the provided time
+#
+###########################################################################
+sub subtractInterval {
+    my ($self, $time, $interval) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT '$time' - INTERVAL $interval; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Formats a date/time string in a way suitable for TIMEDIFF calls
+#
+###########################################################################
+sub formatTimeForDiff {
+    my ($self, $time) = @_;
+
+    if (${$time} =~ m/^[0-9]{4}-[0-9]{2}-[0-9]{2}.*/) {${$time} = $self->getFullDate(${$time});}
+    elsif(${$time} !~ m/^[0-9]+:[0-9]{2}:[0-9]{2}$/) {return 0;}
+
+    return 1;
+}
+
+###########################################################################
+#
 # Finds the difference of two times
+#
+# Should be able to handle most date/time formats
 #
 ###########################################################################
@@ -104 +153 @@
     my ($self, $time1, $time2) = @_;
 
-      my $query = $self->{_db}->prepare(<<SQL);
+    if (!$time1 || !$time2) {return 0;}
+
+    if (!$self->formatTimeForDiff(\$time1) || !$self->formatTimeForDiff(\$time2)) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
          SELECT TIMEDIFF('$time1','$time2'); 
+SQL
+
+    $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns the provided time in seconds
+#
+###########################################################################
+sub getTimeInSecs {
+    my ($self, $time) = @_;
+
+    if (!$time) {return 0;}
+
+      my $query = $self->{_db}->prepare(<<SQL);
+         SELECT TIME_TO_SEC('$time'); 
 SQL
 
@@ -120 +191 @@
     my ($self, $time1, $time2) = @_;
 
-      my $query = $self->{_db}->prepare(<<SQL);
+    if (!$time1 || !$time2) {return 0;}
+    if (!$self->formatTimeForDiff(\$time1) || !$self->formatTimeForDiff(\$time2)) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
          SELECT TIME_TO_SEC(TIMEDIFF('$time1','$time2')); 
 SQL
@@ -146 +220 @@
 ###########################################################################
 #
-# Subtracts the provided interval from the provided time
-#
-###########################################################################
-sub subtractInterval {
-    my ($self, $time, $interval) = @_;
-
-      my $query = $self->{_db}->prepare(<<SQL);
-          SELECT '$time' - INTERVAL $interval; 
-SQL
-    $query->execute;
-
-return scalar $query->fetchrow_array();
-}
-
-###########################################################################
-#
 # Returns whether time1 is before time2
 #
@@ -168 +226 @@
     my ($self, $time1, $time2) = @_;
 
-      my $query = $self->{_db}->prepare(<<SQL);
-          SELECT '$time1' < '$time2'; 
-SQL
-    $query->execute;
-
-return scalar $query->fetchrow_array();
+    if (!$time1 || !$time2) {return 0;}
+    if (!$self->formatTimeForDiff(\$time1) || !$self->formatTimeForDiff(\$time2)) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT TIME_TO_SEC(TIMEDIFF('$time1', '$time2'));
+SQL
+    $query->execute;
+
+    return (scalar $query->fetchrow_array() >= 0) ? 0 : 1;
 }
 

branches/eam_branches/ipp-20101103/tools/czartool/Plotter.pm

@@ -4 +4 @@
 use warnings;
 use strict;
+
+use File::Temp;
 
 my @allStages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
@@ -16 +18 @@
     my $class = shift;
     my $self = {
+        _gpc1Db => shift,
         _czarDb => shift,
         _dateFormat => shift,
@@ -53 +56 @@
 }
 
+
+###########################################################################
+#
+# Figures out a suitable interval for rate plots
+# Could use a function here, but instead we want to choose meaningful intervals
+#
+###########################################################################
+sub getSuitableInterval {
+    my ($self, $begin, $end, $interval) = @_;
+
+    my $timeDiff = $self->{_czarDb}->diffTimes($end, $begin);
+
+
+    if ($self->{_czarDb}->isBefore($timeDiff, "00:00:01")) {return 0;}
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "03:00:00")) {${$interval} = "15 MINUTE";}
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "10:00:00")) {${$interval} = "30 MINUTE";}
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "24:00:00")) {${$interval} = "1 HOUR";} # under 1 day
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "36:00:00")) {${$interval} = "2 HOUR";} # under 1.5 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "48:00:00")) {${$interval} = "3 HOUR";} # under 2 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "72:00:00")) {${$interval} = "4 HOUR";} # under 3 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "96:00:00")) {${$interval} = "5 HOUR";} # under 4 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "144:00:00")) {${$interval} = "6 HOUR";} # under 6 days
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "168:00:00")) {${$interval} = "12 HOUR";} # under week
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "672:00:00")) {${$interval} = "1 DAY";} # under 2 week
+    elsif ($self->{_czarDb}->isBefore($timeDiff, "2016:00:00")) {${$interval} = "1 WEEK";} # under 3 month
+    else {${$interval} = "1 MONTH";}
+
+    return 1;
+}
+
 ###########################################################################
 #
@@ -61 +94 @@
     my ($self, $label, $selectedStage, $beginTime, $endTime, $interval, $isLog) = @_;
 
-    my $minX = 999999999;      
-    my $maxX = -9999999999;
-    my $timeDiff = -1;
-
+    # stages
     my $stages = undef;                 
-
-    if (!$selectedStage) {$stages = \@allStages;}
-    else {$stages = ["$selectedStage"]};        
-
-    $self->{_czarDb}->getTimeMinMaxDiff($label, $selectedStage, $beginTime, $endTime, \$minX, \$maxX, \$timeDiff);
-
-    my $divX;
-    my $timeFormat;
-    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
-
+    if (!$selectedStage) {
+        $stages = \@allStages;
+        $selectedStage = 'all_stages';
+    }
+    else {$stages = ["$selectedStage"]};
+
+    # interval
+    if (!$interval && !$self->getSuitableInterval($beginTime, $endTime, \$interval)) {return 0;}
+
+    # set up files
     my %gnuplotFiles;
     my $stage = undef;
@@ -89 +119 @@
         "set term $self->{_outputFormat};" .
         "set title \"'$label', '$selectedStage' during '$beginTime' to '$endTime'\";" .
-        #"set xdata time;" .
-        "set timefmt \"$self->{_dateFormat}\";" .
-       "set format x \"$timeFormat\";" .
-        "set xtics \"$minX\", $divX, \"$maxX\";" .
-#        "set xrange [\"$minX\":\"$maxX\"];" .
         "set boxwidth;" .
+        "set xtic rotate by -90 scale 0;" .
         "set style data histogram;" .
         "set style histogram rowstacked;" .
@@ -124 +150 @@
     print GP ";\n";
     close GP;
-}                                                    
+
+    # now delete temp dat files
+    foreach my $stage (keys %gnuplotFiles) {unlink($gnuplotFiles{$stage});}
+
+    return 1;
+}
+
 ###########################################################################
 #
@@ -168 +200 @@
     if ($numOfPlots == 0 ) {
 
-        print "Warning: No plots could be generated for stage '$selectedStage' and label '$label' during time period '$beginTime', '$endTime'\n";
+#        print "Warning: No plots could be generated for stage '$selectedStage' and label '$label' during time period '$beginTime', '$endTime'\n";
         return;
     } 
@@ -202 +234 @@
     my $outputFile = createImageFileName($self, $label, undef, "h", 0);
 
-    my $inputFile = "/tmp/czarplot_gnuplot_".$label."_h.dat"; # TODO use stage not table
-        open (GNUDAT, ">$inputFile");
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_h.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    open (GNUDAT, ">".$tmpFile->filename);
 
     my ($processed, $pending, $faults);
     my $stage = undef;
     my $pendingMinusFaults = undef;
-    my $maxY = 0;
-    my $sum;
     foreach $stage (@allStages) {
 
-        $self->{_czarDb}->countProcessedPendingAndFaults(
+        if (!$self->{_czarDb}->countProcessedPendingAndFaults(
                 $label, 
                 $stage, 
@@ -219 +249 @@
                 \$processed, 
                 \$pending, 
-                \$faults);
+                \$faults)) {next;}
 
         $pendingMinusFaults = $pending - $faults;
-        print GNUDAT "$stage $faults $processed, $pendingMinusFaults\n";
-        my $sum = $faults + $processed + $pendingMinusFaults;
-        if ($sum > $maxY) {$maxY = $sum;}
+        print GNUDAT "$stage $faults $processed $pendingMinusFaults\n";
     }
 
@@ -232 +260 @@
     use FileHandle;
     GP->autoflush(1);
-
-    if ($maxY == 0) {$maxY = 1;}
-    else {$maxY = $maxY*1.1;}
-
 
     if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
@@ -243 +267 @@
         "set grid;" .
         "set boxwidth;" .
-        "set yrange [\"0\":\"$maxY\"];" .
         "set style data histogram;" .
         "set style histogram rowstacked;" .
@@ -249 +272 @@
         "set ylabel \"Exposures\";" .
         "set boxwidth 0.75;" .
-        "plot '$inputFile' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . 
+        "plot '".$tmpFile->filename."' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . 
         "\n";
 
     close GP;
-    unlink($inputFile);
 }
 
@@ -295 +317 @@
     my ($self, $timeDiff, $divX, $timeFormat) = @_;
 
+    # if less than a few mins of data, show 1 minute tics
+    if ($timeDiff < 300) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 60;
+    }
+    # if less than a few mins of data, show 5 minute tics
+    elsif ($timeDiff < 1800) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 300;
+    }
+    # if less than an hour of data, show 15 minute tics
+    elsif ($timeDiff < 3600) {
+        ${$timeFormat} = "%H:%M";
+        ${$divX} = 900;
+    }
     # if less than a couple of hour's data plotted, show 30 mins tics
-    if ($timeDiff < 7200) {
+    elsif ($timeDiff < 7200) {
         ${$timeFormat} = "%H:%M";
         ${$divX} = 1800;
@@ -376 +413 @@
 
     my $firstIn = 1;
-    foreach my $stage (keys %$gnuplotFiles) {
+    # loop through stages array so that they are ordered properly (not maintained in hash)
+    foreach my $stage (@allStages) {
+
+        if(!$gnuplotFiles->{$stage}) {next;}
         if (!$firstIn) {print GP ",";}
 
@@ -398 +438 @@
 
     # now delete temp dat files
-    foreach my $stage (keys %$gnuplotFiles) {
-
-        unlink($gnuplotFiles->{$stage});
-    }
+    foreach my $stage (keys %$gnuplotFiles) {unlink($gnuplotFiles->{$stage});}
 }
 
@@ -416 +453 @@
     my ($minX, $maxX, $minY, $maxY, $timeDiff);
 
-    my $gnuplotFile = $self->{_czarDb}->createStorageTimeSeriesData($fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff);
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_storage_timeseries.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $self->{_czarDb}->createStorageTimeSeriesData($tmpFile, $fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff);
 
     my $timeFormat = undef;
@@ -440 +478 @@
         "set ylabel \"Available (TB)\";" .
         "plot " . 
-        "'$gnuplotFile' using 1:2 title \"Available\" with lines lt 2 lw 2\n";
+        "'".$tmpFile->filename."' using 1:2 title \"Available\" with lines lt 2 lw 2\n";
 
     print GP "\n";
     close GP;
-    unlink($gnuplotFile);
 }
 
@@ -458 +495 @@
         my $outputFile = "$prefix/czarplot_hosts_space.png";
     my $limit = 97.0;
-    my $inputFile = $self->{_czarDb}->createHostsData($limit);
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_hosts_space.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $self->{_czarDb}->createHostsData($limit, $tmpFile);
 
     my $totalUsed = $self->{_czarDb}->getTotalClusterStorageAsPercentage();
@@ -477 +515 @@
         "set ylabel \"Space (TB)\";" .
         "set xtic rotate by -90 scale 0;" .
-        "plot '$inputFile' " .
+        "plot '".$tmpFile->filename."' " .
         "using 2:xtic(1) t \"Used\" lt 7," .
         "'' using 3 t \"Over $limit% used\" lt 1," . 
@@ -489 +527 @@
     close GP;
 }
+
+###########################################################################
+#
+# Generates 3D plot of magic mask fraction for provided exposure ID
+#
+###########################################################################
+sub plotMagicMaskFractionForThisExposure {
+    my ($self, $exp_id) = @_;
+
+    # set up file stuff
+    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
+        my $outputFile = "$prefix/czarplot_single_exp_magic_mask_fraction.png";
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_single_exp_magic_mask_fraction.XXXXX", DIR => '/tmp', SUFFIX => '.dat');
+
+    my $fracs;
+    if (!$self->{_gpc1Db}->getMagicMaskFraction($exp_id, \$fracs)) {return 0;}
+
+    open (GNUDAT, ">".$tmpFile->filename);
+
+    print GNUDAT "0 0 0.0\n";
+    foreach my $row ( @{$fracs} ) {
+
+        if (@{$row}[0] =~ m/XY([0-9])([0-9])/) {
+            print GNUDAT "$1 $2 @{$row}[1]\n";
+            if($1 == 0 && $2 == 6) {print GNUDAT "0 7 0.0\n";}
+            if($1 == 6 && $2 == 7) {print GNUDAT "7 0 0.0\n";}
+        }
+    }
+    print GNUDAT "7 7 0.0\n";
+
+
+    close(GNUDAT);
+
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    my $datFile = $tmpFile->filename;
+
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
+    print GP <<PLOT;
+
+    set term $self->{_outputFormat}
+    set title "Magic mask fraction for exposure $exp_id"
+        set grid
+        set size square
+        set xrange [-0.5:7.5] reverse 
+        set yrange [-0.5:7.5]  
+        unset key
+        set palette rgb 22,13,10
+        set ylabel "Y"
+        set xlabel "X"
+        set cbrange [0:1]
+        set datafile missing "NaN"
+        plot "$datFile" using 1:2:3  with image
+PLOT
+
+    close GP;
+
+    return 1;
+}
+
+
+###########################################################################
+#
+# Plots mask fraction histogram for exposures taken between these dates
+#
+###########################################################################
+sub plotMagicMaskFractionHistogram {
+    my ($self, $begin, $end) = @_;
+
+    # get exposures between these dates
+    my $exp_ids = undef;
+    if (!$self->{_gpc1Db}->getProcessedExposures($begin, $end, \$exp_ids)) {return 0;}
+    my $totalExp = @{$exp_ids};
+    if ($totalExp < 1) {return 0;}
+
+    # set up file stuff
+    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
+        my $histoOutputFile = "$prefix/czarplot_magic_mask_fraction_h.png";
+    my $distOutputFile = "$prefix/czarplot_magic_mask_fraction_d.png";
+    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_magic_mask_fraction.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+
+    # set up bins
+    my %histogram; 
+    my @bins;
+    my $interval = 0.05;
+    my $min = 1.0;
+    my $max = 1.0;
+    for (my $n=0.0; $n<$max; $n += $interval){
+
+        push(@bins, $n);
+        $histogram{$n} = 0;
+    }
+
+    # get mask for each exposure, and bin
+    my $mask = undef;
+    my $expCount = 0;
+    foreach my $row ( @{$exp_ids} ) {
+
+        $mask = $self->{_gpc1Db}->getAverageMagicMaskFraction(@{$row}[0]);
+
+        if (!$mask) {next;}
+        $expCount++;
+
+        foreach my $bin (@bins) {
+            if ($mask <= ($bin+$interval)) {
+
+                $histogram{$bin}++;
+                last;
+            }
+        }
+    }
+
+    # write data to GNUPLOT data file
+    print "* Found mask values for $expCount exposures out of $totalExp\n";
+    open (GNUDAT, ">".$tmpFile->filename);
+    my $accum = 0;
+    my $maxBin = 0;
+    foreach my $bin (@bins) {
+
+        $accum += $histogram{$bin};
+        if ($histogram{$bin} > $maxBin) {$maxBin = $histogram{$bin};}
+        print GNUDAT "$bin $histogram{$bin} $accum\n";
+    }
+    close(GNUDAT);
+
+    # now plot
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    $maxBin = $maxBin * 1.1;
+    $accum = $accum * 1.1;
+
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$histoOutputFile\";";}
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set title \"Magic mask fraction for $expCount exposures between '$begin' and '$end'\";" .
+        "set grid;" .
+        "set boxwidth;" .
+        "set style data histogram;" .
+        "set yrange [0:$maxBin];" .
+        "set style histogram rowstacked;" .
+        "set style fill solid border -1;" .
+        "set ylabel \"Number of Exposures\";" .
+        "set xlabel \"Magic Mask Fraction\";" .
+        "set boxwidth 0.75;" .
+        "set xtic rotate by -90 scale 0;" .
+        "plot '".$tmpFile->filename."' using 2:xtic(1) notitle lt 1;" . 
+        "\n";
+
+    close GP;
+
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$distOutputFile\";";}
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set title \"Cumulative distribution of magic streaks per image between '$begin' and '$end'\";" .
+        "set yrange [0:$accum];" .
+        "set key left top;" .
+        "set grid xtics;" .
+        "set xlabel \"Magic Mask Fraction\";" .
+        "set ylabel \"Number of Exposures\";" .
+        "set xtic rotate by -90 scale 0;" .
+        "plot " . 
+        "'".$tmpFile->filename."' using 1:3 notitle with lines lt 2 lw 2\n";
+
+    print GP "\n";
+    close GP;
+
+    return 1;
+}
 1;

branches/eam_branches/ipp-20101103/tools/roboczar.pl

@@ -13 +13 @@
 use czartool::Plotter;
 use czartool::Burntool;
+use czartool::StageMetrics;
 
 my $czarDbName = "czardb"; # TODO variables for other Db stuff, host etc
@@ -56 +57 @@
 
         sendEmail(
-                "roydhenderson\@gmail.com", 
+                "ps-ipp-ops\@ifa.hawaii.edu", 
                 "Roboczar warnings", 
                 "$message\n");
@@ -89 +90 @@
     my $stage;
     foreach $stage (@stages) {
-        my ($started, $finished, $stuck, $processed, $pending, $faults, $totalTime);
-        $czarDb->runAnalysis(
-                "all_stdscience_labels", 
-                $stage, 
-                $begin, 
-                $end, 
-                \$started, 
-                \$finished, 
-                \$stuck, 
-                \$processed, 
-                \$pending, 
-                \$faults, 
-                \$totalTime);
 
-        print "*     $stage $end $begin:\n";
-        if (defined $started) {print "* Processing started at $started\n";}
-        if (defined $finished) {print "* Processing finished at $finished and took $totalTime\n";}
-        else {print "* Processing has not finished\n";}
-        if (defined $stuck) {print "* Processing has been stuck since $stuck\n";}
-        print "* $processed exposures have been processed, with $pending pending and $faults faults\n";
+    my $stageMetrics = new czartool::StageMetrics($stage, "all_stdscience_labels", $begin, $end);
 
-        print "*******************************************************************************\n";
+    if ($czarDb->runAnalysis($stageMetrics)) {$stageMetrics->printMe();}
 
-        if ($stuck && $stage eq "burntool" && !$worryAboutBurntool) {next;}
+        if ($stageMetrics->getStuck() && $stage eq "burntool" && !$worryAboutBurntool) {next;}
 
-        if ($stuck) {
+        if ($stageMetrics->getStuck()) {
         
-            ${$message} = ${$message} ."\n - '$stage' is stuck with $pending pending exposures (and $faults faults)";
+            ${$message} = ${$message} ."\n - '$stage' is stuck with ".$stageMetrics->getFinalPending()." pending exposures (and ".$stageMetrics->getFaults()." faults)";
             $anyStuckStages = 1;
         }

branches/eam_branches/ipp-20101103/tools/runchipimfile.pl

@@ -38 +38 @@
 
 my $query1 = "SELECT chipProcessedImfile.path_base, camera, exp_id, rawImfile.magicked AS raw_magicked, (rawImfile.user_1 is not NULL and rawImfile.user_1 > 0.5)  As deburned, chip_imfile_id, chipProcessedImfile.fault, rawImfile.uri, chipRun.reduction, chipRun.state";
-$query1 .= " FROM chipRun join chipProcessedImfile using(chip_id) JOIN chipImfile USING (chip_id, class_id) JOIN rawExp using(exp_id) JOIN rawImfile USING(exp_id) WHERE chip_id = $chip_id AND class_id = $class_id";
+$query1 .= " FROM chipRun join chipProcessedImfile using(chip_id,exp_id) JOIN chipImfile USING (chip_id, class_id) JOIN rawExp using(exp_id) JOIN rawImfile USING(exp_id, camera, class_id) WHERE chip_id = $chip_id AND class_id = '$class_id'";
 
 my $stmt1 = $dbh->prepare($query1);
@@ -55 +55 @@
 my $exp_id = $results->{exp_id};
 
-notdone
-
 die "cannot update database for a chip that is not faulted\n" if $update and !$fault;
 
@@ -69 +67 @@
     $run_state = 'new';
 } elsif ($state eq 'update') {
-    $state = 'update';
+    $run_state = 'update';
 } else {
-    die "unexpected warpRun.state found: $state\n";
+    die "unexpected chipRun.state found: $state\n";
 }
 
-my $command = "chip_imfile.pl --exp_id $exp_id --chip_id $chip_id --class_id $class_id --chip_imfile_id $chip_imfile_id --uri $uri --camera $camera --run-state $run_state --deburned $deburned --outroot $path_base" ;
+my $command = "chip_imfile.pl --exp_id $exp_id --chip_id $chip_id --chip_imfile_id $chip_imfile_id --class_id $class_id --uri $uri --camera $camera --run-state $run_state --deburned $deburned --outroot $path_base" ;
 
 $command .= " --magicked $magicked" if $magicked > 0;
@@ -83 +81 @@
 $command .= " --verbose" unless $no_verbose;
 $command .= " --dbname $dbname" if $dbname;
-
 
 if ($update) {