Changeset 29373

Timestamp:

Oct 11, 2010, 3:11:19 PM (16 years ago)

Author:

rhenders

Message:

Big changes to introduce rate=of-procesing plots as well as removing misleading cleanup from other plots

Location:

trunk/tools/czartool

Files:

• CzarDb.pm (modified) (8 diffs)
• Plotter.pm (modified) (7 diffs)

trunk/tools/czartool/CzarDb.pm

@@ -43 +43 @@
 SQL
 
-
-    if (!$query->execute) {
-
-        return 0;
-    }
+    if (!$query->execute) {return 0;}
 
     ${$labels} = $query->fetchall_arrayref();
@@ -242 +238 @@
 ###########################################################################
 #
-# Gets time series data and stores it to temp file
-#
-###########################################################################
-sub createTimeSeriesData {
-    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff, $dataFile, $isLog) = @_;
-
-    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
-
-    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+# Gets time min/max/diff for a given period
+#
+###########################################################################
+sub getTimeMinMaxDiff {
+    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff) = @_;
 
     my $query = $self->{_db}->prepare(<<SQL);
     SELECT 
-        MIN(processed), GREATEST(MAX(pending), MAX(faults), max(processed)-min(processed)) AS maxY, 
-        LEAST(MIN(pending), MIN(faults), max(processed)-min(processed)) AS minY,
         DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), 
         DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
@@ -266 +256 @@
     if (!$query->execute) {return 0;}
 
-    my $minProcessed = undef;
-    my ($_maxY, $_minY, $_timeDiff);
-    ($minProcessed, $_maxY, $_minY, ${$maxX}, ${$minX}, $_timeDiff) = $query->fetchrow_array();
-
-    if ($_maxY > ${$maxY}) {${$maxY} = $_maxY;}
-    if ($_minY < ${$minY}) {${$minY} = $_minY;}
+    my ($_timeDiff);
+    (${$maxX}, ${$minX}, $_timeDiff) = $query->fetchrow_array();
+
     if ($_timeDiff > ${$timeDiff}) {${$timeDiff} = $_timeDiff;}
 
+return 1;
+}
+
+
+###########################################################################
+#
+# Gets time series data and stores it to temp file
+#
+###########################################################################
+sub createTimeSeriesData {
+    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $dataFile, $isLog, $showCleanup) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        MIN(processed)
+        FROM $stage 
+        WHERE label LIKE '$label'
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+SQL
+
+    if (!$query->execute) {return 0;}
+    my $minProcessed = scalar  $query->fetchrow_array(); # TODO remove
     if (!defined $minProcessed) {return 0;}
+
+    $self->getTimeMinMaxDiff($label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff);
 
     $query = $self->{_db}->prepare(<<SQL);
     SELECT 
-        DATE_FORMAT(timestamp, '$self->{_dateFormat}'), pending, faults, processed-$minProcessed 
+        DATE_FORMAT(timestamp, '$self->{_dateFormat}'), pending, faults, processed 
         FROM $stage 
         WHERE label LIKE '$label' 
@@ -287 +298 @@
     $query->execute;
 
-    # loop round results
+    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
+    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+
+    my $lastProcessed = -1;
+    my $processed = 0;
     while (my @row = $query->fetchrow_array()) {
-        my ($timestamp, $pending, $faults, $processed) = @row;
+        my ($timestamp, $pending, $faults, $thisProcessed) = @row;
+
+        if ($showCleanup) {
+        
+        $processed = $thisProcessed - $minProcessed;
+        }
+        elsif($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
+            
+            $processed = $processed + ($thisProcessed - $lastProcessed);
+        }
 
         if ($isLog) {
@@ -303 +327 @@
 
         print GNUDAT "$timestamp $pending $faults $processed\n";
+        $lastProcessed = $thisProcessed;
     }
 
@@ -365 +390 @@
 sub countProcessedPendingAndFaults {
     my ($self, $label, $stage, $fromTime, $toTime, $processed, $pending, $faults) = @_;
-
 
     my $query = $self->{_db}->prepare(<<SQL);
@@ -379 +403 @@
     (${$pending}, ${$faults}) = $query->fetchrow_array();    
 
-    $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        MAX(processed) - MIN(processed) 
-        FROM $stage 
-        WHERE label LIKE '$label' 
-        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
-SQL
-    $query->execute;
-    (${$processed}) = $query->fetchrow_array();
+    ${$processed} = $self->countProcessed($label, $stage, $fromTime, $toTime);
+
+    return 1;
 }
 
@@ -510 +528 @@
         print GNUDAT "$timestamp $available\n";
     }
+
     close(GNUDAT);
 
     return $dataFile;
 }
-###########################################################################
-#
-# Determines how much has been processed in the provided interval of time 
-# (format: 1 HOUR, 1 MINUTE 1 DAY etc)
-#
-###########################################################################
-sub countProcessed { # TODO use time not interval
-    my ($self, $label, $stage, $interval) = @_;
+
+###########################################################################
+#
+# TODO implement isLog
+#
+###########################################################################
+sub createProcessingRateData {
+    my ($self, $stage, $label, $startDay, $endDay, $interval, $dataFile, $isLog) = @_;
+
+    my $startTime = $startDay;
+    my $endTime;
+    my $quit = 0;
+    my $processed;
+    my $pending;
+    my $faults;
+    my $timestamp;
+    ${$dataFile} = "/tmp/czarplot_gnuplot_".$label."_".$stage."_r.dat";
+    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+    my $cleanupCarry = 0;
+    while(1) {
+
+        if (!$self->isBefore($startTime, $endDay)) {last;}
+        $endTime = $self->addInterval($startTime, $interval);
+print "NNN $startTime, $endTime\n";
+        $self->countProcessedPendingAndFaults($label, $stage, $startTime, $endTime, \$processed, \$pending, \$faults);
+        $timestamp = $self->getFormattedDate($endTime);
+        print GNUDAT "$timestamp $processed 0 0\n";
+        print  "FILE $timestamp $processed 0 0\n";
+
+        $startTime = $endTime;
+    }
+
+    close(GNUDAT) or print "* Problem closing gnuplot data file for rate plot for '$label' '$stage'\n"
+}
+
+###########################################################################
+#
+# Gets count of processed stuff in a given time period 
+#
+###########################################################################
+sub countProcessed {
+    my ($self, $label, $stage, $begin, $end) = @_;
 
     my $query = $self->{_db}->prepare(<<SQL);
     SELECT 
-        MAX(processed) - MIN(processed) 
-        FROM $stage 
+        processed 
+        FROM $stage
         WHERE label LIKE '$label' 
-        AND timestamp > (now() - INTERVAL $interval); 
-SQL
-        $query->execute;
-
-    return scalar $query->fetchrow_array();
-}
-
+        AND timestamp >= '$begin' 
+        AND timestamp < '$end' 
+SQL
+    $query->execute;
+
+    my $processedArray = $query->fetchall_arrayref();
+
+    my $processed;
+    my $thisCount = -1;
+    my $lastCount = -1;
+    my $count = 0;
+    foreach $processed ( @{$processedArray} ){
+
+        ($thisCount) = @{$processed};
+
+        if ($thisCount > $lastCount && $lastCount != -1) {$count = $count + ($thisCount -  $lastCount);}
+        print "thisCount = $thisCount lastCount=$lastCount processed=$count\n";
+        $lastCount = $thisCount;
+    }
+
+    return $count;
+}
 
 ###########################################################################

trunk/tools/czartool/Plotter.pm

@@ -33 +33 @@
 ###########################################################################
 sub createImageFileName {
-    my ($self, $label, $stage, $suffix, $isLog) = @_;
+    my ($self, $label, $stage, $suffix, $isLog, $isRate) = @_;
 
     my $prefix = $self->{_outputPath} ? $self->{_outputPath} : ".";
@@ -55 +55 @@
 ###########################################################################
 #
+# Plots a time series of processing rate for all stages for this label TODO duplication below. combine
+#
+###########################################################################
+sub createRateTimeSeries {
+    my ($self, $label, $selectedStage, $beginTime, $endTime, $interval, $isLog) = @_;
+
+    my $minX = 999999999;      
+    my $maxX = -9999999999;
+    my $timeDiff = -1;
+
+    my $stages = undef;                 
+
+    if (!$selectedStage) {$stages = \@allStages;}
+    else {$stages = ["$selectedStage"]};        
+
+    $self->{_czarDb}->getTimeMinMaxDiff($label, $selectedStage, $beginTime, $endTime, \$minX, \$maxX, \$timeDiff);
+
+    my $divX;
+    my $timeFormat;
+    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
+
+print "$beginTime, $endTime  | $divX $minX $maxX $timeFormat\n";
+
+    my %gnuplotFiles;
+    my $stage = undef;
+    my $gnuplotFile = undef;
+    my $outputFile = createImageFileName($self, $label, $selectedStage, "r", $isLog);
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set output \"$outputFile\";" .
+        "set title \"'$label', '$selectedStage' during '$beginTime' to '$endTime'\";" .
+        #"set xdata time;" .
+        "set timefmt \"$self->{_dateFormat}\";" .
+       "set format x \"$timeFormat\";" .
+        "set xtics \"$minX\", $divX, \"$maxX\";" .
+#        "set xrange [\"$minX\":\"$maxX\"];" .
+        "set grid;" .
+        "set boxwidth;" .
+        "set style data histogram;" .
+        "set style histogram rowstacked;" .
+        "set style fill solid border -1;" .
+        "set ylabel \"Exposures processed per $interval\";" .
+        "set boxwidth 0.75;" .
+        "plot ";
+
+    my $first = 1;
+    foreach $stage (@{$stages}) {
+
+        if ($self->{_czarDb}->createProcessingRateData(
+                    $stage, 
+                    $label, 
+                    $beginTime, 
+                    $endTime, 
+                    $interval,
+                    \$gnuplotFile,
+                    $isLog)) {
+
+            $gnuplotFiles{$stage} = $gnuplotFile;
+
+            if (!$first) { print GP ","; }
+            print GP "'$gnuplotFile' using 2:xtic(1) title \"$stage\" ";
+            $first = 0;
+        }
+    }
+
+    print GP ";\n";
+    close GP;
+}                                                    
+###########################################################################
+#
 # Plots a time series for all stages for this label
 #
 ###########################################################################
 sub createTimeSeries {
-    my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog) = @_;
-
-    my ($minX, $maxX, $minY, $maxY, $timeDiff);
-    $minX = 999999999;      
-    $maxX = -9999999999;
-    $minY = 999999999;          
-    $maxY = -99999999;
-    $timeDiff = 0;
+    my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog, $showCleanup) = @_;
+
+    my $minX = 999999999;      
+    my $maxX = -9999999999;
+    my $timeDiff = 0;
 
     my $stages = undef;                 
@@ -85 +155 @@
                     \$minX, 
                     \$maxX, 
-                    \$minY, 
-                    \$maxY, 
                     \$timeDiff, 
                     \$gnuplotFile,
-                    $isLog)) {
+                    $isLog,
+                    $showCleanup)) {
 
             $gnuplotFiles{$stage} = $gnuplotFile;
@@ -118 +187 @@
             $maxX, 
             $minX, 
-            $maxY, 
-            $minY, 
             $timeDiff, 
             $isLog);
@@ -161 +228 @@
     close(GNUDAT);
 
-    plotHistogram($self, $inputFile, $outputFile, $label, $beginTime, $endTime, $maxY);
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    if ($maxY == 0) {$maxY = 1;}
+    else {$maxY = $maxY*1.1;}
+
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set output \"$outputFile\";" .
+        "set title \"'$label', '$beginTime' to '$endTime'\";" .
+        "set grid;" .
+        "set boxwidth;" .
+        "set yrange [\"0\":\"$maxY\"];" .
+        "set style data histogram;" .
+        "set style histogram rowstacked;" .
+        "set style fill solid border -1;" .
+        "set ylabel \"Exposures\";" .
+        "set boxwidth 0.75;" .
+        "plot '$inputFile' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . 
+        "\n";
+
+    close GP;
     unlink($inputFile);
 }
-
 
 ###########################################################################
@@ -239 +327 @@
 ###########################################################################
 sub plotTimeSeries {
-    my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $maxY, $minY, $timeDiff, $isLog) = @_;
+    my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $timeDiff, $isLog) = @_;
 
     my $timeFormat = undef;
@@ -346 +434 @@
     close GP;
     unlink($gnuplotFile);
-}
-
-###########################################################################
-#
-# Plots a histogram of processed stuff
-#
-###########################################################################
-sub plotHistogram {
-    my ($self, $inputFile, $outputFile, $label, $fromTime, $toTime, $maxY) = @_;
-
-    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
-    use FileHandle;
-    GP->autoflush(1);
-
-    if ($maxY == 0) {$maxY = 1;}
-    else {$maxY = $maxY*1.1;}
-
-    print GP
-        "set term $self->{_outputFormat};" .
-        "set output \"$outputFile\";" .
-        "set title \"'$label', '$fromTime' to '$toTime'\";" .
-        "set grid;" .
-        "set boxwidth;" .
-        "set yrange [\"0\":\"$maxY\"];" .
-        "set style data histogram;" .
-        "set style histogram rowstacked;" .
-        "set style fill solid border -1;" .
-        "set ylabel \"Exposures\";" .
-        "set boxwidth 0.75;" .
-        "plot '$inputFile' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . 
-        "\n";
-
-    close GP;
 }