Changeset 31886

Timestamp:

Jul 15, 2011, 4:16:20 PM (15 years ago)

Author:

rhenders

Message:

all time-series plots now created at same time for speed

Location:

trunk/tools

Files:

• czarplot.pl (modified) (4 diffs)
• czarpoll.pl (modified) (3 diffs)
• czartool/CzarDb.pm (modified) (7 diffs)
• czartool/Metrics.pm (modified) (1 diff)
• czartool/Plotter.pm (modified) (9 diffs)

trunk/tools/czarplot.pl

@@ -26 +26 @@
 my $rateInterval = undef;
 my $deriv = undef;
-my $showCleanup = undef;
 my $nebulous = undef;
 my $savingToFile = undef;
@@ -46 +45 @@
         "histogram|h" => \$histogram,
         "nebulous|n" => \$nebulous,
-        "cleanup|c" => \$showCleanup,
         "rate|r" => \$rate,
         "deriv|f" => \$deriv,
@@ -81 +79 @@
     $analysis = 0;
     print "* OPTIONAL: include analysis                -a                          (default=$analysis)\n";} 
-if (!$showCleanup) {
-    $showCleanup = 0;
-    print "* OPTIONAL: include cleanup in timeseries   -c                          (default=$showCleanup)\n";} 
 if (!$log) {
     $log = 0;
@@ -166 +161 @@
 }
 
-if ($rate) {
+if ($rate and !$timeSeries) {
 
-    # default to a rate histogram if time series not selected
-    if (!$timeSeries) {$histogram = 1;}
-    $plotter->createRatePlot($label, $stage, $begin, $end, $rateInterval, $histogram, $timeSeries);
+        $plotter->createRateHistogram($label, $stage, $begin, $end, $rateInterval);
+        exit;
+}
+
+if (!$histogram && !$nebulous) {
+    $plotter->createTimeSeries($label, $stage, $begin, $end, $timeSeries, $log, $rate);
     exit;
 }
-if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log, $showCleanup, $deriv);}
+
 if ($histogram) {$plotter->createHistogram($label, $begin, $end);}
 if ($nebulous && $timeSeries) {$plotter->plotStorageTimeSeries($begin, $end);}

trunk/tools/czarpoll.pl

@@ -252 +252 @@
 
             chomp($label);
-            $plotter->createLogAndLinearTimeSeries($label,  $stage, $begin, $end);
-            $plotter->createRatePlot($label, $stage, $begin, $end, undef, 1, 1);
+            $plotter->createTimeSeries($label, $stage, $begin, $end, 1, 1, 1);
+            $plotter->createRateHistogram($label, $stage, $begin, $end, undef);
         }
     }
@@ -263 +263 @@
         my ($label) = @{$row};
 
-        $plotter->createLogAndLinearTimeSeries($label, undef, $begin, $end);
-        $plotter->createRatePlot($label, undef, $begin, $end, undef, 1, 1);
+        $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1);
+        $plotter->createRateHistogram($label, undef, $begin, $end, undef);
         $plotter->createHistogram($label, $begin, $end);
     }
@@ -270 +270 @@
     $allServerLabels = "all_".$server."_labels";
 
-    $plotter->createLogAndLinearTimeSeries($allServerLabels, undef, $begin, $end);
-    $plotter->createRatePlot($allServerLabels, undef, $begin, $end, undef, 1, 1);
+    $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 1, 1);
+    $plotter->createRateHistogram($allServerLabels, undef, $begin, $end, undef);
     $plotter->createHistogram($allServerLabels, $begin, $end);
     foreach $stage (@stages) {
 
-        $plotter->createLogAndLinearTimeSeries($allServerLabels, $stage, $begin, $end); # TODO must be a neater way...
-            $plotter->createRatePlot($allServerLabels, $stage, $begin, $end, undef, 1, 1);
+        $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 1, 1); # TODO must be a neater way...
+        $plotter->createRateHistogram($allServerLabels, $stage, $begin, $end, undef);
     }
 }

trunk/tools/czartool/CzarDb.pm

@@ -590 +590 @@
 ###########################################################################
 #
-# Gets time series data and stores it to temp file
+# Gets time series data and stores it to temp files
 #
 ###########################################################################
 sub createTimeSeriesData {
-    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $dataFile, $isLog, $showCleanup, $firstDeriv) = @_;
-
-    my $query = $self->{_db}->prepare(<<SQL);
-    SELECT 
-        MIN(processed)
+    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $linDataFile, $logDataFile, $ratDataFile) = @_;
+
+    # get total number of data points
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT  COUNT(*)
         FROM $stage 
         WHERE label LIKE '$label'
@@ -605 +605 @@
 
     if (!$query->execute) {return 0;}
-    my $minProcessed = scalar  $query->fetchrow_array();
-    if (!defined $minProcessed) {return 0;}
-
+    my ($count) = $query->fetchrow_array();
+
+    # decide on a number for the running mean for rate plots - use 10% of total data points
+    my $range = int($count/10);
     $self->getTimeMinMaxDiff($label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff);
-
+    # grab all the data for the provided time range
     $query = $self->{_db}->prepare(<<SQL);
     SELECT 
@@ -621 +622 @@
     $query->execute;
 
+    # set up Gnuplot datafiles
     my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
     $tmpFile->unlink_on_destroy( 0 );
-    ${$dataFile} = $tmpFile->filename;
-    open (GNUDAT, ">${$dataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+    ${$linDataFile} = $tmpFile->filename;
+    open (LINDAT, ">${$linDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+
+    $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$logDataFile} = $tmpFile->filename;
+    open (LOGDAT, ">${$logDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
+
+    $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_".$stage."_t.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
+    $tmpFile->unlink_on_destroy( 0 );
+    ${$ratDataFile} = $tmpFile->filename;
+    open (RATDAT, ">${$ratDataFile}") or print "* Problem opening gnuplot data file for plot for '$label' '$stage'\n";
 
     my $lastProcessed = -1;
@@ -633 +645 @@
     my ($logProcessed, $logPending, $logFaults);
     my ($derivProcessed, $derivPending, $derivFaults);
-    my $timestamp = undef;
+    my (@rateProcessed, @ratePending, @rateFaults);
+    my $formattedTimestamp = undef;
+    my $middleTime = undef;
+    my $formattedMiddleTimestamp = undef;
     my $positiveJump = 0;
+    my @rateTimes;
+    my $diffPending = 0;
+    my $diffProcessed = 0;
+    my $diffFaults = 0;
+    my $lastDiffProcessed = 0;
+    $derivProcessed = 0;
 
     $linearProcessed = 0;
     my $someData = 0;
+    my $timeSep;
     while (my @row = $query->fetchrow_array()) {
+
         my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
 
-
-        # get first derivative values
+        $formattedTimestamp = $self->getFormattedDate($thisTimestamp);
+
+        # if not first time in....
         if (defined $lastTimestamp) {
 
-            my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
-                
-            if ($thisProcessed >= $lastProcessed && $timeDiff){
-                $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
+            # times
+            $timeSep = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
+            $middleTime = $self->getMiddleTime($lastTimestamp, $thisTimestamp);
+            $formattedMiddleTimestamp = $self->getFormattedDate($middleTime);
+
+            $diffPending = $thisPending - $lastPending;
+            $diffFaults = $thisFaults - $lastFaults;
+
+            # we only count increases in processing - drops are due to cleanup etc
+            if ($thisProcessed > $lastProcessed) {
+
+                $diffProcessed = ($thisProcessed - $lastProcessed) - $positiveJump;
+                $linearProcessed = $linearProcessed + $diffProcessed;
             }
-            else {$derivProcessed = 0;}
+            else {
+            
+                $diffProcessed = 0;
+                $linearProcessed = $linearProcessed;
+            }
+
+            # calculate first derivative in units of images per hour
+            $derivPending = $diffPending/($timeSep/3600); 
+            $derivFaults = $diffFaults/($timeSep/3600); 
+            $derivProcessed = $diffProcessed/($timeSep/3600); 
+
+            # look for large positive jumps in processed, most likely a label added, and store value to subtract from linear value below
+            if ($derivProcessed > 1000) {
+                $positiveJump = $thisProcessed - $lastProcessed;
+            }
+            else {
+
+                $positiveJump = 0;
+            }
+        }
+        # first time in
+        else {
+        
             $derivPending = 0;
             $derivFaults = 0;
-        #    $derivPending = abs($thisPending - $lastPending)/$timeDiff;    
-        #    $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff;    
-
-            # look for large positive jumps in processed, most likely a label added, and store value to subtract from linear value below
-            if ($derivProcessed > 0.3) {
-
-                $positiveJump = $thisProcessed - $lastProcessed;
-                #print "$derivProcessed at $thisTimestamp   $positiveJump\n";
+            $derivProcessed = 0;
+            $formattedMiddleTimestamp = $formattedTimestamp;
         }
-            else {
-            $positiveJump = 0;
+
+        # deal with rate plot
+        push(@ratePending, $derivPending);
+        push(@rateFaults, $derivFaults);
+        push(@rateProcessed, $derivProcessed);
+        push(@rateTimes, $middleTime);
+
+        # get last few values
+        my $size = scalar (@rateProcessed);
+        if ($size > $range) {
+
+            my $counter = 0;
+            my $oldTime;
+            my $newTime = $rateTimes[$size-1];
+            my $totalProcessed = 0;
+            my $totalPending = 0;
+            my $totalFaults = 0;
+            for (my $i=$size-1; $i>=($size - $range); $i--) { 
+               
+                $totalPending = $totalPending + $ratePending[$i];
+                $totalFaults = $totalFaults + $rateFaults[$i];
+                $totalProcessed = $totalProcessed + $rateProcessed[$i];
+                $oldTime = $rateTimes[$i];
+                $counter++;
             }
+
+            my $meanPending = $totalPending / $counter;
+            my $meanFaults = $totalFaults / $counter;
+            my $meanProcessed = $totalProcessed / $counter;
+
+            my $formattedTimestamp = $self->getFormattedDate($self->getMiddleTime($oldTime, $newTime));
+            print RATDAT "$formattedTimestamp $meanPending $meanFaults $meanProcessed\n";
         }
-        else {
-        
-            $derivProcessed = 0;
-            $derivPending = 0;
-            $derivFaults = 0;
-        }
-
-        # get linear values
-        if ($showCleanup) {
-        
-            $linearProcessed = $thisProcessed - $minProcessed;
-        }
-        elsif($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
-            
-            $linearProcessed = $linearProcessed + ($thisProcessed - $lastProcessed) - $positiveJump;
-        }
+
         $linearPending = $thisPending;
         $linearFaults = $thisFaults;
@@ -694 +757 @@
 
         # print to data file
-        $timestamp = $self->getFormattedDate($thisTimestamp);
-        if ($firstDeriv) {print GNUDAT "$timestamp $derivPending $derivFaults $derivProcessed\n";}
-        elsif($isLog) {print GNUDAT "$timestamp $logPending $logFaults $logProcessed\n";}
-        else {print GNUDAT "$timestamp $linearPending $linearFaults $linearProcessed\n";}
+        print LOGDAT "$formattedTimestamp $logPending $logFaults $logProcessed\n";
+        print LINDAT "$formattedTimestamp $linearPending $linearFaults $linearProcessed\n";
 
         if ($linearPending > 0 || $linearFaults > 0 || $linearProcessed > 0) {$someData = 1;}
@@ -705 +766 @@
         $lastFaults = $thisFaults;
         $lastTimestamp = $thisTimestamp;
-    }
-
-    close(GNUDAT) or print "* Problem closing gnuplot data file for plot for '$label' '$stage'\n";
-
-    #return $someData;
+        $lastDiffProcessed = $diffProcessed;
+    }
+
+    close(LINDAT) or print "* Problem closing linear gnuplot data file for plot for '$label' '$stage'\n";
+    close(LOGDAT) or print "* Problem closing log gnuplot data file for plot for '$label' '$stage'\n";
+    close(RATDAT) or print "* Problem closing rate gnuplot data file for plot for '$label' '$stage'\n";
 }
 
@@ -1017 +1079 @@
 #
 # Deletes all but one row per interval from all tables for the provided date range
-# TODO this is very clumsy, I just have time to thinmk of something more elegant
+# TODO this is very clumsy, I just don't have time to think of something more elegant
 #
 ###########################################################################

trunk/tools/czartool/Metrics.pm

@@ -77 +77 @@
     print $htmlFile "<a name=\"summaryplot\"></a>\n";
     print $htmlFile "<h2  align=\"middle\">Summary plots <a href=\"#top\">(top)</a></h1>\n";
-    $self->{plotter}->createTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, 0, 0, 0);
+    $self->{plotter}->createTimeSeries("all_stdscience_labels", undef, $self->{begin}, $self->{end}, 1, 1, 1);
     print $htmlFile "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_t.png\" alt=\"timeseries\" />\n";
     $self->{plotter}->createHistogram("all_stdscience_labels", $self->{begin}, $self->{end}, 0, 0, 0);
     print $htmlFile "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_h.png\" alt=\"histogram\" />\n";
-    $self->{plotter}->createRatePlot("all_stdscience_labels", undef, $self->{begin}, $self->{end}, undef, 1, 1);
+    $self->{plotter}->createRateHistogram("all_stdscience_labels", undef, $self->{begin}, $self->{end}, undef);
     print $htmlFile "<img src=\"czarplot_linear_all_stdscience_labels_all_stages_rh.png\" alt=\"rate\" />\n";
     $self->{plotter}->plotStorageTimeSeries($self->{begin}, $self->{end});

trunk/tools/czartool/Plotter.pm

@@ -97 +97 @@
     my $type = $isLog ? "log" : "linear";
     return $prefix . "/czarplot_" . $type . "_" . $label . "_" . $stagePart . "_$suffix.png";
-}
-
-###########################################################################
-#
-# Plots a time series for all stages for this label, both log and linear
-#
-###########################################################################
-sub createLogAndLinearTimeSeries {
-    my ($self, $label, $selectedStage, $beginTime, $endTime) = @_;
-
-    $self->createTimeSeries($label, $selectedStage, $beginTime, $endTime, 1);
-    $self->createTimeSeries($label, $selectedStage, $beginTime, $endTime, 0);
 }
 
@@ -146 +134 @@
 #
 ###########################################################################
-sub createRatePlot {
-    my ($self, $label, $selectedStage, $beginTime, $endTime, $interval, $histogram, $timeSeries) = @_;
+sub createRateHistogram {
+    my ($self, $label, $selectedStage, $beginTime, $endTime, $interval) = @_;
 
     # stages
@@ -184 +172 @@
     }
 
-    if ($histogram) {
-
-        my $outputFile = createImageFileName($self, $label, $selectedStage, "rh", 0);
-        $self->plotRateStackedHistogram(\%gnuplotFiles, $outputFile, $label, $selectedStage, $beginTime, $endTime, $interval);
-    }
-    if ($timeSeries) {
-
-        my $outputFile = createImageFileName($self, $label, $selectedStage, "rt", 0);
-        $self->plotTimeSeries(\%gnuplotFiles, $outputFile, $label, $beginTime, $endTime, $maxX, $minX, $timeDiff, "", "Exposures processed per $interval");
-    }
+    my $outputFile = createImageFileName($self, $label, $selectedStage, "rh", 0);
+    $self->plotRateStackedHistogram(\%gnuplotFiles, $outputFile, $label, $selectedStage, $beginTime, $endTime, $interval);
 
     # now delete temp dat files
@@ -250 +230 @@
 ###########################################################################
 sub createTimeSeries {
-    my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog, $showCleanup, $deriv) = @_;
+    my ($self, $label, $selectedStage, $beginTime, $endTime, $linear, $log, $rate) = @_;
 
     my $minX = 999999999;      
@@ -261 +241 @@
     else {$stages = ["$selectedStage"]};        
 
-    my %gnuplotFiles;
+    my %linDataFiles;
+    my %logDataFiles;
+    my %ratDataFiles;
     my $stage = undef;
-    my $gnuplotFile = undef;
+    my $linDataFile = undef;
+    my $logDataFile = undef;
+    my $ratDataFile = undef;
+
     foreach $stage (@{$stages}) {
 
@@ -274 +259 @@
                     \$maxX, 
                     \$timeDiff, 
-                    \$gnuplotFile,
-                    $isLog,
-                    $showCleanup,
-                    $deriv)) {
-
-            $gnuplotFiles{$stage} = $gnuplotFile;
+                    \$linDataFile,
+                    \$logDataFile,
+                    \$ratDataFile)) {
+
+            $linDataFiles{$stage} = $linDataFile;
+            $logDataFiles{$stage} = $logDataFile;
+            $ratDataFiles{$stage} = $ratDataFile;
         }
     }                                                           
 
-    my $numOfPlots =  keys(%gnuplotFiles);
+    my $numOfPlots =  keys(%linDataFiles);
 
     if ($numOfPlots == 0 ) {
@@ -297 +283 @@
     }
 
-    my $outputFile = createImageFileName($self, $label, $selectedStage, "t", $isLog);
-    my $yTitle;
-    if ($isLog) {$yTitle = "Log(Exposures)";}
-    elsif ($deriv) {$yTitle = "dExposures/dTime(secs)";}
-    else {$yTitle = "Exposures";}
-    $self->plotTimeSeries(
-            \%gnuplotFiles, 
-            $outputFile, 
-            $label, 
-            $beginTime, 
-            $endTime, 
-            $maxX, 
-            $minX, 
-            $timeDiff, 
-            "",
-            $yTitle);
-}                                                    
+    if ($linear) {
+
+        my $outputFile = createImageFileName($self, $label, $selectedStage, "t", 0);
+
+        $self->plotTimeSeries(
+                \%linDataFiles, 
+                $outputFile, 
+                $label, 
+                $beginTime, 
+                $endTime, 
+                $maxX, 
+                $minX, 
+                $timeDiff, 
+                "",
+                "Exposures");
+    }
+
+    if ($log) {
+
+        my $outputFile = createImageFileName($self, $label, $selectedStage, "t", 1);
+
+        $self->plotTimeSeries(
+                \%logDataFiles, 
+                $outputFile, 
+                $label, 
+                $beginTime, 
+                $endTime, 
+                $maxX, 
+                $minX, 
+                $timeDiff, 
+                "",
+                "Log(Exposures)");
+    }
+
+    if ($rate) {
+
+        my $outputFile = createImageFileName($self, $label, $selectedStage, "rt", 0);
+
+        $self->plotTimeSeries(
+                \%ratDataFiles, 
+                $outputFile, 
+                $label, 
+                $beginTime, 
+                $endTime, 
+                $maxX, 
+                $minX, 
+                $timeDiff, 
+                "",
+                "Exposures processed per hour");
+}    
+}                                                
 
 ###########################################################################
@@ -468 +488 @@
     my $timeFormat = undef;
     my $divX = undef;
-    #my $yTitle = undef;
-    #if ($isLog) {$yTitle = "Log( Exposures )";}
-    #elsif ($isDeriv) {$yTitle = "dExposures/dTime(secs)";}
-    #else {$yTitle = "Exposures";}
 
     $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
@@ -478 +494 @@
 
     # sort out plot title
-    #my $title = "";
-    #if ($isDeriv) {$title .= "First derivatives of "}
     if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title .= "'".$stage."'";}}
     else {$title .= "'all stages'"}