IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29437


Ignore:
Timestamp:
Oct 15, 2010, 12:34:37 PM (16 years ago)
Author:
rhenders
Message:

Added first derivative plots

Location:
trunk/tools
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/tools/czarplot.pl

    r29375 r29437  
    2323my $timeSeries = undef;
    2424my $rate = undef;
     25my $deriv = undef;
    2526my $showCleanup = undef;
    2627my $nebulous = undef;
    … …  
    2930
    3031GetOptions (
    31         "dbname|d=s" => \$czarDbName,
     32        "dbname=s" => \$czarDbName,
    3233        "label|l=s" => \$label,
    3334        "stage|s=s" => \$stage,
    … …  
    4142        "cleanup|c" => \$showCleanup,
    4243        "rate|r" => \$rate,
     44        "deriv|d" => \$deriv,
    4345        "timeseries|t" => \$timeSeries,
    4446        "verbose|v" => \$verbose,
    … …  
    5961if (!$rate) {
    6062    print "* OPTIONAL: plot timeseries of rate         -r                          (default=off)\n";}
     63if (!$deriv) {
     64    $deriv = 0;
     65    print "* OPTIONAL: plot first derivative           -d                          (default=$deriv)\n";}
    6166if (!$showCleanup) {
    6267    $showCleanup = 0;
    … …  
    97102$czarDb->setDateFormat("%Y%m%d-%H%i%s");
    98103
     104
    99105my $plotter = new czartool::Plotter(
    100106        $czarDb,
    … …  
    128134    exit;
    129135}
    130 if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log, $showCleanup);}
     136if (!$nebulous && $timeSeries) {$plotter->createTimeSeries($label, $stage, $begin, $end, $log, $showCleanup, $deriv);}
    131137if ($histogram) {$plotter->createHistogram($label, $begin, $end);}
    132138if ($nebulous && $timeSeries) {$plotter->plotStorageTimeSeries($begin, $end);}

  • trunk/tools/czartool/CzarDb.pm

    r29377 r29437  
    271271###########################################################################
    272272sub createTimeSeriesData {
    273     my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $dataFile, $isLog, $showCleanup) = @_;
     273    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $timeDiff, $dataFile, $isLog, $showCleanup, $firstDeriv) = @_;
    274274
    275275    my $query = $self->{_db}->prepare(<<SQL);
    … …  
    289289    $query = $self->{_db}->prepare(<<SQL);
    290290    SELECT
    291         DATE_FORMAT(timestamp, '$self->{_dateFormat}'), pending, faults, processed
     291        timestamp, pending, faults, processed
    292292        FROM $stage
    293293        WHERE label LIKE '$label'
    … …  
    302302
    303303    my $lastProcessed = -1;
     304    my $lastPending = -1;
     305    my $lastFaults = -1;
     306    my $lastTimestamp = undef;
    304307    my $processed = 0;
     308    my $runningProcessed = 0;
     309    my $pending = 0;
     310    my $faults = 0;
     311    my $timestamp = undef;
    305312    while (my @row = $query->fetchrow_array()) {
    306         my ($timestamp, $pending, $faults, $thisProcessed) = @row;
    307 
    308         if ($showCleanup) {
     313        my ($thisTimestamp, $thisPending, $thisFaults, $thisProcessed) = @row;
     314
     315        if ($showCleanup ) {
    309316       
    310         $processed = $thisProcessed - $minProcessed;
     317            $runningProcessed = $thisProcessed - $minProcessed;
    311318        }
    312319        elsif($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
    313320           
    314             $processed = $processed + ($thisProcessed - $lastProcessed);
    315         }
    316 
    317         if ($isLog) {
    318 
    319             if ($processed < 0) {$processed = 0;}
    320             if ($pending < 0) {$pending = 0;}
    321             if ($faults < 0) {$faults = 0;}
    322 
    323             $processed = log($processed + 1)/log(10);
    324             $pending = log($pending + 1)/log(10);
     321            $runningProcessed = $runningProcessed + ($thisProcessed - $lastProcessed);
     322        }
     323
     324
     325        if ($firstDeriv) {
     326
     327            if (defined $lastTimestamp) {
     328
     329                my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
     330               
     331                if ($thisProcessed > $lastProcessed){
     332                $processed = abs($thisProcessed - $lastProcessed)/$timeDiff;
     333#print "$processed = ($thisProcessed - $lastProcessed)/$timeDiff\n";
     334                }
     335                else {$processed = 0;}
     336                #$pending = abs($thisPending - $lastPending)/$timeDiff;   
     337                #$faults = abs($thisFaults - $lastFaults)/$timeDiff;   
     338            }
     339            else {
     340                $processed = 0;
     341                $pending = 0;
     342                $faults = 0;
     343            }
     344        }
     345        elsif ($isLog) {
     346
     347            $processed = ($runningProcessed < 0) ? 0 : $runningProcessed;
     348            $pending =  ($thisPending < 0) ? 0 : $thisPending;
     349            $faults =  ($thisFaults < 0) ? 0 : $thisFaults;
     350
     351            $processed = log($runningProcessed + 1)/log(10);
     352            $pending = log($thisPending + 1)/log(10);
    325353            $faults = log($faults + 1)/log(10);
    326354        }
     355        else {
     356       
     357            $processed = $runningProcessed;
     358            $pending = $thisPending;
     359            $faults = $thisFaults;
     360        }
     361
     362        $timestamp = $self->getFormattedDate($thisTimestamp);
    327363
    328364        print GNUDAT "$timestamp $pending $faults $processed\n";
     365
    329366        $lastProcessed = $thisProcessed;
     367        $lastPending = $thisPending;
     368        $lastFaults = $thisFaults;
     369        $lastTimestamp = $thisTimestamp;
    330370    }
    331371
    … …  
    568608###########################################################################
    569609#
     610# When did this stage finish?
     611#
     612###########################################################################
     613sub getFinishTime {
     614    my ($self, $label, $stage, $begin, $end) = @_;
     615
     616    my $query = $self->{_db}->prepare(<<SQL);
     617    SELECT
     618        timestamp, pending, faults
     619        FROM $stage
     620        WHERE label LIKE '$label'
     621        AND timestamp >= '$begin'
     622        AND timestamp <= '$end'
     623SQL
     624    $query->execute;
     625
     626    my $array = $query->fetchall_arrayref();
     627
     628    my $row;
     629    my $thisPending = -1;
     630    my $lastPending = -1;
     631    my $thisFaults = -1;
     632    my $lastFaults = -1;
     633    my $timestamp;
     634    my $started = undef;
     635    my $finished = undef;
     636    my $thisLeft = 0;
     637    my $lastLeft = 0;
     638    my $finishTimeout = "01:00:00";
     639    my $numRows = @{$array};
     640    print "$numRows rows\n";
     641    my $n = 0;
     642    foreach $row ( @{$array} ){
     643
     644        ($timestamp, $thisPending, $thisFaults) = @{$row};
     645
     646#        print "$timestamp, $thisPending, $thisFaults\n";
     647
     648        $thisLeft = $thisPending - $thisFaults;
     649        $lastLeft = $lastPending - $lastFaults;
     650
     651
     652        if ($n == 0 && $thisLeft > 0) {
     653
     654            print "Starting this time period with stuff pending ($thisLeft)\n";
     655            $started = $timestamp;
     656        }
     657
     658        if (!defined $started && $lastLeft == 0 && $thisLeft > 0) {
     659
     660            $started = $timestamp;
     661            print "STARTED at $started\n";
     662        }
     663
     664        if ($started && !defined $finished && $thisLeft == 0) {
     665
     666            $finished = $timestamp;
     667            print "setting FINISHED to $finished\n";
     668        }
     669
     670        if (defined $finished) {
     671
     672            if ($thisLeft != 0) {
     673               
     674                $finished = undef;
     675           
     676                print "NEW pending  at $timestamp\n";
     677           
     678            }
     679            elsif ($thisLeft == 0) {
     680               
     681                my $howLongFinished = $self->diffTimes($timestamp, $finished);
     682
     683print "$howLongFinished = $timestamp, $finished\n";
     684
     685                if ($self->isIntervalGreaterThan($howLongFinished, $finishTimeout)) {
     686                   
     687                    print "0 pending for $howLongFinished. We're done \n";
     688                    last;
     689                }
     690
     691            }
     692
     693        }
     694
     695
     696        $lastPending = $thisPending;
     697        $lastFaults = $thisFaults;
     698        $n++;
     699        if ($n == $numRows) {
     700
     701            if ($thisLeft == 0) {
     702
     703                print "exceeded interval before reaching finished timeout\n";
     704            }
     705            else {
     706                print "Reached end of interval and not finished yet. $thisLeft left\n";
     707            }
     708        }
     709    }
     710
     711    if ($finished) {
     712
     713        my $timeTaken = $self->diffTimes($finished, $started);
     714        print "FINISHED at $finished with $thisFaults remaining faults, time taken = $timeTaken\n";
     715    }
     716
     717}
     718
     719###########################################################################
     720#
     721# Figures out if a stage has plateaued
     722#
     723###########################################################################
     724sub hasPlateaued {
     725    my ($self, $label, $stage, $end, $interval) = @_;
     726
     727    my $query = $self->{_db}->prepare(<<SQL);
     728    SELECT
     729        timestamp, pending, faults
     730        FROM $stage
     731        WHERE label LIKE '$label'
     732        AND timestamp >= '$end - INTERVAL $interval'
     733        AND timestamp <= '$end'
     734        ORDER BY timestamp DESC;
     735SQL
     736    $query->execute;
     737
     738    my $array = $query->fetchall_arrayref();
     739
     740    my $thisPending = -1;
     741    my $lastPending = -1;
     742    my $thisFaults = -1;
     743    my $lastFaults = -1;
     744    my $timestamp;
     745    my $thisLeft = 0;
     746    my $lastLeft = 0;
     747    my $row;
     748    my $numRows = @{$array};
     749    my $n = 0;
     750    foreach $row ( @{$array} ){
     751
     752        ($timestamp, $thisPending, $thisFaults) = @{$row};
     753
     754        $thisLeft = $thisPending - $thisFaults;
     755        $lastLeft = $lastPending - $lastFaults;
     756
     757        print "$timestamp $thisPending\n";
     758
     759        if ($n>0) {
     760
     761            #if () {}
     762
     763        }
     764
     765        $lastPending = $thisPending;
     766        $lastFaults = $thisFaults;
     767        $n++;
     768    }
     769}
     770
     771###########################################################################
     772#
    570773# Gets count of processed stuff in a given time period
    571774#
    … …  
    582785        AND timestamp < '$end'
    583786SQL
    584     $query->execute;
     787        $query->execute;
    585788
    586789    my $processedArray = $query->fetchall_arrayref();
    … …  
    614817
    615818        if (!$self->isBefore($thisDay, $endDay)) {
    616        
     819
    617820            $quit = 1;
    618821        }
    … …  
    645848        $toTime = $self->addInterval($fromTime, $interval);
    646849        if (!$self->isBefore($toTime, $endDay)) {
    647        
     850
    648851            $toTime = $endDay;
    649852            $quit = 1;
    650853        }
    651    
     854
    652855        my $stage = undef;
    653856        $totalDeleted = 0;
    … …  
    703906                    AND server = '$server'
    704907SQL
    705                 $query->execute;
     908                    $query->execute;
    706909
    707910                my $toDelete = scalar $query->fetchrow_array() - 1;
    … …  
    714917                        AND server = '$server' ORDER BY timestamp DESC LIMIT $toDelete
    715918SQL
    716                     $query->execute;
     919                        $query->execute;
    717920
    718921                    $totalDeleted += $toDelete;
    … …  
    728931            AND timestamp <= '$toTime'
    729932SQL
    730         $query->execute;
     933            $query->execute;
    731934
    732935        my $toDelete = scalar $query->fetchrow_array() - 1;
    … …  
    739942                ORDER BY timestamp DESC LIMIT $toDelete
    740943SQL
    741             $query->execute;
     944                $query->execute;
    742945
    743946            $totalDeleted += $toDelete;
    … …  
    755958###########################################################################
    756959sub optimize {
    757         my ($self) = @_;
     960    my ($self) = @_;
    758961
    759962    my $stage = undef;
    … …  
    782985SQL
    783986
    784     $query->execute;
     987        $query->execute;
    785988    my $numOfReadings = scalar $query->fetchrow_array();
    786989
    … …  
    795998SQL
    796999
    797     $query->execute;
     1000        $query->execute;
    7981001    my $numNotRunning = scalar $query->fetchrow_array();
    7991002
    … …  
    8931096    INSERT INTO revision (revision) VALUES ($revision);
    8941097SQL
    895     $query->execute;
     1098        $query->execute;
    8961099}
    8971100
    … …  
    9121115SQL
    9131116
    914     $query->execute;
     1117        $query->execute;
    9151118    my @row = $query->fetchrow_array();
    9161119
    … …  
    9401143    foreach $stage (@stages) {
    9411144
    942     my $query = $self->{_db}->prepare(<<SQL);
     1145        my $query = $self->{_db}->prepare(<<SQL);
    9431146        CREATE TABLE $stage (
    9441147                timestamp TIMESTAMP DEFAULT NOW(),
    … …  
    9691172    foreach $stage (@stages) {
    9701173
    971     my $query = $self->{_db}->prepare(<<SQL);
     1174        my $query = $self->{_db}->prepare(<<SQL);
    9721175        ALTER TABLE $stage
    9731176            ADD COLUMN reverting TINYINT NOT NULL DEFAULT 0;
    … …  
    9941197    foreach $stage (@stages) {
    9951198
    996     my $query = $self->{_db}->prepare(<<SQL);
     1199        my $query = $self->{_db}->prepare(<<SQL);
    9971200        ALTER TABLE $stage
    9981201            ADD COLUMN processed BIGINT NOT NULL DEFAULT 0;
    9991202SQL
    10001203
    1001       $query->execute;
     1204            $query->execute;
    10021205    }
    10031206
    … …  
    10191222SQL
    10201223
    1021       $query->execute;
     1224        $query->execute;
    10221225
    10231226    $self->setRevision(4);
    … …  
    10411244SQL
    10421245
    1043       $query->execute;
     1246        $query->execute;
    10441247
    10451248    $self->setRevision(5);
    … …  
    10651268SQL
    10661269
    1067       $query->execute;
    1068     }
    1069 
    1070         my $query = $self->{_db}->prepare(<<SQL);
    1071         CREATE INDEX serverIndex ON servers (timestamp, server);
    1072 SQL
    1073 
    1074       $query->execute;
     1270            $query->execute;
     1271    }
     1272
     1273    my $query = $self->{_db}->prepare(<<SQL);
     1274    CREATE INDEX serverIndex ON servers (timestamp, server);
     1275SQL
     1276
     1277        $query->execute;
    10751278
    10761279
    … …  
    10931296SQL
    10941297
    1095     $query->execute;
     1298        $query->execute;
    10961299
    10971300    $self->setRevision(7);
    … …  
    11171320SQL
    11181321
    1119     $query->execute;
     1322            $query->execute;
    11201323    }
    11211324
    … …  
    11271330SQL
    11281331
    1129     $query->execute;
     1332        $query->execute;
    11301333
    11311334    # insert stages into revert table
    … …  
    11371340            ('$stage', 0);
    11381341SQL
    1139        $query->execute;
     1342            $query->execute;
    11401343    }
    11411344
    … …  
    11621365SQL
    11631366
    1164       $query->execute;
     1367        $query->execute;
    11651368    $query = $self->{_db}->prepare(<<SQL);
    11661369    CREATE TABLE hosts (
    … …  
    11711374SQL
    11721375
    1173       $query->execute;
    1174 
    1175         $query = $self->{_db}->prepare(<<SQL);
    1176         CREATE INDEX clusterSpaceIndex ON cluster_space (timestamp);
    1177 SQL
    1178 
    1179       $query->execute;
     1376        $query->execute;
     1377
     1378    $query = $self->{_db}->prepare(<<SQL);
     1379    CREATE INDEX clusterSpaceIndex ON cluster_space (timestamp);
     1380SQL
     1381
     1382        $query->execute;
    11801383
    11811384    $self->setRevision(9);
    … …  
    11951398    ALTER TABLE hosts
    11961399        ADD COLUMN writable TINYINT NOT NULL DEFAULT 0,
    1197         ADD COLUMN readable TINYINT NOT NULL DEFAULT 0;
    1198 SQL
    1199 
    1200     $query->execute;
     1400            ADD COLUMN readable TINYINT NOT NULL DEFAULT 0;
     1401SQL
     1402
     1403        $query->execute;
    12011404
    12021405    $self->setRevision(10);
    … …  
    12231426SQL
    12241427
    1225       $query->execute;
    1226         $query = $self->{_db}->prepare(<<SQL);
    1227         CREATE INDEX burntoolIndex ON burntool (timestamp, label);
    1228 SQL
    1229 
    1230       $query->execute;
     1428        $query->execute;
     1429    $query = $self->{_db}->prepare(<<SQL);
     1430    CREATE INDEX burntoolIndex ON burntool (timestamp, label);
     1431SQL
     1432
     1433        $query->execute;
    12311434
    12321435    $self->setRevision(11);
    … …  
    12501453SQL
    12511454
    1252     $query->execute;
     1455        $query->execute;
    12531456
    12541457    $self->setRevision(12);

  • trunk/tools/czartool/Plotter.pm

    r29377 r29437  
    130130###########################################################################
    131131sub createTimeSeries {
    132     my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog, $showCleanup) = @_;
     132    my ($self, $label, $selectedStage, $beginTime, $endTime, $isLog, $showCleanup, $deriv) = @_;
    133133
    134134    my $minX = 999999999;     
    … …  
    156156                    \$gnuplotFile,
    157157                    $isLog,
    158                     $showCleanup)) {
     158                    $showCleanup,
     159                    $deriv)) {
    159160
    160161            $gnuplotFiles{$stage} = $gnuplotFile;
    … …  
    186187            $minX,
    187188            $timeDiff,
    188             $isLog);
     189            $isLog,
     190            $deriv);
    189191}                                                   
    190192
    … …  
    325327###########################################################################
    326328sub plotTimeSeries {
    327     my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $timeDiff, $isLog) = @_;
     329    my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $timeDiff, $isLog, $isDeriv) = @_;
    328330
    329331    my $timeFormat = undef;
    330332    my $divX = undef;
    331333    my $yTitle = undef;
    332     if ($isLog) {$yTitle = "Log( numExposures )";}
    333     else {$yTitle = "numExposures";}
     334    if ($isLog) {$yTitle = "Log( Exposures )";}
     335    elsif ($isDeriv) {$yTitle = "dExposures/dTime";}
     336    else {$yTitle = "Exposures";}
    334337
    335338    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
    336339
    337340    my $numOfPlots = keys %$gnuplotFiles;
    338     my $title = undef;
    339341
    340342    # sort out plot title
    341     if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title = "'".$stage."'";}}
    342     else {$title = "'All stages'"}
     343    my $title = "";
     344    if ($isDeriv) {$title .= "First derivatives of "}
     345    if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title .= "'".$stage."'";}}
     346    else {$title .= "'all stages'"}
    343347
    344348    $title .= " for '$label', '$fromTime' to '$toTime'";
    … …  
    369373        if ($numOfPlots == 1) {
    370374
     375            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4 lw 2,";
    371376            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"Processed\" with lines lt 2 lw 2,";
    372             print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4 lw 2,";
    373377            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"Faults\" with lines lt 7 lw 2";
    374378        }
Note: See TracChangeset for help on using the changeset viewer.