IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 28980


Ignore:
Timestamp:
Aug 19, 2010, 6:50:13 PM (16 years ago)
Author:
eugene
Message:

updates from trunk

Location:
branches/eam_branches/ipp-20100621
Files:
1 deleted
44 edited
25 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20100621/ippMonitor/Makefile.in

    r28794 r28980  
    103103$(DESTWWW)/chipProcessedImfile.php \
    104104$(DESTWWW)/chipProcessedImfile_failure.php \
     105$(DESTWWW)/failedChipProcessedImfile.php \
    105106$(DESTWWW)/camSummary.php \
    106107$(DESTWWW)/camStageExp.php \
    … …  
    109110$(DESTWWW)/camProcessedExp_Images.php \
    110111$(DESTWWW)/camProcessedExp_failure.php \
     112$(DESTWWW)/failedCamProcessedExp.php \
    111113$(DESTWWW)/camIQstats.php \
    112114$(DESTWWW)/camProcessedImfile.php \
    … …  
    118120$(DESTWWW)/fakeProcessedImfile.php \
    119121$(DESTWWW)/fakeProcessedImfile_failure.php \
     122$(DESTWWW)/failedFakeProcessedImfile.php \
    120123$(DESTWWW)/pzPendingExp.php \
    121124$(DESTWWW)/pzPendingImfile.php \
    … …  
    142145$(DESTWWW)/warpProcessedSkyfiles.php \
    143146$(DESTWWW)/warpFailedSkyfiles.php \
     147$(DESTWWW)/failedWarpSkyfiles.php \
    144148$(DESTWWW)/diffSummary.php \
    145149$(DESTWWW)/diffSummary_Images.php \
    … …  
    148152$(DESTWWW)/diffProcessedSkyfile.php \
    149153$(DESTWWW)/diffProcessedSkyfile_Images.php \
    150 $(DESTWWW)/diffFailedSkyfile.php \
     154$(DESTWWW)/failedDiffSkyfile.php \
    151155$(DESTWWW)/stackRun.php \
    152156$(DESTWWW)/stackSummary.php \
    … …  
    156160$(DESTWWW)/stackProcessedSkyfile_Images.php \
    157161$(DESTWWW)/stackFailedSkyfile.php \
     162$(DESTWWW)/failedStackSkyfile.php \
    158163$(DESTWWW)/staticskyRun.php \
    159164$(DESTWWW)/staticskySummary.php \
    … …  
    168173$(DESTWWW)/magicProcessedMasks_Images.php \
    169174$(DESTWWW)/magicFailedTrees.php \
     175$(DESTWWW)/failedMagicTrees.php \
    170176$(DESTWWW)/magicFailedNodes.php \
     177$(DESTWWW)/failedMagicNodes.php \
    171178$(DESTWWW)/magicFailedMasks.php \
     179$(DESTWWW)/failedMagicMasks.php \
    172180$(DESTWWW)/distSummary.php \
    173181$(DESTWWW)/distTarget.php \

  • branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_getplot.php

    r28794 r28980  
    2020else if ($type=="h")
    2121$filePath = "/tmp/czarplot_".$label."_all_stages_".$type.".png";
     22else if ($type=="s")
     23$filePath = "/tmp/czarplot_hosts_space.png";
     24
    2225$file = fopen ($filePath, "r");
    2326if ($file && !$debug) {

  • branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_labels.php

    r28794 r28980  
    1010//$db = dbconnect($ID['proj']);
    1111$czardb = dbconnect("czardb"); // HACK to connect to czarDb
     12$lastUpdateTime = getLastUpdateTime($czardb);
     13
    1214$gpc1db = dbconnect($ID['proj']);
    1315
    … …  
    2224else {$myMenu = "ipp.imfiles.dat";}
    2325
    24 menu($myMenu, 'Czartool', 'ipp.css', $ID['link'], $ID['proj']);
     26menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']);
    2527
    2628$pass = $ID['pass'];
    … …  
    3133$selectedLabel = $_GET[label];
    3234$selectedServer = $_GET[server];
    33 $selectedRevertServer = $_GET[revertserver];
    3435$selectedRevertStage = $_GET[revertstage];
    3536$selectedRevertMode = $_GET[revertmode];
    3637
    37 if ($selectedLabel == "") $selectedLabel = "all_labels";
    38 if ($selectedStage == "") $selectedStage = "all_stages";
     38if ($selectedLabel == "") { $selectedLabel = "all_stdscience_labels"; }
     39if ($selectedStage == "") { $selectedStage = "all_stages"; }
     40
     41echo 'Some documentation can be found <a href="http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing">here</a><br>';
    3942
    4043// deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later
    41 $currentRevertMode;
    42 if ($selectedRevertStage != "" && $selectedRevertMode != "" && $selectedRevertServer != "" ) {
    43 
    44     exec("czartool_revert.pl -s $selectedRevertServer -t $selectedRevertStage -o $selectedRevertMode", $response, $status);
     44if ($selectedRevertStage != "" && $selectedRevertMode != "") {
     45
     46    exec("czartool_revert.pl -t $selectedRevertStage -o $selectedRevertMode", $response, $status);
     47    $currentRevertMode;
    4548    if ($response[0] == "off") $currentRevertMode = 0;
    4649    else if ($response[0] == "on") $currentRevertMode = 1;
     50
     51    setRevertStatus($czardb, $selectedRevertStage, $currentRevertMode);
    4752}
    4853$debug = 0;
    … …  
    5156$distLabels = getLabels($czardb, "distribution");
    5257$pubLabels = getLabels($czardb, "publishing");
     58$updateLabels = getLabels($czardb, "update");
    5359
    5460if ($debug) {
    … …  
    8187echo "<input type=\"hidden\" name=\"menu\" value=\"$menu\">\n";
    8288echo "</form>\n";
    83 $lastUpdateTime = getLastUpdateTime($czardb);
    8489echo "<p  align=\"center\"> Current status of IPP (any faults are shown in parentheses). NOTE: This data is good as of: $lastUpdateTime </p>";
    8590
    8691echo "<table>\n";
    8792  echo "<tr>\n";
     93    echo "<td>\n";
     94      echo "<img src=\"czartool_getplot.php?type=t&label=$selectedLabel&stage=$selectedStage\"><br>";
     95    echo "</td>\n";
    8896
    8997    echo "<td> \n";
    90       echo "<img src=\"czartool_getplot.php?type=t&label=$selectedLabel&stage=$selectedStage\"><br>\n";
    91     echo "</td>\n";
    92 
    93     echo "<td> \n";
    94       createLabelsTable($pass, $proj, $czardb, $stdsLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage, $selectedRevertStage, $currentRevertMode);
    95     echo "</td>\n";
    96 
    97   echo "</tr>\n";
    98   echo "<tr>\n";
    99 
    100 echo "<table>\n";
    101   echo "<tr valign=top>\n";
    102     echo "<td> \n";
    103       echo "<img src=\"czartool_getplot.php?type=h&label=$selectedLabel&stage=$selectedStage\"><br>\n";
    104     echo "</td>\n";
    105     echo "<td> \n";
    106       createServersTable($pass, $proj,$czardb, $servers, $selectedLabel, $selectedStage);
    107     echo "</td>\n";
    108 
    109     echo "<td> \n";
    110       $today = date("Y-m-d");                         // 03.10.01
    111       showSummitData($gpc1db, $today);
     98      createLabelsTable($pass, $proj, $czardb, "stdscience", $stdsLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage);
    11299    echo "</td>\n";
    113100  echo "</tr>\n";
    114 echo "</table>\n";
    115 if ($selectedServer) showServerStatus($selectedServer);
    116 
     101
     102  echo "<tr>\n";
     103    echo "<td>\n";
     104      echo "<img src=\"czartool_getplot.php?type=h&label=$selectedLabel&stage=$selectedStage\"><br>";
     105    echo "</td>\n";
     106
     107    echo "<td> \n";
     108      createLabelsTable($pass, $proj, $czardb, "update", $updateLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage);
     109    echo "</td>\n";
     110  echo "</tr>\n";
     111
     112  echo "<tr>\n";
     113    echo "<table>\n";
     114      echo "<tr valign=top>\n";
     115        echo "<td> \n";
     116          echo "<img src=\"czartool_getplot.php?type=s\"><br>";
     117        echo "</td>\n";
     118        echo "<td>\n";
     119          createServersTable($pass, $proj,$czardb, $servers, $selectedLabel, $selectedStage);
     120        echo "</td>\n";
     121
     122        echo "<td> \n";
     123          $today = date("Y-m-d");
     124          showSummitData($gpc1db, $today);
     125        echo "</td>\n";
     126      echo "</tr>\n";
     127    echo "</table>\n";
     128
     129    echo "<table>\n";
     130      echo "<tr valign=top>\n";
     131        echo "<td> \n";
     132        echo "</td>\n";
     133        echo "<td>\n";
     134      if ($selectedServer) showServerStatus($selectedServer);
     135        echo "</td>\n";
     136      echo "</tr>\n";
     137    echo "</table>\n";
    117138  echo "</tr>\n";
    118139echo "</table>\n";
    … …  
    203224#
    204225###########################################################################
    205 function createLabelsTable($pass, $proj, $db, $stdsLabels, $distLabels, $pubLabels, $stages, $states, $selectedState, $selectedLabel, $selectedStage, $selectedRevertStage, $currentRevertMode) {
     226function createLabelsTable($pass, $proj, $db, $server, $labels, $distLabels, $pubLabels, $stages, $states, $selectedState, $selectedLabel, $selectedStage) {
    206227
    207228    // set up table columns
    … …  
    210231    echo "<tr><td></td>\n";
    211232
     233    echo "<p  align=\"center\"> Current labels for $server server </p>";
     234
    212235    write_header_cell($class, "");
    213236    write_header_cell($class, "");
    … …  
    215238    foreach ($stages as &$stage) {
    216239
    217         $revertOnOff=array();
    218         if ($stage == "destreak" or $stage == "dist") $server = "distribution";
    219         else $server = "stdscience";
    220         if ($stage == $selectedRevertStage) $reverting = $currentRevertMode;
    221         else $reverting = getRevertStatus($db, $stage);
    222         $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&revertserver=" . $server . "&revertstage=" . $stage . "&revertmode=";
     240        $reverting = getRevertStatus($db, $stage);
     241        $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&revertmode=";
    223242        if(!$reverting) {$label =  "Start";$link = $link . "on";}
    224243        if($reverting) {$label = "Stop";$link = $link . "off";}
    … …  
    250269
    251270    // write rows
    252     foreach ($stdsLabels as &$stdsLabel) {
     271    foreach ($labels as &$thisLabel) {
    253272
    254273        $distributing = false;
    … …  
    256275        foreach ($distLabels as &$distLabel) {
    257276
    258             if ($stdsLabel == $distLabel) { $distributing = true; break;}
     277            if ($thisLabel == $distLabel) { $distributing = true; break;}
    259278        }
    260279        foreach ($pubLabels as &$pubLabel) {
    261280
    262             if ($stdsLabel == $pubLabel) { $publishing = true; break;}
     281            if ($thisLabel == $pubLabel) { $publishing = true; break;}
    263282        }
    264283
    265284        // create link to label summary page for each label
    266         if ($stdsLabel == $selectedLabel) $link = "";
    267         else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel . "&stage=" . $selectedStage;
     285        if ($thisLabel == $selectedLabel) $link = "";
     286        else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $thisLabel . "&stage=" . $selectedStage;
    268287
    269288        echo "<tr><td></td>\n";
    270         write_table_cell($class, '%s', $link, $stdsLabel);
     289        write_table_cell($class, '%s', $link, $thisLabel);
    271290        write_table_cell($class, '%s', "", $distributing ? "yes" : "NO");
    272291        write_table_cell($class, '%s', "", $publishing ? "yes" : "NO");
    … …  
    275294        $anyFaults = false;
    276295
    277         $link = "chipProcessedImfile_failure.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.label=" . $stdsLabel . "&chipRun.state=new";
    278         getStateAndFaults($db, $stdsLabel, $selectedState, "chip", $str, $anyFaults);
     296        $link = "failedChipProcessedImfile.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.label=" . $thisLabel . "&chipRun.state=new";
     297        getStateAndFaults($db, $thisLabel, $selectedState, "chip", $str, $anyFaults);
    279298        write_table_cell($class, '%s', $anyFaults ? $link : "", $str);
    280299
    281         $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&camRun.label=" . $stdsLabel . "&camRun.state=new";
    282         getStateAndFaults($db, $stdsLabel, $selectedState, "cam", $str, $anyFaults);
     300        $link = "failedCamProcessedExp.php?pass=" . $pass . "&proj=" . $proj . "&camRun.label=" . $thisLabel . "&camRun.state=new";
     301        getStateAndFaults($db, $thisLabel, $selectedState, "cam", $str, $anyFaults);
    283302        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    284303
     304        $link =  "failedFakeProcessedImfile.php?pass=" . $pass . "&proj=" . $proj . "&fakeRun.label=" . $thisLabel . "&fakeRun.state=new";
     305        getStateAndFaults($db, $thisLabel, $selectedState, "fake", $str, $anyFaults);
     306        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
     307
     308        $link = "failedWarpSkyfiles.php?pass=" . $pass . "&proj=" . $proj . "&warpRun.label=" . $thisLabel . "&warpRun.state=new";
     309        getStateAndFaults($db, $thisLabel, $selectedState, "warp", $str, $anyFaults);
     310        write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
     311
     312        $link = "failedStackSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&stackRun.label=" . $thisLabel . "&stackRun.state=new";
     313        getStateAndFaults($db, $thisLabel, $selectedState, "stack", $str, $anyFaults);
     314        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
     315
     316        $link = "failedDiffSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.label=" . $thisLabel . "&diffRun.state=new";
     317        getStateAndFaults($db, $thisLabel, $selectedState, "diff", $str, $anyFaults);
     318        write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
     319
    285320        $link = $defaultlink;
    286         getStateAndFaults($db, $stdsLabel, $selectedState, "fake", $str, $anyFaults);
     321        getStateAndFaults($db, $thisLabel, $selectedState, "magic", $str, $anyFaults);
    287322        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    288323
    289         $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj . "&warpRun.label=" . $stdsLabel . "&warpRun.state=new";
    290         getStateAndFaults($db, $stdsLabel, $selectedState, "warp", $str, $anyFaults);
    291         write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
    292 
    293         $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&stackRun.label=" . $stdsLabel . "&stackRun.state=new";
    294         getStateAndFaults($db, $stdsLabel, $selectedState, "stack", $str, $anyFaults);
     324        $link = $defaultlink;
     325        getStateAndFaults($db, $thisLabel, $selectedState, "magicDS", $str, $anyFaults);
    295326        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    296327
    297         $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.label=" . $stdsLabel . "&diffRun.state=new";
    298         getStateAndFaults($db, $stdsLabel, $selectedState, "diff", $str, $anyFaults);
    299         write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
    300 
    301328        $link = $defaultlink;
    302         getStateAndFaults($db, $stdsLabel, $selectedState, "magic", $str, $anyFaults);
     329        getStateAndFaults($db, $thisLabel, $selectedState, "dist", $str, $anyFaults);
    303330        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    304331
    305         $link = $defaultlink;
    306         getStateAndFaults($db, $stdsLabel, $selectedState, "magicDS", $str, $anyFaults);
    307         write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    308 
    309         $link = $defaultlink;
    310         getStateAndFaults($db, $stdsLabel, $selectedState, "dist", $str, $anyFaults);
    311         write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    312 
    313332        echo "</tr>\n";
    314333    }
    315334
    316     if ($selectedLabel == "all_labels") $link = "";
    317     else  $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=all_labels&stage=".$selectedStage;
     335    if ($selectedLabel == "all_".$server."_labels") $link = "";
     336    else  $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=all_".$server."_labels&stage=".$selectedStage;
    318337
    319338    echo "<tr><td></td>\n";
    320     write_table_cell($class, '%s', $link, "All labels:");
    321     write_table_cell($class, '%s', "", "-");
    322     write_table_cell($class, '%s', "", "-");
    323 
    324     $str = "";
    325     $anyFaults = false;
    326 
    327     $link = "chipProcessedImfile_failure.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.state=new";
    328     getStateAndFaults($db, "all_labels", $selectedState, "chip", $str, $anyFaults);
    329     write_table_cell($class, '%s', $anyFaults ? $link : "", $str);
    330 
    331     $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&camRun.state=new";
    332     getStateAndFaults($db, "all_labels", $selectedState, "cam", $str, $anyFaults);
    333     write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    334 
    335     $link = $defaultlink;
    336     getStateAndFaults($db, "all_labels", $selectedState, "fake", $str, $anyFaults);
    337     write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    338 
    339     $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj . "&warpRun.state=new";
    340     getStateAndFaults($db, "all_labels", $selectedState, "warp", $str, $anyFaults);
    341     write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
    342 
    343     $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&stackRun.state=new";
    344     getStateAndFaults($db, "all_labels", $selectedState, "stack", $str, $anyFaults);
    345     write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    346 
    347     $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.state=new";
    348     getStateAndFaults($db, "all_labels", $selectedState, "diff", $str, $anyFaults);
    349     write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
    350 
    351     $link = $defaultlink;
    352     getStateAndFaults($db, "all_labels", $selectedState, "magic", $str, $anyFaults);
    353     write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    354 
    355     $link = $defaultlink;
    356     getStateAndFaults($db, "all_labels", $selectedState, "magicDS", $str, $anyFaults);
    357     write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    358 
    359     $link = $defaultlink;
    360     getStateAndFaults($db, "all_labels", $selectedState, "dist", $str, $anyFaults);
    361     write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
    362 
     339    write_table_cell($class, '%s', $link, "All $server labels");
    363340
    364341    echo "</tr>\n";
    … …  
    368345###########################################################################
    369346#
     347# Sets reverts status
     348#
     349###########################################################################
     350function setRevertStatus($db, $stage, $reverting) {
     351
     352    $sql = "UPDATE reverts SET reverting = $reverting WHERE stage LIKE '$stage'";
     353    if ($debug) {echo "$sql<br>";}
     354
     355    $qry = $db->query($sql);
     356    if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
     357}
     358
     359###########################################################################
     360#
    370361# Returns whether this stage is reverting or not
    371362#
    … …  
    374365
    375366    $anyFaults = false;
    376     $sql = "SELECT reverting FROM $stage ORDER BY timestamp DESC LIMIT 1";
     367
     368    $sql = "SELECT reverting FROM reverts WHERE stage LIKE '$stage'";
    377369    if ($debug) {echo "$sql<br>";}
    378370

  • branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.imfiles.dat

    r28794 r28980  
    2525
    2626menutop   | menutop      | plain   | &nbsp;                       |
     27menutop   | menutop      | link    | New failed chip              | failedChipProcessedImfile.php
     28menutop   | menutop      | link    | New failed cam               | failedCamProcessedExp.php
     29menutop   | menutop      | link    | New failed fake              | failedFakeProcessedImfile.php
     30menutop   | menutop      | link    | New failed warp              | failedWarpSkyfiles.php
     31menutop   | menutop      | link    | New failed stack             | failedStackSkyfile.php
     32menutop   | menutop      | link    | New failed diff              | failedDiffSkyfile.php
     33menutop   | menutop      | link    | New failed magic tree        | failedMagicTrees.php
     34menutop   | menutop      | link    | New failed magic node        | failedMagicNodes.php
     35menutop   | menutop      | link    | New failed magic mask        | failedMagicMasks.php
     36
     37menutop   | menutop      | plain   | &nbsp;                       |
    2738menulink  | menuselect   | link    | Tables columns               | columns_in_db.php
    2839menulink  | menuselect   | link    | Clean /tmp directory         | cleanTmpDirectory.php

  • branches/eam_branches/ipp-20100621/ippMonitor/scripts/czartool_revert.pl

    r28794 r28980  
    55use Getopt::Long;
    66
    7 my $server = undef;
    87my $task = undef;
    98my $onoff = undef;
    109
    1110GetOptions (
    12         "server|s=s" => \$server,
    1311        "task|t=s" => \$task,
    1412        "onoff|o=s" => \$onoff);
    1513
    1614my $prob = 0;
    17 if (!$server) {print "ERROR: need to define a server (-s)\n"; $prob=1;}
    1815if (!$task) {print "ERROR: need to define a task (-t)\n"; $prob=1;}
    1916if ($prob) {exit;}
    2017
    2118if ($task eq "cam") {$task = "camera";}
     19
     20my $server = undef;
     21if ($task eq "destreak" or $task eq "dist") {$server = "distribution";}
     22else {$server = "stdscience";}
    2223
    2324my $revert = undef;

  • branches/eam_branches/ipp-20100621/ippMonitor/scripts/generate

    r28794 r28980  
    648648    foreach $word (@words) {
    649649        $fword = &parse_fieldname ($word);
    650         # print "field: $word -> $fword\n";
     650#        print "field: $word -> $fword\n";
    651651        push @outwords, $fword;
    652652        if ($word =~ m|^\$|) {
    … …  
    660660    $opf{$name} = $outline;
    661661
    662     # print "ops: $name : $ops{$name}\n";
    663     # print "opf: $name : $opf{$name}\n";
     662#    print "ops: $name : $ops{$name}\n";
     663#    print "opf: $name : $opf{$name}\n";
    664664
    665665    return 1;

  • branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/Gpc1Db.pm

    r28794 r28980  
    2828        JOIN diffSkyfile USING (diff_id)
    2929        WHERE rawExp.exp_id = $expId AND camRun.magicked
    30 
    3130SQL
    3231
    … …  
    3736###########################################################################
    3837#
    39 # Returns camera-stage smf files for this exposure
     38# Returns camera-stage smf file for this exposure as used in the provided DVO Db
     39#
     40###########################################################################
     41sub getCameraStageSmfForThisDvoDb {
     42    my ($self, $dvoDb, $expId) = @_;
     43
     44    my $query = $self->{_db}->prepare(<<SQL);
     45    SELECT path_base
     46        FROM camProcessedExp
     47        JOIN addRun USING(cam_id)   
     48        JOIN camRun USING(cam_id) 
     49        JOIN chipRun USING(chip_id)   
     50        JOIN rawExp USING(exp_id)   
     51        WHERE addRun.dvodb LIKE '$dvoDb'
     52        AND exp_id = $expId;
     53SQL
     54
     55    $query->execute;
     56    return $query->fetchrow_array();
     57}
     58
     59###########################################################################
     60#
     61# Returns latest processed camera-stage smf file for this exposure ID
    4062#
    4163###########################################################################
    … …  
    6486###########################################################################
    6587#
    66 # Returns a list of exposure IDs for this survey
     88# Returns a list of exposure IDs in a given DVO database 
    6789#
    6890###########################################################################
    69 sub getExposureListFromSurvey {
    70     my ($self, $survey) = @_;
     91sub getExposureListFromDvoDb {
     92    my ($self, $dvoDb, $exposures) = @_;
    7193
    7294    my $query = $self->{_db}->prepare(<<SQL);
    73 
    74     SELECT rawExp.exp_id, rawExp.exp_name, camRun.dist_group
    75         FROM camRun, chipRun, rawExp
    76         WHERE camRun.state = 'full'
    77         AND camRun.chip_id = chipRun.chip_id
    78         AND chipRun.exp_id = rawExp.exp_id
    79         AND camRun.dist_group = '$survey'   
    80         GROUP BY  rawExp.exp_id
    81         ORDER BY rawExp.exp_id ASC;
     95    SELECT exp_id, exp_name, camRun.dist_group
     96        FROM addRun
     97        JOIN camRun USING(cam_id)
     98        JOIN chipRun USING(chip_id)
     99        JOIN rawExp USING(exp_id)
     100        WHERE addRun.dvodb LIKE '$dvoDb'
     101        AND addRun.state = 'full'
     102        ORDER BY exp_id ASC;
    82103SQL
    83 
    84104
    85105    #AND rawExp.exp_id > 133887
    … …  
    93113
    94114    $query->execute;
    95     return $query->fetchall_arrayref();
     115    ${$exposures} = $query->fetchall_arrayref();
     116    my $numOfExposures = scalar @{${$exposures}};
     117    if ($numOfExposures > 0) {
     118   
     119        print "* Found $numOfExposures exposures in DVO Db '$dvoDb'\n";
     120        return 1;
     121    }
     122
     123    print "* No exposures found in DVO Db '$dvoDb'\n";
     124    return 0;
    96125}
    97126
    98127###########################################################################
    99128#
    100 # Returns info for this exposure ID
     129# Returns info for this exposure ID for given DVO Db
    101130#
    102131###########################################################################
    103 sub getExposureListFromExpId {
    104     my ($self, $expId) = @_;
     132sub getSingleExposureFromDvoDb {
     133    my ($self, $dvoDb, $expId, $exposures) = @_;
    105134
    106135    my $query = $self->{_db}->prepare(<<SQL);
    107     SELECT DISTINCT rawExp.exp_id,  rawExp.exp_name, dist_group
    108         FROM rawExp, chipRun
    109         WHERE chipRun.exp_id = rawExp.exp_id
    110         AND rawExp.exp_id = $expId
     136    SELECT exp_id, exp_name, camRun.dist_group
     137        FROM addRun
     138        JOIN camRun USING(cam_id)
     139        JOIN chipRun USING(chip_id)
     140        JOIN rawExp USING(exp_id)
     141        WHERE addRun.dvodb LIKE '$dvoDb'
     142        AND exp_id = $expId;
    111143SQL
    112144
    113145    $query->execute;
    114     return $query->fetchall_arrayref();
     146
     147    ${$exposures} = $query->fetchall_arrayref();
     148
     149    if (scalar @{${$exposures}} > 0) {
     150   
     151        print "* Found exposure $expId in DVO Db '$dvoDb'\n";
     152        return 1;
     153    }
     154
     155    print "* Exposure $expId NOT found in DVO Db '$dvoDb'\n";
     156    return 0;
    115157}
    116158
    117 
    118 
    1191591;

  • branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/IppToPspsDb.pm

    r28794 r28980  
    88use ippToPsps::MySQLDb;
    99our @ISA = qw(ippToPsps::MySQLDb);    # inherits from MySQLDb
     10
     11
     12###########################################################################
     13#
     14# Returns a list of batches that have been processed and loaded to datastore
     15#
     16###########################################################################
     17sub getBatchList {
     18    my ($self, $exposures, $fromTime, $toTime) = @_;
     19
     20    my $query = $self->{_db}->prepare(<<SQL);
     21    SELECT created, exp_id, batch_id, survey_id, deleted
     22        FROM batches
     23        WHERE processed = 1
     24        AND on_datastore = 1
     25        AND created >= '$fromTime'
     26        AND created <= '$toTime';
     27SQL
     28
     29    # TODO remove date restriction
     30    $query->execute;
     31    ${$exposures} = $query->fetchall_arrayref();
     32    my $count = scalar @{${$exposures}};
     33
     34   printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es");
     35   return $count;
     36}
     37
     38###########################################################################
     39#
     40# Returns a info about one particular batch
     41#
     42###########################################################################
     43sub getSingleBatch{
     44    my ($self, $batch_id, $exposures) = @_;
     45
     46    my $query = $self->{_db}->prepare(<<SQL);
     47    SELECT created, exp_id, batch_id, survey_id, deleted
     48        FROM batches
     49        WHERE batch_id = $batch_id
     50        AND processed = 1
     51        AND on_datastore = 1;
     52SQL
     53 
     54    $query->execute;
     55    ${$exposures} = $query->fetchall_arrayref();
     56    my $count = scalar @{${$exposures}};
     57
     58   printf( "* Found %d batch%s\n", $count, ($count==1) ? "" : "es");
     59   return $count;
     60}
    1061
    1162#######################################################################################
    … …  
    56107#
    57108########################################################################################
    58 sub updateDb {
    59     my ($self, $batchId, $expId, $processed, $published, $totalDetections) = @_;
    60 
    61 print "HJHJH '$batchId', '$expId', '$processed', '$published', '$totalDetections'\n";
     109sub updateBatch {
     110    my ($self, $batchId, $expId, $processed, $published, $totalDetections, $minObjId, $maxObjId) = @_;
     111
     112    if (!$minObjId) {$minObjId = -1;}
     113    if (!$maxObjId) {$maxObjId = -1;}
    62114
    63115if (!$totalDetections) {$totalDetections = -1;}
    … …  
    65117    my $query = $self->{_db}->prepare(<<SQL);
    66118    UPDATE batches
    67         SET processed = $processed, on_datastore = $published, total_detections = $totalDetections
     119        SET
     120          processed = $processed,
     121          on_datastore = $published,
     122          total_detections = $totalDetections,
     123          min_obj_id = $minObjId,
     124          max_obj_id = $maxObjId
    68125        WHERE batch_id = $batchId
    69126        AND exp_id = $expId;
    … …  
    84141    SELECT COUNT(*)
    85142        FROM batches
    86         WHERE exp_id = $expId
     143        WHERE exp_id = $expId
     144        AND created > '2010-08-12'
    87145        AND processed = 1;
    88146SQL
    … …  
    102160#
    103161########################################################################################
    104 sub getNewBatchId {
    105     my ($self, $expId, $surveyType, $batchType) = @_;
     162sub createNewBatch {
     163    my ($self, $expId, $surveyType, $batchType, $dvoDb, $datastoreProduct) = @_;
    106164
    107165    my $query = $self->{_db}->prepare(<<SQL);
    … …  
    118176    $batchId++;
    119177
    120     my $query = $self->{_db}->prepare(<<SQL);
     178    $query = $self->{_db}->prepare(<<SQL);
    121179    INSERT INTO batches
    122         (batch_id, exp_id, survey_id, batch_type)
     180        (batch_id, exp_id, survey_id, batch_type, dvo_db, datastore_product)
    123181        VALUES
    124         ($batchId, $expId, '$surveyType', '$batchType');
    125 
     182        ($batchId, $expId, '$surveyType', '$batchType', '$dvoDb', '$datastoreProduct');
    126183SQL
    127184
    … …  
    133190}
    134191
     192#######################################################################################
     193#
     194# Updates an existing database record to show datastore status
     195#
     196#######################################################################################
     197sub setBatchAsDeleted {
     198    my ($self,$batchId, $expId) = @_;
     199
     200    my $query = $self->{_db}->prepare(<<SQL);
     201    UPDATE batches
     202        SET deleted = 1
     203        WHERE batch_id = $batchId
     204        AND exp_id = $expId;
     205SQL
     206
     207        $query->execute; # TODO check response of these
     208}
     209
     210#######################################################################################
     211#
     212# Updates an existing database record with info from ODM
     213#
     214#######################################################################################
     215sub updateODMStatus {
     216    my ($self,$batchId, $expId, $loadedToOdm, $mergeWorthy, $mergeCompleted) = @_;
     217
     218    my $query = $self->{_db}->prepare(<<SQL);
     219    UPDATE batches
     220        SET loaded_to_ODM = $loadedToOdm, merge_worthy = $mergeWorthy, merged = $mergeCompleted
     221        WHERE batch_id = $batchId
     222        AND exp_id = $expId;
     223SQL
     224
     225        $query->execute; # TODO check response of these
     226}
     227
     228
     229
    135230###########################################################################
    136231#
    … …  
    142237
    143238    my $currentRevision = -1;
    144     my $latestRevision = 4;
     239    my $latestRevision = 6;
    145240
    146241    while ($currentRevision != $latestRevision) {
    147242
    148243        $currentRevision = $self->getRevision();
    149         if ($self->{_verbose}) {print "* Current revision = $currentRevision\n";}
     244        if ($self->{_verbose}) {print "* Current database revision = $currentRevision\n";}
    150245
    151246        if ($currentRevision == 0) {$self->createRevision_1();}
    … …  
    153248        elsif ($currentRevision == 2) {$self->createRevision_3();}
    154249        elsif ($currentRevision == 3) {$self->createRevision_4();}
     250        elsif ($currentRevision == 4) {$self->createRevision_5();}
     251        elsif ($currentRevision == 5) {$self->createRevision_6();}
    155252    }
    156253}
    … …  
    164261    my ($self) = @_;
    165262
    166     print "* Creating revision 1 of '$dbname'\n";
     263    print "* Creating revision 1 of '$self->{_dbName}'\n";
    167264
    168265    my $query = $self->{_db}->prepare(<<SQL);
    … …  
    175272        $query->execute;
    176273
    177     my $query = $self->{_db}->prepare(<<SQL);
     274    $query = $self->{_db}->prepare(<<SQL);
    178275    CREATE TABLE batches (
    179276            batch_id BIGINT NOT NULL,
    … …  
    192289        $query->execute;
    193290
    194     setRevision(1);
     291    $self->setRevision(1);
    195292}
    196293
    … …  
    203300    my ($self) = @_;
    204301
    205     print "* Creating revision 2 of '$dbname'\n";
     302    print "* Creating revision 2 of '$self->{_dbName}'\n";
    206303
    207304    my $query = $self->{_db}->prepare(<<SQL);
    … …  
    211308        $query->execute;
    212309
    213     setRevision(2);
     310    $self->setRevision(2);
    214311}
    215312
    … …  
    222319    my ($self) = @_;
    223320
    224     print "* Creating revision 3 of '$dbname'\n";
     321    print "* Creating revision 3 of '$self->{_dbName}'\n";
    225322
    226323    my $query = $self->{_db}->prepare(<<SQL);
    … …  
    230327        $query->execute;
    231328
    232     setRevision(3);
     329    $self->setRevision(3);
    233330}
    234331
    … …  
    241338    my ($self) = @_;
    242339
    243     print "* Creating revision 4 of '$dbname'\n";
     340    print "* Creating revision 4 of '$self->{_dbName}'\n";
    244341
    245342    my $query = $self->{_db}->prepare(<<SQL);
    … …  
    249346        $query->execute;
    250347
    251     setRevision(4);
     348    $self->setRevision(4);
     349}
     350
     351#######################################################################################
     352#
     353# Create revision 5 of the database
     354#
     355#######################################################################################
     356sub createRevision_5 {
     357    my ($self) = @_;
     358
     359    print "* Creating revision 5 of '$self->{_dbName}'\n";
     360
     361    my $query = $self->{_db}->prepare(<<SQL);
     362         CREATE INDEX batchesIndex ON batches (batch_id);
     363SQL
     364    $query->execute;
     365
     366    $self->setRevision(5);
     367}
     368
     369#######################################################################################
     370#
     371# Create revision 6 of the database
     372#
     373#######################################################################################
     374sub createRevision_6 {
     375    my ($self) = @_;
     376
     377    print "* Creating revision 6 of '$self->{_dbName}'\n";
     378
     379    my $query = $self->{_db}->prepare(<<SQL);
     380    ALTER TABLE batches
     381        ADD COLUMN dvo_db VARCHAR(50) DEFAULT "UNKNOWN",
     382        ADD COLUMN datastore_product VARCHAR(50)  DEFAULT "UNKNOWN",
     383        ADD COLUMN min_obj_id BIGINT DEFAULT 0,
     384        ADD COLUMN max_obj_id BIGINT DEFAULT 0
     385SQL
     386    $query->execute;
     387
     388    $self->setRevision(6);
    252389}
    253390

  • branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps_run.pl

    r28794 r28980  
    66use PS::IPP::Config 1.01 qw( :standard );
    77use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
    8 use Pod::Usage qw( pod2usage );
    98use IPC::Cmd 0.36 qw( can_run run );
    109use File::Temp qw(tempfile);
    1110use XML::LibXML;
     11use File::Basename;
    1212
    1313# local classes
    1414use ippToPsps::Gpc1Db;
    1515use ippToPsps::IppToPspsDb;
     16use ippToPsps::Datastore;
    1617
    1718# globals
    1819my $camera = 'GPC1';
    1920my $batchType = undef;
    20 my $survey = undef;
    21 my $dvodb = undef;
     21my $dvoLocation = undef;
     22my $dvoDb = undef;
     23my $fullDvoPath = undef;
    2224my $verbose = undef;
    2325my $save_temps = undef;
    24 my $no_update = undef;
    2526my $output = undef;
    2627my $singleExpId = undef;
    … …  
    3435        'output|o=s' => \$output,
    3536        'batch|b=s' => \$batchType,
    36         'dvodb|d=s' => \$dvodb,
    37         'survey|s=s' => \$survey,
     37        'dvo|d=s' => \$fullDvoPath,
    3838        'expid|e=s' => \$singleExpId,
    3939        'product|p=s' => \$datastoreProduct,
    4040        'verbose|v' => \$verbose,
    4141        'save_temps|t' => \$save_temps,
    42         'no-update|u' => \$no_update,
    4342        'force|f' => \$force,
    4443        'tarnzip|z' => \$dontTarNZip,
    … …  
    4948print "* \n";
    5049if (@ARGV) {
    51     print "* UNKNKOWN: option:                         @ARGV\n"; $quit=1;}
     50    $quit=1;
     51    print "* UNKNKOWN: option                          @ARGV\n";
     52}
    5253if (!defined $output) {
    53     print "* REQUIRED: need to provide an output path: -o <path>\n"; $quit=1;}
     54    $quit=1;
     55    print "* REQUIRED: need to provide an output path  -o <path>\n";
     56}
    5457if (!defined $batchType) {
    55     print "* REQUIRED: need to define a batch type:    -b <init|det|diff|stack>\n"; $quit=1;}
    56 if (!defined $dvodb) {
    57     print "* REQUIRED: need to provide a DVO Db path:  -d <path>\n"; $quit=1;}
    58 if (!defined $survey and !defined $singleExpId) {
    59     print "* REQUIRED: need to provide a survey type:  -s <ThreePi|STS|SAS|M31|MD01|MD02|etc> ***OR***\n";
    60     print "*           an exposure ID                  -e <expID>\n"; $quit=1;}
     58    $quit=1;
     59    print "* REQUIRED: need to define a batch type     -b <init|det|diff|stack>\n";
     60}
     61if (!defined $fullDvoPath) {
     62    $quit=1;
     63    print "* REQUIRED: need to provide a DVO Db        -d <pathToDVO>\n";
     64}
     65if (!defined $singleExpId) {
     66
     67    print "* OPTIONAL: a single exposure ID            -e <expID>           (default = none)\n";
     68}
    6169if (!defined $datastoreProduct) {
    62     print "* OPTIONAL: datastore product:              -p <product>\n";}
     70
     71    print "* OPTIONAL: datastore product               -p <product>         (default = none, i.e. data will not be published)\n";
     72}
    6373if (!defined $verbose) {
    64     print "* OPTIONAL: run in verbose mode:            -v\n"; $verbose = 0;}
     74    $verbose = 0;
     75    print "* OPTIONAL: run in verbose mode             -v                   (default = $verbose)\n";
     76}
    6577if (!defined $save_temps) {
    66     print "* OPTIONAL: keep temp files:                -t\n"; $save_temps = 0;}
    67 if (!defined $no_update) {
    68     print "* OPTIONAL: don't update database:          -u\n"; $no_update = 0;}
     78    $save_temps = 0;
     79    print "* OPTIONAL: keep temp files                 -t                   (default = $save_temps)\n";
     80}
    6981if (!defined $force) {
    70     print "* OPTIONAL: force if already processed :    -f\n"; $force = 0;}
     82    $force = 0;
     83    print "* OPTIONAL: force if already processed      -f                   (default = $force)\n";
     84}
    7185if (!defined $dontTarNZip) {
    72     print "* OPTIONAL: don't tar and zip output :      -z\n"; $dontTarNZip = 0;}
    73     print "*\n*******************************************************************************\n";
     86    $dontTarNZip = 0;
     87    print "* OPTIONAL: don't tar and zip output        -z                   (default = $dontTarNZip)\n";
     88}
     89print "*\n*******************************************************************************\n";
    7490
    7591if ($quit) { exit; }
     92
     93# determine PSPS batch 'type'
     94my $pspsBatchType;
     95if ($batchType eq 'init') {$pspsBatchType = "IN";}
     96elsif ($batchType eq 'det') {$pspsBatchType = "P2";}
     97elsif ($batchType eq 'stack') {$pspsBatchType = "ST";}
     98elsif ($batchType eq 'diff') {$pspsBatchType = "OB";}
     99else {$pspsBatchType = "UNKNOWN";}
     100
     101# spilt full DVO path into Db and path
     102$fullDvoPath =~ s/\/$//; # strip off trailing '\'
     103($dvoDb, $dvoLocation) = fileparse($fullDvoPath);
    76104
    77105my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps);
    78106my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
     107my $datastore = undef;
     108if ($datastoreProduct) {$datastore = new ippToPsps::Datastore($datastoreProduct, $verbose, $save_temps);}
    79109
    80110# check we can run programs and get camera config
    81111my $ippToPsps = can_run('ippToPsps') or (warn "Can't find 'ippToPsps' program" and exit($PS_EXIT_CONFIG_ERROR));
    82 my $dsreg = can_run('dsreg') or (warn "Can't find 'dsreg' program" and exit($PS_EXIT_CONFIG_ERROR));
    83112my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR));
    84113
    85114if ($batchType eq "init") {$initBatch = 1;}
    86115else {$initBatch = 0;}
    87 process();
     116
     117if (!process()) {print "* Finished unsuccessfully\n";}
     118else {print "* Finished successfully\n";}
    88119
    89120#######################################################################################
    … …  
    114145    my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps);
    115146    my $lastExpId = $ippToPspsDb->getLastExpId();
    116     my $rows = undef;
     147    my $exposures;
    117148
    118149    my $query;
    119150    if ($initBatch) {
    120    
    121         my @row = (0, 'NULL', 'ThreePi');
    122         @{$rows} = (\@row);
    123     }
    124     elsif ($singleExpId) { $rows = $gpc1Db->getExposureListFromExpId($singleExpId); }
    125     else { $rows = $gpc1Db->getExposureListFromSurvey($survey); }
    126 
    127 # TODO check if there are no exposures and give a warning
     151
     152        my @exposure = (0, 'NULL', 'ThreePi');
     153        @{$exposures} = (\@exposure);
     154    }
     155    # get single exposure
     156    elsif ($singleExpId) {
     157       
     158        if (!$gpc1Db->getSingleExposureFromDvoDb($dvoDb, $singleExpId, \$exposures)) {return 0;}
     159    }
     160    # get all exposures in this DVO Db
     161    else {
     162       
     163        if (!$gpc1Db->getExposureListFromDvoDb($dvoDb, \$exposures)) {return 0;}
     164    }
    128165
    129166    my $batchId = 0;
    … …  
    132169
    133170    #my $batchId = -1;
    134     my $row;
    135     foreach $row ( @{$rows} ) {
    136         my ($expId, $expName, $distGroup) = @{$row};
    137     #        print "JHGHGHGHGH2 $expId, $expName, $distGroup\n";
    138 
    139     # loop round exposures
    140     #while (my @row = $query->fetchrow_array()) {
    141      #   my ($expId, $expName, $distGroup) = @row;
     171    my $exposure;
     172    foreach $exposure ( @{$exposures} ) {
     173        my ($expId, $expName, $distGroup) = @{$exposure};
    142174
    143175        if (!$initBatch && $ippToPspsDb->isExposureAlreadyProcessed($expId)) {
    144                
    145                 if ($force) {print "* Forcing....\n";}
    146                 else {next};
     176
     177            if ($force) {print "* Forcing....\n";}
     178            else {next};
    147179        }
    148180
    … …  
    150182        if (!$surveyType) {next;}
    151183
    152         $batchId = $ippToPspsDb->getNewBatchId($expId, $surveyType, $batchType);
     184        $batchId = $ippToPspsDb->createNewBatch($expId, $distGroup, $pspsBatchType, $dvoDb, (defined $datastore) ? $datastoreProduct : "NONE");
    153185
    154186        # TODO quit here if no sensible batch ID
    … …  
    156188        # generate batch path from batch IDs
    157189        my $batch = sprintf("B%08d", $batchId);
    158         my $batchDir = sprintf("$output/$batch");
     190        my $dvoDir = "$output/$dvoDb";
     191        my $batchDir = "$dvoDir/$batch";
     192
     193        # make directories
     194        unless(-d $dvoDir) {mkdir($dvoDir, 0777);}
    159195        mkdir($batchDir, 0777);
     196
    160197        $published = 0;
    161198
    … …  
    171208            if (writeBatchManifest($batchDir, $batch, $batchType, $surveyType, $filename, $minObjId, $maxObjId)) {
    172209
     210                # tar n' zip
    173211                if (!$dontTarNZip) {
    174                     my $tarball = tarAndZipBatch($output, $batch);
    175                     if ($tarball && $datastoreProduct && publishToDatastore($batch, $output, $tarball)) {$published = 1;}
     212                    my $tarball = tarAndZipBatch($dvoDir, $batch);
     213
     214                    # and publish
     215                    if ($tarball && defined $datastore ) {
     216                       
     217                        $published = $datastore->register($batch, $dvoDir, $tarball, "IPP_PSPS", "tgz");
     218                    }
    176219                }
    177220            }
    178221
    179             $ippToPspsDb->updateDb($batchId, $expId, 1, $published, $totalDetections);
     222            $ippToPspsDb->updateBatch($batchId, $expId, 1, $published, $totalDetections, $minObjId, $maxObjId);
    180223        }
    181224
    … …  
    330373    my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec;
    331374
    332     # determine batch 'type'
    333     my $type;
    334     if ($batchType eq 'init') {$type = "IN";}
    335     elsif ($batchType eq 'det') {$type = "P2";}
    336     elsif ($batchType eq 'stack') {$type = "ST";}
    337     elsif ($batchType eq 'diff') {$type = "OB";}
    338     else {$type = "UNKNOWN";}
    339 
    340375    # create XML file
    341376    my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2);
    … …  
    372407                $writer->startTag('manifest',
    373408                        "name" => "$batch",
    374                         "type" => $type,
     409                        "type" => $pspsBatchType,
    375410                        "survey" => $pspsSurvey,
    376411                        "timestamp" => "$timeStamp",
    … …  
    382417                $writer->startTag('manifest',
    383418                        "name" => "$batch",
    384                         "type" => $type,
     419                        "type" => $pspsBatchType,
    385420                        "timestamp" => "$timeStamp");
    386421            }
    … …  
    404439
    405440#######################################################################################
    406 #
    407 # register new job with the datastore
    408 #
    409 ########################################################################################
    410 sub publishToDatastore {
    411     my ($batch, $path, $tarball) = @_;
    412 
    413     my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_dsregList.XXXX", UNLINK => !$save_temps);
    414 
    415     print $tempFile $tarball . '|||tgz' . "\n";
    416 
    417     # build dsreg command command
    418     my $command  = "$dsreg";
    419     $command .= " --add $batch";
    420     $command .= " --copy";
    421     $command .= " --datapath $path";
    422     $command .= " --type IPP_PSPS";
    423     $command .= " --product $datastoreProduct";
    424     $command .= " --list $tempName";
    425 
    426     # run command
    427     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    428         run(command => $command, verbose => $verbose);
    429 
    430     if (!$success) { print "* Unable to publish $tarball to datastore\n" and return 0 };
    431 
    432     print "* Successfully published $tarball to datastore\n";
    433 
    434     close($tempFile);
    435 
    436     return 1;
    437 }
    438 
    439 #######################################################################################
    440441#
    441442# runs ippToPsps to produce init batch
    … …  
    457458    my ($expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_;
    458459
    459     my $nebPath = $gpc1Db->getCameraStageSmf($expId);
     460    my $nebPath = $gpc1Db->getCameraStageSmfForThisDvoDb($dvoDb, $expId);
    460461    if (!$nebPath) { return 0; }
    461462
    … …  
    527528    $command .= " -input $input";
    528529    $command .= " -output $output";
    529     $command .= " -D CATDIR $dvodb";
     530    $command .= " -D CATDIR $fullDvoPath";
    530531    $command .= " -config ../config"; # TODO
    531     $command .= " -expid $expid";
     532        $command .= " -expid $expid";
    532533    $command .= " -expname $expName";
    533534    $command .= " -survey $surveyType";

  • branches/eam_branches/ipp-20100621/ippTools/share/Makefile.am

    r28794 r28980  
    66        addtool_checkminidvodbaddrun.sql \
    77        addtool_donecleanup.sql \
     8        addtool_find_cam_id_dvo.sql \
    89        addtool_find_cam_id.sql \
    910        addtool_find_pendingexp.sql \
    … …  
    2627        bgtool_definechip.sql \
    2728        bgtool_definewarp.sql \
     29        bgtool_listchip.sql \
     30        bgtool_listwarp.sql \
    2831        bgtool_revertchip.sql \
    2932        bgtool_revertwarp.sql \
    … …  
    150153        disttool_definebyquery_fake.sql \
    151154        disttool_definebyquery_raw.sql \
     155        disttool_definebyquery_sky.sql \
    152156        disttool_definebyquery_stack.sql \
    153157        disttool_definebyquery_warp.sql \
    … …  
    163167        disttool_pending_fake.sql \
    164168        disttool_pending_raw.sql \
     169        disttool_pending_sky.sql \
    165170        disttool_pending_stack.sql \
    166171        disttool_pending_warp.sql \
    … …  
    172177        disttool_processedcomponent.sql \
    173178        disttool_queuercrun.sql \
     179        disttool_rerun_select.sql \
    174180        disttool_revertrcrun.sql \
    175181        disttool_revertrun.sql \
    … …  
    324330        staticskytool_definebyquery_select.sql \
    325331        staticskytool_definebyquery_inputs.sql \
     332        staticskytool_export_run.sql \
     333        staticskytool_export_input.sql \
     334        staticskytool_export_result.sql \
    326335        staticskytool_inputs.sql \
    327336        staticskytool_todo.sql \

  • branches/eam_branches/ipp-20100621/ippTools/share/chiptool_setimfiletoupdate.sql

    r28022 r28980  
    55                              AND component = class_id)
    66SET chipRun.state = 'update',
    7     chipProcessedImfile.data_state = 'update'
     7    chipProcessedImfile.data_state = 'update',
     8    chipProcessedImfile.fault = 0
    89    -- set hook %s
    910WHERE chip_id = %lld

  • branches/eam_branches/ipp-20100621/ippTools/share/disttool_toadvance.sql

    r28794 r28980  
    240240        AND distComponent.fault = 0
    241241UNION
     242-- staticsky stage
     243-- NOTE this assumes that there is only one component per staticskyRun
     244-- (one skycell)
     245SELECT
     246    distRun.dist_id,
     247    stage,
     248    stage_id,
     249    outroot,
     250    label,
     251    clean
     252    FROM distRun
     253    JOIN staticskyResult on stage_id = sky_id
     254    LEFT JOIN distComponent
     255        ON distRun.dist_id = distComponent.dist_id
     256    WHERE
     257        distRun.state = 'new'
     258        AND distRun.fault = 0
     259        AND distRun.stage = 'sky'
     260        AND distComponent.component IS NOT NULL
     261        AND distComponent.fault = 0
     262UNION
    242263-- SSdiff stage
    243264SELECT

  • branches/eam_branches/ipp-20100621/ippTools/share/magicdstool_torevert_chip.sql

    r27945 r28980  
    22    magic_ds_id,
    33    magicDSRun.state,
    4     exp_id,
     4    rawExp.exp_id,
    55    re_place,
    66    camera,
    … …  
    1919    JOIN camProcessedExp using(cam_id)
    2020    JOIN chipRun ON (stage_id = chipRun.chip_id)
    21     -- JOIN chipProcessedImfile ON (stage_id = chipProcessedImfile.chip_id AND component = class_id)
    22     JOIN chipProcessedImfile USING(chip_id, exp_id)
    23     JOIN rawExp using(exp_id)
     21    JOIN chipProcessedImfile ON (stage_id = chipProcessedImfile.chip_id AND component = class_id)
     22    JOIN rawExp ON chipRun.exp_id = rawExp.exp_id
    2423WHERE magicDSRun.stage = 'chip'
    2524    AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0)

  • branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_create_tables.sql

    r28794 r28980  
    987987       projection_cell VARCHAR(64) NOT NULL,
    988988       path_base     VARCHAR(255) NOT NULL,
    989        PRIMARY KEY(warp_id),
    990        KEY(projection_cell),
     989       PRIMARY KEY(warp_id, projection_cell),
    991990       FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id)
    992991) ENGINE=innodb DEFAULT CHARSET=latin1;
    … …  
    10861085    projection_cell VARCHAR(64) NOT NULL,
    10871086    path_base   VARCHAR(255) NOT NULL,
    1088     PRIMARY KEY(sass_id),
    1089     KEY(projection_cell),
     1087    PRIMARY KEY(sass_id, projection_cell),
    10901088    FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id)
    10911089) ENGINE=innodb DEFAULT CHARSET=latin1;
    … …  
    11921190    projection_cell VARCHAR(64) NOT NULL,
    11931191    path_base   VARCHAR(255) NOT NULL,
    1194     PRIMARY KEY(diff_id),
    1195     KEY(projection_cell),
     1192    PRIMARY KEY(diff_id, projection_cell),
    11961193    FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id)
    11971194) ENGINE=innodb DEFAULT CHARSET=latin1;

  • branches/eam_branches/ipp-20100621/ippTools/share/staticskytool_result.sql

    r28091 r28980  
    1 SELECT
     1SELECT DISTINCT
    22    staticskyResult.*,
    33    staticskyRun.state,
    4     staticskyRun.tess_id,
    5     staticskyRun.skycell_id,
    6     staticskyRun.filter,
    74    staticskyRun.workdir,
    85    staticskyRun.label,
     6    stackRun.tess_id,
     7    stackRun.skycell_id,
     8    stackRun.filter
    99FROM staticskyRun
    10 JOIN staticskyResult
    11 USING(stack_id)
     10JOIN staticskyResult USING(sky_id)
     11JOIN staticskyInput USING(sky_id)
     12JOIN stackRun using(stack_id)

  • branches/eam_branches/ipp-20100621/ippTools/src/addtool.c

    r28295 r28980  
    144144    PXOPT_LOOKUP_BOOL(simple,     config->args, "-simple", false);
    145145
    146 
    147 
    148     // find the cam_id of all the exposures that we want to queue up.
    149     psString bare_query = pxDataGet("addtool_find_cam_id.sql");
    150     if (!bare_query) {
    151         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
    152         psFree(where);
    153         return false;
    154     }
    155 
    156146    // prevent queueing an addRun if a given exposure has already been added to
    157147    // the given dvo database
    158148    psString dvodb_string = NULL;
     149    psString bare_query = NULL;
    159150    if (dvodb) {
    160         // user supplied dvodb
    161         psStringAppend(&dvodb_string, "(previous_dvodb = '%s')", dvodb);
     151      psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_cam_id_dvo.sql\n", dvodb);
     152        // find the cam_id of all the exposures that we want to queue up.
     153        bare_query = pxDataGet("addtool_find_cam_id_dvo.sql");
     154        // user supplied dvodb
     155        psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb);
    162156    } else {
     157        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_cam_id.sql\n");
     158        // find the cam_id of all the exposures that we want to queue up.
     159        bare_query = pxDataGet("addtool_find_cam_id.sql");
    163160        // inherit dvodb from camRun, avoid matching NULL
    164161        psStringAppend(&dvodb_string, "(camRun.dvodb IS NOT NULL AND previous_dvodb = camRun.dvodb)");
     162    }
     163    if (!bare_query) {
     164        psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
     165        psFree(where);
     166        return false;
    165167    }
    166168    // Take the bare query and add the dvodb selector

  • branches/eam_branches/ipp-20100621/ippTools/src/bgtool.c

    r28794 r28980  
    3939static bool advancechipMode(pxConfig *config);
    4040static bool revertchipMode(pxConfig *config);
     41static bool listchipMode(pxConfig *config);
    4142static bool definewarpMode(pxConfig *config);
    4243static bool updatewarpMode(pxConfig *config);
    … …  
    4748static bool advancewarpMode(pxConfig *config);
    4849static bool revertwarpMode(pxConfig *config);
     50static bool listwarpMode(pxConfig *config);
    4951static bool tocleanchipMode(pxConfig *config);
    5052static bool cleanedchipMode(pxConfig *config);
    … …  
    100102        MODECASE(BGTOOL_MODE_ADVANCECHIP, advancechipMode);
    101103        MODECASE(BGTOOL_MODE_REVERTCHIP,  revertchipMode);
     104        MODECASE(BGTOOL_MODE_LISTCHIP,    listchipMode);
    102105        MODECASE(BGTOOL_MODE_DEFINEWARP,  definewarpMode);
    103106        MODECASE(BGTOOL_MODE_UPDATEWARP,  updatewarpMode);
    … …  
    108111        MODECASE(BGTOOL_MODE_ADVANCEWARP, advancewarpMode);
    109112        MODECASE(BGTOOL_MODE_REVERTWARP,  revertwarpMode);
     113        MODECASE(BGTOOL_MODE_LISTWARP,    listwarpMode);
    110114        MODECASE(BGTOOL_MODE_TOCLEANCHIP, tocleanchipMode);
    111115        MODECASE(BGTOOL_MODE_CLEANEDCHIP, cleanedchipMode);
    … …  
    907911    int numDeleted = psDBAffectedRows(config->dbh);
    908912    psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d chipBackgroundImfiles", numDeleted);
     913
     914    return true;
     915}
     916static bool listchipMode(pxConfig *config)
     917{
     918    PS_ASSERT_PTR_NON_NULL(config, false);
     919
     920    psMetadata *where = psMetadataAlloc();
     921    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "==");
     922    PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "==");
     923
     924    // chip_bg_id is required
     925    PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false);
     926
     927    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     928    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     929
     930    psString query = pxDataGet("bgtool_listchip.sql");
     931    if (!query) {
     932        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
     933        return false;
     934    }
     935
     936    psString whereStr = psStringCopy("");
     937    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     938    psStringAppend(&query, "\nWHERE %s", whereClause);
     939    psFree(whereClause);
     940    psFree(where);
     941
     942    if (limit) {
     943        psString limitString = psDBGenerateLimitSQL(limit);
     944        psStringAppend(&query, " %s", limitString);
     945        psFree(limitString);
     946    }
     947
     948    if (!p_psDBRunQuery(config->dbh, query)) {
     949        psError(psErrorCodeLast(), false, "database error");
     950        psFree(query);
     951        return false;
     952    }
     953    psFree(whereStr);
     954    psFree(query);
     955
     956    psArray *output = p_psDBFetchResult(config->dbh);
     957    if (!output) {
     958        psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query);
     959        return false;
     960    }
     961    if (!psArrayLength(output)) {
     962        psTrace("bgtool", PS_LOG_INFO, "no rows found");
     963        psFree(output);
     964        return true;
     965    }
     966
     967    if (psArrayLength(output)) {
     968        if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) {
     969            psError(psErrorCodeLast(), false, "failed to print array");
     970            psFree(output);
     971            return false;
     972        }
     973    }
     974
     975    psFree(output);
    909976
    910977    return true;
    … …  
    16861753    return true;
    16871754}
     1755static bool listwarpMode(pxConfig *config)
     1756{
     1757    PS_ASSERT_PTR_NON_NULL(config, false);
     1758
     1759    psMetadata *where = psMetadataAlloc();
     1760    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "==");
     1761    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
     1762
     1763    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     1764    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     1765
     1766    psString query = pxDataGet("bgtool_listwarp.sql");
     1767    if (!query) {
     1768        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
     1769        return false;
     1770    }
     1771
     1772    psString whereStr = psStringCopy("");
     1773    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     1774    psStringAppend(&query, "\nWHERE %s", whereClause);
     1775    psFree(whereClause);
     1776    psFree(where);
     1777
     1778    if (limit) {
     1779        psString limitString = psDBGenerateLimitSQL(limit);
     1780        psStringAppend(&query, " %s", limitString);
     1781        psFree(limitString);
     1782    }
     1783
     1784    if (!p_psDBRunQuery(config->dbh, query)) {
     1785        psError(psErrorCodeLast(), false, "database error");
     1786        psFree(query);
     1787        return false;
     1788    }
     1789    psFree(whereStr);
     1790    psFree(query);
     1791
     1792    psArray *output = p_psDBFetchResult(config->dbh);
     1793    if (!output) {
     1794        psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query);
     1795        return false;
     1796    }
     1797    if (!psArrayLength(output)) {
     1798        psTrace("bgtool", PS_LOG_INFO, "no rows found");
     1799        psFree(output);
     1800        return true;
     1801    }
     1802
     1803    if (psArrayLength(output)) {
     1804        if (!ippdbPrintMetadatas(stdout, output, "warpSkyfile", !simple)) {
     1805            psError(psErrorCodeLast(), false, "failed to print array");
     1806            psFree(output);
     1807            return false;
     1808        }
     1809    }
     1810
     1811    psFree(output);
     1812
     1813    return true;
     1814}
    16881815
    16891816static bool tocleanwarpMode(pxConfig *config)

  • branches/eam_branches/ipp-20100621/ippTools/src/bgtool.h

    r28794 r28980  
    3434    BGTOOL_MODE_ADVANCECHIP,
    3535    BGTOOL_MODE_REVERTCHIP,
     36    BGTOOL_MODE_LISTCHIP,
    3637    // Warp stage
    3738    BGTOOL_MODE_DEFINEWARP,
    … …  
    4344    BGTOOL_MODE_ADVANCEWARP,
    4445    BGTOOL_MODE_REVERTWARP,
     46    BGTOOL_MODE_LISTWARP,
    4547    // Cleanups
    4648    BGTOOL_MODE_TOCLEANCHIP,

  • branches/eam_branches/ipp-20100621/ippTools/src/bgtoolConfig.c

    r28794 r28980  
    184184    psMetadataAddS16(revertchipArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
    185185    psMetadataAddBool(revertchipArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false);
     186
     187    // -listchip
     188    psMetadata *listchipArgs = psMetadataAlloc();
     189    psMetadataAddS64(listchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id (required)", 0);
     190    psMetadataAddStr(listchipArgs, PS_LIST_TAIL, "-class_id",  0, "search by class_id", NULL);
     191    psMetadataAddBool(listchipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     192    psMetadataAddU64(listchipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
    186193
    187194    // -tocleanchip
    … …  
    347354    psMetadataAddBool(revertwarpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false);
    348355
     356    // -listwarp
     357    psMetadata *listwarpArgs = psMetadataAlloc();
     358    psMetadataAddS64(listwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id (required)", 0);
     359    psMetadataAddStr(listwarpArgs, PS_LIST_TAIL, "-skycell_id",  0, "search by skycell_id", NULL);
     360    psMetadataAddBool(listwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     361    psMetadataAddU64(listwarpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
     362
    349363    // -tocleanwarp
    350364    psMetadata *tocleanwarpArgs = psMetadataAlloc();
    … …  
    382396    PXOPT_ADD_MODE("-advancechip", "", BGTOOL_MODE_ADVANCECHIP, advancechipArgs);
    383397    PXOPT_ADD_MODE("-revertchip",  "", BGTOOL_MODE_REVERTCHIP,  revertchipArgs);
     398    PXOPT_ADD_MODE("-listchip",    "", BGTOOL_MODE_LISTCHIP,    listchipArgs);
    384399    PXOPT_ADD_MODE("-tocleanchip", "", BGTOOL_MODE_TOCLEANCHIP, tocleanchipArgs);
    385400    PXOPT_ADD_MODE("-cleanedchip", "", BGTOOL_MODE_CLEANEDCHIP, cleanedchipArgs);
    … …  
    395410    PXOPT_ADD_MODE("-advancewarp", "", BGTOOL_MODE_ADVANCEWARP, advancewarpArgs);
    396411    PXOPT_ADD_MODE("-revertwarp",  "", BGTOOL_MODE_REVERTWARP,  revertwarpArgs);
     412    PXOPT_ADD_MODE("-listwarp",    "", BGTOOL_MODE_LISTWARP,    listwarpArgs);
    397413    PXOPT_ADD_MODE("-tocleanwarp", "", BGTOOL_MODE_TOCLEANWARP, tocleanwarpArgs);
    398414    PXOPT_ADD_MODE("-cleanedwarp", "", BGTOOL_MODE_CLEANEDWARP, cleanedwarpArgs);

  • branches/eam_branches/ipp-20100621/ippTools/src/disttool.c

    r28794 r28980  
    3434static bool updaterunMode(pxConfig *config);
    3535static bool revertrunMode(pxConfig *config);
     36static bool rerunMode(pxConfig *config);
    3637static bool pendingcomponentMode(pxConfig *config);
    3738static bool addprocessedcomponentMode(pxConfig *config);
    … …  
    8384        MODECASE(DISTTOOL_MODE_UPDATERUN, updaterunMode);
    8485        MODECASE(DISTTOOL_MODE_REVERTRUN, revertrunMode);
     86        MODECASE(DISTTOOL_MODE_RERUN, rerunMode);
    8587        MODECASE(DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentMode);
    8688        MODECASE(DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentMode);
    … …  
    327329        // stack stage doesn't require magic
    328330        no_magic = true;
     331    } else if (!strcmp(stage, "sky")) {
     332        magicRunType = "staticskyRun";
     333        query = pxDataGet("disttool_definebyquery_sky.sql");
     334        if (!query) {
     335            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
     336            psFree(where);
     337            return false;
     338        }
     339
     340        if (label) {
     341            psStringAppend(&query, " AND (staticskyRun.label = '%s')", label);
     342        }
     343        if (dist_group) {
     344            psStringAppend(&query, " AND (sticskyRun.dist_group = '%s')", dist_group);
     345        }
     346        // (static)sky stage doesn't require magic
     347        no_magic = true;
    329348    } else if (!strcmp(stage, "SSdiff")) {
    330349      magicRunType = "diffRun";
    … …  
    587606}
    588607
     608static bool rerunMode(pxConfig *config)
     609{
     610    psMetadata *where = psMetadataAlloc();
     611    PXOPT_COPY_S64(config->args, where, "-dist_id", "distRun.dist_id", "==");
     612    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");;
     613    PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
     614    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
     615    PXOPT_COPY_STR(config->args, where, "-data_group", "data_group", "==");
     616
     617    // require data_group or dist_id to be supplied
     618    PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", true, false);
     619    PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", false, false);
     620    if (!data_group && !dist_id) {
     621        psError(PXTOOLS_ERR_CONFIG, true, "data_group or dist_id is required");
     622        return false;
     623    }
     624    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
     625
     626    psString label_hook = psStringCopy("");
     627    if (set_label) {
     628        psStringAppend(&label_hook, ", label = '%s'", set_label);
     629    }
     630
     631    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
     632    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     633    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     634
     635
     636    // It might be useful to be able to query by the parameters of the underlying runs
     637
     638    if (!psListLength(where->list)) {
     639        psFree(where);
     640        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
     641        return false;
     642    }
     643
     644    psString query = pxDataGet("disttool_rerun_select.sql");
     645    if (!query) {
     646        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
     647        if (!psDBRollback(config->dbh)) {
     648            psError(PS_ERR_UNKNOWN, false, "database error");
     649        }
     650        return false;
     651    }
     652
     653    if (psListLength(where->list)) {
     654        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     655        psStringAppend(&query, " AND %s", whereClause);
     656        psFree(whereClause);
     657    }
     658    psFree(where);
     659
     660    if (limit) {
     661        psString limitString = psDBGenerateLimitSQL(limit);
     662        psStringAppend(&query, " %s", limitString);
     663        psFree(limitString);
     664    }
     665
     666    if (!p_psDBRunQuery(config->dbh, query)) {
     667        psError(PS_ERR_UNKNOWN, false, "database error");
     668        psFree(query);
     669        return false;
     670    }
     671    psFree(query);
     672
     673    psArray *output = p_psDBFetchResult(config->dbh);
     674    if (!output) {
     675        psError(PS_ERR_UNKNOWN, false, "database error");
     676        return false;
     677    }
     678    if (!psArrayLength(output)) {
     679        psTrace("disttool", PS_LOG_INFO, "no rows found");
     680        psFree(output);
     681        return true;
     682    }
     683
     684
     685    if (pretend) {
     686        if (!ippdbPrintMetadatas(stdout, output, "distRunsToUpdate", true)) {
     687            psError(PS_ERR_UNKNOWN, false, "failed to print array");
     688            psFree(output);
     689            return false;
     690        }
     691        psFree(output);
     692        return true;
     693    }
     694
     695    query = "DELETE FROM distComponent where dist_id = %" PRId64;
     696   
     697    for (long i=0; i < psArrayLength(output); i++) {
     698        psMetadata *md = output->data[i];
     699        psS64 dist_id = psMetadataLookupS64(NULL, md, "dist_id");
     700
     701        if (!psDBTransaction(config->dbh)) {
     702            psError(PS_ERR_UNKNOWN, false, "database error");
     703            return false;
     704        }
     705
     706        if (!p_psDBRunQueryF(config->dbh, query, dist_id)) {
     707            psError(PS_ERR_UNKNOWN, false, "database error");
     708            psFree(query);
     709            if (!psDBRollback(config->dbh)) {
     710                psError(PS_ERR_UNKNOWN, false, "database error");
     711            }
     712            return false;
     713        }
     714
     715        if (!p_psDBRunQueryF(config->dbh,
     716        "UPDATE distRun SET state = 'new', fault=0 %s WHERE dist_id =%" PRId64,
     717                label_hook, dist_id)) {
     718            psError(PS_ERR_UNKNOWN, false, "database error");
     719            psFree(query);
     720            if (!psDBRollback(config->dbh)) {
     721                psError(PS_ERR_UNKNOWN, false, "database error");
     722            }
     723            return false;
     724        }
     725        if (!psDBCommit(config->dbh)) {
     726            psError(PS_ERR_UNKNOWN, false, "database error");
     727            return false;
     728        }
     729        printf("re-running distRun %" PRId64 "\n", dist_id);
     730    }
     731
     732    psFree(output);
     733
     734    return true;
     735}
    589736static bool revertrunMode(pxConfig *config)
    590737{

  • branches/eam_branches/ipp-20100621/ippTools/src/disttool.h

    r26193 r28980  
    2828    DISTTOOL_MODE_UPDATERUN,
    2929    DISTTOOL_MODE_REVERTRUN,
     30    DISTTOOL_MODE_RERUN,
    3031    DISTTOOL_MODE_PENDINGCOMPONENT,
    3132    DISTTOOL_MODE_ADDPROCESSEDCOMPONENT,

  • branches/eam_branches/ipp-20100621/ippTools/src/disttoolConfig.c

    r28259 r28980  
    8585    psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault",      0, "define fault code", 0);
    8686
     87    // -rerun
     88    psMetadata *rerunArgs = psMetadataAlloc();
     89    psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-data_group",    0, "define data_group (required)", NULL);
     90    psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0);
     91    psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-stage",    0, "search by stage", NULL);
     92    psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0);
     93    psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-label",    0, "search by label", NULL);
     94    psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-set_label",    0, "define new label", NULL);
     95    psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false);
     96    psMetadataAddU64(rerunArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);
     97    psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     98
    8799    // -revertrun
    88100    psMetadata *revertrunArgs = psMetadataAlloc();
    … …  
    327339    psMetadata *modes = psMetadataAlloc();
    328340
    329     PXOPT_ADD_MODE("-definebyquery",    "", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs);
    330     PXOPT_ADD_MODE("-updaterun",    "", DISTTOOL_MODE_UPDATERUN, updaterunArgs);
    331     PXOPT_ADD_MODE("-revertrun",    "", DISTTOOL_MODE_REVERTRUN, revertrunArgs);
     341    PXOPT_ADD_MODE("-definebyquery",    "define distribution run", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs);
     342    PXOPT_ADD_MODE("-updaterun",    "update distRun", DISTTOOL_MODE_UPDATERUN, updaterunArgs);
     343    PXOPT_ADD_MODE("-rerun",        "reprocess a cleaned distribution run", DISTTOOL_MODE_RERUN, rerunArgs);
     344    PXOPT_ADD_MODE("-revertrun",    "revert distRun.faults", DISTTOOL_MODE_REVERTRUN, revertrunArgs);
    332345    PXOPT_ADD_MODE("-pendingcomponent",   "", DISTTOOL_MODE_PENDINGCOMPONENT,    pendingcomponentArgs);
    333346    PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs);
    334     PXOPT_ADD_MODE("-revertcomponent",    "", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs);
     347    PXOPT_ADD_MODE("-revertcomponent",    "revert faulted components", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs);
    335348    PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs);
    336349    PXOPT_ADD_MODE("-toadvance",          "", DISTTOOL_MODE_TOADVANCE, toadvanceArgs);
    … …  
    340353    PXOPT_ADD_MODE("-updatefileset",         "", DISTTOOL_MODE_UPDATEFILESET, updatefilesetArgs);
    341354    PXOPT_ADD_MODE("-listfilesets",        "", DISTTOOL_MODE_LISTFILESETS, listfilesetsArgs);
    342     PXOPT_ADD_MODE("-revertfileset",      "", DISTTOOL_MODE_REVERTFILESET, revertfilesetArgs);
     355    PXOPT_ADD_MODE("-revertfileset",      "revert faulted filesets", DISTTOOL_MODE_REVERTFILESET, revertfilesetArgs);
    343356    PXOPT_ADD_MODE("-queuercrun",         "", DISTTOOL_MODE_QUEUERCRUN, queuercrunArgs);
    344357    PXOPT_ADD_MODE("-updatercrun",        "", DISTTOOL_MODE_UPDATERCRUN, updatercrunArgs);

  • branches/eam_branches/ipp-20100621/ippTools/src/magicdstool.c

    r28794 r28980  
    613613    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "==");
    614614    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
     615    PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
    615616    pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "==");
    616617

  • branches/eam_branches/ipp-20100621/ippTools/src/magicdstoolConfig.c

    r28794 r28980  
    120120    psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0);
    121121    psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
     122    psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage ID", 0);
    122123    psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL);
    123124    psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);

  • branches/eam_branches/ipp-20100621/ippTools/src/pstamptool.c

    r28794 r28980  
    12201220    PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
    12211221    PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
     1222    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
     1223    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
    12221224    PXOPT_COPY_STR(config->args, where, "-rlabel", "rlabel", "==");
    12231225    pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "==");

  • branches/eam_branches/ipp-20100621/ippTools/src/pstamptoolConfig.c

    r28794 r28980  
    211211    psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-component", 0,    "define component for dependent", NULL);
    212212    psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define dep_id for dependent", 0);
     213    psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-job_id", 0, "define job_id for dependent", 0);
     214    psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-req_id", 0, "define eqp_id for dependent", 0);
    213215    psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK,   "define label for pstampRequest", NULL);
    214216    psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0,   "define label for dependent", NULL);

  • branches/eam_branches/ipp-20100621/ippTools/src/staticskytool.c

    r28154 r28980  
    727727bool exportrunMode(pxConfig *config)
    728728{
    729 # if (0)
    730729  typedef struct ExportTable {
    731730    char tableName[80];
    … …  
    737736  PS_ASSERT_PTR_NON_NULL(config, NULL);
    738737
    739   PXOPT_LOOKUP_S64(det_id, config->args, "-stack_id", true,  false);
     738  PXOPT_LOOKUP_S64(det_id, config->args, "-sky_id", true,  false);
    740739  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
    741740  PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
    … …  
    753752  }
    754753  psMetadata *where = psMetadataAlloc();
    755   PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
     754  PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "==");
    756755
    757756  ExportTable tables [] = {
    758     {"stackRun", "staticskytool_export_run.sql"},
    759     {"stackInputSkyfile", "staticskytool_export_input_skyfile.sql"},
    760     {"stackSumSkyfile", "staticskytool_export_sum_skyfile.sql"},
     757    {"staticskyRun", "staticskytool_export_run.sql"},
     758    {"staticskyInput", "staticskytool_export_input.sql"},
     759    {"staticskyResult", "staticskytool_export_result.sql"},
    761760  };
    762761
    … …  
    800799
    801800    if (clean) {
    802         if (!strcmp(tables[i].tableName, "stackRun")) {
    803             if (!pxSetStateCleaned("stackRun", "state", output)) {
     801        if (!strcmp(tables[i].tableName, "staticskyRun")) {
     802            if (!pxSetStateCleaned("staticskyRun", "state", output)) {
    804803                psFree(output);
    805804                psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
    … …  
    821820    fclose (f);
    822821
    823 # endif
    824822    return true;
    825823}

  • branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_create

    r28043 r28980  
    1717     $query_id,
    1818     $nostage,
     19     $version,
    1920     );
    2021
    … …  
    2223           'input|i=s'    => \$input,
    2324           'output|o=s'   => \$output,
    24            'query_id|q=s'  => \$query_id,
     25           'query_id|q=s' => \$query_id,
     26           'version|v=s'  => \$version,
    2527           'nostage'      => \$nostage,
    2628) or pod2usage( 2 );
    … …  
    3032           -exitval => 3) unless defined $input and defined $output;
    3133
    32 # The header kewords
     34# Read what we've been given
     35my $in;
     36if ($input eq '-') {
     37    $in = \*STDIN;
     38} else {
     39    open $in, "<$input" or die "cannot open $input for reading";
     40}
     41my %colData;
     42my %headerData;
     43my $numRows = read_data_for_table($in,'\s+', \%colData, \%headerData);
     44if (!$numRows) {
     45    print STDERR "no data in $input\n";
     46    exit 1;
     47}
     48
     49# The header keywords
    3350my $header = [
    3451        { name =>  'QUERY_ID',
    … …  
    6279                    value => undef
    6380        },
    64        { name =>  'STAGE',
     81        { name =>  'STAGE',
    6582                   writetype => TSTRING,
    6683                   comment => 'processing stage to examine',
    6784                   value => undef
    68        }
     85        }
    6986];
    70 unless(defined($nostage)) {
    71     push @{ $header },        { name =>  'STAGE',
    72                                 writetype => TSTRING,
    73                                 comment => 'processing stage to examine',
    74                                 value => undef
    75     };
     87
     88# Validate header.
     89
     90if (defined($query_id)) {
     91    $headerData{QUERY_ID} = $query_id;
     92}
     93unless (exists($headerData{QUERY_ID})) {
     94    die "No QUERY_ID specified for header.";
     95}
     96
     97unless (exists($headerData{EXTVER})) {
     98    die "No EXTVER specified for header.";
     99}
     100if ($headerData{EXTVER} == 1) {
     101    unless (exists($headerData{STAGE})) {
     102        warn "No STAGE value specified in header. Assuming default value of 'diff'";
     103        $headerData{STAGE} = 'diff';
     104    }
     105    foreach my $entry_ref (@{ $header }) {
     106        my $name = $entry_ref->{name};
     107        unless (exists($headerData{$name})) {
     108            die "Required header value $name not specified (try EXTVER=2?).";
     109        }
     110        $entry_ref->{value} = $headerData{$name};
     111    }
     112}   
     113elsif ($headerData{EXTVER} == 2) {
     114    unless (exists($headerData{STAGE})) {
     115        warn "No STAGE value specified in header. Assuming default value of 'diff'";
     116        $headerData{STAGE} = 'diff';
     117    }
     118    my $tmp_header;
     119    foreach my $entry_ref (@{ $header }) {
     120        my $name = $entry_ref->{name};
     121        if (exists($headerData{$name})) {
     122            $entry_ref->{value} = $headerData{$name};
     123            push @{ $tmp_header }, $entry_ref;
     124        }
     125    }
     126    $header = $tmp_header;
     127}
     128else {
     129    die "Unknown EXTVER = $headerData{EXTVER}.";
    76130}
    77131
    … …  
    91145        { name => 'MAG',      type => 'D',   writetype => TDOUBLE },
    92146];
    93 
    94 my $in;
    95 if ($input eq '-') {
    96     $in = \*STDIN;
    97 } else {
    98     open $in, "<$input" or die "cannot open $input for reading";
    99 }
    100 
    101 my @colData;
    102 foreach (@$columns) {
    103     push @colData, [];
    104 }
    105 
    106 
    107 my $numRows = read_data_for_table($in,'\s+', \@colData, $header);
    108 if (!$numRows) {
    109     print STDERR "no data in $input\n";
    110     exit 1;
    111 }
    112 
    113 # overwrite the QUERY_ID value from the input file with the command
    114 # line argument
    115 
    116 if ($query_id) {
    117     $header->[0]->{value} = $query_id;
    118 }
    119 
    120 my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $columns, $header);
     147my $columns_v2 = [
     148    { name => 'FPA_ID', type => '20A', writetype => TSTRING },
     149    { name => 'MJD-OBS', type => 'D', writetype => TDOUBLE },
     150    { name => 'FILTER', type => '20A', writetype => TSTRING },
     151    { name => 'OBSCODE', type => '20A', writetype => TSTRING },
     152    { name => 'STAGE', type => '20A', writetype => TSTRING }
     153    ];
     154# Validate the data.
     155if ($headerData{EXTVER} == 1) {
     156    foreach my $entry_ref (@{ $columns }) {
     157        my $name = $entry_ref->{name};
     158        unless (exists($colData{$name})) {
     159            die "Required data column $name not found (try EXTVER=2?).";
     160        }
     161    }
     162}
     163if ($headerData{EXTVER} == 2) {
     164    my $tmp_columns;
     165    foreach my $entry_ref (@{ $columns }) {
     166        my $name = $entry_ref->{name};
     167        if (exists($colData{$name})) {
     168            push @{ $tmp_columns }, $entry_ref;
     169        }
     170    }
     171    foreach my $entry_ref (@{ $columns_v2 }) {
     172        my $name = $entry_ref->{name};
     173        if (exists($colData{$name})) {
     174            push @{ $tmp_columns }, $entry_ref;
     175        }
     176    }
     177    $columns = $tmp_columns;
     178}           
     179
     180# Construct the array of arrays
     181my @colDataAoA = ();
     182foreach my $col_def (@{ $columns }) {
     183    my $name = $col_def->{name};
     184    push @colDataAoA, $colData{$name};
     185}
     186
     187# foreach (@$columns) {
     188#     push @colData, [];
     189# }
     190
     191
     192
     193my $status = make_fits_table($output, EXTNAME, $numRows, \@colDataAoA, $columns, $header);
    121194
    122195exit $status;
    … …  
    230303
    231304    my $line_num = 0;
    232 
    233     # read data for header if any data is expected
    234     if ($header) {
    235         my $nhead = @$header;
    236         while (my $line = <$in>) {
    237             $line_num++;
    238             chomp $line;
    239             next if !$line;             # skip blank lines
    240             next if ($line =~ /^#/);    # skip comment lines
    241             my @vals = split /$sep/, $line;
    242             my $nvals = @vals;
    243             die "number of header columns in input $nvals does not equal expected number of header words $nhead"
    244                     if (@vals != @$header);
    245 
    246             for (my $i=0; $i < @$header; $i++) {
    247                 $header->[$i]->{value} = $vals[$i];
    248             }
    249 
    250             last; # only one header line
    251         }
    252     }
    253 
    254     my $row_num = 0;
    255     my $ncols = @$colData;
    256     while (my $line = <$in>) {
    257         chomp $line;
    258         $line_num++;
    259         next if !$line;             # skip blank lines
    260         next if ($line =~ /^#/);    # skip comment lines
    261 
    262         my @vals = split /$sep/, $line;
    263         my $nvals = @vals;
    264         die "number of columns $nvals in input does not equal expected number of header "
    265                 . " words $ncols on line $line_num" if ($nvals != $ncols);
    266 
    267         for (my $col = 0; $col < @$colData; $col++) {
    268             $colData->[$col]->[$row_num] = $vals[$col];
    269         }
    270         $row_num++;
    271     }
    272 
     305    my @keywords = ();
     306    my $row_num;
     307    while (<$in>) {
     308        chomp;
     309        if ($line_num == 0) {
     310            # Parse header information keywords
     311            $_ =~ s/#//g;
     312            @keywords = split /\s+/;
     313            if ($keywords[0] eq '') {
     314                shift(@keywords);
     315            }
     316        }
     317        elsif ($line_num == 1) {
     318            # Parse header information values
     319            my @values = split /\s+/;
     320            if ($#values != $#keywords) {
     321                die "Number of header columns in input does not equal expected number of header words";
     322            }
     323            for (my $i = 0; $i <= $#values; $i++) {
     324                $header->{$keywords[$i]} = $values[$i];
     325            }           
     326        }
     327        elsif ($line_num == 2) {
     328            # Parse table information keywords, dumping old keywords
     329            $_ =~ s/#//g;
     330            @keywords = split /\s+/;
     331            if ($keywords[0] eq '') {
     332                shift(@keywords);
     333            }
     334        }
     335        else {
     336            # Parse table information values
     337            unless ($_ =~ /^#/) {
     338                my @values = split /\s+/;
     339                if ($#values != $#keywords) {
     340                    die "Number of header columns in input does not equal expected number of header words";
     341                }
     342                for (my $i = 0; $i <= $#values; $i++) {
     343                    push @{ $colData{$keywords[$i]} }, $values[$i];
     344                    $row_num = $#{ $colData{$keywords[$i] } } + 1;
     345                }               
     346            }
     347        }
     348        $line_num++;
     349    }
    273350    # we return the number of rows read
    274351    return $row_num;

  • branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_read

    r28794 r28980  
    2020use Math::Trig;
    2121use Data::Dumper;
     22use IPC::Cmd 0.36 qw( can_run run );
    2223
    2324use constant EXTNAME => 'MOPS_DETECTABILITY_QUERY'; # Extension name for table
    … …  
    3132     $output,                   # Name of output table
    3233     $save_temps,               # Save temporary files?
     34     $dbname,                   # needed to do camtool lookups.
    3335     );
     36
     37my $regtool = can_run('regtool') or (die "Can't find regtool");
     38my $camtool = can_run('camtool') or (die "Can't find regtool");
     39
    3440
    3541GetOptions(
    3642           'input|i=s'    => \$input,
    3743           'output|o=s'   => \$output,
     44           'dbname=s'     => \$dbname,
    3845           'nolabel|l'    => \$no_print_label,
    3946           'noheader|h'   => \$no_print_header,
    … …  
    6572
    6673# The keywords found in the header
    67 # my $header = {
    68 #         'EXTVER'   => { name => 'EXTVER',
    69 #                      writetype => TSTRING,
    70 #                      comment => 'Extension version',
    71 #                      value => undef
    72 #                       },
    73 #         'EXTNAME'  => { name => 'EXTNAME',
    74 #                      writetype => TSTRING,
    75 #                      comment => 'name of this binary table extension',
    76 #                      value => undef
    77 #                     },
    78 #         'QUERY_ID' => { name => 'QUERY_ID',
    79 #                         writetype => TSTRING,
    80 #                         comment => 'MOPS Query ID for this batch query',
    81 #                         value => undef
    82 #                       },
    83 #         'FPA_ID'   => { name => 'FPA_ID',
    84 #                         writetype => TSTRING,
    85 #                         comment => 'orginal FPA_ID used at ingest',
    86 #                         value => undef
    87 #                       },
    88 #         'MJD_OBS'  => { name => 'MJD-OBS',
    89 #                         writetype => TDOUBLE,
    90 #                         comment => 'starting time of the exposure, MJD',
    91 #                         value => undef
    92 #                       },
    93 #         'FILTER'   => { name => 'FILTER',
    94 #                         writetype => TSTRING,
    95 #                         comment => 'effective filter use for the exposure',
    96 #                         value => undef
    97 #                       },
    98 #         'OBSCODE'  => { name => 'OBSCODE',
    99 #                         writetype => TSTRING,
    100 #                         comment => 'site identifier (MPC observatory code)',
    101 #                         value => undef
    102 #                       },
    103 #       'STAGE'    => {
    104 #                       name => 'STAGE',
    105 #                       writetype => TSTRING,
    106 #                       comment => 'processing stage to examine',
    107 #                       value => undef
    108 #                     }
    109 # };
    11074
    11175my $parse_error = 0;
    112 # Parse the header to determine what we expect to find.
     76# Parse the header to determine what we have to work with.
     77# Clean up empty space and remove unneeded single quotes.
    11378foreach my $header_key (keys %{ $inHeader }) {
    11479    $inHeader->{$header_key} =~ s/\s+//g;
    11580    $inHeader->{$header_key} =~ s/\'//g;
    11681}
    117 my ($EXTVER,$headerEXTNAME,$QUERY_ID,$FPA_ID,$MJD_OBS,$FILTER,$OBSCODE,$STAGE) =
    118     ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID},$inHeader->{FPA_ID},
    119      $inHeader->{'MJD-OBS'},$inHeader->{FILTER},$inHeader->{OBSCODE},$inHeader->{STAGE});
     82my ($EXTVER,$headerEXTNAME,$QUERY_ID,
     83    $FPA_ID,$MJD_OBS,$FILTER,
     84    $OBSCODE,$STAGE) =
     85    ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID},
     86     $inHeader->{FPA_ID},$inHeader->{'MJD-OBS'},$inHeader->{FILTER},
     87     $inHeader->{OBSCODE},$inHeader->{STAGE});
     88
    12089unless(defined($EXTVER) && defined($headerEXTNAME) &&
    12190       (($EXTVER == 1)||($EXTVER == 2)) &&
    122        ($headerEXTNAME eq EXTNAME)) {
     91       ($headerEXTNAME eq EXTNAME)&&
     92       (defined($QUERY_ID))) {
    12393    $parse_error = 1;
    12494}
    … …  
    143113        # apparent magnitude
    144114        { name => 'MAG',      type => 'D',   writetype => TDOUBLE, version => 1 },
    145     # v2 query_id: MOPS query ID for this batch query
     115    # v2 query_id: needs to be here.
    146116    { name => 'QUERY_ID', type => '20A', writetype => TSTRING, version => 2, default => $QUERY_ID },
    147117    # v2 fpa_id: original FPA_ID used at ingest
    … …  
    172142
    173143    if ($col->{version} > $EXTVER) {
    174         @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows);
     144        @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1);
    175145        next;
    176146    }
    177     $inFits->get_colnum(0, $col->{name}, $col_num, $status) and check_fitsio($status);
    178     $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status);
    179     $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status)
    180                                                                     and check_fitsio($status);
    181     $colData{$col->{name}} = $col_data;
    182     if ($col->{name} eq 'MJD-OBS') {
    183         print @{ $col_data } . "\n";
     147    else {
     148        if (defined($col->{default})) {
     149            @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1);
     150            next;
     151        }       
     152    }   
     153
     154    $inFits->get_colnum(0, $col->{name}, $col_num, $status);
     155    if ($status == 0) {
     156        $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status);
     157        $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status)
     158            and check_fitsio($status);
     159        $colData{$col->{name}} = $col_data;
     160        if ($col->{name} eq 'QUERY_ID') {
     161            print @{ $col_data } . "\n";
     162        }
     163
     164    }
     165    elsif ($status == 219) {
     166        @{ $colData{$col->{name}} } = map { "Not Set" } (0 .. $numRows - 1);
     167        $status = 0;
     168    }
     169}
     170
     171# Set things that aren't set.
     172for (my $i = 0; $i < $numRows; $i++) {
     173# Simple stuff first.
     174    my %known_filters = ();
     175    if ($colData{STAGE}[$i] eq "Not Set") {
     176        $colData{STAGE}[$i] = 'diff';
     177    }
     178    if ($colData{OBSCODE}[$i] eq "Not Set") {
     179        $colData{OBSCODE}[$i] = 566;
     180    }
     181# Define filter and MJD from FPA_ID
     182    if (($colData{FILTER}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) {
     183        if (exists($known_filters{$colData{FPA_ID}[$i]})) {
     184            $colData{FILTER}[$i] = $known_filters{$colData{FPA_ID}[$i]};
     185        }
     186        else {
     187            my $cmd = "$regtool -processedexp -dbname $dbname -exp_name $colData{FPA_ID}[$i]";
     188            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     189                run(command => $cmd, verbose => 0);
     190            unless ($success) {
     191                # This is a problem, because I'm not sure how we handle a failure to read something.
     192                # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it,
     193                # we can't send that response back.
     194                die("Unable to perform $cmd error code: $error_code");
     195            }
     196            foreach my $entry (split /\n/, (join "", @$full_buf)) {
     197                $entry =~ s/^\s+//;
     198                my @line = split /\s+/, $entry;
     199                if (scalar(@line) != 3) {
     200                    next;
     201                }
     202                my ($key,$type,$value) = @line;
     203#               print "$entry => $key / $type / $value \n";
     204                if ($key =~ /filter/i) {
     205                    $value =~ s/.00000//;
     206                    $colData{FILTER}[$i] = $value;
     207                    $known_filters{$colData{FPA_ID}[$i]} = $value;
     208                }
     209            }
     210        }
     211    }
     212    if (($colData{'MJD-OBS'}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) {
     213        # HACK!
     214        my $mjd = $colData{FPA_ID}[$i];
     215        $mjd =~ s/o(....)g.*/$1/;
     216        $mjd += 50000;
     217        $colData{'MJD-OBS'}[$i] = $mjd;
     218    }
     219    if (($colData{FPA_ID}[$i] eq "Not Set")&&(($colData{'FILTER'}[$i] ne "Not Set")&&
     220                                              ($colData{'MJD-OBS'}[$i] ne "Not Set"))) {
     221        my $dateobs_begin = mjd_to_dateobs($colData{'MJD-OBS'}[$i]);
     222        my $dateobs_end = mjd_to_dateobs($colData{'MJD-OBS'}[$i] + 1);
     223        my $ra = $colData{'RA1_DEG'}[$i];
     224        my $dec = $colData{'DEC1_DEG'}[$i];
     225        my $filter = $colData{'FILTER'}[$i] . ".00000";
     226        my $cmd = "$camtool -processedexp -dbname $dbname -filter $filter ";
     227        $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
     228        $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1";
     229        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     230            run(command => $cmd, verbose => 0);
     231        unless ($success) {
     232            # This is a problem, because I'm not sure how we handle a failure to read something.
     233            # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it,
     234            # we can't send that response back.
     235            die("Unable to perform $cmd error code: $error_code");
     236        }
     237        foreach my $entry (split /\n/, (join "", @$full_buf)) {
     238            $entry =~ s/^\s+//;
     239            my @line = split /\s+/, $entry;
     240            if (scalar(@line) != 3) {
     241                next;
     242            }
     243            my ($key,$type,$value) = @line;
     244#               print "$entry => $key / $type / $value \n";
     245            if ($key =~ /exp_name/i) {
     246                $colData{FPA_ID}[$i] = $value;
     247            }
     248        }
    184249    }
    185250}
    … …  
    189254foreach my $colName (@unique_fields) {
    190255    my %counter = ();
     256    my $i = 0;
    191257    foreach my $row (@{ $colData{$colName} }) {
    192258        $counter{$row} = 1;
    … …  
    232298        my $msg;                # Message to output
    233299        Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
    234         die "CFITSIO error: $msg\n";
     300        die "CFITSIO error: $status => $msg\n";
    235301    }
    236302}
    … …  
    252318}
    253319
     320# Stolen from PStamp/Job.pm.  Thanks, Bill.
     321sub mjd_to_dateobs {
     322    my $mjd = shift;
     323
     324    my $ticks = ($mjd - 40587.0) * 86400;
     325
     326    my ($sec, $min, $hr, $day, $mon, $year) = gmtime($ticks);
     327
     328    return sprintf "'%4d-%02d-%02dT%02d:%02d:%02dZ'", $year+1900, $mon+1, $day, $hr, $min, $sec;
     329}
     330
    254331
    255332

  • branches/eam_branches/ipp-20100621/pstamp/scripts/detectability_respond.pl

    r28794 r28980  
    108108# Parse input request file using detect_query_read (as it's already written).
    109109#
    110     my $dqr_command = "$detect_query_read --input $request_file";
     110    my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file";
    111111    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    112112        run(command => $dqr_command, verbose => $verbose);
    … …  
    308308}
    309309close(WISDOM);
     310
     311# If there is anything that needs to be updated, create the update request list, and then exit the program.
    310312my $exit_code = 0;
    311313my $update_request_file = "${workdir}/update_request.dat";
    … …  
    316318            $exit_code = 25;
    317319        }
     320        elsif ($fault != 0) {
     321            $exit_code = 21;
     322        }
    318323        my $update_request = join ' ', @{ $update_request{$images}{$fault} };
    319324        print UPDATE_REQUEST "$update_request\n";
    … …  
    321326}
    322327close(UPDATE_REQUEST);
    323 # if ($exit_code != 0) {
    324 #     exit($exit_code);
    325 # }
     328if ($exit_code != 0) {
     329    exit($exit_code);
     330}
    326331
    327332# This duplicates stuff returned by PSTAMP, but my thought is to convert that ---^ into a conditional, in which I read
    … …  
    338343    }
    339344}
    340 #exit(10);
    341345
    342346# run ppCoord and psphotForced to calculate the required data.
    … …  
    569573        $columns = [
    570574            # matching rownum from detectability original request
    571             { name => 'ROWNUM',   type => 'V', writetype => TULONG },
     575            { name => 'ROWNUM',   type => '20A', writetype => TSTRING },
    572576            # any errors that occurred during processing
    573577            { name => 'ERROR_CODE',   type => 'V', writetype => TULONG },
    … …  
    715719#       print STDERR "$writeType $i $numRows $colName $status @{ $colData{$colName} }\n";
    716720        $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData{$colName}, $status );
     721#       print "$writeType $i $numRows $colName $status\n";
    717722        check_fitsio( $status );
    718        
    719723    }
    720724    $outFits->close_file( $status );
    … …  
    730734        my $msg;                # Message to output
    731735        Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
     736        carp("CFITSIO error: $status => $msg");
    732737        die "CFITSIO error: $msg\n";
    733738    }

  • branches/eam_branches/ipp-20100621/pstamp/scripts/dqueryparse.pl

    r28794 r28980  
    158158    }
    159159}
    160 else {
     160elsif ($fault == $PSTAMP_NOT_AVAILABLE) {
    161161    # Failed to run correctly, which means that we need to queue a job and flag data for updating.
    162162    # Get the dependency id for the data we're requesting be updated.
    … …  
    191191    close(UPDATE_REQUEST);
    192192}
     193else {
     194    my_die ("Parse fault!!", $PS_EXIT_UNKNOWN_ERROR);
     195}
    193196
    194197# This does not set the request state to stop.  That will happen with the request_finish.pl script,

  • branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_checkdependent.pl

    r28794 r28980  
    147147    #       been updated we need to go and queue it.
    148148
    149     if ($it->{fault}) {
     149    if (($it->{state} ne 'cleaned') and $it->{fault}) {
    150150        my_die("Component faulted on update dep_id: $dep_id",
    151151                $PS_EXIT_SYS_ERROR);

  • branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_job_run.pl

    r28794 r28980  
    9393
    9494    $argString .= " -file $params->{image}";
    95     $argString .= " -mask $params->{mask}";
    96     my @file_list = ($params->{image}, $params->{mask});
     95    my @file_list = ($params->{image});
     96   
     97    my $nan_masked = 1;
     98    if (!$params->{magicked}) {
     99        $nan_masked = 0;
     100    }
     101    if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) {
     102        $argString .= " -mask $params->{mask}";
     103        push @file_list, $params->{mask};
     104    }
    97105    if ($options & $PSTAMP_SELECT_VARIANCE) {
    98106        $argString .= " -variance $params->{weight}";
    … …  
    351359    foreach my $f (@_) {
    352360        if (!$ipprc->file_exists($f)) {
    353             my_die( "file $f does not exist:", $job_id, $PS_EXIT_SYS_ERROR);
     361            my_die( "file $f does not exist:", $job_id, $PSTAMP_GONE, 'stop');
    354362        }
    355363    }
    … …  
    361369    my $job_id = shift;         # job identifier
    362370    my $exit_code = shift;      # Exit code to add
     371    my $job_state = shift;      # new pstampJob.state
    363372
    364373    $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code;
     374    $job_state = 'run' unless $job_state;
    365375
    366376    carp($msg);
    … …  
    370380        $command .= " -set_fault $exit_code";
    371381        # XXX: fix pstamptool to not require -state when -fault with nonzero value is provided
    372         $command .= " -set_state run";
     382        $command .= " -set_state $job_state";
    373383        $command .= " -dbname $dbname" if defined $dbname;
    374384        $command .= " -dbserver $dbserver" if defined $dbserver;

  • branches/eam_branches/ipp-20100621/pstamp/scripts/pstampparse.pl

    r28794 r28980  
    386386    $need_magic    = $proj_hash->{need_magic};
    387387
    388     # Temporary hack so that MOPS can get at non-magicked data
     388    # Temporary hack so that IFA can get at non-magicked data
    389389    my $allow_mops_unmagicked = 1;
    390390    if ($allow_mops_unmagicked) {
    391391        if ($product and (($product eq "mops-pstamp-results") or
    392                           ($product eq "mops-pstamp-results2"))) {
     392                          ($product eq "mops-pstamp-results2") or
     393                          ($product eq "ifa-pstamp-results"))) {
    393394            $need_magic = 0;
    394395        }

  • branches/eam_branches/ipp-20100621/pstamp/src/ppstampMakeStamp.c

    r28310 r28980  
    229229
    230230static int makeStamp(pmConfig *config, ppstampOptions *options, pmFPAfile *input,
    231                 pmChip *inChip, pmFPAview *view, pmAstromObj *center)
     231                pmChip *inChip, pmFPAview *view, pmAstromObj *center, pmFPAfile *astrom)
    232232{
    233233    int status = false;
    … …  
    266266        srcFile = input;
    267267        *srcView = *view;
     268    }
     269
     270    if (astrom->camera != srcFile->camera) {
     271        psError(PS_ERR_UNKNOWN, true, "Input camera and astrometry camera do not match");
     272        return PS_EXIT_CONFIG_ERROR;
    268273    }
    269274
    … …  
    583588    if (!astrom) {
    584589        astrom = input;
    585     } else if (astrom->camera != input->camera) {
    586         psError(PS_ERR_UNKNOWN, true, "Input camera and astrometry camera do not match");
    587         return PS_EXIT_CONFIG_ERROR;
    588590    }
    589591
    … …  
    636638        case PPSTAMP_ON:
    637639        case PPSTAMP_PARTIALLY_ON:
    638             returnval = makeStamp(config, options, input, chip, view, center);
     640            returnval = makeStamp(config, options, input, chip, view, center, astrom);
    639641            allDone = true;
    640642            foundOverlap = true;

  • branches/eam_branches/ipp-20100621/tools/czarplot.pl

    r28794 r28980  
    2121my $histogram = undef;
    2222my $timeSeries = undef;
     23my $nebulous = undef;
    2324my $savingToFile = undef;
    2425
    … …  
    3233        "output|o=s" => \$path,
    3334        "histogram|h" => \$histogram,
     35        "nebulous|n" => \$nebulous,
    3436        "timeseries|t" => \$timeSeries,
    3537        "verbose|v" => \$verbose,
    … …  
    4143if (!$timeSeries) {
    4244    print "* OPTIONAL: plot timeseries                 -t                          (default=on)\n";}
     45if (!$nebulous) {
     46    print "* OPTIONAL: plot nebulous disk space        -n                          (default=off)\n";}
    4347if (!$label) {
    44     print "* OPTIONAL: choose a label                  -l <labellName>             (default='all_labels')\n";}
     48    print "* OPTIONAL: choose a label                  -l <labellName>             (default='all_stdscience_labels')\n";}
    4549if (!$stage) {
    4650    print "* OPTIONAL: choose a stage                  -s <chip|cam|warp|etc>      (default=none)\n";}
    … …  
    5761
    5862# default values
    59 if (!$label) {$label = "all_labels";}
    60 if (!$histogram && !$timeSeries) {$timeSeries = 1; $histogram = 1;}
     63if (!$label) {$label = "all_stdscience_labels";}
     64if (!$nebulous && !$histogram && !$timeSeries) {$timeSeries = 1; $histogram = 0;}
    6165if (!$verbose) {$verbose = 0;}
    6266if (!$save_temps) {$save_temps = 0;}
    … …  
    6771$czarDb->setDateFormat("%Y%m%d-%H%i%s");
    6872
    69 my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png" : "X11", $path, $save_temps);
     73my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" : "X11", $path, $save_temps);
    7074
    7175my @allStages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
    … …  
    7882}
    7983
    80 if ($timeSeries) {$czarplot->createTimeSeries($label, $stage, $begin, $end);}
     84my $diskUsage = 1; # TODO
     85
     86if (!$nebulous && $timeSeries) {$czarplot->createTimeSeries($label, $stage, $begin, $end);}
    8187if ($histogram) {$czarplot->createHistogram($label, $begin, $end);}
     88if ($nebulous && $timeSeries) {$czarplot->plotStorageTimeSeries($begin, $end);}
     89elsif ($nebulous) {$czarplot->plotDiskUsageHistogram();}
    8290

  • branches/eam_branches/ipp-20100621/tools/czartool.pl

    r28794 r28980  
    6464
    6565        if ($key eq "s") {checkServers();}
    66         elsif ($key eq "l") {checkAllLabels("new");}
    67         elsif ($key eq "u") {serverStatus("stdscience");}
     66        elsif ($key eq "l") {checkAllLabels("stdscience");}
     67        elsif ($key eq "u") {$pantasks->getServerCurrentStatus("stdscience");}
    6868        elsif ($key =~ m/[0-9]/) {my $key2=getc; checkOneLabel($stdscienceLabels[($key.$key2)-1]);}
    6969        printInstructions();
    … …  
    9999
    100100    return \@diff;
    101 }
    102 
    103 ###########################################################################
    104 #
    105 # Prints the status of a given pantasks server
    106 #
    107 ###########################################################################
    108 sub serverStatus {
    109     my ($server) = @_;
    110 
    111     my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
    112     my $line;
    113     foreach $line (@cmdOut) {
    114 
    115         chomp($line);
    116 
    117         print "$line\n";
    118     }
    119101}
    120102
    … …  
    263245    my $new = $gpc1Db->countExposures($label, $stage, $state);
    264246
    265     if ($state ne "new") {return $new;}
    266 
    267     my $faults = $gpc1Db->countFaults($label, $stage);
     247    #if ($state ne "new") {return $new;}
     248
     249    my $faults = $gpc1Db->countFaults($label, $stage, $state);
    268250
    269251    if ($faults < 1) {return $new;}

  • branches/eam_branches/ipp-20100621/tools/czartool/CzarDb.pm

    r28794 r28980  
    1616
    1717    # Call the constructor of the parent class, Person.
    18     my $self = $class->SUPER::new( $_[1], $_[2], $_[3], $_[4],  $_[5], $_[6]);
     18    my $self = $class->SUPER::new($_[1], $_[2], $_[3], $_[4],  $_[5], $_[6]);
    1919
    2020    bless $self, $class;
    … …  
    6666
    6767       $query->execute;
     68}
     69
     70###########################################################################
     71#
     72# Gets revert status for this stage
     73#
     74###########################################################################
     75sub getRevertStatus {
     76    my ($self, $stage, $reverting) = @_;
     77
     78    my $query = $self->{_db}->prepare(<<SQL);
     79    SELECT reverting
     80        FROM reverts
     81         WHERE stage LIKE '$stage';
     82SQL
     83
     84    $query->execute;
     85    return scalar $query->fetchrow_array();
     86}
     87
     88###########################################################################
     89#
     90# Updates revert status for this stage
     91#
     92###########################################################################
     93sub updateRevertStatus {
     94    my ($self, $stage, $reverting) = @_;
     95
     96    my $query = $self->{_db}->prepare(<<SQL);
     97     UPDATE reverts
     98         SET reverting = $reverting
     99         WHERE stage LIKE '$stage';
     100SQL
     101
     102    $query->execute;
    68103}
    69104
    … …  
    117152    }
    118153}
     154
     155###########################################################################
     156#
     157# Updates hosts table
     158#
     159###########################################################################
     160sub updateHost {
     161    my ($self, $host, $total, $available, $used, $readable, $writable) = @_;
     162
     163    my $query = $self->{_db}->prepare(<<SQL);
     164    DELETE FROM hosts WHERE host LIKE '$host';
     165SQL
     166
     167    $query->execute;
     168    $query = $self->{_db}->prepare(<<SQL);
     169    INSERT INTO hosts
     170        (host, total, available, used, readable, writable)
     171        VALUES
     172        ('$host', $total, $available, $used, $readable, $writable);
     173SQL
     174    $query->execute;
     175
     176}
     177
     178###########################################################################
     179#
     180# Inserts new cluster space info
     181#
     182###########################################################################
     183sub insertNewClusterSpace {
     184    my ($self, $total, $available, $used, $hostsOver98) = @_;
     185
     186    my $query = $self->{_db}->prepare(<<SQL);
     187    INSERT INTO cluster_space
     188        (total, available, used, hostsOver98)
     189        VALUES
     190        ( $total, $available, $used, $hostsOver98);
     191SQL
     192
     193        $query->execute;
     194}
     195
    119196###########################################################################
    120197#
    … …  
    123200###########################################################################
    124201sub insertNewTimeData {
    125     my ($self, $stage, $label, $pending, $processed, $faults, $reverting) = @_;
     202    my ($self, $stage, $label, $pending, $processed, $faults) = @_;
    126203
    127204    my $query = $self->{_db}->prepare(<<SQL);
    128205    INSERT INTO $stage
    129         (label, pending, processed, faults, reverting)
     206        (label, pending, processed, faults)
    130207        VALUES
    131         ('$label', $pending, $processed, $faults, $reverting);
     208        ('$label', $pending, $processed, $faults);
    132209SQL
    133210
    … …  
    158235SQL
    159236
    160 
    161 
    162237    if (!$query->execute) {return undef;}
    163238
    164     my $minProcessed;
     239    my $minProcessed = undef;
    165240
    166241    ($minProcessed, ${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array();
    167242
    168     if (!$minProcessed) {return undef;}
     243    if (!defined $minProcessed) {return undef;}
    169244
    170245    $query = $self->{_db}->prepare(<<SQL);
    … …  
    273348###########################################################################
    274349#
     350# Gets host data and stores it to temp file
     351#
     352###########################################################################
     353sub createHostsData {
     354    my ($self, $limit) = @_;
     355
     356    my $dataFile = "/tmp/czarplot_hosts_space.dat";
     357    open (GNUDAT, ">$dataFile");
     358
     359    my $query = $self->{_db}->prepare(<<SQL);
     360    SELECT
     361        host, used, available, writable, readable
     362        FROM hosts
     363        ORDER BY host;
     364SQL
     365
     366    $query->execute;
     367
     368    my $underLimit;
     369
     370    # loop round results
     371    while (my @row = $query->fetchrow_array()) {
     372
     373        my ($host, $used, $available, $writable, $readable) = @row;
     374   
     375        if (($used/($used+$available))*100 > $limit) {$underLimit = 0;}
     376        else {$underLimit = 1;}
     377
     378        # Col 1: host
     379        print GNUDAT "$host";
     380       
     381        # Col 2: available, readable used space under limit
     382        if ($readable && $underLimit) {print GNUDAT " $used";}
     383        else {print GNUDAT " 0";}
     384
     385        # Col 3: available, readable used space OVER limit
     386        if ($readable && !$underLimit) {print GNUDAT " $used";}
     387        else {print GNUDAT " 0";}
     388
     389        # Col 4: NOT available used space
     390        if (!$readable) {print GNUDAT " $used";}
     391        else {print GNUDAT " 0";}
     392
     393        # Col 5: available writable space
     394        if($writable) {print GNUDAT " $available";}
     395        else {print GNUDAT " 0";}
     396
     397        # Col 6: NOT available writable space
     398        if(!$writable) {print GNUDAT " $available";}
     399        else {print GNUDAT " 0";}
     400
     401        print GNUDAT "\n";
     402    }
     403    close(GNUDAT);
     404
     405    return $dataFile;
     406}
     407
     408###########################################################################
     409#
     410# Gets total used cluster space as a percentage
     411#
     412############################################################################
     413sub getTotalClusterStorageAsPercentage {
     414    my ($self) = @_;
     415
     416    my $query = $self->{_db}->prepare(<<SQL);
     417    SELECT used/total*100
     418        FROM cluster_space
     419        ORDER BY timestamp DESC LIMIT 1;
     420SQL
     421    $query->execute;
     422    return scalar $query->fetchrow_array();
     423}
     424
     425###########################################################################
     426#
     427# Gets time series data for cluster storage and saves it to file
     428#
     429###########################################################################
     430sub createStorageTimeSeriesData {
     431    my ($self, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
     432
     433    my $dataFile = "/tmp/czarplot_gnuplot_storage_timeseries.dat";
     434    open (GNUDAT, ">$dataFile");
     435
     436    my $query = $self->{_db}->prepare(<<SQL);
     437    SELECT
     438        MAX(available), MIN(available), 
     439        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'),
     440        DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
     441        TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp)))
     442            FROM cluster_space
     443            WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime';
     444SQL
     445
     446
     447    if (!$query->execute) {return undef;}
     448
     449    (${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array();
     450
     451
     452    $query = $self->{_db}->prepare(<<SQL);
     453    SELECT
     454        DATE_FORMAT(timestamp, '$self->{_dateFormat}'), available
     455        FROM cluster_space
     456        WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime'
     457        ORDER BY timestamp;
     458SQL
     459
     460    $query->execute;
     461
     462    # loop round results
     463    while (my @row = $query->fetchrow_array()) {
     464
     465        my ($timestamp, $available) = @row;
     466        print GNUDAT "$timestamp $available\n";
     467    }
     468    close(GNUDAT);
     469
     470    return $dataFile;
     471}
     472###########################################################################
     473#
    275474# Determines how much has been processed in the provided interval of time
    276475# (format: 1 HOUR, 1 MINUTE 1 DAY etc)
    … …  
    301500
    302501    my $currentRevision = -1;
    303     my $latestRevision = 7;
     502    my $latestRevision = 10;
    304503
    305504    while ($currentRevision != $latestRevision) {
    … …  
    315514        elsif ($currentRevision == 5) {$self->createRevision_6();}
    316515        elsif ($currentRevision == 6) {$self->createRevision_7();}
    317     }
     516        elsif ($currentRevision == 7) {$self->createRevision_8();}
     517        elsif ($currentRevision == 8) {$self->createRevision_9();}
     518        elsif ($currentRevision == 9) {$self->createRevision_10();}
     519    }
     520}
     521
     522#######################################################################################
     523#
     524# Sets current revision of ippToPsps database
     525#
     526#######################################################################################
     527sub setRevision {
     528    my ($self, $revision) = @_;
     529
     530    my $query = $self->{_db}->prepare(<<SQL);
     531    INSERT INTO revision (revision) VALUES ($revision);
     532SQL
     533    $query->execute;
     534}
     535
     536#######################################################################################
     537#
     538# Gets current revision of ippToPsps database
     539#
     540#######################################################################################
     541sub getRevision {
     542    my ($self) = @_;
     543
     544    if (!$self->SUPER::doesTableExist("revision")) {return 0;}
     545
     546    my $query = $self->{_db}->prepare(<<SQL);
     547    SELECT revision
     548        FROM revision
     549        ORDER BY revision DESC LIMIT 1;
     550SQL
     551
     552    $query->execute;
     553    my @row = $query->fetchrow_array();
     554
     555    return $row[0];
    318556}
    319557
    … …  
    500738#######################################################################################
    501739#
    502 # Sets current revision of ippToPsps database
    503 #
    504 #######################################################################################
    505 sub setRevision {
    506     my ($self, $revision) = @_;
    507 
    508     my $query = $self->{_db}->prepare(<<SQL);
    509     INSERT INTO revision (revision) VALUES ($revision);
    510 SQL
    511     $query->execute;
    512 }
    513 
    514 #######################################################################################
    515 #
    516 # Gets current revision of ippToPsps database
    517 #
    518 #######################################################################################
    519 sub getRevision {
    520     my ($self) = @_;
    521 
    522     if (!$self->SUPER::doesTableExist("revision")) {return 0;}
    523 
    524     my $query = $self->{_db}->prepare(<<SQL);
    525     SELECT revision
    526         FROM revision
    527         ORDER BY revision DESC LIMIT 1;
    528 SQL
    529 
    530     $query->execute;
    531     my @row = $query->fetchrow_array();
    532 
    533     return $row[0];
     740# Create revision 8 of the database
     741#
     742#######################################################################################
     743sub createRevision_8 {
     744    my ($self) = @_;
     745
     746    print "* Creating revision 8 of '$self->{_dbName}'\n";
     747
     748    # drop reverting column from all stages tables
     749    my $stage = undef;
     750    foreach $stage (@stages) {
     751
     752        my $query = $self->{_db}->prepare(<<SQL);
     753        ALTER TABLE $stage
     754            DROP COLUMN reverting;
     755SQL
     756
     757    $query->execute;
     758    }
     759
     760    # create new 'revert' table
     761    my $query = $self->{_db}->prepare(<<SQL);
     762    CREATE TABLE reverts (
     763            stage VARCHAR(128),
     764            reverting TINYINT);
     765SQL
     766
     767    $query->execute;
     768
     769    # insert stages into revert table
     770    foreach $stage (@stages) {
     771        my $query = $self->{_db}->prepare(<<SQL);
     772        INSERT INTO reverts
     773            (stage, reverting)
     774            VALUES
     775            ('$stage', 0);
     776SQL
     777       $query->execute;
     778    }
     779
     780    $self->setRevision(8);
     781}
     782
     783#######################################################################################
     784#
     785# Create revision 9 of the database
     786#
     787#######################################################################################
     788sub createRevision_9 {
     789    my ($self) = @_;
     790
     791    print "* Creating revision 9 of '$self->{_dbName}'\n";
     792
     793    my $query = $self->{_db}->prepare(<<SQL);
     794    CREATE TABLE cluster_space (
     795            timestamp TIMESTAMP DEFAULT NOW(),
     796            total FLOAT,
     797            available FLOAT,
     798            used FLOAT,
     799            hostsOver98 SMALLINT);
     800SQL
     801
     802      $query->execute;
     803    $query = $self->{_db}->prepare(<<SQL);
     804    CREATE TABLE hosts (
     805            host VARCHAR(128),
     806            total FLOAT,
     807            available FLOAT,
     808            used FLOAT);
     809SQL
     810
     811      $query->execute;
     812
     813        $query = $self->{_db}->prepare(<<SQL);
     814        CREATE INDEX clusterSpaceIndex ON cluster_space (timestamp);
     815SQL
     816
     817      $query->execute;
     818
     819    $self->setRevision(9);
     820}
     821
     822#######################################################################################
     823#
     824# Create revision 10 of the database
     825#
     826#######################################################################################
     827sub createRevision_10 {
     828    my ($self) = @_;
     829
     830    print "* Creating revision 10 of '$self->{_dbName}'\n";
     831
     832    my $query = $self->{_db}->prepare(<<SQL);
     833    ALTER TABLE hosts
     834        ADD COLUMN writable TINYINT NOT NULL DEFAULT 0,
     835        ADD COLUMN readable TINYINT NOT NULL DEFAULT 0;
     836SQL
     837
     838    $query->execute;
     839
     840    $self->setRevision(10);
    534841}
    535842

  • branches/eam_branches/ipp-20100621/tools/czartool/Czarplot.pm

    r28794 r28980  
    154154    my ($self, $minY, $maxY) = @_;
    155155
    156     ${$maxY} = ${$maxY} + (${$maxY}/10);
    157     ${$minY} = ${$minY} - (${$minY}/10);
     156    my $tmp;
     157    if (${$maxY} < ${$minY}) {
     158
     159        $tmp = ${$maxY};
     160        ${$maxY} = ${$minY};
     161        ${$minY} = $tmp;
     162    }
     163
     164    my $border = (${$maxY} - ${$minY})/10;
     165
     166    ${$maxY} += $border;
     167    ${$minY} -= $border ;
    158168    if (${$maxY} == 0) {${$maxY} = 1;}
    159169    if (${$minY} == 0) {${$minY} = -1;}
    160170}
    161171
     172
     173###########################################################################
     174#
     175# Figures out suitable spacing for time tics on time-series pliots
     176#
     177###########################################################################
     178sub getTimeSpacing {
     179    my ($self, $timeDiff, $divX, $timeFormat) = @_;
     180
     181    # if less than a couple of hour's data plotted, show 30 mins tics
     182    if ($timeDiff < 7200) {
     183        ${$timeFormat} = "%H:%M";
     184        ${$divX} = 1800;
     185    }
     186    # if less than half a day's data plotted, show hourly tics
     187    elsif ($timeDiff < 43200) {
     188        ${$timeFormat} = "%H:%M";
     189        ${$divX} = 3600;
     190    }
     191    # if less than one day's data plotted, show 2 hourly tics
     192    elsif ($timeDiff < 86400) {
     193        ${$timeFormat} = "%H:%M";
     194        ${$divX} = 7200;
     195    }
     196    # if less than 1 week's data is data plotted, show daily tics
     197    elsif ($timeDiff < 604800) {
     198
     199        ${$timeFormat} = "%m/%d";
     200        ${$divX} = 86400;
     201    }
     202    else {
     203
     204        ${$timeFormat} = "%m/%d";
     205        ${$divX} = 172800;
     206    }
     207}
     208
    162209###########################################################################
    163210#
    … …  
    171218
    172219    my $timeFormat = undef;
    173 
    174     my $divX = $timeDiff/4;
    175 
    176     # if less than a couple of hour's data plotted, show 30 mins tics
    177     if ($timeDiff < 7200) {
    178         $timeFormat = "%H:%M";
    179         $divX = 1800;
    180     }
    181     # if less than half a day's data plotted, show hourly tics
    182     elsif ($timeDiff < 43200) {
    183         $timeFormat = "%H:%M";
    184         $divX = 3600;
    185     }
    186     # if less than one day's data plotted, show 2 hourly tics
    187     elsif ($timeDiff < 86400) {
    188         $timeFormat = "%H:%M";
    189         $divX = 7200;
    190     }
    191     # if more than one day's data plotted, show daily tics
    192     else {
    193        
    194         $timeFormat = "%m/%d %H:%M";
    195         $divX = 86400;
    196     }
     220    my $divX = undef;
     221
     222    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
    197223
    198224    my $numOfPlots = keys %$gnuplotFiles;
     225    my $title = undef;
     226
     227    # sort out plot title
     228    if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title = "'".$stage."'";}}
     229    else {$title = "'All stages'"}
     230
     231    $title .= " for '$label', '$fromTime' to '$toTime'";
    199232
    200233    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
    … …  
    205238        "set term $self->{_outputFormat};" .
    206239        "set output \"$outputFile\";" .
    207         "set title \"All stages for '$label' since $fromTime\";" .
     240        "set title \"$title\";" .
    208241        "set key left top;" .
    209242        "set xdata time;" .
    210243        "set timefmt \"$self->{_dateFormat}\";" .
     244    #    "set yrange [\"$minY\":\"$maxY\"];" .
    211245        "set xrange [\"$minX\":\"$maxX\"];" .
    212246        "set format x \"$timeFormat\";" .
    … …  
    224258        if ($numOfPlots == 1) {
    225259
    226             print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"processed\" with lines lt 2,";
    227             print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"pending\" with lines lt 3,";
    228             print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"faults\" with lines lt 1";
     260            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"Processed\" with lines lt 2,";
     261            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4,";
     262            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"Faults\" with lines lt 7";
    229263        }
    230264        # when plotting multiple stages, show only processed
    … …  
    241275###########################################################################
    242276#
    243 # Plots a histogram of processed stuff
    244 #
    245 ###########################################################################
    246 sub plotHistogram {
    247     my ($self, $inputFile, $outputFile, $label, $fromTime, $toTime) = @_;
     277# Plots a time-series of cluster storage
     278#
     279###########################################################################
     280sub plotStorageTimeSeries {
     281    my ($self, $fromTime, $toTime) = @_;
     282
     283    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : ".";
     284    my $outputFile = "$prefix/czarplot_cluster.png";
     285    my ($minX, $maxX, $minY, $maxY, $timeDiff);
     286
     287    my $gnuplotFile = $self->{_czarDb}->createStorageTimeSeriesData($fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff);
     288
     289    $self->sortOutMaxMinY(\$minY, \$maxY);
     290
     291    my $timeFormat = undef;
     292    my $divX = undef;
     293
     294    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);
    248295
    249296    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
    … …  
    254301        "set term $self->{_outputFormat};" .
    255302        "set output \"$outputFile\";" .
    256         "set title \"Processing status for '$label' since $fromTime\";" .
     303        "set title \"Total available cluster space over time\";" .
     304        "set key left top;" .
     305        "set xdata time;" .
     306        "set timefmt \"$self->{_dateFormat}\";" .
     307        "set xrange [\"$minX\":\"$maxX\"];" .
     308        "set yrange [\"$minY\":\"$maxY\"];" .
     309        "set format x \"$timeFormat\";" .
     310        "set xtics \"$minX\", $divX, \"$maxX\";" .
     311        "set grid xtics;" .
     312        "set xlabel \"Time\";" .
     313        "set ylabel \"Available (TB)\";" .
     314        "plot " .
     315#        "'$gnuplotFile' using 1:2 title \"Total\" with lines lt 1," .
     316        "'$gnuplotFile' using 1:2 title \"Available\" with lines lt 2\n";
     317
     318    print GP "\n";
     319    close GP;
     320}
     321
     322###########################################################################
     323#
     324# Plots a histogram of processed stuff
     325#
     326###########################################################################
     327sub plotHistogram {
     328    my ($self, $inputFile, $outputFile, $label, $fromTime, $toTime) = @_;
     329
     330    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
     331    use FileHandle;
     332    GP->autoflush(1);
     333
     334    print GP
     335        "set term $self->{_outputFormat};" .
     336        "set output \"$outputFile\";" .
     337        "set title \"'$label', '$fromTime' to '$toTime'\";" .
    257338        "set grid;" .
    258339        "set boxwidth;" .
    … …  
    262343        "set bmargin 5;" .
    263344        "set ylabel \"Exposures\";" .
    264         "plot '$inputFile' using 2:xtic(1) title \"Processed\" lt 2, '' using 3 title \"Pending\" lt 3, '' using 4 title \"Faults\" lt 1\n";
     345        "plot '$inputFile' using 2:xtic(1) title \"Processed\" lt 2, '' using 3 title \"Pending\" lt 4, '' using 4 title \"Faults\" lt 7;" .
     346        "\n";
    265347
    266348    close GP;
    267349}
    268350
     351###########################################################################
     352#
     353# Plots disk usage across cluster as a stacked histogram
     354#
     355###########################################################################
     356sub plotDiskUsageHistogram {
     357    my ($self) = @_;
     358
     359    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
     360        my $outputFile = "$prefix/czarplot_hosts_space.png";
     361    my $limit = 97.0;
     362    my $inputFile = $self->{_czarDb}->createHostsData($limit);
     363
     364    my $totalUsed = $self->{_czarDb}->getTotalClusterStorageAsPercentage();
     365
     366    my $totalPercent = sprintf("%.1f%%", $totalUsed);
     367
     368    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
     369    use FileHandle;
     370    GP->autoflush(1);
     371
     372    print GP
     373        "set term $self->{_outputFormat};" .
     374        "set output \"$outputFile\";" .
     375        "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" .
     376        "set style fill solid 1.00 border -1;" .
     377        "set style histogram rowstacked;" .
     378        "set style data histograms;" .
     379        "set ylabel \"Space (TB)\";" .
     380        "set xtic rotate by -90 scale 0;" .
     381        "plot '$inputFile' " .
     382        "using 2:xtic(1) t \"Used\" lt 7," .
     383        "'' using 3 t \"Over $limit% used\" lt 1," .
     384        "'' using 4 t \"Unavailable\" fs solid 0.50 lt -1 ," .
     385        "'' using 5 t \"Free\" lt 2," .
     386        "'' using 6 notitle fs solid 0.50 lt -1" .
     387        ";" .
     388
     389        "\n";
     390
     391    close GP;
     392}
    2693931;

  • branches/eam_branches/ipp-20100621/tools/czartool/Gpc1Db.pm

    r28794 r28980  
    88use czartool::MySQLDb;
    99our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
     10
     11###########################################################################
     12#
     13# Returns the right join table and ID to join on for for a given stage TODO make private
     14#
     15###########################################################################
     16sub getJoinTableAndId {
     17    my ($self, $stage, $joinTable, $id) = @_;
     18
     19    ${$id} = $stage."_id";
     20
     21    if ($stage eq "chip") {${$joinTable}="chipProcessedImfile";}
     22    elsif ($stage eq "cam") {${$joinTable}="camProcessedExp";}
     23    elsif ($stage eq "fake") {${$joinTable}="fakeProcessedImfile";}
     24    elsif ($stage eq "warp") {${$joinTable}="warpSkyfile";}
     25    elsif ($stage eq "stack") {${$joinTable}="stackSumSkyfile";}
     26    elsif ($stage eq "diff") {${$joinTable}="diffSkyfile";}
     27    elsif ($stage eq "magic") {${$joinTable}="magicNodeResult";}
     28    elsif ($stage eq "magicDS") {${$id} = "magic_ds_id"; ${$joinTable}="magicDSFile";}
     29    elsif ($stage eq "dist") {${$joinTable}="distComponent";}
     30    else {
     31
     32        print "* ERROR: could nod find joinTable and ID for '$stage' stage\n";
     33        return 0;
     34    }
     35
     36    return 1;
     37}
    1038
    1139###########################################################################
    … …  
    3058}
    3159
    32 
    3360###########################################################################
    3461#
    … …  
    3764###########################################################################
    3865sub countFaults {
    39     my ($self, $label, $stage) = @_;
     66    my ($self, $label, $stage, $state) = @_;
    4067
    41 
    42     my $table = getTableForStage($self, $stage);
    43     my $joinTable;
    44     my $id = $stage."_id";
    45 
    46     if ($stage eq "chip") {$joinTable="chipProcessedImfile";}
    47     elsif ($stage eq "cam") {$joinTable="camProcessedExp";}
    48     elsif ($stage eq "fake") {$joinTable="fakeProcessedImfile";}
    49     elsif ($stage eq "warp") {$joinTable="warpSkyfile";}
    50     elsif ($stage eq "stack") {$joinTable="stackSumSkyfile";}
    51     elsif ($stage eq "diff") {$joinTable="diffSkyfile";}
    52     elsif ($stage eq "magic") {$joinTable="magicNodeResult";}
    53     elsif ($stage eq "magicDS") {$id = "magic_ds_id"; $joinTable="magicDSFile";}
    54     elsif ($stage eq "dist") {$joinTable="distComponent";}
    55     else {return -1;}
     68    my $table = undef;
     69    my $joinTable = undef;
     70    my $id = undef;
     71    $table = getTableForStage($self, $stage);
     72    if (!getJoinTableAndId($self, $stage, \$joinTable, \$id)) {return -1;}
    5673
    5774    my $faultCol =  $joinTable.".fault";
    … …  
    6481        WHERE label LIKE '$label'
    6582        AND $faultCol != 0
    66         AND $stateCol = 'new'
     83        AND $stateCol = '$state'
    6784SQL
    6885
    … …  
    85102SQL
    86103
    87         $query->execute;
     104    $query->execute;
    88105    my $priority = scalar $query->fetchrow_array();
    89106    if (!$priority) {return 50000;} # assume labels not given priority in gpc1 Db have highest priority
    … …  
    100117
    101118    my $table = getTableForStage($self, $stage);
    102     if ($state eq "fault") {return countFaults($stage);}
     119    my $query = undef;
     120
     121    if ($state eq "update") {
     122
     123        if ($stage eq "dist") {return 0;}
     124        my $joinTable = undef;
     125        my $id = undef;
     126        if (!getJoinTableAndId($self, $stage, \$joinTable, \$id)) {return -1;}
     127
     128        if ($stage eq "chip" || $stage eq "fake" || $stage eq "warp" || $stage eq "diff" || $stage eq "magicDS") {
     129            $query = $self->{_db}->prepare(<<SQL);
     130            SELECT COUNT(state)
     131                FROM $table JOIN $joinTable
     132                USING($id)
     133                WHERE label LIKE '$label'
     134                AND state = 'update'
     135                AND data_state = 'update';
     136SQL
     137        }
     138        else {
     139       
     140            $query = $self->{_db}->prepare(<<SQL);
     141            SELECT COUNT(state)
     142                FROM $table JOIN $joinTable
     143                USING($id)
     144                WHERE label LIKE '$label'
     145                AND state = 'update';
     146SQL
     147        }
     148
     149    }
     150    else {
     151
     152        $query = $self->{_db}->prepare(<<SQL);
     153        SELECT COUNT(state) 
     154            FROM $table
     155            WHERE label LIKE '$label'
     156            AND state = '$state'
     157SQL
     158    }
    103159
    104160
    105     my $query = $self->{_db}->prepare(<<SQL);
    106     SELECT count(state) 
    107         FROM $table
    108         WHERE label LIKE '$label'
    109         AND state = '$state'
    110 SQL
    111 
    112         $query->execute;
     161    $query->execute;
    113162    return scalar $query->fetchrow_array();
    114163}

  • branches/eam_branches/ipp-20100621/tools/czartool/Pantasks.pm

    r28794 r28980  
    55
    66package czartool::Pantasks;
    7 
    87
    98my @servers = (
    … …  
    2120        );
    2221
     22###########################################################################
     23#
     24# Constructor TODO put in check that we can run pantasks_client
     25#
     26###########################################################################
     27sub new {
     28    my $class = shift;
     29    my $self = {};
     30
     31    bless $self, $class;
     32    return $self;
     33}
    2334
    2435###########################################################################
    … …  
    3546###########################################################################
    3647#
    37 # Constructor
    38 #
    39 ###########################################################################
    40 sub new {
    41     my $class = shift;
    42     my $self = {};
    43 
    44     bless $self, $class;
    45     return $self;
    46 }
    47 
    48 ###########################################################################
    49 #
    5048# Returns the correct server for this processing stage
    5149#
    … …  
    5856}
    5957
     58###########################################################################
     59#
     60# Checks for a meaningful response from pantasks client
     61#
     62###########################################################################
     63sub outputOk {
     64    my ($self, $output) = @_;
     65
     66    if (
     67            $output =~ m/.*show\.labels.*/i ||
     68            $output =~ m/.*Connection reset by peer.*/i ||
     69            $output =~ m/.*server is busy.*/i ) {
     70
     71        print "* WARNING: pantasks returned '$output'\n";
     72        return 0;
     73    }
     74    return 1;
     75}
    6076
    6177###########################################################################
    … …  
    7591
    7692        chomp($line);
    77         if ($line =~ m/pantasks:\s+(.*)/) {
    78 
    79             # HACK quit if we get 'show.labels: Command not found.'
    80             if ($1 =~ m/.*show\.labels.*/i) {return \@labels;};
    81             # HACK quit if we get 'Connection reset by peer'
    82             if ($1 =~ m/.*Connection reset by peer.*/i) {return \@labels;};
    83             # HACK quit if we get 'server is busy' message
    84             if ($1 =~ m/.*server is busy.*/i) {return \@labels;};
    85             # HACK to get around 'dummy' label in stdscience
    86             if ($1 !~ m/.*dummy.*/) {push(@labels, $1);};
    87             $passedHeader=1;
    88             next;
    89         }
    90 
     93        if (!$self->outputOk($line)) {return \@labels;}
     94        if ($line =~ m/pantasks:\s+/) {$passedHeader=1; next;}
    9195        if ($passedHeader) {push(@labels, $line);}
    9296    }
    … …  
    113117        if ($line =~ m/Scheduler is running/) {${$running} = 1; ${$alive}=1;last;}
    114118        if ($line =~ m/Task Status/) {last;}
    115 
    116119    }
    117120}
    … …  
    125128    my ($self, $server) = @_;
    126129
     130    my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
     131    my $line;
     132    my $passedHeader=0;
     133    print "\n";
     134    foreach $line (@cmdOut) {
     135
     136        if ($line =~ m/.*Task Status.*/) {$passedHeader=1;next;}
     137        if ($passedHeader){print "$line";}
     138    }
     139}
     140
     141###########################################################################
     142#
     143# Gets revert status for this stage
     144#
     145###########################################################################
     146sub getRevertStatus {
     147    my ($self, $stage, $reverting) = @_;
     148
     149    my $server = $self->getServerForThisStage($stage);
     150
     151    if ($stage eq "cam") {$stage = "camera";}
    127152
    128153    my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
    129154    my $line;
    130     my $passedHeader=0;
    131155    foreach $line (@cmdOut) {
    132156
    133157        chomp($line);
    134         if ($line =~ m/.*Task Status.*/) {$passedHeader=1;next;}
    135         if ($passedHeader){print "$line<br>";}
     158        if ($line =~ m/\s+([+-])[+-]\s+$stage\.revert.*/) {
     159
     160            if ($1 =~ m/-/) {${$reverting} = 0;}
     161            elsif ($1 =~ m/\+/) {${$reverting}=1;}
     162        }
    136163    }
    137164}
    138 
    139 
     1651;

  • branches/eam_branches/ipp-20100621/tools/examine_burntool_pcontrol.pl

    r27070 r28980  
    3333while(<LAZY>) {
    3434    chomp;
    35     if ($_ =~ /BURNTOOL.STATE.GOOD/) {
     35    if ($_ =~ /BURNTOOL.STATE.GOOD\s/) {
    3636        @line = split /\s+/;
    3737        $burntoolStateGood = $line[2];

  • branches/eam_branches/ipp-20100621/tools/psstatus

    r28355 r28980  
    1010use File::Temp qw(tempfile);
    1111
    12 my $limit = 20;
     12my $limit = 0;
    1313my $running;
     14my $finished;
    1415my $req_faulted;
    1516my $job_faulted;
    … …  
    2324GetOptions(
    2425    'running|r',     \$running,
     26    'finished|f',    \$finished,
    2527    'req_faulted',   \$req_faulted,
    2628    'job_faulted',   \$job_faulted,
    … …  
    3234pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    3335
    34 my $no_args = ! (defined $running or defined $req_faulted or defined $job_faulted );
     36my $no_args = ! (defined $running or defined $finished or defined $req_faulted or defined $job_faulted );
     37
     38die "cannot supply --running and --finished\n" if ($running and $finished);
     39   
    3540
    3641my $ipprc =  PS::IPP::Config->new();
    3742my $dbh = getDBHandle();
    3843
    39 my $select = "SELECT req_id, name, label, reqType, pstampRequest.state, pstampRequest.fault, count(job_id) AS numJobs, timestamp";
     44my $jobLabel = $running ? "Running jobs" : "Num Jobs";
     45my $select = "SELECT req_id, name, label, reqType, pstampRequest.state, pstampRequest.fault, count(job_id) AS '$jobLabel', timestamp";
    4046
    4147$select .= ", outdir" if ($verbose);
    … …  
    4551my $sql = $select . $from . $where;
    4652
    47 if ($no_args and $limit) {
     53$limit = 20 if ($no_args);
     54# if ($no_args and $limit) {
     55if (0 and $limit) {
    4856    # use mysql to get the last_req_id
    4957    # this works but it prints out the @last_req_id value which is distracting. Use dbi for the query
    50     my $stmt = $dbh->prepare("SELECT req_id FROM pstampRequest ORDER BY req_id DESC LIMIT 1");
     58    my $where = "";
     59    if ($running) {
     60        $where = " WHERE pstampRequest.state = 'run' OR pstampRequest.state = 'new'";
     61    } elsif ($finished) {
     62        $where = " WHERE pstampRequest.state = 'stop'";
     63    }
     64    my $stmt = $dbh->prepare("SELECT req_id FROM pstampRequest $where ORDER BY req_id DESC LIMIT 1");
    5165    $stmt->execute();
    5266    my $row = $stmt->fetchrow_hashref();
    … …  
    5973}
    6074if ($running) {
    61     $sql .= " AND (pstampRequest.state = 'new' OR pstampRequest.state = 'run')"
     75    $sql .= " AND (pstampRequest.state = 'new' OR pstampRequest.state = 'run') AND (pstampJob.state = 'run' OR pstampJob.state IS NULL)";
     76} elsif ($finished) {
     77    $sql .= " AND (pstampRequest.state = 'stop') AND date_add(timestamp, interval 1 day) >=utc_timestamp()"
    6278}
    6379if ($req_faulted) {
    … …  
    6985$sql .= " GROUP BY req_id";
    7086
     87if ($finished) {
     88    $sql .= " ORDER BY timestamp DESC";
     89}
    7190if ($limit and !$no_args) {
    7291    $sql .= " LIMIT $limit";

  • branches/eam_branches/ipp-20100621/tools/roboczar.pl

    r28794 r28980  
    1010use czartool::Gpc1Db;
    1111use czartool::Pantasks;
     12use czartool::Nebulous;
    1213use czartool::Czarplot;
    13 
    1414
    1515my $period = 60;
    … …  
    2424my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg
    2525my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
     26my $nebulous = new czartool::Nebulous($czarDb);
    2627my $pantasks = new czartool::Pantasks();
    27 my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", "png", "/tmp", $save_temps);
     28my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8", "/tmp", $save_temps); # TODO hardcoded font path
    2829$czarDb->setDateFormat("%Y%m%d-%H%i%s");
    2930
    … …  
    4142
    4243    print "* Updating labels\n";
    43     my @servers = ("stdscience", "distribution", "publishing");
     44    my @servers = ("stdscience", "distribution", "publishing", "update");
    4445
    4546    my $server = undef;
    … …  
    9192    my $faults;
    9293    my $stage;
    93     my ($totalNew,$totalFull,$totalFaults);
    9494    my $query = undef;
    9595    my $str = undef;
    96     my $reverting = 0;
    97     my $server = undef;
    98     my $rows = undef;
     96    my $labels = undef;
     97    my $updateLabels = undef;
    9998    my $row = undef;
    10099    my $begin = undef;
    101100    my $end = undef;
    102101    my $priority = undef;
     102    my $newState = undef;
    103103
    104104    while (1) {
    105 
    106         updateServerStatus();
    107         updateLabels();
    108         if (!$czarDb->getCurrentLabels("stdscience", \$rows)) {next;}
    109 
    110         my $size = @{$rows};
    111         if($size < 1) {
    112            
    113             print "* WARNING: no stdscience labels found in Db\n";
    114             next;
    115         }
    116 
    117         # get priority
    118         foreach $row ( @{$rows} ) {
    119             my ($label) = @{$row};
    120             $priority = $gpc1Db->getPriority($label);
    121             $czarDb->setLabelPriority($label, $priority);
    122         }
    123105
    124106        # sort out times
    … …  
    131113        }
    132114
    133         foreach $stage (@stages) {
    134 
    135             $server = $pantasks->getServerForThisStage($stage);
    136             $str = `czartool_revert.pl -s $server -t $stage`;
    137             if ($str =~ m/on/) {$reverting = 1;}
    138             else {$reverting = 0;}
    139             print "* Checking labels for $stage stage\n";
    140 
    141             $totalNew=$totalFaults=$totalFull=0;
    142             foreach $row ( @{$rows} ) {
    143                 my ($label) = @{$row};
    144 
    145                 chomp($label);
    146 
    147                 $new = $gpc1Db->countExposures($label, $stage, "new");
    148                 $full = $gpc1Db->countExposures($label, $stage, "full");
    149                 $faults = $gpc1Db->countFaults($label, $stage);
    150 
    151                 #printf("%ld, %s, %s, %d, %d\n", $currentTime, $label, $stage, $new, $faults);
    152                 $totalNew += $new;
    153                 $totalFull += $full;
    154                 $totalFaults += $faults;
    155 
    156                 $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults, $reverting);
    157                 $czarplot->createTimeSeries($label,  $stage, $begin, $end);
     115        # check nebulous
     116        print "* Checking Nebulous\n";
     117        $nebulous->updateClusterSpaceInfo();
     118        $czarplot->plotDiskUsageHistogram();
     119        updateServerStatus();
     120
     121        # check labels
     122        updateLabels();
     123
     124        # servers to check
     125        my @serversToCheck = ("stdscience", "update");
     126
     127        my $thisServer = undef;
     128        foreach $thisServer (@serversToCheck) {
     129
     130            if ($thisServer eq "update") {$newState = "update";}
     131            else {$newState = "new";}
     132
     133            # deal with stdscience labels
     134            if (!$czarDb->getCurrentLabels($thisServer, \$labels)) {next;}
     135            my $size = @{$labels};
     136            if($size > 0) {
     137
     138                # get priority
     139                foreach $row ( @{$labels} ) {
     140                    my ($label) = @{$row};
     141                    $priority = $gpc1Db->getPriority($label);
     142                    $czarDb->setLabelPriority($label, $priority);
     143                }
     144
     145                updateAllStages($thisServer, $newState, $labels, $begin, $end);
     146                createPlots($thisServer, $labels, $begin, $end);
    158147            }
    159 
    160             $czarDb->insertNewTimeData($stage, "all_labels", $totalNew, $totalFull, $totalFaults, $reverting);
    161         }
    162 
    163         print "* Generating plots\n";
    164         foreach $row ( @{$rows} ) {
    165             my ($label) = @{$row};
    166 
    167             $czarplot->createTimeSeries($label, undef, $begin, $end);
    168             $czarplot->createHistogram($label, $begin, $end);
    169 
    170             #routineChecks($label, "1 HOUR");
    171         }
    172         $czarplot->createTimeSeries("all_labels", undef, $begin, $end);
    173         $czarplot->createHistogram("all_labels", $begin, $end);
    174         foreach $stage (@stages) {
    175 
    176             $czarplot->createTimeSeries("all_labels",  $stage, $begin, $end); # TODO must be a neater way...
    177         }
     148            else { print "* WARNING: no $thisServer labels found in Db\n";}
     149        }
     150
    178151        print "--------------------------------------------------------------------------\n";
    179152        print "* Going to sleep\n";
    … …  
    183156        #sendEmail("roydhenderson\@gmail.com", "roboczar\@ipp.com", "Roboczar update", "Some content");
    184157    };
     158}
     159
     160###########################################################################
     161#
     162# Loops through labels and creates time series and histogram plots
     163#
     164###########################################################################
     165sub createPlots {
     166    my ($server, $rows, $begin, $end) = @_;
     167
     168    my $stage = undef;
     169    my $row = undef;
     170
     171    print "* Generating plots\n";
     172
     173    # create plots for each label for each stage
     174    foreach $stage (@stages) {
     175        foreach $row ( @{$rows} ) {
     176            my ($label) = @{$row};
     177
     178            chomp($label);
     179            $czarplot->createTimeSeries($label,  $stage, $begin, $end);
     180        }
     181    }
     182
     183    # create plots for each label for all stages
     184    foreach $row ( @{$rows} ) {
     185        my ($label) = @{$row};
     186
     187        $czarplot->createTimeSeries($label, undef, $begin, $end);
     188        $czarplot->createHistogram($label, $begin, $end);
     189
     190        #routineChecks($label, "1 HOUR");
     191    }
     192    $czarplot->createTimeSeries("all_".$server."_labels", undef, $begin, $end);
     193    $czarplot->createHistogram("all_".$server."_labels", $begin, $end);
     194    foreach $stage (@stages) {
     195
     196        $czarplot->createTimeSeries("all_".$server."_labels",  $stage, $begin, $end); # TODO must be a neater way...
     197    }
     198}
     199
     200###########################################################################
     201#
     202# Loops through some labels and updates processed/pending/faults in the Db
     203#
     204###########################################################################
     205sub updateAllStages {
     206    my ($labelServer, $newState, $rows, $begin, $end) = @_;
     207
     208    print "* Updating stage data\n";
     209    my $totalNew = undef;
     210    my $totalFaults = undef;
     211    my $totalFull = undef;
     212    my $stage = undef;
     213    my $reverting = 0;
     214    my $row = undef;
     215    my $new = undef;
     216    my $full = undef;
     217    my $faults = undef;
     218    my $server = undef;
     219    my $state = undef;
     220   
     221    foreach $stage (@stages) {
     222
     223        $server = $pantasks->getServerForThisStage($stage);
     224        $pantasks->getRevertStatus($stage, \$reverting);
     225        $czarDb->updateRevertStatus($stage, $reverting);
     226
     227        print "* Checking labels for $stage stage\n";
     228
     229        $totalNew=$totalFaults=$totalFull=0;
     230        foreach $row ( @{$rows} ) {
     231            my ($label) = @{$row};
     232
     233            chomp($label);
     234
     235            $new = $gpc1Db->countExposures($label, $stage, $newState);
     236            $full = $gpc1Db->countExposures($label, $stage, "full");
     237            $faults = $gpc1Db->countFaults($label, $stage, $newState);
     238
     239            #printf("%s  %s, %s, %d, %d\n", $labelServer, $label, $stage, $new, $faults);
     240            $totalNew += $new;
     241            $totalFull += $full;
     242            $totalFaults += $faults;
     243
     244            $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults);
     245        }
     246
     247        $czarDb->insertNewTimeData($stage, "all_".$labelServer."_labels", $totalNew, $totalFull, $totalFaults);
     248    }
    185249}
    186250
Note: See TracChangeset for help on using the changeset viewer.