IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 30587


Ignore:
Timestamp:
Feb 11, 2011, 4:42:29 PM (15 years ago)
Author:
watersc1
Message:

Caught up with the trunk. Now to look for bugs.

Location:
branches/czw_branch/20101203
Files:
51 edited
32 copied

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/20101203

  • branches/czw_branch/20101203/Ohana/src/getstar/src/MatchCoords.c

    r27940 r30587  
    5353        }
    5454
    55 #ifdef notdef
    56         printf("%ld %8.1f %8.1f %s\n", i, x, y, dbImages[i].name);
    57 #endif
     55       
     56        if (VERBOSE) fprintf(stderr, "%ld %8.1f %8.1f %s\n", i, x, y, dbImages[i].name);
    5857
    5958        if ((x >= xmin && x <= xmax) && (y >= ymin && y <= ymax)) {

  • branches/czw_branch/20101203/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

    r30586 r30587  
    3434                    $PSTAMP_NO_WAIT_FOR_UPDATE
    3535                    $PSTAMP_SUCCESS
     36                    $PSTAMP_FIRST_ERROR_CODE
    3637                    $PSTAMP_SYSTEM_ERROR
    3738                    $PSTAMP_NOT_IMPLEMENTED
    … …  
    7475# NOTE: these must match the values in pstamp/src/pstamp.h
    7576our $PSTAMP_SUCCESS          = 0;
     77our $PSTAMP_FIRST_ERROR_CODE = 10;
    7678our $PSTAMP_SYSTEM_ERROR     = 10;
    7779our $PSTAMP_NOT_IMPLEMENTED  = 11;

  • branches/czw_branch/20101203/ippMonitor/raw/czartool_exposures.php

    r30586 r30587  
    1717else {$myMenu = "ipp.imfiles.dat";}
    1818
    19 menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']);
    20 
    2119$pass = $ID['pass'];
    2220$proj = $ID['proj'];
    2321$menu = $ID['menu'];
    2422
    25 $today = date("Y-m-d");
    26 
     23$date = $_POST['date'];
     24$survey = $_POST['survey'];
     25
     26// if no date chosen, use today
     27if ($date == "") $date = date("Y-m-d");
     28if ($survey == "") $survey = "";
     29
     30menu($myMenu, 'Exposure summary for '.$date , 'ipp.css', $ID['link'], $ID['proj']);
     31
     32// set up the form
     33echo "<form action=\"czartool_exposures.php\" method=\"POST\">\n";
     34
     35echo "<input type=\"hidden\" name=\"pass\" value=\"$pass\">\n";
     36echo "<input type=\"hidden\" name=\"proj\" value=\"$proj\">\n";
     37echo "<input type=\"hidden\" name=\"menu\" value=\"$menu\">\n";
     38
     39
     40echo "<form method=post action=czartool_exposures.php?pass=$pass&proj=$proj&date=$date&survey=$survey>";
     41echo "<p  align=\"center\"> Current status of all exposures taken on (UTC) <input type=text name=date value='$date'>";
     42echo " for survey <input type=text name=survey value='$survey'> (leave blank for all surveys)";
     43echo "<input type=submit value=Submit>";
    2744echo "</p>";
    28 echo "<p  align=\"center\"> Current status of all exposures taken $today UTC </p>";
    29 
    30 createTableForThisSurvey($gpc1db, $today, "");
     45
     46echo "</form>\n";
     47
     48createTableForThisSurvey($gpc1db, $date, $survey);
    3149
    3250menu_end();
    … …  
    5876    $qry->fetchInto($count);
    5977
    60     if ($surveyStr == "") $surveyText = "all surveys"; 
     78    if ($surveyStr == "") $surveyText = "all surveys";
    6179    else $surveyText = "$surveyStr survey";
    6280
    … …  
    83101    $ssDiffProduced = array(); getSSDiffProduced($db, $date, $surveyStr, &$ssDiffProduced);
    84102
    85     $class = "list";
    86     echo "<table class=$class >\n";
     103    // create table
     104    echo "<table border=\"1\" cellspacing=\"0\" cellpadding=\"4\" style=\"font-size:80%\">\n";
     105
     106    // table headers
    87107    echo "<tr><td></td>\n";
    88     write_header_cell($class, "");
    89     write_header_cell($class, "");
    90     write_header_cell($class, "");
    91     write_header_cell($class, "");
    92     write_header_cell($class, "Raw");
    93     write_header_cell($class, "Chip");
    94     write_header_cell($class, "Camera");
    95     write_header_cell($class, "Warp");
    96     write_header_cell($class, "Stack");
    97     write_header_cell($class, "Warp-Stack Diff");
    98     write_header_cell($class, "Warp-Warp1 Diff");
    99     write_header_cell($class, "Warp-Warp2 Diff");
    100     write_header_cell($class, "Stack-Stack Diff");
     108    echo "<th></th>";
     109    echo "<th></th>";
     110    echo "<th></th>";
     111    echo "<th></th>";
     112    echo "<th>Raw</th>";
     113    echo "<th>Chip</th>";
     114    echo "<th>Camera</th>";
     115    echo "<th>Warp</th>";
     116    echo "<th>Stack</th>";
     117    echo "<th>Warp-Stack Diff</th>";
     118    echo "<th>Warp-Warp1 Diff</th>";
     119    echo "<th>Warp-Warp2 Diff</th>";
     120    echo "<th>Stack-Stack Diff</th>";
    101121    echo "<tr><td></td>\n";
    102     write_header_cell($class, "Exp ID");
    103     write_header_cell($class, "Exp Name");
    104     write_header_cell($class, "Filter");
    105     write_header_cell($class, "Comment");
    106     write_header_cell($class, "state");
    107     write_header_cell($class, "state/dist?");
    108     write_header_cell($class, "state/dist?");
    109     write_header_cell($class, "state/dist?");
    110     write_header_cell($class, "total/dist");
    111     write_header_cell($class, "total/dist");
    112     write_header_cell($class, "total/dist");
    113     write_header_cell($class, "total/dist");
    114     write_header_cell($class, "total/dist");
     122    echo "<th>Exp ID</th>";
     123    echo "<th>Exp Name</th>";
     124    echo "<th>Filter</th>";
     125    echo "<th>Comment</th>";
     126    echo "<th>state</th>";
     127    echo "<th>state/dist?</th>";
     128    echo "<th>state/dist?</th>";
     129    echo "<th>state/dist?</th>";
     130    echo "<th>total/dist</th>";
     131    echo "<th>total/dist</th>";
     132    echo "<th>total/dist</th>";
     133    echo "<th>total/dist</th>";
     134    echo "<th>total/dist</th>";
    115135
    116136    $sql =
    … …  
    147167        echo "<tr><td></td>\n";
    148168
     169        echo "<TD>$exp_id</td>";
     170        echo "<TD>$exp_name</td>";
     171        echo "<TD>$filter</td>";
     172        echo "<TD>$comment</td>";
     173        echo "<TD>$rawExp</td>";
     174
    149175        // chip
    150176        if (!$chipDist[$exp_id]) $chipDistStr = "N";
    151177        else $chipDistStr = $chipDist[$exp_id];
     178        echo "<TD>$chipState/$chipDistStr</td>";
    152179
    153180        // cam
    154181        if (!$camDist[$exp_id]) $camDistStr = "N";
    155182        else $camDistStr = $camDist[$exp_id];
     183        echo "<TD>$camState/$camDistStr</td>";
    156184
    157185        // warp
    158186        if (!$warpDist[$exp_id]) $warpDistStr = "N";
    159187        else $warpDistStr = $warpDist[$exp_id];
     188        echo "<TD>$warpState/$warpDistStr</td>";
    160189
    161190        // stack
    162         if (!$stack_hash[$exp_id] && !$stackDist[$exp_id]) $stackDistStr = "";
     191        if (!$stack_hash[$exp_id] && !$stackDist[$exp_id]) $stackDistStr = "&nbsp;";
    163192        else $stackDistStr = $stack_hash[$exp_id] . "/" . $stackDist[$exp_id];
     193        $color = getColour($stack_hash[$exp_id], $stackDist[$exp_id]);
     194        echo "<TD bgcolor=\"$color\" >$stackDistStr</td>";
    164195
    165196        // WS diff
    166         if (!$wsDiffProduced[$exp_id] && !$wsDiffDist[$exp_id]) $wsDistStr = "";
     197        if (!$wsDiffProduced[$exp_id] && !$wsDiffDist[$exp_id]) $wsDistStr = "&nbsp;";
    167198        else $wsDistStr = $wsDiffProduced[$exp_id] . "/" . $wsDiffDist[$exp_id];
     199        $color = getColour($wsDiffProduced[$exp_id], $wsDiffDist[$exp_id]);
     200        echo "<TD bgcolor=\"$color\" >$wsDistStr</td>";
    168201
    169202        // WW1 diff
    170         if (!$ww1DiffProduced[$exp_id] && !$ww1DiffDist[$exp_id]) $ww1DistStr = "";
     203        if (!$ww1DiffProduced[$exp_id] && !$ww1DiffDist[$exp_id]) $ww1DistStr = "&nbsp;";
    171204        else $ww1DistStr = $ww1DiffProduced[$exp_id] . "/" . $ww1DiffDist[$exp_id];
     205        $color = getColour($ww1DiffProduced[$exp_id], $ww1DiffDist[$exp_id]);
     206        echo "<TD bgcolor=\"$color\" >$ww1DistStr</td>";
    172207
    173208        // WW2 dist
    174         if (!$ww2DiffProduced[$exp_id] && !$ww2DiffDist[$exp_id]) $ww2DistStr = "";
     209        if (!$ww2DiffProduced[$exp_id] && !$ww2DiffDist[$exp_id]) $ww2DistStr = "&nbsp;";
    175210        else $ww2DistStr =  $ww2DiffProduced[$exp_id] . "/" . $ww2DiffDist[$exp_id];
     211        $color = getColour($ww2DiffProduced[$exp_id], $ww2DiffDist[$exp_id]);
     212        echo "<TD bgcolor=\"$color\" >$ww2DistStr</td>";
    176213
    177214        // SS dist
    178         if (!$ssDiffProduced[$exp_id] && !$ssDiffDist[$exp_id]) $ssDistStr = "";
     215        if (!$ssDiffProduced[$exp_id] && !$ssDiffDist[$exp_id]) $ssDistStr = "&nbsp;";
    179216        else $ssDistStr = $ssDiffProduced[$exp_id] . "/" . $ssDiffDist[$exp_id];
    180 
    181         write_table_cell($class, '%s', "", $exp_id);
    182         write_table_cell($class, '%s', "", $exp_name);
    183         write_table_cell($class, '%s', "", $filter);
    184         write_table_cell($class, '%s', "", $comment);
    185         write_table_cell($class, '%s', "", $rawExp);
    186         write_table_cell($class, '%s', "", "$chipState/$chipDistStr");
    187         write_table_cell($class, '%s', "", "$camState/$camDistStr");
    188         write_table_cell($class, '%s', "", "$warpState/$warpDistStr");
    189         write_table_cell($class, '%s', "", $stackDistStr);
    190         write_table_cell($class, '%s', "", $wsDistStr);
    191         write_table_cell($class, '%s', "", $ww1DistStr);
    192         write_table_cell($class, '%s', "", $ww2DistStr);
    193         write_table_cell($class, '%s', "", $ssDistStr);
     217        $color = getColour($ssDiffProduced[$exp_id], $ssDiffDist[$exp_id]);
     218        echo "<TD bgcolor=\"$color\" >$ssDistStr</td>";
    194219
    195220        echo "</tr>\n";
    … …  
    199224}
    200225
     226###########################################################################
     227#
     228# Gets the right cell colour provided number created and distributed
     229#
     230###########################################################################
     231function getColour($created, $distributed) {
     232
     233    if ($distributed > 0 && $created > 0 && $distributed % $created != 0) return "red";
     234    return "";
     235}
    201236
    202237###########################################################################

  • branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl

    r30118 r30587  
    325325        # get the UNIX version of the (possible) neb: or path: filename
    326326        my $uriReal = $ipprc->file_resolve( $uri );
     327        &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
    327328
    328329        # funpack into the temp file.

  • branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl

    r30586 r30587  
    1515my $REALRUN = 1;
    1616
    17 my ( $class_id, $exp_id, $this_uri, $previous_uri, $imfile_state, $camera, $dbname, $logfile, $verbose, $save_temps, $rerun );
     17my ( $class_id, $exp_id, $this_uri, $previous_uri, $imfile_state, $camera, $dbname, $logfile, $verbose, $save_temps );
     18my ( $rerun, $fixit );
     19my $continue = 0;
    1820GetOptions(
    1921    'exp_id=s'         => \$exp_id,   # exposure identifier
    … …  
    2628    'logfile=s'         => \$logfile,
    2729    'rerun'             => \$rerun,
     30#    'fix|f'             => \$fixit,
     31    'continue=i'        => \$continue,
    2832    'verbose'           => \$verbose,   # Print to stdout
    2933    'save-temps'        => \$save_temps, # Save temporary files?
    … …  
    3337pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    3438pod2usage(
    35     -msg => "Required options: --class_id --this_uri --dbname",
     39    -msg => "Required options: --class_id --this_uri --exp_id --dbname",
    3640    -exitval => 3,
    3741    ) unless
    3842    defined $class_id and
    3943    defined $this_uri and
     44    defined $exp_id and
    4045    defined $dbname;
    4146
    … …  
    9398print ">>$burntoolStateGood<<\n";
    9499
     100# Determine what to do if we're fixing.
     101# if ($fixit) {
     102#     ($exp_id,$this_uri,$previous_uri,$continue) = fix_burntool($exp_id,$this_uri,$class_id);
     103# }
     104
     105
     106# Precalculate what we should be able to do if we're continuing
     107my @continue_uris;
     108my @continue_exp_ids;
     109my @continue_states;
     110if ($continue > 0) {
     111    my $regtool_get_date   = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -dbname $dbname";
     112    my $dateobs;
     113    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     114        run ( command => $regtool_get_date, verbose => $verbose );
     115    unless ($success) {
     116        $continue = 0;
     117        last;
     118    }
     119    my $imfile_data = $mdcParser->parse(join "", @$stdout_buf);
     120    my $imfile_data2 = parse_md_list($imfile_data);
     121    if ($#{ $imfile_data2 } > 0) {
     122        &my_die("Too many entries returned for date query", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
     123    }
     124    foreach my $entry (@$imfile_data2) {
     125        $dateobs = $entry->{dateobs};
     126    }
     127    my $date = $dateobs; $date =~ s/T.*//;
     128
     129    my $regtool_status_cmd = "$regtool -checkstatus -dateobs_begin $dateobs -dateobs_end ${date}T17:30:00 -class_id $class_id -dbname $dbname";
     130    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     131        run ( command => $regtool_status_cmd, verbose => $verbose );
     132    unless ($success) {
     133        $continue = 0;
     134    }
     135    if ($continue != 0) {
     136        my $night_data = $mdcParser->parse(join "", @$stdout_buf);
     137        my $night_data2 = parse_md_list($night_data);
     138        my $addable = 0;
     139       
     140        foreach my $entry (@$night_data2) {
     141            my ($this_exp_id,$this_uri,$this_state);
     142            if ($entry->{exp_id} == $exp_id) {
     143                $addable = 1;
     144                next;
     145            }
     146            if (($addable == 1)&&($#continue_exp_ids < $continue)) {
     147                $this_exp_id = $entry->{exp_id};
     148                $this_uri    = $entry->{uri};
     149                $this_state  = $entry->{imfile_state};
     150                print ">> $this_exp_id $this_uri $this_state\n";
     151                if ($this_state eq 'pending_burntool') {
     152                    push @continue_exp_ids, $this_exp_id;
     153                    push @continue_uris, $this_uri;
     154                    push @continue_states, $this_state;
     155                }
     156                else {
     157                    last;
     158                }
     159            }
     160            if ($#continue_exp_ids >= $continue) {
     161                last;
     162            }
     163        }
     164    }
     165}
     166$continue = $#continue_exp_ids + 1;
     167
     168# Look over at least one uri/imfile/exp_id and do burntool on it.
     169my $next_exp_id;
     170for (my $iteration = 0; $iteration <= $continue; $iteration ++) {
    95171# Set up files
    96 my $rawImfile = $this_uri;
    97 my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0);
    98 
    99 my $outTable  = $this_uri;
    100 $outTable  =~ s/fits$/burn.tbl/;
    101 my $outTableReal = $ipprc->file_resolve($outTable, 1);
    102 
    103 my $previousTable;
    104 my $previousTableReal;
    105 if ($previous_uri) {
    106     $previousTable = $previous_uri;
    107     $previousTable =~ s/fits$/burn.tbl/;
    108     $previousTableReal = $ipprc->file_resolve($previousTable, 0);
    109 }
    110 
     172    my $rawImfile = $this_uri;
     173    my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0);
     174   
     175    my $outTable  = $this_uri;
     176    $outTable  =~ s/fits$/burn.tbl/;
     177    my $outTableReal = $ipprc->file_resolve($outTable, 1);
     178   
     179    my $previousTable;
     180    my $previousTableReal;
     181    if ($previous_uri) {
     182        $previousTable = $previous_uri;
     183        $previousTable =~ s/fits$/burn.tbl/;
     184        $previousTableReal = $ipprc->file_resolve($previousTable, 0);
     185    }
     186   
    111187# Set state to processing:
    112 my $burntool_state = 0;
    113 
    114 my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN);
    115 if ($status) {
    116     &my_die("failed to update imfile");
    117 }
    118 $burntool_state = 1;
    119 
    120 
     188    my $burntool_state = 0;
     189   
     190    my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN);
     191    if ($status) {
     192        &my_die("failed to update imfile");
     193    }
     194    $burntool_state = 1;
     195   
     196   
    121197# funpack
    122 my $tempfile = new File::Temp ( TEMPLATE => "burntool.${exp_id}.${class_id}.XXXX",
    123                                 DIR => '/tmp/',
    124                                 UNLINK => !$save_temps,
    125                                 SUFFIX => '.fits');
    126 my $tempPixels = $tempfile->filename;
    127 
    128 $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN);
    129 if ($status) {
    130     &my_die("failed on funpack",$exp_id,$class_id);
    131 }
    132 
     198    my $tempfile = new File::Temp ( TEMPLATE => "burntool.${exp_id}.${class_id}.XXXX",
     199                                    DIR => '/tmp/',
     200                                    UNLINK => !$save_temps,
     201                                    SUFFIX => '.fits');
     202    my $tempPixels = $tempfile->filename;
     203   
     204    $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN);
     205    if ($status) {
     206        &my_die("failed on funpack",$exp_id,$class_id);
     207    }
     208   
    133209# burntool
    134 if (($previousTableReal)&&($previousTableReal ne '')) {
    135     $status = vsystem ("$burntool $tempPixels in=$previousTableReal out=$outTableReal tableonly=t persist=t", $REALRUN);
    136 }
    137 else {
    138     $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN);
    139 }
    140 if ($status) {
    141     &my_die("failed on burntool",$exp_id,$class_id);
    142 }
    143 &my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal);
    144 
     210    if (($previousTableReal)&&($previousTableReal ne '')) {
     211        $status = vsystem ("$burntool $tempPixels in=$previousTableReal out=$outTableReal tableonly=t persist=t", $REALRUN);
     212    }
     213    else {
     214        $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN);
     215    }
     216    if ($status) {
     217        &my_die("failed on burntool",$exp_id,$class_id);
     218    }
     219    &my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal);
     220   
    145221# Replicate files
    146 $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN);
    147 $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN);
    148 
     222    $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN);
     223    $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN);
     224
     225# Prep for next iteration if necessary.
     226    if ($iteration < $continue) {
     227        $next_exp_id = shift(@continue_exp_ids);
     228        my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $next_exp_id -class_id $class_id -set_state check_burntool";
     229        $status = vsystem ($command, $REALRUN);
     230        if ($status) {
     231            $continue = 0;
     232        }
     233    }
    149234# Set state to finished.
    150 my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState";
    151 if ($imfile_state ne 'full') {
    152     $command .= " -set_state full ";
    153 }
    154 $status = vsystem ($command, $REALRUN);
    155 if ($status) {
    156     &my_die("failed to update imfile");
    157 }
    158 
     235    my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState";
     236    if ($imfile_state ne 'full') {
     237        $command .= " -set_state full ";
     238    }
     239    $status = vsystem ($command, $REALRUN);
     240    if ($status) {
     241        &my_die("failed to update imfile");
     242    }
     243
     244# Set values for the next iteration if we are doing another iteration.
     245    $previous_uri = $this_uri;
     246    if ($iteration < $continue) {
     247        $this_uri = shift (@continue_uris);
     248        $exp_id   = $next_exp_id;
     249        $imfile_state = shift(@continue_states);
     250    }
     251
     252}
    159253exit 0;
    160254
    … …  
    183277    exit 1;
    184278}
    185 
    186 
    187    
     279       
     280# sub fix_burntool {
     281#     my ($exp_id,$this_uri,$class_id,$burntoolGoodState) = @_;
     282#     my ($dateobs,$exp_name);
     283#     # Calculate the date for this exposure, and calculate back to find the burntool period.
     284#     my $regtool_get_date   = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -dbname $dbname";
     285#     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     286#       run ( command => $regtool_get_date, verbose => $verbose );
     287#     unless ($success) {
     288#       $continue = 0;
     289#       return($exp_id,$this_uri,undef,0);
     290#     }
     291#     my $imfile_data = $mdcParser->parse(join "", @$stdout_buf);
     292#     foreach my $entry (@$imfile_data) {
     293#       if ($entry->{name} eq 'dateobs') {
     294#           $dateobs = $entry->{value};
     295#       }
     296#       if ($entry->{name} eq 'exp_name') {
     297#           $exp_name = $entry->{value};
     298#       }
     299#     }
     300#     my $date = $dateobs; $date =~ s/T.*//;
     301#     my $time = $dateobs; $time =~ s/.*T//;
     302#     my ($hour,$min,$sec) = split /\:/, $time; #/;
     303#     if ($min >= 30) {
     304#       $min -= 30;
     305#     }
     306#     else {
     307#       $min += 30;
     308#       $hour -= 1;
     309#     }
     310#     my $dateobs_begin = sprintf("%sT%02d:%02d:%02d",$date,$hour,$min,$sec);
     311
     312
     313#     # Get the night data for this date range.
     314#     my $regtool_status_cmd = "$regtool -checkstatus -dateobs_begin $dateobs_begin -dateobs_end $dateobs -class_id $class_id -dbname $dbname";
     315#     ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     316#       run ( command => $regtool_get_date, verbose => $verbose );
     317#     unless ($success) {
     318#       return($exp_id,$this_uri,undef,0);
     319#     }
     320
     321#     my $night_data = $mdcParser->parse(join "", @$stdout_buf);
     322
     323#     my $mode;
     324#     my ($row_exp_id,$row_uri,$row_data_state,$row_bt_state);
     325#     my $entry = @$night_data[-1];
     326#     foreach my $row (@$entry) {  # This is a single exposure, so let's get everything we'll need here.
     327#       if ($row->{name} eq 'exp_id') {
     328#           $row_exp_id = $row->{value};
     329#       }
     330#       elsif ($row->{name} eq 'uri') {
     331#           $row_uri = $row->{uri};
     332#       }
     333#       elsif ($row->{name} eq 'data_state') {
     334#           $row_data_state = $row->{data_state};
     335#       }               
     336#       elsif ($row->{name} eq 'burntool_state') {
     337#           $row_bt_state = $row->{burntool_state};
     338#       }               
     339#     }
     340#     # Determine what to do.
     341#     if ($row_exp_id == $exp_id) {
     342#       if (($row_bt_state != $burntoolStateGood)) {
     343#           # Never been run.
     344#           $mode = 1;
     345#       }
     346#       else {
     347#           $mode = 0;
     348#       }
     349#     }
     350#     if ($mode == 0) {
     351#       # If we're here, then the database thinks we have a table for this uri on disk. Let's see if that's true.
     352#       # Zero byte file/good alternate
     353#       # Inconsistent md5sums
     354#       # Missing this table only.
     355#       my $neb = $ipprc->nebulous();
     356#       my $table_key = $this_uri; $table_key =~ s/fits/burn.tbl/;
     357
     358#       my $neb_instances = $neb->find_instances($table_key, 'any');
     359
     360#       my $Ninstances = $#{ $neb_instances } + 1;
     361
     362#       if (($Ninstances == 0)||($Ninstances == 1)) {
     363#           #Missing this table only.  This should fall through to do a regular pass on this uri
     364#           $continue = 0;
     365#           $entry = @$night_data[-2];
     366#           foreach my $row (@$entry) {  # This is a single exposure, so let's get everything we'll need here.
     367#               if ($row->{name} eq 'exp_id') {
     368#                   $row_exp_id = $row->{value};
     369#               }
     370#               elsif ($row->{name} eq 'uri') {
     371#                   $row_uri = $row->{uri};
     372#               }
     373#               elsif ($row->{name} eq 'data_state') {
     374#                   $row_data_state = $row->{data_state};
     375#               }               
     376#               elsif ($row->{name} eq 'burntool_state') {
     377#                   $row_bt_state = $row->{burntool_state};
     378#               }               
     379#           }
     380#           $previous_uri = $row_uri;
     381#           # Go through to regular processing of this exposure.
     382#       }
     383#       elsif ($Ninstances == 2) {
     384#           $md5sum_A = md5sum(${ $neb_instances }[0]);
     385#           $md5sum_B = md5sum(${ $neb_instances }[1]);
     386#           if ($md5sum_A != $md5sum_B) {
     387#               # md5sums are different
     388#               if ($md5sum_A = 'd41d8cd98f00b204e9800998ecf8427e') {
     389#                   # copy B -> A
     390#               }
     391#               elsif ($md5sum_B = 'd41d8cd98f00b204e9800998ecf8427e') {
     392#                   # copy A -> B
     393#               }
     394#               else {
     395#                   # different but unknown why
     396#               }
     397#           }
     398#       }
     399#     }
     400
     401#     if ($mode == 1) {
     402#       foreach my $entry (reverse(@$night_data)) {  # reverse it so we can start with the exposure we have and go backwards
     403#           foreach my $row (@$entry) {  # This is a single exposure, so let's get everything we'll need here.
     404#               if ($row->{name} eq 'exp_id') {
     405#                   $row_exp_id = $row->{value};
     406#               }
     407#               elsif ($row->{name} eq 'uri') {
     408#                   $row_uri = $row->{uri};
     409#               }
     410#               elsif ($row->{name} eq 'data_state') {
     411#                   $row_data_state = $row->{data_state};
     412#               }               
     413#               elsif ($row->{name} eq 'burntool_state') {
     414#                   $row_bt_state = $row->{burntool_state};
     415#               }               
     416#           }
     417#       }
     418#     }
     419
     420#     return($exp_id,$this_uri,$previous_uri,$continue);
     421# }

  • branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl

    r30586 r30587  
    207207            }
    208208
    209             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    210             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    211             $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
    212             $command .= " -dbname $dbname" if defined $dbname;
    213             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    214                 run(command => $command, verbose => $verbose);
    215             unless ($success) {
    216                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    217                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    218             }
     209            set_destreak_goto_cleaned();
    219210
    220211        } else {
    … …  
    324315        }
    325316
    326         # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    327         my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    328         $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
    329         $command .= " -dbname $dbname" if defined $dbname;
    330         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    331             run(command => $command, verbose => $verbose);
    332         unless ($success) {
    333             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    334             &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    335         }
     317        set_destreak_goto_cleaned();
    336318
    337319    } else {
    … …  
    469451            }
    470452
    471             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    472             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    473             $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
    474             $command .= " -dbname $dbname" if defined $dbname;
    475             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    476                 run(command => $command, verbose => $verbose);
    477             unless ($success) {
    478                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    479                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    480             }
     453            set_destreak_goto_cleaned();
    481454
    482455         } else {
    … …  
    603576            }
    604577
    605             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    606             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    607             $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
    608             $command .= " -dbname $dbname" if defined $dbname;
    609             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    610                 run(command => $command, verbose => $verbose);
    611             unless ($success) {
    612                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    613                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    614             }
     578            set_destreak_goto_cleaned();
    615579
    616580        } else {
    … …  
    759723            }
    760724
    761             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    762             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    763             $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
    764             $command .= " -dbname $dbname" if defined $dbname;
    765             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    766                 run(command => $command, verbose => $verbose);
    767             unless ($success) {
    768                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    769                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    770             }
     725            set_destreak_goto_cleaned();
    771726
    772727        } else {
    … …  
    20622017}
    20632018
     2019# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
     2020my $ds_done = 0;
     2021sub set_destreak_goto_cleaned {
     2022
     2023    return if $ds_done;
     2024
     2025    # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
     2026    my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
     2027    my $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
     2028    $command .= " -dbname $dbname" if defined $dbname;
     2029    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     2030        run(command => $command, verbose => $verbose);
     2031    unless ($success) {
     2032        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     2033            &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
     2034    }
     2035    $ds_done = 1;
     2036}
     2037
    20642038# XXX we currently do not set the error state in the db on my_die
    20652039sub my_die

  • branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl

    r30586 r30587  
    159159        my $component = $comp->{component};
    160160        my $backup_path_base = $comp->{backup_path_base};
    161         my $recovery_path_base = $comp->{recovery_path_base};
    162161        my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom);
    163162        my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom);
    … …  
    197196                $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $component);
    198197            }
    199             if ($recovery_path_base) {
    200                 $rimage  = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $component);
    201 
    202                 if ($dynamicMasks) {
    203                     my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $component);
    204                     # This is kludgey but correct
    205                     $rmask = dirname($recovery_path_base) . "/" . basename($mask);
    206                     $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component);
    207                 } else {
    208                     $rmask = $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component);
    209                 }
    210                 $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $component);
    211             }
    212198        } elsif ($stage eq "camera") {
    213199            if ($backup_path_base) {
    … …  
    221207                $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base);
    222208            }
    223             if ($recovery_path_base) {
    224                 $rimage  = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base);
    225                 $rmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base);
    226                 $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base);
    227                 $rsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $recovery_path_base);
    228             }
    229209        } elsif ($stage eq "diff") {
    230210            my $name = "PPSUB.OUTPUT";
     211            if ($backup_path_base) {
     212                $bimage  = $ipprc->filename($name, $backup_path_base);
     213                $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
     214                $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
     215                # bills 2011-01-24
     216                # don't clean up the uncensored Diff sources file
     217                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
     218            }
     219        }
     220
     221        delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
     222
     223        if ($stage eq "diff" and $warp_warp) {
     224            my $name = "PPSUB.INVERSE";
    231225            if ($backup_path_base) {
    232226                $bimage  = $ipprc->filename($name, $backup_path_base);
    … …  
    237231                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
    238232            }
    239             if ($recovery_path_base) {
    240                 $rimage  = $ipprc->filename($name, $recovery_path_base);
    241                 $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
    242                 $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
    243                 $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base);
    244             }
    245         }
    246 
    247         delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
    248 
    249         if ($stage eq "diff" and $warp_warp) {
    250             my $name = "PPSUB.INVERSE";
    251             if ($backup_path_base) {
    252                 $bimage  = $ipprc->filename($name, $backup_path_base);
    253                 $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
    254                 $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
    255                 # bills 2011-01-24
    256                 # don't clean up the uncensored sources file
    257                 # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
    258             }
    259             if ($recovery_path_base) {
    260                 $rimage  = $ipprc->filename($name, $recovery_path_base);
    261                 $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
    262                 $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
    263                 $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base);
    264             }
    265             # undef is not necessary, it's just to keep the lists in the same order.
    266             delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources);
     233            delete_files($bimage, $bmask, $bweight, $bsources);
    267234        }
    268235        my $command = "$magicdstool -tocleanedfile -magic_ds_id $magic_ds_id -component $component";

  • branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl

    r30586 r30587  
    12661266}
    12671267
     1268sub multi_date_verify_uniqueness_diff {
     1269    my $warp_id_1 = shift;
     1270    my $warp_id_2 = shift;
     1271    my $date = shift;
     1272    my $target = shift;
     1273
     1274    my $db = init_gpc_db();
     1275    $date =~ s/-//g;
     1276    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
     1277
     1278    my $count = 0;
     1279   
     1280    my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
     1281    my $data_ref = $db->selectall_arrayref( $sth );
     1282    $count += $#{ $data_ref } + 1;
     1283
     1284    $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_2 AND warp2 = $warp_id_1";
     1285    $data_ref = $db->selectall_arrayref( $sth );
     1286    $count += $#{ $data_ref } + 1;
     1287
     1288    return($count);
     1289}
     1290
    12681291sub pre_diff_queue {
    12691292    my $date = shift;
    … …  
    13751398        $metadata_out{nsDiffState} = 'DIFFING';
    13761399    }
    1377 
     1400    if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') {
     1401        foreach my $target (sort (keys %science_config)) {
     1402            if ($science_config{$target}{DIFFABLE} == 1) {
     1403                foreach my $filter (@filter_list) {
     1404                    multi_date_diff_queue($date,$target,$filter,$pretend);
     1405                }
     1406            }
     1407        }
     1408    }
    13781409#     if (($Npotential == $Nnoexp)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)) {
    13791410#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
    … …  
    15111542
    15121543}
     1544
     1545sub multi_date_diff_queue {
     1546    my $date = shift;
     1547    my $target = shift;
     1548    my $filter = shift;
     1549    my $pretend = shift;
     1550    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
     1551
     1552    my $db = init_gpc_db();
     1553
     1554    my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
     1555    $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
     1556
     1557    my $object_ref = $db->selectall_arrayref( $obj_sth );
     1558
     1559    my $Npotential = 0;
     1560    my $Nqueued = 0;
     1561   
     1562    foreach my $object_row (@{ $object_ref }) {
     1563        my $this_object = shift @{ $object_row };
     1564#       my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
     1565#       $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
     1566#       $input_sth .= " ORDER BY dateobs ";
     1567       
     1568        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM ";
     1569        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
     1570        $input_sth .=   " WHERE warpRun.label = '$label' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
     1571        $input_sth .=   " ORDER BY dateobs ";
     1572
     1573        my $warps = $db->selectall_arrayref( $input_sth );
     1574
     1575        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
     1576        my %comment_hash = ();
     1577        foreach my $this_warp (@{ $warps }) {
     1578            my $this_comment = ${ $this_warp }[3];
     1579            my $this_exp_id  = ${ $this_warp }[0];
     1580            $comment_hash{$this_comment} = $this_exp_id;
     1581        }
     1582       
     1583        if (($#{ $warps } + 1) % 2 != 0) {
     1584            print STDERR "md_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
     1585            if ($#{ $warps} + 1 == 1) {
     1586                print STDERR ": I can do no diffs with only one exposure.\n";
     1587                next;
     1588            }
     1589            else {
     1590                print STDERR ": I should declare an exposure to be faulty.\n";
     1591                my @keep_warps = ();
     1592#               print "@{ $warps }\n";
     1593                foreach my $this_warp (@{ $warps }) {
     1594                    my $this_comment = ${ $this_warp }[3];
     1595                    my $this_exp_id  = ${ $this_warp }[0];
     1596                    if ($comment_hash{$this_comment} == $this_exp_id) {
     1597                        push @keep_warps, $this_warp;
     1598                    }
     1599                    else {
     1600                        print STDERR "md_diff_queue: excluding $this_exp_id for $this_object\n";
     1601                    }
     1602                }
     1603                @{ $warps } = @keep_warps;
     1604#               print "@{ $warps }\n";
     1605            }
     1606        }
     1607       
     1608        while ($#{ $warps } > -1) {
     1609            my $input_warp = shift @{ $warps };
     1610            my $input_exp_id = ${ $input_warp }[0];
     1611            my $input_comment = ${ $input_warp }[3];
     1612
     1613           
     1614            my $template_warp = shift @{ $warps };
     1615
     1616            my $template_exp_id = ${ $template_warp }[0];
     1617           
     1618            my $input_warp_id = ${ $input_warp }[1];
     1619            my $template_warp_id = ${ $template_warp }[1];
     1620
     1621            my $input_warp_state = ${ $input_warp }[4];
     1622            my $template_warp_state = ${ $template_warp }[4];
     1623           
     1624            unless(defined($template_warp)&& defined($template_exp_id)) {
     1625                print STDERR "md_diff received an undef! $input_exp_id $input_comment $this_object T: $template_warp V: @$template_warp\n";
     1626                next;
     1627            }
     1628            $Npotential++;
     1629           
     1630            unless (defined($input_warp_id) && defined($template_warp_id) &&
     1631                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
     1632                print STDERR "md_Diff for this $date $target $input_exp_id ($input_warp_id $input_warp_state) $template_exp_id ($template_warp_id $template_warp_state) not fully processed\n";
     1633                next;
     1634            }
     1635
     1636            if (multi_date_verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
     1637                $Nqueued++;
     1638                print STDERR "md_Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
     1639                next;
     1640            }
     1641
     1642            my $new_data_group = "${data_group}.multi";
     1643            my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
     1644            $cmd .= "-input_label $label  -template_label $label ";
     1645            $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
     1646            $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $new_data_group ";
     1647            $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
     1648#               $cmd .= " -pretend ";
     1649            if (defined($pretend)) {
     1650                $cmd .= ' -pretend ';
     1651            }
     1652            if ($debug == 1) {
     1653                $cmd .= ' -pretend ';
     1654                print STDERR "md_Diffs would like to queue for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
     1655                print STDERR "md_diff_queue: $cmd\n";
     1656                print STDERR " $input_warp_id $template_warp_id\n";
     1657            }
     1658           
     1659            if (($debug == 0)&&(!defined($pretend))) {
     1660                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1661                    run ( command => $cmd, verbose => $verbose );
     1662                unless ($success) {
     1663                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1664                    &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     1665                }
     1666                $Nqueued++;
     1667            }
     1668        }
     1669    }
     1670    $metadata_out{nsDiffPotential} += $Npotential;
     1671    $metadata_out{nsDiffQueued}    += $Nqueued;
     1672#      if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
     1673#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
     1674#      }       
     1675
     1676}
    15131677           
    15141678

  • branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl

    r30586 r30587  
    255255        if (exists($nightlyscience_config{$target}{OBSMODE})) {
    256256            $possible++;
    257             if ($obsmode =~ /$nightlyscience_config{$target}{OBSMODE}/) {
     257            if ($obsmode =~ /^$nightlyscience_config{$target}{OBSMODE}$/) {
    258258                $match++;
    259259            }
    … …  
    261261        if (exists($nightlyscience_config{$target}{OBJECT})) {
    262262            $possible++;
    263             if ($object =~ /$nightlyscience_config{$target}{OBJECT}/) {
     263            if ($object =~ /^$nightlyscience_config{$target}{OBJECT}$/) {
    264264                $match++;
    265265            }
    … …  
    267267        if (exists($nightlyscience_config{$target}{COMMENT})) {
    268268            $possible++;
    269             if ($comment =~ /$nightlyscience_config{$target}{COMMENT}/) {
     269            if ($comment =~ /^$nightlyscience_config{$target}{COMMENT}$/) {
    270270                $match++;
    271271            }

  • branches/czw_branch/20101203/ippScripts/scripts/warp_skycell.pl

    r27754 r30587  
    7171my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    7272
    73 my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id) or my_die( "Unable to get log filename", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR );
    74 $logDest .= ".update" if ($run_state eq 'update');
     73my ($logDest, $traceDest);
     74if ($run_state eq 'new') {
     75    $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, 0);
     76    $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1);
     77} elsif ($run_state eq 'update')  {
     78    $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 0);
     79    $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, $skycell_id, 1);
     80} else {
     81    &my_die( "invalid run_state: $run_state", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR );
     82}
     83
     84my $neb;
     85my $scheme = file_scheme($outroot);
     86if ($scheme and $scheme eq 'neb') {
     87    $neb = $ipprc->nebulous();
     88}
    7589
    7690$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect;
    … …  
    117131    unless ($success) {
    118132        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    119         &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code);
     133        &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $skycell_id, $tess_dir, $error_code);
    120134    }
    121135    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    122         &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR);
     136        &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR);
    123137    $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
    124138}
    … …  
    141155
    142156
    143 my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outroot, $skycell_id );
    144 my $outputMask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outroot, $skycell_id);
    145 my $outputWeight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id);
    146 my $outputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id);
    147 my $outputPSF = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $outroot);
    148 my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outroot, $skycell_id );
    149 my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outroot, $skycell_id );
    150 my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot, $skycell_id );
    151 my $traceDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id);
    152 my $configuration =  $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id);
    153 
    154 if ($run_state eq 'update') {
    155     $traceDest .= ".update";
    156     $outputStats .= ".update";
    157 }
    158 
    159 my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id );
     157my $outputImage = prepare_output ("PSWARP.OUTPUT", $outroot, $skycell_id, 1);
     158my $outputMask = prepare_output ("PSWARP.OUTPUT.MASK", $outroot, $skycell_id, 1);
     159my $outputWeight = prepare_output ("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id, 1);
     160my $outputSources = prepare_output ("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id, 1);
     161my $outputPSF = prepare_output ("PSPHOT.PSF.SKY.SAVE", $outroot, 1);
     162my $outputBin1 = prepare_output ("PSWARP.BIN1", $outroot, $skycell_id, 1);
     163my $outputBin2 = prepare_output ("PSWARP.BIN2", $outroot, $skycell_id, 1);
     164my $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1);
     165my $configuration;
     166
     167my $dump_config = 1;
     168if ($run_state eq 'new') {
     169    $configuration =  prepare_output ("PSWARP.CONFIG", $outroot, $skycell_id, 1);
     170} else {
     171    $configuration =  $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id) or
     172        &my_die("Missing entry from camera config PSWARP.CONFIG", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
     173    if ($ipprc->file_exists($configuration)) {
     174        $dump_config = 0;
     175    } else {
     176        print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n";
     177
     178        # XXX: should we create a new config dump file?
     179        # I vote yes but only if we can distingusing between temporarily unavailable and GONE.
     180        my $gone = 0;
     181        if (storage_object_exists($configuration, \$gone)) {
     182            if ($gone) {
     183                $configuration = prepare_output('PSWARP.CONFIG', $outroot, $skycell_id, 1);
     184                # if we dump the config we need to insure that the config dump represents
     185                # the full processing
     186            } else {
     187                # file is temporarily not available. Don't dump config.
     188                $dump_config = 0;
     189            }
     190        }
     191    }
     192}
     193
     194
     195my $skyFile = prepare_output ("SKYCELL.TEMPLATE", $outroot, $skycell_id, 1);
    160196$ipprc->skycell_file( $tess_dir, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
    161197## XXX this seems to have insufficient error checking: dvoImageExtract can fail to write and still return a valid exit status
    … …  
    171207foreach my $imfile (@$imfiles) {
    172208    my $image = $imfile->{uri}; # Image name
    173     my $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
     209    my $weight = $ipprc->filename ("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
    174210
    175211    my $mask;                   # Mask name
    176212    if ($dynamicMasks) {
    177         $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id});
     213        $mask = $ipprc->filename ("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id});
    178214    } else {
    179         $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id});
     215        $mask = $ipprc->filename ("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id});
    180216    }
    181217
    … …  
    186222    # Astrometry file: astrometry is done at the camera stage, and always results in a MEF file
    187223    # XXX allow an option to use the image header astrometry?
    188     my $astrom = $ipprc->filename($astromSource, $imfile->{cam_path_base});
     224    my $astrom = $ipprc->filename ($astromSource, $imfile->{cam_path_base});
    189225
    190226    &my_die("Couldn't find input file: $astrom", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless defined $astrom and $ipprc->file_exists($astrom);
    … …  
    245281        $do_stats = 1;
    246282    } else {
    247         #$command .= " -ipprc $configuration";
    248         my $resolved = $ipprc->file_resolve($configuration);
    249         $command .= " -ipprc $resolved";
     283        $command .= " -ipprc $configuration";
    250284    }
    251285    if ($do_stats) {
    … …  
    263297    if ($do_stats) {
    264298        # Check first for the stats file
     299        check_output($outputStats, 0);
    265300        my $outputStatsReal = $ipprc->file_resolve($outputStats);
    266         &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
     301#        &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
    267302        &my_die("Stats file has zero size: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless -s $outputStatsReal;
    268303
    269         # measure chip stats
     304        # measure skycell stats
    270305        $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL";
    271306        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    283318
    284319        if (!$quality) {
    285             &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
    286             &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
    287             &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
    288             &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
    289             &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF);
    290             if ($run_state eq 'new') {
    291                 &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
     320        #    &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
     321        #    &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
     322        #    &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
     323        #    &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
     324        #    &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF);
     325
     326            check_output($outputImage, 0);
     327            check_output($outputMask, 0);
     328            check_output($outputWeight, 0);
     329            check_output($outputSources, 1);
     330            check_output($outputPSF, 1) if metadataLookupBool($recipe, 'PSF')  ;
     331            if ($dump_config)  {
     332                check_output($configuration, 1);
     333#                &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
    292334            }
    293335        }
    … …  
    335377    }
    336378}
     379
     380exit 0;
     381
     382# Prepare to write to an output file
     383#   Lookup the filename in the rules.
     384#   Make sure that if file exists and is a nebulous file that there is only one instance
     385#   Deal with files that have been lost.
     386sub prepare_output
     387{
     388    my $filerule = shift;
     389    my $outroot  = shift;
     390    my $skycell_id = shift;
     391    my $delete = shift;
     392    $delete = 0 if !defined $delete;
     393
     394    my $error;
     395    my $output = $ipprc->prepare_output($filerule, $outroot, $skycell_id, $delete, \$error)
     396                    or &my_die("failed to prepare output file for: $filerule", $warp_id, $skycell_id, $tess_dir, $error);
     397    return $output;
     398}
     399
     400sub check_output
     401{
     402    my $file = shift;
     403    my $replicate = shift;
     404
     405    if (!defined $file) {
     406        return;
     407    }
     408
     409    &my_die("Couldn't find expected output file: $file",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
     410
     411    if ($replicate and $neb) {
     412        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
     413    }
     414}
     415
     416# subroutine to check the status of a nebulous file. Used to distinguish between a storage object that
     417# does not exist and one that all of the instances have been lost.
     418# XXXX This should be implemented properly in Nebulous
     419# For now uses Bill's script 'whichnode' which queries the nebulous database directly
     420
     421my $whichnode;
     422sub storage_object_exists
     423{
     424    return 0 if !$neb;
     425
     426    my $file = shift;
     427    my $ref_all_gone = shift;
     428
     429    my $exists = $neb->storage_object_exists($file);
     430    if (!$exists) {
     431        return 0;
     432    }
     433
     434    if (!$whichnode) {
     435        $whichnode = can_run('whichnode') or
     436            &my_die("Can't find whichnode",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
     437    }
     438
     439    my $command = "$whichnode $file";
     440
     441    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     442        run(command => $command, verbose => $verbose);
     443    unless ($success) {
     444        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     445        &my_die("Unable to perform whichnode: $error_code", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
     446    }
     447
     448    my @lines = split "\n", (join "", @$stdout_buf);
     449
     450    if (scalar @lines == 0) {
     451        # no output the file is really and truely gone
     452        # XXX: this is now caught above
     453        print STDERR "storage object for $file does not exist\n";
     454        return 0;
     455    }
     456
     457    my $numGone = 0;
     458    my $numNotGone = 0;
     459    foreach my $line (@lines) {
     460        chomp $line;
     461
     462        # output lines are either
     463        #   "volume available"
     464        # or
     465        #   "volume not available"
     466
     467        my ($volume, $answer, undef) = split " ", $line;
     468        # our hack is if the volume has an X in the name it's gone
     469        if ($volume =~ /X/) {
     470            print STDERR "$file is on $volume which is gone\n";
     471            $numGone++;
     472        } elsif ($answer eq 'available') {
     473            $numNotGone++;
     474        } elsif ($answer eq 'not') {
     475            print STDERR "$file is on $volume which is not available\n";
     476            $numNotGone++;
     477        } else {
     478            print STDERR "unexpected output from whichnode: $line\n";
     479        }
     480    }
     481    # if there are any instances that are not on a gone volume set all_gone to 0
     482    if ($numNotGone == 0 and $numGone > 0) {
     483        $$ref_all_gone = 1;
     484    } else {
     485        $$ref_all_gone = 0;
     486    }
     487
     488    # storage object exists so return true
     489    return 1;
     490}
     491
    337492
    338493sub my_die

  • branches/czw_branch/20101203/ippTasks

    • Property svn:mergeinfo deleted

  • branches/czw_branch/20101203/ippTasks/pstamp.pro

    r30586 r30587  
    1818$pstampStopFaulted_DB = 0
    1919
     20# give up on dependents with fault_count >= $PSTAMP_MAX_FAULT_COUNT
     21$PSTAMP_MAX_FAULT_COUNT = 5
     22macro set.max.fault.count
     23    $PSTAMP_MAX_FAULT_COUNT = $1
     24end
     25macro get.max.fault.count
     26    echo maximum fault count: $PSTAMP_MAX_FAULT_COUNT
     27end
     28$POLL_DEP = 500
     29
     30macro set.dependent.poll
     31    $POLL_DEP = $1
     32end
     33macro get.dependent.poll
     34    echo dependent poll limit: $POLL_DEP
     35end
     36
    2037# set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config
    2138# warning: no quotes around the two words. That causes the variable to get passed to pstamptool as one word
    … …  
    137154    end
    138155end
     156
    139157macro pstamp.find.on
    140158    task pstamp.request.find
    … …  
    143161end
    144162macro pstamp.find.off
    145     task pstamp.request.find.load
    146         active false
    147     end
    148 end
     163    task pstamp.request.find
     164        active false
     165    end
     166end
     167
     168macro pstamp.dependent.on
     169    task pstamp.dependent.load
     170        active true
     171    end
     172    task pstamp.dependent.run
     173        active true
     174    end
     175end
     176macro pstamp.dependent.off
     177    task pstamp.dependent.load
     178        active false
     179    end
     180    task pstamp.dependent.run
     181        active false
     182    end
     183end
     184
    149185
    150186macro pstamp.status.on
    … …  
    158194    end
    159195end
     196
    160197macro pstamp.status.set.exec
    161198    task pstamp.save.status
    … …  
    525562            echo pstamp.job.run task.exit $JOB_ID status: $JOB_STATUS
    526563        end
    527         showcommand failure
     564#        showcommand failure
    528565        process_exit pstampJob $options:0 $JOB_STATUS
    529566    end
    … …  
    609646        add_poll_args run
    610647        add_poll_labels run
    611         command $run
     648        command $run -limit $POLL_DEP
    612649    end
    613650
    … …  
    664701        book getword pstampDependent $pageName outdir     -var OUTDIR
    665702        book getword pstampDependent $pageName need_magic -var NEED_MAGIC
     703        book getword pstampDependent $pageName fault_count -var FAULT_COUNT
    666704        book getword pstampDependent $pageName dbname     -var DBNAME
    667705
    … …  
    682720        stderr $MYLOGFILE
    683721
    684         $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC
     722        $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT
    685723
    686724        add_standard_args run
    … …  
    705743            echo pstamp.job.run task.exit $DEP_ID status: $JOB_STATUS
    706744        end
    707         showcommand failure
     745#        showcommand failure
    708746        process_exit pstampDependent $options:0 $JOB_STATUS
    709747    end

  • branches/czw_branch/20101203/ippTasks/rcserver.pro

    r30586 r30587  
    4040    active false
    4141  end
    42   task rcserver.revert
    43     active false
    44   end
    4542end
    4643
    … …  
    4845  task rcserver.revert
    4946    active true
     47  end
     48end
     49
     50macro rcserver.revert.off
     51  task rcserver.revert
     52    active false
    5053  end
    5154end

  • branches/czw_branch/20101203/ippTasks/survey.pro

    r30586 r30587  
    11351135    end
    11361136   
    1137     echo $run
     1137    # echo $run
    11381138
    11391139    command $run

  • branches/czw_branch/20101203/ippToPsps/perl/ippToPsps/Gpc1Db.pm

    r29234 r30587  
    3434
    3535    return 1;
     36}
     37
     38###########################################################################
     39#
     40# Returns a count of warps thyat contributed to the stack with this stack ID
     41#
     42###########################################################################
     43sub countContributingWarpsToStack {
     44        my ($self, $stackId) = @_;
     45
     46          my $query = $self->{_db}->prepare(<<SQL);
     47          SELECT COUNT(DISTINCT warp_id)
     48              FROM stackInputSkyfile
     49              WHERE stack_id = $stackId
     50SQL
     51
     52    $query->execute;
     53    return scalar $query->fetchrow_array();
    3654}
    3755

  • branches/czw_branch/20101203/ippTools/share/chiptool_processedimfile.sql

    r29199 r30587  
    2929    rawImfile.magicked AS raw_magicked,
    3030    rawImfile.burntool_state,
    31     magicDSRun.state AS dsRun_state,
    32     IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id
     31    IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id,
     32    IFNULL(magicDSRun.state,0) AS dsRun_state,
     33    IFNULL(magicDSFile.fault,0) AS dsFile_fault,
     34    IFNULL(magicDSFile.data_state,0) AS dsFile_data_state
    3335FROM chipRun
    3436JOIN chipImfile
    … …  
    4345LEFT JOIN magicDSRun
    4446    ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place AND magicDSRun.state != 'drop'
     47LEFT JOIN magicDSFile ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id AND chipProcessedImfile.class_id = magicDSFile.component

  • branches/czw_branch/20101203/ippTools/share/difftool_pendingcleanupskyfile.sql

    r24186 r30587  
    99    USING(diff_id)
    1010WHERE
    11    ((diffRun.state = 'goto_cleaned'  AND diffSkyfile.data_state = 'full')
     11   ((diffRun.state = 'goto_cleaned'  AND (diffSkyfile.data_state = 'full' OR diffSkyfile.data_state = 'update'))
    1212    OR
    1313    (diffRun.state = 'goto_scrubbed' AND diffSkyfile.data_state != 'scrubbed')

  • branches/czw_branch/20101203/ippTools/share/difftool_setskyfiletoupdate.sql

    r29568 r30587  
    11UPDATE diffRun
    22    JOIN diffSkyfile USING(diff_id)
    3     LEFT JOIN magicDSRun ON (stage_id = diff_id AND stage = 'diff')
    4     LEFT JOIN magicDSFile ON (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    5                               AND component = skycell_id)
    63SET diffRun.state = 'update',
    7     diffSkyfile.data_state = 'update'
     4    diffSkyfile.data_state = 'update',
     5    diffSkyfile.fault = 0
    86    -- set hook %s
    97WHERE diff_id = %lld
    108    AND (diffRun.state = 'cleaned' OR diffRun.state = 'update')
    11     AND (diffSkyfile.data_state = 'cleaned')
     9    AND (diffSkyfile.data_state = 'cleaned' OR diffSkyfile.data_state = 'update')

  • branches/czw_branch/20101203/ippTools/share/magicdstool_setfiletoupdate.sql

    r29561 r30587  
    77WHERE magic_ds_id = %lld
    88    AND (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update')
    9     AND (magicDSFile.data_state = 'cleaned')
     9    AND (magicDSFile.data_state = 'cleaned' OR magicDSFile.data_state ='update')

  • branches/czw_branch/20101203/ippTools/share/warptool_change_skyfile_data_state.sql

    r26567 r30587  
    11-- handle changes in warpSkyfile.data_state.
    22-- Used for the modes tofullskyfile, tocleanedskyfile and topurgedskyfile
    3 -- args are new data_state, string for magic hook, warp_id, skycell_id and current expected state for warpRun
     3-- arguments are are new data_state, string for magic hook, warp_id, skycell_id
    44UPDATE warpSkyfile
    55JOIN warpRun USING(warp_id)

  • branches/czw_branch/20101203/ippTools/share/warptool_scmap.sql

    r29349 r30587  
    1212    IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id,
    1313    IFNULL(magicDSRun.state, 0) AS dsRun_state,
     14    IFNULL(magicDSFile.data_state, 0) AS dsFile_data_state,
    1415    IFNULL(magicDSFile.fault, 0) as dsFile_fault
    1516FROM warpRun

  • branches/czw_branch/20101203/ippTools/share/warptool_setskyfiletoupdate.sql

    r29568 r30587  
    11UPDATE warpRun
    22    JOIN warpSkyfile USING(warp_id)
    3     LEFT JOIN magicDSRun ON (stage_id = warp_id AND stage = 'warp')
    4     LEFT JOIN magicDSFile ON (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    5                               AND component = skycell_id)
    63SET warpRun.state = 'update',
    7     warpSkyfile.data_state = 'update'
     4    warpSkyfile.data_state = 'update',
     5    warpSkyfile.fault = 0
    86    -- set hook %s
    97WHERE warp_id = %lld
    108    AND (warpRun.state = 'cleaned' OR warpRun.state = 'update')
    11     AND (warpSkyfile.data_state = 'cleaned')
     9    AND (warpSkyfile.data_state = 'cleaned' OR warpSkyfile.data_state = 'update')

  • branches/czw_branch/20101203/ippTools/share/warptool_towarped.sql

    r29171 r30587  
    1111    exp_tag,
    1212    workdir,
     13    label,
    1314    magicked,
    1415    path_base,
    … …  
    2324        warpRun.state,
    2425        warpRun.reduction,
     26        warpRun.label,
    2527        camRun.cam_id,
    2628        rawExp.camera,
    … …  
    7678        warpRun.state,
    7779        warpRun.reduction,
     80        warpRun.label,
    7881        camRun.cam_id,
    7982        rawExp.camera,

  • branches/czw_branch/20101203/ippTools/src/chiptool.c

    r29982 r30587  
    838838    PXOPT_COPY_S16(config->args, where, "-fault", "chipProcessedImfile.fault", "==");
    839839
     840    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     841
    840842    if (!psListLength(where->list)
    841843        && !psMetadataLookupBool(NULL, config->args, "-all")) {
    … …  
    874876    }
    875877    psFree(query);
     878
     879    if (!fault) {
     880        // If fault has not been supplied, don't revert update faults with the magic value
     881        // We don't do this for new runs because then they would never complete
     882        // quality should be used to drop bad components
     883        psStringAppend(&query_update, " AND (chipProcessedImfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
     884    }
    876885    if (!p_psDBRunQuery(config->dbh, query_update)) {
    877886        psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    19191928    }
    19201929
     1930    // we do not update components with the magic fault value. They are non-updateable
     1931    psStringAppend(&query, " AND (chipProcessedImfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
     1932
    19211933    if (!p_psDBRunQueryF(config->dbh, query, setHook, chip_id)) {
    19221934        psError(PS_ERR_UNKNOWN, false, "database error");

  • branches/czw_branch/20101203/ippTools/src/difftool.c

    r30118 r30587  
    917917    PXOPT_COPY_S16(config->args, where, "-fault",     "fault", "==");
    918918
     919    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     920
    919921    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
    920922        psFree(where);
    … …  
    950952        psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh));
    951953
     954        if (!fault) {
     955            // If fault has not been supplied, don't revert update faults with the magic value
     956            // We don't do this for new runs because then they would never complete
     957            // quality should be used to drop bad components
     958            psStringAppend(&query_updated, " AND (diffSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
     959        }
    952960        if (!p_psDBRunQuery(config->dbh, query_updated)) {
    953961            psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    31983206        psStringAppend(&query, " AND (diffSkyfile.skycell_id = '%s')", skycell_id);
    31993207    }
     3208    // we do not update components with the magic fault value. They are non-updateable
     3209    // (But can be recovered with "difftool -revertwarped -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT)
     3210    psStringAppend(&query, " AND (diffSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
    32003211
    32013212    if (!p_psDBRunQueryF(config->dbh, query, setHook, diff_id)) {

  • branches/czw_branch/20101203/ippTools/src/disttool.c

    r30586 r30587  
    11851185    PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
    11861186    PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
    1187     PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
     1187    pxAddLabelSearchArgs(config, where, "-label", "label", "==");
    11881188    PXOPT_COPY_S16(config->args, where, "-fault", "rcDSFileset.fault", "==");
    11891189

  • branches/czw_branch/20101203/ippTools/src/disttoolConfig.c

    r29067 r30587  
    197197    psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-stage_id",0, "define stage_id", 0);
    198198    psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-state",   0, "define state", NULL);
    199     psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-label",   0, "define label", NULL);
     199    psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-label",   PS_META_DUPLICATE_OK, "define label", NULL);
    200200    psMetadataAddS16(revertfilesetArgs, PS_LIST_TAIL, "-fault",   0, "define fault code", 0);
    201201    psMetadataAddBool(revertfilesetArgs, PS_LIST_TAIL, "-all",    0, "revert all faulted runs", NULL);

  • branches/czw_branch/20101203/ippTools/src/magicdstool.c

    r30586 r30587  
    11121112
    11131113    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
     1114    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
    11141115
    11151116    psString queryFile = NULL;
     1117    bool stateIsUpdate = false;
    11161118    if (state) {
    11171119        if (! strcmp(state, "new") || !strcmp(state, "goto_restored")) {
    … …  
    11191121        } else if (!strcmp(state, "update")) {
    11201122            queryFile = "magicdstool_revertupdated.sql";
     1123            stateIsUpdate = true;
    11211124        } else {
    11221125            psError(PXTOOLS_ERR_SYS, true, "%s is not a valid value for state", state);
    … …  
    11411144    }
    11421145    psFree(where);
     1146
     1147    if (stateIsUpdate && !fault) {
     1148        // If fault has not been supplied, don't revert update faults with
     1149        // the magic "do not update" value
     1150        // We don't do this for new runs because then they would never complete
     1151        // quality should be used to drop bad components
     1152        psStringAppend(&query, " AND magicDSFile.fault != %d", PXTOOL_DO_NOT_REVERT_FAULT);
     1153    }
    11431154
    11441155    if (!p_psDBRunQuery(config->dbh, query)) {
    … …  
    17441755    }
    17451756
     1757    // we do not update components with the magic fault value. They are non-updateable
     1758    // (But can be recovered with "magicdstool -revertdestreakedfile -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT)
     1759    psStringAppend(&query, " AND (magicDSFile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
     1760
    17461761    if (!p_psDBRunQueryF(config->dbh, query, setHook, magic_ds_id)) {
    17471762        psError(PS_ERR_UNKNOWN, false, "database error");

  • branches/czw_branch/20101203/ippTools/src/magictool.c

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/czw_branch/20101203/ippTools/src/pstamptool.c

    r30586 r30587  
    571571    PXOPT_COPY_S32(config->args, where, "-fault",      "fault", "==");
    572572    PXOPT_COPY_STR(config->args, where, "-state",      "state", "==");
     573    PXOPT_COPY_STR(config->args, where, "-reqType",     "reqType", "==");
     574    PXOPT_COPY_STR(config->args, where, "-name",     "name", "LIKE");
    573575    PXOPT_COPY_TIME(config->args, where, "-timestamp_begin", "timestamp", ">=");
    574576    PXOPT_COPY_TIME(config->args, where, "-timestamp_end", "timestamp", "<=");
    … …  
    586588        psStringAppend(&query, ", state = '%s'", state);
    587589        if (!strcmp(state, "goto_cleaned")) {
    588             psStringAppend(&stateCheck, " AND state != 'cleaned'");
     590            psStringAppend(&stateCheck, " AND (state != 'cleaned' AND state != 'goto_cleaned')");
    589591        }
    590592    }

  • branches/czw_branch/20101203/ippTools/src/pstamptoolConfig.c

    r30586 r30587  
    104104    psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0,        "search by fault code", 0);
    105105    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0,        "search by state", NULL);
     106    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0,      "search by reqType", NULL);
     107    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0,      "search by reqType (LIKE comparsion)", NULL);
    106108    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL);
    107109    psMetadataAddTime(updatereqArgs, PS_LIST_TAIL, "-timestamp_begin", 0, "search by timestamp (>=)", NULL);

  • branches/czw_branch/20101203/ippTools/src/pxtools.h

    r28486 r30587  
    5151# define MAX_ROWS 10e9
    5252# define PXTOOL_MODE_NONE 0x0
     53// we do not revert or update components with this fault value
     54// This is the same value as PSTAMP_GONE
     55# define PXTOOL_DO_NOT_REVERT_FAULT 26
    5356
    5457bool pxIsValidState(const char *state);

  • branches/czw_branch/20101203/ippTools/src/regtool.c

    r30586 r30587  
    211211  // convert regular class_id format to summitImfile.class_id format
    212212  rep = psStringCopy(class_id);
    213   psFree(class_id);
     213
    214214  class_id = rep;
    215215 
    … …  
    320320  psStringSubstitute(&query,dateobs_end,"@DATEOBS_END@");
    321321
    322   // fprintf(stderr,"%s",query);
     322  fprintf(stderr,"%s",query);
    323323
    324324  if (!p_psDBRunQuery(config->dbh, query)) {
    … …  
    345345  }
    346346
    347   // fprintf (stderr, "found %ld rows\n", output->n);
     347  fprintf (stderr, "found %ld rows\n", output->n);
    348348  if (!psArrayLength(output)) {
    349349    psTrace("regtool", PS_LOG_INFO, "no rows found");
    … …  
    370370    char *tmp_id = psMetadataLookupStr(&status,row,"summit_class_id");
    371371    if (!status) {
    372       //      fprintf (stderr, "incomplete on %s\n", psMetadataLookupStr(NULL,row,"exp_name"));
     372            fprintf (stderr, "incomplete on %s\n", psMetadataLookupStr(NULL,row,"exp_name"));
    373373        continue;
    374374    }
    … …  
    440440        already_burned = false;
    441441
    442         //      fprintf (stderr, "missing uri: %s %s\n", psMetadataLookupStr(NULL,row,"exp_name"), this_class_id);
     442        fprintf (stderr, "missing uri: %s %s\n", psMetadataLookupStr(NULL,row,"exp_name"), this_class_id);
    443443
    444444        // Save this round for next round.
    … …  
    19031903  psFree(query);
    19041904  query = rep;
    1905   psStringSubstitute(&class_id,"ota","XY");
    19061905  psStringSubstitute(&query,class_id,"@CLASS_ID@");
    1907    
     1906  psStringSubstitute(&query,"ota","XY");
     1907 
    19081908  if (!date) {
    19091909    if (!dateobs_begin || !dateobs_end) {

  • branches/czw_branch/20101203/ippTools/src/warptool.c

    r29903 r30587  
    14851485    PXOPT_COPY_S16(config->args, where, "-fault",      "warpSkyfile.fault", "==");
    14861486
     1487    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     1488
    14871489    if (!psListLength(where->list)
    14881490        && !psMetadataLookupBool(NULL, config->args, "-all")) {
    … …  
    15101512    }
    15111513    psFree(where);
     1514
     1515    if (!fault) {
     1516        // If fault has not been supplied, don't revert update faults with the magic value
     1517        // We don't do this for new runs because then they would never complete
     1518        // quality should be used to drop bad components
     1519        psStringAppend(&query_updated, " AND warpSkyfile.fault != %d", PXTOOL_DO_NOT_REVERT_FAULT);
     1520    }
    15121521
    15131522    if (!p_psDBRunQuery(config->dbh, query)) {
    … …  
    20032012// shared code for the modes -tocleanedskyfile -tofullskyfile -topurgedskyfile
    20042013
    2005 static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state)
     2014static bool change_skyfile_data_state(pxConfig *config, psString data_state)
    20062015{
    20072016    PS_ASSERT_PTR_NON_NULL(config, false);
    … …  
    20652074static bool tocleanedskyfileMode(pxConfig *config)
    20662075{
    2067     return change_skyfile_data_state(config, "cleaned", "goto_cleaned");
     2076    return change_skyfile_data_state(config, "cleaned");
    20682077}
    20692078static bool tofullskyfileMode(pxConfig *config)
    20702079{
    2071     return change_skyfile_data_state(config, "full", "update");
     2080    return change_skyfile_data_state(config, "full");
    20722081}
    20732082static bool topurgedskyfileMode(pxConfig *config)
    20742083{
    2075     return change_skyfile_data_state(config, "purged", "goto_purged");
     2084    return change_skyfile_data_state(config, "purged");
    20762085}
    20772086static bool toscrubbedskyfileMode(pxConfig *config)
    20782087{
    2079      return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed");
     2088     return change_skyfile_data_state(config, "scrubbed");
    20802089}
    20812090
    … …  
    21012110    else {
    21022111      if (strcmp(state,"error_cleaned") == 0) {
    2103         change_skyfile_data_state(config,"error_cleaned","goto_cleaned");
     2112        change_skyfile_data_state(config,"error_cleaned");
    21042113      }
    21052114      else if (strcmp(state, "error_scrubbed") == 0) {
    2106         change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed");
     2115        change_skyfile_data_state(config,"error_scrubbed");
    21072116      }
    21082117      else if (strcmp(state, "error_purged") == 0) {
    2109         change_skyfile_data_state(config,"error_purged","goto_purged");
     2118        change_skyfile_data_state(config,"error_purged");
    21102119      }
    21112120      else {
    … …  
    25262535        psStringAppend(&query, " AND (warpSkyfile.skycell_id = '%s')", skycell_id);
    25272536    }
     2537    // we do not update components with the magic fault value. They are non-updateable
     2538    // (But can be recovered with "warptool -revertwarped -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT)
     2539    psStringAppend(&query, " AND (warpSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT);
    25282540
    25292541    if (!p_psDBRunQueryF(config->dbh, query, setHook, warp_id)) {

  • branches/czw_branch/20101203/ippconfig/recipes/filerules-split.mdc

    r30586 r30587  
    341341CENSOR.OUTPUT                OUTPUT {OUTPUT}.smf                      CMF             NONE       FPA        TRUE      MEF
    342342
    343 MAGIC.ORIGINAL.PNG           OUTPUT {OUTPUT}_original.png             JPEG            NONE       FPA        TRUE      NONE
    344 MAGIC.RESIDUAL.PNG           OUTPUT {OUTPUT}_residual.png             JPEG            NONE       FPA        TRUE      NONE
    345 MAGIC.MASK.PNG               OUTPUT {OUTPUT}_mask.png                 JPEG            NONE       FPA        TRUE      NONE
    346 MAGIC.DUPLICATE.PNG          OUTPUT {OUTPUT}_duplicate.png            JPEG            NONE       FPA        TRUE      NONE
     343MAGIC.ORIGINAL.PNG           OUTPUT {OUTPUT}.original.png             JPEG            NONE       FPA        TRUE      NONE
     344MAGIC.RESIDUAL.PNG           OUTPUT {OUTPUT}.residual.png             JPEG            NONE       FPA        TRUE      NONE
     345MAGIC.MASK.PNG               OUTPUT {OUTPUT}.mask.png                 JPEG            NONE       FPA        TRUE      NONE
     346MAGIC.DUPLICATE.PNG          OUTPUT {OUTPUT}.duplicate.png            JPEG            NONE       FPA        TRUE      NONE
    347347
    348348# TYPE                         OUTPUT FILENAME.RULE                     FILE.TYPE       FITS.TYPE  DATA.LEVEL FILE.SAVE FILE.FORMAT

  • branches/czw_branch/20101203/magic/remove/src/streaksreplace.c

    r30586 r30587  
    7575                if (sfiles->inWeight) {
    7676                    double recWeightValue = psImageGet(sfiles->recWeight->image, x, y);
    77                     if (recWeightValue != NAN) {
     77                    if (!isnan(recWeightValue)) {
    7878                        psImageSet(sfiles->inWeight->image, x, y, recWeightValue);
    7979                    }
    … …  
    8686
    8787    } while (streakFilesNextExtension(sfiles));
     88
     89    // check the weight and mask files for extra extensions that might be in files
     90    // (covariance matrix for example)
     91    copyExtraExtensions(sfiles);
    8892
    8993    // close all files
    … …  
    235239            usage();
    236240        }
    237         psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.MASK", 0,
     241        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY.MASK", 0,
    238242                "name of input mask image", argv[argnum]);
    239243        psArgumentRemove(argnum, &argc, argv);
    … …  
    251255            usage();
    252256        }
    253         psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.WEIGHT", 0,
     257        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY.WEIGHT", 0,
    254258                "name of input weight image", argv[argnum]);
    255259        psArgumentRemove(argnum, &argc, argv);
    … …  
    321325        addDestreakKeyword(sf->outMask->header, false);
    322326
    323         // XXX: TODO
    324327        setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false);
    325328
    … …  
    333336        sf->outWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header);
    334337        addDestreakKeyword(sf->outWeight->header, false);
    335         setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false);
     338        setupImageRefs(sf->outWeight, NULL, sf->inWeight, sf->extnum, false);
    336339
    337340        copyFitsOptions(sf->outWeight, NULL, sf->inWeight, sf->tiles);

  • branches/czw_branch/20101203/ppTranslate/src/ppMopsMerge.c

    r28623 r30587  
    3535
    3636    ppMopsDetections *merged = NULL;    // Merged list
    37     int num = 1;                                                         // Number of merged files
     37    int num = 0;                                                         // Number of merged files
    3838    for (int i = 0; i < detections->n; i++) {
    3939        ppMopsDetections *det = detections->data[i]; // Detections of interest
    … …  
    163163    }
    164164
     165    if (num == 0) {
     166        //All detections were NULL?!
     167        psTrace("ppMops.merge", 3, "All %d detections were NULL\n", detections->n);
     168        return NULL;
     169    }
    165170    psTrace("ppMops.merge", 2, "%ld sources in merged detections list\n", merged->num);
    166171

  • branches/czw_branch/20101203/psLib/src/imageops

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/czw_branch/20101203/psLib/test/math

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/czw_branch/20101203/psModules/src/detrend/pmFringeStats.c

    r25930 r30587  
    10081008        }
    10091009    }
     1010    // Allocate array of vectors to hold data.
     1011    psArray *bins = psArrayAlloc(4000);
     1012    for (int j = 0; j < bins->n; j++) {
     1013      bins->data[j] = psVectorAllocEmpty(1,PS_TYPE_F32);
     1014      //      psVector *v = psVectorAllocEmpty(1,PS_TYPE_F32);
     1015      //      bins = psArrayAdd(bins,1,v);
     1016    }
     1017
     1018    // Fill vectors
     1019    pmFringeStats *fringe = fringes->data[0];
     1020    for (int j = 0; j < numRegions; j++) {
     1021      if (mask->data.PS_TYPE_VECTOR_MASK_DATA[j] == 0) {
     1022        int array_bin = (int) ((fringe->f->data.F32[j] - -0.1) / 5e-5);
     1023        psVector *bin = bins->data[array_bin];
     1024        psVectorAppend(bin,science->f->data.F32[j]);
     1025      }
     1026    }
    10101027   
     1028    psVector *fringe_positions = psVectorAllocEmpty(4000,PS_TYPE_F32);
     1029    psVector *science_values   = psVectorAllocEmpty(4000,PS_TYPE_F32);
     1030    psVector *science_errors   = psVectorAllocEmpty(4000,PS_TYPE_F32);
     1031
     1032    psStats *binStats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV);
     1033    for (int i = 0; i < 4000; i++) {
     1034      psVector *bin = bins->data[i];
     1035      if (bin->n > 2) {
     1036        psStatsInit(binStats);
     1037
     1038        psVectorStats(binStats,bin,NULL,NULL,1);
     1039       
     1040        if (isfinite(binStats->robustStdev) &&
     1041            isfinite(binStats->robustMedian) &&
     1042            binStats->robustStdev > 0) {
     1043          psVectorAppend(fringe_positions,-0.1 + i * 5e-5);
     1044          psVectorAppend(science_values, binStats->robustMedian);
     1045          psVectorAppend(science_errors, binStats->robustStdev);
     1046        }
     1047      }
     1048      psFree(bins->data[i]);
     1049    }
     1050    psFree(bins);
     1051    psFree(binStats);
     1052
     1053    for (int i = 0; i < fringe_positions->n; i++) {
     1054      psTrace("psModules.detrend",7,"FITDATA: %f %f %f\n",
     1055              fringe_positions->data.F32[i],
     1056              science_values->data.F32[i],
     1057              science_errors->data.F32[i]);
     1058    }
    10111059/*     // Begin switch from old outlier removal and fitting code. */
    10121060
    10131061    psPolynomial1D *poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1);
    10141062
    1015     pmFringeStats *fringe = fringes->data[0];
    1016     psVector *errors = psVectorAlloc(science->df->n,PS_TYPE_F32);
    1017     for (int j = 0; j < errors->n; j++) {
    1018       errors->data.F32[j] = 1 / science->df->data.F32[j];
    1019     }
    1020     psVectorFitPolynomial1D(poly,mask,0xff,science->f,errors,fringe->f);
    1021 
     1063    //    pmFringeStats *fringe = fringes->data[0];
     1064/*     psVector *errors = psVectorAlloc(science->df->n,PS_TYPE_F32); */
     1065/*     for (int j = 0; j < errors->n; j++) { */
     1066/*       errors->data.F32[j] = 1 / science->df->data.F32[j]; */
     1067/*     } */
     1068/*     psVectorFitPolynomial1D(poly,mask,0xff,science->f,errors,fringe->f); */
     1069    psVectorFitPolynomial1D(poly,NULL,0xff,science_values,science_errors,fringe_positions);
     1070    psFree(fringe_positions);
     1071    psFree(science_values);
     1072    psFree(science_errors);
     1073   
    10221074    for (int i = 0; i <= poly->nX; i++) {
    10231075      scale->coeff->data.F32[i] = poly->coeff[i];
    … …  
    10271079    psFree(poly);
    10281080    //    psFree(fringe);
    1029     psFree(errors);
     1081    //    psFree(errors);
    10301082
    10311083    psFree(median);

  • branches/czw_branch/20101203/psModules/src/objects

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/czw_branch/20101203/psphot/src/psphotApResid.c

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/czw_branch/20101203/psphot/src/psphotEfficiency.c

    r30118 r30587  
    3131    assert(covarFactor);
    3232
     33    // apply the amplitude of kernel^2 to the covarFactor (why?)
     34    // XXX this is simply undoing the scale scalculate in psImageCovarianceCalculateFactor
     35    // if this is the right solution, make this extra calculation optional (and explain...)
    3336    psKernel *kernel = psImageSmoothKernel(smoothSigma, smoothNsigma); // Kernel used for smoothing
    34     *covarFactor = psImageCovarianceCalculateFactor(kernel, ro->covariance);
     37    double sum2 = 0.0;                                               // Sum of kernel squared
     38    for (int y = kernel->yMin; y <= kernel->yMax; y++) {
     39        for (int x = kernel->xMin; x <= kernel->xMax; x++) {
     40            sum2 += PS_SQR(kernel->kernel[y][x]);
     41        }
     42    }
     43    *covarFactor = sum2 * psImageCovarianceCalculateFactor(kernel, ro->covariance);
    3544    psFree(kernel);
    3645
    … …  
    7079    float fluxLim = 2.0 * *norm * peakLim; // Limiting flux in original
    7180    *magLim = -2.5 * log10f(fluxLim);
     81    psTrace("psphot.fake", 1, "Covar Factor:  %f\n", *covarFactor);
    7282    psTrace("psphot.fake", 1, "Limiting peak: %f\n", peakLim);
    7383    psTrace("psphot.fake", 1, "Limiting flux: %f\n", fluxLim);

  • branches/czw_branch/20101203/psphot/src/psphotMakeFluxScale.c

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/czw_branch/20101203/psphot/src/psphotOutput.c

    r30118 r30587  
    307307    psMetadataItemSupplement (&status, header, analysis, "PSPHOT.CR.MAX.MAG");
    308308
     309    psMetadataItemSupplement (&status, header, analysis, "EFFECTIVE_AREA");
     310    psMetadataItemSupplement (&status, header, analysis, "SIGNIFICANCE_SCALE_FACTOR");
     311
    309312    // sky background model statistics
    310313    psMetadataItemSupplement (&status, header, analysis, "MSKY_MN");

  • branches/czw_branch/20101203/pstamp/scripts/psstatus

    r30586 r30587  
    1313my $running;
    1414my $finished;
    15 my $req_faulted;
    16 my $job_faulted;
     15my $finishing;
     16my $totals;
    1717my $dbname;
    1818my $dbserver;
    … …  
    2020my $dbpassword;
    2121my $verbose;
    22 my $finishing;
    2322
    2423GetOptions(
    … …  
    2625    'finished|f',    \$finished,
    2726    'finishing',     \$finishing,
    28     'req_faulted',   \$req_faulted,
    29     'job_faulted',   \$job_faulted,
     27    'totals|t',      \$totals,
    3028    'limit|l=i',     \$limit,
    3129    'dbname=s',      \$dbname,
    … …  
    3533pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV ;
    3634
    37 $running = 1 if (!$finished and !$finishing and !$running);
    38 
    39 my $no_args = ! (defined $running or defined $finished or defined $req_faulted or defined $job_faulted );
     35$running = 1 if (!$finished and !$finishing and !$running and !$totals);
     36
     37my $no_args = ! (defined $running or defined $finished or defined $finishing or defined $totals );
    4038
    4139die "cannot supply --running and --finished\n" if ($running and $finished);
    … …  
    4442my $ipprc =  PS::IPP::Config->new();
    4543my $dbh = getDBHandle();
     44
     45my $totals_query = "
     46SELECT label, count(job_id) AS 'Unfinished Jobs', priority
     47FROM pstampRequest join pstampJob USING(req_id)
     48JOIN Label USING(label)
     49WHERE pstampJob.state ='run' and pstampRequest.state ='run'
     50GROUP by label";
    4651
    4752my $running_query = "
    … …  
    5156    label,
    5257    reqType,
    53     priority,
     58   -- priority,
    5459    state,
    5560    fault,
    … …  
    233238
    234239my $sql;
    235 if ($finished) {
     240if ($totals) {
     241    $sql = $totals_query;
     242} elsif ($finished) {
    236243    $sql = $finished_query;
    237244} elsif ($running) {

  • branches/czw_branch/20101203/pstamp/scripts/pstamp_checkdependent.pl

    r30586 r30587  
    44#
    55# Check the status of a pending pstampDependent insuring that
    6 # the dependent processing has been queued and whether it is
    7 # finished or not
     6# the any update processing that is needed has been queued and determine check
     7# whether the processing has  finished or not
    88
    99use warnings;
    … …  
    2727my $IPP_DIFF_MODE_STACK_STACK = 4;
    2828
    29 my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic);
    30 my ($dbname, $dbserver, $verbose, $save_temps, $no_update);
     29my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count);
     30my ($dbname, $ps_dbserver, $verbose, $save_temps, $no_update);
    3131
    3232GetOptions(
    … …  
    3535    'stage_id=i'    =>  \$stage_id,
    3636    'component=s'   =>  \$component,
    37     'imagedb=s'     =>  \$imagedb,      # dbname for images.
     37    'imagedb=s'     =>  \$imagedb,      # dbname for images lookups.
    3838    'rlabel=s'      =>  \$rlabel,
    3939    'need_magic'    =>  \$need_magic,
     40    'fault_count=i' =>  \$fault_count,
     41    'max_fault_count=i' =>  \$max_fault_count,
    4042    'dbname=s'      =>  \$dbname,       # postage stamp server dbname
    41     'dbserver=s'    =>  \$dbserver,     # postage stamp server dbserver
     43    'dbserver=s'    =>  \$ps_dbserver,  # postage stamp server dbserver
    4244    'verbose'       =>  \$verbose,
    4345    'save-temps'    =>  \$save_temps,
    … …  
    4850    if !(defined $dep_id and defined $stage and defined $stage_id and
    4951        defined $component and defined $imagedb);
     52
     53$max_fault_count = 5 if !$max_fault_count;
     54$fault_count = 0 if !defined $fault_count;
    5055
    5156my $missing_tools;
    … …  
    6166}
    6267
     68my $ipprc = PS::IPP::Config->new();
     69
     70if (!$ps_dbserver) {
     71    $ps_dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
     72}
     73$pstamptool  .= " -dbname $dbname" if $dbname;
     74$pstamptool  .= " -dbserver $ps_dbserver";
     75
    6376# Append imagedb to the ippTools
     77# Note: configured DBSERVER is used for this server
    6478$chiptool    .= " -dbname $imagedb";
    6579$warptool    .= " -dbname $imagedb";
    … …  
    6882$magicdstool .= " -dbname $imagedb";
    6983
    70 my $ipprc = PS::IPP::Config->new();
    71 
    72 if (!$dbserver) {
    73     $dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
    74 }
    75 
    7684
    7785my $tool;
    7886my $cmd;
    7987my $dsRun_state = "";
    80 my $whole_run = ($component eq 'all');
    81 
    82 # XXX: whole_run was a concept that isn't practical. Having one dependent for
    83 # a whole run makes finding errors too hard. We always have a dependent for each
    84 # component.
    85 # XXX: remove the unneeded code
    86 my_die("component = 'all' not supported", $PS_EXIT_PROG_ERROR) if $whole_run;;
    87 
    88 if ($whole_run) {
    89     if ($stage eq "chip") {
    90         $cmd = "$chiptool -listrun -chip_id $stage_id";
    91     } elsif ($stage eq "warp") {
    92         $cmd = "$warptool -listrun -warp_id $stage_id";
    93     } elsif ($stage eq "diff") {
    94         $cmd = "$difftool -listrun -diff_id $stage_id";
    95     } else {
    96         my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR);
    97     }
     88
     89if ($stage eq "chip") {
     90    $cmd = "$chiptool -processedimfile -allfiles -chip_id $stage_id -class_id $component";
     91} elsif ($stage eq "warp") {
     92    $cmd = "$warptool -warped -warp_id $stage_id -skycell_id $component";
     93} elsif ($stage eq "diff") {
     94    $cmd = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $component";
    9895} else {
    99     if ($stage eq "chip") {
    100         $cmd = "$chiptool -processedimfile -allfiles -chip_id $stage_id -class_id $component";
    101     } elsif ($stage eq "warp") {
    102         $cmd = "$warptool -warped -warp_id $stage_id -skycell_id $component";
    103     } elsif ($stage eq "diff") {
    104         $cmd = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $component";
    105     } else {
    106         my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR);
    107     }
     96    my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR);
    10897}
    10998
    … …  
    124113}
    125114my $status = 0;
    126 if (($it->{state} eq 'full') or ($it->{state} eq 'update') and ($whole_run or ($it->{data_state} eq 'full'))
     115if ((($it->{state} eq 'full') or ($it->{state} eq 'update')) and ($it->{data_state} eq 'full')
    127116        and (!$need_magic or $magic_ok or $it->{magicked} > 0)) {
    128117
    129     # This Dependency is satisfied. All done!
     118    # This Dependency is satisfied. All done. Release the pstampJobs
     119    #
    130120    my $command = "$pstamptool -updatedependent -set_state full -dep_id $dep_id";
    131     $command .= " -dbname $dbname" if $dbname;
    132     $command .= " -dbserver $dbserver" if $dbserver;
    133121    if (!$no_update) {
    134122        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    143131} elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update')) {
    144132    #       For warp and diff stages we need to call the 'queue_update' subroutines even if the
    145     #       data_state is update in order to check the earlier stages in the pipeline
    146     #       For example if warpSkyfile is in update state but the chip run that it depends on hasn't
    147     #       been updated we need to go and queue it.
    148 
    149     if (($it->{state} ne 'cleaned') and $it->{fault}) {
    150         my_die("Component faulted on update dep_id: $dep_id",
    151                 $PS_EXIT_SYS_ERROR);
     133    #       data_state is update in order to check the state of inputs in earlier stages in the pipeline
     134    #       For example if warpSkyfile is in update state but the chipRun that it depends on hasn't
     135    #       been updated we need to go and queue the chips for processing.
     136
     137    my $fault = $it->{fault};
     138    if (($fault eq $PSTAMP_GONE) or (($it->{state} eq 'update') and $fault)) {
     139        $fault_count++;
     140        print "$stage $stage_id $component has fault $fault\n";
     141        if ($it->{fault} eq $PSTAMP_GONE) {
     142            faultJobs($PSTAMP_GONE);
     143            exit 0;
     144        } elsif ($fault_count >= $max_fault_count) {
     145            print "$stage $stage_id $component has faulted $fault_count times. Giving up\n";
     146
     147            faultComponent($stage, $stage_id, $component, $PSTAMP_GONE);
     148
     149            # fault the jobs
     150            faultJobs($PSTAMP_GONE);
     151            exit 0;
     152        }
     153
     154        # assume the fault is transient.
     155        my_die("Component faulted on update dep_id: $dep_id", $PS_EXIT_SYS_ERROR);
    152156    }
    153157
    154158    if ($stage eq 'chip') {
    155         # check_states_chip takes an array so that check_states_warp can pass it set of chips
     159        # check_states_chip takes an array so that check_states_warp can pass it a set of chips
    156160        my $chips = [$it];
    157         $status = check_states_chip($it->{chip_id}, $whole_run, $chips, $rlabel, $need_magic);
     161        $status = check_states_chip($it->{chip_id}, $chips, $rlabel, $need_magic);
    158162    } elsif ($stage eq 'warp') {
    159         $status = check_states_warp($it, $whole_run, $rlabel, $need_magic);
     163        $status = check_states_warp($it, $rlabel, $need_magic);
    160164    } elsif ($stage eq 'diff') {
    161         $status = check_states_diff($it, $whole_run, $rlabel, $need_magic);
     165        $status = check_states_diff($it, $rlabel, $need_magic);
    162166    } else {
    163167        my_die("Unexpected stage found $stage", $PS_EXIT_PROG_ERROR);
     168    }
     169    if ($status >= $PSTAMP_FIRST_ERROR_CODE) {
     170        faultJobs($status);
    164171    }
    165172} else {
    … …  
    173180    my $job_fault = 0;
    174181
    175 if (0) {
    176     if ($stage eq 'chip') {
    177         # XXX: There is no need to check this anymore. All magicked chipRuns have abs(burntool_state) >= 13
    178         # If something changes that causes an error, we will figure that out from the chip failure
    179         my $burntool_state = $it->{burntool_state};
    180         # XXX: get the value of 13 from the ppImage recipe
    181         if ($burntool_state and (abs($burntool_state) < 13)) {
    182             print STDERR "chip $it->{chip_id} $it->{class_id} has burntool_state $burntool_state. Not avaiable.\n";
    183             $job_fault = $PSTAMP_NOT_AVAILABLE;
    184         }
    185     }
    186 }
    187182    if ($state eq 'error_cleaned') {
    188183        $job_fault = $PSTAMP_NOT_AVAILABLE;
    189     } elsif (($state =~ /scrub/) or ($state =~ /purge/)) {
    190         # jobs must have changed state since depenency was made
     184    } elsif (($state =~ /scrub/) or ($state =~ /purge/) or ($state eq 'drop')) {
     185        # Component state must have been changed state since dependency was inserted.
    191186        print STDERR "Dependency cannot be satisfied\n";
    192187        $job_fault = $PSTAMP_GONE;
    … …  
    195190        my_die ("Unexpected state for ${stage}Run $stage_id $state", $PS_EXIT_PROG_ERROR);
    196191    }
     192
    197193    if (!$job_fault and ($stage eq 'chip')) {
     194        # should only get here with data_state 'full' and perhaps destreaking not done
     195        my_die ("Unexpected state for ${stage}Run $stage_id $state", $PS_EXIT_PROG_ERROR)
     196            if $it->{data_state} ne 'full';
     197
    198198        # chip processing is done, start destreaking.
    199199        my @chips;
    … …  
    201201        $job_fault = check_states_magicDSRun($stage, $stage_id, \@chips, $rlabel, $need_magic, $it->{raw_magicked}, $it->{magic_ds_id}, $it->{dsRun_state});
    202202    }
    203     if ($job_fault) {
    204         faultJobs($state, $stage, $stage_id, $job_fault);
     203
     204    if ($job_fault >= $PSTAMP_FIRST_ERROR_CODE) {
     205        faultJobs($job_fault);
    205206    }
    206207}
    … …  
    211212sub check_states_chip {
    212213    my $chip_id = shift;
    213     my $whole_run = shift;  # if true queue entire run for update
    214     my $metadatas = shift;  # an array of hashes, either from -processedimfile or -listrun
     214    my $metadatas = shift;  # an array of hashes, either from chiptool -processedimfile or warptool -scmap
    215215    my $rlabel = shift;     # if defined a new label for the chipRun
    216216    my $need_magic = shift;
    217 
    218217
    219218    my $dsRun_state;
    … …  
    221220    my @chips;
    222221    my $magic_ds_id;
    223     if (!$whole_run) {
    224         foreach my $chip (@$metadatas) {
    225             $dsRun_state = $chip->{dsRun_state};
    226             $raw_all_magicked &= ($chip->{raw_magicked} > 0);
    227             $magic_ds_id = $chip->{magic_ds_id};
    228 
    229             push @chips, $chip->{class_id};
    230 
    231             my $state = $chip->{state};
    232             my $data_state = $chip->{data_state};
    233             if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
    234                 ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/)) {
    235 
    236                 print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state cannot update\n";
    237                 faultJobs('stop', undef, undef, $PSTAMP_GONE);
    238 
    239                 return 0;
    240             } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
    241 
    242                 my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id -class_id $chip->{class_id}";
    243                 $command .= " -set_label $rlabel" if $rlabel;
    244 
    245                 if (!$no_update) {
    246                     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    247                                 run(command => $command, verbose => $verbose);
    248                     unless ($success) {
    249                         my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
    250                     }
    251                 } else {
    252                     print "skipping $command\n";
    253                 }
    254             } elsif ($chip->{fault}) {
    255                 # fault the dependent
    256                 my_die("chip $chip->{chip_id} $chip->{class_id} faulted: $chip->{fault}", $chip->{fault});
    257             } elsif ($chip->{dsFile_fault}) {
    258                 # fault the dependent
    259                 my_die("magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} faulted: $chip->{dsFile_fault}", $chip->{dsFile_fault});
    260             }
    261         }
    262     } else {
    263         my $run = $metadatas->[0];
    264         $dsRun_state = $run->{dsRun_state};
    265         $raw_all_magicked = ($run->{raw_magicked} > 0);
    266         my $state = $run->{state};
    267         if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/)) {
    268             print "chipRun state is $run->{chip_id} is in state $state cannot update\n";
    269             faultJobs('stop', undef, undef, $PSTAMP_GONE);
    270             return 0;
    271         }
    272 
    273         # providing no -class_id arguments changes all imfiles with data_state = 'cleaned' to 'update'
    274         my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id";
    275         $command .= " -set_label $rlabel" if $rlabel;
    276 
    277         if (!$no_update) {
    278             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    279                         run(command => $command, verbose => $verbose);
    280             unless ($success) {
    281                 my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
    282             }
    283         } else {
    284             print "skipping $command\n";
    285         }
    286     }
    287 
    288     my $status = check_states_magicDSRun('chip', $chip_id, \@chips, $rlabel, $need_magic, $raw_all_magicked, $magic_ds_id, $dsRun_state);
    289 
    290     return $status;
    291 }
    292 
    293 sub check_states_warp {
    294     # check status of input chips. If state is not updatable set error code for job
    295 
    296     # if chipProcessedImfile.state is cleaned call check_states_chip
    297 
    298     my $metadata = shift;
    299     my $whole_run = shift;  # if true queue entire run for update
    300     my $rlabel = shift;     # if defined a new label for the chipRun
    301     my $need_magic = shift;
    302 
    303     my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked
    304 
    305     my $warp_id = $metadata->{warp_id};
    306     my $skycell_id = $metadata->{skycell_id};
    307     my $state = $metadata->{state};
    308     my $data_state = $metadata->{data_state};
    309     if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
    310          ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/)) {
    311         print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs\n";
    312         faultJobs('stop', undef, undef, $PSTAMP_GONE);
    313         exit 0;
    314     }
    315     if (($state eq 'update') and ($metadata->{fault})) {
    316         # fault dependent.
    317         my $fault = $metadata->{fault};
    318         my_die("warp $warp_id $skycell_id faulted: $fault", $fault);
    319     }
    320 
    321     if (!$whole_run) {
    322         my $skycell = $metadata;
    323         my $skycell_id = $skycell->{skycell_id};
    324 
    325         my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id";
    326         my $data = runToolAndParse($command, $verbose);
    327         if (!$data or scalar @$data == 0) {
    328             # This happens if the chipProcessedImfile disappears which happened when earlier
    329             # versions of chiptool -revertprocessedimfile didn't check the chipRun.state before
    330             # deleing the row.
    331             # Fault the jobs so that the Request can finish ...
    332             faultJobs('stop', undef, undef, $PSTAMP_GONE);
    333             # ... and fault the dependent so that we have a record of the error
    334             my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id",
    335                 $PSTAMP_GONE);
    336         }
    337 
    338 
    339         my $chips_ready = 1;
    340         my @chipsToUpdate;
    341         my $chip_id;
    342         foreach my $chip (@$data) {
    343             $chip_id = $chip->{chip_id};
    344             if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
    345                 $chips_ready = 0;
    346                 $chip->{fault} = $chip->{chip_fault};
    347                 push @chipsToUpdate, $chip;
     222
     223    my $queued_update = 0;
     224    foreach my $chip (@$metadatas) {
     225        $dsRun_state = $chip->{dsRun_state};
     226        $raw_all_magicked &= ($chip->{raw_magicked} > 0);
     227        $magic_ds_id = $chip->{magic_ds_id};
     228
     229        push @chips, $chip->{class_id};
     230
     231        my $state = $chip->{state};
     232        my $data_state = $chip->{data_state};
     233        if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
     234            ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) {
     235
     236            print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state cannot update\n";
     237            my $error_code;
     238            if (($state eq 'error_cleaned') or ($data_state ='error_cleaned')) {
     239                $error_code = $PSTAMP_NOT_AVAILABLE;
    348240            } else {
    349                 # this chip is good to go
    350             }
    351         }
    352 
    353         if ($chips_ready and $skycell->{data_state} ne 'update') {
    354             # the reason we defer setting the warp to update is so that we can handle error conditions at previous
    355             # stages more easily.
    356             my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
     241                $error_code = $PSTAMP_GONE;
     242            }
     243           
     244            # caller will fault the jobs
     245            return $error_code;
     246        } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
     247
     248            # chiptool does more state checking to insure this isn't done prematurely.
     249            my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id -class_id $chip->{class_id}";
    357250            $command .= " -set_label $rlabel" if $rlabel;
    358251
    … …  
    366259                print "skipping $command\n";
    367260            }
    368         } elsif (scalar @chipsToUpdate > 0) {
    369             return check_states_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic);
    370         }
    371     } else {
    372         my $warpRun = $metadata;
    373         my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}";
    374         my $data = runToolAndParse($command, $verbose);
    375         my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR)
    376             if !$data or scalar @$data != 1;
    377 
    378         my $chipRun = $data->[0];
    379 
    380         my $chipRunState = $chipRun->{state};
    381         if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) {
    382             print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n";
    383             faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE);
    384             return 0;
    385         }
    386         my $warpRunState = $warpRun->{state};
    387         if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) {
    388             # The inputs and outputs are ready. Queue the warpRun for update.
    389 
    390             # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update'
    391             my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id";
    392             $command .= " -set_label $rlabel" if $rlabel;
    393 
    394             if (!$no_update) {
    395                 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    396                             run(command => $command, verbose => $verbose);
    397                 unless ($success) {
    398                     my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
    399                 }
    400             } else {
    401                 print "skipping $command\n";
    402             }
    403         } elsif ($chipRunState eq 'cleaned' or
    404             (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) {
    405             my $data = [$chipRun];
    406             return check_states_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic);
    407         }
    408     }
    409 
    410     # return value may be the return status of script so zero is good
    411     return 0;
    412 }
     261            $queued_update = 1;
     262        } elsif ($chip->{fault}) {
     263            $fault_count++;
     264            my $fault =  $chip->{fault};
     265
     266            if ($fault eq $PSTAMP_GONE) {
     267                # caller will fault jobs
     268                return $PSTAMP_GONE;
     269            } elsif ($fault_count > $max_fault_count) {
     270                print "$stage $stage_id has faulted $fault_count times. Giving up\n";
     271                $fault = $PSTAMP_GONE;
     272                faultComponent('chip', $chip->{chip_id}, $chip->{class_id}, $PSTAMP_GONE);
     273                return $PSTAMP_GONE;
     274            }
     275            # fault the dependent
     276            my_die("chip $chip->{chip_id} $chip->{class_id} faulted: $chip->{fault}", $chip->{fault});
     277        } elsif ($chip->{dsFile_fault} eq $PSTAMP_GONE) {
     278            print STDERR "magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} is GONE";
     279            return $PSTAMP_GONE;
     280        } elsif ($chip->{dsFile_fault} and ($chip->{dsFile_data_state} eq 'update')) {
     281            # fault the dependent
     282            my_die("magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} faulted: $chip->{dsFile_fault}", $chip->{dsFile_fault});
     283        }
     284    }
     285
     286    my $status = 0;
     287    if (!$queued_update) {
     288        $status = check_states_magicDSRun('chip', $chip_id, \@chips, $rlabel, $need_magic, $raw_all_magicked, $magic_ds_id, $dsRun_state);
     289    }
     290
     291    return $status;
     292}
     293
     294sub check_states_warp {
     295    my $metadata = shift;
     296    my $rlabel = shift;     # if defined a new label for the chipRun
     297    my $need_magic = shift;
     298   
     299    my $exit_status = 0;
     300
     301    my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked
     302
     303    my $warp_id = $metadata->{warp_id};
     304    my $skycell_id = $metadata->{skycell_id};
     305    my $state = $metadata->{state};
     306    my $data_state = $metadata->{data_state};
     307
     308    if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
     309         ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) {
     310        print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs\n";
     311        my $error_code;
     312        if (($state eq 'error_cleaned') or ($data_state eq 'error_cleaned')) {
     313            $error_code = $PSTAMP_NOT_AVAILABLE;
     314        } else {
     315            $error_code = $PSTAMP_GONE;
     316        }
     317        return $error_code
     318    }
     319    if (($state eq 'update') and ($metadata->{fault})) {
     320        # fault dependent.
     321        my $fault = $metadata->{fault};
     322        print STDERR "warp $warp_id $skycell_id faulted: $fault";
     323        return $fault;
     324    }
     325
     326    my $skycell = $metadata;
     327
     328    # get the list of input chips for this skycell
     329    my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id";
     330    my $data = runToolAndParse($command, $verbose);
     331    if (!$data or scalar @$data == 0) {
     332        # This happens if the chipProcessedImfile disappears which happened when earlier
     333        # versions of chiptool -revertprocessedimfile didn't check the chipRun.state before
     334        # deleting the row.
     335        print STDERR "failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id";
     336        return $PSTAMP_GONE;
     337    }
     338
     339    my $chips_ready = 1;
     340    my @chipsToUpdate;
     341    my $chip_id;
     342    foreach my $chip (@$data) {
     343        $chip_id = $chip->{chip_id};
     344        if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
     345            $chips_ready = 0;
     346            $chip->{fault} = $chip->{chip_fault};
     347            push @chipsToUpdate, $chip;
     348        } else {
     349            # this chip is done
     350        }
     351    }
     352
     353    if ($chips_ready and $skycell->{data_state} ne 'update') {
     354        # the reason we defer setting the warp to update is so that we can handle error conditions at previous
     355        # stages more easily.
     356        my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
     357        $command .= " -set_label $rlabel" if $rlabel;
     358
     359        if (!$no_update) {
     360            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     361                        run(command => $command, verbose => $verbose);
     362            unless ($success) {
     363                my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
     364            }
     365        } else {
     366            print "skipping $command\n";
     367        }
     368    } elsif (scalar @chipsToUpdate > 0) {
     369        my $fault = check_states_chip($chip_id, \@chipsToUpdate, $rlabel, $need_magic);
     370        if ($fault) {
     371            if ($fault eq $PSTAMP_GONE) {
     372                # chip or dsfile that this skycell depends on has faulted in a way that is not recoverable
     373                # fault the skycell
     374                faultComponent('warp', $warp_id, $skycell->{skycell_id}, $PSTAMP_GONE);
     375            }
     376            $exit_status = $fault;
     377        }
     378    }
     379
     380    # return value may be used as the return status of script so we use zero as success
     381    return $exit_status;
     382}
     383
    413384sub check_states_diff {
    414385    my $metadata = shift;
    415     my $whole_run = shift;  # if true queue entire run for update
    416386    my $rlabel = shift;     # if defined a new label for the chipRun
    417387    my $need_magic = shift;
    … …  
    419389    my $diff_id   = $metadata->{diff_id};
    420390    my $diff_mode = $metadata->{diff_mode};
    421     if (!$whole_run) {
    422         my $skycell = $metadata;
    423         my $skycell_id = $skycell->{skycell_id};
    424 
    425         if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {
    426             # check the state of the template stack
    427             my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
    428             my $stack = runToolAndParseExpectOne($command, $verbose);
    429             my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack;
    430 
    431             if ($stack->{state} ne 'full') {
    432                 print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
    433                 # this faults all jobs depending on this dep_id
    434                 faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE);
    435                 return $PSTAMP_GONE;
    436             }
    437 
    438             # now check the warp
    439             $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
    440             my $warp = runToolAndParseExpectOne($command, $verbose);
    441             my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
    442 
    443             if ($warp->{data_state} ne 'full') {
    444                 return check_states_warp($warp, 0, $rlabel, $need_magic);
    445             }
    446             # warps are ready fall through and queue the diff update
    447         } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) {
    448             my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
    449             my $warp1 = runToolAndParseExpectOne($command, $verbose);
    450             my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1;
    451 
    452             my $warps_ready = 1;
    453             my $status = 0;
    454             if ($warp1->{data_state} ne 'full') {
    455                 $warps_ready = 0;
    456                 $status = check_states_warp($warp1, 0, $rlabel, $need_magic);
    457                 if ($status) {
    458                     return $status;
     391    my $skycell = $metadata;
     392    my $skycell_id = $skycell->{skycell_id};
     393
     394    if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {
     395        # check the state of the template stack
     396        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
     397        my $stack = runToolAndParseExpectOne($command, $verbose);
     398        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack;
     399
     400        if ($stack->{state} ne 'full') {
     401            print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
     402            faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
     403            return $PSTAMP_GONE;
     404        }
     405
     406        # now check the warp
     407        $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
     408        my $warp = runToolAndParseExpectOne($command, $verbose);
     409        my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
     410
     411        if ($warp->{data_state} ne 'full') {
     412            my $warp_status = check_states_warp($warp, 0, $rlabel, $need_magic);
     413            if ($warp_status eq $PSTAMP_GONE) {
     414                faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
     415            }
     416            return $warp_status;
     417        }
     418        # warps are ready fall through and queue the diff update
     419    } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) {
     420        my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
     421        my $warp1 = runToolAndParseExpectOne($command, $verbose);
     422        my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1;
     423
     424        my $warps_ready = 1;
     425        my $warp_status = 0;
     426        if ($warp1->{data_state} ne 'full') {
     427            $warps_ready = 0;
     428            $warp_status = check_states_warp($warp1, 0, $rlabel, $need_magic);
     429            if ($warp_status) {
     430                if ($warp_status eq $PSTAMP_GONE) {
     431                    faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
    459432                }
    460             }
    461             $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
    462             my $warp2 = runToolAndParseExpectOne($command, $verbose);
    463             my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2;
    464 
    465             if ($warp2->{data_state} ne 'full') {
    466                 $warps_ready = 0;
    467                 $status = check_states_warp($warp2, 0, $rlabel, $need_magic);
    468             }
    469 
    470             if (!$warps_ready) {
    471                 # don't queue the diff update yet
    472                 return $status;
    473             }
    474 
    475         } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_STACK ) {
    476             # check the state of the input stack
    477             my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
    478             my $stack1 = runToolAndParseExpectOne($command, $verbose);
    479             my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1;
    480 
    481             if ($stack1->{state} ne 'full') {
    482                 print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
    483                 # this faults all jobs depending on this dep_id
    484                 faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE);
    485                 return $PSTAMP_GONE;
    486             }
    487             # check the state of the template stack
    488             $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
    489             my $stack2 = runToolAndParseExpectOne($command, $verbose);
    490             my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack2;
    491 
    492             if ($stack2->{state} ne 'full') {
    493                 print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
    494                 # this faults all jobs depending on this dep_id
    495                 faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE);
    496                 return $PSTAMP_GONE;
    497             }
    498 
    499             # inputs are ready fall through and queue the diff update
    500         } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_WARP ) {
    501             # check the state of the input stack
    502             my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}";
    503             my $stack = runToolAndParseExpectOne($command, $verbose);
    504             my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack;
    505 
    506             if ($stack->{state} ne 'full') {
    507                 print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
    508                 # this faults all jobs depending on this dep_id
    509                 faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE);
    510                 return $PSTAMP_GONE;
    511             }
    512 
    513             # now check the template warp
    514             $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
    515             my $warp = runToolAndParseExpectOne($command, $verbose);
    516             my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
    517 
    518             if ($warp->{data_state} ne 'full') {
    519                 return check_states_warp($warp, 0, $rlabel, $need_magic);
    520             }
    521             # warps are ready fall through and queue the diff update
     433                return $warp_status;
     434            }
     435        }
     436        $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
     437        my $warp2 = runToolAndParseExpectOne($command, $verbose);
     438        my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2;
     439
     440        if ($warp2->{data_state} ne 'full') {
     441            $warps_ready = 0;
     442            $warp_status = check_states_warp($warp2, 0, $rlabel, $need_magic);
     443            if ($warp_status eq $PSTAMP_GONE) {
     444                faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
     445            }
     446        }
     447
     448        if (!$warps_ready) {
     449            # don't queue the diff update yet
     450            return $warp_status;
     451        }
     452        # inputs are ready fall through and queue the diff update
     453
     454    } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_STACK ) {
     455        # check the state of the input stack
     456        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
     457        my $stack1 = runToolAndParseExpectOne($command, $verbose);
     458        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1;
     459
     460        if ($stack1->{state} ne 'full') {
     461            print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
     462            faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
     463            return $PSTAMP_GONE;
     464        }
     465        # check the state of the template stack
     466        $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
     467        my $stack2 = runToolAndParseExpectOne($command, $verbose);
     468        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack2;
     469
     470        if ($stack2->{state} ne 'full') {
     471            print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
     472            faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
     473            return $PSTAMP_GONE;
     474        }
     475
     476        # inputs are ready fall through and queue the diff update
     477    } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_WARP ) {
     478        # check the state of the input stack
     479        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}";
     480        my $stack = runToolAndParseExpectOne($command, $verbose);
     481        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack;
     482
     483        if ($stack->{state} ne 'full') {
     484            print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
     485            faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
     486            return $PSTAMP_GONE;
     487        }
     488
     489        # now check the template warp
     490        $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
     491        my $warp = runToolAndParseExpectOne($command, $verbose);
     492        my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
     493
     494        if ($warp->{data_state} ne 'full') {
     495            my $warp_status = check_states_warp($warp, 0, $rlabel, $need_magic);
     496            if ($warp_status eq $PSTAMP_GONE) {
     497                faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
     498            }
     499            return $warp_status;
     500        }
     501        # warps are ready fall through and queue the diff update
     502    } else {
     503        my_die("unexpected diff_mode found: $diff_mode", $PS_EXIT_PROG_ERROR);
     504    }
     505
     506    if ($skycell->{data_state} ne 'update') {
     507        my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id";
     508        $command .= " -set_label $rlabel" if $rlabel;
     509
     510        if (!$no_update) {
     511            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     512                        run(command => $command, verbose => $verbose);
     513            unless ($success) {
     514                my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
     515            }
    522516        } else {
    523             my_die("unexpected diff_mode found: $diff_mode", $PS_EXIT_PROG_ERROR);
    524         }
    525 
    526         if ($skycell->{data_state} ne 'update') {
    527             my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id";
    528             $command .= " -set_label $rlabel" if $rlabel;
    529 
    530             if (!$no_update) {
    531                 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    532                             run(command => $command, verbose => $verbose);
    533                 unless ($success) {
    534                     my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
    535                 }
    536             } else {
    537                 print "skipping $command\n";
    538             }
    539         }
    540     } else {
    541         # XXX todo whole_run
    542         return $PS_EXIT_PROG_ERROR;
     517            print "skipping $command\n";
     518        }
    543519    }
    544520
    … …  
    633609}
    634610
    635 sub faultJobs {
    636     my ($state, $stage, $stage_id, $job_fault) = @_;
    637 
    638     my $command = "$pstamptool -updatejob -set_state stop -set_fault $job_fault -dep_id $dep_id";
    639     $command .= " -dbname $dbname" if $dbname;
    640     $command .= " -dbserver $dbserver" if $dbserver;
    641     if (!$no_update) {
    642         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    643                     run(command => $command, verbose => $verbose);
    644         unless ($success) {
    645             my_die("failed to set pstampJob.fault for dep_id: $dep_id",
    646                 $PS_EXIT_UNKNOWN_ERROR);
    647         }
    648     } else {
    649         print "skipping $command\n";
    650     }
    651 }
    652 
     611# Check the data_state of the magicDSFile associated with this component (currently only chip stage is supported)
     612# Returns zero on success.
     613# returns the PSTAMP fault code to use for the jobs if an unrecoverable error is detected
     614# And faults the dependent if transient errors occur
    653615sub check_states_magicDSRun {
    654616    my $stage = shift;
    … …  
    661623    my $dsRun_state = shift;
    662624
    663     # XXX: this code assumes that destreaking is handled at the chip stage
     625    # XXX: this code assumes that for update destreaking is only performed for chip stage
    664626    my_die ("check_states_magicDSRun only implemented for chip stage", $PS_EXIT_PROG_ERROR) if $stage ne 'chip';
    665627
    … …  
    675637    if ($need_magic and !$input_magicked) {
    676638        if (!defined($dsRun_state) or ($dsRun_state eq 'NULL')) {
     639            # it is arguably a programming error if we get here
    677640            print "No magicDSRun for chipRun $stage_id and magic is required\n";
    678             faultJobs('stop', undef, undef, $PSTAMP_NOT_DESTREAKED);
     641            return $PSTAMP_NOT_DESTREAKED;
    679642        } elsif (($dsRun_state eq 'cleaned') or ($dsRun_state eq 'update')) {
    680643            foreach my $c (@$components) {
    … …  
    684647                    my_die("failed to find magicDSFile for ${stage}Run $stage_id $c", $PS_EXIT_UNKNOWN_ERROR);
    685648                }
    686                 if ($dsfile->{fault} > 0) {
     649                if ($dsfile->{fault} eq $PSTAMP_GONE) {
     650                    print "magicDSFile has fault $PSTAMP_GONE\n";
     651                    return $PSTAMP_GONE;
     652                }
     653                #  destreak faults get cleared when the component is set to be updated
     654                if (($dsfile->{data_state} eq 'update') and ($dsfile->{fault} > 0)) {
     655                    $fault_count++;
     656                    if ($fault_count > $max_fault_count) {
     657                        print "Destreak file $magic_ds_id $component for $stage $stage_id has faulted $fault_count times. Giving up\n";
     658                        faultComponent('destreak', $magic_ds_id, $component, $PSTAMP_GONE);
     659                        return $PSTAMP_GONE;
     660                    }
     661                    # Assume fault is transient
    687662                    my_die("faulted magicDSFile for ${stage}Run $stage_id $c fault: $dsfile->{fault}",
    688663                        $PS_EXIT_UNKNOWN_ERROR);
    … …  
    690665                if ($dsfile->{data_state} eq 'cleaned') {
    691666                    $command = "$magicdstool -setfiletoupdate -magic_ds_id $magic_ds_id -component $c";
    692                     # XXX: get the recoveryroot from a config file (it isn't actually used except to check whether it is in nebulous)
     667                    # XXX: get the recoveryroot from a config file
     668                    # (It isn't actually used except to check whether it is a nebulous path)
    693669                    $command .= " -set_recoveryroot neb://any/gpc1/destreak/recover";
    694670                    $command .= " -set_label $rlabel" if $rlabel;
    … …  
    708684            }
    709685        } elsif ($dsRun_state eq 'failed_revert') {
     686            # XXX: revert failures are rarely fixed. give up but say it's just not available not GONE
    710687            print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update\n";
    711             faultJobs('stop', undef, undef, $PSTAMP_NOT_AVAILABLE);
     688            return $PSTAMP_NOT_AVAILABLE;
    712689        } else {
    713690            print "magicDSRun.state = $dsRun_state for chipRun $stage_id";
    … …  
    716693        }
    717694    }
     695
    718696    return 0;
     697}
     698
     699sub faultJobs {
     700    my ($job_fault) = @_;
     701
     702    my $command = "$pstamptool -updatejob -set_state stop -set_fault $job_fault -dep_id $dep_id";
     703    if (!$no_update) {
     704        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     705                    run(command => $command, verbose => $verbose);
     706        unless ($success) {
     707            my_die("failed to set pstampJob.fault for dep_id: $dep_id",
     708                $PS_EXIT_UNKNOWN_ERROR);
     709        }
     710    } else {
     711        print "skipping $command\n";
     712    }
     713}
     714
     715sub faultComponent {
     716    my ($stage, $stage_id, $component, $fault) = @_;
     717
     718    my $command;
     719    if ($stage eq 'chip') {
     720        $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $component";
     721    } elsif ($stage eq 'warp') {
     722        $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $component";
     723    } elsif ($stage eq 'diff') {
     724        $command = "$difftool -updatediffskyfile -diff_id $stage_id -skycell_id $component";
     725    } elsif ($stage eq 'destreak') {
     726        $command = "$magicdstool -updatedestreakedfile -magic_ds_id $stage_id -component $component";
     727    } else {
     728        my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR);
     729    }
     730
     731    $command .= " -fault $fault";
     732
     733    if (!$no_update) {
     734        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     735                    run(command => $command, verbose => $verbose);
     736        unless ($success) {
     737            carp "$cmd failed";
     738        }
     739    } else {
     740        print "skipping $command\n";
     741    }
    719742}
    720743
    … …  
    726749
    727750    my $command = "$pstamptool -updatedependent -set_fault $fault -dep_id $dep_id";
    728     $command .= " -dbname $dbname" if $dbname;
    729     $command .= " -dbserver $dbserver" if $dbserver;
    730751    if (!$no_update) {
    731752        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

  • branches/czw_branch/20101203/pstamp/scripts/pstamp_job_run.pl

    r30586 r30587  
    1414use File::Basename;
    1515use File::Copy;
    16 use File::Temp qw(tempfile);
     16use File::Temp qw(tempfile tempdir);
    1717use Digest::MD5::File qw( file_md5_hex );
    1818use PS::IPP::PStamp::RequestFile qw( :standard );
    … …  
    144144        } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) {
    145145            # user required uncensored but since stage isn't chip we can't rebuild them
    146             my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE);
    147         } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($stage ne 'chip')) {
     146            my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE, 'stop');
     147        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} eq 'update') and ($stage ne 'chip')) {
     148            # we can only restore pixels for chip stage images if the data has been updated.
     149            # XXX: this test is not quite good enough. If all components have been updated then the
     150            # state will be 'full' But this will get us going.
     151            print "Run state is update: will make stamps from destreaked $stage images.\n";
    148152            # make stamps from uncensored images
    149153            $muggle = 0;
    … …  
    151155            # Try and replace the streaks from the recovery images
    152156
    153             @file_list = ();
    154             $tmproot = "$outdir/$job_id";
    155             mkdir $tmproot or
    156                 my_die( "failed to create temporary directory $tmproot", $job_id, $PS_EXIT_UNKNOWN_ERROR);
     157            my $temp_dir = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR") or &my_die("Unable to find temporary directory in site configuration", $job_id, $PS_EXIT_CONFIG_ERROR);
     158            $tmproot = tempdir("$temp_dir/psjob.$job_id.XXXX", CLEANUP => !$save_temps);
    157159            my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot";
    158160            # find the "directory" of the input path_base
    … …  
    160162            my $base = basename($image);
    161163            $tmpImage = "$tmproot/$base";
     164
     165            @file_list = ();
    162166
    163167            # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and
    … …  
    180184                $base = basename($variance);
    181185                $tmpVariance = "$tmproot/$base";
    182                 $newFileArgs .= " -weight $tmpVariance";
     186                $newFileArgs .= " -variance $tmpVariance";
    183187                my $recVariance = "$inputdir/REC_$base";
    184188                $muggle_command .= " -weight $variance -recweight $recVariance";
    … …  
    225229    }
    226230
    227     if (!$save_temps) {
    228         unlink $tmpImage if $tmpImage;
    229         unlink $tmpMask if $tmpMask;
    230         unlink $tmpVariance if $tmpVariance;
    231         rmdir $tmproot if $tmproot;
    232     }
    233 
    234231    if ($exitStatus == 0) {
    235232        my $reglist = "$outdir/reglist$job_id";

  • branches/czw_branch/20101203/pstamp/scripts/pstamp_server_status

    r30586 r30587  
    4646
    4747
    48 my $down = 0;
    49 if ($down) {
    50     print "Postage Stamp Server will be down for maintenance it will back at approximately 00:00 2010-12-11 UTC\n";
    51     exit 0;
    52 }
    5348
    5449my $now = `date -u`;
    … …  
    5651print "<b>Status updated: &nbsp;&nbsp;&nbsp;</b> $now<br /><br />\n";
    5752
    58 my ($pts, $pantasks_script) = tempfile ('/tmp/pts.XXXX', UNLINK => !$save_temps);
     53my $down = 0;
     54if ($down) {
     55    print "Postage Stamp Server will be down for maintenance it will back at approximately 00:00 2010-12-11 UTC\n";
     56    # exit 0;
     57} else {
     58    # talk to pantasks_server and get the status
    5959
    60 print $pts "status\n";
    61 print $pts "quit\n";
    62 close $pts;
     60    my ($pts, $pantasks_script) = tempfile ('/tmp/pts.XXXX', UNLINK => !$save_temps);
    6361
    64 my $command = "$pantasks_client < $pantasks_script";
    65 my  ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    66     run(command => $command, verbose => $verbose);
    67 unless ($success) {
    68     $error_code = (($error_code >> 8) or 1);
    69     if ($error_code == 12) {
    70         #print "Postage Stamp Server is not running\n";
    71         print "Postage Stamp Server will be down for maintenance it will back shortly.\n";
    72         exit 0;
     62    print $pts "status\n";
     63    print $pts "quit\n";
     64    close $pts;
     65
     66
     67    my $command = "$pantasks_client < $pantasks_script";
     68    my  ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     69        run(command => $command, verbose => $verbose);
     70    unless ($success) {
     71        $error_code = (($error_code >> 8) or 1);
     72        if ($error_code == 12) {
     73            #print "Postage Stamp Server is not running\n";
     74            print "Postage Stamp Server will be down for maintenance it will back shortly.\n";
     75            exit 0;
     76        } else {
     77            warn("$command failed. exit status: $error_code");
     78            exit $error_code;
     79        }
    7380    }
    74     warn("$command failed. exit status: $error_code");
    75     exit $error_code;
    76 }
    7781
    78 my $output = join "", @$stdout_buf;
     82    my $output = join "", @$stdout_buf;
    7983
    80 print "$output" if $verbose;
     84    print "$output" if $verbose;
    8185
    82 my @lines = split "\n", $output;
     86    my @lines = split "\n", $output;
    8387
    84 my ($scheduler_state, $controller_state);
    85 my $got_state = 0;
    86 my %tasks;
    87 foreach my $line (@lines) {
    88     chomp $line;
    89     next if !$line;
    90     my ($first, $second, $third);
     88    my ($scheduler_state, $controller_state);
     89    my $got_state = 0;
     90    my %tasks;
     91    foreach my $line (@lines) {
     92        chomp $line;
     93        next if !$line;
     94        my ($first, $second, $third);
    9195
    92     if ($line =~ /Scheduler is/) {
    93         ($first, $second, $scheduler_state) = split " ", $line;
    94         $got_state = 1;
    95 #        print "$first $second $third $scheduler_state\n";
    96     } elsif ($line =~ /Controller is/) {
    97         ($first, $second, $controller_state) = split " ", $line;
    98 #        print "$first $second $third $controller_state\n";
    99     } elsif ($got_state == 1 and $line =~ /\+|\-/) {
    100 #        print "$line\n";
    101         my ($task_state, $task, $nrun, $njobs, $ngood, $nfail, $ntime, $cmd) = split " ", $line;
    102 #        print "$task $task_state\n";
    103         my %this_task = ( name => $task, nrun=> $nrun, njobs => $njobs, ngood => $ngood, nfail => $nfail, ntime => $ntime, cmd => $cmd );
     96        if ($line =~ /Scheduler is/) {
     97            ($first, $second, $scheduler_state) = split " ", $line;
     98            $got_state = 1;
     99    #        print "$first $second $third $scheduler_state\n";
     100        } elsif ($line =~ /Controller is/) {
     101            ($first, $second, $controller_state) = split " ", $line;
     102    #        print "$first $second $third $controller_state\n";
     103        } elsif ($got_state == 1 and $line =~ /\+|\-/) {
     104    #        print "$line\n";
     105            my ($task_state, $task, $nrun, $njobs, $ngood, $nfail, $ntime, $cmd) = split " ", $line;
     106    #        print "$task $task_state\n";
     107            my %this_task = ( name => $task, nrun=> $nrun, njobs => $njobs, ngood => $ngood, nfail => $nfail, ntime => $ntime, cmd => $cmd );
    104108
    105         # change '.' in task name to '_' so we have a valid hash key
    106         $task =~ s/\./\_/g;
    107         $tasks{$task} =  \%this_task;
     109            # change '.' in task name to '_' so we have a valid hash key
     110            $task =~ s/\./\_/g;
     111            $tasks{$task} =  \%this_task;
     112        }
     113    }
     114
     115    if ($scheduler_state) {
     116        print "Pantasks Scheduler $scheduler_state.\n";
     117        if ($controller_state) {
     118            print $br . "Pantasks Controller $controller_state.\n";
     119        } else {
     120            print STDERR "Controller state not found";
     121            exit 1;
     122        }
     123        print "$br$br\n";
     124    } else {
     125        print STDERR "Scheduler state not found";
     126        exit 1;
     127    }
     128
     129    my $request_run = $tasks{'pstamp_request_run'};
     130    my $request_fin = $tasks{'request_finish_run'};
     131    my $job_run = $tasks{'pstamp_job_run'};
     132
     133
     134    if ($request_run) {
     135        # print "<br >\n";
     136        print "<b>Parser Status:</b> $request_run->{nrun} running. $request_run->{ngood} parsed successfully.&nbsp; $request_run->{nfail} failed to parse.";
     137        if ($request_fin) {
     138            print "  $request_fin->{ngood} Requests finished.";
     139        } else {
     140            print "Task request.finish.run not found.\n";
     141        }
     142        print "\n";
     143        print "\n";
     144        if ($job_run) {
     145            print "<br /><b>Job Status:</b> &nbsp;&nbsp; $job_run->{nrun} running. &nbsp; $job_run->{ngood} completed successfully. &nbsp; $job_run->{nfail} failed";
     146            print "<br />\n";
     147        } else {
     148            print "Task pstamp.job.run not found.<br />\n";
     149        }
     150    } else {
     151        print "Task pstamp.request.run not found.\n";
    108152    }
    109153}
    110154
    111 if ($scheduler_state) {
    112     print "Pantasks Scheduler $scheduler_state.\n";
    113     if ($controller_state) {
    114         print $br . "Pantasks Controller $controller_state.\n";
    115     } else {
    116         print STDERR "Controller state not found";
    117         exit 1;
    118     }
    119     print "$br$br\n";
    120 } else {
    121     print STDERR "Scheduler state not found";
    122     exit 1;
    123 }
     155# now run the script psstatus to check the status of running requests and finished requests
     156print "<br /><b>Unfinished Requests</b><br />\n";
     157print "<pre>\n";
     158system "$status_cmd --totals";
     159print "</pre>\n";
    124160
    125 my $request_run = $tasks{'pstamp_request_run'};
    126 my $request_fin = $tasks{'request_finish_run'};
    127 my $job_run = $tasks{'pstamp_job_run'};
     161#print "<br />\n";
    128162
    129163
    130 if ($request_run) {
    131     print "<br /><b>Parser Status:</b> $request_run->{nrun} running. $request_run->{ngood} parsed successfully.&nbsp; $request_run->{nfail} failed to parse.";
    132     if ($request_fin) {
    133         print "  $request_fin->{ngood} Requests finished.";
    134     } else {
    135         print "Task request.finish.run not found.\n";
    136     }
    137     print "\n";
    138     print "\n";
    139     if ($job_run) {
    140         print "<br /><b>Job Status:</b> &nbsp;&nbsp; $job_run->{nrun} running. &nbsp; $job_run->{ngood} completed successfully. &nbsp; $job_run->{nfail} failed";
    141         print "<br />\n";
    142     } else {
    143         print "Task pstamp.job.run not found.<br />\n";
    144     }
    145 
    146     # now run the script psstatus to check the status of running requests and finished requests
    147     print "<br /><b>Unfinished Requests</b><br />\n";
    148     print "<pre>\n";
    149     system "$status_cmd --running";
    150     print "</pre>\n";
     164# print "<br /><b>Unfinished Requests</b><br />\n";
     165print "<pre>\n";
     166system "$status_cmd --running";
     167print "</pre>\n";
    151168
    152169if (0) {
    … …  
    158175}
    159176
    160     print "<br /><b>Requests completed in last 24 hours (most recently completed first)</b><br />\n";
    161     print "<pre>\n";
    162     system "$status_cmd --finished";
    163     print "</pre>\n";
    164 
    165 } else {
    166     print "Task pstamp.request.run not found.\n";
    167 }
    168 
     177print "<br /><b>Requests completed in last 24 hours (most recently completed first)</b><br />\n";
     178print "<pre>\n";
     179system "$status_cmd --finished";
     180print "</pre>\n";
    169181
    170182

  • branches/czw_branch/20101203/pstamp/scripts/pstampparse.pl

    r30586 r30587  
    932932        my $data_state = $image->{data_state};
    933933        $data_state = $run_state if $stage eq 'stack';
    934         if (($run_state =~ /purged/) or ($run_state =~ /scrubbed/) or ($run_state eq 'drop')) {
     934        my $component_fault = $image->{fault};
     935        my $stage_id = $image->{stage_id};
     936        my $component = $image->{component};
     937
     938        if ($component_fault eq $PSTAMP_GONE) {
    935939            # image is gone and it's not coming back
     940            print STDERR "$stage $stage_id $component is gone\n";
     941            $$r_newState = 'stop';
     942            $$r_fault = $PSTAMP_GONE;
     943        } elsif (($run_state =~ /purged/) or ($run_state =~ /scrubbed/) or ($run_state eq 'drop')) {
     944            # image is gone and it's not coming back
     945            print STDERR "$stage $stage_id $component has state: $run_state data_state: $data_state\n";
    936946            $$r_newState = 'stop';
    937947            $$r_fault = $PSTAMP_GONE;
    938948        } elsif (($run_state eq "error_cleaned") or ($data_state eq 'error_cleaned')) {
    939949            # if cleanup had an error don't get the user's hopes up set fault to gone
     950            print STDERR "$stage $stage_id $component has state: $run_state data_state: $data_state\n";
    940951            $$r_newState = 'stop';
    941952            $$r_fault = $PSTAMP_GONE;
Note: See TracChangeset for help on using the changeset viewer.