IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 18460


Ignore:
Timestamp:
Jul 9, 2008, 5:28:06 PM (18 years ago)
Author:
Paul Price
Message:

Only promote to "master" status on the last iteration.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/detrend_reject_exp.pl

    r18445 r18460  
    3030my ($det_id, $iter, $det_type, $camera, $outroot, $filter, $dbname, $verbose, $no_update, $no_op);
    3131GetOptions(
    32            'det_id|d=s'        => \$det_id,
    33            'iteration=s'       => \$iter,
    34            'det_type|t=s'      => \$det_type,
    35            'camera=s'          => \$camera,
    36            'outroot|w=s'       => \$outroot,   # output file base name
    37            'filter=s'          => \$filter,
    38            'dbname|d=s'        => \$dbname, # Database name
    39            'verbose'           => \$verbose,   # Print to stdout
    40            'no-update'         => \$no_update,
    41            'no-op'             => \$no_op,
     32           'det_id|d=s'        => \$det_id,
     33           'iteration=s'       => \$iter,
     34           'det_type|t=s'      => \$det_type,
     35           'camera=s'          => \$camera,
     36           'outroot|w=s'       => \$outroot,   # output file base name
     37           'filter=s'          => \$filter,
     38           'dbname|d=s'        => \$dbname, # Database name
     39           'verbose'           => \$verbose,   # Print to stdout
     40           'no-update'         => \$no_update,
     41           'no-op'             => \$no_op,
    4242) or pod2usage( 2 );
    4343
    4444pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    4545pod2usage( -msg => "Required options: --det_id --iteration --det_type --camera --outroot",
    46            -exitval => 3) unless
    47     defined $det_id   and 
    48     defined $iter     and 
    49     defined $det_type and 
     46           -exitval => 3) unless
     47    defined $det_id   and
     48    defined $iter     and
     49    defined $det_type and
    5050    defined $camera   and
    5151    defined $outroot;
    … …  
    5555# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
    5656# XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
    57 my $STATS = 
    58    [   
     57my $STATS =
     58   [
    5959       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
    6060       { name => "bg",             type => "mean",  flag => "-bg",            dtype => "float" },
    … …  
    6666# these stats are used it the rejections but not passed to the database
    6767# there is some duplication with the above, but the calculation time is minimal
    68 my $REJSTATS = 
    69    [   
     68my $REJSTATS =
     69   [
    7070       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
    7171       { name => "bg",             type => "clipmean",  flag => "ensMeanMean",       dtype => "float" },
    … …  
    8484my $missing_tools;
    8585my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
    86 if ($missing_tools) { 
     86if ($missing_tools) {
    8787    warn("Can't find required tools.");
    88     exit($PS_EXIT_CONFIG_ERROR); 
     88    exit($PS_EXIT_CONFIG_ERROR);
    8989}
    9090
    … …  
    9898    $command .= " -dbname $dbname" if defined $dbname;
    9999    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    100         run(command => $command, verbose => $verbose);
     100        run(command => $command, verbose => $verbose);
    101101    unless ($success) {
    102         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    103         &my_die("Unable to perform dettool -residexp: $error_code", $det_id, $iter, $error_code);
     102        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     103        &my_die("Unable to perform dettool -residexp: $error_code", $det_id, $iter, $error_code);
    104104    }
    105105
    106106    # Parse the stdout buffer into a metadata
    107     my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
     107    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
    108108    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    109         &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR);
     109        &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR);
    110110
    111111    # parse the file info in the metadata
    112112    $exposures = parse_md_list($metadata) or
    113         &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR);
     113        &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR);
    114114
    115115    # Parse the statistics on the residual image
    … …  
    122122# each image has a mean mean (average of means over all chips)
    123123# a standard deviation mean (average of standard deviations over all chips)
    124 # and a 
     124# and a
    125125
    126126
    … …  
    128128# we use the statistics of the ensemble to accept/reject exposurs
    129129my $ensMeanMean       = $rejstats->value_for_flag ("ensMeanMean");       # average of all exposure means (in turn averaged over all chips)
    130 my $ensMeanStdev      = $rejstats->value_for_flag ("ensMeanStdev");      # standard deviation of all exposure means 
     130my $ensMeanStdev      = $rejstats->value_for_flag ("ensMeanStdev");      # standard deviation of all exposure means
    131131my $ensMeanStdevMean  = $rejstats->value_for_flag ("ensMeanStdevMean");  # average over all exposures of the stdev of the means for each chip
    132132my $ensMeanStdevStdev = $rejstats->value_for_flag ("ensMeanStdevStdev"); # standard deviation over all exposures of the stdev of the means for each chip
    133 my $ensStdevMean      = $rejstats->value_for_flag ("ensStdevMean");      # average over all exposures of the sum of the squares of the stdevs for each chip 
    134 my $ensStdevStdev     = $rejstats->value_for_flag ("ensStdevStdev");     # standard deviation over all exposures of the sum of the squares of the stdevs for each chip 
     133my $ensStdevMean      = $rejstats->value_for_flag ("ensStdevMean");      # average over all exposures of the sum of the squares of the stdevs for each chip
     134my $ensStdevStdev     = $rejstats->value_for_flag ("ensStdevStdev");     # standard deviation over all exposures of the sum of the squares of the stdevs for each chip
    135135
    136136$ipprc->define_camera($camera);
    … …  
    155155    print $logFile "Ensemble mean $ensMeanMean +/- $ensMeanStdev\n";
    156156    print $logFile "Ensemble stdev $ensStdevMean +/- $ensStdevStdev\n";
    157     print $logFile "Ensemble mean rms (over imfiles) $ensMeanStdevMean +/- $ensMeanStdevStdev\n\n"; 
     157    print $logFile "Ensemble mean rms (over imfiles) $ensMeanStdevMean +/- $ensMeanStdevStdev\n\n";
    158158}
    159159
    160160# Go through again to do rejection, and update the database for each exposure
    161 my $numChanges = 0;             # Number of exposures with changed status
    162 my $numReject = 0;              # Number of exposures rejected
     161my $numChanges = 0;             # Number of exposures with changed status
     162my $numReject = 0;              # Number of exposures rejected
    163163my $firstElement = 1;
    164164
    165165foreach my $exposure (@$exposures) {
    166     my $mean      = $exposure->{bg};    # Mean for this exposure
     166    my $mean      = $exposure->{bg};    # Mean for this exposure
    167167    my $stdev     = $exposure->{bg_stdev}; # Stdev for this exposure
    168168    my $meanStdev = $exposure->{bg_mean_stdev}; # Stdev of Means for this exposure
    … …  
    175175    &my_die("Unable to find include.\n",     $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $include;
    176176
    177     my $reject = 0;             # Reject this exposure?
     177    my $reject = 0;             # Reject this exposure?
    178178
    179179    $command  = "$dettool -updateresidexp";
    … …  
    182182    $command .= " -exp_id $expID";
    183183    $command .= " -dbname $dbname" if defined $dbname;
    184    
     184
    185185    if (not $accept) {
    186         # Rejected this at an earlier stage
    187         unless ($no_op) {
    188             print $logFile "Rejecting $expID based on earlier determination.\n";
    189         }
    190         $reject = 1;
    191         goto UPDATE;
     186        # Rejected this at an earlier stage
     187        unless ($no_op) {
     188            print $logFile "Rejecting $expID based on earlier determination.\n";
     189        }
     190        $reject = 1;
     191        goto UPDATE;
    192192    }
    193193
    194194    # Cop-out if we're not doing any operations
    195195    if ($no_op) {
    196         # Make sure something gets rejected (just once!), just so that
    197         # we can trace the full range of the workflow
    198         if ($firstElement and $iter == 0) {
    199             $reject = 1;
    200         }
    201         goto UPDATE;
    202     }
    203    
     196        # Make sure something gets rejected (just once!), just so that
     197        # we can trace the full range of the workflow
     198        if ($firstElement and $iter == 0) {
     199            $reject = 1;
     200        }
     201        goto UPDATE;
     202    }
     203
    204204    if ($reject_mean > 0 and $ensMeanStdev > 0) {
    205         my $delta = abs($mean - $ensMeanMean);
    206         if ($delta > ($reject_mean * $ensMeanStdev)) {
    207             print $logFile "Rejecting $expID based on ensemble mean value: ";
    208             $reject = 1;
    209             #goto UPDATE;
    210         } else {
    211             print $logFile "$expID OK against ensemble mean: ";
    212         }
    213         print $logFile "$mean --> $delta vs " . $reject_mean * $ensMeanStdev . "\n";
     205        my $delta = abs($mean - $ensMeanMean);
     206        if ($delta > ($reject_mean * $ensMeanStdev)) {
     207            print $logFile "Rejecting $expID based on ensemble mean value: ";
     208            $reject = 1;
     209            #goto UPDATE;
     210        } else {
     211            print $logFile "$expID OK against ensemble mean: ";
     212        }
     213        print $logFile "$mean --> $delta vs " . $reject_mean * $ensMeanStdev . "\n";
    214214    } else {
    215         print $logFile "No rejection of $expID for ensemble mean\n";
     215        print $logFile "No rejection of $expID for ensemble mean\n";
    216216    }
    217217
    218218    if ($reject_stdev > 0 and $ensStdevStdev > 0) {
    219         my $delta = abs($stdev - $ensStdevMean);
    220         if ($delta > ($reject_stdev * $ensStdevStdev)) {
    221             print $logFile "Rejecting $expID based on ensemble stdev: ";
    222             $reject = 1;
    223             #goto UPDATE;
    224         } else {
    225             print $logFile "$expID OK against ensemble stdev: ";
    226         }
    227         print $logFile "$stdev --> $delta sigma vs " . $reject_stdev * $ensStdevStdev . "\n";
     219        my $delta = abs($stdev - $ensStdevMean);
     220        if ($delta > ($reject_stdev * $ensStdevStdev)) {
     221            print $logFile "Rejecting $expID based on ensemble stdev: ";
     222            $reject = 1;
     223            #goto UPDATE;
     224        } else {
     225            print $logFile "$expID OK against ensemble stdev: ";
     226        }
     227        print $logFile "$stdev --> $delta sigma vs " . $reject_stdev * $ensStdevStdev . "\n";
    228228    } else {
    229         print $logFile "No rejection of $expID for ensemble stdev\n";
    230     }
    231    
     229        print $logFile "No rejection of $expID for ensemble stdev\n";
     230    }
     231
    232232    if ($reject_meanstdev > 0 and $ensMeanStdevStdev > 0) {
    233         my $delta = abs($meanStdev - $ensMeanStdevMean);
    234         if ($delta > ($reject_meanstdev * $ensMeanStdevStdev)) {
    235             print $logFile "Rejecting $expID based on ensemble mean stdev: ";
    236             $reject = 1;
    237             #goto UPDATE;
    238         } else {
    239             print $logFile "$expID OK against ensemble mean stdev: ";
    240         }
    241         print $logFile "$meanStdev --> $delta sigma vs " . $reject_meanstdev * $ensMeanStdevStdev. "\n";
     233        my $delta = abs($meanStdev - $ensMeanStdevMean);
     234        if ($delta > ($reject_meanstdev * $ensMeanStdevStdev)) {
     235            print $logFile "Rejecting $expID based on ensemble mean stdev: ";
     236            $reject = 1;
     237            #goto UPDATE;
     238        } else {
     239            print $logFile "$expID OK against ensemble mean stdev: ";
     240        }
     241        print $logFile "$meanStdev --> $delta sigma vs " . $reject_meanstdev * $ensMeanStdevStdev. "\n";
    242242    } else {
    243         print $logFile "No rejection of $expID for ensemble mean stdev\n";
     243        print $logFile "No rejection of $expID for ensemble mean stdev\n";
    244244    }
    245245
    246246  UPDATE:
    247247    if ($reject) {
    248         $command .= ' -reject';
    249         $numReject++;
    250     }
    251    
     248        $command .= ' -reject';
     249        $numReject++;
     250    }
     251
    252252    # Check for status changes
    253253    if ((not $include and not $reject) or ($include and $reject)) {
    254         unless ($no_op) {
    255             print $logFile "Status of $expID has changed.\n";
    256         }
    257         $numChanges++;
     254        unless ($no_op) {
     255            print $logFile "Status of $expID has changed.\n";
     256        }
     257        $numChanges++;
    258258    }
    259259
    260260    unless ($no_update) {
    261         # Update
    262         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    263             run(command => $command, verbose => $verbose);
    264         unless ($success) {
    265             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    266             &my_die("Unable to perform dettool -updateresidexp: $error_code", $det_id, $iter, $error_code);
    267         }
     261        # Update
     262        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     263            run(command => $command, verbose => $verbose);
     264        unless ($success) {
     265            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     266            &my_die("Unable to perform dettool -updateresidexp: $error_code", $det_id, $iter, $error_code);
     267        }
    268268    }
    269269}
    270270
    271271# Decide if the current is sufficient to use as a master, and if we can stop iterating
    272 my $master = 1;                 # This is good enough for a master
    273 my $stop = 1;                   # Stop iterating
     272my $master = 1;                 # This is good enough for a master
     273my $stop = 1;                   # Stop iterating
    274274
    275275if ($numChanges > 0) {
    … …  
    282282    $master = 0;
    283283    $stop = 1;
     284    carp "All inputs rejected!\n";
    284285}
    285286
    … …  
    290291}
    291292
     293# Allow iteration to be turned off
     294my $allow_iter = metadataLookupBool($ipprc->{rejection}, "ITERATION"); # Allow iteration?
     295my $force_master = metadataLookupBool($ipprc->{rejection}, "MASTER"); # Force the stack to be accepted
     296$stop = 1 unless $allow_iter;
     297$master = 1 if $force_master and $stop;
     298
    292299# attempt to prevent endless, pathological iterations
    293300if ($iter >= $ITER_LIMIT) {
    … …  
    295302    exit($PS_EXIT_PROG_ERROR);
    296303}
    297 
    298 # Allow iteration to be turned off
    299 my $allow_iter = metadataLookupBool($ipprc->{rejection}, "ITERATION"); # Allow iteration?
    300 my $force_master = metadataLookupBool($ipprc->{rejection}, "MASTER"); # Force the stack to be accepted
    301 $stop = 1 unless $allow_iter;
    302 $master = 1 if $force_master;
    303 
    304304
    305305## add the summary statistics, and request a new iteration if needed
    … …  
    310310$command .= " -dbname $dbname" if defined $dbname;
    311311### XXX WE NEED to make this a recipe-driven option
    312 $command .= " -again" unless $stop; 
     312$command .= " -again" unless $stop;
    313313$command .= $stats->cmdflags();
    314314
    … …  
    316316unless ($no_update) {
    317317    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    318         run(command => $command, verbose => $verbose);
     318        run(command => $command, verbose => $verbose);
    319319    unless ($success) {
    320         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    321         warn("Unable to perform dettool -adddetrunsummary: $error_code");
    322         exit($error_code);
     320        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     321        warn("Unable to perform dettool -adddetrunsummary: $error_code");
     322        exit($error_code);
    323323    }
    324324} else {
    … …  
    337337{
    338338    my $msg = shift; # Warning message on die
    339     my $det_id = shift;         # Detrend identifier
    340     my $iter = shift;           # Iteration
     339    my $det_id = shift;         # Detrend identifier
     340    my $iter = shift;           # Iteration
    341341    my $exit_code = shift; # Exit code to add
    342342
    343343    carp($msg);
    344344    if (defined $det_id and defined $iter and not $no_update) {
    345         my $command = "$dettool -adddetrunsummary";
    346         $command .= " -det_id $det_id";
    347         $command .= " -iteration $iter";
    348         $command .= " -code $exit_code";
    349         $command .= " -dbname $dbname" if defined $dbname;
     345        my $command = "$dettool -adddetrunsummary";
     346        $command .= " -det_id $det_id";
     347        $command .= " -iteration $iter";
     348        $command .= " -code $exit_code";
     349        $command .= " -dbname $dbname" if defined $dbname;
    350350        system ($command);
    351351    }
    … …  
    356356sub rejection_limit
    357357{
    358     my $name = shift;           # Rejection limit to
    359     my $type = shift;           # Type of exposure
    360     my $filter = shift;         # Filter
     358    my $name = shift;           # Rejection limit to
     359    my $type = shift;           # Type of exposure
     360    my $filter = shift;         # Filter
    361361
    362362    my $value = $ipprc->rejection( $name, $det_type, $filter );
    363363    if (not defined $value) {
    364         $filter = "(no filter)" unless defined $filter;
    365         die "Unable to determine $name rejection limit for $det_type with $filter.\n";
     364        $filter = "(no filter)" unless defined $filter;
     365        die "Unable to determine $name rejection limit for $det_type with $filter.\n";
    366366    }
    367367
Note: See TracChangeset for help on using the changeset viewer.