IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 9505


Ignore:
Timestamp:
Oct 12, 2006, 10:14:50 AM (20 years ago)
Author:
Paul Price
Message:

Making detrend rejection limits configurable per camera, by placing them in the camera configuration and using PS::IPP::Config to access them.

Location:
trunk/ippScripts/scripts
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/detrend_reject_exp.pl

    r9458 r9505  
    1212use Statistics::Descriptive;
    1313
     14use PS::IPP::Config;
     15my $ipprc = PS::IPP::Config->new(); # IPP configuration
     16
    1417use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
    1518use Pod::Usage qw( pod2usage );
    1619
    17 my ($det_id, $iter, $det_type, $no_update);
     20my ($det_id, $iter, $det_type, $camera, $filter, $no_update);
    1821GetOptions(
    19     'det_id|d=s'        => \$det_id,
    20     'iteration=s'       => \$iter,
    21     'det_type|t=s'      => \$det_type,
    22     'no-update'         => \$no_update
     22           'det_id|d=s'        => \$det_id,
     23           'iteration=s'       => \$iter,
     24           'det_type|t=s'      => \$det_type,
     25           'camera=s'          => \$camera,
     26           'filter=s'          => \$filter,
     27           'no-update'         => \$no_update
    2328) or pod2usage( 2 );
    2429
    2530pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    2631pod2usage(
    27     -msg => "Required options: --det_id --iteration --det_type",
     32    -msg => "Required options: --det_id --iteration --det_type --camera",
    2833    -exitval => 3,
    2934) unless defined $det_id
    3035    and defined $iter
    31     and defined $det_type;
    32 
    33 # Rejection threshold for the mean, in terms of the standard deviation
    34 # This measures how close it is to what's expected
    35 use constant REJECT_MEAN => {
    36     'bias' => 0,                # Should be fairly flat; some CRs
    37     'dark' => 0,                # Lots of CRs
    38     'shutter' => 10,            # Should be less than 10 sec
    39     'flat' => undef             # Can't define expected value (depends on exposure level)
    40     };
    41 
    42 # Rejection threshold for the standard deviation, in terms of the standard deviation of the stdevs
    43 # This measures how much variation there is within the components of an exposure, compared to "typical"
    44 use constant REJECT_STDEV => {
    45     'bias' => 0, # Components should have the same degree of variation
    46     'dark' => 0, # Components should have the same degree of variation
    47     'shutter' => undef,         # Might be significant variation
    48     'flat' => 0 # Components should have the same degree of variation
    49     };
    50 
    51 # Rejection threshold for the mean of the stdev, in terms of the standard deviation of the mean of the stdevs
    52 # This measures how structured the images are, compared to "typical"
    53 use constant REJECT_MEAN_STDEV => {
    54     'bias' => 0,                # All images should be equally structured
    55     'dark' => 0,                # All images should be equally structured
    56     'shutter' => 0,             # All images should be equally structured
    57     'flat' => 0                 # All images should be equally structured
    58     };
     36    and defined $det_type
     37    and defined $camera;
     38
    5939
    6040# Look for programs we need
    … …  
    10383$meanStdevStats->add_data(@meanStdevs);
    10484
    105 # Check the existence of the required rejection levels as a function of detrend type
    106 die "Unknown mean rejection level for detrend type $det_type\n"
    107     if not exists REJECT_MEAN->{$det_type};
    108 die "Unknown stdev rejection level for detrend type $det_type\n"
    109     if not exists REJECT_STDEV->{$det_type};
    110 die "Unknown mean stdev rejection level for detrend type $det_type\n"
    111     if not exists REJECT_MEAN_STDEV->{$det_type};
     85$ipprc->define_camera($camera);
     86# Rejection thresholds
     87my $reject_mean = rejection_limit( 'ENSEMBLE.MEAN', $det_type, $filter );
     88my $reject_stdev = rejection_limit( 'ENSEMBLE.STDEV', $det_type, $filter );
     89my $reject_meanstdev = rejection_limit( 'ENSEMBLE.MEANSTDEV', $det_type, $filter );
    11290
    11391# rejections based on comparison with ensemble statistics
    … …  
    124102# Go through again to do rejection, and update the database for each exposure
    125103my $numChanges = 0;             # Number of exposures with changed status
     104my $numReject = 0;              # Number of exposures rejected
     105my $command;
    126106for (my $i = 0; $i < scalar @means; $i++) {
    127107    my $expTag = $expTags[$i];  # Exposure ID
    128     my $command = "$dettool -updateresidexp -det_id $det_id -iteration $iter -exp_tag $expTag"; # Command to run
     108    $command = "$dettool -updateresidexp -det_id $det_id -iteration $iter -exp_tag $expTag"; # Command to run
    129109    my $reject = 0;             # Reject this exposure?
    130110
    … …  
    134114        $reject = 1;
    135115        goto UPDATE;
    136     } 
    137     if (REJECT_MEAN->{$det_type} && ($meanStdev > 0)) {
     116    }
     117    if ($reject_mean && ($meanStdev > 0)) {
    138118        my $nSigma = abs($means[$i] - $mean) / $meanStdev;
    139         if ($nSigma > REJECT_MEAN->{$det_type}) {
     119        if ($nSigma > $reject_mean) {
    140120            print "Rejecting $expTag based on outlier mean value: " .
    141                 "$means[$i] is $nSigma vs " . REJECT_MEAN->{$det_type} . "\n";
     121                "$means[$i] is $nSigma vs " . $reject_mean . "\n";
    142122            $reject = 1;
    143123            goto UPDATE;
    … …  
    146126        print "no rejection for exposure mean\n";
    147127    }
    148     if (REJECT_STDEV->{$det_type} && ($varStdev > 0)) {
     128    if ($reject_stdev && ($varStdev > 0)) {
    149129        my $nSigma = abs($variances[$i] - $var) / $varStdev;
    150         if ($nSigma > REJECT_STDEV->{$det_type}) {
     130        if ($nSigma > $reject_stdev) {
    151131            print "Rejecting $expTag based on outlier stdev: " .
    152                 sqrt($variances[$i]) . " is $nSigma vs " . REJECT_STDEV->{$det_type} . "\n";
     132                sqrt($variances[$i]) . " is $nSigma vs " . $reject_stdev . "\n";
    153133            $reject = 1;
    154134            goto UPDATE;
    … …  
    161141    if ($reject) {
    162142        $command .= ' -reject';
     143        $numReject++;
    163144    }
    164145   
    … …  
    184165    $master = 0;
    185166    $stop = 0;
     167}
     168
     169# Rejecting everything --- stop before something bad happens!
     170if ($numReject == scalar @means) {
     171    $master = 0;
     172    $stop = 1;
    186173}
    187174
    … …  
    217204END { system("sync") == 0 or die "failed to execute sync: $!" }
    218205
     206# Retrieve the requested rejection limit, dying if not extant
     207sub rejection_limit
     208{
     209    my $name = shift;           # Rejection limit to
     210    my $type = shift;           # Type of exposure
     211    my $filter = shift;         # Filter
     212
     213    my $value = $ipprc->rejection( $name, $det_type, $filter );
     214    if (not defined $value) {
     215        $filter = "(no filter)" if not defined $filter;
     216        die "Unable to determine $name rejection limit for $det_type with $filter.\n";
     217    }
     218
     219    return $value;
     220}
     221
     222
    219223__END__

  • trunk/ippScripts/scripts/detrend_reject_imfile.pl

    r9458 r9505  
    1919use Pod::Usage qw( pod2usage );
    2020
    21 my ($det_id, $iter, $exp_tag, $det_type, $no_update, $reject);
     21my ($det_id, $iter, $exp_tag, $det_type, $camera, $filter, $no_update, $reject);
    2222GetOptions(
    2323           'det_id|d=s'        => \$det_id,
    … …  
    2626           'det_type|t=s'      => \$det_type,
    2727           'no-update'         => \$no_update,
     28           'camera=s'          => \$camera,
     29           'filter=s'          => \$filter,
    2830           'reject'            => \$reject
    2931           ) or pod2usage( 2 );
    … …  
    3133pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    3234pod2usage(
    33           -msg => "Required options: --det_id --iteration --exp_tag --det_type",
     35          -msg => "Required options: --det_id --iteration --exp_tag --det_type --camera",
    3436          -exitval => 3,
    3537          ) unless defined $det_id
    3638    and defined $iter
    3739    and defined $exp_tag
    38     and defined $det_type;
     40    and defined $det_type
     41    and defined $camera;
    3942
    4043use constant RECIPE1 => 'PPIMAGE_J1'; # Recipe to use for ppImage to make JPEGs
    4144use constant RECIPE2 => 'PPIMAGE_J2'; # Recipe to use for ppImage to make JPEGs
    42 
    43 #### XXXXX these values must come from the config system, and may depend on filter!!!
    44 # XXX it is valid to reject on more than one criterion
    45 
    46 # The expected mean, as a function of detrend type
    47 use constant EXPECTED_MEAN => {
    48     'bias' => 0,                # Bias should be zero
    49     'dark' => 0,                # Dark should be zero
    50     'shutter' => undef,         # Shutter could be anything (depends on exposure level)
    51     'flat' => 0                 # Flat could be anything (depends on exposure level)
    52     };
    53 
    54 # Rejection threshold for the mean
    55 # This measures how close it is to what's expected
    56 use constant REJECT_IMFILE_MEAN => {
    57     'bias' => 0,                # Should be fairly flat; some CRs
    58     'dark' => 0,                # Lots of CRs
    59     'shutter' => undef,         # Can't define expected value (depends on exposure level)
    60     'flat' => 0         # Can't define expected value (depends on exposure level)
    61     };
    62 
    63 # Rejection threshold for the standard deviation, in ADUs
    64 # This measures how much variation there is in each imfile
    65 use constant REJECT_IMFILE_STDEV => {
    66     'bias' => 15,               # Should be fairly flat; some CRs
    67     'dark' => 0,                # Lots of CRs
    68     'shutter' => undef,         # Can be significant structure
    69     'flat' => 0         # Stars and galaxies
    70     };
    71 
    72 # Rejection threshold for the mean of the exposure, in terms of the standard deviation of the exposure
    73 # This measures how close it is to what's expected
    74 use constant REJECT_EXPOSURE_MEAN => {
    75     'bias' => 0,                # Should be little variation between chips
    76     'dark' => 0,                # Could be some glow on some chips
    77     'shutter' => undef,         # Can't define expected value (depends on exposure level)
    78     'flat' => 0         # Can't define expected value (depends on exposure level)
    79     };
    80 
    81 # Rejection threshold for the stdev of the exposure, in ADUs
    82 # This measures how much variation there is across the imfiles
    83 use constant REJECT_EXPOSURE_STDEV => {
    84     'bias' => 15,               # Should be little variation between chips
    85     'dark' => 0,                # Could be some glow on some chips
    86     'shutter' => undef,         # Can be significant structure
    87     'flat' => 0                 # Could be features on some chips, but all should be about the same
    88     };
    89 
    90 # Rejection threshold for the stdev of the exposure, in ADUs
    91 # This measures how much variation there is across the imfiles
    92 use constant REJECT_EXPOSURE_MEAN_STDEV => {
    93     'bias' => 0,                # Should be little variation between chips
    94     'dark' => 0,                # Could be some glow on some chips
    95     'shutter' => undef,         # Can be significant structure
    96     'flat' => 0                 # Could be features on some chips, but all should be about the same
    97     };
    98 
    9945
    10046# Look for programs we need
    … …  
    13177open my $list2File, '>' . $list2Name;
    13278foreach my $file (@$files) {
    133     print $list1File File::Spec->rel2abs( $file->{b1_uri}, $ipprc->workdir() ) . "\n";
    134     print $list2File File::Spec->rel2abs( $file->{b2_uri}, $ipprc->workdir() ) . "\n";
     79    print $list1File (File::Spec->rel2abs( $file->{b1_uri}, $ipprc->workdir() ) . "\n");
     80    print $list2File (File::Spec->rel2abs( $file->{b2_uri}, $ipprc->workdir() ) . "\n");
    13581    push @means, $file->{bg};
    13682    ## calculate the root-mean-square of the bd_stdevs
    … …  
    160106}
    161107
    162 # Check the existence of the required rejection levels as a function of detrend type
    163 die "Unknown expected mean for detrend type $det_type\n"
    164     if not exists EXPECTED_MEAN->{$det_type};
    165 die "Unknown imfile mean rejection level for detrend type $det_type\n"
    166     if not exists REJECT_IMFILE_MEAN->{$det_type};
    167 die "Unknown imfile stdev rejection level for detrend type $det_type\n"
    168     if not exists REJECT_IMFILE_STDEV->{$det_type};
    169 die "Unknown exposure mean rejection level for detrend type $det_type\n"
    170     if not exists REJECT_EXPOSURE_MEAN->{$det_type};
    171 die "Unknown exposure stdev rejection level for detrend type $det_type\n"
    172     if not exists REJECT_EXPOSURE_STDEV->{$det_type};
    173 die "Unknown exposure mean stdev rejection level for detrend type $det_type\n"
    174     if not exists REJECT_EXPOSURE_MEAN_STDEV->{$det_type};
     108$ipprc->define_camera( $camera);
     109my $expected = rejection_limit( 'EXPECTED', $det_type, $filter ); # Expected mean
     110# Rejection thresholds
     111my $reject_imfile_mean   = rejection_limit( 'IMFILE.MEAN',   $det_type, $filter );
     112my $reject_imfile_stdev  = rejection_limit( 'IMFILE.STDEV',  $det_type, $filter );
     113my $reject_exp_mean      = rejection_limit( 'EXP.MEAN',      $det_type, $filter );
     114my $reject_exp_stdev     = rejection_limit( 'EXP.STDEV',     $det_type, $filter );
     115my $reject_exp_meanstdev = rejection_limit( 'EXP.MEANSTDEV', $det_type, $filter );
    175116
    176117# Reject based on the stats of the imfiles
    … …  
    179120for (my $i = 0; $i < scalar @means; $i++) {
    180121    my $mean = $means[$i];      # Mean for this imfile
    181     $mean -= EXPECTED_MEAN->{$det_type} if defined EXPECTED_MEAN->{$det_type};
     122    $mean -= $expected;
    182123    my $stdev = sqrt($variances[$i]);   # Stdev for this imfile
    183124
    184     if (REJECT_IMFILE_MEAN->{$det_type}) {
    185         if (abs($mean) > REJECT_IMFILE_MEAN->{$det_type}) {
     125    if ($reject_imfile_mean) {
     126        if (abs($mean) > $reject_imfile_mean) {
    186127            print "Rejecting exposure based on bad imfile mean for imfile $i: " .
    187                 $mean . " vs " . REJECT_IMFILE_MEAN->{$det_type} . "\n";
     128                $mean . " vs " . $reject_imfile_mean . "\n";
    188129            $reject = 1;
    189130            last;
    … …  
    192133        print "no rejection for imfile mean\n";
    193134    }
    194     if (REJECT_IMFILE_STDEV->{$det_type}) {
    195         if ($stdev > REJECT_IMFILE_STDEV->{$det_type}) {
     135    if ($reject_imfile_stdev) {
     136        if ($stdev > $reject_imfile_stdev) {
    196137            print "Rejecting exposure based on bad imfile stdev for imfile $i: " .
    197                 $stdev . " vs " . REJECT_IMFILE_STDEV->{$det_type} . "\n";
     138                $stdev . " vs " . $reject_imfile_stdev . "\n";
    198139            $reject = 1;
    199140            last;
    … …  
    221162## Reject based on the exposure ensemble stats
    222163# reject if the exposure ensemble mean is deviant
    223 if (REJECT_EXPOSURE_MEAN->{$det_type}) {
    224     if (abs($mean) > REJECT_EXPOSURE_MEAN->{$det_type}) {
     164if ($reject_exp_mean) {
     165    if (abs($mean) > $reject_exp_mean) {
    225166        print "Rejecting exposure based on bad mean: " . ($mean / $stdev) . " vs " .
    226             REJECT_EXPOSURE_MEAN->{$det_type} . "\n";
     167            $reject_exp_mean . "\n";
    227168        $reject = 1;
    228169    }
    … …  
    231172}
    232173# reject if the exposure ensemble stdev is deviant
    233 if (REJECT_EXPOSURE_STDEV->{$det_type}) {
    234     if ($stdev > REJECT_EXPOSURE_STDEV->{$det_type}) {
     174if ($reject_exp_stdev) {
     175    if ($stdev > $reject_exp_stdev) {
    235176        print "Rejecting exposure based on bad mean stdev: " . $stdev . " vs " .
    236             REJECT_EXPOSURE_STDEV->{$det_type} . "\n";
     177            $reject_exp_stdev . "\n";
    237178        $reject = 1;
    238179    }
    … …  
    241182}
    242183# reject if the exposure ensemble mean stdev is deviant
    243 if (REJECT_EXPOSURE_MEAN_STDEV->{$det_type}) {
    244     if ($meanStdev > REJECT_EXPOSURE_MEAN_STDEV->{$det_type}) {
     184if ($reject_exp_meanstdev) {
     185    if ($meanStdev > $reject_exp_meanstdev) {
    245186        print "Rejecting exposure based on bad mean stdev: " . $meanStdev . " vs " .
    246             REJECT_EXPOSURE_MEAN_STDEV->{$det_type} . "\n";
     187            $reject_exp_meanstdev . "\n";
    247188        $reject = 1;
    248189    }
    … …  
    268209
    269210END { system("sync") == 0 or die "failed to execute sync: $!" }
     211
     212
     213# Retrieve the requested rejection limit, dying if not extant
     214sub rejection_limit
     215{
     216    my $name = shift;           # Rejection limit to
     217    my $type = shift;           # Type of exposure
     218    my $filter = shift;         # Filter
     219
     220    my $value = $ipprc->rejection( $name, $det_type, $filter );
     221    if (not defined $value) {
     222        $filter = "(no filter)" if not defined $filter;
     223        die "Unable to determine $name rejection limit for $det_type with $filter.\n";
     224    }
     225
     226    return $value;
     227}
     228
     229
     230
    270231
    271232__END__
Note: See TracChangeset for help on using the changeset viewer.