IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Oct 10, 2006, 8:52:00 AM (20 years ago)
Author:
eugene
Message:

updating bg_stdev, bg_mean_stdev tests

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/detrend_reject_imfile.pl

    r9454 r9457  
    2121my ($det_id, $iter, $exp_tag, $det_type, $no_update, $reject);
    2222GetOptions(
    23     'det_id|d=s'        => \$det_id,
    24     'iteration=s'       => \$iter,
    25     'exp_tag|e=s'       => \$exp_tag,
    26     'det_type|t=s'      => \$det_type,
    27     'no-update'         => \$no_update,
    28     'reject'            => \$reject
    29 ) or pod2usage( 2 );
     23           'det_id|d=s'        => \$det_id,
     24           'iteration=s'       => \$iter,
     25           'exp_tag|e=s'       => \$exp_tag,
     26           'det_type|t=s'      => \$det_type,
     27           'no-update'         => \$no_update,
     28           'reject'            => \$reject
     29           ) or pod2usage( 2 );
    3030
    3131pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    3232pod2usage(
    33     -msg => "Required options: --det_id --iteration --exp_tag --det_type",
    34     -exitval => 3,
    35 ) unless defined $det_id
     33          -msg => "Required options: --det_id --iteration --exp_tag --det_type",
     34          -exitval => 3,
     35          ) unless defined $det_id
    3636    and defined $iter
    3737    and defined $exp_tag
    … …  
    4242
    4343#### XXXXX these values must come from the config system, and may depend on filter!!!
     44# XXX it is valid to reject on more than one criterion
     45
    4446# The expected mean, as a function of detrend type
    4547use constant EXPECTED_MEAN => {
    … …  
    5052    };
    5153
    52 # Rejection threshold for the mean, in terms of the standard deviation
     54# Rejection threshold for the mean
    5355# This measures how close it is to what's expected
    54 # XXX it is valid to reject on more than one criterion
    55 use constant REJECT_COMPONENT_MEAN => {
     56use constant REJECT_IMFILE_MEAN => {
    5657    'bias' => 0,                # Should be fairly flat; some CRs
    5758    'dark' => 0,                # Lots of CRs
    … …  
    6162
    6263# Rejection threshold for the standard deviation, in ADUs
    63 # This measures how much variation there is in each imfile component
    64 use constant REJECT_COMPONENT_STDEV => {
     64# This measures how much variation there is in each imfile
     65use constant REJECT_IMFILE_STDEV => {
    6566    'bias' => 0,                # Should be fairly flat; some CRs
    6667    'dark' => 0,                # Lots of CRs
    … …  
    6970    };
    7071
    71 # Rejection threshold for the mean of the sample, in terms of the standard deviation of the sample
     72# Rejection threshold for the mean of the exposure, in terms of the standard deviation of the exposure
    7273# This measures how close it is to what's expected
    73 use constant REJECT_SAMPLE_MEAN => {
     74use constant REJECT_EXPOSURE_MEAN => {
    7475    'bias' => 0,                # Should be little variation between chips
    7576    'dark' => 0,                # Could be some glow on some chips
    … …  
    7879    };
    7980
    80 # Rejection threshold for the stdev of the sample, in ADUs
    81 # This measures how much variation there is across the imfile components
    82 use constant REJECT_SAMPLE_STDEV => {
     81# Rejection threshold for the stdev of the exposure, in ADUs
     82# This measures how much variation there is across the imfiles
     83use constant REJECT_EXPOSURE_STDEV => {
    8384    'bias' => 0,                # Should be little variation between chips
    8485    'dark' => 0,                # Could be some glow on some chips
    … …  
    8788    };
    8889
    89 # Rejection threshold for the stdev of the sample, in ADUs
    90 # This measures how much variation there is across the imfile components
    91 use constant REJECT_SAMPLE_MEAN_STDEV => {
     90# Rejection threshold for the stdev of the exposure, in ADUs
     91# This measures how much variation there is across the imfiles
     92use constant REJECT_EXPOSURE_MEAN_STDEV => {
    9293    'bias' => 0,                # Should be little variation between chips
    9394    'dark' => 0,                # Could be some glow on some chips
    … …  
    105106my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    106107
    107 # Get list of component files
    108 my $files;                      # Array of component files
     108# Get list of imfile files
     109my $files;                      # Array of imfile files
    109110{
    110111    my $command = "$dettool -residimfile -det_id $det_id -iteration $iter -exp_tag $exp_tag"; # Command to run
    … …  
    126127my $list2Name = $outputRoot . '.b2.list'; # Name for the input file list for binning 2
    127128my @means;                      # Array of means
    128 my @stdevs;                     # Array of stdevs
     129my @variances;                  # Array of variances
    129130open my $list1File, '>' . $list1Name;
    130131open my $list2File, '>' . $list2Name;
    … …  
    134135    push @means, $file->{bg};
    135136    ## calculate the root-mean-square of the bd_stdevs
    136     push @stdevs, $file->{bg_mean_stdev}*$file->{bg_mean_stdev};
     137    push @variances, $file->{bg_stdev}*$file->{bg_stdev};
    137138}
    138139close $list1File;
    … …  
    162163die "Unknown expected mean for detrend type $det_type\n"
    163164    if not exists EXPECTED_MEAN->{$det_type};
    164 die "Unknown component mean rejection level for detrend type $det_type\n"
    165     if not exists REJECT_COMPONENT_MEAN->{$det_type};
    166 die "Unknown component stdev rejection level for detrend type $det_type\n"
    167     if not exists REJECT_COMPONENT_STDEV->{$det_type};
    168 die "Unknown sample mean rejection level for detrend type $det_type\n"
    169     if not exists REJECT_SAMPLE_MEAN->{$det_type};
    170 die "Unknown sample stdev rejection level for detrend type $det_type\n"
    171     if not exists REJECT_SAMPLE_STDEV->{$det_type};
    172 
    173 # Reject based on the stats of the component imfile
    174 die "Number of means and number of stdevs differ!\n" if scalar @means != scalar @stdevs;
     165die "Unknown imfile mean rejection level for detrend type $det_type\n"
     166    if not exists REJECT_IMFILE_MEAN->{$det_type};
     167die "Unknown imfile stdev rejection level for detrend type $det_type\n"
     168    if not exists REJECT_IMFILE_STDEV->{$det_type};
     169die "Unknown exposure mean rejection level for detrend type $det_type\n"
     170    if not exists REJECT_EXPOSURE_MEAN->{$det_type};
     171die "Unknown exposure stdev rejection level for detrend type $det_type\n"
     172    if not exists REJECT_EXPOSURE_STDEV->{$det_type};
     173die "Unknown exposure mean stdev rejection level for detrend type $det_type\n"
     174    if not exists REJECT_EXPOSURE_MEAN_STDEV->{$det_type};
     175
     176# Reject based on the stats of the imfiles
     177# it is VALID to reject on more than one criterion
     178die "Number of means and number of variances differ!\n" if scalar @means != scalar @variances;
    175179for (my $i = 0; $i < scalar @means; $i++) {
    176     my $mean = $means[$i];      # Mean for this component
     180    my $mean = $means[$i];      # Mean for this imfile
    177181    $mean -= EXPECTED_MEAN->{$det_type} if defined EXPECTED_MEAN->{$det_type};
    178     my $stdev = sqrt($stdevs[$i]);      # Stdev for this component
    179 
    180     ## IT IS VALID to reject on more than one criterion
    181     if (REJECT_COMPONENT_MEAN->{$det_type} && ($stdev > 0) ) {
    182         if ($mean / $stdev > REJECT_COMPONENT_MEAN->{$det_type}) {
    183             print "Rejecting exposure based on bad component mean for component $i: " .
    184                 ($mean / $stdev) . " vs " . REJECT_COMPONENT_MEAN->{$det_type} . "\n";
     182    my $stdev = sqrt($variances[$i]);   # Stdev for this imfile
     183
     184    if (REJECT_IMFILE_MEAN->{$det_type}) {
     185        if (abs($mean) > REJECT_IMFILE_MEAN->{$det_type}) {
     186            print "Rejecting exposure based on bad imfile mean for imfile $i: " .
     187                $mean . " vs " . REJECT_IMFILE_MEAN->{$det_type} . "\n";
    185188            $reject = 1;
    186189            last;
    187190        }
    188     }
    189     if (REJECT_COMPONENT_STDEV->{$det_type}) {
    190         if ($stdev > REJECT_COMPONENT_STDEV->{$det_type}) {
    191             print "Rejecting exposure based on bad component stdev for component $i: " .
    192                 $stdev . " vs " . REJECT_COMPONENT_STDEV->{$det_type} . "\n";
     191    }  else {
     192        print "no rejection for imfile mean\n";
     193    }
     194    if (REJECT_IMFILE_STDEV->{$det_type}) {
     195        if ($stdev > REJECT_IMFILE_STDEV->{$det_type}) {
     196            print "Rejecting exposure based on bad imfile stdev for imfile $i: " .
     197                $stdev . " vs " . REJECT_IMFILE_STDEV->{$det_type} . "\n";
    193198            $reject = 1;
    194199            last;
    195200        }
     201    } else {
     202        print "no rejection for imfile stdev\n";
    196203    }
    197204}
    198205
    199 # calculate the imfile ensemble statistics
     206# calculate the exposure ensemble statistics
    200207my $meanStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for means
    201208$meanStats->add_data(@means);
    202 my $stdevStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for stdevs
    203 $stdevStats->add_data(@stdevs);
    204 my $mean = $meanStats->mean();  # Mean of the sample of means
    205 my $meanStdev = $meanStats->standard_deviation(); # Stdev of the sample of means
    206 if (not defined $stdev) {
    207     $stdev = 0;
    208 }
    209 my $stdev = sqrt($stdevStats->mean()); # Root-Mean-Square of the sample of stdevs
    210 print "calculating root-mean-square of the sample stdevs: $meanStdev\n";
    211 
    212 ## Reject based on the imfile ensemble stats
    213 # reject if the imfile ensemble
    214 if (REJECT_SAMPLE_MEAN->{$det_type} && ($stdev > 0)) {
    215     if ($mean / $stdev > REJECT_SAMPLE_MEAN->{$det_type}) {
     209my $varianceStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for variances
     210$varianceStats->add_data(@variances);
     211
     212my $mean = $meanStats->mean();  # Mean of the imfile means
     213my $meanStdev = $meanStats->standard_deviation(); # Stdev of the imfile means
     214if (not defined $meanStdev) {
     215    # this is the case for Nimfile == 1
     216    $meanStdev = 0;
     217}
     218my $stdev = sqrt($varianceStats->mean()); # Root-Mean-Square of the imfile stdevs (root mean of variances)
     219print "exposure mean $mean, stdev $stdev, mean stdev $meanStdev\n";
     220
     221## Reject based on the exposure ensemble stats
     222# reject if the exposure ensemble mean is deviant
     223if (REJECT_EXPOSURE_MEAN->{$det_type}) {
     224    if (abs($mean) > REJECT_EXPOSURE_MEAN->{$det_type}) {
    216225        print "Rejecting exposure based on bad mean: " . ($mean / $stdev) . " vs " .
    217             REJECT_SAMPLE_MEAN->{$det_type} . "\n";
     226            REJECT_EXPOSURE_MEAN->{$det_type} . "\n";
    218227        $reject = 1;
    219228    }
    220 }
    221 if (REJECT_SAMPLE_STDEV->{$det_type}) {
    222     if ($stdev > REJECT_SAMPLE_STDEV->{$det_type}) {
     229} else {
     230    print "no rejection for imfile mean\n";
     231}
     232# reject if the exposure ensemble stdev is deviant
     233if (REJECT_EXPOSURE_STDEV->{$det_type}) {
     234    if ($stdev > REJECT_EXPOSURE_STDEV->{$det_type}) {
    223235        print "Rejecting exposure based on bad mean stdev: " . $stdev . " vs " .
    224             REJECT_SAMPLE_STDEV->{$det_type} . "\n";
     236            REJECT_EXPOSURE_STDEV->{$det_type} . "\n";
    225237        $reject = 1;
    226238    }
    227 }
    228 if (REJECT_SAMPLE_MEAN_STDEV->{$det_type}) {
    229     if ($meanStdev > REJECT_SAMPLE_MEAN_STDEV->{$det_type}) {
     239} else {
     240    print "no rejection for imfile stdev\n";
     241}
     242# reject if the exposure ensemble mean stdev is deviant
     243if (REJECT_EXPOSURE_MEAN_STDEV->{$det_type}) {
     244    if ($meanStdev > REJECT_EXPOSURE_MEAN_STDEV->{$det_type}) {
    230245        print "Rejecting exposure based on bad mean stdev: " . $meanStdev . " vs " .
    231             REJECT_SAMPLE_MEAN_STDEV->{$det_type} . "\n";
     246            REJECT_EXPOSURE_MEAN_STDEV->{$det_type} . "\n";
    232247        $reject = 1;
    233248    }
     249} else {
     250    print "no rejection for imfile mean stdev\n";
    234251}
    235252
Note: See TracChangeset for help on using the changeset viewer.