IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 8612


Ignore:
Timestamp:
Aug 25, 2006, 3:18:17 PM (20 years ago)
Author:
Paul Price
Message:

We get the statistics out of the database, not from ppImage. We are
given binned FITS files, and just have to JPEG them; need to do this
twice --- once for each binning level (this should be cleaned up).
Set the rejection levels high --- they will eventually come back down.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/detrend_reject_imfile.pl

    r8507 r8612  
    77use PS::IPP::Metadata::Config;
    88use PS::IPP::Metadata::List qw( parse_md_list );
     9use Statistics::Descriptive;
    910use Data::Dumper;
    1011
    … …  
    2122# This measures how close it is to what's expected
    2223use constant REJECT_INDIVIDUAL_MEAN => {
    23     'bias' => 0.01,             # Should be fairly flat; some CRs
    24     'dark' => 0.05,             # Lots of CRs
     24    'bias' => 5,                # Should be fairly flat; some CRs
     25    'dark' => 10,               # Lots of CRs
    2526    'flat' => undef             # Can't define expected value (depends on exposure level)
    2627    };
    … …  
    2930# This measures how much variation there is in each component
    3031use constant REJECT_INDIVIDUAL_STDEV => {
    31     'bias' => 0.1,              # Should be fairly flat; some CRs
    32     'dark' => 0.5,              # Lots of CRs
    33     'flat' => 100               # Stars and galaxies
     32    'bias' => 10,               # Should be fairly flat; some CRs
     33    'dark' => 100,              # Lots of CRs
     34    'flat' => 1000              # Stars and galaxies
    3435    };
    3536
    … …  
    3738# This measures how close it is to what's expected
    3839use constant REJECT_SAMPLE_MEAN => {
    39     'bias' => 0.01,             # Should be little variation between chips
    40     'dark' => 0.05,             # Could be some glow on some chips
     40    'bias' => 10,               # Should be little variation between chips
     41    'dark' => 100,              # Could be some glow on some chips
    4142    'flat' => undef             # Can't define expected value (depends on exposure level)
    4243    };
    … …  
    4546# This measures how much variation there is across the components
    4647use constant REJECT_SAMPLE_STDEV => {
    47     'bias' => 0.1,              # Should be little variation between chips
    48     'dark' => 0.5,              # Could be some glow on some chips
    49     'flat' => 10                # Could be features on some chips, but all should be about the same
     48    'bias' => 10,               # Should be little variation between chips
     49    'dark' => 100,              # Could be some glow on some chips
     50    'flat' => 500               # Could be features on some chips, but all should be about the same
    5051    };
    5152
    … …  
    7172my $files;                      # Array of component files
    7273{
    73     my $command = "$dettool -residimfile -det_id $detId -iter $iter -exp_id $expId"; # Command to run
     74    my $command = "$dettool -residimfile -det_id $detId -iteration $iter -exp_id $expId"; # Command to run
    7475    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    7576        run(command => $command, verbose => 1);
    … …  
    7980}
    8081
    81 # Generate the file list
    82 my $outputName = $detId . '_' . $iter; # Root output name
    83 my $statName = $outputName . '.stats';
    84 my $listName = $outputName . '_jpeg.list'; # Name for the input file list
    85 open my $listFile, '>' . $listName;
     82# Generate the file list, and get the statistics
     83my $outputName = $detType . '_' . $detId . '_' . $iter; # Root output name
     84my $list1Name = $outputName . '_jpeg1.list'; # Name for the input file list for binning 1
     85my $list2Name = $outputName . '_jpeg2.list'; # Name for the input file list for binning 2
     86my @means;                      # Array of means
     87my @stdevs;                     # Array of stdevs
     88open my $list1File, '>' . $list1Name;
     89open my $list2File, '>' . $list2Name;
    8690foreach my $file (@$files) {
    87     print $listFile $file->{uri} . "\n";
     91    print $list1File $file->{b1_uri} . "\n";
     92    print $list2File $file->{b2_uri} . "\n";
     93    push @means, $file->{bg};
     94    push @stdevs, $file->{bg_stdev};
    8895}
    89 close $listFile;
     96close $list1File;
     97close $list2File;
    9098
    9199# Output products --- need to synch with the camera configuration!
    … …  
    93101my $jpeg2Name = $outputName . ".b2.jpeg"; # Binned JPEG #2
    94102
    95 # Make the jpeg
     103# Make the jpeg for binning 1
    96104{
    97     my $command = "$ppImage -list $listName $outputName -stat $statName" .
    98         "-recipe PPIMAGE " .  RECIPE; # Command to run
     105    my $command = "$ppImage -list $list1Name $outputName -recipe PPIMAGE " . RECIPE; # Command to run
    99106    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    100107        run(command => $command, verbose => 1);
    101     die "Unable to find expected output file: $jpeg1Name\n" if not -f $jpeg1Name;
    102     die "Unable to find expected output file: $jpeg2Name\n" if not -f $jpeg2Name;
    103     die "Unable to find expected output file: $statName\n" if not -f $statName;
     108#    die "Unable to find expected output file: $jpeg1Name\n" if not -f $jpeg1Name;
    104109}
    105110
    106 # Parse the statistics
    107 my $stats;                      # Statistics from ppImage
     111# Make the jpeg for binning 2
    108112{
    109     open my $statsFile, $statName or die "Can't open statistics file $statName: $!\n";
    110     my @contents = <$statsFile>; # Contents of file
    111     close $statsFile;
    112     my $metadata = $mdcParser->parse(join "", @contents); # Metadata from the stats
    113     $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
    114     $stats->parse($metadata) or die "Unable to find all values in statistics output.\n";
     113    my $command = "$ppImage -list $list2Name $outputName -recipe PPIMAGE " . RECIPE; # Command to run
     114    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     115        run(command => $command, verbose => 1);
     116#    die "Unable to find expected output file: $jpeg2Name\n" if not -f $jpeg2Name;
    115117}
    116118
    … …  
    128130
    129131# Reject based on the individual stats
    130 my @means = $stats->bg_data();  # Array of means
    131 my @stdevs = $stats->bg_stdev_data(); # Array of stdevs
    132132my $reject = 0;                 # Rejection flag
    133133die "Number of means and number of stdevs differ!\n" if scalar @means != scalar @stdevs;
    … …  
    137137    my $stdev = $stdevs[$i];    # Stdev for this component
    138138
    139     if ((defined REJECT_INDIVIDUAL_MEAN->{$detType} and $mean / $stdev > REJECT_INDIVIDUAL_MEAN->{$detType})
    140         or
    141         (defined REJECT_INDIVIDUAL_STDEV->{$detType} and $stdev > REJECT_INDIVIDUAL_STDEV->{$detType})) {
     139    if (defined REJECT_INDIVIDUAL_MEAN->{$detType}
     140        and $stdev > 0 and
     141        $mean / $stdev > REJECT_INDIVIDUAL_MEAN->{$detType}) {
     142        print "Rejecting exposure based on bad individual mean for component $i: " .
     143            ($mean / $stdev) . " vs " . REJECT_INDIVIDUAL_MEAN->{$detType} . "\n";
     144        $reject = 1;
     145        last;
     146    } elsif (defined REJECT_INDIVIDUAL_STDEV->{$detType} and
     147             $stdev > REJECT_INDIVIDUAL_STDEV->{$detType}) {
     148        print "Rejecting exposure based on bad individual stdev for component $i: " .
     149            $stdev . " vs " . REJECT_INDIVIDUAL_STDEV->{$detType} . "\n";
    142150        $reject = 1;
    143151        last;
    … …  
    146154
    147155# Reject based on the sample stats
    148 my $mean = $stats->bg_mean;     # Mean of the sample of means
    149 my $stdev = $stats->bg_stdev; # Stdev of the sample of means
    150 my $meanStdev = $stats->bg_mean_stdev; # Mean of the sample of stdevs
    151 
    152 if ((defined REJECT_SAMPLE_MEAN->{$detType} and $mean / $stdev > REJECT_SAMPLE_MEAN->{$detType})
    153     or
    154     (defined REJECT_SAMPLE_STDEV->{$detType} and $meanStdev > REJECT_SAMPLE_STDEV->{$detType})) {
     156my $meanStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for means
     157$meanStats->add_data(@means);
     158my $stdevStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for stdevs
     159$stdevStats->add_data(@stdevs);
     160my $mean = $meanStats->mean();  # Mean of the sample of means
     161my $stdev = $meanStats->standard_deviation(); # Stdev of the sample of means
     162if (not defined $stdev) {
     163    $stdev = 0;
     164}
     165my $meanStdev = $stdevStats->mean(); # Mean of the sample of stdevs
     166if (defined REJECT_SAMPLE_MEAN->{$detType} and
     167    $stdev != 0 and
     168    $mean / $stdev > REJECT_SAMPLE_MEAN->{$detType}) {
     169    print "Rejecting exposure based on bad mean: " . ($mean / $stdev) . " vs " .
     170        REJECT_SAMPLE_MEAN->{$detType} . "\n";
     171    $reject = 1;
     172} elsif (defined REJECT_SAMPLE_STDEV->{$detType} and
     173         $meanStdev > REJECT_SAMPLE_STDEV->{$detType}) {
     174    print "Rejecting exposure based on bad mean stdev: " . $meanStdev . " vs " .
     175        REJECT_SAMPLE_STDEV->{$detType} . "\n";
    155176    $reject = 1;
    156177}
    157178
    158 
    159179# Add the result into the database
    160180{
    161     my $command = "dettool -addresidexp -det_id $detId -iter $iter -exp_id $expId " .
     181    my $command = "$dettool -addresidexp -det_id $detId -iteration $iter -exp_id $expId " .
    162182        "-recip " . RECIPE() . " -b1_uri $jpeg1Name -b2_uri $jpeg2Name -bg $mean -bg_stdev $stdev " .
    163183        "-bg_mean_stdev $meanStdev";    # Command to run
Note: See TracChangeset for help on using the changeset viewer.