IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jul 4, 2007, 1:53:22 PM (19 years ago)
Author:
eugene
Message:

extensive changes to use new stats calculation tools; general cleanups

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/detrend_norm_exp.pl

    r13760 r14009  
    1717use PS::IPP::Metadata::Config;
    1818use PS::IPP::Metadata::List qw( parse_md_list );
    19 use Statistics::Descriptive;
    2019use File::Temp qw( tempfile );
    2120
    … …  
    4746
    4847pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    49 pod2usage(
    50     -msg => "Required options: --det_id --iteration --camera --det_type",
    51     -exitval => 3,
    52 ) unless defined $det_id
    53      and defined $iter
    54      and defined $camera
    55      and defined $det_type;
     48pod2usage( -msg => "Required options: --det_id --iteration --camera --det_type",
     49           -exitval => 3)
     50    unless defined $det_id
     51    and defined $iter
     52    and defined $camera
     53    and defined $det_type;
    5654
    5755$ipprc->define_camera($camera);
    5856
    59 # Recipes to use, as a function of the detrend type
    60 use constant RECIPES => {
    61     'bin1' => {         # We're creating a master --- already processed the input
    62         'bias'     => 'PPIMAGE_J1_IMAGE_B',     # Bias only
    63         'dark'     => 'PPIMAGE_J1_IMAGE_B',     # Dark only
    64         'shutter'  => 'PPIMAGE_J1_IMAGE_F',     # Shutter only
    65         'flat'     => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
    66         'domeflat' => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
    67         'skyflat'  => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
    68         'fringe'   => 'PPIMAGE_J1_IMAGE_R',     # Fringe only
    69     },
    70     'bin2' => {         # We're checking the master --- input is not already processed
    71         'bias'     => 'PPIMAGE_J2_IMAGE_B',     # Bias only
    72         'dark'     => 'PPIMAGE_J2_IMAGE_B',     # Dark only
    73         'shutter'  => 'PPIMAGE_J2_IMAGE_F',     # Shutter only
    74         'flat'     => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
    75         'domeflat' => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
    76         'skyflat'  => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
    77         'fringe'   => 'PPIMAGE_J2_IMAGE_R',     # Fringe only
    78     },
    79 };
     57# Recipes to use based on reduction class
     58$reduction = 'DETREND' unless defined $reduction;
     59
     60my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE' . uc($det_type); # Recipe to use
     61&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
     62
     63my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE' . uc($det_type); # Recipe to use
     64&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
     65
     66# values to extract from output metadata and the stats to calculate
     67# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
     68# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
     69# XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
     70my $STATS =
     71   [   
     72       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
     73       { name => "bg",             type => "mean",  flag => "-bg" },
     74       { name => "bg",             type => "stdev", flag => "-bg_mean_stdev" },
     75       { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
     76       # { name => "bg_mean_stdev",  type => "rms",   flag => "-bg_mean_stdev" },
     77   ];
    8078
    8179# Look for programs we need
    … …  
    8785    exit($PS_EXIT_CONFIG_ERROR);
    8886}
    89 
    90 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    9187
    9288# Get list of component files
    … …  
    10197        &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $error_code);
    10298    }
     99
     100    # convert stdout to a metadata
     101    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
    103102    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    104103        &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR);
     104
     105    # parse the file info in the metadata
    105106    $files = parse_md_list($metadata) or
    106107        &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR);
    107 }
    108 
    109 # Gather the statistics
    110 my ($bg, $bg_stdev, $bg_mean_stdev); # The statistics triplet
    111 {
    112     my @backgrounds;            # Array of backgrounds in each component
    113     my @variances;    # Array of variances for each component
    114     foreach my $file (@$files) {
    115         &my_die("Unable to find class id", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $file->{class_id};
    116         my $class_id = $file->{class_id};
    117         &my_die("Unable to find bg for class_id=$class_id", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $file->{bg};
    118         &my_die("Unable to find bg_mean_stdev for class_id=$class_id", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $file->{bg_mean_stdev};
    119         push @backgrounds, $file->{bg};
    120         push @variances, $file->{bg_stdev}**2;
    121     }
    122 
    123     {
    124         my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
    125         $stats->add_data(@backgrounds);
    126         $bg = ($stats->mean() or 'NAN');
    127         $bg_mean_stdev = ($stats->standard_deviation() or 'NAN');
    128     }
    129     {
    130         my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
    131         $stats->add_data(@variances);
    132         $bg_stdev = (sqrt( $stats->mean() ) or 'NAN');
     108
     109    # parse the stats in the metadata
     110    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
     111    unless ($stats->parse($metadata)) {
     112        &my_die("Unable to find all values in statistics output.\n", $det_id, $iter, $PS_EXIT_PROG_ERROR);
    133113    }
    134114}
    … …  
    136116my ($list1File, $list1Name) = tempfile( "$camera.$det_type.norm.$det_id.$iter.b1.list.XXXX", UNLINK => 1 );
    137117my ($list2File, $list2Name) = tempfile( "$camera.$det_type.norm.$det_id.$iter.b2.list.XXXX", UNLINK => 1 );
    138 my @means;                      # Array of means
    139 my @stdevs;                     # Array of stdevs
    140118foreach my $file (@$files) {
    141119    print $list1File ( $ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n");
    142120    print $list2File ( $ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n");
    143     push @means, $file->{bg};
    144     push @stdevs, $file->{bg_stdev};
    145121}
    146122close $list1File;
    … …  
    152128my $jpeg1Name = $ipprc->filename("PPIMAGE.JPEG1", $outputRoot); # Binned JPEG #1
    153129my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2
    154 
    155 # Recipes to use in processing
    156 my $recipe1 = RECIPES->{"bin1"}->{lc($det_type)};
    157 &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
    158 
    159 my $recipe2 = RECIPES->{"bin2"}->{lc($det_type)};
    160 &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
    161130
    162131unless ($no_op) {
    … …  
    178147}
    179148
     149# command to update the database
     150my $command = "$dettool -addnormalizedexp";
     151$command .= " -det_id $det_id";
     152$command .= " -iteration $iter";
     153$command .= " -recip $recipe1,$recipe2";
     154$command .= " -path_base $outputRoot ";
     155$command .= " -dbname $dbname" if defined $dbname;
     156
     157# add in the elements from the selected stats above
     158foreach my $entry (@$STATS) {
     159    my $value = $entry->{value};
     160    my $flag = $entry->{flag};
     161    $command .= " $flag $value";
     162}
     163
     164# Add the processed file to the database
    180165unless ($no_update) {
    181     my $command = "$dettool -addnormalizedexp -det_id $det_id -iteration $iter";
    182     $command .= " -recip $recipe1,$recipe2 -path_base $outputRoot ";
    183     $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
    184     $command .= " -dbname $dbname" if defined $dbname;
    185166    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    186167        run(command => $command, verbose => 1);
    … …  
    189170        &my_die("Unable to perform dettool -addnormalizedexp: $error_code", $det_id, $iter, $error_code);
    190171    }
    191 }
    192 
     172} else {
     173    print "skipping command: $command\n";
     174}
    193175
    194176sub my_die
    … …  
    201183    carp($msg);
    202184    if ($det_id and $iter and not $no_update) {
    203         my $command = "$dettool -addprocessedimfile -det_id $det_id -iter $iter -code $exit_code";
     185        my $command = "$dettool -addprocessedimfile";
     186        $command .= " -det_id $det_id";
     187        $command .= " -iter $iter";
     188        $command .= " -code $exit_code";
    204189        $command .= " -dbname $dbname" if defined $dbname;
    205190###        system ($command);
Note: See TracChangeset for help on using the changeset viewer.