IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jul 4, 2007, 1:53:22 PM (19 years ago)
Author:
eugene
Message:

extensive changes to use new stats calculation tools; general cleanups

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/detrend_process_exp.pl

    r13760 r14009  
    1717use PS::IPP::Metadata::Config;
    1818use PS::IPP::Metadata::List qw( parse_md_list );
    19 use Statistics::Descriptive;
     19use File::Temp qw( tempfile );
    2020
    2121use PS::IPP::Config qw($PS_EXIT_SUCCESS
    … …  
    2929                       );
    3030my $ipprc = PS::IPP::Config->new(); # IPP configuration
    31 use File::Temp qw( tempfile );
    3231
    3332use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
    … …  
    4746
    4847pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    49 pod2usage(
    50     -msg => "Required options: --det_id --det_type --exp_tag --camera",
    51     -exitval => 3,
    52 ) unless defined $det_id
     48pod2usage( -msg => "Required options: --det_id --det_type --exp_tag --camera",
     49           -exitval => 3)
     50    unless defined $det_id
    5351    and defined $det_type
    5452    and defined $exp_tag
    … …  
    5755$ipprc->define_camera($camera);
    5856
    59 # Recipes to use, as a function of the detrend type
    60 use constant RECIPES => {
    61     'bin1' => {         # We're creating a master --- already processed the input
    62         'bias'     => 'PPIMAGE_J1_IMAGE_B',     # Bias only
    63         'dark'     => 'PPIMAGE_J1_IMAGE_B',     # Dark only
    64         'shutter'  => 'PPIMAGE_J1_IMAGE_F',     # Shutter only
    65         'flat'     => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
    66         'domeflat' => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
    67         'skyflat'  => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
    68         'fringe'   => 'PPIMAGE_J1_IMAGE_R',     # Fringe only
    69     },
    70     'bin2' => {         # We're checking the master --- input is not already processed
    71         'bias'     => 'PPIMAGE_J2_IMAGE_B',     # Bias only
    72         'dark'     => 'PPIMAGE_J2_IMAGE_B',     # Dark only
    73         'shutter'  => 'PPIMAGE_J2_IMAGE_F',     # Shutter only
    74         'flat'     => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
    75         'domeflat' => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
    76         'skyflat'  => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
    77         'fringe'   => 'PPIMAGE_J2_IMAGE_R',     # Fringe only
    78     },
    79 };
     57# Recipes to use based on reduction class
     58$reduction = 'DETREND' unless defined $reduction;
     59
     60my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE' . uc($det_type); # Recipe to use
     61&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
     62
     63my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE' . uc($det_type); # Recipe to use
     64&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
     65
     66# values to extract from output metadata and the stats to calculate
     67# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
     68# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
     69# XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
     70my $STATS =
     71   [   
     72       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
     73       { name => "bg",             type => "mean",  flag => "-bg" },
     74       { name => "bg",             type => "stdev", flag => "-bg_mean_stdev" },
     75       { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
     76       # { name => "bg_mean_stdev",  type => "rms",   flag => "-bg_mean_stdev" },
     77   ];
    8078
    8179# Look for programs we need
    … …  
    8785    exit($PS_EXIT_CONFIG_ERROR);
    8886}
    89 
    90 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    9187
    9288# Get list of component files
    … …  
    10197        &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $exp_tag, $error_code);
    10298    }
     99
     100    # convert stdout to a metadata
     101    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
    103102    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    104103        &my_die("Unable to parse metadata config doc", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR);
     104
     105    # parse the file info in the metadata
    105106    $files = parse_md_list($metadata) or
    106107        &my_die("Unable to parse metadata list", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR);
    107 }
    108 
    109 # Gather the statistics
    110 my ($bg, $bg_stdev, $bg_mean_stdev); # The statistics triplet
    111 {
    112     my @backgrounds;            # Array of backgrounds in each component
    113     my @variances;    # Array of variances for each component
    114     foreach my $file (@$files) {
    115         &my_die("Unable to find class id", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{class_id};
    116         my $class_id = $file->{class_id};
    117         &my_die("Unable to find bg for class_id=$class_id", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{bg};
    118         &my_die("Unable to find bg_mean_stdev for class_id=$class_id", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{bg_mean_stdev};
    119         push @backgrounds, $file->{bg};
    120         push @variances, $file->{bg_stdev}**2;
    121     }
    122 
    123     {
    124         my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
    125         $stats->add_data(@backgrounds);
    126         $bg = ($stats->mean() or 'NAN');
    127         $bg_mean_stdev = ($stats->standard_deviation() or 'NAN');
    128     }
    129     {
    130         my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
    131         $stats->add_data(@variances);
    132         $bg_stdev = (sqrt( $stats->mean() ) or 'NAN');
     108
     109    # parse the stats in the metadata
     110    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
     111    unless ($stats->parse($metadata)) {
     112        &my_die("Unable to find all values in statistics output.\n", $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
    133113    }
    134114}
    … …  
    137117my ($list1File, $list1Name) = tempfile( "$exp_tag.detproc.$det_id.b1.list.XXXX", UNLINK => 1 );
    138118my ($list2File, $list2Name) = tempfile( "$exp_tag.detproc.$det_id.b2.list.XXXX", UNLINK => 1 );
    139 my @means;                      # Array of means
    140 my @stdevs;                     # Array of stdevs
    141119foreach my $file (@$files) {
    142120    print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n");
    143121    print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n");
    144     push @means, $file->{bg};
    145     push @stdevs, $file->{bg_stdev};
    146122}
    147123close $list1File;
    … …  
    153129my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outputRoot); # Binned JPEG #1
    154130my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2
    155 
    156 # Recipes to use in processing
    157 my $recipe1 = RECIPES->{"bin1"}->{lc($det_type)};
    158 &my_die("Unrecognised detrend type: $det_type", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
    159 
    160 my $recipe2 = RECIPES->{"bin2"}->{lc($det_type)};
    161 &my_die("Unrecognised detrend type: $det_type", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
    162131
    163132unless ($no_op) {
    … …  
    187156}
    188157
     158# Command to update the database
     159my $command = "$dettool -addprocessedexp";
     160$command .= " -det_id $det_id";
     161$command .= " -exp_tag $exp_tag";
     162$command .= " -recip $recipe1,$recipe2 -path_base $outputRoot";
     163$command .= " -dbname $dbname" if defined $dbname;
     164
     165# add in the elements from the selected stats above
     166foreach my $entry (@$STATS) {
     167    my $value = $entry->{value};
     168    my $flag = $entry->{flag};
     169    $command .= " $flag $value";
     170}
     171
     172# Add the processed file to the database
    189173unless ($no_update) {
    190     my $command = "$dettool -addprocessedexp -det_id $det_id -exp_tag $exp_tag";
    191     $command .= " -recip $recipe1,$recipe2 -path_base $outputRoot";
    192     $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
    193     $command .= " -dbname $dbname" if defined $dbname;
    194174    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    195175        run(command => $command, verbose => 1);
    … …  
    199179        exit($error_code);
    200180    }
    201 }
    202 
     181} else {
     182    print "skipping command: $command\n";
     183}
    203184
    204185sub my_die
    … …  
    211192    carp($msg);
    212193    if ($det_id and $exp_tag and not $no_update) {
    213         my $command = "$dettool -addprocessedexp -det_id $det_id -exp_tag $exp_tag -code $exit_code";
     194        my $command = "$dettool -addprocessedexp";
     195        $command .= " -det_id $det_id";
     196        $command .= " -exp_tag $exp_tag";
     197        $command .= " -code $exit_code";
    214198        $command .= " -dbname $dbname" if defined $dbname;
    215199###        system ($command);
Note: See TracChangeset for help on using the changeset viewer.