IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 19627


Ignore:
Timestamp:
Sep 21, 2008, 5:31:39 PM (18 years ago)
Author:
eugene
Message:

converting statistics analysis from perl script support to ppStatsFromMetadata (which can use pslib robust stats)

Location:
trunk/ippScripts/scripts
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/detrend_resid_exp.pl

    r19332 r19627  
    2121
    2222use IPC::Cmd 0.36 qw( can_run run );             # tools to run UNIX programs with control over I/O
     23# use IPC::Run qw ( start finish timeout );
     24use IPC::Run;
     25
    2326use PS::IPP::Metadata::Config;                   # tools to parse the psMetadataConfig files
    24 use PS::IPP::Metadata::Stats;
     27
     28# XXX drop: moved to ppStatsFromMetadata
     29# use PS::IPP::Metadata::Stats;
    2530
    2631use PS::IPP::Metadata::List qw( parse_md_list ); # tools to parse a metadata into a hash list
    … …  
    6974# load IPP config information for the specified camera
    7075$ipprc->define_camera($camera);
    71 if ($redirect) {
    72     my $logDest = $ipprc->filename("LOG.EXP", $outroot)
    73        or &my_die("Missing entry from camera config", $det_id, $iter, $exp_id, $PS_EXIT_CONFIG_ERROR);
    74     $ipprc->redirect_output($logDest);
    75 }
     76
     77my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $det_id, $iter, $exp_id, $PS_EXIT_CONFIG_ERROR);
     78
     79$ipprc->redirect_output($logDest) if $redirect;
    7680
    7781# Recipes to use based on reduction class
    … …  
    8387my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_RESID_' . uc($det_type)); # Recipe to use
    8488&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
    85 
    86 # values to extract from output metadata and the stats to calculate
    87 # XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
    88 # XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
    89 # XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
    90 my $STATS =
    91     [
    92         #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
    93         { name => "bg",             type => "mean",  flag => "-bg",            dtype => "float" },
    94         { name => "bg_mean_stdev",  type => "stdev", flag => "-bg_mean_stdev", dtype => "float" },
    95         { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev",      dtype => "float" },
    96         { name => "bg_skewness",    type => "mean",  flag => "-bg_skewness",   dtype => "float" },
    97         { name => "bg_kurtosis",    type => "mean",  flag => "-bg_kurtosis",   dtype => "float" },
    98         { name => "bin_stdev",      type => "rms",   flag => "-bin_stdev",     dtype => "float" },
    99         { name => "fringe_0",       type => "mean",  flag => "-fringe_0",      dtype => "float" },
    100         { name => "fringe_1",       type => "rms",   flag => "-fringe_1",      dtype => "float" },
    101         { name => "fringe_2",       type => "stdev", flag => "-fringe_2",      dtype => "float" },
    102         { name => "user_1",         type => "mean",  flag => "-user_1",        dtype => "float" }, # fringe residual
    103         { name => "user_2",         type => "rms",   flag => "-user_2",        dtype => "float" }, # fringe residual
    104         { name => "user_1",         type => "stdev", flag => "-user_3",        dtype => "float" }, # fringe residual
    105         ];
    106 my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
    10789
    10890# Look for programs we need
    … …  
    11092my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
    11193my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
     94my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
    11295if ($missing_tools) {
    11396    warn("Can't find required tools.");
    … …  
    11699
    117100# Get list of imfile files
     101my $cmdflags;
    118102my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
    119103{
    … …  
    142126        &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
    143127
    144     # Parse the statistics on the residual image
    145     $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
     128    # since I can't figure out how to do input and output within PERL, I'm writing to a temp file
     129    my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.stats.XXXX", UNLINK => !$save_temps );
     130    foreach my $line (@$stdout_buf) {
     131        print $statFile $line;
     132    }
     133    close $statFile;
     134
     135    $command = "$ppStatsFromMetadata $statName - DETREND_RESID_EXP";
     136    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     137        run(command => $command, verbose => $verbose);
     138    unless ($success) {
     139        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     140        warn("Unable to perform ppSTatsFromMetadata: $error_code\n");
     141        exit($error_code);
     142    }
     143
     144    foreach my $line (@$stdout_buf) {
     145        $cmdflags .= " $line";
     146    }
     147    print "cmdflags: $cmdflags\n";
    146148}
    147149
    … …  
    377379
    378380# basic ensemble stats
    379 my $mean               = $stats->value_for_flag ("-bg");
    380 my $meanStdev          = $stats->value_for_flag ("-bg_mean_stdev");
    381 my $stdev              = $stats->value_for_flag ("-bg_stdev");
    382 my $binStdev           = $stats->value_for_flag ("-bin_stdev");
    383 my $fringe_mean        = $stats->value_for_flag ("-fringe_0");
    384 my $fringe_err         = $stats->value_for_flag ("-fringe_1");
    385 my $fringe_mean_stdev  = $stats->value_for_flag ("-fringe_2");
    386 my $dfringe_mean       = $stats->value_for_flag ("-fringe_resid_0");
    387 my $dfringe_err        = $stats->value_for_flag ("-fringe_resid_1");
    388 my $dfringe_mean_stdev = $stats->value_for_flag ("-fringe_resid_2");
     381my $mean               = &value_for_flag ($cmdflags, "-bg");
     382my $meanStdev          = &value_for_flag ($cmdflags, "-bg_mean_stdev");
     383my $stdev              = &value_for_flag ($cmdflags, "-bg_stdev");
     384my $binStdev           = &value_for_flag ($cmdflags, "-bin_stdev");
     385my $fringe_mean        = &value_for_flag ($cmdflags, "-fringe_0");
     386my $fringe_err         = &value_for_flag ($cmdflags, "-fringe_1");
     387my $fringe_mean_stdev  = &value_for_flag ($cmdflags, "-fringe_2");
     388my $dfringe_mean       = &value_for_flag ($cmdflags, "-fringe_resid_0");
     389my $dfringe_err        = &value_for_flag ($cmdflags, "-fringe_resid_1");
     390my $dfringe_mean_stdev = &value_for_flag ($cmdflags, "-fringe_resid_2");
    389391
    390392# other stats (flux depends on bg and exp_time)
    … …  
    506508$command .= ' -reject' if $reject;
    507509$command .= " -dbname $dbname" if defined $dbname;
    508 $command .= $stats->cmdflags();
     510$command .= $cmdflags;
    509511
    510512unless ($no_update) {
    … …  
    527529}
    528530
     531
     532sub value_for_flag
     533{
     534    my $cmdflags = shift;
     535    my $flag = shift;
     536
     537    my $value = 0.0;
     538    if ($cmdflags =~ m|$flag|) {
     539        ($value) = $cmdflags =~ m|$flag\s+(\S+)|;
     540    }
     541    $value;
     542}
    529543
    530544sub my_die

  • trunk/ippScripts/scripts/detrend_resid_imfile.pl

    r19621 r19627  
    1616use IPC::Cmd 0.36 qw( can_run run );
    1717use PS::IPP::Metadata::Config;
    18 use PS::IPP::Metadata::Stats;
    1918use PS::IPP::Config 1.01 qw( :standard );
     19
     20# XXX drop (moved to ppStatsFromMetadata)
     21# use PS::IPP::Metadata::Stats;
    2022
    2123my $ipprc = PS::IPP::Config->new(); # IPP configuration
    … …  
    8890print "real recipe: $recipe\n";
    8991
    90 # values to extract from output metadata and the stats to calculate
    91 my $STATS =
    92    [
    93        #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
    94        { name => "ROBUST_MEDIAN",      type => "mean",  flag => "-bg",             dtype => "float" },
    95        { name => "ROBUST_MEDIAN",      type => "stdev", flag => "-bg_mean_stdev",  dtype => "float" },
    96        { name => "ROBUST_STDEV",       type => "rms",   flag => "-bg_stdev",       dtype => "float" },
    97        { name => "SAMPLE_SKEWNESS",    type => "mean",  flag => "-bg_skewness",    dtype => "float" },
    98        { name => "SAMPLE_KURTOSIS",    type => "mean",  flag => "-bg_kurtosis",    dtype => "float" },
    99        { name => "FRINGE_0",           type => "mean",  flag => "-fringe_0",       dtype => "float" },
    100        { name => "FRINGE_ERR_0",       type => "rms",   flag => "-fringe_1",       dtype => "float" },
    101        { name => "FRINGE_0",           type => "stdev", flag => "-fringe_2",       dtype => "float" },
    102        { name => "FRINGE_RESID_0",     type => "mean",  flag => "-fringe_resid_0", dtype => "float" },
    103        { name => "FRINGE_RESID_ERR_0", type => "rms",   flag => "-fringe_resid_1", dtype => "float" },
    104        { name => "FRINGE_RESID_0",     type => "stdev", flag => "-fringe_resid_2", dtype => "float" },
    105    ];
    106 my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
    107 
    108 my $BINNED_STATS =
    109    [
    110        { name => "ROBUST_STDEV",   type => "rms",   flag => "-bin_stdev" },
    111    ];
    112 my $binnedStats = PS::IPP::Metadata::Stats->new($BINNED_STATS); # Stats parser
    113 
    11492# Flags to specify the particular detrend to use
    11593use constant DETRENDS => {
    … …  
    140118my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
    141119my $ppStats = can_run('ppStats') or (warn "Can't find ppStats" and $missing_tools = 1);
     120my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
    142121if ($missing_tools) {
    143122    warn("Can't find required tools.");
    … …  
    161140my $outputStats = $ipprc->filename("PPIMAGE.STATS",  $outroot, $class_id);
    162141my $traceDest   = $ipprc->filename("TRACE.IMFILE",   $outroot, $class_id);
     142
     143my $cmdflags;
    163144
    164145# Run ppImage & ppStats
    … …  
    184165        &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code);
    185166    }
     167
    186168    &my_die("Couldn't find expected output file: $outputName", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputName);
    187     &my_die("Couldn't find expected output file: $outputStats", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats);
    188169    &my_die("Couldn't find expected output file: $bin1Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin1Name);
    189170    &my_die("Couldn't find expected output file: $bin2Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin2Name);
    190171
    191     # Load the raw output stats file
    192     my $statsFile;              # File handle
    193     open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
    194     my @contents = <$statsFile>; # Contents of file
    195     close $statsFile;
    196 
    197     # Parse the stats file contents into a metadata
    198     my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
    199     my $metadata = $mdcParser->parse(join "", @contents) or &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
    200 
    201     # Parse the statistics on the residual image
    202     $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
     172    my $outputStatsReal = $ipprc->file_resolve($outputStats);
     173    &my_die("Couldn't find expected output file: $outputStats", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal;
     174
     175    # ppStatsFromMetadata $outputStats - DETREND_RESID_IMFILE
     176    $command = "$ppStatsFromMetadata $outputStatsReal - DETREND_RESID_IMFILE";
     177    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     178        run(command => $command, verbose => $verbose);
     179    unless ($success) {
     180        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     181        &my_die("Unable to perform ppStatsFromMetadata: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code);
     182    }
     183    $cmdflags = $stdout_buf; chomp $cmdflags;
    203184
    204185    # run ppStats on the binned image
    205     $command = "$ppStats -recipe PPSTATS RESIDUAL $bin2Name";
     186    $command = "$ppStats -recipe PPSTATS RESIDUAL $bin2Name | $ppStatsFromMetadata - - DETREND_RESID_IMFILE_BINNED";
    206187    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    207188        run(command => $command, verbose => $verbose);
    … …  
    210191        &my_die("Unable to perform ppStats: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code);
    211192    }
    212 
    213     # Parse the output contents into a metadata
    214     my $binnedMetadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata output", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
    215 
    216     # parse the binned image statistics
    217     $binnedStats->parse($binnedMetadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
     193    $cmdflags .= $stdout_buf; chomp $cmdflags;
    218194}
    219195
    … …  
    230206$command .= " -path_base $outroot";
    231207$command .= " -dbname $dbname" if defined $dbname;
    232 $command .= $stats->cmdflags();
    233 $command .= $binnedStats->cmdflags();
     208$command .= $cmdflags;
    234209
    235210# Add the processed file to the database
Note: See TracChangeset for help on using the changeset viewer.