IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 19942


Ignore:
Timestamp:
Oct 7, 2008, 9:56:11 AM (18 years ago)
Author:
eugene
Message:

converting to ppStatsFromMetadata

Location:
trunk/ippScripts/scripts
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/camera_exp.pl

    r19938 r19942  
    1919use IPC::Cmd 0.36 qw( can_run run );
    2020use PS::IPP::Metadata::Config;
    21 use PS::IPP::Metadata::Stats;
    2221use PS::IPP::Metadata::List qw( parse_md_list );
    2322use PS::IPP::Config 1.01 qw( :standard );
    … …  
    7978&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe1;
    8079
    81 # values to extract from output metadata and the stats to calculate
    82 # these should be coming from the psastro results
    83 my $CHIPSTATS =
    84     [
    85         #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
    86         { name => "bg",             type => "mean",  flag => "-bg",             dtype => "float" },
    87         { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev",       dtype => "float" },
    88         { name => "bg_mean_stdev",  type => "stdev", flag => "-bg_mean_stdev",  dtype => "float" },
    89         { name => "bias",           type => "mean",  flag => "-bias",           dtype => "float" },
    90         { name => "bias_stdev",     type => "rms",   flag => "-bias_stdev",     dtype => "float" },
    91         { name => "fringe_0",       type => "mean",  flag => "-fringe_0",       dtype => "float" },
    92         { name => "fringe_1",       type => "rms",   flag => "-fringe_1",       dtype => "float" },
    93         { name => "fringe_2",       type => "stdev", flag => "-fringe_2",       dtype => "float" },
    94         { name => "ap_resid",       type => "mean",  flag => "-ap_resid",       dtype => "float" },
    95         { name => "ap_resid_stdev", type => "rms",   flag => "-ap_resid_stdev", dtype => "float" },
    96         { name => "fwhm_major",     type => "mean",  flag => "-fwhm_major",     dtype => "float" },
    97         { name => "fwhm_minor",     type => "mean",  flag => "-fwhm_minor",     dtype => "float" },
    98         { name => "n_stars",        type => "sum",   flag => "-n_stars",        dtype => "int"   },
    99         { name => "n_extended",     type => "sum",   flag => "-n_extended",     dtype => "int"   },
    100         { name => "n_cr",           type => "sum",   flag => "-n_cr",           dtype => "int"   },
    101         ];
    102 my $chipStats = PS::IPP::Metadata::Stats->new($CHIPSTATS); # Stats parser
    103 
    104 # values to extract from camera-level output metadata and the stats to calculate
    105 my $CAMSTATS =
    106    [
    107        #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
    108       { name => "CERROR",         type => "rms",   flag => "-sigma_ra",       dtype => "float" },
    109       { name => "CERROR",         type => "rms",   flag => "-sigma_dec",      dtype => "float" },
    110 #      { name => "DT_ASTR",        type => "sum",   flag => "-dtime_astrom",   dtype => "float" },
    111       { name => "NASTRO",         type => "sum",   flag => "-n_astrom",       dtype => "int"   },
    112 #     { name => "ZP??",           type => "mean",  flag => "-zp_mean",        dtype => "float" },
    113 #     { name => "ZP??",           type => "rms",   flag => "-zp_stdev",       dtype => "float" },
    114    ];
    115 my $camStats = PS::IPP::Metadata::Stats->new($CAMSTATS); # Stats parser
    116 
    11780# Look for programs we need
    11881my $missing_tools;
    … …  
    12083my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
    12184my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
     85my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
    12286
    12387# test for addstar and psastro:
    … …  
    13195
    13296my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
     97
     98my $cmdflags;
    13399
    134100# Get list of component files
    … …  
    150116        &my_die("Unable to parse metadata list", $cam_id, $PS_EXIT_PROG_ERROR);
    151117
    152     # extract the stats from the metadata
    153     unless ($chipStats->parse($metadata)) {
    154         &my_die("Unable to find all values in statistics output.\n", $cam_id, $PS_EXIT_PROG_ERROR);
     118    # since I can't figure out how to do input and output within PERL, I'm writing to a temp file
     119    my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.cm.$cam_id.stats.XXXX", UNLINK => !$save_temps );
     120    print "saving stats to $statName\n";
     121    foreach my $line (@$stdout_buf) {
     122        print $statFile $line;
     123    }
     124    close $statFile;
     125
     126    # parse the stats in the metadata file
     127    $command = "$ppStatsFromMetadata $statName - CAMERA_EXP_IMFILE";
     128    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     129        run(command => $command, verbose => $verbose);
     130    unless ($success) {
     131        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     132        warn("Unable to perform ppStatsFromMetadata: $error_code\n");
     133        exit($error_code);
     134    }
     135
     136    foreach my $line (@$stdout_buf) {
     137        $cmdflags .= " $line";
    155138    }
    156139}
    … …  
    248231    if ($chipObjectsExist) {
    249232        # run psastro on the chipObjects, producing fpaObjects
    250         # XXX add a ppStats call which will collect the astrometry stats
    251233        my $command;
    252234        $command  = "$psastro -list $list3Name $outroot";
    … …  
    256238        my $do_stats;
    257239        if ($run_state eq 'new') {
    258             $command .= " -stats $fpaStats -recipe PPSTATS CHIPSTATS";
     240            $command .= " -stats $fpaStats -recipe PPSTATS CAMSTATS";
    259241            $command .= " -dumpconfig $configuration";
    260242            $do_stats = 1;
    … …  
    275257
    276258        if ($do_stats) {
    277             &my_die("Unable to find expected output file: $fpaStats",   $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects);
    278 
    279             # Get the statistics on the processed image
    280             my $statsFile;              # File handle
    281             open $statsFile, $ipprc->file_resolve($fpaStats) or &my_die("Can't open statistics file $fpaStats: $!", $cam_id, $PS_EXIT_SYS_ERROR);
    282             my @contents = <$statsFile>; # Contents of file
    283             close $statsFile;
    284 
    285             # parse the statistics MDC file
    286             my $mdcParser = PS::IPP::Metadata::Config->new();   # Parser for metadata config files
    287             my $metadata = $mdcParser->parse(join "", @contents);
    288             unless ($metadata) {
    289                 &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR);
    290             }
    291 
    292             # extract the stats from the metadata
    293             unless ($camStats->parse($metadata)) {
    294                 &my_die("Failure extracting metadata from the statistics output file.\n", $cam_id, $PS_EXIT_PROG_ERROR);
    295             }
     259            my $fpaStatsReal = $ipprc->file_resolve($fpaStats);
     260            &my_die("Couldn't find expected output file: $fpaStats", $cam_id, $PS_EXIT_SYS_ERROR) unless -f $fpaStatsReal;
     261
     262            # parse stats from metadata
     263            $command = "$ppStatsFromMetadata $fpaStatsReal - CAMERA_EXP_FPA";
     264            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     265                run(command => $command, verbose => $verbose);
     266            unless ($success) {
     267                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     268                &my_die("Unable to perform ppStatsFromMetadata: $error_code", $cam_id, $error_code);
     269            }
     270            foreach my $line (@$stdout_buf) {
     271                $cmdflags .= " $line";
     272            }
     273            chomp $cmdflags;
    296274        }
    297275
    … …  
    340318}
    341319
    342 
    343320my $fpaCommand = "$camtool -cam_id $cam_id";
    344321if ($run_state eq 'new') {
    … …  
    346323    $fpaCommand .= " -uri UNKNOWN";
    347324    $fpaCommand .= " -path_base $outroot";
    348     $fpaCommand .= $chipStats->cmdflags();
    349     $fpaCommand .= $camStats->cmdflags();
     325    $fpaCommand .= " $cmdflags";
    350326    $fpaCommand .= " -hostname $host" if defined $host;
    351327    $fpaCommand .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));

  • trunk/ippScripts/scripts/chip_imfile.pl

    r19938 r19942  
    1919use IPC::Cmd 0.36 qw( can_run run );
    2020use PS::IPP::Metadata::Config;
    21 use PS::IPP::Metadata::Stats;
    2221use PS::IPP::Config 1.01 qw( :standard );
    2322
    … …  
    7473}
    7574
    76 # values to extract from output metadata and the stats to calculate
    77 # XXX commented-out entries are not yet defined in the output files
    78 my $STATS =
    79    [
    80        #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
    81        { name => "ROBUST_MEDIAN",  type => "mean",  flag => "-bg",             dtype => "float" },
    82        { name => "ROBUST_MEDIAN",  type => "stdev", flag => "-bg_mean_stdev",  dtype => "float" },
    83        { name => "ROBUST_STDEV",   type => "rms",   flag => "-bg_stdev",       dtype => "float" },
    84        { name => "OVER_VAL",       type => "mean",  flag => "-bias",           dtype => "float" },
    85        { name => "OVER_VAL",       type => "stdev", flag => "-bias_stdev",     dtype => "float" },
    86        { name => "FRINGE_0",       type => "rms",   flag => "-fringe_0",       dtype => "float" },
    87        { name => "FRINGE_RESID_0", type => "rms",   flag => "-fringe_1",       dtype => "float" },
    88        { name => "FRINGE_ERR_0",   type => "rms",   flag => "-fringe_2",       dtype => "float" },
    89        { name => "APMIFIT",        type => "mean",  flag => "-ap_resid",       dtype => "float" },
    90        { name => "DAPMIFIT",       type => "rms",   flag => "-ap_resid_stdev", dtype => "float" },
    91        { name => "FWHM_X",         type => "mean",  flag => "-fwhm_major",     dtype => "float" },
    92        { name => "FWHM_Y",         type => "mean",  flag => "-fwhm_minor",     dtype => "float" },
    93        { name => "DT_DET",         type => "sum",   flag => "-dtime_detrend",  dtype => "float" },
    94        { name => "DT_PHOT",        type => "sum",   flag => "-dtime_photom",   dtype => "float" },
    95        { name => "DT_TOTAL",       type => "sum",   flag => "-dtime_total",    dtype => "float" },
    96        { name => "NSTARS",         type => "sum",   flag => "-n_stars",        dtype => "int"   },
    97 #      { name => "?",              type => "sum",   flag => "-n_extended",     dtype => "int"   },
    98 #      { name => "?",              type => "sum",   flag => "-n_cr",           dtype => "int"   },
    99    ];
    100 my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
    101 
    10275# Look for programs we need
    10376my $missing_tools;
    … …  
    10578my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
    10679my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
     80my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
    10781if ($missing_tools) {
    10882    warn("Can't find required tools.");
    … …  
    137111}
    138112
     113my $cmdflags;
     114
    139115# Run ppImage
    140116unless ($no_op) {
    141     ## XXX can we convert ppImage log and trace to use the filerules to generate consistent names
    142     ## XXX also stats: output should be implied by $outroot
    143117    my $command;
    144118    my $do_stats;
    … …  
    205179        &my_die("Couldn't find expected output file: $outputStats\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats);
    206180
    207         # Get the statistics on the processed image
    208         my $statsFile;              # File handle
    209         open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
    210         my @contents = <$statsFile>; # Contents of file
    211         close $statsFile;
    212 
    213         # parse the statistics MDC file
    214         my $mdcParser = PS::IPP::Metadata::Config->new();   # Parser for metadata config files
    215         my $metadata = $mdcParser->parse(join "", @contents);
    216         unless ($metadata) {
    217             &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
    218         }
    219 
    220         # extract the stats from the metadata
    221         unless ($stats->parse($metadata)) {
    222             &my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
    223         }
     181        my $outputStatsReal = $ipprc->file_resolve($outputStats);
     182        &my_die("Couldn't find expected output file: $outputStats", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal;
     183
     184        # measure chip stats
     185        $command = "$ppStatsFromMetadata $outputStatsReal - CHIP_IMFILE";
     186        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     187            run(command => $command, verbose => $verbose);
     188        unless ($success) {
     189            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     190            &my_die("Unable to perform ppStatsFromMetadata: $error_code", $exp_id, $chip_id, $class_id, $error_code);
     191        }
     192        foreach my $line (@$stdout_buf) {
     193            $cmdflags .= " $line";
     194        }
     195        chomp $cmdflags;
    224196    }
    225197}
    … …  
    236208    $command .= " -hostname $host" if defined $host;
    237209    $command .= " -dbname $dbname" if defined $dbname;
    238     $command .= $stats->cmdflags();
     210    $command .= " $cmdflags";
    239211    $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    240212} else {

  • trunk/ippScripts/scripts/detrend_norm_apply.pl

    r19817 r19942  
    109109my $outFile = ($det_type_real eq "mask") ? 'PPIMAGE.OUTPUT.DETMASK' : 'PPIMAGE.OUTPUT';; # XXXX something of a hack (too many places to control things...)
    110110
    111 my $RECIPE_PPSTATS = ($det_type_real eq "dark") ? 'DARKSTATS' : 'CHIPSTATS';; # XXXX something of a hack (too many places to control things...)
     111my $RECIPE_PPSTATS = ($det_type_real eq "dark") ? 'DARKSTATS' : 'DETSTATS';; # XXXX something of a hack (too many places to control things...)
    112112
    113113my $output    = $ipprc->filename($outFile,        $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR);
    … …  
    153153    &my_die("Can't find expected output file: $statsName", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $statsNameReal;
    154154
    155     # ppStatsFromMetadata $outputStats - DETREND_RESID_IMFILE
    156     $command = "$ppStatsFromMetadata $statsNameReal - DETREND_RESID_IMFILE";
     155    # parse stats from metadata
     156    $command = "$ppStatsFromMetadata $statsNameReal - DETREND_NORM_APPLY";
    157157    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    158158        run(command => $command, verbose => $verbose);

  • trunk/ippScripts/scripts/detrend_process_exp.pl

    r19658 r19942  
    1616use IPC::Cmd 0.36 qw( can_run run );
    1717use PS::IPP::Metadata::Config;
    18 use PS::IPP::Metadata::Stats;
    1918use PS::IPP::Config 1.01 qw( :standard );
    2019use PS::IPP::Metadata::List qw( parse_md_list );
    … …  
    6867&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe;
    6968
    70 # values to extract from output metadata and the stats to calculate
    71 # XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
    72 # XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
    73 # XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
    74 my $STATS =
    75    [
    76        #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
    77        { name => "bg",             type => "mean",  flag => "-bg",            dtype => "float" },
    78        { name => "bg",             type => "stdev", flag => "-bg_mean_stdev", dtype => "float" },
    79        { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev",      dtype => "float" },
    80        # { name => "bg_mean_stdev",  type => "rms",   flag => "-bg_mean_stdev" },
    81    ];
    82 my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
    83 
    8469# Look for programs we need
    8570my $missing_tools;
    8671my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
    8772my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
     73my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
    8874if ($missing_tools) {
    8975    warn("Can't find required tools.");
    9076    exit($PS_EXIT_CONFIG_ERROR);
    9177}
     78
     79my $cmdflags;
    9280
    9381# Get list of component files
    … …  
    112100        &my_die("Unable to parse metadata list", $det_id, $exp_id, $PS_EXIT_PROG_ERROR);
    113101
    114     # parse the stats in the metadata
    115     unless ($stats->parse($metadata)) {
    116         &my_die("Unable to find all values in statistics output.\n", $det_id, $exp_id, $PS_EXIT_PROG_ERROR);
     102    # since I can't figure out how to do input and output within PERL, I'm writing to a temp file
     103    my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.detproc.$det_id.stats.XXXX", UNLINK => !$save_temps );
     104    print "saving stats to $statName\n";
     105    foreach my $line (@$stdout_buf) {
     106        print $statFile $line;
     107    }
     108    close $statFile;
     109
     110    # parse the stats in the metadata file
     111    $command = "$ppStatsFromMetadata $statName - DETREND_PROCESS_EXP";
     112    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     113        run(command => $command, verbose => $verbose);
     114    unless ($success) {
     115        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     116        warn("Unable to perform ppStatsFromMetadata: $error_code\n");
     117        exit($error_code);
     118    }
     119
     120    foreach my $line (@$stdout_buf) {
     121        $cmdflags .= " $line";
    117122    }
    118123}
    … …  
    173178$command .= " -recip $recipe -path_base $outroot";
    174179$command .= " -dbname $dbname" if defined $dbname;
    175 $command .= $stats->cmdflags();
     180$command .= " $cmdflags";
    176181
    177182# Add the processed file to the database

  • trunk/ippScripts/scripts/detrend_process_imfile.pl

    r19652 r19942  
    1616use IPC::Cmd 0.36 qw( can_run run );
    1717use PS::IPP::Metadata::Config;
    18 use PS::IPP::Metadata::Stats;
    1918use PS::IPP::Config 1.01 qw( :standard );
    2019
    … …  
    6867my $jpeg_recipe = $ipprc->reduction($reduction, uc($det_type) . '_JPEG_IMAGE'); # Recipe name for JPEG
    6968
    70 # values to extract from output metadata and the stats to calculate
    71 my $STATS =
    72    [
    73        #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
    74        { name => "ROBUST_MEDIAN",  type => "mean",  flag => "-bg",            dtype => "float" },
    75        { name => "ROBUST_MEDIAN",  type => "stdev", flag => "-bg_mean_stdev", dtype => "float" },
    76        { name => "ROBUST_STDEV",   type => "rms",   flag => "-bg_stdev",      dtype => "float" },
    77    ];
    78 my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
    79 
    8069# Look for programs we need
    8170my $missing_tools;
    8271my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
    8372my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
     73my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
    8474if ($missing_tools) {
    8575    warn("Can't find required tools.");
    … …  
    9787$ipprc->outroot_prepare($outroot);
    9888
     89my $cmdflags;
     90
    9991my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
    10092my $outputBin1  = $ipprc->filename("PPIMAGE.BIN1",   $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
    … …  
    10799    $command .= " -recipe PPIMAGE $ppimage_recipe";
    108100    $command .= " -recipe JPEG $jpeg_recipe";
    109     $command .= " -recipe PPSTATS CHIPSTATS";
     101    $command .= " -recipe PPSTATS DETSTATS";
    110102    $command .= " -stats $outputStats";
    111103    $command .= " -tracedest $traceDest -log $logDest";
    … …  
    121113
    122114    &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage);
    123     &my_die("Couldn't find expected output file: $outputStats", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats);
    124115    &my_die("Couldn't find expected output file: $outputBin1",  $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin1);
    125116    &my_die("Couldn't find expected output file: $outputBin2",  $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin2);
    126117
    127     # Get the statistics on the processed image
    128     my $statsFile;              # File handle
    129     open $statsFile, $ipprc->file_resolve("$outputStats") or die "Can't open statistics file $outputStats: $!\n";
    130     my @contents = <$statsFile>; # Contents of file
    131     close $statsFile;
     118    my $outputStatsReal = $ipprc->file_resolve($outputStats);
     119    &my_die("Couldn't find expected output file: $outputStats", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal;
    132120
    133     # parse the statistics MDC file
    134     my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
    135     my $metadata = $mdcParser->parse(join "", @contents)
    136         or &my_die("Unable to parse metadata config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
    137 
    138     # extract the stats from the metadata
    139     $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
     121    # parse stats from metadata
     122    $command = "$ppStatsFromMetadata $outputStatsReal - DETREND_PROCESS_IMFILE";
     123    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     124        run(command => $command, verbose => $verbose);
     125    unless ($success) {
     126        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     127        &my_die("Unable to perform ppStatsFromMetadata: $error_code", $det_id, $exp_id, $class_id, $error_code);
     128    }
     129    foreach my $line (@$stdout_buf) {
     130        $cmdflags .= " $line";
     131    }
     132    chomp $cmdflags;
    140133}
    141134
    … …  
    148141$command .= " -uri $outputImage -path_base $outroot";
    149142$command .= " -dbname $dbname" if defined $dbname;
    150 $command .= $stats->cmdflags();
     143$command .= " $cmdflags";
    151144
    152145# Add the processed file to the database

  • trunk/ippScripts/scripts/detrend_resid_exp.pl

    r19818 r19942  
    128128    close $statFile;
    129129
     130    # parse the stats in the metadata file
    130131    $command = "$ppStatsFromMetadata $statName - DETREND_RESID_EXP";
    131132    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

  • trunk/ippScripts/scripts/detrend_stack.pl

    r19374 r19942  
    1616use IPC::Cmd 0.36 qw( can_run run );
    1717use PS::IPP::Metadata::Config;
    18 use PS::IPP::Metadata::Stats;
    1918use PS::IPP::Metadata::List qw( parse_md_list );
    2019use PS::IPP::Config 1.01 qw( :standard );
    … …  
    6766my $recipe = $ipprc->reduction($reduction, $det_type . '_STACK'); # Recipe name to use
    6867
    69 # values to extract from output metadata and the stats to calculate
    70 # XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
    71 # XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
    72 # XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
    73 my $STATS =
    74    [
    75        #          KEYWORD          STATISTIC            DETTOOL FLAG
    76        { name => "ROBUST_MEDIAN",  type => "clipmean",  flag => "-bg",             dtype => "float" },
    77        { name => "ROBUST_MEDIAN",  type => "clipstdev", flag => "-bg_mean_stdev",  dtype => "float" },
    78        { name => "ROBUST_STDEV",   type => "rms",       flag => "-bg_stdev",       dtype => "float" },
    79    ];
    80 my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
    81 
    8268# Look for programs we need
    8369my $missing_tools;
    8470my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
    8571my $ppMerge = can_run('ppMerge') or (warn "Can't find ppMerge" and $missing_tools = 1);
     72my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
    8673if ($missing_tools) {
    8774    warn("Can't find required tools.");
    … …  
    11299
    113100# The stats recipe depends on the detrend type
    114 my $STATRECIPES = {'FLATMASK'         => 'CHIPSTATS',
    115                    'DARKMASK'         => 'CHIPSTATS',
    116                    'MASK'             => 'CHIPSTATS',
    117                    'BIAS'             => 'CHIPSTATS',
     101my $STATRECIPES = {'FLATMASK'         => 'DETSTATS',
     102                   'DARKMASK'         => 'DETSTATS',
     103                   'MASK'             => 'DETSTATS',
     104                   'BIAS'             => 'DETSTATS',
    118105                   'DARK'             => 'DARKSTATS',
    119106                   'DARK_PREMASK'     => 'DARKSTATS',
    120107                   'SHUTTER'          => 'DARKSTATS',
    121                    'FLAT_PREMASK'     => 'CHIPSTATS',
    122                    'DOMEFLAT_PREMASK' => 'CHIPSTATS',
    123                    'SKYFLAT_PREMASK'  => 'CHIPSTATS',
    124                    'FLAT_RAW'         => 'CHIPSTATS',
    125                    'DOMEFLAT_RAW'     => 'CHIPSTATS',
    126                    'SKYFLAT_RAW'      => 'CHIPSTATS',
    127                    'FLAT'             => 'CHIPSTATS',
    128                    'DOMEFLAT'         => 'CHIPSTATS',
    129                    'SKYFLAT'          => 'CHIPSTATS',
    130                    'FRINGE'           => 'CHIPSTATS',
     108                   'FLAT_PREMASK'     => 'DETSTATS',
     109                   'DOMEFLAT_PREMASK' => 'DETSTATS',
     110                   'SKYFLAT_PREMASK'  => 'DETSTATS',
     111                   'FLAT_RAW'         => 'DETSTATS',
     112                   'DOMEFLAT_RAW'     => 'DETSTATS',
     113                   'SKYFLAT_RAW'      => 'DETSTATS',
     114                   'FLAT'             => 'DETSTATS',
     115                   'DOMEFLAT'         => 'DETSTATS',
     116                   'SKYFLAT'          => 'DETSTATS',
     117                   'FRINGE'           => 'DETSTATS',
    131118              };
    132119my $statrecipe = $STATRECIPES->{$det_type}; # File rule for output
     120
     121my $cmdflags;
    133122
    134123# Get list of files to stack
    … …  
    217206    &my_die("Unable to find expected output file: $outputCount\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputCount);
    218207    &my_die("Unable to find expected output file: $outputSigma\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputSigma);
    219     &my_die("Unable to find expected output file: $outputStats\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats);
    220 
    221     # Get the statistics on the stacked image
    222     open(my $statsFile, $ipprc->file_resolve("$outputStats")) or
    223         &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR);
    224     my $contents = do { local $/; <$statsFile> }; # Contents of file
    225     close($statsFile);
    226 
    227     my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
    228     my $metadata = $mdcParser->parse($contents) or
    229         &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR);
    230 
    231     $stats->parse($metadata)  or
    232         &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR);
     208
     209    my $outputStatsReal = $ipprc->file_resolve($outputStats);
     210    &my_die("Couldn't find expected output file: $outputStats", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal;
     211
     212    # parse stats from metadata
     213    $command = "$ppStatsFromMetadata $outputStatsReal - DETREND_STACK";
     214    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     215        run(command => $command, verbose => $verbose);
     216    unless ($success) {
     217        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     218        &my_die("Unable to perform ppStatsFromMetadata: $error_code", $det_id, $iter, $class_id, $error_code);
     219    }
     220    foreach my $line (@$stdout_buf) {
     221        $cmdflags .= " $line";
     222    }
     223    chomp $cmdflags;
    233224}
    234225
    … …  
    241232$command .= " -recip $recipe";
    242233$command .= " -dbname $dbname" if defined $dbname;
    243 $command .= $stats->cmdflags();
     234$command .= " $cmdflags";
    244235
    245236# Add the resultant into the database
Note: See TracChangeset for help on using the changeset viewer.