IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 18020


Ignore:
Timestamp:
Jun 9, 2008, 11:23:08 AM (18 years ago)
Author:
eugene
Message:

finished fake script and task

Location:
trunk
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/Build.PL

    r17887 r18020  
    4747        scripts/chip_imfile.pl
    4848        scripts/camera_exp.pl
     49        scripts/fake_imfile.pl
    4950        scripts/warp_overlap.pl
    5051        scripts/warp_skycell.pl

  • trunk/ippScripts/scripts/fake_imfile.pl

    r17671 r18020  
    1414## to select the desired locations in pixel coord. 
    1515
     16# XXX : seeing needs to be determined from the input PSF (not currently done in ppSim)
     17# ppSim output -input input.fits -cmf input.cmf -psf input.psf -seeing 0.563760 -recipe PPSIM FAKEPHOT
     18
    1619use Carp;
    1720use warnings;
    … …  
    3841
    3942# Parse the command-line arguments
    40 my ( $exp_id, $exp_tag, $chip_id, $class_id, $uri, $camera, $outroot, $dbname, $reduction, $verbose,
    41      $no_update, $no_op );
     43my ( $exp_id, $fake_id, $class_id, $chiproot, $camroot, $camera, $outroot,
     44     $dbname, $reduction, $verbose, $no_update, $no_op );
    4245GetOptions(
    4346           'exp_id=s'      => \$exp_id,    # Exposure identifier
    44            'exp_tag=s'     => \$exp_tag,   # Exposure identifier
    45            'chip_id=s'     => \$chip_id,   # Chiptool identifier
     47           'fake_id=s'     => \$fake_id,   # Chiptool identifier
    4648           'class_id=s'    => \$class_id,  # Class identifier
    47            'uri|u=s'       => \$uri,       # Input FITS file
     49           'chiproot=s'    => \$chiproot,  # Input Chip files (root)
     50           'camroot=s'     => \$camroot,   # Input Camera files (root)
    4851           'camera|c=s'    => \$camera,    # Camera
    4952           'outroot|w=s'   => \$outroot,   # output file base name
    … …  
    5659
    5760pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    58 pod2usage( -msg => "Required options: --exp_id --exp_tag --chip_id --class_id --uri --camera --outroot",
     61pod2usage( -msg => "Required options: --exp_id --fake_id --class_id --chiproot --camroot --camera --outroot",
    5962           -exitval => 3) unless
    6063    defined $exp_id and
    61     defined $exp_tag and
    62     defined $chip_id and
     64    defined $fake_id and
    6365    defined $class_id and
    64     defined $uri and
     66    defined $chiproot and
     67    defined $camroot and
    6568    defined $camera and
    6669    defined $outroot;
    … …  
    6972
    7073# Recipes to use based on reduction class
    71 # the CHIP recipe should not perform astrometry anymore (2008.04.08)
    72 
    7374$reduction = 'DEFAULT' unless defined $reduction;
    74 my $recipe = $ipprc->reduction($reduction, 'CHIP'); # Recipe to use
     75my $recipe = $ipprc->reduction($reduction, 'FAKEPHOT'); # Recipe to use
    7576unless ($recipe) {
    76     &my_die("Couldn't find selected reduction for CHIP: $reduction\n", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     77    &my_die("Couldn't find selected reduction for FAKE: $reduction\n", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    7778}
    7879
    7980# values to extract from output metadata and the stats to calculate
    80 # XXX commented-out entries are not yet defined in the output files
    81 # XXX EAM : we are removing the astrometry analysis from chip, so we
    82 # do not bother to calculate those stats.
    83 my $STATS =
    84    [
    85        #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
    86        { name => "ROBUST_MEDIAN",  type => "mean",  flag => "-bg",             dtype => "float" },
    87        { name => "ROBUST_MEDIAN",  type => "stdev", flag => "-bg_mean_stdev",  dtype => "float" },
    88        { name => "ROBUST_STDEV",   type => "rms",   flag => "-bg_stdev",       dtype => "float" },
    89        { name => "OVER_VAL",       type => "mean",  flag => "-bias",           dtype => "float" },
    90        { name => "OVER_VAL",       type => "stdev", flag => "-bias_stdev",     dtype => "float" },
    91        { name => "FRINGE_0",       type => "rms",   flag => "-fringe_0",       dtype => "float" },
    92        { name => "FRINGE_RESID_0", type => "rms",   flag => "-fringe_1",       dtype => "float" },
    93        { name => "FRINGE_ERR_0",   type => "rms",   flag => "-fringe_2",       dtype => "float" },
    94 #      { name => "CERROR",         type => "rms",   flag => "-sigma_ra",       dtype => "float" },
    95 #      { name => "CERROR",         type => "rms",   flag => "-sigma_dec",      dtype => "float" },
    96        { name => "APMIFIT",        type => "mean",  flag => "-ap_resid",       dtype => "float" },
    97        { name => "DAPMIFIT",       type => "rms",   flag => "-ap_resid_stdev", dtype => "float" },
    98 #      { name => "ZP??",           type => "mean",  flag => "-zp_mean",        dtype => "float" },
    99 #      { name => "ZP??",           type => "rms",   flag => "-zp_stdev",       dtype => "float" },
    100        { name => "FWHM_X",         type => "mean",  flag => "-fwhm_major",     dtype => "float" },
    101        { name => "FWHM_Y",         type => "mean",  flag => "-fwhm_minor",     dtype => "float" },
    102        { name => "DT_DET",         type => "sum",   flag => "-dtime_detrend",  dtype => "float" },
    103        { name => "DT_PHOT",        type => "sum",   flag => "-dtime_photom",   dtype => "float" },
    104 #       { name => "DT_ASTR",        type => "sum",   flag => "-dtime_astrom",   dtype => "float" },
    105        { name => "NSTARS",         type => "sum",   flag => "-n_stars",        dtype => "int"   },
    106 #      { name => "?",              type => "sum",   flag => "-n_extended",     dtype => "int"   },
    107 #      { name => "?",              type => "sum",   flag => "-n_cr",           dtype => "int"   },
    108 #      { name => "NASTRO",         type => "sum",   flag => "-n_astrom",       dtype => "int"   },
    109    ];
    110 my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
     81# XXX not sure if this is needed by fake_imfile.pl
     82# my $STATS =
     83#    [
     84#        #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
     85#        { name => "ROBUST_MEDIAN",  type => "mean",  flag => "-bg",             dtype => "float" },
     86#        { name => "ROBUST_MEDIAN",  type => "stdev", flag => "-bg_mean_stdev",  dtype => "float" },
     87#        { name => "ROBUST_STDEV",   type => "rms",   flag => "-bg_stdev",       dtype => "float" },
     88#    ];
     89# my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
    11190
    11291# Look for programs we need
    11392my $missing_tools;
    114 my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
    115 my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
     93my $faketool = can_run('faketool') or (warn "Can't find faketool" and $missing_tools = 1);
     94my $ppSim = can_run('ppSim') or (warn "Can't find ppSim" and $missing_tools = 1);
    11695my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
    11796if ($missing_tools) {
    … …  
    122101my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    123102
    124 &my_die("Couldn't find input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri);
    125 
    126103# outroot examples (HOST components must be set)
    127104# file://data/ipp004.0/gpc1/20080130
    … …  
    133110
    134111## these names are used in ppImage, and thus may be URIs
    135 my $outputImage   = $ipprc->filename("PPIMAGE.CHIP",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    136 my $outputMask    = $ipprc->filename("PPIMAGE.CHIP.MASK",   $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    137 my $outputWeight  = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    138 my $outputBin1    = $ipprc->filename("PPIMAGE.BIN1",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    139 my $outputBin2    = $ipprc->filename("PPIMAGE.BIN2",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    140 my $outputStats   = $ipprc->filename("PPIMAGE.STATS",       $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    141 my $traceDest     = $ipprc->filename("TRACE.IMFILE",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    142 my $logDest       = $ipprc->filename("LOG.IMFILE",          $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    143 
    144 # Run ppImage
    145 unless ($no_op) {
    146     ## XXX can we convert ppImage log and trace to use the filerules to generate consistent names
    147     ## XXX also stats: output should be implied by $outroot
    148     my $command = "$ppImage -file $uri $outroot";
    149     $command .= " -recipe PPIMAGE $recipe";
    150     $command .= " -recipe PPSTATS CHIPSTATS";
    151     $command .= " -stats $outputStats";
     112my $chipImage     = $ipprc->filename("PPIMAGE.CHIP",        $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     113my $chipMask      = $ipprc->filename("PPIMAGE.CHIP.MASK",   $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     114my $chipWeight    = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     115my $chipPSF       = $ipprc->filename("PSPHOT.PSF.SAVE",     $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     116my $cameraObjects = $ipprc->filename("PSASTRO.OUTPUT",      $camroot)             or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     117my $traceDest     = $ipprc->filename("TRACE.IMFILE",        $outroot, $class_id)  or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     118my $logDest       = $ipprc->filename("LOG.IMFILE",          $outroot, $class_id)  or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     119
     120# XXX check for existence of input data
     121# &my_die("Couldn't find input file: $uri\n", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri);
     122
     123## get the PPSIM recipe for this camera and FAKEPHOT reduction
     124my $command = "$ppConfigDump -camera $camera -dump-recipe PPSIM -recipe PPSIM $recipe -";
     125my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     126        run(command => $command, verbose => $verbose);
     127unless ($success) {
     128        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     129        &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR);
     130}
     131my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
     132        &my_die("Unable to parse metadata config doc", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR);
     133
     134## allow the output images to be optional, depending on the recipe / reduction class
     135my $skipFake = metadataLookupBool($recipeData, 'SKIP.FAKE');
     136
     137# Run ppSim
     138unless ($no_op || $skipFake) {
     139    # examine the PPSIM recipe to decide if we need to run this or opt out
     140
     141    my $command = "$ppSim $outroot";
     142    $command .= " -input $chipImage";
     143    # XXX add input mask and weight to ppSim
     144    # $command .= " -mask $chipMask";
     145    # $command .= " -weight $chipWeight";
     146    $command .= " -cmf $cameraObjects";
     147    $command .= " -psf $chipPSF";
     148    $command .= " -recipe PPSIM $recipe";
    152149    $command .= " -dbname $dbname" if defined $dbname;
    153     $command .= " -tracedest $traceDest -log $logDest";
     150    $command .= " -tracedest $traceDest";
     151    $command .= " -log $logDest";
    154152
    155153    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    157155    unless ($success) {
    158156        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    159         &my_die("Unable to perform ppImage: $error_code", $exp_id, $chip_id, $class_id, $error_code);
     157        &my_die("Unable to perform ppSim: $error_code", $exp_id, $fake_id, $class_id, $error_code);
    160158    }
    161159
    162     ## get the ppImage recipe for this camera and CHIP reduction
    163     $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe -";
    164     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    165         run(command => $command, verbose => $verbose);
    166     unless ($success) {
    167         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    168         &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
    169     }
    170     my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
    171         &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
    172 
    173     ## allow the output images to be optional, depending on the recipe / reduction class
    174     my $outputImageExpect = metadataLookupBool($recipeData, 'CHIP.FITS');
    175     if ($outputImageExpect) {
    176         &my_die("Couldn't find expected output file: $outputImage\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
    177     }
    178 
    179     my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS');
    180     if ($outputMaskExpect) {
    181         &my_die("Couldn't find expected output file: $outputMask\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
    182     }
    183 
    184     my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.WEIGHT.FITS');
    185     if ($outputWeightExpect) {
    186         &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
    187     }
    188 
    189     &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
    190     &my_die("Couldn't find expected output file: $outputBin2\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
    191     &my_die("Couldn't find expected output file: $outputStats\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats);
    192 
     160    # XXX check for output files?
     161    # &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
     162
     163    # XXX use this to parse the output metadata : eg, detection limits, Nfakes, Nforced
    193164    # Get the statistics on the processed image
    194     my $statsFile;              # File handle
    195     open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
    196     my @contents = <$statsFile>; # Contents of file
    197     close $statsFile;
    198 
    199     # parse the statistics MDC file
    200     my $mdcParser = PS::IPP::Metadata::Config->new();   # Parser for metadata config files
    201     my $metadata = $mdcParser->parse(join "", @contents);
    202     unless ($metadata) {
    203         &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
    204     }
    205 
    206     # extract the stats from the metadata
    207     unless ($stats->parse($metadata)) {
    208         &my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
    209     }
     165    # my $statsFile;            # File handle
     166    # open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR);
     167    # my @contents = <$statsFile>; # Contents of file
     168    # close $statsFile;
     169    #
     170    # # parse the statistics MDC file
     171    # my $mdcParser = PS::IPP::Metadata::Config->new(); # Parser for metadata config files
     172    # my $metadata = $mdcParser->parse(join "", @contents);
     173    # unless ($metadata) {
     174    #   &my_die("Unable to parse metadata config doc", $exp_id, $fake_id, $class_id, $PS_EXIT_PROG_ERROR);
     175    # }
     176    #
     177    # # extract the stats from the metadata
     178    # unless ($stats->parse($metadata)) {
     179    #   &my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $fake_id, $class_id, $PS_EXIT_PROG_ERROR);
     180    # }
     181} else {
     182    print "skipping ppSim processing\n";
    210183}
    211184
    212185# command to update database
    213 my $command = "$chiptool -addprocessedimfile";
     186$command = "$faketool -addprocessedimfile";
     187$command .= " -fake_id $fake_id";
    214188$command .= " -exp_id $exp_id";
    215 $command .= " -chip_id $chip_id";
    216189$command .= " -class_id $class_id";
    217 $command .= " -uri $outputImage";
    218190$command .= " -path_base $outroot";
    219191$command .= " -dbname $dbname" if defined $dbname;
    220 $command .= $stats->cmdflags();
     192# XXX add this after defined
     193# $command .= $stats->cmdflags();
    221194
    222195# Add the processed file to the database
    … …  
    226199    unless ($success) {
    227200        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    228         warn("Unable to perform chiptool -addprocessedimfile: $error_code\n");
     201        warn("Unable to perform faketool -addprocessedimfile: $error_code\n");
    229202        exit($error_code);
    230203    }
    … …  
    236209{
    237210    my $msg = shift; # Warning message on die
    238     my $exp_id = shift; # Chiptool identifier
    239     my $chip_id = shift; # Chiptool identifier
     211    my $exp_id = shift; # rawExp identifier
     212    my $fake_id = shift; # fakeRun identifier
    240213    my $class_id = shift; # Class identifier
    241214    my $exit_code = shift; # Exit code to add
    242215
    243216    carp($msg);
    244     if (defined $chip_id and defined $class_id and not $no_update) {
    245         my $command = "$chiptool -addprocessedimfile";
     217    if (defined $exp_id and defined $fake_id and defined $class_id and not $no_update) {
     218        my $command = "$faketool -addprocessedimfile";
     219        $command .= " -fake_id $fake_id";
    246220        $command .= " -exp_id $exp_id";
    247         $command .= " -chip_id $chip_id";
    248221        $command .= " -class_id $class_id";
     222        $command .= " -path_base $outroot";
    249223        $command .= " -code $exit_code";
    250         $command .= " -uri $outputImage";
    251224        $command .= " -dbname $dbname" if defined $dbname;
    252225        system ($command);

  • trunk/ippTasks/Makefile.am

    r17593 r18020  
    1111        chip.pro \
    1212        camera.pro \
     13        fake.pro \
    1314        warp.pro \
    1415        magic.pro \

  • trunk/ippTasks/fake.pro

    r17631 r18020  
    112112    book getword fakePendingImfile $pageName workdir -var WORKDIR_TEMPLATE
    113113    book getword fakePendingImfile $pageName class_id -var CLASS_ID
    114     book getword fakePendingImfile $pageName uri -var URI
     114    book getword fakePendingImfile $pageName chip_path_base -var CHIPROOT
     115    book getword fakePendingImfile $pageName cam_path_base -var CAMROOT
    115116    book getword fakePendingImfile $pageName dbname -var DBNAME
    116117    book getword fakePendingImfile $pageName reduction -var REDUCTION
    … …  
    137138    mkdir $outpath
    138139
    139     $run = fake_imfile.pl --exp_id $EXP_ID --fake_id $FAKE_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --uri $URI --camera $CAMERA --outroot $outroot
     140    $run = fake_imfile.pl --exp_id $EXP_ID --fake_id $FAKE_ID --class_id $CLASS_ID --chiproot=$CHIPROOT --camroot=$CAMROOT --camera $CAMERA --outroot $outroot
    140141    if ("$REDUCTION" != "NULL")
    141142      $run = $run --reduction $REDUCTION

  • trunk/ippTasks/pantasks.pro

    r17966 r18020  
    148148macro module.tasks
    149149  module register.pro
    150   module chip.pro
    151   module camera.pro
    152150  module detrend.process.pro
    153151  module detrend.stack.pro
    … …  
    155153  module detrend.resid.pro
    156154  module detrend.reject.pro
     155  module chip.pro
     156  module camera.pro
     157  module fake.pro
    157158  module warp.pro
    158159  module diff.pro
Note: See TracChangeset for help on using the changeset viewer.