Index: trunk/ippScripts/scripts/fake_imfile.pl
===================================================================
--- trunk/ippScripts/scripts/fake_imfile.pl	(revision 17671)
+++ trunk/ippScripts/scripts/fake_imfile.pl	(revision 18020)
@@ -14,4 +14,7 @@
 ## to select the desired locations in pixel coord.  
 
+# XXX : seeing needs to be determined from the input PSF (not currently done in ppSim)
+# ppSim output -input input.fits -cmf input.cmf -psf input.psf -seeing 0.563760 -recipe PPSIM FAKEPHOT
+
 use Carp;
 use warnings;
@@ -38,12 +41,12 @@
 
 # Parse the command-line arguments
-my ( $exp_id, $exp_tag, $chip_id, $class_id, $uri, $camera, $outroot, $dbname, $reduction, $verbose,
-     $no_update, $no_op );
+my ( $exp_id, $fake_id, $class_id, $chiproot, $camroot, $camera, $outroot, 
+     $dbname, $reduction, $verbose, $no_update, $no_op );
 GetOptions(
 	   'exp_id=s'      => \$exp_id,    # Exposure identifier
-	   'exp_tag=s'     => \$exp_tag,   # Exposure identifier
-	   'chip_id=s'     => \$chip_id,   # Chiptool identifier
+	   'fake_id=s'     => \$fake_id,   # Chiptool identifier
 	   'class_id=s'    => \$class_id,  # Class identifier
-	   'uri|u=s'       => \$uri,       # Input FITS file
+	   'chiproot=s'    => \$chiproot,  # Input Chip files (root)
+	   'camroot=s'     => \$camroot,   # Input Camera files (root)
 	   'camera|c=s'    => \$camera,	   # Camera
 	   'outroot|w=s'   => \$outroot,   # output file base name
@@ -56,11 +59,11 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --exp_id --exp_tag --chip_id --class_id --uri --camera --outroot",
+pod2usage( -msg => "Required options: --exp_id --fake_id --class_id --chiproot --camroot --camera --outroot",
 	   -exitval => 3) unless
     defined $exp_id and
-    defined $exp_tag and
-    defined $chip_id and
+    defined $fake_id and
     defined $class_id and
-    defined $uri and
+    defined $chiproot and
+    defined $camroot and
     defined $camera and
     defined $outroot;
@@ -69,49 +72,25 @@
 
 # Recipes to use based on reduction class
-# the CHIP recipe should not perform astrometry anymore (2008.04.08)
-
 $reduction = 'DEFAULT' unless defined $reduction;
-my $recipe = $ipprc->reduction($reduction, 'CHIP'); # Recipe to use
+my $recipe = $ipprc->reduction($reduction, 'FAKEPHOT'); # Recipe to use
 unless ($recipe) {
-    &my_die("Couldn't find selected reduction for CHIP: $reduction\n", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+    &my_die("Couldn't find selected reduction for FAKE: $reduction\n", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
 }
 
 # values to extract from output metadata and the stats to calculate
-# XXX commented-out entries are not yet defined in the output files
-# XXX EAM : we are removing the astrometry analysis from chip, so we
-# do not bother to calculate those stats.
-my $STATS =
-   [
-       #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
-       { name => "ROBUST_MEDIAN",  type => "mean",  flag => "-bg",             dtype => "float" },
-       { name => "ROBUST_MEDIAN",  type => "stdev", flag => "-bg_mean_stdev",  dtype => "float" },
-       { name => "ROBUST_STDEV",   type => "rms",   flag => "-bg_stdev",       dtype => "float" },
-       { name => "OVER_VAL",       type => "mean",  flag => "-bias", 	       dtype => "float" },
-       { name => "OVER_VAL",       type => "stdev", flag => "-bias_stdev",     dtype => "float" },
-       { name => "FRINGE_0",       type => "rms",   flag => "-fringe_0",       dtype => "float" },
-       { name => "FRINGE_RESID_0", type => "rms",   flag => "-fringe_1",       dtype => "float" },
-       { name => "FRINGE_ERR_0",   type => "rms",   flag => "-fringe_2",       dtype => "float" },
-#      { name => "CERROR",         type => "rms",   flag => "-sigma_ra",       dtype => "float" },
-#      { name => "CERROR",         type => "rms",   flag => "-sigma_dec",      dtype => "float" },
-       { name => "APMIFIT",        type => "mean",  flag => "-ap_resid",       dtype => "float" },
-       { name => "DAPMIFIT",       type => "rms",   flag => "-ap_resid_stdev", dtype => "float" },
-#      { name => "ZP??",           type => "mean",  flag => "-zp_mean",        dtype => "float" },
-#      { name => "ZP??",           type => "rms",   flag => "-zp_stdev",       dtype => "float" },
-       { name => "FWHM_X",         type => "mean",  flag => "-fwhm_major",     dtype => "float" },
-       { name => "FWHM_Y",         type => "mean",  flag => "-fwhm_minor",     dtype => "float" },
-       { name => "DT_DET",         type => "sum",   flag => "-dtime_detrend",  dtype => "float" },
-       { name => "DT_PHOT",        type => "sum",   flag => "-dtime_photom",   dtype => "float" },
-#       { name => "DT_ASTR",        type => "sum",   flag => "-dtime_astrom",   dtype => "float" },
-       { name => "NSTARS",         type => "sum",   flag => "-n_stars",        dtype => "int"   },
-#      { name => "?",              type => "sum",   flag => "-n_extended",     dtype => "int"   },
-#      { name => "?",              type => "sum",   flag => "-n_cr",           dtype => "int"   },
-#      { name => "NASTRO",         type => "sum",   flag => "-n_astrom",       dtype => "int"   },
-   ];
-my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
+# XXX not sure if this is needed by fake_imfile.pl
+# my $STATS =
+#    [
+#        #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
+#        { name => "ROBUST_MEDIAN",  type => "mean",  flag => "-bg",             dtype => "float" },
+#        { name => "ROBUST_MEDIAN",  type => "stdev", flag => "-bg_mean_stdev",  dtype => "float" },
+#        { name => "ROBUST_STDEV",   type => "rms",   flag => "-bg_stdev",       dtype => "float" },
+#    ];
+# my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
 
 # Look for programs we need
 my $missing_tools;
-my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
-my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
+my $faketool = can_run('faketool') or (warn "Can't find faketool" and $missing_tools = 1);
+my $ppSim = can_run('ppSim') or (warn "Can't find ppSim" and $missing_tools = 1);
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 if ($missing_tools) {
@@ -122,6 +101,4 @@
 my $mdcParser = PS::IPP::Metadata::Config->new;	# Parser for metadata config files
 
-&my_die("Couldn't find input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri);
-
 # outroot examples (HOST components must be set)
 # file://data/ipp004.0/gpc1/20080130
@@ -133,23 +110,44 @@
 
 ## these names are used in ppImage, and thus may be URIs
-my $outputImage   = $ipprc->filename("PPIMAGE.CHIP",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $outputMask    = $ipprc->filename("PPIMAGE.CHIP.MASK",   $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $outputWeight  = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $outputBin1    = $ipprc->filename("PPIMAGE.BIN1",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $outputBin2    = $ipprc->filename("PPIMAGE.BIN2",  	    $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $outputStats   = $ipprc->filename("PPIMAGE.STATS", 	    $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $traceDest     = $ipprc->filename("TRACE.IMFILE", 	    $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $logDest       = $ipprc->filename("LOG.IMFILE", 	    $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-
-# Run ppImage
-unless ($no_op) {
-    ## XXX can we convert ppImage log and trace to use the filerules to generate consistent names
-    ## XXX also stats: output should be implied by $outroot
-    my $command = "$ppImage -file $uri $outroot";
-    $command .= " -recipe PPIMAGE $recipe";
-    $command .= " -recipe PPSTATS CHIPSTATS";
-    $command .= " -stats $outputStats";
+my $chipImage     = $ipprc->filename("PPIMAGE.CHIP",        $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $chipMask      = $ipprc->filename("PPIMAGE.CHIP.MASK",   $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $chipWeight    = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $chipPSF       = $ipprc->filename("PSPHOT.PSF.SAVE",     $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $cameraObjects = $ipprc->filename("PSASTRO.OUTPUT",      $camroot)             or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $traceDest     = $ipprc->filename("TRACE.IMFILE", 	    $outroot, $class_id)  or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+my $logDest       = $ipprc->filename("LOG.IMFILE", 	    $outroot, $class_id)  or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+
+# XXX check for existence of input data
+# &my_die("Couldn't find input file: $uri\n", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri);
+
+## get the PPSIM recipe for this camera and FAKEPHOT reduction
+my $command = "$ppConfigDump -camera $camera -dump-recipe PPSIM -recipe PPSIM $recipe -";
+my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR);
+}
+my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata config doc", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR);
+
+## allow the output images to be optional, depending on the recipe / reduction class
+my $skipFake = metadataLookupBool($recipeData, 'SKIP.FAKE');
+
+# Run ppSim
+unless ($no_op || $skipFake) {
+    # examine the PPSIM recipe to decide if we need to run this or opt out
+
+    my $command = "$ppSim $outroot";
+    $command .= " -input $chipImage";
+    # XXX add input mask and weight to ppSim
+    # $command .= " -mask $chipMask";
+    # $command .= " -weight $chipWeight";
+    $command .= " -cmf $cameraObjects";
+    $command .= " -psf $chipPSF";
+    $command .= " -recipe PPSIM $recipe";
     $command .= " -dbname $dbname" if defined $dbname;
-    $command .= " -tracedest $traceDest -log $logDest";
+    $command .= " -tracedest $traceDest";
+    $command .= " -log $logDest";
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -157,66 +155,41 @@
     unless ($success) {
 	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	&my_die("Unable to perform ppImage: $error_code", $exp_id, $chip_id, $class_id, $error_code);
+	&my_die("Unable to perform ppSim: $error_code", $exp_id, $fake_id, $class_id, $error_code);
     }
 
-    ## get the ppImage recipe for this camera and CHIP reduction
-    $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe -";
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run(command => $command, verbose => $verbose);
-    unless ($success) {
-	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	&my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-    }
-    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
-	&my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-
-    ## allow the output images to be optional, depending on the recipe / reduction class
-    my $outputImageExpect = metadataLookupBool($recipeData, 'CHIP.FITS');
-    if ($outputImageExpect) {
-	&my_die("Couldn't find expected output file: $outputImage\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
-    }
-
-    my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS');
-    if ($outputMaskExpect) {
-	&my_die("Couldn't find expected output file: $outputMask\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
-    }
-
-    my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.WEIGHT.FITS');
-    if ($outputWeightExpect) {
-	&my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
-    }
-
-    &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
-    &my_die("Couldn't find expected output file: $outputBin2\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
-    &my_die("Couldn't find expected output file: $outputStats\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats);
-
+    # XXX check for output files?
+    # &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
+
+    # XXX use this to parse the output metadata : eg, detection limits, Nfakes, Nforced
     # Get the statistics on the processed image
-    my $statsFile;		# File handle
-    open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-    my @contents = <$statsFile>; # Contents of file
-    close $statsFile;
-
-    # parse the statistics MDC file
-    my $mdcParser = PS::IPP::Metadata::Config->new();	# Parser for metadata config files
-    my $metadata = $mdcParser->parse(join "", @contents);
-    unless ($metadata) {
-	&my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
-    }
-
-    # extract the stats from the metadata
-    unless ($stats->parse($metadata)) {
-	&my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
-    }
+    # my $statsFile;		# File handle
+    # open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR);
+    # my @contents = <$statsFile>; # Contents of file
+    # close $statsFile;
+    # 
+    # # parse the statistics MDC file
+    # my $mdcParser = PS::IPP::Metadata::Config->new();	# Parser for metadata config files
+    # my $metadata = $mdcParser->parse(join "", @contents);
+    # unless ($metadata) {
+    # 	&my_die("Unable to parse metadata config doc", $exp_id, $fake_id, $class_id, $PS_EXIT_PROG_ERROR);
+    # }
+    # 
+    # # extract the stats from the metadata
+    # unless ($stats->parse($metadata)) {
+    # 	&my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $fake_id, $class_id, $PS_EXIT_PROG_ERROR);
+    # }
+} else {
+    print "skipping ppSim processing\n";
 }
 
 # command to update database
-my $command = "$chiptool -addprocessedimfile";
+$command = "$faketool -addprocessedimfile";
+$command .= " -fake_id $fake_id";
 $command .= " -exp_id $exp_id";
-$command .= " -chip_id $chip_id";
 $command .= " -class_id $class_id";
-$command .= " -uri $outputImage";
 $command .= " -path_base $outroot";
 $command .= " -dbname $dbname" if defined $dbname;
-$command .= $stats->cmdflags();
+# XXX add this after defined
+# $command .= $stats->cmdflags();
 
 # Add the processed file to the database
@@ -226,5 +199,5 @@
     unless ($success) {
 	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	warn("Unable to perform chiptool -addprocessedimfile: $error_code\n");
+	warn("Unable to perform faketool -addprocessedimfile: $error_code\n");
 	exit($error_code);
     }
@@ -236,17 +209,17 @@
 {
     my $msg = shift; # Warning message on die
-    my $exp_id = shift; # Chiptool identifier
-    my $chip_id = shift; # Chiptool identifier
+    my $exp_id = shift; # rawExp identifier
+    my $fake_id = shift; # fakeRun identifier
     my $class_id = shift; # Class identifier
     my $exit_code = shift; # Exit code to add
 
     carp($msg);
-    if (defined $chip_id and defined $class_id and not $no_update) {
-	my $command = "$chiptool -addprocessedimfile";
+    if (defined $exp_id and defined $fake_id and defined $class_id and not $no_update) {
+	my $command = "$faketool -addprocessedimfile";
+	$command .= " -fake_id $fake_id";
 	$command .= " -exp_id $exp_id";
-	$command .= " -chip_id $chip_id";
 	$command .= " -class_id $class_id";
+	$command .= " -path_base $outroot";
 	$command .= " -code $exit_code";
-	$command .= " -uri $outputImage";
 	$command .= " -dbname $dbname" if defined $dbname;
         system ($command);