Index: /branches/czw_branch/cleanup/ippScripts/scripts/addstar_run.pl
===================================================================
--- /branches/czw_branch/cleanup/ippScripts/scripts/addstar_run.pl	(revision 25088)
+++ /branches/czw_branch/cleanup/ippScripts/scripts/addstar_run.pl	(revision 25088)
@@ -0,0 +1,407 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+use Carp;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use DateTime;
+my $mjd_start = DateTime->now->mjd;   # MJD of starting script
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+use File::Temp qw( tempfile );
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Look for programs we need
+my $missing_tools;
+#my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1);
+#my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+#my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+#my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1);
+my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+my ( $exp_tag, $add_id, $camera, $outroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update,
+     $no_op, $redirect, $save_temps, $run_state);
+GetOptions(
+    'exp_tag=s'          => \$exp_tag, # Exposure identifier
+    'add_id=s'          => \$add_id, # Camtool identifier
+    'recipe=s'          => \$recipe, # Recipe to use
+    'camera|c=s'        => \$camera, # Camera
+    'dbname|d=s'        => \$dbname, # Database name
+    'outroot|w=s'       => \$outroot, # output file base name
+    'reduction=s'       => \$reduction, # Reduction class
+    'dvodb|w=s'         => \$dvodb,  # output DVO database
+    'run-state=s'       => \$run_state, # 'new' or 'update'
+    'verbose'           => \$verbose,   # Print to stdout
+    'no-update'         => \$no_update, # Update the database?
+    'no-op'             => \$no_op, # Don't do any operations?
+    'redirect-output'   => \$redirect,
+    'save-temps'        => \$save_temps, # Save temporary files?
+    ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage(
+          -msg => "Required options: --exp_tag --add_id --camera --outroot",
+          -exitval => 3,
+          ) unless
+    defined $exp_tag and
+    defined $add_id and
+    defined $outroot and
+    defined $camera;
+
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+
+my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR);
+
+if (not defined $run_state) { $run_state = 'new'; }
+if ($run_state eq 'update') {
+    $logDest .= '.update';
+}
+
+if ($redirect) {
+    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $add_id, $PS_EXIT_SYS_ERROR );
+    print "\n\n";
+    print "Starting script $0 on $host\n\n";
+    print "COMMAND IS: @ARGV\n\n";
+}
+
+# Recipes to use based on reduction class
+$reduction = 'DEFAULT' unless defined $reduction;
+
+my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use
+&my_die("Unrecognised ADDSTAR recipe", $add_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar;
+
+#my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use
+#&my_die("Unrecognised PSASTRO recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro;
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $cmdflags;
+
+# Get list of component files
+my $files;                      # Array of component files
+{
+    my $command = "$addtool -pendingexp -add_id $add_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform addtool: $error_code", $add_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_PROG_ERROR);
+
+    # extract the metadata for the files into a hash list
+    $files = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
+
+#     # since I can't figure out how to do input and output within PERL, I'm writing to a temp file
+#     my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.add.$add_id.stats.XXXX", UNLINK => !$save_temps );
+#     print "saving stats to $statName\n";
+#     foreach my $line (@$stdout_buf) {
+#         print $statFile $line;
+#     }
+#     close $statFile;
+
+#     # parse the stats in the metadata file
+#     $command = "$ppStatsFromMetadata $statName - CAMERA_EXP_IMFILE";
+#     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#         run(command => $command, verbose => $verbose);
+#     unless ($success) {
+#         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+#         warn("Unable to perform ppStatsFromMetadata: $error_code\n");
+#         exit($error_code);
+#     }
+
+#     foreach my $line (@$stdout_buf) {
+#         $cmdflags .= " $line";
+#     }
+#     chomp $cmdflags;
+}
+
+# ### not needed to have such an extensive temp file name.
+# my ($list1File, $list1Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b1.list.XXXX", UNLINK => !$save_temps ); # For binning 1
+# my ($list2File, $list2Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b2.list.XXXX", UNLINK => !$save_temps ); # For binning 2
+# my ($list3File, $list3Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b3.list.XXXX", UNLINK => !$save_temps ); # For astrometry
+
+# ### XXX for the moment, always generate the bright-star mask
+# my ($list4File, $list4Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b4.list.XXXX", UNLINK => !$save_temps ); # For astrometry
+
+# # XXX we perform astrometry iff photometry output exists
+my $chipObjectsExist = 0;
+# foreach my $file (@$files) {
+#     # use the path_base as OUTPUT root and convert the filenames with ipprc->filename:
+#     my $class_id = $file->{class_id};
+
+#     # If there is only one chip, we use this name for the input to addstar
+#     # we expect the chip analysis stage to produce psphot output (cmf file) and two binned images
+#     my $chipObjects = $ipprc->filename("PSPHOT.OUTPUT", $file->{path_base}, $class_id);
+#     my $chipMask   = $ipprc->filename("PPIMAGE.CHIP.MASK", $file->{path_base}, $class_id);
+
+#     # if any of the output chip photometry files exist, we can run psastro / addstar below
+#     if ($ipprc->file_exists($chipObjects)) {
+#         $chipObjectsExist = 1;
+#     } else {
+#         if ($verbose) { print "skipping $chipObjects (not found)\n"; }
+#         next;
+#     }
+
+#     print $list1File ($ipprc->filename("PPIMAGE.BIN1", $file->{path_base}, $class_id) . "\n");
+#     print $list2File ($ipprc->filename("PPIMAGE.BIN2", $file->{path_base}, $class_id) . "\n");
+#     print $list3File ($chipObjects . "\n");
+#     print $list4File ($chipMask . "\n");
+# }
+# close $list1File;
+# close $list2File;
+# close $list3File;
+# close $list4File;
+
+# Output products
+$ipprc->outroot_prepare($outroot);
+
+# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs
+# my $jpeg1      = $ipprc->filename("PPIMAGE.JPEG1",      $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
+# my $jpeg2      = $ipprc->filename("PPIMAGE.JPEG2",      $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
+my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",     $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR);
+# my $fpaStats   = $ipprc->filename("PSASTRO.STATS",      $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
+my $traceDest  = $ipprc->filename("TRACE.EXP",          $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR);
+#my $configuration = $ipprc->filename("PSASTRO.CONFIG",  $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
+
+if ($run_state eq 'update') {
+    $traceDest .= '.update';
+#    $fpaStats .= '.update';
+}
+
+# convert supplied DVO database name to UNIX filename
+my $dvodbReal;
+if (defined $dvodb) {
+    $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO
+    $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal );
+}
+
+my $dtime_addstar = 0;
+
+unless ($no_op) {
+
+    ## build the output JPEG images first so we get them even if the astrometry fails
+
+    # Make the jpeg for binning 1
+#     if ($run_state eq 'new') {
+#         my $command = "$ppImage -list $list1Name $outroot"; # Command to run
+#         $command .= " -recipe PPIMAGE $recipe1";
+#         $command .= " -dbname $dbname" if defined $dbname;
+
+#         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#             run(command => $command, verbose => $verbose);
+#         unless ($success) {
+#             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+#             &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code);
+#         }
+#         &my_die("Unable to find expected output file: $jpeg1", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg1);
+#     }
+
+#     # Make the jpeg for binning 2
+#     if ($run_state eq 'new') {
+#         my $command = "$ppImage -list $list2Name $outroot"; # Command to run
+#         $command .= " -recipe PPIMAGE $recipe2";
+#         $command .= " -dbname $dbname" if defined $dbname;
+
+#         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#             run(command => $command, verbose => $verbose);
+#         unless ($success) {
+#             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+#             &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code);
+#         }
+#         &my_die("Unable to find expected output file: $jpeg2", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg2);
+#     }
+
+    # only run psastro / addstar if any of the output chip astrometry files exist (should we test for successful astrometry?)
+#    if ($chipObjectsExist) {
+        # run psastro on the chipObjects, producing fpaObjects
+#         my $command;
+#         $command  = "$psastro";
+# 	$command .= " -list $list3Name";
+# 	$command .= " -masklist $list4Name";
+# 	$command .= " $outroot";
+#         $command .= " -recipe PSASTRO $recipe_psastro";
+#         $command .= " -tracedest $traceDest -log $logDest";
+#         $command .= " -dbname $dbname" if defined $dbname;
+
+#         my $do_stats;
+#         if ($run_state eq 'new') {
+#             $command .= " -stats $fpaStats -recipe PPSTATS CAMSTATS";
+#             $command .= " -dumpconfig $configuration";
+#             $do_stats = 1;
+#         } elsif ($run_state eq 'update') {
+#             $command .= " -ipprc $configuration";
+#         } else {
+#             &my_die("invalid value for run-state: $run_state", $cam_id, $PS_EXIT_CONFIG_ERROR);
+#         }
+
+#         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#             run(command => $command, verbose => $verbose);
+#         unless ($success) {
+#             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+#             &my_die("Unable to perform psastro: $error_code", $cam_id, $error_code);
+#         }
+
+#         my $quality;            # Quality flag
+#         if ($do_stats) {
+#             my $fpaStatsReal = $ipprc->file_resolve($fpaStats);
+#             &my_die("Couldn't find expected output file: $fpaStats", $cam_id, $PS_EXIT_SYS_ERROR) unless -f $fpaStatsReal;
+
+#             # parse stats from metadata
+#             $command = "$ppStatsFromMetadata $fpaStatsReal - CAMERA_EXP_FPA";
+#             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#                 run(command => $command, verbose => $verbose);
+#             unless ($success) {
+#                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+#                 &my_die("Unable to perform ppStatsFromMetadata: $error_code", $cam_id, $error_code);
+#             }
+#             foreach my $line (@$stdout_buf) {
+#                 $cmdflags .= " $line";
+#             }
+#             chomp $cmdflags;
+
+#             ($quality) = $cmdflags =~ /-quality (\d+)/;
+#         }
+
+#         if (!$quality) {
+#             &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects);
+#         }
+
+#         # run addstar on the output fpaObjects (if a DVO database is defined)
+        if (defined $dvodbReal and ($run_state eq 'new')) {
+
+            ## XXX the camera analysis can either save the full set of
+            ## detections, or just the image metadata, in the dvodb
+
+            ## get the addstar recipe for this camera and CAMERA reduction
+            my $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform ppConfigDump: $error_code", $add_id, $PS_EXIT_SYS_ERROR);
+            }
+            my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+                &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_SYS_ERROR);
+
+            ## allow the dvodb to save only images, or the full detection set
+            my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY');
+
+            # XXX this construct requires the user to have a valid .ptolemyrc
+            # XXX which in turn points at ippconfig/dvo.site
+            # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default)
+            # XXX this needs to be converted to addstar_client...
+
+            my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar
+            $command  = "$addstar -D CAMERA $camdir -update";
+            $command .= " -image" if $imagesOnly;
+            $command .= " -D CATDIR $dvodbReal";
+
+            my $realFile = $ipprc->file_resolve($fpaObjects);
+            $command .= " $realFile";
+
+            my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
+
+            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform addstar: $error_code", $add_id, $error_code);
+            }
+            $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);   # MJD of starting script
+        }
+}
+
+
+my $dtime_script = (DateTime->now->mjd - $mjd_start) * 86400;
+
+# This needs to be updated when addtool is written. BROKEN
+my $fpaCommand = "$addtool -add_id $add_id";
+if ($run_state eq 'new') {
+    $fpaCommand .= " -addprocessedexp";
+    $fpaCommand .= " -path_base $outroot";
+    $fpaCommand .= " $cmdflags";
+    $fpaCommand .= " -hostname $host" if defined $host;
+    $fpaCommand .= " -dtime_script $dtime_script";
+    $fpaCommand .= " -dtime_addstar $dtime_addstar";
+} else {
+    $fpaCommand .= " -updaterun -state full";
+}
+$fpaCommand .= " -dbname $dbname" if defined $dbname;
+
+# Add the result into the database
+unless ($no_update) {
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $fpaCommand, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("Unable to add result to database: $error_code\n");
+        exit($error_code);
+    }
+} else {
+    print "skipping command: $fpaCommand\n";
+}
+
+
+sub my_die
+{
+    my $msg = shift; # Warning message on die
+    my $add_id = shift; # Camtool identifier
+    my $exit_code = shift; # Exit code to add
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+    carp($msg);
+    if (defined $add_id and not $no_update) {
+# This needs to be updated when addtool is written. BROKEN
+        my $command = "$addtool -add_id $add_id";
+        if ($run_state eq 'new') {
+            $command .= " -addprocessedexp";
+            $command .= " -uri UNKNOWN";
+            $command .= " -fault $exit_code";
+            $command .= " -path_base $outroot";
+            $command .= " -path_base $outroot" if defined $outroot;
+            $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
+        } else {
+            $command .= " -updateprocessedexp";
+            $command .= " -fault $exit_code";
+        }
+        $command .= " -hostname $host" if defined $host;
+        $command .= " -dbname $dbname" if defined $dbname;
+        system ($command);
+    }
+    exit $exit_code;
+}
+
+
+END {
+    my $status = $?;
+    system("sync") == 0
+        or die "failed to execute sync: $!" ;
+    $? = $status;
+}
+
+__END__