Index: /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl
===================================================================
--- /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl	(revision 37279)
+++ /tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_chip.pl	(revision 37280)
@@ -24,4 +24,6 @@
 my $threads_req   = 4;
 
+my $fail_state = "prep_fail";
+
 # Look for programs we need
 my $missing_tools;
@@ -36,5 +38,5 @@
 
 # Options
-my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose);
+my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose,$dbverbose);
 GetOptions(
     'remote_id=s'    => \$remote_id,
@@ -45,4 +47,5 @@
     'no_update'      => \$no_update,
     'verbose'        => \$verbose,
+    'dbverbose'      => \$dbverbose,
     ) or pod2usage( 2 );
 
@@ -55,5 +58,5 @@
     defined($dbname);
 
-my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -61,7 +64,7 @@
 # Detrend concept holders (name, ppImage option)
 my %det_types = (
-    'MASK' 	=> '-mask',
-    'FLAT' 	=> '-flat',
-    'DARK' 	=> '-dark',
+    'MASK'      => '-mask',
+    'FLAT'      => '-flat',
+    'DARK'      => '-dark',
     'VIDEODARK' => '-dark',
     'LINEARITY' => '-linearity',
@@ -73,5 +76,4 @@
 
 # STEP 0: Open output files
-my $uri_command = $path_base . ".cmd";
 my $uri_transfer= $path_base . ".transfer";
 my $uri_check   = $path_base . ".check";
@@ -80,5 +82,4 @@
 my $uri_return  = $path_base . ".return";
 
-my $disk_command = $ipprc->file_resolve($uri_command,1);
 my $disk_transfer= $ipprc->file_resolve($uri_transfer,1);
 my $disk_check   = $ipprc->file_resolve($uri_check,1);
@@ -89,37 +90,33 @@
 my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it.
 
-open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer",$chip_id, $PS_EXIT_SYS_ERROR);
-open(CHECK,    ">$disk_check")     || &my_die("Couldn't open file? $disk_check",   $chip_id, $PS_EXIT_SYS_ERROR);
-open(CONFIG,   ">$disk_config")    || &my_die("Couldn't open file? $disk_config",  $chip_id, $PS_EXIT_SYS_ERROR);
-open(GENERATE, ">$disk_generate")  || &my_die("Couldn't open file? $disk_generate",$chip_id, $PS_EXIT_SYS_ERROR);
-open(RETURN,   ">$disk_return")    || &my_die("Couldn't open file? $disk_return",  $chip_id, $PS_EXIT_SYS_ERROR);
+open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CHECK,    ">$disk_check")     || &my_die("Couldn't open file? $disk_check",    $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CONFIG,   ">$disk_config")    || &my_die("Couldn't open file? $disk_config",   $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(GENERATE, ">$disk_generate")  || &my_die("Couldn't open file? $disk_generate", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(RETURN,   ">$disk_return")    || &my_die("Couldn't open file? $disk_return",   $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
 
 my $job_index = 0;
-my @pre_commands = ();
-my @main_commands = ();
-my @post_commands = ();
 
 # STEP 1 : Get exposure level information for this chipRun
-my $exp_id, $filter, $altfilt, $dateobs;
+my ($exp_id, $filter, $altfilt, $dateobs);
 {
     my $command = "$chiptool -listrun -chip_id $chip_id ";
     $command   .= " -dbname $dbname " if defined($dbname);
-    
+
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run(command => $command, verbose => 0);
+        run(command => $command, verbose => $dbverbose);
     unless ($success) {
-	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	
-	&my_die("Unable to run chiptool to determine stage parameters.",
-		$chip_id,$error_code);
-    }
-    
-    # Parse chipRun level data to determine exposure information 
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+        &my_die("Unable to run chiptool to determine stage parameters.", $remote_id, $chip_id, $error_code, $fail_state);
+    }
+
+    # Parse chipRun level data to determine exposure information
     my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
-	&my_die("Unable to determine chip component information.", $chip_id,$PS_EXIT_PROG_ERROR);
+        &my_die("Unable to determine chip component information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
     my $metadata = parse_md_list($MDlist);
-    my $chipEntry = $metadata[0];
-    
-    $exp_id  = $chipEntry->{exp_id};    
+    my $chipEntry = $metadata->[0];
+
+    $exp_id  = $chipEntry->{exp_id};
     $filter  = $chipEntry->{filter};
     $altfilt = $filter;
@@ -134,24 +131,29 @@
 foreach my $det_type (keys (%det_types)) {
     if (($filter !~ /y/) && ($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y
-    
+
     my $command = "detselect -search -det_type $det_type -time $dateobs";
     $command   .= " -dbname $dbname " if defined($dbname);
-    
-    if (($det_type eq 'FLAT') || ($det_type eq 'FRINGE')) { 
-	$command .= " -filter $altfilt";
-    }
-    
+
+    if (($det_type eq 'FLAT') || ($det_type eq 'FRINGE')) {
+        $command .= " -filter $altfilt";
+    }
+
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run(command => $command, verbose => 0);
+        run(command => $command, verbose => $dbverbose);
     unless ($success) {
-	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	
-	&my_die("No valid detrend available for this image: $det_type", $chip_id, $error_code);
-    }
-    
-    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or 
-	&my_die("Could not parse detrend information for this image: $det_type", $chip_id,$error_code);
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+        &my_die("No valid detrend available for this image: $det_type", $remote_id, $chip_id, $error_code, $fail_state);
+    }
+
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Could not parse detrend information for this image: $det_type", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
     my $metadata = parse_md_list($MDlist);
-    my $detEntry = $metadata[0];
+    my $detEntry = $metadata->[0];
+
+    if ($verbose) {
+        print STDERR "det_id: $detEntry->{det_id}\n";
+        print STDERR "det_iter: $detEntry->{iteration}\n";
+    }
 
     # save det_id and iteration
@@ -165,124 +167,119 @@
     my $command = "$chiptool -pendingimfile -chip_id $chip_id";
     $command   .= " -dbname $dbname "   if defined($dbname);
-    
+
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run(command => $command, verbose => $verbose);
+        run(command => $command, verbose => $dbverbose);
 
     unless ($success) {
-	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	
-	&my_die("Unable to run chiptool -pendingimfile",
-		$chip_id,$error_code);
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+        &my_die("Unable to run chiptool -pendingimfile", $remote_id, $chip_id, $error_code, $fail_state);
     }
 
     my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
-	&my_die("Unable to parse chiptool -pendingimfile information.", $chip_id,$error_code);
+        &my_die("Unable to parse chiptool -pendingimfile information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
     my $metadata = parse_md_list($MDlist);
-    
+
     # Iterate over the chipProcessedImfile level data.
     foreach my $chipEntry (@$metadata) {
-	# Get information we need to pass to ppImage
-	my $uri            = $chipEntry->{uri};
-	my $class_id       = $chipEntry->{class_id};
-	my $video_cells    = $chipEntry->{video_cells};
-	my $reduction      = $chipEntry->{reduction};
-	my $exp_tag        = $chipEntry->{exp_tag};
-	my $workdir        = $chipEntry->{workdir};
-	my $chip_imfile_id = $chipEntry->{chip_imfile_id};
-	
-	# Process the image and burntool table
-	my ($ipp_uri, $remote_uri) = uri_to_outputs_raw($uri);
-	my $btt = $uri;
-	$btt =~ s/fits$/burn.tbl/;
-	my ($ipp_btt, $remote_btt) = uri_to_outputs_raw($btt);
-	
-	# Initialize the ppI command
-	my $ppImage_command = "ppImage -file $remote_uri";
-	$ppImage_command   .= " -burntool $remote_btt ";
-	
-	foreach my $det_type (keys (%det_types)) {
-	    if (( $video_cells) && ($det_type eq 'DARK')) next;
-	    if ((!$video_cells) && ($det_type eq 'VIDEODARK')) next;
-
-	    my $det_id   = $det_ids{$det_type};
-	    my $det_iter = $det_iters{$det_type};
-
-	    # Add detrend information to the command line
-	    my $detselect_command = "detselect -select ";
-	    $detselect_command   .= " -det_id $det_id";
-	    $detselect_command   .= " -iteration $det_iter";
-	    $detselect_command   .= " -class_id $class_id";
-	    $detselect_command   .= " -dbname $dbname " if defined($dbname);
-	    
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		run(command => $detselect_command, verbose => 0);
-
-	    unless ($success) {
-		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		
-		&my_die("No valid detrend available for this image: $det_type $det_id $det_iter",
-			$chip_id,$error_code);
-	    }
-	
-	    my $detMDlist = $mdcParser->parse(join "", @$stdout_buf) or 
-		&my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter",
-			$chip_id,$error_code);
-	    my $detImfileMD = parse_md_list($detMDlist);
-	    my $duri = $detImfileMD[0]->{uri};
-	    
-	    my $det_code = $det_types{$det_type};
-	    my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri);
-	    $ppImage_command .= " $det_code $remote_det_uri ";
-	}
-	
-	# Add output root
-	my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}";
-	my $remote_outroot = uri_local_to_remote($ipp_outroot);
-	$ppImage_command .= " $remote_outroot ";
-	print STDERR "$remote_outroot $ipp_outroot $class_id\n";
-	# Complete reduction information.
-	$reduction = 'DEFAULT' unless defined $reduction;
-	my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage
-	my $recipe_psphot  = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot
-	
-	$ppImage_command .= " -recipe PPIMAGE $recipe_ppImage ";
-	$ppImage_command .= " -recipe PSPHOT $recipe_psphot ";
-	$ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats ";
-	$ppImage_command .= " -threads $threads_req ";
-	$ppImage_command .= " -image_id $chip_imfile_id ";
-	$ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace ";
-	$ppImage_command .= " -log ${remote_outroot}.${class_id}.log ";
-
-#	push @main_commands, $ppImage_command;
-
-	# Calculate pre and post commands
-	my $remote_outroot_dir = dirname($remote_outroot);
-	my $pre_command =  "mkdir -p $remote_outroot_dir";
-	
-	my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id ";
-	$post_commandA   .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot ";
-	$post_commandA   .= " -magicked 0 -hostname REMOTE -dtime_script 0 ";
-	$post_commandA   .= " -dbname $dbname " if defined $dbname;
-
-	my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo  ";
-	my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo";
-	
-#	push @post_commands, "$post_commandB $post_commandC";
-
-	$job_index++;
-
-	print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}";
-
-	# Determine which output files need to be returned.
-	foreach my $component (@return_component_list) {
-#	    uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) );
-	    my $filename = $ipprc->filename($component, $ipp_outroot, $class_id);
-	    my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename );
-	    my $remote_outroot_dir = dirname($ipp_disk);
-	    print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
-#	    print "  $filename $ipp_disk $remote_disk\n";
-#	    die;
-	}
-	print CONFIG "\n";
+        # Get information we need to pass to ppImage
+        my $uri            = $chipEntry->{uri};
+        my $class_id       = $chipEntry->{class_id};
+        my $video_cells    = $chipEntry->{video_cells};
+        my $reduction      = $chipEntry->{reduction};
+        my $exp_tag        = $chipEntry->{exp_tag};
+        my $workdir        = $chipEntry->{workdir};
+        my $chip_imfile_id = $chipEntry->{chip_imfile_id};
+
+        # Process the image and burntool table
+        my ($ipp_uri, $remote_uri) = uri_to_outputs_raw($uri);
+        my $btt = $uri;
+        $btt =~ s/fits$/burn.tbl/;
+        my ($ipp_btt, $remote_btt) = uri_to_outputs_raw($btt);
+
+        # Initialize the ppI command
+        my $ppImage_command = "ppImage -file $remote_uri";
+        $ppImage_command   .= " -burntool $remote_btt ";
+
+        foreach my $det_type (keys (%det_types)) {
+            if (( $video_cells) && ($det_type eq 'DARK')) { next; }
+            if ((!$video_cells) && ($det_type eq 'VIDEODARK')) { next; }
+
+            my $det_id   = $det_ids{$det_type};
+            my $det_iter = $det_iters{$det_type};
+
+            if (not defined $det_id) { next; }
+
+            if ($verbose) { print STDERR "det_type, det_id, det_iter: $det_type, $det_id, $det_iter\n"; }
+
+            # Add detrend information to the command line
+            my $detselect_command = "detselect -select ";
+            $detselect_command   .= " -det_id $det_id";
+            $detselect_command   .= " -iteration $det_iter";
+            $detselect_command   .= " -class_id $class_id";
+            $detselect_command   .= " -dbname $dbname " if defined($dbname);
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $detselect_command, verbose => $dbverbose);
+
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+                &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $error_code, $fail_state);
+            }
+
+            my $detMDlist = $mdcParser->parse(join "", @$stdout_buf) or
+                &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
+            my $detImfileMD = parse_md_list($detMDlist);
+            my $detImfile = $detImfileMD->[0];
+            my $duri = $detImfile->{uri};
+
+            my $det_code = $det_types{$det_type};
+            my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri);
+            $ppImage_command .= " $det_code $remote_det_uri ";
+        }
+
+        # Add output root
+        my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}";
+        my $remote_outroot = uri_local_to_remote($ipp_outroot);
+        $ppImage_command .= " $remote_outroot ";
+        print STDERR "$remote_outroot $ipp_outroot $class_id\n";
+        # Complete reduction information.
+        $reduction = 'DEFAULT' unless defined $reduction;
+        my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage
+        my $recipe_psphot  = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot
+
+        $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage ";
+        $ppImage_command .= " -recipe PSPHOT $recipe_psphot ";
+        $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats ";
+        $ppImage_command .= " -threads $threads_req ";
+        $ppImage_command .= " -image_id $chip_imfile_id ";
+        $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace ";
+        $ppImage_command .= " -log ${remote_outroot}.${class_id}.log ";
+
+        # Calculate pre and post commands
+        my $remote_outroot_dir = dirname($remote_outroot);
+        my $pre_command =  "mkdir -p $remote_outroot_dir";
+
+        my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id ";
+        $post_commandA   .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot ";
+        $post_commandA   .= " -magicked 0 -hostname REMOTE -dtime_script 0 ";
+        $post_commandA   .= " -dbname $dbname " if defined $dbname;
+
+        my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo  ";
+        my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo";
+
+        $job_index++;
+
+        print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}";
+
+        # Determine which output files need to be returned.
+        foreach my $component (@return_component_list) {
+            my $filename = $ipprc->filename($component, $ipp_outroot, $class_id);
+            my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename );
+            my $remote_outroot_dir = dirname($ipp_disk);
+            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
+        }
+        print CONFIG "\n";
     }
 }
@@ -314,7 +311,7 @@
     my $ipp_disk= $ipprc->file_resolve( $neb_uri );
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-	my_die();
+        my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -330,7 +327,7 @@
     my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1);
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-	my_die();
+        my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -345,10 +342,7 @@
     my $neb_uri = shift;
     my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
-    
-    unless (exists($file_filter{$neb_uri})) {
-	$file_filter{$neb_uri} = 1;
-	print TRANSFER "$ipp_disk\n";
-	print CHECK    "$remote_disk\n";
-    }
+
+    print TRANSFER "$ipp_disk\n";
+    print CHECK    "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
@@ -358,9 +352,7 @@
     my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
     $remote_disk = $remote_raw . $ipp_disk;
-    unless (exists($file_filter{$neb_uri})) {
-	$file_filter{$neb_uri} = 1;
-	print TRANSFER "$ipp_disk\n";
-	print CHECK    "$remote_disk\n";
-    }
+
+    print TRANSFER "$ipp_disk\n";
+    print CHECK    "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
@@ -369,9 +361,6 @@
     my $neb_uri = shift;
     my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri );
-    unless (exists($file_filter{$neb_uri})) {
-	$file_filter{$neb_uri} = 1;
-	print RETURN "$ipp_disk\n";
-	print GENERATE "$remote_disk\n";
-    }
+    print RETURN "$ipp_disk\n";
+    print GENERATE "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
@@ -387,5 +376,5 @@
     return($remote_uri);
 }
- 
+
 sub uri_remote_to_local {
     # This needs to replace the remote root directory with the nebulous tag.
@@ -393,10 +382,12 @@
     $remote_uri =~ s%${remote_root}%%;
     my $local_uri  = "neb:///" . $remote_uri;
-    
+
     return($local_uri);
 }
+
 sub my_die {
     my $msg = shift;
-    my $id  = shift;
+    my $remote_id  = shift;
+    my $stage_id  = shift;
     my $exit_code = shift;
     my $exit_state = shift;
@@ -405,15 +396,11 @@
 
     carp($msg);
-    
-    I need stage id here
-
-    
-    if (defined $id and not $no_update) {
-	my $command = "remotetool -updatecomponent -remote_id $id";
-	$command .= " -fault $exit_code " if defined $exit_code;
-	$command .= " -set_state $exit_state " if defined $exit_state;
-	$command .= " -dbname $dbname " if defined $dbname;
-
-	system($command);
+
+    if (defined $remote_id and defined $stage_id and not $no_update) {
+        my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id";
+        $command .= " -set_state $exit_state " if defined $exit_state;
+        $command .= " -dbname $dbname " if defined $dbname;
+
+        system($command);
     }