Index: trunk/ippScripts/scripts/automate_stacks.pl
===================================================================
--- trunk/ippScripts/scripts/automate_stacks.pl	(revision 28916)
+++ trunk/ippScripts/scripts/automate_stacks.pl	(revision 28982)
@@ -46,4 +46,5 @@
 my ( $check_stacks, $queue_stacks, $check_sweetspot, $queue_sweetspot, $check_diffs, $queue_diffs, $clean_old);
 my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats);
+my ( $confirm_stacks, $check_confirm_stacks );
 
 GetOptions(
@@ -70,4 +71,6 @@
     'check_stacks'         => \$check_stacks,
     'queue_stacks'         => \$queue_stacks,
+    'confirm_stacks'       => \$confirm_stacks,
+    'check_confirm_stacks' => \$check_confirm_stacks,
     'check_sweetspot'      => \$check_sweetspot,
     'queue_sweetspot'      => \$queue_sweetspot,
@@ -132,4 +135,5 @@
 my %stackable_list = ();
 my %reduction_class = ();
+my %macro_formats = ();
 my @unrecoverable_quality = ();
 my @detrend_list = ();
@@ -181,4 +185,16 @@
     elsif (${ $entry }{name} eq 'UNRECOVERABLE_QUALITY') {
         push @unrecoverable_quality, ${ $entry }{value};
+    }
+    elsif (${ $entry }{name} eq 'MACRO_DEFINITIONS') {
+	my @macro_data = @{ ${ $entry }{value} };
+	my $this_mode = '';
+	foreach my $mentry (@macro_data) {
+	    if (${ $mentry }{name} eq 'PROC_MODE') {
+		$this_mode = ${ $mentry }{value};
+	    }
+	    elsif (${ $mentry }{name} eq 'MACRO') {
+		$macro_formats{$this_mode} = ${ $mentry }{value};
+	    }
+	}
     }
     elsif (${ $entry }{name} eq 'TARGETS') {
@@ -306,4 +322,7 @@
 	print "CLEAN: $z $clean_commands{$z} $clean_retention{$z}\n";
     }
+    foreach $z (keys (%macro_formats)) {
+	print "MACROS: $z $macro_formats{$z}\n";
+    }
     
 }
@@ -385,4 +404,17 @@
 	$metadata_out{nsState} = 'STACKING_POSSIBLE_ERROR';
     }
+    return_metadata($date);
+    exit(0);
+}
+
+if (defined($check_confirm_stacks) || defined($test_mode) || defined($check_mode)) {
+    $metadata_out{nsState} = 'CONFIRM_STACKING';
+    &confirm_stacks($date,"pretend");
+    return_metadata($date);
+    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
+}
+if (defined($confirm_stacks)) {
+    $metadata_out{nsState} = 'CONFIRM_STACKING';
+    &confirm_stacks($date);
     return_metadata($date);
     exit(0);
@@ -582,5 +614,5 @@
     $metadata_out{btN} = $N - 1;
     $metadata_out{btNCounter} = 0;
-
+    add_to_macro_list('define_burntool',1,$date);
     return(0);
 }
@@ -589,4 +621,5 @@
 # Chips
 ################################################################################
+
 
 sub construct_chiptool_cmd {
@@ -680,4 +713,5 @@
     }
     $metadata_out{nsState} = 'TOWARP';
+
     return(0);
 }
@@ -696,9 +730,9 @@
         if ($Nexposures == 0) {
 	    print STDERR "execute_chips: Target $target on $date had no exposures.\n";
-	    next;
+#	    next;
         }
         if ($Nalready != 0) {
 	    print STDERR "execute_chips: Not queueing $target on $date due to already existing exposures.\n";
-            next;
+#            next;
         }
         if ($Nimfiles != $Nburntooled) {
@@ -712,4 +746,11 @@
             chip_queue($date,$target);
         }
+	if (defined($pretend)) {
+	    add_to_macro_list('check_chips',$stackable_list{$target},$date,$target);
+	}
+	else {
+	    add_to_macro_list('queue_chips',$stackable_list{$target},$date,$target);
+	}
+		    
     }
     if ($exposures == 0) {
@@ -717,4 +758,5 @@
     }
 }
+
 
 #
@@ -742,14 +784,4 @@
     my $date = shift;
     
-#     my $db = init_gpc_db();
-#     my $trunc_date = $date; $trunc_date =~ s/-//g;
-
-#     my $where = " label LIKE '%.nightlyscience' AND data_group' ";
-#     my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new' OR state = 'cleaned') AND $where ";
-#     my $cam_sth = "SELECT * from camRun WHERE state = 'full' AND $where ";
-
-#     my $chip_ref = $db->selectall_arrayref( $chip_sth );
-#     my $cam_ref = $db->selectall_arrayref( $cam_sth );
-    
     my $command = construct_dqstats_cmd($date) . ' -pretend ';
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -762,5 +794,5 @@
     my @input_exposures = split /\n/, (join '', @$stdout_buf);
 
-    return($#input_exposures + 1,1,1); # $#{ $chip_ref } + 1, $#{ $cam_ref } + 1);
+    return($#input_exposures + 1,1,1); 
 }
  
@@ -791,4 +823,10 @@
 	unless(defined($pretend)) {
 	    dqstats_queue($date);
+	}
+	if (defined($pretend)) {
+	    add_to_macro_list('check_dqstats',1,$date);
+	}
+	else {
+	    add_to_macro_list('queue_dqstats',1,$date);
 	}
     }
@@ -887,4 +925,10 @@
 	    detrend_queue($date,$target);
 	}
+	if (defined($pretend)) {
+	    add_to_macro_list('check_detrends',1,$date,$target,"dettool");
+	}
+	else {
+	    add_to_macro_list('queue_detrends',1,$date,$target,"dettool");
+	}
     }
     if ($exposures == 0) {
@@ -1023,4 +1067,10 @@
 	$metadata_out{nsState} = 'SS_DONE';
 	sweetspot_queue($date);
+    }
+    if (defined($pretend)) {
+	add_to_macro_list('check_sweetspot',1,$date);
+    }
+    else {
+	add_to_macro_list('queue_sweetspot',1,$date);
     }
 }
@@ -1173,4 +1223,10 @@
                     stack_queue($date,$target,$filter);
                 }
+		if (defined($pretend)) {
+		    add_to_macro_list('check_stacks',$stackable_list{$target},$date,$target,$filter);
+		}
+		else {
+		    add_to_macro_list('queue_stacks',$stackable_list{$target},$date,$target,$filter);
+		}
             }
         }
@@ -1181,4 +1237,49 @@
 
 }
+
+sub post_stack_queue {
+    my $date = shift;
+    my $target = shift;
+
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+    # check warp stage == chip stage
+    my $db = init_gpc_db();
+
+    my $trunc_date = $date; $trunc_date =~ s/-//g;
+
+    my $where = " label = '$label' AND data_group = '$data_group' ";
+
+    my $stack_full_sth = "SELECT * from stackRun where $where ";
+    my $stack_done_sth = "SELECT * from stackRun LEFT JOIN stackSumSkyfile USING(stack_id) WHERE $where ";
+    $stack_done_sth .= " AND ((state = 'full') OR (state = 'new' && fault != 0)) ";
+
+    my $stack_full_ref = $db->selectall_arrayref( $stack_full_sth );
+    my $stack_done_ref = $db->selectall_arrayref( $stack_done_sth );
+
+    return($#{ $stack_full_ref } + 1, $#{ $stack_done_ref } + 1);
+}
+
+sub confirm_stacks {
+    my $date = shift;
+    my $pretend = shift;
+    
+    foreach my $target (@target_list) {
+        if ($stackable_list{$target} == 1) {
+	    my ($Nstacks, $Nattempted) = post_stack_queue($date,$target);
+	    if ($Nstacks != $Nattempted) {
+		if ($debug == 1) {
+		    print STDERR "confirm_stacks: Target $target on $date is not done stacking. $Nstacks $Nattempted\n"
+		}
+		next;
+	    }
+	    if (defined($pretend)) {
+		add_to_macro_list('check_confirm_stacks',$stackable_list{$target},$date,$target);
+	    }
+	    else {
+		add_to_macro_list('confirm_stacks',$stackable_list{$target},$date,$target);
+	    }
+	}
+    }
+}	    
 
 #
@@ -1233,47 +1334,27 @@
 
     foreach my $mode (@mode_list) {
-# 	if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) {
-# 	    my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode);
-# 	    if ($cleaning_date eq 'no clean') {
-# 		next;
-# 	    }
-# 	    print STDERR "$command\n";
-# 	    if (!(defined($pretend) || $debug == 1)) {
-# #           print STDERR "BEAR IS DRIVING!?\n";
-# 		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-# 		    run ( command => $command, verbose => $verbose );
-# 		unless ($success) {
-# 		    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-# 		    &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
-# 		}
-# 	    }
-# 	}
-# 	else {
-	    foreach my $target (@target_list) {
-		if (exists($noclean_list{$target})) {
-		    next;
+	foreach my $target (@target_list) {
+	    if (exists($noclean_list{$target})) {
+		next;
+	    }
+	    my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
+	    if ($cleaning_date eq 'no clean') {
+		next;
+	    }		
+	    print STDERR "$command\n";
+	    if (!(defined($pretend) || $debug == 1)) {
+#           print STDERR "BEAR IS DRIVING!?\n";
+		my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		    run ( command => $command, verbose => $verbose );
+		unless ($success) {
+		    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		    &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
 		}
-		my ($cleaning_date,$command) = construct_cleantool_args($date,$target,$mode);
-		if ($cleaning_date eq 'no clean') {
-		    next;
-		}		
-		print STDERR "$command\n";
-		if (!(defined($pretend) || $debug == 1)) {
-#           print STDERR "BEAR IS DRIVING!?\n";
-		    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-			run ( command => $command, verbose => $verbose );
-		    unless ($success) {
-			$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-			&my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
-		    }
-		}
-	    }
-# 	}
+		add_to_macro_list('clean_old',1,$date,$target,$mode);
+	    }
+	}
     }
     return(0);
 }
-
-
-
 
 #
@@ -1324,4 +1405,42 @@
     my $workdir = 'neb://@HOST@.0/' . $dbname . "/detverify.nt/${workdir_date}/${lc_type}${internal_filter}";
     return($label,$workdir,$det_filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN);
+}
+
+sub add_to_macro_list {
+    my $proc_mode = shift;
+    my $do_or_do_not = shift;
+    my $date = shift;
+    my $target = shift;
+    my $mode = shift;
+
+    if ((defined($macro_formats{$proc_mode}))&&($do_or_do_not)) {
+	unless (defined($metadata_out{N_MACROS})) {
+	    $metadata_out{N_MACROS} = 0;
+	}
+	my $N = $metadata_out{N_MACROS};
+	$metadata_out{"ns${N}Macro"} = $macro_formats{$proc_mode};
+	print STDERR "WORKING ON A MACRO: ns${N}Macro $proc_mode $macro_formats{$proc_mode}\n";
+	if (defined($date)&&(defined($target))) {
+
+	    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) 
+		= get_tool_parameters($date,$target);
+	    $metadata_out{"ns${N}Macro"} =~ s/\@LABEL\@/$label/;
+	    $metadata_out{"ns${N}Macro"} =~ s/\@WORKDIR\@/$workdir/;
+	    $metadata_out{"ns${N}Macro"} =~ s/\@OBS_MODE\@/$obs_mode/;
+	    $metadata_out{"ns${N}Macro"} =~ s/\@OBJECT\@/$object/;
+	    $metadata_out{"ns${N}Macro"} =~ s/\@COMMENT\@/$comment/;
+	    $metadata_out{"ns${N}Macro"} =~ s/\@TESS_ID\@/$tess_id/;
+	    $metadata_out{"ns${N}Macro"} =~ s/\@DIST_GROUP\@/$dist_group/;
+	    $metadata_out{"ns${N}Macro"} =~ s/\@DATA_GROUP\@/$data_group/;
+	    $metadata_out{"ns${N}Macro"} =~ s/\@REDUCTION\@/$reduction/;
+	}
+	if (defined($mode)) {
+	    $metadata_out{"ns${N}Macro"} =~ s/\@EXTRA\@/$mode/;
+	}
+	if (defined($date)) {
+	    $metadata_out{"ns${N}Macro"} =~ s/\@DATE\@/$date/;
+	}
+	$metadata_out{N_MACROS} ++;
+    }
 }