Index: trunk/ippScripts/scripts/automate_stacks.pl
===================================================================
--- trunk/ippScripts/scripts/automate_stacks.pl	(revision 27938)
+++ trunk/ippScripts/scripts/automate_stacks.pl	(revision 28222)
@@ -20,4 +20,5 @@
 my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1);
@@ -59,4 +60,5 @@
 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
 my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);
+my ( $check_detrends, $queue_detrends);
 
 GetOptions(
@@ -81,4 +83,6 @@
     'check_stacks'         => \$check_stacks,
     'queue_stacks'         => \$queue_stacks,
+    'check_detrends'          => \$check_detrends,
+    'queue_detrends'          => \$queue_detrends,
     'check_diffs'          => \$check_diffs,
     'queue_diffs'          => \$queue_diffs,
@@ -105,4 +109,6 @@
            --check_stacks         Confirm that stacks can be built.
            --queue_stacks         Issue stacktool commands to queue stacks.
+           --check_detrends       Confirm that detrend verify runs can be built.
+           --queue_detrends       Issue dettool commands to queue detrend verify runs.
            --check_diffs          Confirm that diffs can be done.
            --queue_diffs          Issue difftool commands to queue diffs.
@@ -115,6 +121,6 @@
           ) unless
     defined $check_registration or defined $define_burntool or defined $queue_burntool or
-    defined $queue_chips or defined $queue_stacks or
-    defined $check_chips or defined $check_stacks or
+    defined $queue_chips or defined $queue_stacks or $queue_detrends or
+    defined $check_chips or defined $check_stacks or $check_detrends or
     defined $test_mode or defined $clean_old or defined $check_mode;
 
@@ -128,4 +134,11 @@
 my %stackable_list = ();
 my %reduction_class = ();
+my @detrend_list = ();
+my %dettype_list = ();
+my %exptype_list = ();
+my %refID_list = ();
+my %refIter_list = ();
+my %detfilter_list = ();
+my %detmax_list = ();
 my %clean_commands = ();
 my %clean_retention = ();
@@ -158,7 +171,4 @@
 	}
     }
-#    if (${ $entry }{name} eq 'RETENTION_TIME') {
-#        $retention_time = ${ $entry }{value};
-#    }
     elsif (${ $entry }{name} eq 'FILTERS') {
         push @filter_list, ${ $entry }{value};
@@ -195,8 +205,33 @@
         }
     }
-}
-
-
-
+    elsif (${ $entry }{name} eq 'DETRENDS') {
+	my @detrend_data = @{ ${ $entry }{value} };
+	my $this_detrend = '';
+	foreach my $dentry (@detrend_data) {
+	    if (${ $dentry }{name} eq 'NAME') {
+		$this_detrend = ${ $dentry }{value};
+		push @detrend_list, $this_detrend;
+	    }
+	    elsif (${ $dentry }{name} eq 'DETTYPE') {
+		$dettype_list{$this_detrend} = ${ $dentry }{value};
+	    }
+	    elsif (${ $dentry }{name} eq 'EXPTYPE') {
+		$exptype_list{$this_detrend} = ${ $dentry }{value};
+	    }
+	    elsif (${ $dentry }{name} eq 'REF_ID') {
+		$refID_list{$this_detrend} = ${ $dentry }{value};
+	    }
+	    elsif (${ $dentry }{name} eq 'REF_ITER') {
+		$refIter_list{$this_detrend} = ${ $dentry }{value};
+	    }
+	    elsif (${ $dentry }{name} eq 'FILTER') {
+		$detfilter_list{$this_detrend} = ${ $dentry }{value};
+	    }
+	    elsif (${ $dentry }{name} eq 'MAX') {
+		$detmax_list{$this_detrend} = ${ $dentry }{value};
+	    }
+	}
+    }		
+}
 
 unless(defined($date)) {
@@ -256,4 +291,16 @@
     return_metadata($date);
     unless (defined($test_mode) || defined($check_mode)) { exit(0); }
+}
+if (defined($check_detrends) || defined($test_mode)) {
+    $metadata_out{nsState} = 'CHECKDETRENDS';
+    &execute_detrends($date,"pretend");
+    return_metadata($date);
+    unless (defined($test_mode) || defined($check_mode)) { exit(0); }
+}
+if (defined($queue_detrends)) {
+    $metadata_out{nsState} = 'QUEUEDETRENDS';
+    &execute_detrends($date,"pretend");
+    return_metadata($date);
+    exit(0);
 }
 if (defined($define_burntool) || defined($test_mode)) {
@@ -498,5 +545,5 @@
 
     my $cmd = "$chiptool";
-    $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp ';
+    $cmd .= " -simple -dbname $dbname -definebyquery -set_end_stage warp ";
     $cmd .= " -set_label $label ";
     $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
@@ -566,4 +613,6 @@
 }
 
+
+
 sub chip_queue {
     my $date = shift;
@@ -612,4 +661,101 @@
 
 #
+# Detrend verification
+################################################################################
+
+sub construct_dettool_cmd {
+    my $date = shift;
+    my $target = shift;
+
+    my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target);
+    
+    my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
+    $date =~ s/-//g;
+
+    my $cmd = "$dettool";
+    $cmd .= " -pretend ";
+    $cmd .= " -simple -dbname $dbname -definebyquery -det_type $det_type ";
+    $cmd .= " -mode verify -ref_det_id $ref_det_id -ref_iter $ref_iter ";
+    $cmd .= " $select ";
+    $cmd .= " -inst $camera ";
+    $cmd .= " -select_exp_type $exp_type ";
+    $cmd .= " -select_filter $filter " if defined($filter);
+    $cmd .= " -workdir $workdir ";
+    $cmd .= " -label $label ";
+    if ($maxN > 0) {
+	$cmd .= " -random_subset -random_limit $maxN ";
+    }
+    if ($debug == 1) {
+	$cmd .= ' -pretend ';
+    }
+    print STDERR "$cmd\n";
+    return($cmd);
+}    
+
+sub verify_uniqueness_detverify {
+    my $date = shift;
+    my $target = shift;
+
+    my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target);
+    
+    my $db = init_gpc_db();
+    my $sth = "SELECT * FROM detRun WHERE workdir = '$workdir' AND ref_det_id = $ref_det_id AND ref_iter = $ref_iter";
+    my $data_ref = $db->selectall_arrayref( $sth );
+
+    return($#{ $data_ref } + 1);
+}
+
+sub pre_detrend_queue {
+    my $date = shift;
+    my $target = shift;
+    
+    my $command = construct_dettool_cmd($date,$target) . ' -pretend ';
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run ( command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR);
+    }
+    
+    my @input_exposures = split /\n/, (join '', @$stdout_buf);
+    return($#input_exposures + 1);
+}
+
+sub detrend_queue {
+    my $date = shift;
+    my $target = shift;
+    
+    my $command = construct_dettool_cmd($date,$target);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run ( command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+    }
+    $metadata_out{nsState} = 'DETREND_QUEUED';
+    return(0);
+}
+
+sub execute_detrends {
+    my $date = shift;
+    my $pretend = shift;
+    my $exposures = 0;
+    foreach my $target (@detrend_list) {
+	my ($Nexposures) = pre_detrend_queue($date,$target);
+	if ($Nexposures == 0) {
+	    print STDERR "execute_detrends: Target $target on $date had no exposures.\n";
+	    next;
+	}
+	$exposures++;
+	unless(defined($pretend)) {
+	    detrend_queue($date,$target);
+	}
+    }
+    if ($exposures == 0) {
+	$metadata_out{nsState} = 'DETREND_DROP';
+    }
+}
+
+#
 # Stacking
 ################################################################################
@@ -627,5 +773,5 @@
     my $cmd = "$stacktool";
 #    $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 ';  # Probably silly, but I want to be safe and not overwrite
-    $cmd .= ' -simple -dbname gpc1 -definebyquery ';
+    $cmd .= " -simple -dbname $dbname -definebyquery ";
     $cmd .= " -set_label $label -select_label $label ";
     $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
@@ -779,5 +925,5 @@
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target);
     my $args = $command;
-    $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
+    $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group ";
     if ($debug == 1) {
         $args .= ' -pretend ';
@@ -826,5 +972,5 @@
 
     my $label = "${target}.nightlyscience";
-    my $workdir = "neb://\@HOST\@.0/gpc1/${target}.nt/${workdir_date}";
+    my $workdir = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}";
     my $obs_mode = $obsmode_list{$target};
     my $object   = $object_list{$target};
@@ -837,4 +983,30 @@
 }
 
+sub get_dettool_parameters {
+    my $date = shift;
+    my $target = shift;
+    my $workdir_date = $date; $workdir_date =~ s%-%/%g;
+    my $trunc_date = $date; $trunc_date =~ s/-//g;
+
+    my $exp_type = $exptype_list{$target};
+    my $det_type = $dettype_list{$target};
+    my $ref_det_id = $refID_list{$target};
+    my $ref_iter = $refIter_list{$target};
+    my $det_filter = $detfilter_list{$target};
+    my $internal_filter;
+    if (defined($det_filter)) {
+	$internal_filter = $det_filter; $internal_filter =~ s/\..*//;
+	$internal_filter = '.' . $internal_filter;
+    }
+    else {
+	$internal_filter = '';
+    }
+    my $maxN = $detmax_list{$target};    
+    
+    my $lc_type = lc($exp_type);
+    my $label = "${lc_type}${internal_filter}.$trunc_date";
+    my $workdir = 'neb://@HOST@.0/' . $dbname . "/detverify.nt/${workdir_date}/${lc_type}${internal_filter}";
+    return($label,$workdir,$det_filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN);
+}
 
 sub init_gpc_db {