Index: /branches/eam_branches/ipp-20110505/ippScripts/scripts/camera_exp.pl
===================================================================
--- /branches/eam_branches/ipp-20110505/ippScripts/scripts/camera_exp.pl	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippScripts/scripts/camera_exp.pl	(revision 31587)
@@ -153,4 +153,28 @@
     }
     chomp $cmdflags;
+
+    { # Determine if FWHM is too large to bother continuing.
+	my $command = "$ppConfigDump -camera $camera -dump-camera -";
+	my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $command, verbose => $verbose);
+	unless ($success) {
+	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	    &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_CONFIG_ERROR);
+	}
+	my $cameraConfig = $mdcParser->parse(join "", @$stdout_buf) or
+	    &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR);
+
+	my $maxFWHM = metadataLookupStr($cameraConfig, 'MAX_ALLOWED_FWHM');
+	if ($maxFWHM) {
+	    my $expFWHM;
+	    ($expFWHM) = $cmdflags =~ /-fwhm_major (\d+)/;	
+	    
+	    if ($expFWHM > $maxFWHM) {
+		print "Setting quality to 4007 due to large FWHM: exposure: $expFWHM  maximum: $maxFWHM\n";
+		$cmdflags .= " -quality 4007 "; # This corresponds to PSASTRO_ERR_DATA
+		$no_op = 1;
+	    }
+	}
+    }
 }
 
Index: /branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_advancerun.pl
===================================================================
--- /branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_advancerun.pl	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_advancerun.pl	(revision 31587)
@@ -114,5 +114,5 @@
     $tool_cmd = "$staticskytool -sky_id";
     $list_mode = "-result";
-    $component_key = "";
+    $component_key = "skycell_id";
 } elsif ($stage eq "diff") {
     $tool_cmd = "$difftool -diff_id";
Index: /branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_bundle.pl
===================================================================
--- /branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_bundle.pl	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_bundle.pl	(revision 31587)
@@ -62,4 +62,5 @@
 my $streaksrelease   = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1);
 my $bgtool   = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
+my $staticskytool   = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
 my $file_cmd   = can_run('file') or (warn "can't find program file" and $missing_tools = 1);
 my $zcat   = can_run('zcat') or (warn "can't find program zcat" and $missing_tools = 1);
@@ -106,25 +107,30 @@
     defined $stage_id and
     defined $component and
-    defined $exp_type and
     defined $path_base and
     defined $outroot;
 
+if ($stage eq 'raw' and !$clean and !$no_magic) {
+    
+    # for raw stage need to have exposure type defined and if the type is OBJECT we need
+    # a chip_path_base so we can find the chip mask file
+    if (!defined $exp_type or ($exp_type eq 'OBJECT' and !defined $chip_path_base)) {
+        pod2usage( -msg => "Required options: --chip_path_base --exp_type for raw stage", -exitval => 3);
+    }
+}
+
 $ipprc->redirect_output($logfile) if $logfile;
 
-if ($stage eq 'raw' and !$clean and !$no_magic) {
-    # need to be able to find chip mask file
-    if ($exp_type eq 'OBJECT' and !defined $chip_path_base) {
-        pod2usage( -msg => "Required options: --chip_path_base for raw stage", -exitval => 3);
-    }
-}
-
 $ipprc->define_camera($camera);
 
 $ipprc->outroot_prepare($outroot);
 
+my $num_sky_inputs;
+if ($stage eq 'sky') {
+    $num_sky_inputs = get_num_sky_inputs($stage_id);
+}
 # Get the list of data products for this component
 # note: We my_die in get_file_list if something goes wrong.
 
-my $file_list = get_file_list($stage, $component, $path_base, $clean);
+my $file_list = get_file_list($stage, $component, $path_base, $clean, $num_sky_inputs);
 
 if (($stage ne 'raw') and ($stage ne 'fake') and !$poor_quality) {
@@ -184,4 +190,8 @@
         # it must exist)
         next if $poor_quality;
+
+        # skip file stats file. Due to a bug the update process destroys them sometimes
+        # XXX: perhaps only do this for stages where we know that this happens
+        next if $file_rule =~ /STATS/;
 
         &my_die("failed to resolve  $file_name", $component, $PS_EXIT_DATA_ERROR);
@@ -480,4 +490,5 @@
     my $path_base = shift;
     my $clean = shift;
+    my $num_sky_inputs = shift;
 
     my @file_list;
@@ -531,5 +542,9 @@
         $config_file_rule = "PPSTACK.CONFIG";
     } elsif ($stage eq "sky") {
-        $config_file_rule = "PSPHOT.STACK.CONFIG";
+        if ($num_sky_inputs > 1) {
+            $config_file_rule = "PSPHOT.STACK.CONFIG";
+        } else {
+            $config_file_rule = "PSPHOT.SKY.CONFIG";
+        }
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -627,4 +642,24 @@
     return \@file_list;
 }
+sub get_num_sky_inputs {
+    my $sky_id = shift;
+
+    my $command = "$staticskytool -inputs -sky_id $sky_id -simple";
+    $command .= " -dbname $dbname" if $dbname;
+    $command .= " | wc";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $component, $error_code);
+    }
+    my ($num_inputs, $words, $chars) = split " ", (join "", @$stdout_buf);
+    if (!$num_inputs) {
+        $num_inputs = "undefined" if !defined $num_inputs;
+        &my_die("unexpected number of static sky inputs $num_inputs",  $PS_EXIT_PROG_ERROR, $component, $error_code);
+    }
+
+    return $num_inputs;
+}
 
 sub my_die
Index: /branches/eam_branches/ipp-20110505/ippScripts/scripts/lap_science.pl
===================================================================
--- /branches/eam_branches/ipp-20110505/ippScripts/scripts/lap_science.pl	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippScripts/scripts/lap_science.pl	(revision 31587)
@@ -99,5 +99,5 @@
     my $status = queue_chips($lap_id);
 
-    if ($status) {
+    if (!$status) { # This is the culprit.
 	my $command = "$laptool -updaterun -lap_id $lap_id";
 	$command .= " -dbname $dbname " if defined $dbname;
@@ -120,5 +120,5 @@
     # This is a puzzler... chiptool doesn't actually return a useful metadata.  We'll just scrape it from the database for now.
 
-    my $command = "$chiptool -listrun -exp_id $exp_id -label $label -data_group $data_group";
+    my $command = "$chiptool -listrun -pstamp_order -exp_id $exp_id -label $label -data_group $data_group";
     $command .= " -dbname $dbname " if defined $dbname;
 
@@ -236,4 +236,29 @@
 	my $comment = $exposure->{comment};
 
+	# This is a hack to fix old exposures that have no object.
+	unless(defined($comment)) {
+	    $comment = '';
+	}
+	if ((!defined($object))||($object eq 'NULL')||($object eq '')) {
+	    if ($comment =~ /3pi_/) {
+		$object = $comment;
+		$object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/$1/;
+	    }
+	    elsif ($comment =~ / ps1_/) {
+		$object = $comment;
+		$object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/$1/;
+	    }
+	    else {
+		$exposure->{data_state} = 'new';
+		$exposure->{pairwise} = 0;
+		$exposure->{private} = 1;
+		$exposure->{pair_id} = 9223372036854775807;
+		update_this_exposure($exposure);
+		$counter++;
+		next;
+	    }
+	}
+
+
 	if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id
 	    if (($pairwise) && !($pair_id)) {
@@ -255,4 +280,5 @@
 	    $counter++;
 	}
+	print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n";
     }
 
@@ -262,4 +288,5 @@
 	foreach my $comment (keys %{ $matching{$object} }) {
 	    push @exp_ids_to_diff, $matching{$object}{$comment};
+	    print "$object $comment $matching{$object}{$comment} $indexing{$matching{$object}{$comment}} $exp_ids_to_diff[-1]\n";
 	}
 	@exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff;
@@ -275,5 +302,5 @@
 	    my $exp_A = ${ $exposures }[$indexing{$exp_id_A}];
 	    my $exp_B = ${ $exposures }[$indexing{$exp_id_B}];
-
+	    print "$exp_A $exp_B $exp_id_A $exp_id_B $indexing{$exp_id_A} $indexing{$exp_id_B}\n";
 	    $exp_A->{pairwise} = 1;
 	    $exp_A->{private} = 0;
@@ -292,4 +319,5 @@
     # Scan all exposures, and ensure that pairwise and private are set correctly
     foreach my $exposure (@$exposures) { 
+	print "YY: $exposure\n";
 	if ($exposure->{pairwise} && !($exposure->{pair_id})) {
 	    $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error.
@@ -299,7 +327,7 @@
 	}
 
-	$exposure = update_this_exposure($exposure);
-    }
-    return(1);
+	update_this_exposure($exposure);
+    }
+    return(0);
 }
 
@@ -432,7 +460,13 @@
 	my $companion;
 
-	if ($exposure->{pair_id}) { # Load companion exposure information
-	    if (exists($match_hash{$exposure->{chip_id}})) {
+	if ($exposure->{pairwise}) {
+            # Load companion exposure information
+	    if (($exposure->{pair_id})&&(exists($match_hash{$exposure->{chip_id}}))) {
 		$companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; # Match!
+	    }
+	    else { # We claimed to be pairwise, but do not have a valid pair_id.
+		$exposure->{pairwise} = 0;
+		$exposure->{private} = 1;
+		&update_this_exposure($exposure);
 	    }
 	}
@@ -446,26 +480,49 @@
 	    $exposure = remake_this_exposure($exposure);
 	}
-	if ($exposure->{cam_quality}) {
-	    $needs_qstack = 1;
-	    $needs_something_private = 1;
-	    if ($companion) {
-		$companion->{private} = 1;
-		$companion->{pairwise} = 0;
-		&update_this_exposure($companion);
+	# Do quality checks here
+	my $is_bad_quality = 0;
+	if ((defined($exposure->{chipRun_state}))&&($exposure->{chipRun_state} eq 'full')&&
+	    ($exposure->{chip_component_count} > 0)&&($exposure->{chip_bad_quality} / $exposure->{chip_component_count} > 0.05)) {
+	    printf("QUALITY: $exposure->{exp_id} has bad chip quality: %d / %d\n",
+		   $exposure->{chip_bad_quality} , $exposure->{chip_component_count});
+	    $is_bad_quality = 1;
+	}
+	elsif ((defined($exposure->{camRun_state}))&&($exposure->{camRun_state} eq 'full')&&
+	       ($exposure->{cam_bad_quality} / $exposure->{cam_component_count} > 0)) {
+	    printf("QUALITY: $exposure->{exp_id} has bad cam quality: %d / %d\n",
+		   $exposure->{cam_bad_quality} , $exposure->{cam_component_count});
+	    $is_bad_quality = 1;
+	}
+	elsif ((defined($exposure->{warpRun_state}))&&($exposure->{warpRun_state} eq 'full')&&
+	       ($exposure->{warp_bad_quality} / $exposure->{warp_component_count} > 0.2)) {
+	    printf("QUALITY: $exposure->{exp_id} has bad warp quality: %d / %d\n",
+		   $exposure->{warp_bad_quality} , $exposure->{warp_component_count});
+	    $is_bad_quality = 1;
+	}
+	elsif ((defined($exposure->{diffRun_state}))&&($exposure->{diffRun_state} eq 'full')&&
+	       ($exposure->{diff_bad_quality} / $exposure->{diff_component_count} > 0.5)) {
+	    printf("QUALITY: $exposure->{exp_id} has bad diff quality: %d / %d\n",
+		    $exposure->{diff_bad_quality} , $exposure->{diff_component_count});
+	    $is_bad_quality = 1;
+	}
+	# If we've detected a bad quality exposure, drop it, and tell the companion.
+	if ($is_bad_quality) {
+	    unless ((defined($exposure->{diffRun_state}))&&
+		    ($exposure->{diffRun_state} eq 'full')) {
+		$needs_qstack = 1;
+		$needs_something_private = 1;
+		if ($companion) {
+		    $companion->{private} = 1;
+		    $companion->{pairwise} = 0;
+		    &update_this_exposure($companion);
+		}
+		$exposure->{private} = 1;
+		$exposure->{pairwise} = 0;
 	    }
-	    $exposure->{private} = 1;
-	    $exposure->{pairwise} = 0;
 	    $exposure->{data_state} = 'drop';
 	    &update_this_exposure($exposure);
 
 	}
-# 	if ($companion) { # Validate that there are no problems with the companion exposure
-# 	    if ($companion->{cam_quality}) { # Maybe other things here?
-# 		$exposure->{private} = 1;
-# 		$exposure->{data_state} = 'drop';
-# 		&update_this_exposure($exposure);
-# 		$needs_qstack = 1;
-# 	    }
-# 	}
+	
 	if  ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through.
 	    $can_qstack ++;
@@ -564,4 +621,12 @@
     }
 
+    my $warps = '';
+    foreach $exposure (@$exposures) {
+	if (($exposure->{data_state} != 'drop')&&
+	    (S64_IS_NOT_NULL($exposure->{warp_id}))) {
+	    $warps .= " -warp_id $exposure->{warp_id} ";
+	}
+    }
+
     my @utctime = gmtime();
     $utctime[5] += 1900;
@@ -570,5 +635,5 @@
     my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
     my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
-    my $data_group = "${label}.${date}";
+    my $data_group = "${label}.${proj_cell}.quick.${date}";
 
     my $command = "$stacktool ";
@@ -576,7 +641,8 @@
     $command .= " -dbname $dbname " if defined $dbname;
     $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter ";
-    $command .= " -set_label ${label} -set_data_group ${proj_cell}.quick.${date} ";
+    $command .= " -set_label ${label} -set_data_group $data_group ";
     $command .= "  -set_workdir $workdir  -set_dist_group NODIST ";
     $command .= " -min_num 2 -set_reduction QUICKSTACK ";
+    $command .= " $warps ";
 
     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -589,5 +655,5 @@
     $command = "$stacktool ";
     $command .= " -dbname $dbname " if defined $dbname;
-    $command .= " -sassskyfile -data_group ${proj_cell}.quick.${date} ";
+    $command .= " -sassskyfile -data_group $data_group ";
     $command .= " -filter $filter -projection_cell ${proj_cell} ";
 
@@ -632,4 +698,13 @@
     unless (defined($label) && defined($filter) && defined($proj_cell)) {
 	&my_die("Unable to perform stacktool. Insufficient information.", $lap_id);
+    }
+
+    my $warps = '';
+    foreach $exposure (@$exposures) {
+	if (($exposure->{data_state} != 'drop')&&
+	    (S64_IS_NOT_NULL($exposure->{magicked}))&&
+	    (S64_IS_NOT_NULL($exposure->{warp_id}))) {
+	    $warps .= " -warp_id $exposure->{warp_id} ";
+	}
     }
 
@@ -640,5 +715,5 @@
     my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
     my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
-    my $data_group = "${label}.${date}";
+    my $data_group = "${label}.${proj_cell}.final.${date}";
 
     my $command = "$stacktool ";
@@ -646,6 +721,7 @@
     $command .= " -dbname $dbname " if defined $dbname;
     $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter ";
-    $command .= " -set_label ${label} -set_workdir $workdir -set_data_group ${proj_cell}.final.${date} ";
-    $command .= " -min_num 2 -set_reduction THREEPI_STACK ";
+    $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group ";
+    $command .= " -min_num 2 -set_reduction THREEPI_STACK -set_dist_group ${label} ";
+    $command .= " $warps ";
 
     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -659,5 +735,5 @@
     $command = "$stacktool ";
     $command .= " -dbname $dbname " if defined $dbname;
-    $command .= " -sassskyfile -data_group ${proj_cell}.final.${date} ";
+    $command .= " -sassskyfile -data_group $data_group ";
     $command .= " -filter $filter -projection_cell ${proj_cell} ";
 
@@ -734,10 +810,12 @@
 	    $command .= " -set_dist_group $exposure->{dist_group} ";
 	}
-	
-	if ($exposure->{pairwise}) { # warpwarp
+	my $retry_command;
+	if (($exposure->{pairwise})&&(defined(${ $exposures }[$match_hash{$exposure->{chip_id}}]))) { # warpwarp
 	    my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}];
 	    $command .= " -definewarpwarp ";
-	    $command .= "-input_label $label -template_label $label -backwards ";
+	    $command .= "-input_label $label -template_label $label ";
 	    $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} ";
+	    $retry_command = $command;
+	    $command .= " -backwards "; # This usually works.
 	    $already_queued{$exposure->{warp_id}} = 1;
 	    $already_queued{$companion->{warp_id}} = 1;
@@ -763,8 +841,23 @@
 	my $diff_id = $diff->{diff_id};
 	unless (defined($diff_id)) {
-	    $exposure->{data_state} = 'drop';
-	    &update_this_exposure($exposure);
-	}
-	
+	    if ($retry_command) {
+		($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+		    run(command => $retry_command, verbose => $verbose);
+		unless ($success) {
+		    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+		    &my_die("unable to perform difftool -definewarp(warp|stack): $error_code", $exposure->{lap_id}, $exposure->{proj_cell});
+		}
+		
+		$diffs = $mdcParser->parse_list(join "", @$stdout_buf) or
+		    &my_die("Unable to parse metadata from difftool -definewarp(warp|stack)", $lap_id, "");
+		
+		$diff = ${ $diffs }[0];
+		$diff_id = $diff->{diff_id};
+	    }
+	    unless (defined($diff_id)) {
+		$exposure->{data_state} = 'drop';
+		&update_this_exposure($exposure);
+	    }
+	}
     }
 }
@@ -872,4 +965,5 @@
     
     my $command = "$laptool -updateexp -lap_id $lap_id -exp_id $exp_id ";
+    $command .= " -dbname $dbname " if defined $dbname;
     if (($exposure->{chip_id})&&(S64_IS_NOT_NULL($exposure->{chip_id}))) {
 	$command .= " -set_chip_id  $exposure->{chip_id} ";
Index: /branches/eam_branches/ipp-20110505/ippScripts/scripts/receive_file.pl
===================================================================
--- /branches/eam_branches/ipp-20110505/ippScripts/scripts/receive_file.pl	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippScripts/scripts/receive_file.pl	(revision 31587)
@@ -224,6 +224,6 @@
         $stage = 'sky';
         # XXX: This should be skycell, but the distribution code uses exposure
-        $comp_name = 'exposure';
-        $current_component = $comp_name;
+        $comp_name = 'skycell_id';
+#        $current_component = $comp_name;
     } else {
         &my_die( "unexpected run type line found in $filename: $runType\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
@@ -237,4 +237,13 @@
         $new_workdir_value = "$workdir/$destdir";
     }
+
+    if ($stage eq 'sky') {
+        # the dbinfo file for a skyRun only has one component and it doesn't contain
+        # skycell_id which is the way components are listed in the dirinfo file.
+        my @ids = keys %$components;
+        &my_die( "unexpected number of components scalar @ids found in staticsky dirinfo file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if scalar @ids != 1;
+        $current_component = $ids[0];
+    }
+
     my $component_dir;
     if ($current_component) {
Index: /branches/eam_branches/ipp-20110505/ippScripts/scripts/warp_skycell.pl
===================================================================
--- /branches/eam_branches/ipp-20110505/ippScripts/scripts/warp_skycell.pl	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippScripts/scripts/warp_skycell.pl	(revision 31587)
@@ -72,7 +72,9 @@
 
 my ($logDest, $traceDest);
+my $do_stats;
 if ($run_state eq 'new') {
     $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, 0);
     $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1);
+    $do_stats = 1;
 } elsif ($run_state eq 'update')  {
     $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 1);
@@ -162,5 +164,8 @@
 my $outputBin1 = prepare_output ("PSWARP.BIN1", $outroot, $skycell_id, 1);
 my $outputBin2 = prepare_output ("PSWARP.BIN2", $outroot, $skycell_id, 1);
-my $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1);
+my $outputStats;
+if ($do_stats) {
+    $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1) if $do_stats;
+}
 my $configuration;
 
@@ -257,5 +262,4 @@
 # Run pswarp
 my $cmdflags;
-my $do_stats;
 unless ($no_op) {
     my $command = "$pswarp";
@@ -279,5 +283,4 @@
     if ($run_state eq 'new') {
         $command .= " -dumpconfig $configuration";
-        $do_stats = 1;
     } else {
         $command .= " -ipprc $configuration";
Index: /branches/eam_branches/ipp-20110505/ippTasks/lap.pro
===================================================================
--- /branches/eam_branches/ipp-20110505/ippTasks/lap.pro	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippTasks/lap.pro	(revision 31587)
@@ -17,8 +17,19 @@
 
 macro lap.show.books
+    echo "lapNewRuns"
     book listbook lapNewRuns
+    echo "lapRunRuns"
     book listbook lapRunRuns
+    echo "lapDoneRuns"
     book listbook lapDoneRuns
+    echo "lapFullRuns"
     book listbook lapFullRuns
+end
+
+macro lap.clear.books
+    book init lapNewRuns
+    book init lapRunRuns
+    book init lapFullRuns
+    book init lapDoneRuns
 end
 
@@ -36,10 +47,4 @@
       active true
     end
-    task lap.cleanup.load
-      active true
-    end
-    task lap.cleanup.run
-      active true
-    end
 end
 
@@ -57,8 +62,17 @@
       active false
     end
-    task lap.cleanup.load
-      active false
-    end
-    task lap.cleanup.run
+end
+
+macro lap.debug.mode
+    task lap.initial.load
+      active true
+    end
+    task lap.initial.run
+      active false
+    end
+    task lap.monitor.load
+      active true
+    end
+    task lap.monitor.run
       active false
     end
@@ -156,7 +170,8 @@
   # success
   task.exit  0
-#    book delpage lapNewRuns $options:0
     ipptool2book stdout lapNewRuns -uniq -key lap_id -setword dbname $options:0 -setword pantaskState INIT
 
+    process_cleanup lapNewRuns
+
     if ($VERBOSE > 2)
       book listbook lapNewRuns
@@ -181,4 +196,5 @@
   periods      -exec $LOADEXEC
   periods      -timeout 600
+  active       false
 # This can probably be increased and spread over hosts in the future.
   npending     1            
@@ -193,11 +209,12 @@
 
 
-    book getpage lapNewRuns $lap_NewPage -var lapNewPageName 
+    book getpage lapNewRuns 0 -var lapNewPageName -key pantaskState INIT
 
     $lap_NewPage ++
     if ($lap_NewPage >= $N) set lap_NewPage = 0
 
-
     if ("$lapNewPageName" == "NULL") break
+
+
 
     book setword lapNewRuns $lapNewPageName pantaskState RUN
@@ -205,15 +222,15 @@
     book getword lapNewRuns $lapNewPageName dbname -var DBNAME
 
+    option $LAP_ID
+
     $run = lap_science.pl --chip_mode --dbname $DBNAME --lap_id $LAP_ID
-
-    command $run
-
-  end
-
-  # success
-  task.exit  0
-#    book delpage lapNewRuns $options:0
-#    ipptool2book stdout lapNewRuns -uniq -key lap_id
-
+    
+    command $run
+
+  end
+
+  # success
+  task.exit  0
+    process_exit lapNewRuns $options:0 0
     if ($VERBOSE > 2)
       book listbook lapNewRuns
@@ -222,11 +239,14 @@
   # locked list
   task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  #operation times out?
-  task.exit    timeout
+    process_exit lapNewRuns $options:0 0
+    showcommand failure
+  end
+  task.exit    crash
+    process_exit lapNewRuns $options:0 0
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    process_exit lapNewRuns $options:0 0
     showcommand timeout
   end
@@ -267,11 +287,11 @@
 
     add_poll_labels run
-#    echo $run
-    command $run
-  end
-  # success
-  task.exit  0
-#    book delpage lapRunRuns $options:0
+    command $run
+  end
+  # success
+  task.exit  0
     ipptool2book stdout lapRunRuns -uniq -key lap_id -setword dbname $options:0 -setword pantaskState INIT
+    
+    process_cleanup lapRunRuns
 
     if ($VERBOSE > 2)
@@ -297,4 +317,5 @@
   periods      -exec $LOADEXEC
   periods      -timeout 600
+  active       false
 # This can probably be increased and spread over hosts in the future.
   npending     1            
@@ -306,15 +327,13 @@
     book npages lapRunRuns -var N
 
-#    echo $N $NETWORK
     if ($N == 0) break
     if ($NETWORK == 0) break
 
 
-    book getpage lapRunRuns $lap_RunPage -var lapRunPageName
+    book getpage lapRunRuns 0 -var lapRunPageName -key pantaskState INIT
 
     $lap_RunPage ++
     if ($lap_RunPage >= $N) set lap_RunPage = 0
 
-#    echo $lapRunPageName
     if ("$lapRunPageName" == "NULL") break
 
@@ -323,4 +342,6 @@
     book getword lapRunRuns $lapRunPageName dbname -var DBNAME
 
+    option $LAP_ID
+
     $run = lap_science.pl --monitor_mode --dbname $DBNAME --lap_id $LAP_ID
 
@@ -331,8 +352,7 @@
   # success
   task.exit  0
-#    book delpage lapRunRuns $options:0
-#    ipptool2book stdout lapRunRuns -uniq -key lap_id
-
-    if ($VERBOSE > 2)
+    process_exit lapRunRuns $options:0 0
+    if ($VERBOSE > 2)
+
       book listbook lapRunRuns
     end
@@ -340,11 +360,14 @@
   # locked list
   task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  #operation times out?
-  task.exit    timeout
+    process_exit lapRunRuns $options:0 0
+    showcommand failure
+  end
+  task.exit    crash
+    process_exit lapRunRuns $options:0 0
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    process_exit lapRunRuns $options:0 0
     showcommand timeout
   end
@@ -358,4 +381,5 @@
   periods      -exec $LOADEXEC
   periods      -timeout 30
+  active       false
   npending     1
 
@@ -390,4 +414,5 @@
     ipptool2book stdout lapDoneRuns -uniq -key lap_id  -setword dbname $options:0 -setword pantaskState INIT
 
+    process_cleanup lapDoneRuns
     if ($VERBOSE > 2)
       book listbook lapRuns
@@ -412,4 +437,5 @@
   periods      -exec $LOADEXEC
   periods      -timeout 600
+  active       false
 # This can probably be increased and spread over hosts in the future.
   npending     1            
@@ -424,5 +450,5 @@
 
 
-    book getpage lapDoneRuns $lap_DonePage -var lapDonePageName 
+    book getpage lapDoneRuns 0 -var lapDonePageName -key pantaskState INIT
 
     $lap_DonePage ++
@@ -435,4 +461,5 @@
     book getword lapDoneRuns $lapDonePageName dbname -var DBNAME
 
+    option $LAP_ID
     $run = lap_science.pl --cleanup_mode --dbname $DBNAME --lap_id $LAP_ID
 
@@ -443,6 +470,5 @@
   # success
   task.exit  0
-#    ipptool2book stdout lapDoneRuns -uniq -key lap_id
-
+    process_exit lapDoneRuns $options:0 0
     if ($VERBOSE > 2)
       book listbook lapDoneRuns
@@ -451,12 +477,15 @@
   # locked list
   task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  #operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
+    process_exit lapDoneRuns $options:0 0
+    showcommand failure
+  end
+  task.exit    crash
+    process_exit lapDoneRuns $options:0 0
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    process_exit lapDoneRuns $options:0 0
+    showcommand timeout
+  end
+end
Index: /branches/eam_branches/ipp-20110505/ippToPsps/jython/batch.py
===================================================================
--- /branches/eam_branches/ipp-20110505/ippToPsps/jython/batch.py	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippToPsps/jython/batch.py	(revision 31587)
@@ -32,5 +32,17 @@
     "../config/2/tables.vot"
     '''
-    def __init__(self, logger, batchType, inputFitsPath="", survey="", useFullTables=False):
+    def __init__(self, 
+                 logger, 
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 id,
+                 batchType, 
+                 inputFitsPath="", 
+                 survey="", 
+                 useFullTables=False):
+
+        self.everythingOK = False
+        self.readHeader = False
 
         # set up logging
@@ -40,4 +52,8 @@
 
         # set up class variables
+        self.id = id
+        self.gpc1Db = gpc1Db
+        self.ippToPspsDb = ippToPspsDb
+        self.scratchDb = scratchDb
         self.batchType = batchType;
         self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot"
@@ -46,4 +62,11 @@
         self.useFullTables = useFullTables
 
+        if self.alreadyProcessed(): return
+
+        # do we have an input file?
+        if self.inputFitsPath != "":
+
+            if not self.readPrimaryHeader(): return
+
         # TODO
         self.tablesToExport = []
@@ -51,9 +74,4 @@
         # open config
         doc = ElementTree(file="config.xml")
-
-        # create Gpc1Db object
-        self.gpc1Db = Gpc1Db(self.logger)
-        self.ippToPspsDb = IppToPspsDb(logger)
-        self.scratchDb = ScratchDb(logger, self.useFullTables)
 
         if self.survey != "":
@@ -88,13 +106,9 @@
         self.dateStr = now.strftime("%Y-%m-%d")
 
-        if self.inputFitsPath != "": 
-            file = open(self.inputFitsPath)
-            self.header = self.parseFitsHeader(file)
-            self.logger.info("Read primary and found " + str(len(self.header)) + " header cards") 
-            # TODO close file?
-
         # create DVO tables if accessing DVO directly
         if not self.useFullTables: self.scratchDb.createDvoTables()
 
+        self.everythingOK = True
+
     '''
     Destructor
@@ -103,4 +117,27 @@
 
         self.logger.debug("Batch destructor")
+
+
+    '''
+    Reads the primary header of the FITS file
+    '''
+    def readPrimaryHeader(self):
+
+        if self.readHeader: return True
+
+        # does it exist?
+        if not os.path.isfile(self.inputFitsPath):
+
+            self.logger.error("Cannot read file at '" + self.inputFitsPath + "'")
+            return False
+
+        file = open(self.inputFitsPath)
+        self.header = self.parseFitsHeader(file)
+        self.logger.info("Read primary header and found " + str(len(self.header)) + " header cards")
+        # TODO close file?
+
+        self.readHeader = True
+
+        return True
 
 
@@ -138,6 +175,8 @@
             file.seek(index + 2880, 0)
             
-        if found != True: self.logger.error("...could not find extension '" + name + "'")
-        else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards") 
+        if found != True: 
+            self.logger.error("...could not read header in extension '" + name + "'")
+            return
+        #else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards") 
 
         return header
@@ -262,9 +301,13 @@
 
         first = True
+
+        self.totalDetections = 0
         for table in tables:
 
-            sql = "SELECT MIN(objID), MAX(objID) FROM " + table
-            rs = self.scratchDb.stmt.executeQuery(sql)
+            sql = "SELECT MIN(objID), MAX(objID), COUNT(objID) FROM " + table
+            rs = self.scratchDb.executeQuery(sql)
             rs.first()
+
+            self.totalDetections = self.totalDetections + rs.getLong(3)
 
             if first:
@@ -276,6 +319,9 @@
 
             first = False
+            rs.close()
 
         self.ippToPspsDb.updateMinMaxObjID(self.batchID, self.minObjID, self.maxObjID)
+        self.logger.info("Total detections = %ld min objID = %ld max objID = %ld" % (self.totalDetections, self.minObjID, self.maxObjID))
+
 
     '''
@@ -313,5 +359,5 @@
          self.pspsTables = stilts.treads(self.pspsVoTableFilePath)
          for table in self.pspsTables:
-             self.logger.info("Creating PSPS table: " + table.name)
+             self.logger.debug("Creating PSPS table: " + table.name)
              table.write(self.scratchDb.url + '#' + table.name)
              self.tablesToExport.append(table.name)
@@ -337,5 +383,5 @@
     Accepts a regular expression filter so not all tables need to be imported
     '''
-    def importIppTables(self, filter):
+    def importIppTables(self, filter=""):
 
       self.logger.info("Attempting to import tables from input FITS file")
@@ -347,12 +393,12 @@
           match = re.match(filter, table.name)
           if not match: continue
-          self.logger.info("   Reading IPP table " + table.name + " from FITS file")
+          self.logger.info("Reading IPP table " + table.name + " from FITS file")
           table = stilts.tpipe(table, cmd='explodeall')
 
           # drop any previous tables before import
-          self.scratchDb.dropTable(table.name)
+          #self.scratchDb.dropTable(table.name)
 
           # IPP FITS files are littered with infinities, so remove these
-          self.logger.info("   Removing Infinity values from all columns")
+          self.logger.debug("Removing Infinity values from all columns")
           table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
           table = stilts.tpipe(table, cmd='replaceval Infinity null *')
@@ -360,10 +406,11 @@
           try:
               table.write(self.scratchDb.url + '#' + table.name)
+              self.scratchDb.killLastConnectionID()
+              count = count + 1
           except:
-              self.logger.exception("   Problem writing table '" + table.name + "' to the database")
-          count = count + 1
+              self.logger.exception("Problem writing table '" + table.name + "' to the database")
+
 
       self.logger.info("Done. Imported %d tables" % count)
-
       self.indexIppTables()
 
@@ -373,8 +420,8 @@
     def exportPspsTablesToFits(self, regex="(.*)"):
 
-        self.logger.info("Replacing NULLs with -999 then exporting all PSPS tables to FITS")
+        self.logger.info("Replacing NULLs with -999, changing tables names using regex: " + regex)
         _tables = []
 
-        self.logger.info("    Selecting database tables")
+        self.logger.info("Selecting database tables")
         for table in self.tablesToExport:
 
@@ -383,6 +430,11 @@
 
            # get everything from table
-           _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table)
-
+           try:
+               _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table)
+               self.scratchDb.killLastConnectionID()
+           except:
+               self.logger.exception("Could not read from DB table: " + table)
+               return False
+               
            # replace nulls and empty fields with weird PSPS -999 pseudo-null
            _table = stilts.tpipe(_table, cmd='replaceval "" -999 *')
@@ -395,8 +447,13 @@
            _tables.append(_table)
 
-        self.logger.info("    Writing to FITS file '" + self.outputFitsPath + "'...")
-        stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
-        self.logger.info("    ...done")
-        self.ippToPspsDb.updateProcessed(self.batchID, 1)
+        self.logger.info("Writing to FITS file '" + self.outputFitsPath + "'...")
+        try:
+            stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
+            self.ippToPspsDb.updateProcessed(self.batchID, 1)
+        except:
+            self.logger.exception("Could not write to FITS")
+            return False
+
+        return True
 
     '''
@@ -447,6 +504,24 @@
     '''
     def alreadyProcessed(self):
-           self.logger.info("Not implemented")
-
-
-
+        self.logger.info("Not implemented")
+
+
+    '''
+    Creates and publishes a batch
+    '''
+    def run(self):
+
+        if not self.everythingOK: return
+
+        self.createEmptyPspsTables()
+        self.importIppTables()
+        if self.populatePspsTables():
+            if self.exportPspsTablesToFits():
+                self.writeBatchManifest()
+                self.createTarball()
+                self.publishToDatastore()
+                #self.reportNullsInAllPspsTables(False)
+                #sys.exit()
+        self.logger.info("Finished.")
+
+
Index: /branches/eam_branches/ipp-20110505/ippToPsps/jython/detectionbatch.py
===================================================================
--- /branches/eam_branches/ipp-20110505/ippToPsps/jython/detectionbatch.py	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippToPsps/jython/detectionbatch.py	(revision 31587)
@@ -6,6 +6,9 @@
 from java.lang import *
 from java.sql import *
+
 from batch import Batch
 from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
 
 import logging.config
@@ -19,22 +22,35 @@
     Constructor
     '''
-    def __init__(self, logger, camID, inputFile, test=False, useFullTables=False):
+    def __init__(self, 
+                 logger,
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 camID, 
+                 inputFile, 
+                 test=False, 
+                 useFullTables=False):
+
        super(DetectionBatch, self).__init__(
                logger,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               camID,
                "detection", 
                inputFile, 
-               "MD04",
-               useFullTables) # TODO
+               "MD04", # TODO
                #"3PI") # TODO
-
-       self.logger.info("DetectionBatch constructor. Creating batch from: '" + inputFile + "'")
-
-       meta = self.gpc1Db.getCameraStageMeta(camID)
-      
-       self.expID = meta[0];
-       self.expName = meta[1];
-       self.distGroup = meta[2];
-
-       self.logger.info("Processing exposure with ID: %d, name: %s and distribution group: %s" % (self.expID, self.expName, self.distGroup))
+               useFullTables)
+
+       if not self.everythingOK: return
+
+       # meta data to the log
+       self.logger.info("New Detection Batch:")
+       self.logger.info("Cam ID:             %d" % self.id)
+       self.logger.info("file:               %s" % inputFile)
+       self.logger.info("Exp ID:             %d" % self.expID)
+       self.logger.info("Exp name:           %s" % self.expName)
+       self.logger.info("Distribution group: %s" % self.distGroup)
 
        # create an output filename, which is {expID}.FITS
@@ -167,5 +183,5 @@
         ," + self.header['PCA2X0Y2'] + " \
         )"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("FrameMeta", "surveyID", str(self.surveyID))
@@ -184,5 +200,5 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE ImageMeta"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -310,9 +326,9 @@
                )"
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         self.scratchDb.updateFilterID(tableName, self.filter)
         self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
-        self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO'])
+        if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO'])
         self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0")
 
@@ -327,5 +343,5 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE Detection"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -375,10 +391,9 @@
                ,EXT_NSIGMA \
                FROM " + ota + "_psf"
-
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         # set obsTime
         sql = "UPDATE " + tableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr)
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
         self.scratchDb.updateAllRows(tableName, "historyModNum", "0")
@@ -387,5 +402,5 @@
         self.scratchDb.updateFilterID(tableName, self.filter)
 
-        # now delete bad flux
+        # now delete bad flux and bad chip positions
         self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "instFlux")
         self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "peakADU")
@@ -401,5 +416,5 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE SkinnyObject"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -415,5 +430,5 @@
                ,surveyID \
                FROM Detection_" + ota
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
@@ -429,5 +444,5 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE ObjectCalColor"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -443,5 +458,5 @@
                ,filterID \
                FROM Detection_" + ota
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
@@ -484,5 +499,5 @@
 
         imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID)
-        self.logger.info("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
+        self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
         sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \
                a.ippObjID = b.ippObjID, \
@@ -494,5 +509,5 @@
                AND b.imageID = " + str(imageID)
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
 
@@ -511,4 +526,5 @@
 
         # loop through all OTAs and populate ImageMeta extensions
+        self.logger.info("Reading all fits headers and populating ImageMeta tables")
         for x in range(self.startX, self.endX):
             for y in range(self.startY, self.endY):
@@ -524,4 +540,8 @@
                 # load corresponding header into memory
                 header = self.findAndReadFITSHeader(ota + ".hdr", file)
+                if not header:
+                    self.logger.error("No header found for OTA " + ota)
+                    continue
+
 
                 # store sourceID/imageID combo in Db so DVO can look up later
@@ -578,6 +598,7 @@
 
                 # update ImageMeta with count of detections for this OTA and photoCodeID
-                sql = "UPDATE ImageMeta_" + ota + " SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota]))
-                self.scratchDb.stmt.execute(sql)
+                sql = "UPDATE ImageMeta_" + ota + " \
+                       SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota]))
+                self.scratchDb.execute(sql)
 
                 self.populateSkinnyObjectTable(ota)
@@ -603,5 +624,5 @@
         # update FrameMeta with count OTAs in this file and total number of photometric reference sources
         sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef)
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         
         return True
@@ -613,5 +634,5 @@
 
         sql = "UPDATE " + tableName + " SET imageID = %d%d%d" % (self.expID, x, y) 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -619,4 +640,9 @@
     '''
     def alreadyProcessed(self):
+
+        meta = self.gpc1Db.getCameraStageMeta(self.id)
+        self.expID = meta[0];
+        self.expName = meta[1];
+        self.distGroup = meta[2];
 
         return self.ippToPspsDb.alreadyProcessed("detection", "exp_id", self.expID)
@@ -634,9 +660,31 @@
 
 
+    '''
+    Overriding this method. Filter to only import *.psf extensions
+    '''
+    def importIppTables(self, filter=""):
+       return super(DetectionBatch, self).importIppTables(".*.psf")
+
+
+    '''
+    Overriding this method. Use regex to trim off, eg _XY33 extension
+    '''
+    def exportPspsTablesToFits(self, regex="(.*)"):
+       return super(DetectionBatch, self).exportPspsTablesToFits("([a-zA-Z]+)")
+
+
+# TODO put in config
+useFullTables=True
+testMode=False
+
 logging.config.fileConfig("logging.conf")
 logger = logging.getLogger("detectionbatch")
+logger.setLevel(logging.INFO)
 logger.info("Starting")
 
 gpc1Db = Gpc1Db(logger)
+ippToPspsDb = IppToPspsDb(logger)
+scratchDb = ScratchDb(logger, useFullTables)
+
 camIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.V2", "cam")
 logger.info("Found %d exposures" % len(camIDs))
@@ -645,25 +693,16 @@
 for camID in camIDs:
 
-    logger.info("-------------------------------------------------- cam ID: %d" % camID)
+    #if camID < 43764: continue # TODO 
 
     file = gpc1Db.getCameraStageSmf(camID)
-    if not os.path.isfile(file):
-        logger.error("Cannot read file at '" + file)
-        continue
-
-    detectionBatch = DetectionBatch(logger, camID, file, False, True)
-
-    if not detectionBatch.alreadyProcessed():
-
-        detectionBatch.createEmptyPspsTables()
-        detectionBatch.importIppTables(".*.psf")
-        if detectionBatch.populatePspsTables():
-            detectionBatch.exportPspsTablesToFits("([a-zA-Z]+)")
-            detectionBatch.writeBatchManifest()
-            #detectionBatch.reportNullsInAllPspsTables(False)
-            #detectionBatch.createTarball()
-            #detectionBatch.publishToDatastore()
-    
-            i = i+1
-           # if i > 0: sys.exit()
-
+
+    detectionBatch = DetectionBatch(logger,
+                                    gpc1Db,
+                                    ippToPspsDb,
+                                    scratchDb,
+                                    camID, 
+                                    file, 
+                                    testMode, 
+                                    useFullTables)
+    detectionBatch.run()
+
Index: /branches/eam_branches/ipp-20110505/ippToPsps/jython/gpc1db.py
===================================================================
--- /branches/eam_branches/ipp-20110505/ippToPsps/jython/gpc1db.py	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippToPsps/jython/gpc1db.py	(revision 31587)
@@ -29,4 +29,33 @@
         self.logger.debug("Gpc1Db destructor")
 
+
+    '''
+    TODO
+    '''
+    def getIDsInThisDVODbForThisStageFudge(self):
+
+        sql = "SELECT staticskyRun.sky_id \
+               FROM staticskyInput, staticskyRun, stackRun, staticskyResult \
+               WHERE staticskyRun.sky_id = staticskyInput.sky_id \
+               AND staticskyInput.stack_id = stackRun.stack_id \
+               AND staticskyInput.sky_id = staticskyResult.sky_id \
+               AND staticskyRun.label like 'MD04.staticsky' \
+               AND stackRun.filter like 'i%'"
+
+        try:
+            rs = self.executeQuery(sql)
+        except:
+            self.logger.exception("Can't query for ids in DVO")
+
+        ids = []
+        while (rs.next()):
+            ids.append(rs.getInt(1))
+
+        rs.close()
+
+        self.logger.info("Found %d items in DVO database '" % (len(ids)))
+
+        return ids
+
     '''
     Gets a list of ids in this DVO database for this stage, could be cam or staticsky (so far)
@@ -40,5 +69,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
         except:
             self.logger.exception("Can't query for ids in DVO")
@@ -50,5 +79,5 @@
         rs.close()
 
-        self.logger.debug("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage))
+        self.logger.info("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage))
 
         return ids
@@ -73,5 +102,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
         except:
             self.logger.exception("Can't query for imageIDs")
@@ -105,5 +134,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             meta.append(rs.getInt(1))
@@ -113,4 +142,5 @@
 
         return meta
+
     '''
     Gets some camera-stage meta data for this cam_id
@@ -127,5 +157,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             meta.append(rs.getInt(1))
@@ -150,5 +180,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
         except:
@@ -171,4 +201,6 @@
             files = glob.glob(path + "/*.cmf")
 
+        if len(files) < 1: return "NULL"
+
         return files[0] # TODO just returning first file - check
 
@@ -186,5 +218,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
         except:
@@ -200,8 +232,6 @@
 
             f=os.popen("neb-ls -p "+path+"%cmf")
-            print "neb-ls -p "+path+"%cmf"
             for i in f.readlines():
                 files.append(i.rstrip())
-                print i.rstrip()
 
         # or not a neb path
@@ -211,2 +241,32 @@
         return files
 
+
+    '''
+    TODO hack to get exposure time for a stack
+    '''
+    def getStackExpTime(self, stackID):
+
+        self.logger.debug("Querying GPC1 for stack exposure time")
+
+        sql = "SELECT SUM(exp_time) * (COUNT(warp_id) - reject_images) / COUNT(warp_id) as EXPTIME \
+               FROM staticskyRun JOIN staticskyInput using(sky_id) \
+               JOIN stackRun using(stack_id) \
+               JOIN stackSumSkyfile using(stack_id) \
+               JOIN stackInputSkyfile using(stack_id) \
+               JOIN warpRun using(warp_id) \
+               JOIN fakeRun using(fake_id) \
+               JOIN camRun using(cam_id) \
+               JOIN chipRun using(chip_id) \
+               JOIN rawExp using(exp_id) \
+               WHERE stack_id = %d" % stackID
+
+        try:
+            rs = self.executeQuery(sql)
+            rs.first()
+            return rs.getInt(1)
+        except:
+            self.logger.exception("Can't query for exposure time")
+
+        return 0.0
+
+
Index: /branches/eam_branches/ipp-20110505/ippToPsps/jython/ipptopspsdb.py
===================================================================
--- /branches/eam_branches/ipp-20110505/ippToPsps/jython/ipptopspsdb.py	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippToPsps/jython/ipptopspsdb.py	(revision 31587)
@@ -37,5 +37,5 @@
                )"
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
         sql = "SELECT MAX(batch_id) FROM batch"
@@ -44,5 +44,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             batchID = rs.getInt(1)
@@ -64,5 +64,5 @@
                WHERE batch_id = " + str(batchID)
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -75,5 +75,5 @@
                WHERE batch_id = " + str(batchID)
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -86,5 +86,5 @@
                WHERE batch_id = " + str(batchID)
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -101,5 +101,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             if rs.getInt(1) > 0: 
@@ -128,5 +128,5 @@
                )"
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -149,5 +149,5 @@
                )"
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
 
Index: /branches/eam_branches/ipp-20110505/ippToPsps/jython/mysql.py
===================================================================
--- /branches/eam_branches/ipp-20110505/ippToPsps/jython/mysql.py	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippToPsps/jython/mysql.py	(revision 31587)
@@ -37,5 +37,15 @@
         self.url = "jdbc:mysql://"+self.dbHost+"/"+self.dbName+"?user="+self.dbUser+"&password="+self.dbPass
         self.con = DriverManager.getConnection(self.url)
-        self.stmt = self.con.createStatement()
+        self.connectionID = self.getLastConnectionID()
+        self.logger.info("MySQL connection to %s with ID %d" % (dbType, self.connectionID))
+
+        #self.stmt = self.con.createStatement()
+
+
+    '''
+    Disconnect from database
+    '''
+    def disconnect(self):
+        self.con.close()
 
     '''
@@ -45,6 +55,31 @@
 
         self.logger.debug("MySql destructor")
-        self.stmt.close()
-        self.con.close()
+        self.disconnect()
+
+    '''
+    Kills the last connection ID, so long as it's not THIS connection ID
+    '''
+    def killLastConnectionID(self):
+   
+        connectionID = self.getLastConnectionID()
+        if connectionID == self.connectionID:
+            self.logger.error("NOT going to kill THIS connection ID")
+            return
+
+        sql = "KILL %d" % connectionID
+        self.execute(sql)
+
+    '''
+    Gets the last connection ID
+    '''
+    def getLastConnectionID(self):
+
+        sql = "SELECT ID \
+               FROM INFORMATION_SCHEMA.PROCESSLIST \
+               WHERE DB='" + self.dbName + "' \
+               ORDER BY ID"
+        rs = self.executeQuery(sql)
+        rs.last()
+        return rs.getInt(1)
 
     '''
@@ -54,5 +89,5 @@
 
         sql = "UPDATE " + table + " SET " + column + " = " + value
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -62,5 +97,5 @@
 
         sql = "DROP TABLE " + table
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: return
 
@@ -74,5 +109,5 @@
         sql = "ALTER TABLE " + table + " ADD UNIQUE (" + column + ")"
         try:
-            self.stmt.execute(sql)
+            self.execute(sql)
         except: pass
             #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'")
@@ -82,11 +117,29 @@
     def createIndex(self, table, column):
 
-        self.logger.debug("Creating index on column '"+column+"' for table '"+table+"'")
+        #self.logger.debug("Creating index on column '"+column+"' for table '"+table+"'")
 
         sql = "CREATE INDEX "+table+"_"+column+"_index ON "+table+" ("+column+")"
         try:
-            self.stmt.execute(sql)
+            self.execute(sql)
         except: pass
             #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'")
+    '''
+    TODO
+    '''
+    def execute(self, sql):
+
+        stmt = self.con.createStatement()
+        stmt.execute(sql)
+        stmt.close()
+
+    '''
+    TODO
+    '''
+    def executeQuery(self, sql):
+
+        stmt = self.con.createStatement()
+        rs = stmt.executeQuery(sql)
+        #stmt.close()
+        return rs
 
     '''
@@ -96,5 +149,5 @@
 
        sql = "SHOW COLUMNS FROM " + tableName
-       rs = self.stmt.executeQuery(sql)
+       rs = self.executeQuery(sql)
        columns = []
        while (rs.next()): columns.append(rs.getString(1))
@@ -109,5 +162,5 @@
 
       sql = "UPDATE " + tableName + " SET " + column + " = " + sub + " WHERE " + column + " IS NULL"
-      self.stmt.execute(sql)
+      self.execute(sql)
 
     '''
@@ -123,5 +176,5 @@
           
           sql = "UPDATE " + tableName + " SET " + column + " = " + sub + " WHERE " + column + " IS NULL"
-          self.stmt.execute(sql)
+          self.execute(sql)
 
     '''
@@ -131,11 +184,11 @@
 
         sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + columnName + " = " + value
-        rs = self.stmt.executeQuery(sql)
+        rs = self.executeQuery(sql)
         rs.first()
-        nBadFlux = rs.getInt(1)
-        self.logger.info("%d NULL %s values in table %s. Deleting." % (nBadFlux, columnName, tableName))
+        nBad = rs.getInt(1)
+        self.logger.info("%5d NULL %s values in table %s. Deleting." % (nBad, columnName, tableName))
 
         sql="DELETE from " + tableName + " WHERE " + columnName + " = " + value
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -145,11 +198,11 @@
 
         sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + columnName + " IS NULL"
-        rs = self.stmt.executeQuery(sql)
+        rs = self.executeQuery(sql)
         rs.first()
-        nBadFlux = rs.getInt(1)
-        self.logger.info("%d NULL %s values in table %s. Deleting." % (nBadFlux, columnName, tableName))
+        nBad = rs.getInt(1)
+        self.logger.info("%5d NULL %s values in table %s. Deleting." % (nBad, columnName, tableName))
 
         sql="DELETE from " + tableName + " WHERE " + columnName + " IS NULL"
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -160,5 +213,5 @@
        # first, count rows
        sql = "SELECT COUNT(*) FROM " + tableName
-       rs = self.stmt.executeQuery(sql)
+       rs = self.executeQuery(sql)
        rs.first()
        numRows = rs.getInt(1)
@@ -175,5 +228,5 @@
           
           sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + column + " IS NULL"
-          rs = self.stmt.executeQuery(sql)
+          rs = self.executeQuery(sql)
           rs.first()
           if rs.getInt(1) == numRows:
@@ -191,5 +244,5 @@
         sql = "SELECT COUNT(*) FROM " + table
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             return rs.getInt(1)
Index: /branches/eam_branches/ipp-20110505/ippToPsps/jython/scratchdb.py
===================================================================
--- /branches/eam_branches/ipp-20110505/ippToPsps/jython/scratchdb.py	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippToPsps/jython/scratchdb.py	(revision 31587)
@@ -45,5 +45,5 @@
         sql = "SELECT surveyID FROM Survey WHERE name = '" + name + "'"
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             return rs.getInt(1)
@@ -61,5 +61,5 @@
         sql = "SELECT flags FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             flags = rs.getInt(1)
@@ -78,5 +78,5 @@
         sql = "SELECT imageID FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             imageID = rs.getInt(1)
@@ -95,5 +95,5 @@
         sql = "SELECT photcode FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             photcode = rs.getInt(1)
@@ -109,5 +109,5 @@
 
         sql = "UPDATE "+table+" AS a, Filter AS b SET a.filterID=b.filterID WHERE b.filterType = '" + filter + "'"
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -123,5 +123,5 @@
                " + str(imageID) + "    \
                )"
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -131,5 +131,5 @@
 
         sql = "INSERT INTO dvoDone (name) VALUES ('" + name + "')"
-        self.stmt.execute(sql)
+        self.execute(sql)
         
     '''
@@ -141,5 +141,5 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             if rs.getInt(1) > 0:
@@ -160,9 +160,9 @@
 
         sql = "DROP TABLE dvoMeta"
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: pass
         
         sql = "DROP TABLE dvoDetection"
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: pass
 
@@ -175,5 +175,5 @@
                )"
 
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: 
             self.logger.error("Unable to create DVO meta-data database table")
@@ -193,5 +193,5 @@
                #INDEX (ippDetectID) \
 
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: 
             self.logger.error("Unable to create DVO detection database table")
Index: /branches/eam_branches/ipp-20110505/ippToPsps/jython/stackbatch.py
===================================================================
--- /branches/eam_branches/ipp-20110505/ippToPsps/jython/stackbatch.py	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippToPsps/jython/stackbatch.py	(revision 31587)
@@ -8,6 +8,9 @@
 from java.sql import *
 
+from batch import Batch
 from gpc1db import Gpc1Db
-from batch import Batch
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
+
 import logging.config
 
@@ -20,7 +23,20 @@
     Constructor
     '''
-    def __init__(self, logger, skyID, inputFile, stackType, useFullTables=False):
+    def __init__(self, 
+                 logger, 
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 skyID, 
+                 inputFile, 
+                 stackType, 
+                 useFullTables=False):
+
        super(StackBatch, self).__init__(
                logger,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               skyID,
                "stack", 
                inputFile, 
@@ -28,24 +44,18 @@
                useFullTables) # TODO
 
-       self.logger.info("StackBatch constructor. Creating batch from: '" + inputFile + "'")
-
-       self.skyID = skyID
-
-       # get filterID using init table
-       self.filter = self.header['FPA.FILTER']
-       self.filter = self.filter[0:1]
-
-       self.stackType = stackType
-       meta = self.gpc1Db.getStackStageMeta(self.skyID, self.header['FPA.FILTER'])
-       if len(meta) < 1: return
-       self.stackID = meta[0];
-       self.skycell = meta[1];
-
-       # determine skycell from header value
-       #self.skycell = "skycell.34" #= self.header['SKYCELL']
-       self.skycell = self.skycell[8:]
-
-       self.logger.info("Processing stack with ID: %d, type: %s and skycell: %s filter: %s" % (self.stackID, self.stackType, self.skycell, self.filter))
-
+       if not self.everythingOK: return
+
+       self.expTime = gpc1Db.getStackExpTime(self.stackID)
+
+       self.logger.info("got exp time of %d" % self.expTime)
+
+       # meta data to the log
+       self.logger.info("New Stack Batch:")
+       self.logger.info("Sky ID:     %d" % self.id)
+       self.logger.info("File:       %s" % inputFile)
+       self.logger.info("Stack ID:   %d" % self.stackID)
+       self.logger.info("Stack type: %s" % self.stackType)
+       self.logger.info("Skycell:    %s" % self.skycell)
+       self.logger.info("Filter:     %s" % self.filter)
 
        # delete PSPS tables
@@ -58,16 +68,6 @@
        self.scratchDb.dropTable("ObjectCalColor")
 
-       # delete IPP tables
-       #self.scratchDb.dropTable("SkyChip_psf")
-       #self.scratchDb.dropTable("SkyChip_xsrc")
-       #self.scratchDb.dropTable("SkyChip_xfit")
-       #self.scratchDb.dropTable("SkyChip_xrad")
-
-       self.logger.info("Stack type: " + self.safeDictionaryAccess(self.header, self.stackType))
-       # obs time makes no sense except for nightly stacks
-       #if self.header['STK_TYPE'] != "NIGHTLY_STACK": self.header['MJD-OBS'] = "-999"
-
        # create an output filename, which is {filterID}{skycellID}.FITS
-       self.outputFitsFile = "%s%07d.FITS" % (self.filter, int(self.skycell))
+       self.outputFitsFile = "%08d.FITS" % self.stackID
        self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
 
@@ -77,5 +77,5 @@
 
        # insert what we know about this stack batch into the stack table
-       self.ippToPspsDb.insertStackMeta(self.batchID, self.skyID, self.stackID, self.filter, self.stackType)
+       self.ippToPspsDb.insertStackMeta(self.batchID, self.id, self.stackID, self.filter, self.stackType)
 
        # insert sourceID/imageID combo so DVO can look it up
@@ -89,5 +89,5 @@
 
         sql = "UPDATE " + table + "  SET stackMetaID=" + str(self.stackID)
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -97,5 +97,5 @@
 
         sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '" + self.stackType + "'"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
 
@@ -148,5 +148,5 @@
         WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -196,5 +196,5 @@
         WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'"
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         # sersic fit has an extra parameter
@@ -213,5 +213,5 @@
             WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'"
 
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
 
 
@@ -220,4 +220,5 @@
     '''
     def populateStackMeta(self):
+
         self.logger.info("Procesing StackMeta table")
 
@@ -246,5 +247,5 @@
         ," + str(self.scratchDb.getPhotoCalID(self.header['SOURCEID'], self.header['IMAGEID'])) + " \
         ," + self.header['FPA.ZP'] + " \
-        ," + self.header['EXPTIME'] + " \
+        ," + str(self.expTime) + " \
         ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \
         ,'" + self.header['CTYPE1'] + "' \
@@ -261,5 +262,5 @@
         ," + self.header['PC002002'] + " \
         )"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID))
@@ -272,4 +273,5 @@
     '''
     def populateStackDetection(self):
+
         self.logger.info("Procesing StackDetection table")
 
@@ -321,7 +323,7 @@
                ,X_PSF_SIG \
                ,Y_PSF_SIG \
-               ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+" \
-               ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+")) / 1.085736) \
-               ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \
+               ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+str(self.expTime)+" \
+               ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+str(self.expTime)+")) / 1.085736) \
+               ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+str(self.expTime)+" \
                ,SKY \
                ,SKY_SIGMA \
@@ -352,5 +354,5 @@
                FROM SkyChip_psf"
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID))
@@ -362,6 +364,17 @@
         self.updateStackTypeID("StackDetection")
         self.updateDvoIDs("StackDetection")
-
-        # now delete bad flux
+        sql = "ALTER TABLE StackDetection ADD PRIMARY KEY (objID, stackDetectID)"
+        self.scratchDb.execute(sql)
+
+        if self.stackType == "DEEP_STACK": 
+
+            #if deep stack and instFlux = null and err not null
+            sql = "UPDATE StackDetection AS a, SkyChip_psf AS b \
+                   SET instFlux = 2*b.PSF_INST_FLUX_SIG \
+                   WHERE instFlux IS NULL \
+                   AND b.PSF_INST_FLUX_SIG IS NOT NULL"
+            self.scratchDb.execute(sql)
+            #    instFlux = 2*PSF_INST_FLUX_SIG
+            
         self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux")
         self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID")
@@ -372,4 +385,5 @@
     '''
     def populateStackApFlx(self):
+
         self.logger.info("Procesing StackApFlx table")
  
@@ -381,16 +395,16 @@
 
         try:
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
         except: return
 
         # TODO temporarily loading 1st convolved fluxes into unconvolved fields
-        self.logger.info("    Adding un-convolved fluxes")
+        self.logger.info("Adding un-convolved fluxes")
         self.updateApFlxs("", "< 7.0")
-        self.logger.info("    Adding 1st convolved fluxes")
+        self.logger.info("Adding 1st convolved fluxes")
         self.updateApFlxs("c1", "< 7.0")
-        self.logger.info("    Adding 2nd convolved fluxes")
+        self.logger.info("Adding 2nd convolved fluxes")
         self.updateApFlxs("c2", "> 7.0")
 
-        self.logger.info("    Adding petrosians for extended sources")
+        self.logger.info("Adding petrosians for extended sources")
         sql = "UPDATE StackApFlx AS a, SkyChip_xsrc AS b SET \
         petRadius=b.PETRO_RADIUS \
@@ -403,5 +417,5 @@
         ,petR90Err=b.PETRO_RADIUS_90_ERR \
         WHERE a.ippDetectID=b.IPP_IDET"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID))
@@ -413,4 +427,7 @@
         self.updateStackTypeID("StackApFlx")
         self.updateDvoIDs("StackApFlx")
+        self.scratchDb.reportAndDeleteRowsWithNULLS("StackApFlx", "objID")
+        self.deleteDetectionsNotInStackDetection("StackApFlx")
+
 
     '''
@@ -418,4 +435,5 @@
     '''
     def populateStackModelFit(self):
+
         self.logger.info("Procesing StackModelFit table")
 
@@ -423,15 +441,14 @@
         sql = "INSERT INTO StackModelFit (ippDetectID) SELECT DISTINCT IPP_IDET from SkyChip_xfit"
         try:
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
         except:
             return
 
-
         # populate model parameters
-        self.logger.info("    Adding deVaucouleurs fit")
+        self.logger.info("Adding deVaucouleurs fit")
         self.updateModelFit("deV", "PS_MODEL_DEV")
-        self.logger.info("    Adding exponential fit")
+        self.logger.info("Adding exponential fit")
         self.updateModelFit("exp", "PS_MODEL_EXP")
-        self.logger.info("    Adding sersic fit")
+        self.logger.info("Adding sersic fit")
         self.updateModelFit("ser", "PS_MODEL_SERSIC")
 
@@ -444,4 +461,23 @@
         self.updateStackTypeID("StackModelFit")
         self.updateDvoIDs("StackModelFit")
+        self.scratchDb.reportAndDeleteRowsWithNULLS("StackModelFit", "objID")
+        self.deleteDetectionsNotInStackDetection("StackModelFit")
+
+    '''
+    Reports and deletes detections in this table that are not in StackDetection
+    '''
+    def deleteDetectionsNotInStackDetection(self, table):
+
+        sql = "SELECT COUNT(*) FROM " + table + " WHERE ippDetectID NOT IN (SELECT ippDetectID FROM StackDetection)"
+        rs = self.scratchDb.executeQuery(sql)
+        rs.first()
+        nMissing = rs.getInt(1)
+        self.logger.info("%5d detections in %s table that are not in StackDetection. Deleting" % (nMissing, table))
+  
+        if nMissing < 1: return
+        
+        sql = "DELETE FROM " + table + " WHERE ippDetectID NOT IN (SELECT ippDetectID FROM StackDetection)"
+        self.scratchDb.execute(sql)
+       
 
     '''
@@ -449,4 +485,5 @@
     '''
     def populateStackToImage(self):
+
         self.logger.info("Procesing StackToImage table")
 
@@ -457,9 +494,9 @@
                    VALUES (\
                    " + str(self.stackID) + ", " + imageID + ")"
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
 
         # now update StackMeta with correct number of inputs
         sql = "UPDATE StackMeta SET nP2Images = (SELECT COUNT(*) FROM StackToImage)"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -467,4 +504,5 @@
     '''
     def populateSkinnyObject(self):
+
         self.logger.info("Procesing SkinnyObject table")
 
@@ -474,8 +512,8 @@
                ) \
                SELECT \
-               objID \
+               DISTINCT objID \
                ,ippObjID \
                FROM StackDetection"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID))
@@ -486,4 +524,5 @@
     '''
     def populateObjectCalColor(self):
+
         self.logger.info("Procesing ObjectCalColor table")
 
@@ -493,8 +532,8 @@
                ) \
                SELECT \
-               objID \
+               DISTINCT objID \
                ,ippObjID \
                FROM StackDetection"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateFilterID("ObjectCalColor", self.filter)
@@ -508,5 +547,5 @@
 
         self.logger.info("Altering PSPS tables")
-        self.scratchDb.makeColumnUnique("StackDetection", "objID")
+        #self.scratchDb.makeColumnUnique("StackDetection", "objID")
         self.scratchDb.createIndex("StackDetection", "ippDetectID")
         self.scratchDb.createIndex("StackApFlx", "ippDetectID")
@@ -531,5 +570,5 @@
         imageID = self.scratchDb.getImageIDFromExternID(self.header['SOURCEID'], self.header['IMAGEID'])
 
-        self.logger.info("Updating table '" + table + "' with DVO IDs...")
+        self.logger.debug("Updating table '" + table + "' with DVO IDs...")
         sql = "UPDATE IGNORE " + table + " AS a, dvoDetectionFull AS b SET \
                a.ippObjID = b.ippObjID, \
@@ -539,6 +578,5 @@
                AND b.sourceID = " + self.header['SOURCEID'] + "\
                AND b.imageID = " + str(imageID)
-        self.scratchDb.stmt.execute(sql)
-
+        self.scratchDb.execute(sql)
 
     '''
@@ -553,6 +591,9 @@
         self.populateStackMeta()
         self.populateStackDetection()
-        self.populateStackModelFit()
-        self.populateStackApFlx()
+
+        if self.stackType != "NIGHTLY_STACK": 
+            self.populateStackModelFit()
+            self.populateStackApFlx()
+   
         self.populateStackToImage()
         self.populateSkinnyObject()
@@ -560,5 +601,10 @@
 
         self.setMinMaxObjID(["StackDetection"])
-        
+       
+        if self.totalDetections < 1: 
+
+            self.logger.error("No detections to publish")
+            return False
+
         return True
 
@@ -568,22 +614,49 @@
     def alreadyProcessed(self):
 
-        return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID)
+        # sadly, we have to read the FITS primary header first
+        if not self.readPrimaryHeader(): return False
+
+        # get filterID using init table
+        self.filter = self.header['FPA.FILTER']
+        self.filter = self.filter[0:1]
+
+        self.stackType = stackType
+        meta = self.gpc1Db.getStackStageMeta(self.id, self.header['FPA.FILTER'])
+        if len(meta) < 1: return False
+        self.stackID = meta[0];
+        self.skycell = meta[1];
+        self.skycell = self.skycell[8:]
+
+        #return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID)
+        return False # TODOI
+
+
+useFullTables=True
 
 logging.config.fileConfig("logging.conf")
 logger = logging.getLogger("stackbatch")
+logger.setLevel(logging.INFO)
 logger.info("Starting")
+
 gpc1Db = Gpc1Db(logger)
-stackType = "NIGHTLY_STACK"
-skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky")
-#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky")
-#stackType = "DEEP_STACK"
-#skyIDs = [689]
+ippToPspsDb = IppToPspsDb(logger)
+scratchDb = ScratchDb(logger, useFullTables)
+
+#stackType = "NIGHTLY_STACK"
+#skyIDs = gpc1Db.getIDsInThisDVODbForThisStageFudge()
+#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky")
+
+stackType = "DEEP_STACK"
+skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky")
+
+#skyIDs = [942]
 #skyIDs = [299]
 #skyIDs = [302]
 #skyIDs = [8508]
-i = 0
+#i = 0
 for skyID in skyIDs:
-
-    logger.info("-------------------------------------------------- sky ID: %d" % skyID)
+    
+    #if skyID < 1340: continue # nightly
+    #if skyID < 238: continue # deep
 
     cmfFiles = gpc1Db.getStackStageCmfs(skyID)
@@ -591,24 +664,13 @@
     for file in cmfFiles:
 
-        if not os.path.isfile(file):
-            logger.error("Cannot read file at '" + file)
-            continue
-
-        stackBatch = StackBatch(logger, skyID, file, stackType, True)
-
-        if not stackBatch.alreadyProcessed():
-
-            stackBatch.createEmptyPspsTables()
-            stackBatch.importIppTables("")
-            if stackBatch.populatePspsTables():
- 
-                #stackBatch.reportNullsInAllPspsTables(False)
-                stackBatch.exportPspsTablesToFits()
-                stackBatch.writeBatchManifest()
-                #stackBatch.createTarball()
-                #stackBatch.publishToDatastore()
-
-                i = i + 1
-                #if i > 0: sys.exit()
-
-logger.info("Finished")
+        stackBatch = StackBatch(logger,
+                                gpc1Db,
+                                ippToPspsDb,
+                                scratchDb,
+                                skyID, 
+                                file, 
+                                stackType, 
+                                useFullTables)
+
+        stackBatch.run()
+
Index: /branches/eam_branches/ipp-20110505/ippTools/share/laptool_definerun.sql
===================================================================
--- /branches/eam_branches/ipp-20110505/ippTools/share/laptool_definerun.sql	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippTools/share/laptool_definerun.sql	(revision 31587)
@@ -1,3 +1,3 @@
-SELECT want.exp_id, have.chip_id, false as private, true as active, false as pairwise
+SELECT DISTINCT want.exp_id, have.chip_id, false as private, true as active, false as pairwise
   FROM
   (SELECT exp_id FROM rawExp 
@@ -8,5 +8,5 @@
   ) AS want
   LEFT JOIN 
-  (SELECT *
+  (SELECT exp_id,MAX(chip_id) AS chip_id
      FROM lapExp 
      where private IS FALSE 
Index: /branches/eam_branches/ipp-20110505/ippTools/share/laptool_exposures.sql
===================================================================
--- /branches/eam_branches/ipp-20110505/ippTools/share/laptool_exposures.sql	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippTools/share/laptool_exposures.sql	(revision 31587)
@@ -1,43 +1,41 @@
-SELECT DISTINCT 
-    D.*,diffRun.state,
-    coalesce(CONVERT(sum(others.private),SIGNED),0) AS needs_remade 
---      0 AS needs_remade
-    FROM (
-  SELECT DISTINCT 
-      W.*,CONVERT(IFNULL(diff1.diff_id,diff2.diff_id),SIGNED) AS diff_id FROM (
-    SELECT DISTINCT
-        lap_id,lapRun.tess_id,projection_cell,filter,lapRun.state as lapRun_state, lapRun.registered, lapRun.fault, lapRun.label, lapRun.dist_group,
-        lapExp.exp_id,lapExp.chip_id,lapExp.pair_id,private,pairwise,active,lapExp.data_state,
-        chipRun.state as chipRun_state, 
-	coalesce(CONVERT(sum(chipProcessedImfile.fault),SIGNED),0) as chip_faults, 
-	coalesce(CONVERT(sum(chipProcessedImfile.quality),SIGNED),0) as chip_quality,
-        camRun.cam_id, camRun.state as camRun_state,   
-	coalesce(CONVERT(sum(camProcessedExp.fault),SIGNED),0) AS cam_faults, 
-	coalesce(CONVERT(sum(camProcessedExp.quality),SIGNED),0) AS cam_quality,
-	fakeRun.fake_id, fakeRun.state as fakeRun_state, 
-	coalesce(CONVERT(sum(fakeProcessedImfile.fault),SIGNED),0) as fake_faults,
-  	warpRun.warp_id, warpRun.state as warpRun_state, 
-	coalesce(CONVERT(sum(warpSkyfile.fault),SIGNED),0) as warp_faults, 
-	coalesce(CONVERT(sum(warpSkyfile.quality),SIGNED),0) as warp_quality,
-        warpRun.magicked
-    FROM lapRun JOIN lapExp USING(lap_id)
-    LEFT JOIN chipRun USING(chip_id)
-    LEFT JOIN chipProcessedImfile USING(chip_id)
-    LEFT JOIN camRun USING(chip_id) LEFT JOIN camProcessedExp USING(cam_id)
-    LEFT JOIN fakeRun USING(cam_id) LEFT JOIN fakeProcessedImfile USING(fake_id)
-    LEFT JOIN warpRun USING(fake_id) LEFT JOIN warpSkyfile USING(warp_id)
-    WHERE    @WHERE@
-    AND (warpSkyfile.quality IS NULL OR
-         (warpSkyfile.quality != 8007      -- known cases where quality != 0, but everything's fine.
-          AND warpSkyfile.quality != 3006  -- known cases where quality != 0, but everything's fine.
-	  ))
-    GROUP BY lap_id,exp_id
-    ) AS W
--- This was unreasonably slow in testing, so that's why I'm using a subquery here.
-  LEFT JOIN diffInputSkyfile AS diff1 ON (W.warp_id = diff1.warp1)
-  LEFT JOIN diffInputSkyfile AS diff2 ON (W.warp_id = diff2.warp2)
-) AS D
+select DISTINCT V3.*,
+       diffRun.diff_id,diffRun.state as diffRun_state,
+       coalesce(CONVERT(sum(diffSkyfile.quality != 0),SIGNED),0) AS diff_bad_quality,
+       coalesce(CONVERT(count(diffSkyfile.diff_id),SIGNED),0) AS diff_component_count
+       FROM
+( select V2.*,
+       warpRun.warp_id,warpRun.state as warpRun_state,
+       coalesce(CONVERT(sum(warpSkyfile.quality != 0),SIGNED),0) AS warp_bad_quality,
+       coalesce(CONVERT(count(warpSkyfile.warp_id),SIGNED),0) AS warp_component_count,
+       warpRun.magicked
+       FROM
+( select V1.*,
+       camRun.cam_id,camRun.state as camRun_state,
+       coalesce(CONVERT(sum(camProcessedExp.quality != 0),SIGNED),0) AS cam_bad_quality,
+       coalesce(CONVERT(count(camProcessedExp.cam_id),SIGNED),0) AS cam_component_count,
+       fakeRun.fake_id,fakeRun.state as fakeRun_state FROM
+( SELECT DISTINCT
+       lap_id,lapRun.tess_id,projection_cell,filter,lapRun.state as lapRun_state, lapRun.registered, lapRun.fault, lapRun.label, lapRun.dist_group,
+       lapExp.exp_id,lapExp.chip_id,lapExp.pair_id,private,pairwise,active,lapExp.data_state,
+       chipRun.state as chipRun_state,
+       coalesce(CONVERT(sum(chipProcessedImfile.quality != 0),SIGNED),0) AS chip_bad_quality,
+       coalesce(CONVERT(count(chipProcessedImfile.chip_id),SIGNED),0) AS chip_component_count
+       FROM lapRun JOIN lapExp USING(lap_id)
+       LEFT JOIN chipRun USING(chip_id) LEFT JOIN chipProcessedImfile USING(chip_id)
+WHERE @WHERE@ 
+       GROUP BY lap_id,exp_id
+       ) AS V1
+       LEFT JOIN camRun USING(chip_id) LEFT JOIN camProcessedExp USING(cam_id)
+       LEFT JOIN fakeRun USING(cam_id)
+       GROUP BY lap_id,exp_id
+  ) AS V2
+  LEFT JOIN warpRun USING(fake_id) LEFT JOIN warpSkyfile USING(warp_id)
+  GROUP BY lap_id,exp_id
+) AS V3
+LEFT JOIN
+  (SELECT DISTINCT diff_id,warp1,warp2 FROM diffInputSkyfile) AS DI ON
+  (DI.warp1 = warp_id OR DI.warp2 = warp_id)
 LEFT JOIN diffRun USING(diff_id)
-LEFT JOIN lapExp AS others ON (D.chip_id = others.chip_id AND D.lap_id != others.lap_id)
+LEFT JOIN diffSkyfile USING(diff_id)
 GROUP BY lap_id,exp_id
 
Index: /branches/eam_branches/ipp-20110505/ippTools/src/disttool.c
===================================================================
--- /branches/eam_branches/ipp-20110505/ippTools/src/disttool.c	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippTools/src/disttool.c	(revision 31587)
@@ -447,5 +447,5 @@
 
     if (pretend) {
-        if (!ippdbPrintMetadatas(stdout, output, "newdistRuns", true)) {
+        if (!ippdbPrintMetadatas(stdout, output, "newdistRuns", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
Index: /branches/eam_branches/ipp-20110505/ippTools/src/laptool.c
===================================================================
--- /branches/eam_branches/ipp-20110505/ippTools/src/laptool.c	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippTools/src/laptool.c	(revision 31587)
@@ -384,8 +384,10 @@
   PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
   PXOPT_COPY_STR(config->args, where, "-filter", "filter", "==");
-  PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+  //  PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
   PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
   PXOPT_COPY_STR(config->args, where, "-fault", "fault", "==");
 
+  pxAddLabelSearchArgs(config, where, "-label", "lapRun.label", "==");
+  
   psString query = pxDataGet("laptool_pendingrun.sql");
   if (!query) {
@@ -560,5 +562,5 @@
   psMetadata *where = psMetadataAlloc();
   PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "==");
-  PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-exp_id", "lapExp.exp_id", "==");
   
   psString query = pxDataGet("laptool_exposures.sql");
Index: /branches/eam_branches/ipp-20110505/ippTools/src/laptoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20110505/ippTools/src/laptoolConfig.c	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippTools/src/laptoolConfig.c	(revision 31587)
@@ -69,8 +69,21 @@
   ADD_OPT(Str, pendingrunArgs, "-projection_cell",            "search by projection cell", NULL);
   ADD_OPT(Str, pendingrunArgs, "-filter",                     "search by filter", NULL);
-  ADD_OPT(Str, pendingrunArgs, "-label",                      "search by LAP run label", NULL);
+  //  ADD_OPT(Str, pendingrunArgs, "-label",                      "search by LAP run label", NULL);
+  psMetadataAddStr(pendingrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by LAP run label", NULL);
   ADD_OPT(Str, pendingrunArgs, "-state",                      "search by LAP run state", NULL);
   ADD_OPT(Str, pendingrunArgs, "-fault",                      "search by LAP run fault", NULL);
   ADD_OPT(Bool,pendingrunArgs, "-simple",                     "use the simple output format", false);
+
+  // -listrun
+  psMetadata *listrunArgs = psMetadataAlloc();
+  ADD_OPT(S64, listrunArgs, "-seq_id",                     "search by LAP sequence ID", 0);
+  ADD_OPT(S64, listrunArgs, "-lap_id",                     "search by LAP run ID", 0);
+  ADD_OPT(Str, listrunArgs, "-projection_cell",            "search by projection cell", NULL);
+  ADD_OPT(Str, listrunArgs, "-filter",                     "search by filter", NULL);
+  //  ADD_OPT(Str, listrunArgs, "-label",                      "search by LAP run label", NULL);
+  psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by LAP run label", NULL);
+  ADD_OPT(Str, listrunArgs, "-state",                      "search by LAP run state", NULL);
+  ADD_OPT(Str, listrunArgs, "-fault",                      "search by LAP run fault", NULL);
+  ADD_OPT(Bool,listrunArgs, "-simple",                     "use the simple output format", false);
 
   // -updaterun
@@ -128,9 +141,10 @@
   psMetadata *argSets = psMetadataAlloc();
   psMetadata *modes = psMetadataAlloc();
-
+  
   PXOPT_ADD_MODE("-definesequence",          "", LAPTOOL_MODE_DEFINESEQUENCE,   definesequenceArgs);
   PXOPT_ADD_MODE("-listsequence",            "", LAPTOOL_MODE_LISTSEQUENCE,     listsequenceArgs);
   PXOPT_ADD_MODE("-definerun",               "", LAPTOOL_MODE_DEFINERUN,        definerunArgs);
   PXOPT_ADD_MODE("-pendingrun",              "", LAPTOOL_MODE_PENDINGRUN,       pendingrunArgs);
+  PXOPT_ADD_MODE("-listrun",                 "", LAPTOOL_MODE_PENDINGRUN,       listrunArgs);
   PXOPT_ADD_MODE("-updaterun",               "", LAPTOOL_MODE_UPDATERUN,        updaterunArgs);
   PXOPT_ADD_MODE("-pendingexp",              "", LAPTOOL_MODE_PENDINGEXP,       pendingexpArgs);
Index: /branches/eam_branches/ipp-20110505/ippTools/src/pstamptool.c
===================================================================
--- /branches/eam_branches/ipp-20110505/ippTools/src/pstamptool.c	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippTools/src/pstamptool.c	(revision 31587)
@@ -843,5 +843,5 @@
     psFree(where);
 
-    psStringAppend(&query, " ORDER BY priority DESC, req_id");
+    psStringAppend(&query, " ORDER BY priority DESC, req_id, job_id");
 
     // treat limit == 0 as "no limit"
@@ -1323,5 +1323,5 @@
     psFree(where);
 
-    psStringAppend(&query, " ORDER BY priority DESC, req_id");
+    psStringAppend(&query, " ORDER BY priority DESC, req_id, dep_id");
 
     // treat limit == 0 as "no limit"
Index: /branches/eam_branches/ipp-20110505/ippTools/src/pubtoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20110505/ippTools/src/pubtoolConfig.c	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippTools/src/pubtoolConfig.c	(revision 31587)
@@ -48,5 +48,7 @@
     psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-product", 0, "define product (required)", NULL);
     psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
+    psMetadataAddS16(defineclientArgs, PS_LIST_TAIL, "-output_format", 0, "define output format", 2);
     psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL);
+    psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-name", 0, "define name", NULL);
     psMetadataAddBool(defineclientArgs, PS_LIST_TAIL, "-unmagicked", 0, "allow unmagicked data?", false);
 
Index: /branches/eam_branches/ipp-20110505/ippTools/src/stacktool.c
===================================================================
--- /branches/eam_branches/ipp-20110505/ippTools/src/stacktool.c	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippTools/src/stacktool.c	(revision 31587)
@@ -246,5 +246,6 @@
     PXOPT_COPY_STR(config->args,  where, "-select_data_group",         "warpRun.data_group", "==");
     pxAddLabelSearchArgs (config, where, "-select_label",              "warpRun.label", "LIKE"); // define using warp label
-
+    pxAddLabelSearchArgs (config, where, "-warp_id",                   "warpRun.warp_id", "==");
+    
     // these are used to build the HAVING restriction
     PXOPT_COPY_S32(config->args, having, "-min_num", "num_warp", ">=");
@@ -508,4 +509,7 @@
           association->sass_id = sass_id;
         }
+	else {
+	  sass_id = association->sass_id;
+	}
         // Insert the map entry for this row.
         stackAssociationMapRow *maprow = stackAssociationMapRowAlloc(sass_id,stack_id);
Index: /branches/eam_branches/ipp-20110505/ippTools/src/stacktoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20110505/ippTools/src/stacktoolConfig.c	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ippTools/src/stacktoolConfig.c	(revision 31587)
@@ -91,4 +91,5 @@
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_zpt_obs_max", 0, "define max zero point", NAN);
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_astrom", 0, "define max astrometry rms", NAN);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", PS_META_DUPLICATE_OK, "include this warp ID (multiple OK)", 0);
     psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0);
     psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_num", 0, "minimum number of inputs", 0);
Index: /branches/eam_branches/ipp-20110505/ppStack/src/ppStackPrepare.c
===================================================================
--- /branches/eam_branches/ipp-20110505/ppStack/src/ppStackPrepare.c	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/ppStack/src/ppStackPrepare.c	(revision 31587)
@@ -144,4 +144,5 @@
     int numCols = 0, numRows = 0;   // Size of image
     options->sumExposure = 0.0;
+    int numWithSources = 0;
     for (int i = 0; i < num; i++) {
         pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); // File of interest
@@ -188,5 +189,5 @@
             pmReadout *ro = pmFPAviewThisReadout(view, file->fpa); // Readout with sources
             detections = psMetadataLookupPtr(NULL, ro->analysis, "PSPHOT.DETECTIONS"); // Sources
-            if (!detections || !detections->allSources) {
+            if (!detections || !detections->allSources || !detections->allSources->n) {
                 psWarning("No detections found for image %d --- rejecting.", i);
                 options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_CAL;
@@ -196,4 +197,5 @@
 
             options->sourceLists->data[i] = psMemIncrRefCounter(detections->allSources);
+            numWithSources++;
         }
 
@@ -230,4 +232,15 @@
             ppStackFileActivation(config, PPSTACK_FILES_PREPARE, true);
         }
+    }
+    if (numWithSources < 2) {
+        // This can happen if the inputs have been destreaked
+        psErrorStackPrint(stderr, "Not enough inputs have sources");
+        psWarning("No inputs have sources --- suspect bad data quality.");
+        if (options->quality == 0) {
+            options->quality = PPSTACK_ERR_DATA;
+        }
+        psErrorClear();
+        psFree(view);
+        return false;
     }
 
@@ -308,5 +321,21 @@
         psFree(psfs);
         if (!options->psf) {
+#if 1
             psError(psErrorCodeLast(), false, "Unable to determine output PSF.");
+#else
+            // This will repair the problem reproted in ticket 1427 but we aren't yet sure
+            // why ppStackPSF is failing so we are going to continue to fault for now
+            int errorCode = psErrorCodeLast();
+            if (errorCode == PPSTACK_ERR_PSF) {
+                psErrorStackPrint(stderr, "Unable to determine output PSF.");
+                psWarning("Unable to determine output PSF --- suspect bad data quality.");
+                if (options->quality == 0) {
+                    options->quality = errorCode;
+                }
+                psErrorClear();
+            } else {
+                psError(psErrorCodeLast(), false, "Unable to determine output PSF.");
+            }
+#endif // notyet
             psFree(view);
             return false;
Index: /branches/eam_branches/ipp-20110505/psModules/src/imcombine/pmSubtractionStamps.c
===================================================================
--- /branches/eam_branches/ipp-20110505/psModules/src/imcombine/pmSubtractionStamps.c	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/psModules/src/imcombine/pmSubtractionStamps.c	(revision 31587)
@@ -834,15 +834,21 @@
             }
 
-            psStatsInit (stats);
-            if (!psVectorStats (stats, flux1, NULL, NULL, 0)) {
-                psAbort ("failed to generate stats");
-            }
-            float f1 = stats->sampleMedian;
-
-            psStatsInit (stats);
-            if (!psVectorStats (stats, flux2, NULL, NULL, 0)) {
-                psAbort ("failed to generate stats");
-            }
-            float f2 = stats->sampleMedian;
+            float f1 = NAN;
+            if (flux1->n > 0) {
+                psStatsInit (stats);
+                if (!psVectorStats (stats, flux1, NULL, NULL, 0)) {
+                    psAbort ("failed to generate stats");
+                }
+                f1 = stats->sampleMedian;
+            }
+
+            float f2 = NAN;
+            if (flux2->n > 0) {
+                psStatsInit (stats);
+                if (!psVectorStats (stats, flux2, NULL, NULL, 0)) {
+                    psAbort ("failed to generate stats");
+                }
+                f2 = stats->sampleMedian;
+            }
 
             stamps->window1->kernel[y][x] = f1;
@@ -893,4 +899,14 @@
     float R2 = Sr2 / Sf2;
 
+    if (!isfinite(R1) || !isfinite(R2)) {
+        psError(PM_ERR_STAMPS, true, "Kron Radii are not finite (failure to converge).");
+	psFree (stats);
+	psFree (flux1);
+	psFree (flux2);
+	psFree (norm1);
+	psFree (norm2);
+        return false;
+    }
+
     // Compare the Kron Radii (R1 & R2) to above to the FWHMs : if they are too discrepant, we will need to rescale
     psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "Kron Radii vs FWHMs 1: fwhm: %f, kron %f\n", fwhm1, R1);
@@ -901,4 +917,5 @@
     stamps->normWindow2 = 2.75*R2;
     psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "Windows from Kron Radii: %f for 1, %f for 2\n", stamps->normWindow1, stamps->normWindow2);
+
 
     // if the calculated normWindows are too large, we will fall off the stamps.  In this case, we need to try again.
@@ -913,14 +930,4 @@
 	psFree (norm2);
 	return false; 
-    }
-
-    if (!isfinite(R1) || !isfinite(R2)) {
-        psError(PM_ERR_STAMPS, true, "Kron Radii are not finite (failure to converge).");
-	psFree (stats);
-	psFree (flux1);
-	psFree (flux2);
-	psFree (norm1);
-	psFree (norm2);
-        return false;
     }
 
Index: /branches/eam_branches/ipp-20110505/psconfig/psbuild
===================================================================
--- /branches/eam_branches/ipp-20110505/psconfig/psbuild	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/psconfig/psbuild	(revision 31587)
@@ -386,4 +386,5 @@
     if (@ARGV != 2) { die "USAGE: psbuild -bootstrap (install_dir)\n"; }
     $psconfdir = $ARGV[1];
+    die "Target directory must be absolute, not relative: $psconfdir\n" unless $psconfdir =~ m|^/|;
 
     # copy psconfig.csh and psconfig.bash to psconfdir
Index: /branches/eam_branches/ipp-20110505/pstamp/scripts/pstamp_job_run.pl
===================================================================
--- /branches/eam_branches/ipp-20110505/pstamp/scripts/pstamp_job_run.pl	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/pstamp/scripts/pstamp_job_run.pl	(revision 31587)
@@ -146,9 +146,9 @@
             # user required uncensored but since stage isn't chip we can't rebuild them
             my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE, 'stop');
-        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} eq 'update') and ($stage ne 'chip')) {
+        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} ne 'full') and ($stage ne 'chip')) {
             # we can only restore pixels for chip stage images if the data has been updated.
-            # XXX: this test is not quite good enough. If all components have been updated then the
-            # state will be 'full' But this will get us going.
-            print "Run state is update: will make stamps from destreaked $stage images.\n";
+            # the data will have been updated if the params->{state} the state when the job was queued is not 'full' (
+            # XXX: we should probably be looking explicitly at the job and checking for a dep_id
+            print "Run state was $params->{state}: will make stamps from destreaked $stage images.\n";
             # make stamps from uncensored images
             $muggle = 0;
Index: /branches/eam_branches/ipp-20110505/pstamp/scripts/pstampparse.pl
===================================================================
--- /branches/eam_branches/ipp-20110505/pstamp/scripts/pstampparse.pl	(revision 31586)
+++ /branches/eam_branches/ipp-20110505/pstamp/scripts/pstampparse.pl	(revision 31587)
@@ -754,6 +754,7 @@
     if ( $num_jobs == 0 ) {
         print STDERR "no jobs for row $rownum\n" if $verbose;
-        insertFakeJobForRow($row, 1, $PSTAMP_NO_JOBS_QUEUED);
+        insertFakeJobForRow($row, 1, $PSTAMP_NO_IMAGE_MATCH);
         $num_jobs = 1;
+	$row->{job_num} = 1;
     }
     return $num_jobs;