Changeset 31587

Timestamp:

May 29, 2011, 11:26:31 AM (15 years ago)

Author:

eugene

Message:

merging changes from trunk

Location:

branches/eam_branches/ipp-20110505

Files:

• ippScripts/scripts/camera_exp.pl (modified) (1 diff)
• ippScripts/scripts/dist_advancerun.pl (modified) (1 diff)
• ippScripts/scripts/dist_bundle.pl (modified) (6 diffs)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/lap_science.pl (modified) (21 diffs)
• ippScripts/scripts/receive_file.pl (modified) (2 diffs)
• ippScripts/scripts/warp_skycell.pl (modified) (4 diffs)
• ippTasks/lap.pro (modified) (21 diffs)
• ippToPsps/jython/batch.py (modified) (17 diffs)
• ippToPsps/jython/detectionbatch.py (modified) (22 diffs)
• ippToPsps/jython/gpc1db.py (modified) (12 diffs)
• ippToPsps/jython/ipptopspsdb.py (modified) (8 diffs)
• ippToPsps/jython/mysql.py (modified) (14 diffs)
• ippToPsps/jython/scratchdb.py (modified) (11 diffs)
• ippToPsps/jython/stackbatch.py (modified) (37 diffs)
• ippTools/share/laptool_definerun.sql (modified) (2 diffs)
• ippTools/share/laptool_exposures.sql (modified) (1 diff)
• ippTools/src/disttool.c (modified) (1 diff)
• ippTools/src/laptool.c (modified) (2 diffs)
• ippTools/src/laptoolConfig.c (modified) (2 diffs)
• ippTools/src/pstamptool.c (modified) (2 diffs)
• ippTools/src/pubtoolConfig.c (modified) (1 diff)
• ippTools/src/stacktool.c (modified) (2 diffs)
• ippTools/src/stacktoolConfig.c (modified) (1 diff)
• ppStack/src/ppStackPrepare.c (modified) (5 diffs)
• psModules/src/imcombine/pmSubtractionStamps.c (modified) (4 diffs)
• psModules/src/objects/models (modified) (1 prop)
• psconfig/psbuild (modified) (1 diff)
• pstamp/scripts/pstamp_job_run.pl (modified) (1 diff)
• pstamp/scripts/pstampparse.pl (modified) (1 diff)

branches/eam_branches/ipp-20110505/ippScripts/scripts/camera_exp.pl

@@ -153 +153 @@
     }
     chomp $cmdflags;
+
+    { # Determine if FWHM is too large to bother continuing.
+        my $command = "$ppConfigDump -camera $camera -dump-camera -";
+        my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_CONFIG_ERROR);
+        }
+        my $cameraConfig = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR);
+
+        my $maxFWHM = metadataLookupStr($cameraConfig, 'MAX_ALLOWED_FWHM');
+        if ($maxFWHM) {
+            my $expFWHM;
+            ($expFWHM) = $cmdflags =~ /-fwhm_major (\d+)/;      
+            
+            if ($expFWHM > $maxFWHM) {
+                print "Setting quality to 4007 due to large FWHM: exposure: $expFWHM  maximum: $maxFWHM\n";
+                $cmdflags .= " -quality 4007 "; # This corresponds to PSASTRO_ERR_DATA
+                $no_op = 1;
+            }
+        }
+    }
 }
 

branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_advancerun.pl

@@ -114 +114 @@
     $tool_cmd = "$staticskytool -sky_id";
     $list_mode = "-result";
-    $component_key = "";
+    $component_key = "skycell_id";
 } elsif ($stage eq "diff") {
     $tool_cmd = "$difftool -diff_id";

branches/eam_branches/ipp-20110505/ippScripts/scripts/dist_bundle.pl

@@ -62 +62 @@
 my $streaksrelease   = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1);
 my $bgtool   = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
+my $staticskytool   = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
 my $file_cmd   = can_run('file') or (warn "can't find program file" and $missing_tools = 1);
 my $zcat   = can_run('zcat') or (warn "can't find program zcat" and $missing_tools = 1);
@@ -106 +107 @@
     defined $stage_id and
     defined $component and
-    defined $exp_type and
     defined $path_base and
     defined $outroot;
 
+if ($stage eq 'raw' and !$clean and !$no_magic) {
+    
+    # for raw stage need to have exposure type defined and if the type is OBJECT we need
+    # a chip_path_base so we can find the chip mask file
+    if (!defined $exp_type or ($exp_type eq 'OBJECT' and !defined $chip_path_base)) {
+        pod2usage( -msg => "Required options: --chip_path_base --exp_type for raw stage", -exitval => 3);
+    }
+}
+
 $ipprc->redirect_output($logfile) if $logfile;
 
-if ($stage eq 'raw' and !$clean and !$no_magic) {
-    # need to be able to find chip mask file
-    if ($exp_type eq 'OBJECT' and !defined $chip_path_base) {
-        pod2usage( -msg => "Required options: --chip_path_base for raw stage", -exitval => 3);
-    }
-}
-
 $ipprc->define_camera($camera);
 
 $ipprc->outroot_prepare($outroot);
 
+my $num_sky_inputs;
+if ($stage eq 'sky') {
+    $num_sky_inputs = get_num_sky_inputs($stage_id);
+}
 # Get the list of data products for this component
 # note: We my_die in get_file_list if something goes wrong.
 
-my $file_list = get_file_list($stage, $component, $path_base, $clean);
+my $file_list = get_file_list($stage, $component, $path_base, $clean, $num_sky_inputs);
 
 if (($stage ne 'raw') and ($stage ne 'fake') and !$poor_quality) {
@@ -184 +190 @@
         # it must exist)
         next if $poor_quality;
+
+        # skip file stats file. Due to a bug the update process destroys them sometimes
+        # XXX: perhaps only do this for stages where we know that this happens
+        next if $file_rule =~ /STATS/;
 
         &my_die("failed to resolve  $file_name", $component, $PS_EXIT_DATA_ERROR);
@@ -480 +490 @@
     my $path_base = shift;
     my $clean = shift;
+    my $num_sky_inputs = shift;
 
     my @file_list;
@@ -531 +542 @@
         $config_file_rule = "PPSTACK.CONFIG";
     } elsif ($stage eq "sky") {
-        $config_file_rule = "PSPHOT.STACK.CONFIG";
+        if ($num_sky_inputs > 1) {
+            $config_file_rule = "PSPHOT.STACK.CONFIG";
+        } else {
+            $config_file_rule = "PSPHOT.SKY.CONFIG";
+        }
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -627 +642 @@
     return \@file_list;
 }
+sub get_num_sky_inputs {
+    my $sky_id = shift;
+
+    my $command = "$staticskytool -inputs -sky_id $sky_id -simple";
+    $command .= " -dbname $dbname" if $dbname;
+    $command .= " | wc";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $component, $error_code);
+    }
+    my ($num_inputs, $words, $chars) = split " ", (join "", @$stdout_buf);
+    if (!$num_inputs) {
+        $num_inputs = "undefined" if !defined $num_inputs;
+        &my_die("unexpected number of static sky inputs $num_inputs",  $PS_EXIT_PROG_ERROR, $component, $error_code);
+    }
+
+    return $num_inputs;
+}
 
 sub my_die

branches/eam_branches/ipp-20110505/ippScripts/scripts/lap_science.pl

@@ -99 +99 @@
     my $status = queue_chips($lap_id);
 
-    if ($status) {
+    if (!$status) { # This is the culprit.
         my $command = "$laptool -updaterun -lap_id $lap_id";
         $command .= " -dbname $dbname " if defined $dbname;
@@ -120 +120 @@
     # This is a puzzler... chiptool doesn't actually return a useful metadata.  We'll just scrape it from the database for now.
 
-    my $command = "$chiptool -listrun -exp_id $exp_id -label $label -data_group $data_group";
+    my $command = "$chiptool -listrun -pstamp_order -exp_id $exp_id -label $label -data_group $data_group";
     $command .= " -dbname $dbname " if defined $dbname;
 
@@ -236 +236 @@
         my $comment = $exposure->{comment};
 
+        # This is a hack to fix old exposures that have no object.
+        unless(defined($comment)) {
+            $comment = '';
+        }
+        if ((!defined($object))||($object eq 'NULL')||($object eq '')) {
+            if ($comment =~ /3pi_/) {
+                $object = $comment;
+                $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/$1/;
+            }
+            elsif ($comment =~ / ps1_/) {
+                $object = $comment;
+                $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/$1/;
+            }
+            else {
+                $exposure->{data_state} = 'new';
+                $exposure->{pairwise} = 0;
+                $exposure->{private} = 1;
+                $exposure->{pair_id} = 9223372036854775807;
+                update_this_exposure($exposure);
+                $counter++;
+                next;
+            }
+        }
+
+
         if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id
             if (($pairwise) && !($pair_id)) {
@@ -255 +280 @@
             $counter++;
         }
+        print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n";
     }
 
@@ -262 +288 @@
         foreach my $comment (keys %{ $matching{$object} }) {
             push @exp_ids_to_diff, $matching{$object}{$comment};
+            print "$object $comment $matching{$object}{$comment} $indexing{$matching{$object}{$comment}} $exp_ids_to_diff[-1]\n";
         }
         @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff;
@@ -275 +302 @@
             my $exp_A = ${ $exposures }[$indexing{$exp_id_A}];
             my $exp_B = ${ $exposures }[$indexing{$exp_id_B}];
-
+            print "$exp_A $exp_B $exp_id_A $exp_id_B $indexing{$exp_id_A} $indexing{$exp_id_B}\n";
             $exp_A->{pairwise} = 1;
             $exp_A->{private} = 0;
@@ -292 +319 @@
     # Scan all exposures, and ensure that pairwise and private are set correctly
     foreach my $exposure (@$exposures) { 
+        print "YY: $exposure\n";
         if ($exposure->{pairwise} && !($exposure->{pair_id})) {
             $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error.
@@ -299 +327 @@
         }
 
-        $exposure = update_this_exposure($exposure);
-    }
-    return(1);
+        update_this_exposure($exposure);
+    }
+    return(0);
 }
 
@@ -432 +460 @@
         my $companion;
 
-        if ($exposure->{pair_id}) { # Load companion exposure information
-            if (exists($match_hash{$exposure->{chip_id}})) {
+        if ($exposure->{pairwise}) {
+            # Load companion exposure information
+            if (($exposure->{pair_id})&&(exists($match_hash{$exposure->{chip_id}}))) {
                 $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; # Match!
+            }
+            else { # We claimed to be pairwise, but do not have a valid pair_id.
+                $exposure->{pairwise} = 0;
+                $exposure->{private} = 1;
+                &update_this_exposure($exposure);
             }
         }
@@ -446 +480 @@
             $exposure = remake_this_exposure($exposure);
         }
-        if ($exposure->{cam_quality}) {
-            $needs_qstack = 1;
-            $needs_something_private = 1;
-            if ($companion) {
-                $companion->{private} = 1;
-                $companion->{pairwise} = 0;
-                &update_this_exposure($companion);
+        # Do quality checks here
+        my $is_bad_quality = 0;
+        if ((defined($exposure->{chipRun_state}))&&($exposure->{chipRun_state} eq 'full')&&
+            ($exposure->{chip_component_count} > 0)&&($exposure->{chip_bad_quality} / $exposure->{chip_component_count} > 0.05)) {
+            printf("QUALITY: $exposure->{exp_id} has bad chip quality: %d / %d\n",
+                   $exposure->{chip_bad_quality} , $exposure->{chip_component_count});
+            $is_bad_quality = 1;
+        }
+        elsif ((defined($exposure->{camRun_state}))&&($exposure->{camRun_state} eq 'full')&&
+               ($exposure->{cam_bad_quality} / $exposure->{cam_component_count} > 0)) {
+            printf("QUALITY: $exposure->{exp_id} has bad cam quality: %d / %d\n",
+                   $exposure->{cam_bad_quality} , $exposure->{cam_component_count});
+            $is_bad_quality = 1;
+        }
+        elsif ((defined($exposure->{warpRun_state}))&&($exposure->{warpRun_state} eq 'full')&&
+               ($exposure->{warp_bad_quality} / $exposure->{warp_component_count} > 0.2)) {
+            printf("QUALITY: $exposure->{exp_id} has bad warp quality: %d / %d\n",
+                   $exposure->{warp_bad_quality} , $exposure->{warp_component_count});
+            $is_bad_quality = 1;
+        }
+        elsif ((defined($exposure->{diffRun_state}))&&($exposure->{diffRun_state} eq 'full')&&
+               ($exposure->{diff_bad_quality} / $exposure->{diff_component_count} > 0.5)) {
+            printf("QUALITY: $exposure->{exp_id} has bad diff quality: %d / %d\n",
+                    $exposure->{diff_bad_quality} , $exposure->{diff_component_count});
+            $is_bad_quality = 1;
+        }
+        # If we've detected a bad quality exposure, drop it, and tell the companion.
+        if ($is_bad_quality) {
+            unless ((defined($exposure->{diffRun_state}))&&
+                    ($exposure->{diffRun_state} eq 'full')) {
+                $needs_qstack = 1;
+                $needs_something_private = 1;
+                if ($companion) {
+                    $companion->{private} = 1;
+                    $companion->{pairwise} = 0;
+                    &update_this_exposure($companion);
+                }
+                $exposure->{private} = 1;
+                $exposure->{pairwise} = 0;
             }
-            $exposure->{private} = 1;
-            $exposure->{pairwise} = 0;
             $exposure->{data_state} = 'drop';
             &update_this_exposure($exposure);
 
         }
-#       if ($companion) { # Validate that there are no problems with the companion exposure
-#           if ($companion->{cam_quality}) { # Maybe other things here?
-#               $exposure->{private} = 1;
-#               $exposure->{data_state} = 'drop';
-#               &update_this_exposure($exposure);
-#               $needs_qstack = 1;
-#           }
-#       }
+        
         if  ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through.
             $can_qstack ++;
@@ -564 +621 @@
     }
 
+    my $warps = '';
+    foreach $exposure (@$exposures) {
+        if (($exposure->{data_state} != 'drop')&&
+            (S64_IS_NOT_NULL($exposure->{warp_id}))) {
+            $warps .= " -warp_id $exposure->{warp_id} ";
+        }
+    }
+
     my @utctime = gmtime();
     $utctime[5] += 1900;
@@ -570 +635 @@
     my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
     my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
-    my $data_group = "${label}.${date}";
+    my $data_group = "${label}.${proj_cell}.quick.${date}";
 
     my $command = "$stacktool ";
@@ -576 +641 @@
     $command .= " -dbname $dbname " if defined $dbname;
     $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter ";
-    $command .= " -set_label ${label} -set_data_group ${proj_cell}.quick.${date} ";
+    $command .= " -set_label ${label} -set_data_group $data_group ";
     $command .= "  -set_workdir $workdir  -set_dist_group NODIST ";
     $command .= " -min_num 2 -set_reduction QUICKSTACK ";
+    $command .= " $warps ";
 
     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -589 +655 @@
     $command = "$stacktool ";
     $command .= " -dbname $dbname " if defined $dbname;
-    $command .= " -sassskyfile -data_group ${proj_cell}.quick.${date} ";
+    $command .= " -sassskyfile -data_group $data_group ";
     $command .= " -filter $filter -projection_cell ${proj_cell} ";
 
@@ -632 +698 @@
     unless (defined($label) && defined($filter) && defined($proj_cell)) {
         &my_die("Unable to perform stacktool. Insufficient information.", $lap_id);
+    }
+
+    my $warps = '';
+    foreach $exposure (@$exposures) {
+        if (($exposure->{data_state} != 'drop')&&
+            (S64_IS_NOT_NULL($exposure->{magicked}))&&
+            (S64_IS_NOT_NULL($exposure->{warp_id}))) {
+            $warps .= " -warp_id $exposure->{warp_id} ";
+        }
     }
 
@@ -640 +715 @@
     my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
     my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
-    my $data_group = "${label}.${date}";
+    my $data_group = "${label}.${proj_cell}.final.${date}";
 
     my $command = "$stacktool ";
@@ -646 +721 @@
     $command .= " -dbname $dbname " if defined $dbname;
     $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter ";
-    $command .= " -set_label ${label} -set_workdir $workdir -set_data_group ${proj_cell}.final.${date} ";
-    $command .= " -min_num 2 -set_reduction THREEPI_STACK ";
+    $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group ";
+    $command .= " -min_num 2 -set_reduction THREEPI_STACK -set_dist_group ${label} ";
+    $command .= " $warps ";
 
     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -659 +735 @@
     $command = "$stacktool ";
     $command .= " -dbname $dbname " if defined $dbname;
-    $command .= " -sassskyfile -data_group ${proj_cell}.final.${date} ";
+    $command .= " -sassskyfile -data_group $data_group ";
     $command .= " -filter $filter -projection_cell ${proj_cell} ";
 
@@ -734 +810 @@
             $command .= " -set_dist_group $exposure->{dist_group} ";
         }
-        
-        if ($exposure->{pairwise}) { # warpwarp
+        my $retry_command;
+        if (($exposure->{pairwise})&&(defined(${ $exposures }[$match_hash{$exposure->{chip_id}}]))) { # warpwarp
             my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}];
             $command .= " -definewarpwarp ";
-            $command .= "-input_label $label -template_label $label -backwards ";
+            $command .= "-input_label $label -template_label $label ";
             $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} ";
+            $retry_command = $command;
+            $command .= " -backwards "; # This usually works.
             $already_queued{$exposure->{warp_id}} = 1;
             $already_queued{$companion->{warp_id}} = 1;
@@ -763 +841 @@
         my $diff_id = $diff->{diff_id};
         unless (defined($diff_id)) {
-            $exposure->{data_state} = 'drop';
-            &update_this_exposure($exposure);
-        }
-        
+            if ($retry_command) {
+                ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $retry_command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("unable to perform difftool -definewarp(warp|stack): $error_code", $exposure->{lap_id}, $exposure->{proj_cell});
+                }
+                
+                $diffs = $mdcParser->parse_list(join "", @$stdout_buf) or
+                    &my_die("Unable to parse metadata from difftool -definewarp(warp|stack)", $lap_id, "");
+                
+                $diff = ${ $diffs }[0];
+                $diff_id = $diff->{diff_id};
+            }
+            unless (defined($diff_id)) {
+                $exposure->{data_state} = 'drop';
+                &update_this_exposure($exposure);
+            }
+        }
     }
 }
@@ -872 +965 @@
     
     my $command = "$laptool -updateexp -lap_id $lap_id -exp_id $exp_id ";
+    $command .= " -dbname $dbname " if defined $dbname;
     if (($exposure->{chip_id})&&(S64_IS_NOT_NULL($exposure->{chip_id}))) {
         $command .= " -set_chip_id  $exposure->{chip_id} ";

branches/eam_branches/ipp-20110505/ippScripts/scripts/receive_file.pl

@@ -224 +224 @@
         $stage = 'sky';
         # XXX: This should be skycell, but the distribution code uses exposure
-        $comp_name = 'exposure';
-        $current_component = $comp_name;
+        $comp_name = 'skycell_id';
+#        $current_component = $comp_name;
     } else {
         &my_die( "unexpected run type line found in $filename: $runType\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
@@ -237 +237 @@
         $new_workdir_value = "$workdir/$destdir";
     }
+
+    if ($stage eq 'sky') {
+        # the dbinfo file for a skyRun only has one component and it doesn't contain
+        # skycell_id which is the way components are listed in the dirinfo file.
+        my @ids = keys %$components;
+        &my_die( "unexpected number of components scalar @ids found in staticsky dirinfo file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if scalar @ids != 1;
+        $current_component = $ids[0];
+    }
+
     my $component_dir;
     if ($current_component) {

branches/eam_branches/ipp-20110505/ippScripts/scripts/warp_skycell.pl

@@ -72 +72 @@
 
 my ($logDest, $traceDest);
+my $do_stats;
 if ($run_state eq 'new') {
     $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, 0);
     $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1);
+    $do_stats = 1;
 } elsif ($run_state eq 'update')  {
     $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 1);
@@ -162 +164 @@
 my $outputBin1 = prepare_output ("PSWARP.BIN1", $outroot, $skycell_id, 1);
 my $outputBin2 = prepare_output ("PSWARP.BIN2", $outroot, $skycell_id, 1);
-my $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1);
+my $outputStats;
+if ($do_stats) {
+    $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1) if $do_stats;
+}
 my $configuration;
 
@@ -257 +262 @@
 # Run pswarp
 my $cmdflags;
-my $do_stats;
 unless ($no_op) {
     my $command = "$pswarp";
@@ -279 +283 @@
     if ($run_state eq 'new') {
         $command .= " -dumpconfig $configuration";
-        $do_stats = 1;
     } else {
         $command .= " -ipprc $configuration";

branches/eam_branches/ipp-20110505/ippTasks/lap.pro

@@ -17 +17 @@
 
 macro lap.show.books
+    echo "lapNewRuns"
     book listbook lapNewRuns
+    echo "lapRunRuns"
     book listbook lapRunRuns
+    echo "lapDoneRuns"
     book listbook lapDoneRuns
+    echo "lapFullRuns"
     book listbook lapFullRuns
+end
+
+macro lap.clear.books
+    book init lapNewRuns
+    book init lapRunRuns
+    book init lapFullRuns
+    book init lapDoneRuns
 end
 
@@ -36 +47 @@
       active true
     end
-    task lap.cleanup.load
-      active true
-    end
-    task lap.cleanup.run
-      active true
-    end
 end
 
@@ -57 +62 @@
       active false
     end
-    task lap.cleanup.load
-      active false
-    end
-    task lap.cleanup.run
+end
+
+macro lap.debug.mode
+    task lap.initial.load
+      active true
+    end
+    task lap.initial.run
+      active false
+    end
+    task lap.monitor.load
+      active true
+    end
+    task lap.monitor.run
       active false
     end
@@ -156 +170 @@
   # success
   task.exit  0
-#    book delpage lapNewRuns $options:0
     ipptool2book stdout lapNewRuns -uniq -key lap_id -setword dbname $options:0 -setword pantaskState INIT
 
+    process_cleanup lapNewRuns
+
     if ($VERBOSE > 2)
       book listbook lapNewRuns
@@ -181 +196 @@
   periods      -exec $LOADEXEC
   periods      -timeout 600
+  active       false
 # This can probably be increased and spread over hosts in the future.
   npending     1            
@@ -193 +209 @@
 
 
-    book getpage lapNewRuns $lap_NewPage -var lapNewPageName 
+    book getpage lapNewRuns 0 -var lapNewPageName -key pantaskState INIT
 
     $lap_NewPage ++
     if ($lap_NewPage >= $N) set lap_NewPage = 0
 
-
     if ("$lapNewPageName" == "NULL") break
+
+
 
     book setword lapNewRuns $lapNewPageName pantaskState RUN
@@ -205 +222 @@
     book getword lapNewRuns $lapNewPageName dbname -var DBNAME
 
+    option $LAP_ID
+
     $run = lap_science.pl --chip_mode --dbname $DBNAME --lap_id $LAP_ID
-
-    command $run
-
-  end
-
-  # success
-  task.exit  0
-#    book delpage lapNewRuns $options:0
-#    ipptool2book stdout lapNewRuns -uniq -key lap_id
-
+    
+    command $run
+
+  end
+
+  # success
+  task.exit  0
+    process_exit lapNewRuns $options:0 0
     if ($VERBOSE > 2)
       book listbook lapNewRuns
@@ -222 +239 @@
   # locked list
   task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  #operation times out?
-  task.exit    timeout
+    process_exit lapNewRuns $options:0 0
+    showcommand failure
+  end
+  task.exit    crash
+    process_exit lapNewRuns $options:0 0
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    process_exit lapNewRuns $options:0 0
     showcommand timeout
   end
@@ -267 +287 @@
 
     add_poll_labels run
-#    echo $run
-    command $run
-  end
-  # success
-  task.exit  0
-#    book delpage lapRunRuns $options:0
+    command $run
+  end
+  # success
+  task.exit  0
     ipptool2book stdout lapRunRuns -uniq -key lap_id -setword dbname $options:0 -setword pantaskState INIT
+    
+    process_cleanup lapRunRuns
 
     if ($VERBOSE > 2)
@@ -297 +317 @@
   periods      -exec $LOADEXEC
   periods      -timeout 600
+  active       false
 # This can probably be increased and spread over hosts in the future.
   npending     1            
@@ -306 +327 @@
     book npages lapRunRuns -var N
 
-#    echo $N $NETWORK
     if ($N == 0) break
     if ($NETWORK == 0) break
 
 
-    book getpage lapRunRuns $lap_RunPage -var lapRunPageName
+    book getpage lapRunRuns 0 -var lapRunPageName -key pantaskState INIT
 
     $lap_RunPage ++
     if ($lap_RunPage >= $N) set lap_RunPage = 0
 
-#    echo $lapRunPageName
     if ("$lapRunPageName" == "NULL") break
 
@@ -323 +342 @@
     book getword lapRunRuns $lapRunPageName dbname -var DBNAME
 
+    option $LAP_ID
+
     $run = lap_science.pl --monitor_mode --dbname $DBNAME --lap_id $LAP_ID
 
@@ -331 +352 @@
   # success
   task.exit  0
-#    book delpage lapRunRuns $options:0
-#    ipptool2book stdout lapRunRuns -uniq -key lap_id
-
-    if ($VERBOSE > 2)
+    process_exit lapRunRuns $options:0 0
+    if ($VERBOSE > 2)
+
       book listbook lapRunRuns
     end
@@ -340 +360 @@
   # locked list
   task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  #operation times out?
-  task.exit    timeout
+    process_exit lapRunRuns $options:0 0
+    showcommand failure
+  end
+  task.exit    crash
+    process_exit lapRunRuns $options:0 0
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    process_exit lapRunRuns $options:0 0
     showcommand timeout
   end
@@ -358 +381 @@
   periods      -exec $LOADEXEC
   periods      -timeout 30
+  active       false
   npending     1
 
@@ -390 +414 @@
     ipptool2book stdout lapDoneRuns -uniq -key lap_id  -setword dbname $options:0 -setword pantaskState INIT
 
+    process_cleanup lapDoneRuns
     if ($VERBOSE > 2)
       book listbook lapRuns
@@ -412 +437 @@
   periods      -exec $LOADEXEC
   periods      -timeout 600
+  active       false
 # This can probably be increased and spread over hosts in the future.
   npending     1            
@@ -424 +450 @@
 
 
-    book getpage lapDoneRuns $lap_DonePage -var lapDonePageName 
+    book getpage lapDoneRuns 0 -var lapDonePageName -key pantaskState INIT
 
     $lap_DonePage ++
@@ -435 +461 @@
     book getword lapDoneRuns $lapDonePageName dbname -var DBNAME
 
+    option $LAP_ID
     $run = lap_science.pl --cleanup_mode --dbname $DBNAME --lap_id $LAP_ID
 
@@ -443 +470 @@
   # success
   task.exit  0
-#    ipptool2book stdout lapDoneRuns -uniq -key lap_id
-
+    process_exit lapDoneRuns $options:0 0
     if ($VERBOSE > 2)
       book listbook lapDoneRuns
@@ -451 +477 @@
   # locked list
   task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  #operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
+    process_exit lapDoneRuns $options:0 0
+    showcommand failure
+  end
+  task.exit    crash
+    process_exit lapDoneRuns $options:0 0
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    process_exit lapDoneRuns $options:0 0
+    showcommand timeout
+  end
+end

branches/eam_branches/ipp-20110505/ippToPsps/jython/batch.py

@@ -32 +32 @@
     "../config/2/tables.vot"
     '''
-    def __init__(self, logger, batchType, inputFitsPath="", survey="", useFullTables=False):
+    def __init__(self, 
+                 logger, 
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 id,
+                 batchType, 
+                 inputFitsPath="", 
+                 survey="", 
+                 useFullTables=False):
+
+        self.everythingOK = False
+        self.readHeader = False
 
         # set up logging
@@ -40 +52 @@
 
         # set up class variables
+        self.id = id
+        self.gpc1Db = gpc1Db
+        self.ippToPspsDb = ippToPspsDb
+        self.scratchDb = scratchDb
         self.batchType = batchType;
         self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot"
@@ -46 +62 @@
         self.useFullTables = useFullTables
 
+        if self.alreadyProcessed(): return
+
+        # do we have an input file?
+        if self.inputFitsPath != "":
+
+            if not self.readPrimaryHeader(): return
+
         # TODO
         self.tablesToExport = []
@@ -51 +74 @@
         # open config
         doc = ElementTree(file="config.xml")
-
-        # create Gpc1Db object
-        self.gpc1Db = Gpc1Db(self.logger)
-        self.ippToPspsDb = IppToPspsDb(logger)
-        self.scratchDb = ScratchDb(logger, self.useFullTables)
 
         if self.survey != "":
@@ -88 +106 @@
         self.dateStr = now.strftime("%Y-%m-%d")
 
-        if self.inputFitsPath != "": 
-            file = open(self.inputFitsPath)
-            self.header = self.parseFitsHeader(file)
-            self.logger.info("Read primary and found " + str(len(self.header)) + " header cards") 
-            # TODO close file?
-
         # create DVO tables if accessing DVO directly
         if not self.useFullTables: self.scratchDb.createDvoTables()
 
+        self.everythingOK = True
+
     '''
     Destructor
@@ -103 +117 @@
 
         self.logger.debug("Batch destructor")
+
+
+    '''
+    Reads the primary header of the FITS file
+    '''
+    def readPrimaryHeader(self):
+
+        if self.readHeader: return True
+
+        # does it exist?
+        if not os.path.isfile(self.inputFitsPath):
+
+            self.logger.error("Cannot read file at '" + self.inputFitsPath + "'")
+            return False
+
+        file = open(self.inputFitsPath)
+        self.header = self.parseFitsHeader(file)
+        self.logger.info("Read primary header and found " + str(len(self.header)) + " header cards")
+        # TODO close file?
+
+        self.readHeader = True
+
+        return True
 
 
@@ -138 +175 @@
             file.seek(index + 2880, 0)
             
-        if found != True: self.logger.error("...could not find extension '" + name + "'")
-        else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards") 
+        if found != True: 
+            self.logger.error("...could not read header in extension '" + name + "'")
+            return
+        #else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards") 
 
         return header
@@ -262 +301 @@
 
         first = True
+
+        self.totalDetections = 0
         for table in tables:
 
-            sql = "SELECT MIN(objID), MAX(objID) FROM " + table
-            rs = self.scratchDb.stmt.executeQuery(sql)
+            sql = "SELECT MIN(objID), MAX(objID), COUNT(objID) FROM " + table
+            rs = self.scratchDb.executeQuery(sql)
             rs.first()
+
+            self.totalDetections = self.totalDetections + rs.getLong(3)
 
             if first:
@@ -276 +319 @@
 
             first = False
+            rs.close()
 
         self.ippToPspsDb.updateMinMaxObjID(self.batchID, self.minObjID, self.maxObjID)
+        self.logger.info("Total detections = %ld min objID = %ld max objID = %ld" % (self.totalDetections, self.minObjID, self.maxObjID))
+
 
     '''
@@ -313 +359 @@
          self.pspsTables = stilts.treads(self.pspsVoTableFilePath)
          for table in self.pspsTables:
-             self.logger.info("Creating PSPS table: " + table.name)
+             self.logger.debug("Creating PSPS table: " + table.name)
              table.write(self.scratchDb.url + '#' + table.name)
              self.tablesToExport.append(table.name)
@@ -337 +383 @@
     Accepts a regular expression filter so not all tables need to be imported
     '''
-    def importIppTables(self, filter):
+    def importIppTables(self, filter=""):
 
       self.logger.info("Attempting to import tables from input FITS file")
@@ -347 +393 @@
           match = re.match(filter, table.name)
           if not match: continue
-          self.logger.info("   Reading IPP table " + table.name + " from FITS file")
+          self.logger.info("Reading IPP table " + table.name + " from FITS file")
           table = stilts.tpipe(table, cmd='explodeall')
 
           # drop any previous tables before import
-          self.scratchDb.dropTable(table.name)
+          #self.scratchDb.dropTable(table.name)
 
           # IPP FITS files are littered with infinities, so remove these
-          self.logger.info("   Removing Infinity values from all columns")
+          self.logger.debug("Removing Infinity values from all columns")
           table = stilts.tpipe(table, cmd='replaceval -Infinity null *')
           table = stilts.tpipe(table, cmd='replaceval Infinity null *')
@@ -360 +406 @@
           try:
               table.write(self.scratchDb.url + '#' + table.name)
+              self.scratchDb.killLastConnectionID()
+              count = count + 1
           except:
-              self.logger.exception("   Problem writing table '" + table.name + "' to the database")
-          count = count + 1
+              self.logger.exception("Problem writing table '" + table.name + "' to the database")
+
 
       self.logger.info("Done. Imported %d tables" % count)
-
       self.indexIppTables()
 
@@ -373 +420 @@
     def exportPspsTablesToFits(self, regex="(.*)"):
 
-        self.logger.info("Replacing NULLs with -999 then exporting all PSPS tables to FITS")
+        self.logger.info("Replacing NULLs with -999, changing tables names using regex: " + regex)
         _tables = []
 
-        self.logger.info("    Selecting database tables")
+        self.logger.info("Selecting database tables")
         for table in self.tablesToExport:
 
@@ -383 +430 @@
 
            # get everything from table
-           _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table)
-
+           try:
+               _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table)
+               self.scratchDb.killLastConnectionID()
+           except:
+               self.logger.exception("Could not read from DB table: " + table)
+               return False
+               
            # replace nulls and empty fields with weird PSPS -999 pseudo-null
            _table = stilts.tpipe(_table, cmd='replaceval "" -999 *')
@@ -395 +447 @@
            _tables.append(_table)
 
-        self.logger.info("    Writing to FITS file '" + self.outputFitsPath + "'...")
-        stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
-        self.logger.info("    ...done")
-        self.ippToPspsDb.updateProcessed(self.batchID, 1)
+        self.logger.info("Writing to FITS file '" + self.outputFitsPath + "'...")
+        try:
+            stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
+            self.ippToPspsDb.updateProcessed(self.batchID, 1)
+        except:
+            self.logger.exception("Could not write to FITS")
+            return False
+
+        return True
 
     '''
@@ -447 +504 @@
     '''
     def alreadyProcessed(self):
-           self.logger.info("Not implemented")
-
-
-
+        self.logger.info("Not implemented")
+
+
+    '''
+    Creates and publishes a batch
+    '''
+    def run(self):
+
+        if not self.everythingOK: return
+
+        self.createEmptyPspsTables()
+        self.importIppTables()
+        if self.populatePspsTables():
+            if self.exportPspsTablesToFits():
+                self.writeBatchManifest()
+                self.createTarball()
+                self.publishToDatastore()
+                #self.reportNullsInAllPspsTables(False)
+                #sys.exit()
+        self.logger.info("Finished.")
+
+

branches/eam_branches/ipp-20110505/ippToPsps/jython/detectionbatch.py

@@ -6 +6 @@
 from java.lang import *
 from java.sql import *
+
 from batch import Batch
 from gpc1db import Gpc1Db
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
 
 import logging.config
@@ -19 +22 @@
     Constructor
     '''
-    def __init__(self, logger, camID, inputFile, test=False, useFullTables=False):
+    def __init__(self, 
+                 logger,
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 camID, 
+                 inputFile, 
+                 test=False, 
+                 useFullTables=False):
+
        super(DetectionBatch, self).__init__(
                logger,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               camID,
                "detection", 
                inputFile, 
-               "MD04",
-               useFullTables) # TODO
+               "MD04", # TODO
                #"3PI") # TODO
-
-       self.logger.info("DetectionBatch constructor. Creating batch from: '" + inputFile + "'")
-
-       meta = self.gpc1Db.getCameraStageMeta(camID)
-      
-       self.expID = meta[0];
-       self.expName = meta[1];
-       self.distGroup = meta[2];
-
-       self.logger.info("Processing exposure with ID: %d, name: %s and distribution group: %s" % (self.expID, self.expName, self.distGroup))
+               useFullTables)
+
+       if not self.everythingOK: return
+
+       # meta data to the log
+       self.logger.info("New Detection Batch:")
+       self.logger.info("Cam ID:             %d" % self.id)
+       self.logger.info("file:               %s" % inputFile)
+       self.logger.info("Exp ID:             %d" % self.expID)
+       self.logger.info("Exp name:           %s" % self.expName)
+       self.logger.info("Distribution group: %s" % self.distGroup)
 
        # create an output filename, which is {expID}.FITS
@@ -167 +183 @@
         ," + self.header['PCA2X0Y2'] + " \
         )"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("FrameMeta", "surveyID", str(self.surveyID))
@@ -184 +200 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE ImageMeta"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -310 +326 @@
                )"
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         self.scratchDb.updateFilterID(tableName, self.filter)
         self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
-        self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO'])
+        if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO'])
         self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0")
 
@@ -327 +343 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE Detection"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -375 +391 @@
                ,EXT_NSIGMA \
                FROM " + ota + "_psf"
-
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         # set obsTime
         sql = "UPDATE " + tableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr)
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
         self.scratchDb.updateAllRows(tableName, "historyModNum", "0")
@@ -387 +402 @@
         self.scratchDb.updateFilterID(tableName, self.filter)
 
-        # now delete bad flux
+        # now delete bad flux and bad chip positions
         self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "instFlux")
         self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "peakADU")
@@ -401 +416 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE SkinnyObject"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -415 +430 @@
                ,surveyID \
                FROM Detection_" + ota
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease))
@@ -429 +444 @@
         self.scratchDb.dropTable(tableName)
         sql = "CREATE TABLE " + tableName + " LIKE ObjectCalColor"
-        try: self.scratchDb.stmt.execute(sql)
+        try: self.scratchDb.execute(sql)
         except: pass
 
@@ -443 +458 @@
                ,filterID \
                FROM Detection_" + ota
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum))
@@ -484 +499 @@
 
         imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID)
-        self.logger.info("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
+        self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID)
         sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \
                a.ippObjID = b.ippObjID, \
@@ -494 +509 @@
                AND b.imageID = " + str(imageID)
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
 
@@ -511 +526 @@
 
         # loop through all OTAs and populate ImageMeta extensions
+        self.logger.info("Reading all fits headers and populating ImageMeta tables")
         for x in range(self.startX, self.endX):
             for y in range(self.startY, self.endY):
@@ -524 +540 @@
                 # load corresponding header into memory
                 header = self.findAndReadFITSHeader(ota + ".hdr", file)
+                if not header:
+                    self.logger.error("No header found for OTA " + ota)
+                    continue
+
 
                 # store sourceID/imageID combo in Db so DVO can look up later
@@ -578 +598 @@
 
                 # update ImageMeta with count of detections for this OTA and photoCodeID
-                sql = "UPDATE ImageMeta_" + ota + " SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota]))
-                self.scratchDb.stmt.execute(sql)
+                sql = "UPDATE ImageMeta_" + ota + " \
+                       SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota]))
+                self.scratchDb.execute(sql)
 
                 self.populateSkinnyObjectTable(ota)
@@ -603 +624 @@
         # update FrameMeta with count OTAs in this file and total number of photometric reference sources
         sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef)
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
         
         return True
@@ -613 +634 @@
 
         sql = "UPDATE " + tableName + " SET imageID = %d%d%d" % (self.expID, x, y) 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -619 +640 @@
     '''
     def alreadyProcessed(self):
+
+        meta = self.gpc1Db.getCameraStageMeta(self.id)
+        self.expID = meta[0];
+        self.expName = meta[1];
+        self.distGroup = meta[2];
 
         return self.ippToPspsDb.alreadyProcessed("detection", "exp_id", self.expID)
@@ -634 +660 @@
 
 
+    '''
+    Overriding this method. Filter to only import *.psf extensions
+    '''
+    def importIppTables(self, filter=""):
+       return super(DetectionBatch, self).importIppTables(".*.psf")
+
+
+    '''
+    Overriding this method. Use regex to trim off, eg _XY33 extension
+    '''
+    def exportPspsTablesToFits(self, regex="(.*)"):
+       return super(DetectionBatch, self).exportPspsTablesToFits("([a-zA-Z]+)")
+
+
+# TODO put in config
+useFullTables=True
+testMode=False
+
 logging.config.fileConfig("logging.conf")
 logger = logging.getLogger("detectionbatch")
+logger.setLevel(logging.INFO)
 logger.info("Starting")
 
 gpc1Db = Gpc1Db(logger)
+ippToPspsDb = IppToPspsDb(logger)
+scratchDb = ScratchDb(logger, useFullTables)
+
 camIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.V2", "cam")
 logger.info("Found %d exposures" % len(camIDs))
@@ -645 +693 @@
 for camID in camIDs:
 
-    logger.info("-------------------------------------------------- cam ID: %d" % camID)
+    #if camID < 43764: continue # TODO 
 
     file = gpc1Db.getCameraStageSmf(camID)
-    if not os.path.isfile(file):
-        logger.error("Cannot read file at '" + file)
-        continue
-
-    detectionBatch = DetectionBatch(logger, camID, file, False, True)
-
-    if not detectionBatch.alreadyProcessed():
-
-        detectionBatch.createEmptyPspsTables()
-        detectionBatch.importIppTables(".*.psf")
-        if detectionBatch.populatePspsTables():
-            detectionBatch.exportPspsTablesToFits("([a-zA-Z]+)")
-            detectionBatch.writeBatchManifest()
-            #detectionBatch.reportNullsInAllPspsTables(False)
-            #detectionBatch.createTarball()
-            #detectionBatch.publishToDatastore()
-    
-            i = i+1
-           # if i > 0: sys.exit()
-
+
+    detectionBatch = DetectionBatch(logger,
+                                    gpc1Db,
+                                    ippToPspsDb,
+                                    scratchDb,
+                                    camID, 
+                                    file, 
+                                    testMode, 
+                                    useFullTables)
+    detectionBatch.run()
+

branches/eam_branches/ipp-20110505/ippToPsps/jython/gpc1db.py

@@ -29 +29 @@
         self.logger.debug("Gpc1Db destructor")
 
+
+    '''
+    TODO
+    '''
+    def getIDsInThisDVODbForThisStageFudge(self):
+
+        sql = "SELECT staticskyRun.sky_id \
+               FROM staticskyInput, staticskyRun, stackRun, staticskyResult \
+               WHERE staticskyRun.sky_id = staticskyInput.sky_id \
+               AND staticskyInput.stack_id = stackRun.stack_id \
+               AND staticskyInput.sky_id = staticskyResult.sky_id \
+               AND staticskyRun.label like 'MD04.staticsky' \
+               AND stackRun.filter like 'i%'"
+
+        try:
+            rs = self.executeQuery(sql)
+        except:
+            self.logger.exception("Can't query for ids in DVO")
+
+        ids = []
+        while (rs.next()):
+            ids.append(rs.getInt(1))
+
+        rs.close()
+
+        self.logger.info("Found %d items in DVO database '" % (len(ids)))
+
+        return ids
+
     '''
     Gets a list of ids in this DVO database for this stage, could be cam or staticsky (so far)
@@ -40 +69 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
         except:
             self.logger.exception("Can't query for ids in DVO")
@@ -50 +79 @@
         rs.close()
 
-        self.logger.debug("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage))
+        self.logger.info("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage))
 
         return ids
@@ -73 +102 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
         except:
             self.logger.exception("Can't query for imageIDs")
@@ -105 +134 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             meta.append(rs.getInt(1))
@@ -113 +142 @@
 
         return meta
+
     '''
     Gets some camera-stage meta data for this cam_id
@@ -127 +157 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             meta.append(rs.getInt(1))
@@ -150 +180 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
         except:
@@ -171 +201 @@
             files = glob.glob(path + "/*.cmf")
 
+        if len(files) < 1: return "NULL"
+
         return files[0] # TODO just returning first file - check
 
@@ -186 +218 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
         except:
@@ -200 +232 @@
 
             f=os.popen("neb-ls -p "+path+"%cmf")
-            print "neb-ls -p "+path+"%cmf"
             for i in f.readlines():
                 files.append(i.rstrip())
-                print i.rstrip()
 
         # or not a neb path
@@ -211 +241 @@
         return files
 
+
+    '''
+    TODO hack to get exposure time for a stack
+    '''
+    def getStackExpTime(self, stackID):
+
+        self.logger.debug("Querying GPC1 for stack exposure time")
+
+        sql = "SELECT SUM(exp_time) * (COUNT(warp_id) - reject_images) / COUNT(warp_id) as EXPTIME \
+               FROM staticskyRun JOIN staticskyInput using(sky_id) \
+               JOIN stackRun using(stack_id) \
+               JOIN stackSumSkyfile using(stack_id) \
+               JOIN stackInputSkyfile using(stack_id) \
+               JOIN warpRun using(warp_id) \
+               JOIN fakeRun using(fake_id) \
+               JOIN camRun using(cam_id) \
+               JOIN chipRun using(chip_id) \
+               JOIN rawExp using(exp_id) \
+               WHERE stack_id = %d" % stackID
+
+        try:
+            rs = self.executeQuery(sql)
+            rs.first()
+            return rs.getInt(1)
+        except:
+            self.logger.exception("Can't query for exposure time")
+
+        return 0.0
+
+

branches/eam_branches/ipp-20110505/ippToPsps/jython/ipptopspsdb.py

@@ -37 +37 @@
                )"
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
         sql = "SELECT MAX(batch_id) FROM batch"
@@ -44 +44 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             batchID = rs.getInt(1)
@@ -64 +64 @@
                WHERE batch_id = " + str(batchID)
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -75 +75 @@
                WHERE batch_id = " + str(batchID)
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -86 +86 @@
                WHERE batch_id = " + str(batchID)
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -101 +101 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             if rs.getInt(1) > 0: 
@@ -128 +128 @@
                )"
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -149 +149 @@
                )"
 
-        self.stmt.execute(sql)
+        self.execute(sql)
 
 

branches/eam_branches/ipp-20110505/ippToPsps/jython/mysql.py

@@ -37 +37 @@
         self.url = "jdbc:mysql://"+self.dbHost+"/"+self.dbName+"?user="+self.dbUser+"&password="+self.dbPass
         self.con = DriverManager.getConnection(self.url)
-        self.stmt = self.con.createStatement()
+        self.connectionID = self.getLastConnectionID()
+        self.logger.info("MySQL connection to %s with ID %d" % (dbType, self.connectionID))
+
+        #self.stmt = self.con.createStatement()
+
+
+    '''
+    Disconnect from database
+    '''
+    def disconnect(self):
+        self.con.close()
 
     '''
@@ -45 +55 @@
 
         self.logger.debug("MySql destructor")
-        self.stmt.close()
-        self.con.close()
+        self.disconnect()
+
+    '''
+    Kills the last connection ID, so long as it's not THIS connection ID
+    '''
+    def killLastConnectionID(self):
+   
+        connectionID = self.getLastConnectionID()
+        if connectionID == self.connectionID:
+            self.logger.error("NOT going to kill THIS connection ID")
+            return
+
+        sql = "KILL %d" % connectionID
+        self.execute(sql)
+
+    '''
+    Gets the last connection ID
+    '''
+    def getLastConnectionID(self):
+
+        sql = "SELECT ID \
+               FROM INFORMATION_SCHEMA.PROCESSLIST \
+               WHERE DB='" + self.dbName + "' \
+               ORDER BY ID"
+        rs = self.executeQuery(sql)
+        rs.last()
+        return rs.getInt(1)
 
     '''
@@ -54 +89 @@
 
         sql = "UPDATE " + table + " SET " + column + " = " + value
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -62 +97 @@
 
         sql = "DROP TABLE " + table
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: return
 
@@ -74 +109 @@
         sql = "ALTER TABLE " + table + " ADD UNIQUE (" + column + ")"
         try:
-            self.stmt.execute(sql)
+            self.execute(sql)
         except: pass
             #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'")
@@ -82 +117 @@
     def createIndex(self, table, column):
 
-        self.logger.debug("Creating index on column '"+column+"' for table '"+table+"'")
+        #self.logger.debug("Creating index on column '"+column+"' for table '"+table+"'")
 
         sql = "CREATE INDEX "+table+"_"+column+"_index ON "+table+" ("+column+")"
         try:
-            self.stmt.execute(sql)
+            self.execute(sql)
         except: pass
             #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'")
+    '''
+    TODO
+    '''
+    def execute(self, sql):
+
+        stmt = self.con.createStatement()
+        stmt.execute(sql)
+        stmt.close()
+
+    '''
+    TODO
+    '''
+    def executeQuery(self, sql):
+
+        stmt = self.con.createStatement()
+        rs = stmt.executeQuery(sql)
+        #stmt.close()
+        return rs
 
     '''
@@ -96 +149 @@
 
        sql = "SHOW COLUMNS FROM " + tableName
-       rs = self.stmt.executeQuery(sql)
+       rs = self.executeQuery(sql)
        columns = []
        while (rs.next()): columns.append(rs.getString(1))
@@ -109 +162 @@
 
       sql = "UPDATE " + tableName + " SET " + column + " = " + sub + " WHERE " + column + " IS NULL"
-      self.stmt.execute(sql)
+      self.execute(sql)
 
     '''
@@ -123 +176 @@
           
           sql = "UPDATE " + tableName + " SET " + column + " = " + sub + " WHERE " + column + " IS NULL"
-          self.stmt.execute(sql)
+          self.execute(sql)
 
     '''
@@ -131 +184 @@
 
         sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + columnName + " = " + value
-        rs = self.stmt.executeQuery(sql)
+        rs = self.executeQuery(sql)
         rs.first()
-        nBadFlux = rs.getInt(1)
-        self.logger.info("%d NULL %s values in table %s. Deleting." % (nBadFlux, columnName, tableName))
+        nBad = rs.getInt(1)
+        self.logger.info("%5d NULL %s values in table %s. Deleting." % (nBad, columnName, tableName))
 
         sql="DELETE from " + tableName + " WHERE " + columnName + " = " + value
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -145 +198 @@
 
         sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + columnName + " IS NULL"
-        rs = self.stmt.executeQuery(sql)
+        rs = self.executeQuery(sql)
         rs.first()
-        nBadFlux = rs.getInt(1)
-        self.logger.info("%d NULL %s values in table %s. Deleting." % (nBadFlux, columnName, tableName))
+        nBad = rs.getInt(1)
+        self.logger.info("%5d NULL %s values in table %s. Deleting." % (nBad, columnName, tableName))
 
         sql="DELETE from " + tableName + " WHERE " + columnName + " IS NULL"
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -160 +213 @@
        # first, count rows
        sql = "SELECT COUNT(*) FROM " + tableName
-       rs = self.stmt.executeQuery(sql)
+       rs = self.executeQuery(sql)
        rs.first()
        numRows = rs.getInt(1)
@@ -175 +228 @@
           
           sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + column + " IS NULL"
-          rs = self.stmt.executeQuery(sql)
+          rs = self.executeQuery(sql)
           rs.first()
           if rs.getInt(1) == numRows:
@@ -191 +244 @@
         sql = "SELECT COUNT(*) FROM " + table
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             return rs.getInt(1)

branches/eam_branches/ipp-20110505/ippToPsps/jython/scratchdb.py

@@ -45 +45 @@
         sql = "SELECT surveyID FROM Survey WHERE name = '" + name + "'"
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             return rs.getInt(1)
@@ -61 +61 @@
         sql = "SELECT flags FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             flags = rs.getInt(1)
@@ -78 +78 @@
         sql = "SELECT imageID FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             imageID = rs.getInt(1)
@@ -95 +95 @@
         sql = "SELECT photcode FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID)
         try:
-            rs = self.stmt.executeQuery(sql)  
+            rs = self.executeQuery(sql)  
             rs.first()
             photcode = rs.getInt(1)
@@ -109 +109 @@
 
         sql = "UPDATE "+table+" AS a, Filter AS b SET a.filterID=b.filterID WHERE b.filterType = '" + filter + "'"
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -123 +123 @@
                " + str(imageID) + "    \
                )"
-        self.stmt.execute(sql)
+        self.execute(sql)
 
     '''
@@ -131 +131 @@
 
         sql = "INSERT INTO dvoDone (name) VALUES ('" + name + "')"
-        self.stmt.execute(sql)
+        self.execute(sql)
         
     '''
@@ -141 +141 @@
 
         try:
-            rs = self.stmt.executeQuery(sql)
+            rs = self.executeQuery(sql)
             rs.first()
             if rs.getInt(1) > 0:
@@ -160 +160 @@
 
         sql = "DROP TABLE dvoMeta"
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: pass
         
         sql = "DROP TABLE dvoDetection"
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: pass
 
@@ -175 +175 @@
                )"
 
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: 
             self.logger.error("Unable to create DVO meta-data database table")
@@ -193 +193 @@
                #INDEX (ippDetectID) \
 
-        try: self.stmt.execute(sql)
+        try: self.execute(sql)
         except: 
             self.logger.error("Unable to create DVO detection database table")

branches/eam_branches/ipp-20110505/ippToPsps/jython/stackbatch.py

@@ -8 +8 @@
 from java.sql import *
 
+from batch import Batch
 from gpc1db import Gpc1Db
-from batch import Batch
+from ipptopspsdb import IppToPspsDb
+from scratchdb import ScratchDb
+
 import logging.config
 
@@ -20 +23 @@
     Constructor
     '''
-    def __init__(self, logger, skyID, inputFile, stackType, useFullTables=False):
+    def __init__(self, 
+                 logger, 
+                 gpc1Db,
+                 ippToPspsDb,
+                 scratchDb,
+                 skyID, 
+                 inputFile, 
+                 stackType, 
+                 useFullTables=False):
+
        super(StackBatch, self).__init__(
                logger,
+               gpc1Db,
+               ippToPspsDb,
+               scratchDb,
+               skyID,
                "stack", 
                inputFile, 
@@ -28 +44 @@
                useFullTables) # TODO
 
-       self.logger.info("StackBatch constructor. Creating batch from: '" + inputFile + "'")
-
-       self.skyID = skyID
-
-       # get filterID using init table
-       self.filter = self.header['FPA.FILTER']
-       self.filter = self.filter[0:1]
-
-       self.stackType = stackType
-       meta = self.gpc1Db.getStackStageMeta(self.skyID, self.header['FPA.FILTER'])
-       if len(meta) < 1: return
-       self.stackID = meta[0];
-       self.skycell = meta[1];
-
-       # determine skycell from header value
-       #self.skycell = "skycell.34" #= self.header['SKYCELL']
-       self.skycell = self.skycell[8:]
-
-       self.logger.info("Processing stack with ID: %d, type: %s and skycell: %s filter: %s" % (self.stackID, self.stackType, self.skycell, self.filter))
-
+       if not self.everythingOK: return
+
+       self.expTime = gpc1Db.getStackExpTime(self.stackID)
+
+       self.logger.info("got exp time of %d" % self.expTime)
+
+       # meta data to the log
+       self.logger.info("New Stack Batch:")
+       self.logger.info("Sky ID:     %d" % self.id)
+       self.logger.info("File:       %s" % inputFile)
+       self.logger.info("Stack ID:   %d" % self.stackID)
+       self.logger.info("Stack type: %s" % self.stackType)
+       self.logger.info("Skycell:    %s" % self.skycell)
+       self.logger.info("Filter:     %s" % self.filter)
 
        # delete PSPS tables
@@ -58 +68 @@
        self.scratchDb.dropTable("ObjectCalColor")
 
-       # delete IPP tables
-       #self.scratchDb.dropTable("SkyChip_psf")
-       #self.scratchDb.dropTable("SkyChip_xsrc")
-       #self.scratchDb.dropTable("SkyChip_xfit")
-       #self.scratchDb.dropTable("SkyChip_xrad")
-
-       self.logger.info("Stack type: " + self.safeDictionaryAccess(self.header, self.stackType))
-       # obs time makes no sense except for nightly stacks
-       #if self.header['STK_TYPE'] != "NIGHTLY_STACK": self.header['MJD-OBS'] = "-999"
-
        # create an output filename, which is {filterID}{skycellID}.FITS
-       self.outputFitsFile = "%s%07d.FITS" % (self.filter, int(self.skycell))
+       self.outputFitsFile = "%08d.FITS" % self.stackID
        self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile)
 
@@ -77 +77 @@
 
        # insert what we know about this stack batch into the stack table
-       self.ippToPspsDb.insertStackMeta(self.batchID, self.skyID, self.stackID, self.filter, self.stackType)
+       self.ippToPspsDb.insertStackMeta(self.batchID, self.id, self.stackID, self.filter, self.stackType)
 
        # insert sourceID/imageID combo so DVO can look it up
@@ -89 +89 @@
 
         sql = "UPDATE " + table + "  SET stackMetaID=" + str(self.stackID)
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -97 +97 @@
 
         sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '" + self.stackType + "'"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
 
@@ -148 +148 @@
         WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -196 +196 @@
         WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'"
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         # sersic fit has an extra parameter
@@ -213 +213 @@
             WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'"
 
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
 
 
@@ -220 +220 @@
     '''
     def populateStackMeta(self):
+
         self.logger.info("Procesing StackMeta table")
 
@@ -246 +247 @@
         ," + str(self.scratchDb.getPhotoCalID(self.header['SOURCEID'], self.header['IMAGEID'])) + " \
         ," + self.header['FPA.ZP'] + " \
-        ," + self.header['EXPTIME'] + " \
+        ," + str(self.expTime) + " \
         ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \
         ,'" + self.header['CTYPE1'] + "' \
@@ -261 +262 @@
         ," + self.header['PC002002'] + " \
         )"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID))
@@ -272 +273 @@
     '''
     def populateStackDetection(self):
+
         self.logger.info("Procesing StackDetection table")
 
@@ -321 +323 @@
                ,X_PSF_SIG \
                ,Y_PSF_SIG \
-               ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+" \
-               ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+")) / 1.085736) \
-               ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \
+               ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+str(self.expTime)+" \
+               ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+str(self.expTime)+")) / 1.085736) \
+               ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+str(self.expTime)+" \
                ,SKY \
                ,SKY_SIGMA \
@@ -352 +354 @@
                FROM SkyChip_psf"
 
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID))
@@ -362 +364 @@
         self.updateStackTypeID("StackDetection")
         self.updateDvoIDs("StackDetection")
-
-        # now delete bad flux
+        sql = "ALTER TABLE StackDetection ADD PRIMARY KEY (objID, stackDetectID)"
+        self.scratchDb.execute(sql)
+
+        if self.stackType == "DEEP_STACK": 
+
+            #if deep stack and instFlux = null and err not null
+            sql = "UPDATE StackDetection AS a, SkyChip_psf AS b \
+                   SET instFlux = 2*b.PSF_INST_FLUX_SIG \
+                   WHERE instFlux IS NULL \
+                   AND b.PSF_INST_FLUX_SIG IS NOT NULL"
+            self.scratchDb.execute(sql)
+            #    instFlux = 2*PSF_INST_FLUX_SIG
+            
         self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux")
         self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID")
@@ -372 +385 @@
     '''
     def populateStackApFlx(self):
+
         self.logger.info("Procesing StackApFlx table")
  
@@ -381 +395 @@
 
         try:
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
         except: return
 
         # TODO temporarily loading 1st convolved fluxes into unconvolved fields
-        self.logger.info("    Adding un-convolved fluxes")
+        self.logger.info("Adding un-convolved fluxes")
         self.updateApFlxs("", "< 7.0")
-        self.logger.info("    Adding 1st convolved fluxes")
+        self.logger.info("Adding 1st convolved fluxes")
         self.updateApFlxs("c1", "< 7.0")
-        self.logger.info("    Adding 2nd convolved fluxes")
+        self.logger.info("Adding 2nd convolved fluxes")
         self.updateApFlxs("c2", "> 7.0")
 
-        self.logger.info("    Adding petrosians for extended sources")
+        self.logger.info("Adding petrosians for extended sources")
         sql = "UPDATE StackApFlx AS a, SkyChip_xsrc AS b SET \
         petRadius=b.PETRO_RADIUS \
@@ -403 +417 @@
         ,petR90Err=b.PETRO_RADIUS_90_ERR \
         WHERE a.ippDetectID=b.IPP_IDET"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID))
@@ -413 +427 @@
         self.updateStackTypeID("StackApFlx")
         self.updateDvoIDs("StackApFlx")
+        self.scratchDb.reportAndDeleteRowsWithNULLS("StackApFlx", "objID")
+        self.deleteDetectionsNotInStackDetection("StackApFlx")
+
 
     '''
@@ -418 +435 @@
     '''
     def populateStackModelFit(self):
+
         self.logger.info("Procesing StackModelFit table")
 
@@ -423 +441 @@
         sql = "INSERT INTO StackModelFit (ippDetectID) SELECT DISTINCT IPP_IDET from SkyChip_xfit"
         try:
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
         except:
             return
 
-
         # populate model parameters
-        self.logger.info("    Adding deVaucouleurs fit")
+        self.logger.info("Adding deVaucouleurs fit")
         self.updateModelFit("deV", "PS_MODEL_DEV")
-        self.logger.info("    Adding exponential fit")
+        self.logger.info("Adding exponential fit")
         self.updateModelFit("exp", "PS_MODEL_EXP")
-        self.logger.info("    Adding sersic fit")
+        self.logger.info("Adding sersic fit")
         self.updateModelFit("ser", "PS_MODEL_SERSIC")
 
@@ -444 +461 @@
         self.updateStackTypeID("StackModelFit")
         self.updateDvoIDs("StackModelFit")
+        self.scratchDb.reportAndDeleteRowsWithNULLS("StackModelFit", "objID")
+        self.deleteDetectionsNotInStackDetection("StackModelFit")
+
+    '''
+    Reports and deletes detections in this table that are not in StackDetection
+    '''
+    def deleteDetectionsNotInStackDetection(self, table):
+
+        sql = "SELECT COUNT(*) FROM " + table + " WHERE ippDetectID NOT IN (SELECT ippDetectID FROM StackDetection)"
+        rs = self.scratchDb.executeQuery(sql)
+        rs.first()
+        nMissing = rs.getInt(1)
+        self.logger.info("%5d detections in %s table that are not in StackDetection. Deleting" % (nMissing, table))
+  
+        if nMissing < 1: return
+        
+        sql = "DELETE FROM " + table + " WHERE ippDetectID NOT IN (SELECT ippDetectID FROM StackDetection)"
+        self.scratchDb.execute(sql)
+       
 
     '''
@@ -449 +485 @@
     '''
     def populateStackToImage(self):
+
         self.logger.info("Procesing StackToImage table")
 
@@ -457 +494 @@
                    VALUES (\
                    " + str(self.stackID) + ", " + imageID + ")"
-            self.scratchDb.stmt.execute(sql)
+            self.scratchDb.execute(sql)
 
         # now update StackMeta with correct number of inputs
         sql = "UPDATE StackMeta SET nP2Images = (SELECT COUNT(*) FROM StackToImage)"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
     '''
@@ -467 +504 @@
     '''
     def populateSkinnyObject(self):
+
         self.logger.info("Procesing SkinnyObject table")
 
@@ -474 +512 @@
                ) \
                SELECT \
-               objID \
+               DISTINCT objID \
                ,ippObjID \
                FROM StackDetection"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID))
@@ -486 +524 @@
     '''
     def populateObjectCalColor(self):
+
         self.logger.info("Procesing ObjectCalColor table")
 
@@ -493 +532 @@
                ) \
                SELECT \
-               objID \
+               DISTINCT objID \
                ,ippObjID \
                FROM StackDetection"
-        self.scratchDb.stmt.execute(sql)
+        self.scratchDb.execute(sql)
 
         self.scratchDb.updateFilterID("ObjectCalColor", self.filter)
@@ -508 +547 @@
 
         self.logger.info("Altering PSPS tables")
-        self.scratchDb.makeColumnUnique("StackDetection", "objID")
+        #self.scratchDb.makeColumnUnique("StackDetection", "objID")
         self.scratchDb.createIndex("StackDetection", "ippDetectID")
         self.scratchDb.createIndex("StackApFlx", "ippDetectID")
@@ -531 +570 @@
         imageID = self.scratchDb.getImageIDFromExternID(self.header['SOURCEID'], self.header['IMAGEID'])
 
-        self.logger.info("Updating table '" + table + "' with DVO IDs...")
+        self.logger.debug("Updating table '" + table + "' with DVO IDs...")
         sql = "UPDATE IGNORE " + table + " AS a, dvoDetectionFull AS b SET \
                a.ippObjID = b.ippObjID, \
@@ -539 +578 @@
                AND b.sourceID = " + self.header['SOURCEID'] + "\
                AND b.imageID = " + str(imageID)
-        self.scratchDb.stmt.execute(sql)
-
+        self.scratchDb.execute(sql)
 
     '''
@@ -553 +591 @@
         self.populateStackMeta()
         self.populateStackDetection()
-        self.populateStackModelFit()
-        self.populateStackApFlx()
+
+        if self.stackType != "NIGHTLY_STACK": 
+            self.populateStackModelFit()
+            self.populateStackApFlx()
+   
         self.populateStackToImage()
         self.populateSkinnyObject()
@@ -560 +601 @@
 
         self.setMinMaxObjID(["StackDetection"])
-        
+       
+        if self.totalDetections < 1: 
+
+            self.logger.error("No detections to publish")
+            return False
+
         return True
 
@@ -568 +614 @@
     def alreadyProcessed(self):
 
-        return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID)
+        # sadly, we have to read the FITS primary header first
+        if not self.readPrimaryHeader(): return False
+
+        # get filterID using init table
+        self.filter = self.header['FPA.FILTER']
+        self.filter = self.filter[0:1]
+
+        self.stackType = stackType
+        meta = self.gpc1Db.getStackStageMeta(self.id, self.header['FPA.FILTER'])
+        if len(meta) < 1: return False
+        self.stackID = meta[0];
+        self.skycell = meta[1];
+        self.skycell = self.skycell[8:]
+
+        #return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID)
+        return False # TODOI
+
+
+useFullTables=True
 
 logging.config.fileConfig("logging.conf")
 logger = logging.getLogger("stackbatch")
+logger.setLevel(logging.INFO)
 logger.info("Starting")
+
 gpc1Db = Gpc1Db(logger)
-stackType = "NIGHTLY_STACK"
-skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky")
-#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky")
-#stackType = "DEEP_STACK"
-#skyIDs = [689]
+ippToPspsDb = IppToPspsDb(logger)
+scratchDb = ScratchDb(logger, useFullTables)
+
+#stackType = "NIGHTLY_STACK"
+#skyIDs = gpc1Db.getIDsInThisDVODbForThisStageFudge()
+#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky")
+
+stackType = "DEEP_STACK"
+skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky")
+
+#skyIDs = [942]
 #skyIDs = [299]
 #skyIDs = [302]
 #skyIDs = [8508]
-i = 0
+#i = 0
 for skyID in skyIDs:
-
-    logger.info("-------------------------------------------------- sky ID: %d" % skyID)
+    
+    #if skyID < 1340: continue # nightly
+    #if skyID < 238: continue # deep
 
     cmfFiles = gpc1Db.getStackStageCmfs(skyID)
@@ -591 +664 @@
     for file in cmfFiles:
 
-        if not os.path.isfile(file):
-            logger.error("Cannot read file at '" + file)
-            continue
-
-        stackBatch = StackBatch(logger, skyID, file, stackType, True)
-
-        if not stackBatch.alreadyProcessed():
-
-            stackBatch.createEmptyPspsTables()
-            stackBatch.importIppTables("")
-            if stackBatch.populatePspsTables():
- 
-                #stackBatch.reportNullsInAllPspsTables(False)
-                stackBatch.exportPspsTablesToFits()
-                stackBatch.writeBatchManifest()
-                #stackBatch.createTarball()
-                #stackBatch.publishToDatastore()
-
-                i = i + 1
-                #if i > 0: sys.exit()
-
-logger.info("Finished")
+        stackBatch = StackBatch(logger,
+                                gpc1Db,
+                                ippToPspsDb,
+                                scratchDb,
+                                skyID, 
+                                file, 
+                                stackType, 
+                                useFullTables)
+
+        stackBatch.run()
+

branches/eam_branches/ipp-20110505/ippTools/share/laptool_definerun.sql

@@ -1 @@
-SELECT want.exp_id, have.chip_id, false as private, true as active, false as pairwise
+SELECT DISTINCT want.exp_id, have.chip_id, false as private, true as active, false as pairwise
   FROM
   (SELECT exp_id FROM rawExp 
@@ -8 +8 @@
   ) AS want
   LEFT JOIN 
-  (SELECT *
+  (SELECT exp_id,MAX(chip_id) AS chip_id
      FROM lapExp 
      where private IS FALSE 

branches/eam_branches/ipp-20110505/ippTools/share/laptool_exposures.sql

@@ -1 @@
-SELECT DISTINCT 
-    D.*,diffRun.state,
-    coalesce(CONVERT(sum(others.private),SIGNED),0) AS needs_remade 
---      0 AS needs_remade
-    FROM (
-  SELECT DISTINCT 
-      W.*,CONVERT(IFNULL(diff1.diff_id,diff2.diff_id),SIGNED) AS diff_id FROM (
-    SELECT DISTINCT
-        lap_id,lapRun.tess_id,projection_cell,filter,lapRun.state as lapRun_state, lapRun.registered, lapRun.fault, lapRun.label, lapRun.dist_group,
-        lapExp.exp_id,lapExp.chip_id,lapExp.pair_id,private,pairwise,active,lapExp.data_state,
-        chipRun.state as chipRun_state, 
-        coalesce(CONVERT(sum(chipProcessedImfile.fault),SIGNED),0) as chip_faults, 
-        coalesce(CONVERT(sum(chipProcessedImfile.quality),SIGNED),0) as chip_quality,
-        camRun.cam_id, camRun.state as camRun_state,   
-        coalesce(CONVERT(sum(camProcessedExp.fault),SIGNED),0) AS cam_faults, 
-        coalesce(CONVERT(sum(camProcessedExp.quality),SIGNED),0) AS cam_quality,
-        fakeRun.fake_id, fakeRun.state as fakeRun_state, 
-        coalesce(CONVERT(sum(fakeProcessedImfile.fault),SIGNED),0) as fake_faults,
-        warpRun.warp_id, warpRun.state as warpRun_state, 
-        coalesce(CONVERT(sum(warpSkyfile.fault),SIGNED),0) as warp_faults, 
-        coalesce(CONVERT(sum(warpSkyfile.quality),SIGNED),0) as warp_quality,
-        warpRun.magicked
-    FROM lapRun JOIN lapExp USING(lap_id)
-    LEFT JOIN chipRun USING(chip_id)
-    LEFT JOIN chipProcessedImfile USING(chip_id)
-    LEFT JOIN camRun USING(chip_id) LEFT JOIN camProcessedExp USING(cam_id)
-    LEFT JOIN fakeRun USING(cam_id) LEFT JOIN fakeProcessedImfile USING(fake_id)
-    LEFT JOIN warpRun USING(fake_id) LEFT JOIN warpSkyfile USING(warp_id)
-    WHERE    @WHERE@
-    AND (warpSkyfile.quality IS NULL OR
-         (warpSkyfile.quality != 8007      -- known cases where quality != 0, but everything's fine.
-          AND warpSkyfile.quality != 3006  -- known cases where quality != 0, but everything's fine.
-          ))
-    GROUP BY lap_id,exp_id
-    ) AS W
--- This was unreasonably slow in testing, so that's why I'm using a subquery here.
-  LEFT JOIN diffInputSkyfile AS diff1 ON (W.warp_id = diff1.warp1)
-  LEFT JOIN diffInputSkyfile AS diff2 ON (W.warp_id = diff2.warp2)
-) AS D
+select DISTINCT V3.*,
+       diffRun.diff_id,diffRun.state as diffRun_state,
+       coalesce(CONVERT(sum(diffSkyfile.quality != 0),SIGNED),0) AS diff_bad_quality,
+       coalesce(CONVERT(count(diffSkyfile.diff_id),SIGNED),0) AS diff_component_count
+       FROM
+( select V2.*,
+       warpRun.warp_id,warpRun.state as warpRun_state,
+       coalesce(CONVERT(sum(warpSkyfile.quality != 0),SIGNED),0) AS warp_bad_quality,
+       coalesce(CONVERT(count(warpSkyfile.warp_id),SIGNED),0) AS warp_component_count,
+       warpRun.magicked
+       FROM
+( select V1.*,
+       camRun.cam_id,camRun.state as camRun_state,
+       coalesce(CONVERT(sum(camProcessedExp.quality != 0),SIGNED),0) AS cam_bad_quality,
+       coalesce(CONVERT(count(camProcessedExp.cam_id),SIGNED),0) AS cam_component_count,
+       fakeRun.fake_id,fakeRun.state as fakeRun_state FROM
+( SELECT DISTINCT
+       lap_id,lapRun.tess_id,projection_cell,filter,lapRun.state as lapRun_state, lapRun.registered, lapRun.fault, lapRun.label, lapRun.dist_group,
+       lapExp.exp_id,lapExp.chip_id,lapExp.pair_id,private,pairwise,active,lapExp.data_state,
+       chipRun.state as chipRun_state,
+       coalesce(CONVERT(sum(chipProcessedImfile.quality != 0),SIGNED),0) AS chip_bad_quality,
+       coalesce(CONVERT(count(chipProcessedImfile.chip_id),SIGNED),0) AS chip_component_count
+       FROM lapRun JOIN lapExp USING(lap_id)
+       LEFT JOIN chipRun USING(chip_id) LEFT JOIN chipProcessedImfile USING(chip_id)
+WHERE @WHERE@ 
+       GROUP BY lap_id,exp_id
+       ) AS V1
+       LEFT JOIN camRun USING(chip_id) LEFT JOIN camProcessedExp USING(cam_id)
+       LEFT JOIN fakeRun USING(cam_id)
+       GROUP BY lap_id,exp_id
+  ) AS V2
+  LEFT JOIN warpRun USING(fake_id) LEFT JOIN warpSkyfile USING(warp_id)
+  GROUP BY lap_id,exp_id
+) AS V3
+LEFT JOIN
+  (SELECT DISTINCT diff_id,warp1,warp2 FROM diffInputSkyfile) AS DI ON
+  (DI.warp1 = warp_id OR DI.warp2 = warp_id)
 LEFT JOIN diffRun USING(diff_id)
-LEFT JOIN lapExp AS others ON (D.chip_id = others.chip_id AND D.lap_id != others.lap_id)
+LEFT JOIN diffSkyfile USING(diff_id)
 GROUP BY lap_id,exp_id
 

branches/eam_branches/ipp-20110505/ippTools/src/disttool.c

@@ -447 +447 @@
 
     if (pretend) {
-        if (!ippdbPrintMetadatas(stdout, output, "newdistRuns", true)) {
+        if (!ippdbPrintMetadatas(stdout, output, "newdistRuns", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);

branches/eam_branches/ipp-20110505/ippTools/src/laptool.c

@@ -384 +384 @@
   PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
   PXOPT_COPY_STR(config->args, where, "-filter", "filter", "==");
-  PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+  //  PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
   PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
   PXOPT_COPY_STR(config->args, where, "-fault", "fault", "==");
 
+  pxAddLabelSearchArgs(config, where, "-label", "lapRun.label", "==");
+  
   psString query = pxDataGet("laptool_pendingrun.sql");
   if (!query) {
@@ -560 +562 @@
   psMetadata *where = psMetadataAlloc();
   PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "==");
-  PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-exp_id", "lapExp.exp_id", "==");
   
   psString query = pxDataGet("laptool_exposures.sql");

branches/eam_branches/ipp-20110505/ippTools/src/laptoolConfig.c

@@ -69 +69 @@
   ADD_OPT(Str, pendingrunArgs, "-projection_cell",            "search by projection cell", NULL);
   ADD_OPT(Str, pendingrunArgs, "-filter",                     "search by filter", NULL);
-  ADD_OPT(Str, pendingrunArgs, "-label",                      "search by LAP run label", NULL);
+  //  ADD_OPT(Str, pendingrunArgs, "-label",                      "search by LAP run label", NULL);
+  psMetadataAddStr(pendingrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by LAP run label", NULL);
   ADD_OPT(Str, pendingrunArgs, "-state",                      "search by LAP run state", NULL);
   ADD_OPT(Str, pendingrunArgs, "-fault",                      "search by LAP run fault", NULL);
   ADD_OPT(Bool,pendingrunArgs, "-simple",                     "use the simple output format", false);
+
+  // -listrun
+  psMetadata *listrunArgs = psMetadataAlloc();
+  ADD_OPT(S64, listrunArgs, "-seq_id",                     "search by LAP sequence ID", 0);
+  ADD_OPT(S64, listrunArgs, "-lap_id",                     "search by LAP run ID", 0);
+  ADD_OPT(Str, listrunArgs, "-projection_cell",            "search by projection cell", NULL);
+  ADD_OPT(Str, listrunArgs, "-filter",                     "search by filter", NULL);
+  //  ADD_OPT(Str, listrunArgs, "-label",                      "search by LAP run label", NULL);
+  psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by LAP run label", NULL);
+  ADD_OPT(Str, listrunArgs, "-state",                      "search by LAP run state", NULL);
+  ADD_OPT(Str, listrunArgs, "-fault",                      "search by LAP run fault", NULL);
+  ADD_OPT(Bool,listrunArgs, "-simple",                     "use the simple output format", false);
 
   // -updaterun
@@ -128 +141 @@
   psMetadata *argSets = psMetadataAlloc();
   psMetadata *modes = psMetadataAlloc();
-
+  
   PXOPT_ADD_MODE("-definesequence",          "", LAPTOOL_MODE_DEFINESEQUENCE,   definesequenceArgs);
   PXOPT_ADD_MODE("-listsequence",            "", LAPTOOL_MODE_LISTSEQUENCE,     listsequenceArgs);
   PXOPT_ADD_MODE("-definerun",               "", LAPTOOL_MODE_DEFINERUN,        definerunArgs);
   PXOPT_ADD_MODE("-pendingrun",              "", LAPTOOL_MODE_PENDINGRUN,       pendingrunArgs);
+  PXOPT_ADD_MODE("-listrun",                 "", LAPTOOL_MODE_PENDINGRUN,       listrunArgs);
   PXOPT_ADD_MODE("-updaterun",               "", LAPTOOL_MODE_UPDATERUN,        updaterunArgs);
   PXOPT_ADD_MODE("-pendingexp",              "", LAPTOOL_MODE_PENDINGEXP,       pendingexpArgs);

branches/eam_branches/ipp-20110505/ippTools/src/pstamptool.c

@@ -843 +843 @@
     psFree(where);
 
-    psStringAppend(&query, " ORDER BY priority DESC, req_id");
+    psStringAppend(&query, " ORDER BY priority DESC, req_id, job_id");
 
     // treat limit == 0 as "no limit"
@@ -1323 +1323 @@
     psFree(where);
 
-    psStringAppend(&query, " ORDER BY priority DESC, req_id");
+    psStringAppend(&query, " ORDER BY priority DESC, req_id, dep_id");
 
     // treat limit == 0 as "no limit"

branches/eam_branches/ipp-20110505/ippTools/src/pubtoolConfig.c

@@ -48 +48 @@
     psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-product", 0, "define product (required)", NULL);
     psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
+    psMetadataAddS16(defineclientArgs, PS_LIST_TAIL, "-output_format", 0, "define output format", 2);
     psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL);
+    psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-name", 0, "define name", NULL);
     psMetadataAddBool(defineclientArgs, PS_LIST_TAIL, "-unmagicked", 0, "allow unmagicked data?", false);
 

branches/eam_branches/ipp-20110505/ippTools/src/stacktool.c

@@ -246 +246 @@
     PXOPT_COPY_STR(config->args,  where, "-select_data_group",         "warpRun.data_group", "==");
     pxAddLabelSearchArgs (config, where, "-select_label",              "warpRun.label", "LIKE"); // define using warp label
-
+    pxAddLabelSearchArgs (config, where, "-warp_id",                   "warpRun.warp_id", "==");
+    
     // these are used to build the HAVING restriction
     PXOPT_COPY_S32(config->args, having, "-min_num", "num_warp", ">=");
@@ -508 +509 @@
           association->sass_id = sass_id;
         }
+        else {
+          sass_id = association->sass_id;
+        }
         // Insert the map entry for this row.
         stackAssociationMapRow *maprow = stackAssociationMapRowAlloc(sass_id,stack_id);

branches/eam_branches/ipp-20110505/ippTools/src/stacktoolConfig.c

@@ -91 +91 @@
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_zpt_obs_max", 0, "define max zero point", NAN);
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_astrom", 0, "define max astrometry rms", NAN);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", PS_META_DUPLICATE_OK, "include this warp ID (multiple OK)", 0);
     psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0);
     psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_num", 0, "minimum number of inputs", 0);

branches/eam_branches/ipp-20110505/ppStack/src/ppStackPrepare.c

@@ -144 +144 @@
     int numCols = 0, numRows = 0;   // Size of image
     options->sumExposure = 0.0;
+    int numWithSources = 0;
     for (int i = 0; i < num; i++) {
         pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); // File of interest
@@ -188 +189 @@
             pmReadout *ro = pmFPAviewThisReadout(view, file->fpa); // Readout with sources
             detections = psMetadataLookupPtr(NULL, ro->analysis, "PSPHOT.DETECTIONS"); // Sources
-            if (!detections || !detections->allSources) {
+            if (!detections || !detections->allSources || !detections->allSources->n) {
                 psWarning("No detections found for image %d --- rejecting.", i);
                 options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_CAL;
@@ -196 +197 @@
 
             options->sourceLists->data[i] = psMemIncrRefCounter(detections->allSources);
+            numWithSources++;
         }
 
@@ -230 +232 @@
             ppStackFileActivation(config, PPSTACK_FILES_PREPARE, true);
         }
+    }
+    if (numWithSources < 2) {
+        // This can happen if the inputs have been destreaked
+        psErrorStackPrint(stderr, "Not enough inputs have sources");
+        psWarning("No inputs have sources --- suspect bad data quality.");
+        if (options->quality == 0) {
+            options->quality = PPSTACK_ERR_DATA;
+        }
+        psErrorClear();
+        psFree(view);
+        return false;
     }
 
@@ -308 +321 @@
         psFree(psfs);
         if (!options->psf) {
+#if 1
             psError(psErrorCodeLast(), false, "Unable to determine output PSF.");
+#else
+            // This will repair the problem reproted in ticket 1427 but we aren't yet sure
+            // why ppStackPSF is failing so we are going to continue to fault for now
+            int errorCode = psErrorCodeLast();
+            if (errorCode == PPSTACK_ERR_PSF) {
+                psErrorStackPrint(stderr, "Unable to determine output PSF.");
+                psWarning("Unable to determine output PSF --- suspect bad data quality.");
+                if (options->quality == 0) {
+                    options->quality = errorCode;
+                }
+                psErrorClear();
+            } else {
+                psError(psErrorCodeLast(), false, "Unable to determine output PSF.");
+            }
+#endif // notyet
             psFree(view);
             return false;

branches/eam_branches/ipp-20110505/psModules/src/imcombine/pmSubtractionStamps.c

@@ -834 +834 @@
             }
 
-            psStatsInit (stats);
-            if (!psVectorStats (stats, flux1, NULL, NULL, 0)) {
-                psAbort ("failed to generate stats");
-            }
-            float f1 = stats->sampleMedian;
-
-            psStatsInit (stats);
-            if (!psVectorStats (stats, flux2, NULL, NULL, 0)) {
-                psAbort ("failed to generate stats");
-            }
-            float f2 = stats->sampleMedian;
+            float f1 = NAN;
+            if (flux1->n > 0) {
+                psStatsInit (stats);
+                if (!psVectorStats (stats, flux1, NULL, NULL, 0)) {
+                    psAbort ("failed to generate stats");
+                }
+                f1 = stats->sampleMedian;
+            }
+
+            float f2 = NAN;
+            if (flux2->n > 0) {
+                psStatsInit (stats);
+                if (!psVectorStats (stats, flux2, NULL, NULL, 0)) {
+                    psAbort ("failed to generate stats");
+                }
+                f2 = stats->sampleMedian;
+            }
 
             stamps->window1->kernel[y][x] = f1;
@@ -893 +899 @@
     float R2 = Sr2 / Sf2;
 
+    if (!isfinite(R1) || !isfinite(R2)) {
+        psError(PM_ERR_STAMPS, true, "Kron Radii are not finite (failure to converge).");
+        psFree (stats);
+        psFree (flux1);
+        psFree (flux2);
+        psFree (norm1);
+        psFree (norm2);
+        return false;
+    }
+
     // Compare the Kron Radii (R1 & R2) to above to the FWHMs : if they are too discrepant, we will need to rescale
     psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "Kron Radii vs FWHMs 1: fwhm: %f, kron %f\n", fwhm1, R1);
@@ -901 +917 @@
     stamps->normWindow2 = 2.75*R2;
     psLogMsg ("psModules.imcombine", PS_LOG_DETAIL, "Windows from Kron Radii: %f for 1, %f for 2\n", stamps->normWindow1, stamps->normWindow2);
+
 
     // if the calculated normWindows are too large, we will fall off the stamps.  In this case, we need to try again.
@@ -913 +930 @@
         psFree (norm2);
         return false; 
-    }
-
-    if (!isfinite(R1) || !isfinite(R2)) {
-        psError(PM_ERR_STAMPS, true, "Kron Radii are not finite (failure to converge).");
-        psFree (stats);
-        psFree (flux1);
-        psFree (flux2);
-        psFree (norm1);
-        psFree (norm2);
-        return false;
     }
 

branches/eam_branches/ipp-20110505/psconfig/psbuild

@@ -386 +386 @@
     if (@ARGV != 2) { die "USAGE: psbuild -bootstrap (install_dir)\n"; }
     $psconfdir = $ARGV[1];
+    die "Target directory must be absolute, not relative: $psconfdir\n" unless $psconfdir =~ m|^/|;
 
     # copy psconfig.csh and psconfig.bash to psconfdir

branches/eam_branches/ipp-20110505/pstamp/scripts/pstamp_job_run.pl

@@ -146 +146 @@
             # user required uncensored but since stage isn't chip we can't rebuild them
             my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE, 'stop');
-        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} eq 'update') and ($stage ne 'chip')) {
+        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} ne 'full') and ($stage ne 'chip')) {
             # we can only restore pixels for chip stage images if the data has been updated.
-            # XXX: this test is not quite good enough. If all components have been updated then the
-            # state will be 'full' But this will get us going.
-            print "Run state is update: will make stamps from destreaked $stage images.\n";
+            # the data will have been updated if the params->{state} the state when the job was queued is not 'full' (
+            # XXX: we should probably be looking explicitly at the job and checking for a dep_id
+            print "Run state was $params->{state}: will make stamps from destreaked $stage images.\n";
             # make stamps from uncensored images
             $muggle = 0;

branches/eam_branches/ipp-20110505/pstamp/scripts/pstampparse.pl

@@ -754 +754 @@
     if ( $num_jobs == 0 ) {
         print STDERR "no jobs for row $rownum\n" if $verbose;
-        insertFakeJobForRow($row, 1, $PSTAMP_NO_JOBS_QUEUED);
+        insertFakeJobForRow($row, 1, $PSTAMP_NO_IMAGE_MATCH);
         $num_jobs = 1;
+        $row->{job_num} = 1;
     }
     return $num_jobs;