Changeset 32630 for branches/eam_branches/ipp-20110906/ippScripts

Timestamp:

Nov 8, 2011, 2:44:12 PM (15 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20110906

Files:

• . (modified) (1 prop)
• ippScripts/scripts/camera_exp.pl (modified) (2 diffs)
• ippScripts/scripts/chip_imfile.pl (modified) (9 diffs)
• ippScripts/scripts/diff_skycell.pl (modified) (4 diffs)
• ippScripts/scripts/dist_bundle.pl (modified) (1 diff)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/lap_science.pl (modified) (22 diffs)
• ippScripts/scripts/magic_destreak.pl (modified) (8 diffs)
• ippScripts/scripts/nightly_science.pl (modified) (1 diff)
• ippScripts/scripts/register_exp.pl (modified) (1 diff)
• ippScripts/scripts/register_imfile.pl (modified) (3 diffs)
• ippScripts/scripts/stack_skycell.pl (modified) (3 diffs)
• ippScripts/scripts/videophot_process.pl (modified) (10 diffs)
• ippScripts/scripts/warp_overlap.pl (modified) (1 diff)
• ippScripts/scripts/warp_skycell.pl (modified) (8 diffs)

branches/eam_branches/ipp-20110906/ippScripts/scripts/camera_exp.pl

@@ -88 +88 @@
 
 if ($redirect) {
-    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $cam_id, $PS_EXIT_SYS_ERROR );
+    $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $cam_id, $PS_EXIT_SYS_ERROR );
     print "\n\n";
     print "Starting script $0 on $host\n\n";
@@ -404 +404 @@
         $ipprc->file_exists($file);
 
+    # Funpack to confirm we've really made things correctly
+    my $diskfile = $ipprc->file_resolve($file);
+    if ($diskfile =~ /fits/) {
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack",  $cam_id, $PS_EXIT_SYS_ERROR);
+        my $check_command = "$funpack -S $diskfile > /dev/null";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $check_command, verbose => $verbose);
+        if (!$success) {
+            &my_die("Output file not a valid fits file: $file",  $cam_id, $PS_EXIT_SYS_ERROR);
+        }
+    }
+    #####
+
+
     my $scheme = file_scheme($file);
     if ($replicate and $scheme and (file_scheme($file) eq 'neb')) {

branches/eam_branches/ipp-20110906/ippScripts/scripts/chip_imfile.pl

@@ -34 +34 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1);
+
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -44 +46 @@
 my ( $exp_id, $chip_id, $class_id, $chip_imfile_id, $uri, $camera, $outroot, $dbname, $run_state, $reduction, $threads, $verbose,
      $no_update, $save_temps, $no_op, $redirect, $magicked, $deburned );
-
-my $zaplog = 0;
 
 GetOptions(
@@ -65 +65 @@
     'no-op'             => \$no_op,     # Don't do any operations?
     'redirect-output'   => \$redirect,
-    'zaplog'            => \$zaplog,
     'save-temps'        => \$save_temps, # Save temporary files?
     ) or pod2usage( 2 );
@@ -92 +91 @@
 }
 
-my ($logDest, $traceDest);
+my ($logRule, $traceDest);
 if ($run_state eq 'new') {
-    $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, $zaplog);
+    $logRule = "LOG.IMFILE";
     $traceDest = prepare_output("TRACE.IMFILE",  $outroot, $class_id, 1);
 } else {
-    $logDest = prepare_output("LOG.IMFILE.UPDATE", $outroot, $class_id, 1);
+    $logRule = "LOG.IMFILE.UPDATE";
     $traceDest = prepare_output("TRACE.IMFILE.UPDATE",  $outroot, $class_id, 1);
 }
 
 if ($redirect) {
-    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR );
+    my $logDest = $ipprc->filename($logRule, $outroot, $class_id);
+
+    $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR );
+
     print STDOUT "\n\n";
     print STDOUT "Starting script $0 on $host\n\n";
@@ -330 +332 @@
         # get the UNIX version of the (possible) neb: or path: filename
         my $uriReal = $ipprc->file_resolve( $uri );
+        # Catch errors here
+        if (!$uriReal) {
+            my $repair_cmd = "$nebrepair $uri";
+            my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+            unless ($repair_success) {
+                &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+            $uriReal = $ipprc->file_resolve( $uri );
+        }
+            
         &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
 
@@ -336 +348 @@
         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
         unless ($success) {
+            # Catch errors here
+            my $repair_cmd = "$nebrepair $uri";
+            my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+            unless ($repair_success) {
+                &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
+        }
+        unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
             &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
@@ -348 +369 @@
             $burntoolTable_uri =~ s/fits$/burn.tbl/;
             $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+            if ((!$burntoolTable_uriReal)||(!($ipprc->file_exists($burntoolTable_uri)))) {
+                my $repair_cmd = "$nebrepair $burntoolTable_uri";
+                my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+                unless ($repair_success) {
+                    &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+                }
+                $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+            }
             unless ($ipprc->file_exists($burntoolTable_uri)) {
+                # Catch errors here
                 &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR);
             }
@@ -451 +481 @@
     $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
     $command .= " -source_id $source_id" if defined $source_id;
-    $command .= " -tracedest $traceDest -log $logDest";
+    $command .= " -tracedest $traceDest";
     $command .= " -dbname $dbname" if defined $dbname;
 
@@ -727 +757 @@
         &my_die("Couldn't find expected output file: $file",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     }
+    # Funpack to confirm we've really made things correctly
+    my $diskfile = $ipprc->file_resolve($file);
+    if ($diskfile =~ /fits/) {
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        my $check_command = "$funpack -S $diskfile > /dev/null";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $check_command, verbose => $verbose);
+        if (!$success) {
+            &my_die("Output file not a valid fits file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        }
+    }
+    #####
 
     if ($replicate and $neb) {

branches/eam_branches/ipp-20110906/ippScripts/scripts/diff_skycell.pl

@@ -40 +40 @@
 my ($skycell_id, $diff_skyfile_id);
 my ($use_convolved);
-my $zaplog = 0;
 
 GetOptions(
@@ -57 +56 @@
     'no-op'             => \$no_op, # Don't do any operations?
     'redirect-output'   => \$redirect,
-    'zaplog'            => \$zaplog, # delete any previous log file
 ) or pod2usage( 2 );
 
@@ -79 +77 @@
 
 # XXX camera is not known here; cannot use filerules... 
-# bills 2011-03-07 This is unfortunate because it prevents us from using prepare_output ...
-# my $logDest = $ipprc->filename("LOG.EXP", $outroot);
 my $logDest = "$outroot.log";
 if ($run_state eq 'update') {
     $logDest .= '.update';
-    # ... well we can just do the dirty work here
-    $ipprc->kill_file($logDest);
-} elsif ($zaplog) {
-    $ipprc->kill_file($logDest);
-}
-$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
+}
+$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 
 my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF);
@@ -468 +460 @@
     &my_die("Couldn't find expected output file: $file",  $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
 
+    # Funpack to confirm we've really made things correctly
+    my $diskfile = $ipprc->file_resolve($file);
+    if ($diskfile =~ /fits/) {
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR);
+        my $check_command = "$funpack -S $diskfile > /dev/null";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $check_command, verbose => $verbose);
+        if (!$success) {
+            &my_die("Output file not a valid fits file: $file", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR);
+        }
+    }
+    #####
+
+
     if ($replicate and $neb) {
         $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR);

branches/eam_branches/ipp-20110906/ippScripts/scripts/dist_bundle.pl

@@ -636 +636 @@
         $file{file_rule} = $key;
         $file{name} = $val;
+        if ($val eq "STDERR" or $val eq "STDOUT") {
+            print STDERR "Skipping $key because filename is $val\n";
+            next;
+        }
         push @file_list, \%file;
     }

branches/eam_branches/ipp-20110906/ippScripts/scripts/lap_science.pl

@@ -27 +27 @@
 my ( $camera, $dbname);
 my ( $lap_id );
+my ( $queue_list );
 my ( $chip_mode, $monitor_mode, $cleanup_mode);
 
@@ -44 +45 @@
     
     'lap_id=s'     => \$lap_id,
+    'queue_list=s' => \$queue_list,
 
     'chip_mode'    => \$chip_mode,
@@ -200 +202 @@
     return($exposure);
 }
+
+sub remake_this_exposure_by_update {
+    my $exposure = shift;
+
+    my @utctime = gmtime();
+    $utctime[5] += 1900;
+    $utctime[4] += 1;
+
+    my $label = $exposure->{label};
+
+    my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
+    my $data_group = "${label}.${date}";
+
+    my $chiptool_info_cmd = "chiptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} ";
+    $chiptool_info_cmd   .= " -dbname $dbname " if defined $dbname;
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $chiptool_info_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform chiptool -listrun: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    my $chips = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata from chiptool -listrun", $exposure->{exp_id}, $data_group);
+    # There should be only one.
+    my $chip = ${ $chips }[0];
+    my $chip_magicDS_id = 0;
+    if ($chip) {
+        $chip_magicDS_id = $chip->{magic_ds_id};
+    }
+    if ($chip_magicDS_id == 0) {
+#       return(&remake_this_exposure($exposure));
+    }
+
+    my $warptool_info_cmd = "warptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} ";
+    $warptool_info_cmd   .= " -dbname $dbname " if defined $dbname;
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $warptool_info_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform warptool -listrun: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    my $warps = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata from warptool -listrun", $exposure->{exp_id}, $data_group);
+    # There should be only one.
+    my $warp = ${ $warps }[0];
+    my $warp_id = 0;
+    if ($warp) {
+        $warp_id = $warp->{warp_id};
+    }
+    else {
+        $exposure->{data_state} = 'drop';
+        return($exposure);
+    }   
+    if ($warp_id == 0) {
+        # Handle this correctly. Probably want to drop the exposure here.
+        $exposure->{data_state} = 'drop';
+        return($exposure);
+    }
+
+    if (($chip->{state} eq 'goto_cleaned')||
+        ($warp->{state} eq 'goto_cleaned')||
+        ($chip->{dsRun_state} eq 'goto_cleaned')) {
+        $exposure->{data_state} = 'pending_update';
+        &update_this_exposure($exposure);
+        return($exposure);
+    }
+
+    
+    my $chiptool_update_cmd = "chiptool -setimfiletoupdate -chip_id $exposure->{chip_id} -set_label $label";
+    $chiptool_update_cmd   .= " -dbname $dbname " if defined $dbname;
+    my $magicDS_update_cmd  = "magicdstool -setfiletoupdate -magic_ds_id $chip_magicDS_id -set_label $label";
+    $magicDS_update_cmd    .= " -dbname $dbname " if defined $dbname;
+    my $warptool_update_cmd = "warptool -setskyfiletoupdate -warp_id $warp_id -set_label $label";
+    $warptool_update_cmd   .= " -dbname $dbname " if defined $dbname;
+
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $chiptool_update_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform chiptool -setimfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    if ($chip_magicDS_id != 0) {    
+        ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $magicDS_update_cmd, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
+        }
+    }
+
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $warptool_update_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform warptool -setskyfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    
+    $exposure->{active} = 1;
+    return($exposure);
+}
+        
+sub determine_if_can_update {
+    my $exposure = shift;
+    
+    if (S64_IS_NOT_NULL($exposure->{chip_id})) {
+        if (($exposure->{chip_state} eq 'cleaned')||
+            ($exposure->{chip_state} eq 'goto_cleaned')||
+            ($exposure->{chip_state} eq 'error_cleaned')) {
+            return(1);
+        }
+    }
+
+    return(0);
+}
+        
 
 # This is the "user level" subroutine.
@@ -240 +360 @@
         my $comment = $exposure->{comment};
 
-        # This is a hack to fix old exposures that have no object.
+        my $updateable = determine_if_can_update($exposure);
+
+        # This is a hack to fix old exposures that have no usable object/comment data.
         unless(defined($comment)) {
             $comment = '';
@@ -247 +369 @@
             if ($comment =~ /3pi_/) {
                 $object = $comment;
-                $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/$1/;
+                $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/F1 $1/;
             }
             elsif ($comment =~ / ps1_/) {
                 $object = $comment;
-                $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/$1/;
-            }
-            else {
-                $exposure->{data_state} = 'new';
+                $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/F2 $1/;
+            }
+            elsif ($comment =~ /ThreePi.*3pi_/) {
+                $object = $comment;
+                $object =~ s/^ThreePi . (\S+? 3pi_\d\d_\d\d\d\d) .*$/F3 $1/;
+            }           
+            elsif ($comment =~ /ThreePi.*ps1_/) {
+                $object = $comment;
+                $object =~ s/^ThreePi . (\S+? ps1_\d\d_\d\d\d\d) .*$/F4 $1/;
+            }           
+            elsif ($comment =~ /ThreePi /) {
+                $object = $comment;
+                $object =~ s/^ThreePi . (\S+? \d\d\d\d) .*$/F5 $1/;
+            }           
+            elsif (($comment =~ /focus/i)||
+                   ($comment =~ /test/i)||
+                   ($comment =~ /hyster/i)||
+                   ($comment =~ /dither/i)||
+                   ($comment =~ /camera/i)
+                ){
+                # This is junk that shouldn't exist.
+                $object = 'DROP';
+                $exposure->{data_state} = 'drop';
                 $exposure->{pairwise} = 0;
-                $exposure->{private} = 1;
-                $exposure->{pair_id} = 9223372036854775807;
-                update_this_exposure($exposure);
-                $exposure = remake_this_exposure($exposure); 
-                $counter++;
+                $exposure->{pair_id} =  9223372036854775807;
+                $exposure->{active} = 0;
+                &update_this_exposure($exposure);
+                $counter++; # To ensure everyone else is consistent
                 next;
             }
         }
 
+        # Determine the current state of chipRuns for these exposures, and update/remake as needed.
         if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id
             if (($pairwise) && !($pair_id)) {
@@ -270 +411 @@
                 &my_die("Exposure $exp_id for $lap_id is declared pairwise without a defined pair", $lap_id);
             }
+            $exposure->{data_state} = 'exists';
+            if ($updateable) { # We know about this run, but the data needs to be regenerated.
+                $exposure->{data_state} = 'update';
+                $exposure = remake_this_exposure_by_update($exposure);
+            }
+            &update_this_exposure($exposure);
             $matching{$object}{$comment} = $exp_id;
             $indexing{$exp_id} = $counter;
             $counter++; 
-            next;
         }
         else { # We do not already have a chip_id.  
             # Make a chipRun, and update the exposure.
+            $exposure->{data_state} = 'run';
             $exposure = remake_this_exposure($exposure); 
-            
             # Save our information for diff pairing.
+            &update_this_exposure($exposure);
             $matching{$object}{$comment} = $exp_id;
             $indexing{$exp_id} = $counter;
             $counter++;
         }
+
         if ($verbose) {
             print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n";
@@ -296 +444 @@
             print "$object $comment $matching{$object}{$comment} $indexing{$matching{$object}{$comment}} $exp_ids_to_diff[-1]\n";
         }
-        @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff;
+        @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; # This is effectively a sort by dateobs.
         
         if (( $#exp_ids_to_diff + 1) % 2 != 0) { # We have an odd number of exposures, even after comment filtering
@@ -309 +457 @@
             my $exp_B = ${ $exposures }[$indexing{$exp_id_B}];
             print "$exp_A $exp_B $exp_id_A $exp_id_B $indexing{$exp_id_A} $indexing{$exp_id_B}\n";
-            $exp_A->{pairwise} = 1;
-            $exp_A->{private} = 0;
-            $exp_A->{pair_id} = $exp_B->{chip_id};
-            
-            $exp_B->{pairwise} = 1;
-            $exp_B->{private} = 0;
-            $exp_B->{pair_id} = $exp_A->{chip_id};
-            
-            if ($verbose) {
-                print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n";
+            if ($exp_A->{diff_id} == $exp_B->{diff_id}) {
+                $exp_A->{pairwise} = 1;
+                $exp_A->{pair_id} = $exp_B->{chip_id};
+                
+                $exp_B->{pairwise} = 1;
+                $exp_B->{pair_id} = $exp_A->{chip_id};
+                if ($verbose) {
+                    print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n";
+                }
+            }
+            else {
+                $exp_A->{pairwise} = 0;
+                $exp_B->{pairwise} = 0;
             }
         }
@@ -331 +482 @@
             $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error.
         }
-        if (!($exposure->{pairwise})) {
-            $exposure->{private} = 1; # If this isn't being pairwise diffed, it needs to be private
-        }
-
+        
         update_this_exposure($exposure);
     }
@@ -457 +605 @@
     # Things I want to know before I'm through
 #     my $needs_qstack = 0;
-    my $needs_something_remade = 0;
-    my $needs_something_private = 0;
+    my $needs_something_remade = 0;  # I never need something remade, because we're committed to one-exp=one-chip
+    my $needs_something_private = 0; # I never need something private, as that is calculated by the difference engine.
 #     my $can_qstack = 0;
      my $have_diff = 0;
@@ -480 +628 @@
             else { # We claimed to be pairwise, but do not have a valid pair_id.
                 $exposure->{pairwise} = 0;
-                $exposure->{private} = 1;
                 &update_this_exposure($exposure);
             }
@@ -487 +634 @@
         if  ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through.
             $are_warped++;
-#           $can_qstack ++;
             $can_diff ++;
             $have_diff ++;
             $are_magicked ++;
-#           $can_fstack ++;         
+            next;
+        }
+
+        if ($exposure->{data_state} eq 'pending_update') {
+            $exposure->{data_state} = 'update';
+            $exposure = &remake_this_exposure_by_update($exposure);
+            &update_this_exposure($exposure);
             next;
         }
         
         if ($exposure->{private}) { # I've declared this exposure private to this lapRun.
-#           $needs_qstack++;
             push @lonely_exposures, $exposure;
-#           $lonely_exposures++;
         }
         
-        if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun
-            $needs_something_remade = 1;
-            $exposure = remake_this_exposure($exposure);
-        }
+#       if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun
+#           $needs_something_remade = 1;
+#           $exposure = remake_this_exposure($exposure);
+#       }
         # Do quality checks here
         my $is_bad_quality = 0;
@@ -536 +686 @@
                     ($exposure->{diffRun_state} eq 'full')) {
                 $are_warped++;
-#               $needs_qstack++;
-                $needs_something_private = 1;
                 if ($companion) {
-                    $companion->{private} = 1;
                     $companion->{pairwise} = 0;
                     &update_this_exposure($companion);
-                    push @lonely_exposures, $companion;
-#                   $lonely_exposures++;
-                }
-                $exposure->{private} = 1;
+#                   push @lonely_exposures, $companion;
+                }
                 $exposure->{pairwise} = 0;
             }
             $exposure->{data_state} = 'drop';
             &update_this_exposure($exposure);
+            next;
         }
         
@@ -558 +704 @@
 #           $can_qstack ++;
             $can_diff ++;
-        }
-        if (($exposure->{magicked}&&
-             &S64_IS_NOT_NULL($exposure->{magicked}))) {  # This exposure has been magicked, so it is through with diff.
-            $are_magicked++;
-#           $can_fstack ++;
+            $exposure->{data_state} = 'to_diff';
         }
         if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) {
             $have_diff ++;
-        }
+            $exposure->{data_state} = 'to_magic';
+        }
+        if (($exposure->{magicked})&&
+            ($exposure->{warpRun_state})&&
+            ($exposure->{warpRun_state} eq 'full')&&
+            (&S64_IS_NOT_NULL($exposure->{magicked}))) {  # This exposure has been magicked, so it is through with diff.
+            $are_magicked++;
+            $exposure->{data_state} = 'full';
+#           $can_fstack ++;
+        }
+        &update_this_exposure($exposure);
     }
 
@@ -575 +727 @@
     print "STATUS: HAVE_FSTACK:     $have_fstack\n";
 
-#    print "STATUS: NEEDS_QSTACK:    $needs_qstack\n";
     print "STATUS: NEEDS_REMADE:    $needs_something_remade\n";
     print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n";
     print "STATUS: ARE WARPED:      $are_warped\n";
     print "STATUS: ARE MAGICKED:    $are_magicked\n";
-#    print "STATUS: CAN_QSTACK:      $can_qstack\n";
     print "STATUS: CAN_DIFF:        $can_diff\n";
     print "STATUS: HAVE_DIFF:       $have_diff\n";
-#    print "STATUS: CAN_FSTACK:      $can_fstack\n";
 
     print "STATUS: TOTAL_EXPOSURES: $total_exposures\n";
@@ -924 +1073 @@
             next;
         }
-        if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # Not sure how this would happen, but still. ## This happens when we inherit a complete exposure.
+        if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # This happens when we inherit a complete exposure.
             next;
         }
@@ -964 +1113 @@
         }
         else { # warp-qstack
+            # We need to decide if we can make a warp-stack diff:
+            if (&can_warp_stack_diff_be_made($exposure)) {
 #           next; 
 #           $command .= '-pretend';
-            $command .= " -definewarpstack -available -good_frac 0.2 ";
-            $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} ";
-            $already_queued{$exposure->{warp_id}} = 1;
+                $command .= " -definewarpstack -available -good_frac 0.2 ";
+                $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} ";
+                $already_queued{$exposure->{warp_id}} = 1;
+                $exposure->{private} = 0;
+                &update_this_exposure($exposure);
+            }
+            else {
+                $exposure->{private} = 1;
+                $exposure->{data_state} = 'drop';
+                $already_queued{$exposure->{warp_id}} = 1;
+                &update_this_exposure($exposure);
+                next;
+            }
         }
         
@@ -1006 +1167 @@
 }
 
+sub can_warp_stack_diff_be_made {
+    my $exposure = shift;
+    my $lap_id   = $exposure->{lap_id};
+    my $warp_id  = $exposure->{warp_id};
+
+    my $command = "$laptool -diffcheck -lap_id $lap_id -warp_id $warp_id";
+    $command .= " -dbname $dbname " if defined $dbname;
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform laptool -diffcheck: $error_code", $lap_id, $warp_id);
+    }
+    if (@$stdout_buf == 0) {
+        return(0);
+    }
+    my $skycells = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata from laptool -diffcheck", $lap_id, $warp_id);
+    foreach my $skycell (@$skycells) {
+        unless (defined($skycell->{stack_state})) {
+            return(0);
+        }
+        if ($skycell->{stack_state} ne 'full') {
+            return(0);
+        }
+        if ($skycell->{warp_state} ne 'full') {
+            return(0);
+        }
+        unless (S64_IS_NOT_NULL($skycell->{stack_id})) {
+            return(0);
+        }
+    }
+    return(1);
+}
+
+
 # Deactivate all exposures in this run.
 sub deactivate_exposures {
     my $exposures = shift;
     foreach my $exposure (@$exposures) {
-        $exposure->{active} = 0;
-        update_this_exposure($exposure);
+        if ($exposure->{private} == 1) {  # This can probably be relaxed since we can update an undestreaked warp.
+            $exposure->{active} = 1;
+        }
+        else {
+            $exposure->{active} = 0;
+        }
+        update_this_exposure($exposure);        
     }
 }
@@ -1022 +1225 @@
 sub cleanup_mode {
     my $lap_id = shift;
+
+    if (defined($queue_list)) {
+        my $successful = 1;
+        open(Q,$queue_list) or ($successful = 0);
+        if ($successful == 1) {
+            while(<Q>) {
+                chomp;
+                my $cmd = $_;
+                my ($projection_cell,$seq_id,$filter,$label);
+                $projection_cell = $cmd;
+                $projection_cell =~ s/.*?-projection_cell (skycell.\w+?) .*/$1/;
+                $seq_id = $cmd;
+                $seq_id =~ s/.*-seq_id (\d+?) .*/$1/;
+                $filter = $cmd;
+                $filter =~ s/.*-filter (\w\.00000) .*/$1/;
+                if ($filter =~ /\s+?/) {
+                    $filter =~ s/.*-filter (\w\.00000)$/$1/;
+                }
+                $label = $cmd;
+                $label =~ s/.*-label (.+?) .*/$1/;
+                
+                my $response;
+                chomp($response = 
+                      `laptool -dbname gpc1 -listrun -projection_cell $projection_cell -seq_id $seq_id -filter $filter -label $label -simple`);
+                my $state = (split /\s+/, $response)[5];
+                
+                unless (defined($state)) {
+                    if ($verbose) {
+                        print "Queuing: $cmd\n";
+                    }
+                    system($cmd);
+                    
+                    my $i = 0;
+                    do {
+                        sleep(5);
+                        chomp($response = 
+                              `laptool -dbname gpc1 -listrun -projection_cell $projection_cell -seq_id $seq_id -filter $filter -label $label -simple`);
+                        $state = (split /\s+/, $response)[5];
+                        $i++;
+                        unless(defined($state)) {
+                            $successful = 0;
+                        }
+                    } while (($state ne 'run')&&($i < 20)&&($successful));
+                    last;
+                }
+            }
+            close(Q);
+        }
+    }
+
     my $command = "$laptool -inactiveexp -lap_id $lap_id ";
     $command .= " -dbname $dbname " if defined $dbname;
@@ -1043 +1296 @@
         'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@',
         'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@',
+        'magictool -dbname @DBNAME@ -updaterun -set_state full -set_label @LABEL@.old -magic_id @MAGIC_ID@',
         'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage chip -stage_id @CHIP_ID@',
         'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage warp -stage_id @WARP_ID@',
         'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage diff -stage_id @DIFF_ID@');
+    
     foreach my $exposure (@$exposures) {
         if ($exposure->{is_in_use}) {
@@ -1072 +1327 @@
                 if (S64_IS_NOT_NULL($exposure->{diff_id})) {
                     $command =~ s/\@DIFF_ID\@/$exposure->{diff_id}/;
+                }
+                else {
+                    next;
+                }
+            }
+            if ($command =~ /\@MAGIC_ID\@/) {
+                if ((S64_IS_NOT_NULL($exposure->{magicked}))&&
+                     ($exposure->{magicked} > 0)) {
+                    $command =~ s/\@MAGIC_ID\@/$exposure->{magicked}/;
                 }
                 else {
@@ -1139 +1403 @@
         &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id);
     }
-
+    return(0);
 }
     

branches/eam_branches/ipp-20110906/ippScripts/scripts/magic_destreak.pl

@@ -35 +35 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $dvoImageOverlaps = can_run('dvoImageOverlaps') or (warn "Can't find dvoImageOverlaps" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -45 +46 @@
 my ($outroot, $recoveryroot, $magicked);
 my ($replace, $release);
+my ($diff_tess_id, $mismatched_tess);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
@@ -60 +62 @@
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
            'component=s'    => \$component,  # the class_id or skycell_id
+           'mismatched_tess'=> \$mismatched_tess, # true if tess_id of input does not match tess_id of the diff used for magic
+           'diff_tess_id=s' => \$diff_tess_id, # tess_id of diffRun used to compute the streaks
            'uri=s'          => \$uri,        # uri of the input image
            'path_base=s'    => \$path_base,  # path_base of the input
@@ -106 +110 @@
 if (($stage eq "raw") or ($stage eq "chip") or $stage eq "chip_bg") {
     $class_id = $component;
-    $skycell_args = " -class_id $component";
+    $skycell_args = " -class_id";
 } elsif ($stage eq "warp" or $stage eq "warp_bg") {
     $skycell_id = $component;
-    $skycell_args = " -skycell_id $component";
+    $skycell_args = " -skycell_id";
 } elsif ($stage eq "diff") {
     $skycell_id = $component;
@@ -215 +219 @@
     my ($sfh, $skycell_list);
     if ($skycell_args) {
-        my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells $skycell_args";
-        $command .= " -dbname $dbname" if defined $dbname;
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
-        }
-
-        my $getskycells_output = join "", @$stdout_buf;
-        if ($getskycells_output) {
-            my $metadata = $mdcParser->parse($getskycells_output) or
-                &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
-
-            my $skycells = parse_md_list($metadata) or
-                    &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
-
-            ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
-
-            foreach my $skycell (@$skycells) {
-                my $skycell_uri;
-                if ($skycell->{data_state} eq "full" and $skycell->{quality} = 0) {
-                    $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
-                } else {
-                    # diff run must have been cleaned up, need to create this skycell file on the fly
-                    if (!defined $temp_dir ) {
-                        $temp_dir = tempdir( CLEANUP => !$save_temps);
+        my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells";
+        $command .= " $skycell_args";
+        my @diff_components;
+        if ($mismatched_tess) {
+            # tessellation for this skycell does not match the tessellation used for the magic analysis
+            # used to calculate the streaks file
+            @diff_components = get_overlaps($path_base, $component, $diff_tess_id);
+        } else {
+            @diff_components = ($component);
+        }
+
+        # hash of uris by skycell.
+        my %diff_skycells;
+        foreach my $component (@diff_components) {
+            my $this_command = "$command $component";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $this_command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
+            }
+
+            my $getskycells_output = join "", @$stdout_buf;
+            if ($getskycells_output) {
+                my $metadata = $mdcParser->parse($getskycells_output) or
+                    &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+                my $skycells = parse_md_list($metadata) or
+                        &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+                foreach my $skycell (@$skycells) {
+                    my $skycell_uri;
+                    if ($skycell->{data_state} eq "full" and $skycell->{quality} == 0) {
+                        $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                        $diff_skycells{$skycell_id} = $skycell_uri;
+                    } else {
+                        # diff run must have been cleaned up, need to create this skycell file on the fly
+                        if (!defined $temp_dir ) {
+                            $temp_dir = tempdir( CLEANUP => !$save_temps);
+                        }
+                        my $skycell_id = $skycell->{skycell_id};
+                        if (!$diff_skycells{$skycell_id}) {
+                            $skycell_uri = "$temp_dir/$skycell_id";
+                            $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
+                                &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+                        }
+                        $diff_skycells{$skycell_id} = $skycell_uri;
                     }
-                    my $skycell_id = $skycell->{skycell_id};
-                    $skycell_uri = "$temp_dir/$skycell_id";
-                    $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
-                        &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
                 }
-                print $sfh "$skycell_uri\n";
-            }
-            close $sfh
-        }
+            }
+        }
+        # write the skycell list file
+        ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
+        foreach my $skycell_id (keys %diff_skycells) {
+            print $sfh $diff_skycells{$skycell_id} . "\n";
+        }
+        close $sfh
     }
 
@@ -360 +384 @@
         $command .= " -chip_mask $ch_mask" if defined $ch_mask;
         $command .= " -weight $weight" if defined $weight;
-        if ((defined($sources))&&($ipprc->file_exists($sources))) {
-            $command .= " -sources $sources" if defined $sources;
-        }
-        else {
-            print "Did not add sources because they do not appear to exist. This may be an error.\n";
-        }
+        $command .= " -sources $sources" if defined $sources;
         $command .= " -skycelllist $skycell_list" if defined $skycell_list;
         $command .= " -replace" if $replace;
@@ -475 +494 @@
 
 my $statsFlags;
-if ($stage ne "camera") {
+if (!$no_op and $stage ne "camera") {
     file_check($statsFile);
     {
@@ -595 +614 @@
 }
 
+            # @diff_components = get_overlaps($path_base, $component, $tess_id, $diff_tess_id);
+sub get_overlaps {
+    my $path_base = shift;
+    my $component = shift;
+    my $diff_tess_id = shift;
+
+    my $tess_dir = $ipprc->tessellation_catdir( $diff_tess_id ); 
+    my $catdir = $ipprc->convert_filename_absolute($tess_dir);
+
+    my $cmf = $ipprc->file_resolve("$path_base.cmf");
+    &my_die("Unable to resolve $path_base.$cmf", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless $cmf;
+
+    my $command = "$dvoImageOverlaps -D CATDIR $catdir -accept-astrom $cmf";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dvoImageOverlaps $error_code", $magic_ds_id, $component,
+            $error_code);
+    }
+    my @list;
+    my $output = join "\n", @$stdout_buf;
+    if ($output) {
+        foreach my $line (split "\n", $output) {
+            my (undef, undef, $skycell_id) = split " ", $line;
+            push @list, $skycell_id;
+        }
+    }
+    return @list
+}
+
 
 sub my_die

branches/eam_branches/ipp-20110906/ippScripts/scripts/nightly_science.pl

@@ -1708 +1708 @@
     $cmd .= "-input_label $label  -template_label $label ";
     $cmd .= "-set_workdir $workdir  -set_dist_group $new_dist_group  -set_data_group $new_data_group ";
+    $cmd .= " -mintimediff 40000 ";
     $cmd .= " -simple  -set_label $label -filter $filter ";
     $cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 ";

branches/eam_branches/ipp-20110906/ippScripts/scripts/register_exp.pl

@@ -54 +54 @@
 
 my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $PS_EXIT_CONFIG_ERROR );
-$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $PS_EXIT_SYS_ERROR ) if $logfile;
+$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $exp_id, $PS_EXIT_SYS_ERROR ) if $logfile;
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;

branches/eam_branches/ipp-20110906/ippScripts/scripts/register_imfile.pl

@@ -60 +60 @@
 
 my $ipprc = PS::IPP::Config->new() or my_die_for_add( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
-$ipprc->redirect_output($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
+$ipprc->redirect_to_logfile($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
@@ -404 +404 @@
         }
     }
+    if ($exp_name =~ /a$/) {
+        return(1);
+    }
     return(0);
 }
@@ -517 +520 @@
 
     carp($msg);
-    if (defined $exp_id && defined $tmp_class_id and not $no_update) {
+    if (defined $exp_id && defined $class_id and not $no_update) {
         my $command = "$regtool -updateprocessedimfile";
         $command .= " -exp_id $exp_id";

branches/eam_branches/ipp-20110906/ippScripts/scripts/stack_skycell.pl

@@ -104 +104 @@
 }
 
-$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $stack_id, $PS_EXIT_SYS_ERROR ) if $redirect;
+$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $stack_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 
 my $neb;
@@ -243 +243 @@
     print $listFile "END\n\n";
 }
+close($listFile);
 
 print "I've checked everything: $stack_id\n";
@@ -491 +492 @@
     &my_die("Couldn't find expected output file: $file",  $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
 
+    # Funpack to confirm we've really made things correctly
+    my $diskfile = $ipprc->file_resolve($file);
+    if ($diskfile =~ /fits/) {
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $stack_id, $PS_EXIT_SYS_ERROR);
+        my $check_command = "$funpack -S $diskfile > /dev/null";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $check_command, verbose => $verbose);
+        if (!$success) {
+            &my_die("Output file not a valid fits file: $file", $stack_id, $PS_EXIT_SYS_ERROR);
+        }
+    }
+    #####
+
     if ($replicate and $neb) {
         $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $stack_id, $PS_EXIT_SYS_ERROR);

branches/eam_branches/ipp-20110906/ippScripts/scripts/videophot_process.pl

@@ -8 +8 @@
 my $host = hostname();
 my $date = `date`;
-print "\n\n";
-print "Starting script $0 on $host at $date\n\n";
-
 use DateTime;
 my $mjd_start = DateTime->now->mjd;   # MJD of starting script
@@ -25 +22 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename );
 
 # Look for programs we need
 my $missing_tools;
 my $vptool = can_run('vptool') or (warn "Can't find vptool" and $missing_tools = 1);
-my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
 my $psvideophot = can_run('psvideophot') or (warn "Can't find psvideophot" and $missing_tools = 1);
+my $dumpvideo = can_run('dumpvideo') or (warn "Can't find dumpvideo" and $missing_tools = 1);
 my $listvideocells = can_run('listvideocells.pl') or (warn "Can't find listvideocells.pl" and $missing_tools = 1);
+my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
     exit($PS_EXIT_CONFIG_ERROR);
 }
-my ($vp_id, $exp_id, $camera, $outroot, $dbname, $verbose, $no_update, $no_op, $redirect);
+my ($vp_id, $camera, $outroot, $dest_id, $dbname, $verbose, $no_update, $no_op, $redirect, $save_temps);
+my ($product, $ds_dbname, $ds_dbhost);
 
 GetOptions(
     'vp_id=s'           => \$vp_id,
-    'exp_id=s'          => \$exp_id,
     'camera=s'          => \$camera, 
     'outroot=s'         => \$outroot,
+    'dest_id=s'         => \$dest_id,
+    'product=s'         => \$product,
+    'ds_dbname=s'       => \$ds_dbname,
+    'ds_dbhost=s'       => \$ds_dbhost,
     'dbname|d=s'        => \$dbname,
     'verbose'           => \$verbose,
+    'save-temps'        => \$save_temps,
     'no-update'         => \$no_update, # Don't update the database?
     'no-op'             => \$no_op,     # Don't do any operations?
@@ -52 +57 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
-    -msg => "Required options: --vp_id --exp_id --outroot --camera",
+    -msg => "Required options: --vp_id --outroot --camera",
     -exitval => 3,
           ) unless defined $vp_id
-            and defined $exp_id
             and defined $camera
             and defined $outroot;
 
+pod2usage(
+    -msg => " --product -ds_dbname and --dsdbhost are required if --dest_id",
+    -exitval => 3,
+          ) if ($dest_id and !( defined $product and defined $ds_dbname and defined $ds_dbhost));
+
 
 $no_update = 1 if $no_op;
-
-my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $vp_id, 'nochip', 'nocell', $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$vptool .= " -dbname $dbname" if $dbname;
+
+my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $vp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 
 my $logDest = $ipprc->filename("LOG.EXP", $outroot);
-$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $vp_id, 'nochip', 'nocell', $PS_EXIT_SYS_ERROR ) if $redirect;
-
-my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF);
-
-# Get list of components for subtraction
+if ($redirect) {
+    $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $vp_id, $PS_EXIT_SYS_ERROR );
+
+    print "\n\n";
+    print "Starting script $0 on $host at $date\n\n";
+}
+
+# Get list of chips with pending video cells
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 my $files;
 {
-#    my $command = "$vptool -pendingimfile -vp_id $vp_id";
-#    $command .= " -dbname $dbname" if defined $dbname;
-    my $command = "$regtool -processedimfile -video_cells -exp_id $exp_id";
-    $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, 'nochip', 'nocell', $error_code);
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $vp_id, 'nochip', 'nocell', $PS_EXIT_PROG_ERROR);
-    $files = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $vp_id, 'nochip', 'nocell', $PS_EXIT_PROG_ERROR);
-}
+    my $command = "$vptool -pendingimfile -vp_id $vp_id";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, $error_code);
+    }
+
+    my $output = join "", @$stdout_buf;
+    if ($output) {
+        my $metadata = $mdcParser->parse($output) or
+            &my_die("Unable to parse metadata config doc", $vp_id, $PS_EXIT_PROG_ERROR);
+        $files = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $vp_id, $PS_EXIT_PROG_ERROR);
+    }
+}
+
 
 foreach my $file (@$files) {
@@ -95 +109 @@
 
     my $resolved = $ipprc->file_resolve($uri);
-    &my_die("Unable to resolve $uri", $vp_id, $class_id, 'nocell', $PS_EXIT_UNKNOWN_ERROR) unless $resolved;
+    &my_die("Unable to resolve $uri", $vp_id, $PS_EXIT_UNKNOWN_ERROR) unless $resolved;
 
     my $command = "$listvideocells --file $resolved";
@@ -102 +116 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform $command: $error_code", $vp_id, $class_id, 
-            'nocell', $error_code);
+        &my_die("Unable to perform $command: $error_code", $vp_id, $error_code);
     }
 
     my @video_cells = split "\n", (join "", @$stdout_buf);
-    &my_die("No video cells found in $uri", $vp_id, $class_id, 
-            'nocell', $PS_EXIT_UNKNOWN_ERROR) unless scalar @video_cells;
+    &my_die("No video cells found in $uri", $vp_id, $PS_EXIT_UNKNOWN_ERROR) unless scalar @video_cells;
 
     foreach my $cell_id (@video_cells) {
-        my $command = "$psvideophot $outroot";
+        my $path_base =  "$outroot.$class_id.$cell_id";
+        my $output = $ipprc->filename("PSVIDEOPHOT.OUTPUT", $path_base);
+        my $command = "$psvideophot $output";
         $command .= " -file $uri";
+        $command .= " -class_id $class_id";
         $command .= " -cell_id $cell_id";
-        $command .= " -dbname $dbname" if defined $dbname;
+        my $vpstart = DateTime->now->mjd;
         unless ($no_op) {
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -120 +135 @@
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform ppSub: $error_code", $vp_id, $class_id, $cell_id, $error_code);
+                &my_die("Unable to perform psvideophot: $error_code", $vp_id, $error_code);
             }
-
-            # check_output($outputStats, 1);
         } else {
             print "Not executing: $command\n";
         }
+        # dump the video cell and the video table to a file
+        unless ($no_op) {
+            my $output = $path_base . ".fits";
+            my $extname = "$cell_id";
+            my $command = "$dumpvideo $uri $extname $output -includetable";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dumpvideo: $error_code", $vp_id, $error_code);
+            }
+        } else {
+            print "Not executing: $command\n";
+        }
+
         unless ($no_update) {
-            my $command = "$vptool -addvpcell -vp_id $vp_id -class_id $class_id -cell_id $cell_id";
-            $command .= " -path_base $outroot";
-
-            $command .= " -dbname $dbname" if defined $dbname;
+            my $command = "$vptool -addprocessedcell -vp_id $vp_id -class_id $class_id -cell_id $cell_id";
+            $command .= " -path_base $path_base";
+            $command .= " -hostname $host";
+            $command .= " -dtime_photom " . ((DateTime->now->mjd - $vpstart) * 86400);
 
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -138 +166 @@
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                 my $err_message = "Unable to perform vptool -addvpcell";
-                &my_die("$err_message: $error_code", $vp_id, $class_id, $cell_id, $error_code);
+                &my_die("$err_message: $error_code", $vp_id, $error_code);
             }
         }
+    }
+}
+if ($dest_id) {
+    my $command = "$vptool -processedcell -vp_id $vp_id";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, $error_code);
+    }
+
+    my $output = join "", @$stdout_buf;
+    my $cells;
+    if ($output) {
+        my $metadata = $mdcParser->parse($output) or
+            &my_die("Unable to parse metadata config doc", $vp_id, $PS_EXIT_PROG_ERROR);
+        $cells = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", $vp_id, $PS_EXIT_PROG_ERROR);
+    }
+    my ($reglist, $reglistName) = tempfile("/tmp/filelist.$vp_id.XXXX", UNLINK => !$save_temps);
+
+    if ($redirect) {
+        print $reglist "$logDest|||text|\n";
+    }
+    my @exts = qw( vpt fits );
+    my @types = qw( table fits );
+    foreach my $cell (@$cells) {
+        my $path_base = $cell->{path_base};
+        for (my $i = 0; $i < scalar @exts; $i++) {
+            my $file = "$path_base.$exts[$i]";
+            my $type = $types[$i];
+
+            print $reglist "$file|||$type|\n";
+        }
+            
+    }
+    close $reglist or &my_die("failed to close $reglistName", $vp_id, $PS_EXIT_UNKNOWN_ERROR);
+    unless ($no_update or $no_op) {
+        my $fileset = basename($outroot);
+        my $command = "$dsreg --add $fileset --product $product --type dump --list $reglistName";
+        $command .= " --abspath --link";
+        $command .= " --dbname $ds_dbname --dbhost $ds_dbhost";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+             &my_die("Unable to perform $command error_code: $error_code", $vp_id, $error_code);
+        }
+    }
+}
+
+unless ($no_update) {
+    my $command = "$vptool -updaterun -vp_id $vp_id -set_state full"; 
+    $command .= " -set_outroot $outroot";
+    $command .= " -set_hostname $host";
+    $command .= " -set_dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400);
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        my $err_message = "Unable to perform vptool -updaterun";
+        &my_die("$err_message: $error_code", $vp_id, $error_code);
     }
 }
@@ -149 +240 @@
 {
     my $msg = shift;            # Warning message on die
-    my $vp_id = shift;
-    my $class_id = shift;       # chip identifier
-    my $cell_id = shift;       # chip identifier
+    my $vp_id = shift;          # vpRun id
     my $exit_code = shift;      # Exit code to add
 
@@ -157 +246 @@
 
     warn($msg);
-    if (defined $vp_id and defined $class_id and not $no_update) {
-        my $command = "$vptool -vp_id $vp_id -class_id $class_id -cell_id -fault $exit_code";
-        $command .= " -addvpcell";
-        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
-        $command .= " -hostname $host" if defined $host;
-        $command .= " -path_base $outroot" if defined $outroot;
-        $command .= " -dbname $dbname" if defined $dbname;
+    if (defined $vp_id and not $no_update) {
+        my $command = "$vptool -vp_id $vp_id";
+        $command .= " -updaterun";
+        $command .= " -set_fault $exit_code";
+        $command .= " -set_hostname $host" if defined $host;
+        $command .= " -set_outroot $outroot" if defined $outroot;
+        $command .= " -set_dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400);
+
         run(command => $command, verbose => $verbose);
     }
@@ -169 +259 @@
 }
 
-END {
-    my $exit = $?;
-    system("sync") == 0 or die "failed to execute sync: $!";
-    $? = $exit;
-}
-
 __END__

branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_overlap.pl

@@ -57 +57 @@
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
-$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $warp_id, $PS_EXIT_SYS_ERROR ) if $logfile;
+$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $warp_id, $PS_EXIT_SYS_ERROR ) if $logfile;
 
 &my_die("Tessellation identifier not provided: $tess_dir", $warp_id, $PS_EXIT_SYS_ERROR) unless $tess_dir ne "NULL";

branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_skycell.pl

@@ -38 +38 @@
 
 my ($warp_id, $skycell_id, $warp_skyfile_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $run_state, $magicked, $verbose, $no_update, $no_op, $redirect, $save_temps);
-my $zaplog = 0;
 
 GetOptions(
@@ -55 +54 @@
     'no-update'           => \$no_update, # Don't update the database?
     'no-op'               => \$no_op, # Don't do any operations?
-    'zaplog'              => \$zaplog, # delete previous logfile
     'redirect-output'     => \$redirect,
     'save-temps'          => \$save_temps, # Save temporary files?
@@ -77 +75 @@
 my $do_stats;
 if ($run_state eq 'new') {
-    $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, $zaplog);
+    $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id);
     $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1);
     $do_stats = 1;
 } elsif ($run_state eq 'update')  {
-    $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 1);
+    $logDest = $ipprc->filename("LOG.EXP.UPDATE", $outroot, $skycell_id);
     $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, $skycell_id, 1);
 } else {
@@ -93 +91 @@
 }
 
-$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect;
+$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect;
 
 # Recipes to use based on reduction class
@@ -333 +331 @@
             }
         }
-
+        print "Quality: $quality\n";
         unless ($no_update) {
             # XXX change -tess_id to -tess_dir when db is updated
@@ -358 +356 @@
         }
     } else {
+
+        my $quality = 0;
+
+        unless ($ipprc->file_exists($outputPSF)) {
+            $quality = 8006; # bad data.
+        }
+#         if (!$quality) {
+#             check_output($outputImage, 0);
+#             check_output($outputMask, 0);
+#             check_output($outputWeight, 0);
+#             check_output($outputSources, 1);
+#             check_output($outputPSF, 1) if metadataLookupBool($recipe, 'PSF')  ;
+#         }
+        print "Quality: $quality\n";
+
         # $run_state eq 'update'
         unless ($no_update) {
@@ -364 +377 @@
             $command .= " -skycell_id $skycell_id";
             $command .= " -set_magicked $magicked" if $magicked;
+            $command .= " -set_quality $quality" if $quality;
             $command .= " -dbname $dbname"   if defined $dbname;
 
@@ -407 +421 @@
 
     &my_die("Couldn't find expected output file: $file",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
+
+    # Funpack to confirm we've really made things correctly
+    my $diskfile = $ipprc->file_resolve($file);
+    if ($diskfile =~ /fits/) {
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
+        my $check_command = "$funpack -S $diskfile > /dev/null";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $check_command, verbose => $verbose);
+        if (!$success) {
+            &my_die("Output file not a valid fits file: $file", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
+        }
+    }
+    #####
 
     if ($replicate and $neb) {