Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/camera_exp.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/camera_exp.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/camera_exp.pl	(revision 32630)
@@ -88,5 +88,5 @@
 
 if ($redirect) {
-    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $cam_id, $PS_EXIT_SYS_ERROR );
+    $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $cam_id, $PS_EXIT_SYS_ERROR );
     print "\n\n";
     print "Starting script $0 on $host\n\n";
@@ -404,4 +404,18 @@
         $ipprc->file_exists($file);
 
+    # Funpack to confirm we've really made things correctly
+    my $diskfile = $ipprc->file_resolve($file);
+    if ($diskfile =~ /fits/) {
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack",  $cam_id, $PS_EXIT_SYS_ERROR);
+	my $check_command = "$funpack -S $diskfile > /dev/null";
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $check_command, verbose => $verbose);
+	if (!$success) {
+	    &my_die("Output file not a valid fits file: $file",  $cam_id, $PS_EXIT_SYS_ERROR);
+	}
+    }
+    #####
+
+
     my $scheme = file_scheme($file);
     if ($replicate and $scheme and (file_scheme($file) eq 'neb')) {
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/chip_imfile.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/chip_imfile.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/chip_imfile.pl	(revision 32630)
@@ -34,4 +34,6 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1);
+
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -44,6 +46,4 @@
 my ( $exp_id, $chip_id, $class_id, $chip_imfile_id, $uri, $camera, $outroot, $dbname, $run_state, $reduction, $threads, $verbose,
      $no_update, $save_temps, $no_op, $redirect, $magicked, $deburned );
-
-my $zaplog = 0;
 
 GetOptions(
@@ -65,5 +65,4 @@
     'no-op'             => \$no_op,     # Don't do any operations?
     'redirect-output'   => \$redirect,
-    'zaplog'            => \$zaplog,
     'save-temps'        => \$save_temps, # Save temporary files?
     ) or pod2usage( 2 );
@@ -92,15 +91,18 @@
 }
 
-my ($logDest, $traceDest);
+my ($logRule, $traceDest);
 if ($run_state eq 'new') {
-    $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, $zaplog);
+    $logRule = "LOG.IMFILE";
     $traceDest = prepare_output("TRACE.IMFILE",  $outroot, $class_id, 1);
 } else {
-    $logDest = prepare_output("LOG.IMFILE.UPDATE", $outroot, $class_id, 1);
+    $logRule = "LOG.IMFILE.UPDATE";
     $traceDest = prepare_output("TRACE.IMFILE.UPDATE",  $outroot, $class_id, 1);
 }
 
 if ($redirect) {
-    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR );
+    my $logDest = $ipprc->filename($logRule, $outroot, $class_id);
+
+    $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR );
+
     print STDOUT "\n\n";
     print STDOUT "Starting script $0 on $host\n\n";
@@ -330,4 +332,14 @@
         # get the UNIX version of the (possible) neb: or path: filename
         my $uriReal = $ipprc->file_resolve( $uri );
+	# Catch errors here
+	if (!$uriReal) {
+	    my $repair_cmd = "$nebrepair $uri";
+	    my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+	    unless ($repair_success) {
+		&my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+	    }
+	    $uriReal = $ipprc->file_resolve( $uri );
+	}
+	    
         &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
 
@@ -336,4 +348,13 @@
         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
         unless ($success) {
+	    # Catch errors here
+	    my $repair_cmd = "$nebrepair $uri";
+	    my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+	    unless ($repair_success) {
+		&my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+	    }
+	    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
+	}
+        unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
             &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
@@ -348,5 +369,14 @@
             $burntoolTable_uri =~ s/fits$/burn.tbl/;
             $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+	    if ((!$burntoolTable_uriReal)||(!($ipprc->file_exists($burntoolTable_uri)))) {
+		my $repair_cmd = "$nebrepair $burntoolTable_uri";
+		my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+		unless ($repair_success) {
+		    &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+		}
+		$burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+	    }
             unless ($ipprc->file_exists($burntoolTable_uri)) {
+		# Catch errors here
                 &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR);
             }
@@ -451,5 +481,5 @@
     $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
     $command .= " -source_id $source_id" if defined $source_id;
-    $command .= " -tracedest $traceDest -log $logDest";
+    $command .= " -tracedest $traceDest";
     $command .= " -dbname $dbname" if defined $dbname;
 
@@ -727,4 +757,16 @@
         &my_die("Couldn't find expected output file: $file",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     }
+    # Funpack to confirm we've really made things correctly
+    my $diskfile = $ipprc->file_resolve($file);
+    if ($diskfile =~ /fits/) {
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+	my $check_command = "$funpack -S $diskfile > /dev/null";
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $check_command, verbose => $verbose);
+	if (!$success) {
+	    &my_die("Output file not a valid fits file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+	}
+    }
+    #####
 
     if ($replicate and $neb) {
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/diff_skycell.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/diff_skycell.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/diff_skycell.pl	(revision 32630)
@@ -40,5 +40,4 @@
 my ($skycell_id, $diff_skyfile_id);
 my ($use_convolved);
-my $zaplog = 0;
 
 GetOptions(
@@ -57,5 +56,4 @@
     'no-op'             => \$no_op, # Don't do any operations?
     'redirect-output'   => \$redirect,
-    'zaplog'            => \$zaplog, # delete any previous log file
 ) or pod2usage( 2 );
 
@@ -79,15 +77,9 @@
 
 # XXX camera is not known here; cannot use filerules... 
-# bills 2011-03-07 This is unfortunate because it prevents us from using prepare_output ...
-# my $logDest = $ipprc->filename("LOG.EXP", $outroot);
 my $logDest = "$outroot.log";
 if ($run_state eq 'update') {
     $logDest .= '.update';
-    # ... well we can just do the dirty work here
-    $ipprc->kill_file($logDest);
-} elsif ($zaplog) {
-    $ipprc->kill_file($logDest);
-}
-$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
+}
+$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 
 my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF);
@@ -468,4 +460,18 @@
     &my_die("Couldn't find expected output file: $file",  $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
 
+    # Funpack to confirm we've really made things correctly
+    my $diskfile = $ipprc->file_resolve($file);
+    if ($diskfile =~ /fits/) {
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR);
+	my $check_command = "$funpack -S $diskfile > /dev/null";
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $check_command, verbose => $verbose);
+	if (!$success) {
+	    &my_die("Output file not a valid fits file: $file", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR);
+	}
+    }
+    #####
+
+
     if ($replicate and $neb) {
         $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR);
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/dist_bundle.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/dist_bundle.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/dist_bundle.pl	(revision 32630)
@@ -636,4 +636,8 @@
         $file{file_rule} = $key;
         $file{name} = $val;
+	if ($val eq "STDERR" or $val eq "STDOUT") {
+	    print STDERR "Skipping $key because filename is $val\n";
+	    next;
+	}
         push @file_list, \%file;
     }
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/lap_science.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/lap_science.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/lap_science.pl	(revision 32630)
@@ -27,4 +27,5 @@
 my ( $camera, $dbname);
 my ( $lap_id );
+my ( $queue_list );
 my ( $chip_mode, $monitor_mode, $cleanup_mode);
 
@@ -44,4 +45,5 @@
     
     'lap_id=s'     => \$lap_id,
+    'queue_list=s' => \$queue_list,
 
     'chip_mode'    => \$chip_mode,
@@ -200,4 +202,122 @@
     return($exposure);
 }
+
+sub remake_this_exposure_by_update {
+    my $exposure = shift;
+
+    my @utctime = gmtime();
+    $utctime[5] += 1900;
+    $utctime[4] += 1;
+
+    my $label = $exposure->{label};
+
+    my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]);
+    my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}";
+    my $data_group = "${label}.${date}";
+
+    my $chiptool_info_cmd = "chiptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} ";
+    $chiptool_info_cmd   .= " -dbname $dbname " if defined $dbname;
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $chiptool_info_cmd, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform chiptool -listrun: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    my $chips = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from chiptool -listrun", $exposure->{exp_id}, $data_group);
+    # There should be only one.
+    my $chip = ${ $chips }[0];
+    my $chip_magicDS_id = 0;
+    if ($chip) {
+	$chip_magicDS_id = $chip->{magic_ds_id};
+    }
+    if ($chip_magicDS_id == 0) {
+#	return(&remake_this_exposure($exposure));
+    }
+
+    my $warptool_info_cmd = "warptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} ";
+    $warptool_info_cmd   .= " -dbname $dbname " if defined $dbname;
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $warptool_info_cmd, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform warptool -listrun: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    my $warps = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from warptool -listrun", $exposure->{exp_id}, $data_group);
+    # There should be only one.
+    my $warp = ${ $warps }[0];
+    my $warp_id = 0;
+    if ($warp) {
+	$warp_id = $warp->{warp_id};
+    }
+    else {
+	$exposure->{data_state} = 'drop';
+	return($exposure);
+    }	
+    if ($warp_id == 0) {
+	# Handle this correctly. Probably want to drop the exposure here.
+	$exposure->{data_state} = 'drop';
+	return($exposure);
+    }
+
+    if (($chip->{state} eq 'goto_cleaned')||
+	($warp->{state} eq 'goto_cleaned')||
+	($chip->{dsRun_state} eq 'goto_cleaned')) {
+	$exposure->{data_state} = 'pending_update';
+	&update_this_exposure($exposure);
+	return($exposure);
+    }
+
+    
+    my $chiptool_update_cmd = "chiptool -setimfiletoupdate -chip_id $exposure->{chip_id} -set_label $label";
+    $chiptool_update_cmd   .= " -dbname $dbname " if defined $dbname;
+    my $magicDS_update_cmd  = "magicdstool -setfiletoupdate -magic_ds_id $chip_magicDS_id -set_label $label";
+    $magicDS_update_cmd    .= " -dbname $dbname " if defined $dbname;
+    my $warptool_update_cmd = "warptool -setskyfiletoupdate -warp_id $warp_id -set_label $label";
+    $warptool_update_cmd   .= " -dbname $dbname " if defined $dbname;
+
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $chiptool_update_cmd, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform chiptool -setimfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    if ($chip_magicDS_id != 0) {    
+	($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $magicDS_update_cmd, verbose => $verbose);
+	unless ($success) {
+	    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	    &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
+	}
+    }
+
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $warptool_update_cmd, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform warptool -setskyfiletoupdate: $error_code", $exposure->{exp_id}, $data_group);
+    }
+    
+    $exposure->{active} = 1;
+    return($exposure);
+}
+        
+sub determine_if_can_update {
+    my $exposure = shift;
+    
+    if (S64_IS_NOT_NULL($exposure->{chip_id})) {
+	if (($exposure->{chip_state} eq 'cleaned')||
+	    ($exposure->{chip_state} eq 'goto_cleaned')||
+	    ($exposure->{chip_state} eq 'error_cleaned')) {
+	    return(1);
+	}
+    }
+
+    return(0);
+}
+	
 
 # This is the "user level" subroutine.
@@ -240,5 +360,7 @@
 	my $comment = $exposure->{comment};
 
-	# This is a hack to fix old exposures that have no object.
+	my $updateable = determine_if_can_update($exposure);
+
+	# This is a hack to fix old exposures that have no usable object/comment data.
 	unless(defined($comment)) {
 	    $comment = '';
@@ -247,22 +369,41 @@
 	    if ($comment =~ /3pi_/) {
 		$object = $comment;
-		$object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/$1/;
+		$object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/F1 $1/;
 	    }
 	    elsif ($comment =~ / ps1_/) {
 		$object = $comment;
-		$object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/$1/;
-	    }
-	    else {
-		$exposure->{data_state} = 'new';
+		$object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/F2 $1/;
+	    }
+	    elsif ($comment =~ /ThreePi.*3pi_/) {
+		$object = $comment;
+		$object =~ s/^ThreePi . (\S+? 3pi_\d\d_\d\d\d\d) .*$/F3 $1/;
+	    }           
+	    elsif ($comment =~ /ThreePi.*ps1_/) {
+		$object = $comment;
+		$object =~ s/^ThreePi . (\S+? ps1_\d\d_\d\d\d\d) .*$/F4 $1/;
+	    }           
+	    elsif ($comment =~ /ThreePi /) {
+		$object = $comment;
+		$object =~ s/^ThreePi . (\S+? \d\d\d\d) .*$/F5 $1/;
+	    }           
+	    elsif (($comment =~ /focus/i)||
+		   ($comment =~ /test/i)||
+		   ($comment =~ /hyster/i)||
+		   ($comment =~ /dither/i)||
+		   ($comment =~ /camera/i)
+		){
+		# This is junk that shouldn't exist.
+		$object = 'DROP';
+		$exposure->{data_state} = 'drop';
 		$exposure->{pairwise} = 0;
-		$exposure->{private} = 1;
-		$exposure->{pair_id} = 9223372036854775807;
-		update_this_exposure($exposure);
-		$exposure = remake_this_exposure($exposure); 
-		$counter++;
+		$exposure->{pair_id} =  9223372036854775807;
+		$exposure->{active} = 0;
+		&update_this_exposure($exposure);
+		$counter++; # To ensure everyone else is consistent
 		next;
 	    }
 	}
 
+	# Determine the current state of chipRuns for these exposures, and update/remake as needed.
 	if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id
 	    if (($pairwise) && !($pair_id)) {
@@ -270,18 +411,25 @@
 		&my_die("Exposure $exp_id for $lap_id is declared pairwise without a defined pair", $lap_id);
 	    }
+	    $exposure->{data_state} = 'exists';
+	    if ($updateable) { # We know about this run, but the data needs to be regenerated.
+		$exposure->{data_state} = 'update';
+		$exposure = remake_this_exposure_by_update($exposure);
+	    }
+	    &update_this_exposure($exposure);
 	    $matching{$object}{$comment} = $exp_id;
 	    $indexing{$exp_id} = $counter;
 	    $counter++; 
-	    next;
 	}
 	else { # We do not already have a chip_id.  
 	    # Make a chipRun, and update the exposure.
+	    $exposure->{data_state} = 'run';
 	    $exposure = remake_this_exposure($exposure); 
-	    
 	    # Save our information for diff pairing.
+	    &update_this_exposure($exposure);
 	    $matching{$object}{$comment} = $exp_id;
 	    $indexing{$exp_id} = $counter;
 	    $counter++;
 	}
+
 	if ($verbose) {
 	    print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n";
@@ -296,5 +444,5 @@
 	    print "$object $comment $matching{$object}{$comment} $indexing{$matching{$object}{$comment}} $exp_ids_to_diff[-1]\n";
 	}
-	@exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff;
+	@exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; # This is effectively a sort by dateobs.
 	
 	if (( $#exp_ids_to_diff + 1) % 2 != 0) { # We have an odd number of exposures, even after comment filtering
@@ -309,14 +457,17 @@
 	    my $exp_B = ${ $exposures }[$indexing{$exp_id_B}];
 	    print "$exp_A $exp_B $exp_id_A $exp_id_B $indexing{$exp_id_A} $indexing{$exp_id_B}\n";
-	    $exp_A->{pairwise} = 1;
-	    $exp_A->{private} = 0;
-	    $exp_A->{pair_id} = $exp_B->{chip_id};
-	    
-	    $exp_B->{pairwise} = 1;
-	    $exp_B->{private} = 0;
-	    $exp_B->{pair_id} = $exp_A->{chip_id};
-	    
-	    if ($verbose) {
-		print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n";
+	    if ($exp_A->{diff_id} == $exp_B->{diff_id}) {
+		$exp_A->{pairwise} = 1;
+		$exp_A->{pair_id} = $exp_B->{chip_id};
+		
+		$exp_B->{pairwise} = 1;
+		$exp_B->{pair_id} = $exp_A->{chip_id};
+		if ($verbose) {
+		    print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n";
+		}
+	    }
+	    else {
+		$exp_A->{pairwise} = 0;
+		$exp_B->{pairwise} = 0;
 	    }
 	}
@@ -331,8 +482,5 @@
 	    $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error.
 	}
-	if (!($exposure->{pairwise})) {
-	    $exposure->{private} = 1; # If this isn't being pairwise diffed, it needs to be private
-	}
-
+	
 	update_this_exposure($exposure);
     }
@@ -457,6 +605,6 @@
     # Things I want to know before I'm through
 #     my $needs_qstack = 0;
-    my $needs_something_remade = 0;
-    my $needs_something_private = 0;
+    my $needs_something_remade = 0;  # I never need something remade, because we're committed to one-exp=one-chip
+    my $needs_something_private = 0; # I never need something private, as that is calculated by the difference engine.
 #     my $can_qstack = 0;
      my $have_diff = 0;
@@ -480,5 +628,4 @@
 	    else { # We claimed to be pairwise, but do not have a valid pair_id.
 		$exposure->{pairwise} = 0;
-		$exposure->{private} = 1;
 		&update_this_exposure($exposure);
 	    }
@@ -487,22 +634,25 @@
 	if  ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through.
 	    $are_warped++;
-#	    $can_qstack ++;
 	    $can_diff ++;
 	    $have_diff ++;
 	    $are_magicked ++;
-#	    $can_fstack ++;	    
+	    next;
+	}
+
+	if ($exposure->{data_state} eq 'pending_update') {
+	    $exposure->{data_state} = 'update';
+	    $exposure = &remake_this_exposure_by_update($exposure);
+	    &update_this_exposure($exposure);
 	    next;
 	}
 	
 	if ($exposure->{private}) { # I've declared this exposure private to this lapRun.
-#	    $needs_qstack++;
 	    push @lonely_exposures, $exposure;
-#	    $lonely_exposures++;
 	}
 	
-	if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun
-	    $needs_something_remade = 1;
-	    $exposure = remake_this_exposure($exposure);
-	}
+# 	if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun
+# 	    $needs_something_remade = 1;
+# 	    $exposure = remake_this_exposure($exposure);
+# 	}
 	# Do quality checks here
 	my $is_bad_quality = 0;
@@ -536,18 +686,14 @@
 		    ($exposure->{diffRun_state} eq 'full')) {
 		$are_warped++;
-#		$needs_qstack++;
-		$needs_something_private = 1;
 		if ($companion) {
-		    $companion->{private} = 1;
 		    $companion->{pairwise} = 0;
 		    &update_this_exposure($companion);
-		    push @lonely_exposures, $companion;
-#		    $lonely_exposures++;
-		}
-		$exposure->{private} = 1;
+#		    push @lonely_exposures, $companion;
+		}
 		$exposure->{pairwise} = 0;
 	    }
 	    $exposure->{data_state} = 'drop';
 	    &update_this_exposure($exposure);
+	    next;
 	}
 	
@@ -558,13 +704,19 @@
 #	    $can_qstack ++;
 	    $can_diff ++;
-	}
-	if (($exposure->{magicked}&&
-	     &S64_IS_NOT_NULL($exposure->{magicked}))) {  # This exposure has been magicked, so it is through with diff.
-	    $are_magicked++;
-#	    $can_fstack ++;
+	    $exposure->{data_state} = 'to_diff';
 	}
 	if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) {
 	    $have_diff ++;
-	}
+	    $exposure->{data_state} = 'to_magic';
+	}
+	if (($exposure->{magicked})&&
+	    ($exposure->{warpRun_state})&&
+	    ($exposure->{warpRun_state} eq 'full')&&
+	    (&S64_IS_NOT_NULL($exposure->{magicked}))) {  # This exposure has been magicked, so it is through with diff.
+	    $are_magicked++;
+	    $exposure->{data_state} = 'full';
+#	    $can_fstack ++;
+	}
+	&update_this_exposure($exposure);
     }
 
@@ -575,13 +727,10 @@
     print "STATUS: HAVE_FSTACK:     $have_fstack\n";
 
-#    print "STATUS: NEEDS_QSTACK:    $needs_qstack\n";
     print "STATUS: NEEDS_REMADE:    $needs_something_remade\n";
     print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n";
     print "STATUS: ARE WARPED:      $are_warped\n";
     print "STATUS: ARE MAGICKED:    $are_magicked\n";
-#    print "STATUS: CAN_QSTACK:      $can_qstack\n";
     print "STATUS: CAN_DIFF:        $can_diff\n";
     print "STATUS: HAVE_DIFF:       $have_diff\n";
-#    print "STATUS: CAN_FSTACK:      $can_fstack\n";
 
     print "STATUS: TOTAL_EXPOSURES: $total_exposures\n";
@@ -924,5 +1073,5 @@
 	    next;
 	}
-	if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # Not sure how this would happen, but still. ## This happens when we inherit a complete exposure.
+	if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # This happens when we inherit a complete exposure.
 	    next;
 	}
@@ -964,9 +1113,21 @@
 	}
 	else { # warp-qstack
+	    # We need to decide if we can make a warp-stack diff:
+	    if (&can_warp_stack_diff_be_made($exposure)) {
 #	    next; 
 #	    $command .= '-pretend';
- 	    $command .= " -definewarpstack -available -good_frac 0.2 ";
- 	    $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} ";
- 	    $already_queued{$exposure->{warp_id}} = 1;
+		$command .= " -definewarpstack -available -good_frac 0.2 ";
+		$command .= " -warp_id $exposure->{warp_id} -stack_label ${label} ";
+		$already_queued{$exposure->{warp_id}} = 1;
+		$exposure->{private} = 0;
+		&update_this_exposure($exposure);
+	    }
+	    else {
+		$exposure->{private} = 1;
+		$exposure->{data_state} = 'drop';
+		$already_queued{$exposure->{warp_id}} = 1;
+		&update_this_exposure($exposure);
+		next;
+	    }
 	}
 	
@@ -1006,10 +1167,52 @@
 }
 
+sub can_warp_stack_diff_be_made {
+    my $exposure = shift;
+    my $lap_id   = $exposure->{lap_id};
+    my $warp_id  = $exposure->{warp_id};
+
+    my $command = "$laptool -diffcheck -lap_id $lap_id -warp_id $warp_id";
+    $command .= " -dbname $dbname " if defined $dbname;
+
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    unless ($success) {
+	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	&my_die("Unable to perform laptool -diffcheck: $error_code", $lap_id, $warp_id);
+    }
+    if (@$stdout_buf == 0) {
+	return(0);
+    }
+    my $skycells = $mdcParser->parse_list(join "", @$stdout_buf) or
+	&my_die("Unable to parse metadata from laptool -diffcheck", $lap_id, $warp_id);
+    foreach my $skycell (@$skycells) {
+	unless (defined($skycell->{stack_state})) {
+	    return(0);
+	}
+	if ($skycell->{stack_state} ne 'full') {
+	    return(0);
+	}
+	if ($skycell->{warp_state} ne 'full') {
+	    return(0);
+	}
+	unless (S64_IS_NOT_NULL($skycell->{stack_id})) {
+	    return(0);
+	}
+    }
+    return(1);
+}
+
+
 # Deactivate all exposures in this run.
 sub deactivate_exposures {
     my $exposures = shift;
     foreach my $exposure (@$exposures) {
-	$exposure->{active} = 0;
-	update_this_exposure($exposure);
+	if ($exposure->{private} == 1) {  # This can probably be relaxed since we can update an undestreaked warp.
+	    $exposure->{active} = 1;
+	}
+	else {
+	    $exposure->{active} = 0;
+	}
+	update_this_exposure($exposure);	
     }
 }
@@ -1022,4 +1225,54 @@
 sub cleanup_mode {
     my $lap_id = shift;
+
+    if (defined($queue_list)) {
+	my $successful = 1;
+	open(Q,$queue_list) or ($successful = 0);
+	if ($successful == 1) {
+	    while(<Q>) {
+		chomp;
+		my $cmd = $_;
+		my ($projection_cell,$seq_id,$filter,$label);
+		$projection_cell = $cmd;
+		$projection_cell =~ s/.*?-projection_cell (skycell.\w+?) .*/$1/;
+		$seq_id = $cmd;
+		$seq_id =~ s/.*-seq_id (\d+?) .*/$1/;
+		$filter = $cmd;
+		$filter =~ s/.*-filter (\w\.00000) .*/$1/;
+		if ($filter =~ /\s+?/) {
+		    $filter =~ s/.*-filter (\w\.00000)$/$1/;
+		}
+		$label = $cmd;
+		$label =~ s/.*-label (.+?) .*/$1/;
+		
+		my $response;
+		chomp($response = 
+		      `laptool -dbname gpc1 -listrun -projection_cell $projection_cell -seq_id $seq_id -filter $filter -label $label -simple`);
+		my $state = (split /\s+/, $response)[5];
+		
+		unless (defined($state)) {
+		    if ($verbose) {
+			print "Queuing: $cmd\n";
+		    }
+		    system($cmd);
+		    
+		    my $i = 0;
+		    do {
+			sleep(5);
+			chomp($response = 
+			      `laptool -dbname gpc1 -listrun -projection_cell $projection_cell -seq_id $seq_id -filter $filter -label $label -simple`);
+			$state = (split /\s+/, $response)[5];
+			$i++;
+			unless(defined($state)) {
+			    $successful = 0;
+			}
+		    } while (($state ne 'run')&&($i < 20)&&($successful));
+		    last;
+		}
+	    }
+	    close(Q);
+	}
+    }
+
     my $command = "$laptool -inactiveexp -lap_id $lap_id ";
     $command .= " -dbname $dbname " if defined $dbname;
@@ -1043,7 +1296,9 @@
 	'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@',
 	'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@',
+	'magictool -dbname @DBNAME@ -updaterun -set_state full -set_label @LABEL@.old -magic_id @MAGIC_ID@',
 	'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage chip -stage_id @CHIP_ID@',
 	'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage warp -stage_id @WARP_ID@',
 	'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage diff -stage_id @DIFF_ID@');
+    
     foreach my $exposure (@$exposures) {
 	if ($exposure->{is_in_use}) {
@@ -1072,4 +1327,13 @@
 		if (S64_IS_NOT_NULL($exposure->{diff_id})) {
 		    $command =~ s/\@DIFF_ID\@/$exposure->{diff_id}/;
+		}
+		else {
+		    next;
+		}
+	    }
+	    if ($command =~ /\@MAGIC_ID\@/) {
+		if ((S64_IS_NOT_NULL($exposure->{magicked}))&&
+		     ($exposure->{magicked} > 0)) {
+		    $command =~ s/\@MAGIC_ID\@/$exposure->{magicked}/;
 		}
 		else {
@@ -1139,5 +1403,5 @@
 	&my_die("Unable to perform laptool -updaterun: $error_code", $lap_id);
     }
-
+    return(0);
 }
     
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/magic_destreak.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/magic_destreak.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/magic_destreak.pl	(revision 32630)
@@ -35,4 +35,5 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $dvoImageOverlaps = can_run('dvoImageOverlaps') or (warn "Can't find dvoImageOverlaps" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -45,4 +46,5 @@
 my ($outroot, $recoveryroot, $magicked);
 my ($replace, $release);
+my ($diff_tess_id, $mismatched_tess);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
@@ -60,4 +62,6 @@
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
            'component=s'    => \$component,  # the class_id or skycell_id
+           'mismatched_tess'=> \$mismatched_tess, # true if tess_id of input does not match tess_id of the diff used for magic
+           'diff_tess_id=s' => \$diff_tess_id, # tess_id of diffRun used to compute the streaks
            'uri=s'          => \$uri,        # uri of the input image
            'path_base=s'    => \$path_base,  # path_base of the input
@@ -106,8 +110,8 @@
 if (($stage eq "raw") or ($stage eq "chip") or $stage eq "chip_bg") {
     $class_id = $component;
-    $skycell_args = " -class_id $component";
+    $skycell_args = " -class_id";
 } elsif ($stage eq "warp" or $stage eq "warp_bg") {
     $skycell_id = $component;
-    $skycell_args = " -skycell_id $component";
+    $skycell_args = " -skycell_id";
 } elsif ($stage eq "diff") {
     $skycell_id = $component;
@@ -215,41 +219,61 @@
     my ($sfh, $skycell_list);
     if ($skycell_args) {
-        my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells $skycell_args";
-        $command .= " -dbname $dbname" if defined $dbname;
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
-        }
-
-        my $getskycells_output = join "", @$stdout_buf;
-        if ($getskycells_output) {
-            my $metadata = $mdcParser->parse($getskycells_output) or
-                &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
-
-            my $skycells = parse_md_list($metadata) or
-                    &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
-
-            ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
-
-            foreach my $skycell (@$skycells) {
-                my $skycell_uri;
-                if ($skycell->{data_state} eq "full" and $skycell->{quality} = 0) {
-                    $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
-                } else {
-                    # diff run must have been cleaned up, need to create this skycell file on the fly
-                    if (!defined $temp_dir ) {
-                        $temp_dir = tempdir( CLEANUP => !$save_temps);
+        my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells";
+        $command .= " $skycell_args";
+        my @diff_components;
+        if ($mismatched_tess) {
+            # tessellation for this skycell does not match the tessellation used for the magic analysis
+            # used to calculate the streaks file
+            @diff_components = get_overlaps($path_base, $component, $diff_tess_id);
+        } else {
+            @diff_components = ($component);
+        }
+
+        # hash of uris by skycell.
+        my %diff_skycells;
+        foreach my $component (@diff_components) {
+            my $this_command = "$command $component";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $this_command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
+            }
+
+            my $getskycells_output = join "", @$stdout_buf;
+            if ($getskycells_output) {
+                my $metadata = $mdcParser->parse($getskycells_output) or
+                    &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+                my $skycells = parse_md_list($metadata) or
+                        &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+                foreach my $skycell (@$skycells) {
+                    my $skycell_uri;
+                    if ($skycell->{data_state} eq "full" and $skycell->{quality} == 0) {
+                        $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                        $diff_skycells{$skycell_id} = $skycell_uri;
+                    } else {
+                        # diff run must have been cleaned up, need to create this skycell file on the fly
+                        if (!defined $temp_dir ) {
+                            $temp_dir = tempdir( CLEANUP => !$save_temps);
+                        }
+                        my $skycell_id = $skycell->{skycell_id};
+                        if (!$diff_skycells{$skycell_id}) {
+                            $skycell_uri = "$temp_dir/$skycell_id";
+                            $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
+                                &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+                        }
+                        $diff_skycells{$skycell_id} = $skycell_uri;
                     }
-                    my $skycell_id = $skycell->{skycell_id};
-                    $skycell_uri = "$temp_dir/$skycell_id";
-                    $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
-                        &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
                 }
-                print $sfh "$skycell_uri\n";
-            }
-            close $sfh
-        }
+            }
+        }
+        # write the skycell list file
+        ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
+        foreach my $skycell_id (keys %diff_skycells) {
+            print $sfh $diff_skycells{$skycell_id} . "\n";
+        }
+        close $sfh
     }
 
@@ -360,10 +384,5 @@
         $command .= " -chip_mask $ch_mask" if defined $ch_mask;
         $command .= " -weight $weight" if defined $weight;
-	if ((defined($sources))&&($ipprc->file_exists($sources))) {
-	    $command .= " -sources $sources" if defined $sources;
-	}
-	else {
-	    print "Did not add sources because they do not appear to exist. This may be an error.\n";
-	}
+        $command .= " -sources $sources" if defined $sources;
         $command .= " -skycelllist $skycell_list" if defined $skycell_list;
         $command .= " -replace" if $replace;
@@ -475,5 +494,5 @@
 
 my $statsFlags;
-if ($stage ne "camera") {
+if (!$no_op and $stage ne "camera") {
     file_check($statsFile);
     {
@@ -595,4 +614,35 @@
 }
 
+            # @diff_components = get_overlaps($path_base, $component, $tess_id, $diff_tess_id);
+sub get_overlaps {
+    my $path_base = shift;
+    my $component = shift;
+    my $diff_tess_id = shift;
+
+    my $tess_dir = $ipprc->tessellation_catdir( $diff_tess_id ); 
+    my $catdir = $ipprc->convert_filename_absolute($tess_dir);
+
+    my $cmf = $ipprc->file_resolve("$path_base.cmf");
+    &my_die("Unable to resolve $path_base.$cmf", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless $cmf;
+
+    my $command = "$dvoImageOverlaps -D CATDIR $catdir -accept-astrom $cmf";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dvoImageOverlaps $error_code", $magic_ds_id, $component,
+            $error_code);
+    }
+    my @list;
+    my $output = join "\n", @$stdout_buf;
+    if ($output) {
+        foreach my $line (split "\n", $output) {
+            my (undef, undef, $skycell_id) = split " ", $line;
+            push @list, $skycell_id;
+        }
+    }
+    return @list
+}
+
 
 sub my_die
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/nightly_science.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/nightly_science.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/nightly_science.pl	(revision 32630)
@@ -1708,4 +1708,5 @@
     $cmd .= "-input_label $label  -template_label $label ";
     $cmd .= "-set_workdir $workdir  -set_dist_group $new_dist_group  -set_data_group $new_data_group ";
+    $cmd .= " -mintimediff 40000 ";
     $cmd .= " -simple  -set_label $label -filter $filter ";
     $cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 ";
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/register_exp.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/register_exp.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/register_exp.pl	(revision 32630)
@@ -54,5 +54,5 @@
 
 my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $PS_EXIT_CONFIG_ERROR );
-$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $PS_EXIT_SYS_ERROR ) if $logfile;
+$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $exp_id, $PS_EXIT_SYS_ERROR ) if $logfile;
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/register_imfile.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/register_imfile.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/register_imfile.pl	(revision 32630)
@@ -60,5 +60,5 @@
 
 my $ipprc = PS::IPP::Config->new() or my_die_for_add( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
-$ipprc->redirect_output($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
+$ipprc->redirect_to_logfile($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
@@ -404,4 +404,7 @@
 	}
     }
+    if ($exp_name =~ /a$/) {
+	return(1);
+    }
     return(0);
 }
@@ -517,5 +520,5 @@
 
     carp($msg);
-    if (defined $exp_id && defined $tmp_class_id and not $no_update) {
+    if (defined $exp_id && defined $class_id and not $no_update) {
         my $command = "$regtool -updateprocessedimfile";
         $command .= " -exp_id $exp_id";
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/stack_skycell.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/stack_skycell.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/stack_skycell.pl	(revision 32630)
@@ -104,5 +104,5 @@
 }
 
-$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $stack_id, $PS_EXIT_SYS_ERROR ) if $redirect;
+$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $stack_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 
 my $neb;
@@ -243,4 +243,5 @@
     print $listFile "END\n\n";
 }
+close($listFile);
 
 print "I've checked everything: $stack_id\n";
@@ -491,4 +492,17 @@
     &my_die("Couldn't find expected output file: $file",  $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
 
+    # Funpack to confirm we've really made things correctly
+    my $diskfile = $ipprc->file_resolve($file);
+    if ($diskfile =~ /fits/) {
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $stack_id, $PS_EXIT_SYS_ERROR);
+	my $check_command = "$funpack -S $diskfile > /dev/null";
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $check_command, verbose => $verbose);
+	if (!$success) {
+	    &my_die("Output file not a valid fits file: $file", $stack_id, $PS_EXIT_SYS_ERROR);
+	}
+    }
+    #####
+
     if ($replicate and $neb) {
         $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $stack_id, $PS_EXIT_SYS_ERROR);
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/videophot_process.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/videophot_process.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/videophot_process.pl	(revision 32630)
@@ -8,7 +8,4 @@
 my $host = hostname();
 my $date = `date`;
-print "\n\n";
-print "Starting script $0 on $host at $date\n\n";
-
 use DateTime;
 my $mjd_start = DateTime->now->mjd;   # MJD of starting script
@@ -25,24 +22,32 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename );
 
 # Look for programs we need
 my $missing_tools;
 my $vptool = can_run('vptool') or (warn "Can't find vptool" and $missing_tools = 1);
-my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
 my $psvideophot = can_run('psvideophot') or (warn "Can't find psvideophot" and $missing_tools = 1);
+my $dumpvideo = can_run('dumpvideo') or (warn "Can't find dumpvideo" and $missing_tools = 1);
 my $listvideocells = can_run('listvideocells.pl') or (warn "Can't find listvideocells.pl" and $missing_tools = 1);
+my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
     exit($PS_EXIT_CONFIG_ERROR);
 }
-my ($vp_id, $exp_id, $camera, $outroot, $dbname, $verbose, $no_update, $no_op, $redirect);
+my ($vp_id, $camera, $outroot, $dest_id, $dbname, $verbose, $no_update, $no_op, $redirect, $save_temps);
+my ($product, $ds_dbname, $ds_dbhost);
 
 GetOptions(
     'vp_id=s'           => \$vp_id,
-    'exp_id=s'          => \$exp_id,
     'camera=s'          => \$camera, 
     'outroot=s'         => \$outroot,
+    'dest_id=s'         => \$dest_id,
+    'product=s'         => \$product,
+    'ds_dbname=s'       => \$ds_dbname,
+    'ds_dbhost=s'       => \$ds_dbhost,
     'dbname|d=s'        => \$dbname,
     'verbose'           => \$verbose,
+    'save-temps'        => \$save_temps,
     'no-update'         => \$no_update, # Don't update the database?
     'no-op'             => \$no_op,     # Don't do any operations?
@@ -52,41 +57,50 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
-    -msg => "Required options: --vp_id --exp_id --outroot --camera",
+    -msg => "Required options: --vp_id --outroot --camera",
     -exitval => 3,
           ) unless defined $vp_id
-            and defined $exp_id
             and defined $camera
             and defined $outroot;
 
+pod2usage(
+    -msg => " --product -ds_dbname and --dsdbhost are required if --dest_id",
+    -exitval => 3,
+          ) if ($dest_id and !( defined $product and defined $ds_dbname and defined $ds_dbhost));
+
 
 $no_update = 1 if $no_op;
-
-my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $vp_id, 'nochip', 'nocell', $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$vptool .= " -dbname $dbname" if $dbname;
+
+my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $vp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 
 my $logDest = $ipprc->filename("LOG.EXP", $outroot);
-$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $vp_id, 'nochip', 'nocell', $PS_EXIT_SYS_ERROR ) if $redirect;
-
-my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF);
-
-# Get list of components for subtraction
+if ($redirect) {
+    $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $vp_id, $PS_EXIT_SYS_ERROR );
+
+    print "\n\n";
+    print "Starting script $0 on $host at $date\n\n";
+}
+
+# Get list of chips with pending video cells
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 my $files;
 {
-#    my $command = "$vptool -pendingimfile -vp_id $vp_id";
-#    $command .= " -dbname $dbname" if defined $dbname;
-    my $command = "$regtool -processedimfile -video_cells -exp_id $exp_id";
-    $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, 'nochip', 'nocell', $error_code);
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $vp_id, 'nochip', 'nocell', $PS_EXIT_PROG_ERROR);
-    $files = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $vp_id, 'nochip', 'nocell', $PS_EXIT_PROG_ERROR);
-}
+    my $command = "$vptool -pendingimfile -vp_id $vp_id";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, $error_code);
+    }
+
+    my $output = join "", @$stdout_buf;
+    if ($output) {
+        my $metadata = $mdcParser->parse($output) or
+            &my_die("Unable to parse metadata config doc", $vp_id, $PS_EXIT_PROG_ERROR);
+        $files = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $vp_id, $PS_EXIT_PROG_ERROR);
+    }
+}
+
 
 foreach my $file (@$files) {
@@ -95,5 +109,5 @@
 
     my $resolved = $ipprc->file_resolve($uri);
-    &my_die("Unable to resolve $uri", $vp_id, $class_id, 'nocell', $PS_EXIT_UNKNOWN_ERROR) unless $resolved;
+    &my_die("Unable to resolve $uri", $vp_id, $PS_EXIT_UNKNOWN_ERROR) unless $resolved;
 
     my $command = "$listvideocells --file $resolved";
@@ -102,17 +116,18 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform $command: $error_code", $vp_id, $class_id, 
-            'nocell', $error_code);
+        &my_die("Unable to perform $command: $error_code", $vp_id, $error_code);
     }
 
     my @video_cells = split "\n", (join "", @$stdout_buf);
-    &my_die("No video cells found in $uri", $vp_id, $class_id, 
-            'nocell', $PS_EXIT_UNKNOWN_ERROR) unless scalar @video_cells;
+    &my_die("No video cells found in $uri", $vp_id, $PS_EXIT_UNKNOWN_ERROR) unless scalar @video_cells;
 
     foreach my $cell_id (@video_cells) {
-        my $command = "$psvideophot $outroot";
+        my $path_base =  "$outroot.$class_id.$cell_id";
+        my $output = $ipprc->filename("PSVIDEOPHOT.OUTPUT", $path_base);
+        my $command = "$psvideophot $output";
         $command .= " -file $uri";
+        $command .= " -class_id $class_id";
         $command .= " -cell_id $cell_id";
-        $command .= " -dbname $dbname" if defined $dbname;
+        my $vpstart = DateTime->now->mjd;
         unless ($no_op) {
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -120,16 +135,29 @@
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform ppSub: $error_code", $vp_id, $class_id, $cell_id, $error_code);
+                &my_die("Unable to perform psvideophot: $error_code", $vp_id, $error_code);
             }
-
-            # check_output($outputStats, 1);
         } else {
             print "Not executing: $command\n";
         }
+        # dump the video cell and the video table to a file
+        unless ($no_op) {
+            my $output = $path_base . ".fits";
+            my $extname = "$cell_id";
+            my $command = "$dumpvideo $uri $extname $output -includetable";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dumpvideo: $error_code", $vp_id, $error_code);
+            }
+        } else {
+            print "Not executing: $command\n";
+        }
+
         unless ($no_update) {
-            my $command = "$vptool -addvpcell -vp_id $vp_id -class_id $class_id -cell_id $cell_id";
-            $command .= " -path_base $outroot";
-
-            $command .= " -dbname $dbname" if defined $dbname;
+            my $command = "$vptool -addprocessedcell -vp_id $vp_id -class_id $class_id -cell_id $cell_id";
+            $command .= " -path_base $path_base";
+            $command .= " -hostname $host";
+            $command .= " -dtime_photom " . ((DateTime->now->mjd - $vpstart) * 86400);
 
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -138,7 +166,70 @@
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                 my $err_message = "Unable to perform vptool -addvpcell";
-                &my_die("$err_message: $error_code", $vp_id, $class_id, $cell_id, $error_code);
+                &my_die("$err_message: $error_code", $vp_id, $error_code);
             }
         }
+    }
+}
+if ($dest_id) {
+    my $command = "$vptool -processedcell -vp_id $vp_id";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, $error_code);
+    }
+
+    my $output = join "", @$stdout_buf;
+    my $cells;
+    if ($output) {
+        my $metadata = $mdcParser->parse($output) or
+            &my_die("Unable to parse metadata config doc", $vp_id, $PS_EXIT_PROG_ERROR);
+        $cells = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", $vp_id, $PS_EXIT_PROG_ERROR);
+    }
+    my ($reglist, $reglistName) = tempfile("/tmp/filelist.$vp_id.XXXX", UNLINK => !$save_temps);
+
+    if ($redirect) {
+        print $reglist "$logDest|||text|\n";
+    }
+    my @exts = qw( vpt fits );
+    my @types = qw( table fits );
+    foreach my $cell (@$cells) {
+        my $path_base = $cell->{path_base};
+        for (my $i = 0; $i < scalar @exts; $i++) {
+            my $file = "$path_base.$exts[$i]";
+            my $type = $types[$i];
+
+            print $reglist "$file|||$type|\n";
+        }
+            
+    }
+    close $reglist or &my_die("failed to close $reglistName", $vp_id, $PS_EXIT_UNKNOWN_ERROR);
+    unless ($no_update or $no_op) {
+        my $fileset = basename($outroot);
+        my $command = "$dsreg --add $fileset --product $product --type dump --list $reglistName";
+        $command .= " --abspath --link";
+        $command .= " --dbname $ds_dbname --dbhost $ds_dbhost";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+             &my_die("Unable to perform $command error_code: $error_code", $vp_id, $error_code);
+        }
+    }
+}
+
+unless ($no_update) {
+    my $command = "$vptool -updaterun -vp_id $vp_id -set_state full"; 
+    $command .= " -set_outroot $outroot";
+    $command .= " -set_hostname $host";
+    $command .= " -set_dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400);
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        my $err_message = "Unable to perform vptool -updaterun";
+        &my_die("$err_message: $error_code", $vp_id, $error_code);
     }
 }
@@ -149,7 +240,5 @@
 {
     my $msg = shift;            # Warning message on die
-    my $vp_id = shift;
-    my $class_id = shift;       # chip identifier
-    my $cell_id = shift;       # chip identifier
+    my $vp_id = shift;          # vpRun id
     my $exit_code = shift;      # Exit code to add
 
@@ -157,11 +246,12 @@
 
     warn($msg);
-    if (defined $vp_id and defined $class_id and not $no_update) {
-        my $command = "$vptool -vp_id $vp_id -class_id $class_id -cell_id -fault $exit_code";
-        $command .= " -addvpcell";
-        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
-        $command .= " -hostname $host" if defined $host;
-        $command .= " -path_base $outroot" if defined $outroot;
-        $command .= " -dbname $dbname" if defined $dbname;
+    if (defined $vp_id and not $no_update) {
+        my $command = "$vptool -vp_id $vp_id";
+        $command .= " -updaterun";
+        $command .= " -set_fault $exit_code";
+        $command .= " -set_hostname $host" if defined $host;
+        $command .= " -set_outroot $outroot" if defined $outroot;
+        $command .= " -set_dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400);
+
         run(command => $command, verbose => $verbose);
     }
@@ -169,9 +259,3 @@
 }
 
-END {
-    my $exit = $?;
-    system("sync") == 0 or die "failed to execute sync: $!";
-    $? = $exit;
-}
-
 __END__
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_overlap.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_overlap.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_overlap.pl	(revision 32630)
@@ -57,5 +57,5 @@
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
-$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $warp_id, $PS_EXIT_SYS_ERROR ) if $logfile;
+$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $warp_id, $PS_EXIT_SYS_ERROR ) if $logfile;
 
 &my_die("Tessellation identifier not provided: $tess_dir", $warp_id, $PS_EXIT_SYS_ERROR) unless $tess_dir ne "NULL";
Index: branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_skycell.pl
===================================================================
--- branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_skycell.pl	(revision 32363)
+++ branches/eam_branches/ipp-20110906/ippScripts/scripts/warp_skycell.pl	(revision 32630)
@@ -38,5 +38,4 @@
 
 my ($warp_id, $skycell_id, $warp_skyfile_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $run_state, $magicked, $verbose, $no_update, $no_op, $redirect, $save_temps);
-my $zaplog = 0;
 
 GetOptions(
@@ -55,5 +54,4 @@
     'no-update'           => \$no_update, # Don't update the database?
     'no-op'               => \$no_op, # Don't do any operations?
-    'zaplog'              => \$zaplog, # delete previous logfile
     'redirect-output'     => \$redirect,
     'save-temps'          => \$save_temps, # Save temporary files?
@@ -77,9 +75,9 @@
 my $do_stats;
 if ($run_state eq 'new') {
-    $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, $zaplog);
+    $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id);
     $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1);
     $do_stats = 1;
 } elsif ($run_state eq 'update')  {
-    $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 1);
+    $logDest = $ipprc->filename("LOG.EXP.UPDATE", $outroot, $skycell_id);
     $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, $skycell_id, 1);
 } else {
@@ -93,5 +91,5 @@
 }
 
-$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect;
+$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect;
 
 # Recipes to use based on reduction class
@@ -333,5 +331,5 @@
             }
         }
-
+	print "Quality: $quality\n";
         unless ($no_update) {
             # XXX change -tess_id to -tess_dir when db is updated
@@ -358,4 +356,19 @@
         }
     } else {
+
+	my $quality = 0;
+
+	unless ($ipprc->file_exists($outputPSF)) {
+	    $quality = 8006; # bad data.
+	}
+#         if (!$quality) {
+#             check_output($outputImage, 0);
+#             check_output($outputMask, 0);
+#             check_output($outputWeight, 0);
+#             check_output($outputSources, 1);
+#             check_output($outputPSF, 1) if metadataLookupBool($recipe, 'PSF')  ;
+#         }
+	print "Quality: $quality\n";
+
         # $run_state eq 'update'
         unless ($no_update) {
@@ -364,4 +377,5 @@
             $command .= " -skycell_id $skycell_id";
             $command .= " -set_magicked $magicked" if $magicked;
+	    $command .= " -set_quality $quality" if $quality;
             $command .= " -dbname $dbname"   if defined $dbname;
 
@@ -407,4 +421,17 @@
 
     &my_die("Couldn't find expected output file: $file",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
+
+    # Funpack to confirm we've really made things correctly
+    my $diskfile = $ipprc->file_resolve($file);
+    if ($diskfile =~ /fits/) {
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
+	my $check_command = "$funpack -S $diskfile > /dev/null";
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $check_command, verbose => $verbose);
+	if (!$success) {
+	    &my_die("Output file not a valid fits file: $file", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
+	}
+    }
+    #####
 
     if ($replicate and $neb) {