Changeset 25299

Timestamp:

Sep 8, 2009, 5:32:55 PM (17 years ago)

Author:

watersc1

Message:

Merge my branch back into the trunk. This includes:

ipp_cleanup.pl : and associated edits to allow cleanup to work
burntool : updated to newest version, ipp_apply_burntool.pl modified to use persist=t options
ppImageBurntoolMask.c : masks region burntool identifies
psastro : allow selection of wcs header format
addtool : splits addstar step off from the camera stage

Location:

trunk

Files:

• . (modified) (1 prop)
• DataStore/scripts/dsget (modified) (1 diff)
• dbconfig/add.md (copied) (copied from branches/czw_branch/cleanup/dbconfig/add.md )
• dbconfig/changes.txt (modified) (1 diff)
• dbconfig/ipp.m4 (modified) (1 diff)
• doc/misc/docgen.pl (modified) (1 prop)
• extsrc/gpcsw/gpcsrc/fits/burntool/burnparams.h (modified) (1 diff)
• extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c (modified) (2 diffs)
• extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 (modified) (3 diffs)
• extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.c (modified) (6 diffs)
• extsrc/gpcsw/gpcsrc/fits/burntool/persistio.c (modified) (4 diffs)
• extsrc/gpcsw/gpcsrc/fits/burntool/trailfit.c (modified) (1 diff)
• extsrc/gpcsw/gpcsrc/fits/libfh/fh.c (modified) (3 diffs)
• extsrc/gpcsw/gpcsrc/fits/libfh/fh.h (modified) (3 diffs)
• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/scripts/addstar_run.pl (copied) (copied from branches/czw_branch/cleanup/ippScripts/scripts/addstar_run.pl )
• ippScripts/scripts/camera_exp.pl (modified) (4 diffs)
• ippScripts/scripts/ipp_cleanup.pl (modified) (18 diffs)
• ippScripts/scripts/magic_destreak_revert.pl (modified) (1 prop)
• ippScripts/scripts/stack_skycell.pl (modified) (8 diffs)
• ippTasks/Makefile.am (modified) (1 diff)
• ippTasks/addstar.pro (copied) (copied from branches/czw_branch/cleanup/ippTasks/addstar.pro )
• ippTasks/pantasks.pro (modified) (3 diffs)
• ippTasks/summit.copy.pro (modified) (1 diff)
• ippTools/doc/addstar_flow.txt (copied) (copied from branches/czw_branch/cleanup/ippTools/doc/addstar_flow.txt )
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/addtool_find_pendingexp.sql (copied) (copied from branches/czw_branch/cleanup/ippTools/share/addtool_find_pendingexp.sql )
• ippTools/share/addtool_queue_cam_id.sql (copied) (copied from branches/czw_branch/cleanup/ippTools/share/addtool_queue_cam_id.sql )
• ippTools/share/addtool_reset_faulted_runs.sql (copied) (copied from branches/czw_branch/cleanup/ippTools/share/addtool_reset_faulted_runs.sql )
• ippTools/share/addtool_revertprocessedexp.sql (copied) (copied from branches/czw_branch/cleanup/ippTools/share/addtool_revertprocessedexp.sql )
• ippTools/share/dettool_pendingcleanup_normalizedimfile.sql (modified) (1 diff)
• ippTools/share/dettool_pendingcleanup_stacked.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (2 diffs)
• ippTools/share/pxadmin_drop_tables.sql (modified) (1 diff)
• ippTools/src/Makefile.am (modified) (5 diffs)
• ippTools/src/addtool.c (copied) (copied from branches/czw_branch/cleanup/ippTools/src/addtool.c )
• ippTools/src/addtool.h (copied) (copied from branches/czw_branch/cleanup/ippTools/src/addtool.h )
• ippTools/src/addtoolConfig.c (copied) (copied from branches/czw_branch/cleanup/ippTools/src/addtoolConfig.c )
• ippTools/src/camtool.c (modified) (1 diff)
• ippTools/src/dettoolConfig.c (modified) (2 diffs)
• ippTools/src/pxadd.c (copied) (copied from branches/czw_branch/cleanup/ippTools/src/pxadd.c )
• ippTools/src/pxadd.h (copied) (copied from branches/czw_branch/cleanup/ippTools/src/pxadd.h )
• ippTools/src/pxtools.h (modified) (1 diff)
• ippTools/src/regtool.c (modified) (3 diffs)
• ippTools/src/warptool.c (modified) (1 diff)
• ippconfig/gpc1/format_20090220.config (modified) (2 diffs)
• ippconfig/recipes/masks.16bit.config (modified) (1 diff)
• ippconfig/recipes/ppImage.config (modified) (2 diffs)
• ippconfig/recipes/psastro.config (modified) (1 diff)
• ppImage/src/Makefile.am (modified) (1 diff)
• ppImage/src/ppImage.h (modified) (4 diffs)
• ppImage/src/ppImageBurntoolMask.c (copied) (copied from branches/czw_branch/cleanup/ppImage/src/ppImageBurntoolMask.c )
• ppImage/src/ppImageDetrendReadout.c (modified) (1 diff)
• ppImage/src/ppImageOptions.c (modified) (4 diffs)
• ppImage/src/ppImageSetMaskBits.c (modified) (1 diff)
• psModules/src/astrom/pmAstrometryWCS.c (modified) (6 diffs)
• psModules/src/astrom/pmAstrometryWCS.h (modified) (1 diff)
• psModules/src/camera/pmFPA.h (modified) (1 diff)
• psModules/src/camera/pmReadoutFake.c (modified) (1 prop)
• psastro/src/psastroChipAstrom.c (modified) (1 diff)
• tools/ipp_apply_burntool.pl (modified) (1 diff)

trunk/DataStore/scripts/dsget

@@ -69 +69 @@
         undef $copies;
     }
-    if ($copies < 1) {
+    elsif ($copies < 1) {
         die "--copies must be >= 1";
     }

trunk/dbconfig/changes.txt

@@ -1211 +1211 @@
 ALTER TABLE publishDone ADD COLUMN hostname VARCHAR(64) AFTER path_base;
 ALTER TABLE publishDone ADD COLUMN dtime_script FLOAT AFTER hostname;
+
+-- Version 1.1.55
+-- addstar stage tables:
+
+CREATE TABLE addRun (
+    add_id BIGINT AUTO_INCREMENT,
+    cam_id BIGINT,
+    state VARCHAR(64),
+    workdir VARCHAR(255),
+    workdir_state VARCHAR(64),
+    label VARCHAR(64),
+    dvodb VARCHAR(255),
+    magicked BIGINT,
+    PRIMARY KEY(add_id),
+    KEY(add_id),
+    KEY(cam_id),
+    KEY(state),
+    KEY(workdir_state),
+    KEY(label),
+    INDEX(add_id, cam_id),
+    FOREIGN KEY(cam_id) REFERENCES camRun(cam_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE addProcessedExp (
+    add_id BIGINT AUTO_INCREMENT,
+    dtime_addstar FLOAT,
+    n_stars INT,
+    path_base VARCHAR(255),
+    fault SMALLINT NOT NULL,
+    PRIMARY KEY(add_id),
+    FOREIGN KEY(add_id) REFERENCES addRun(add_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE addMask (
+    label VARCHAR(64),
+    PRIMARY KEY(label)
+) ENGINE=innodb DEFAULT CHARSET=latin1;

trunk/dbconfig/ipp.m4

@@ -14 +14 @@
 include(chip.md)
 include(cam.md)
+include(add.md)
 include(fake.md)
 include(warp.md)

trunk/extsrc/gpcsw/gpcsrc/fits/burntool/burnparams.h

@@ -55 +55 @@
 EXTERN int EXPIRE_TRAIL_TIME;   /* Expire trails after this interval */
 
+EXTERN int PERSIST_RETAIN;      /* Retain bad-slope persistence fits */
+
 #endif /* _INCLUDED_burnparams_ */

trunk/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c

@@ -100 +100 @@
    EXPIRE_TRAIL_TIME = 2000;    /* Expire a persist after this [sec] */
 
+   PERSIST_RETAIN = 0;          /* Retain persists with bad slopes? */
+
 /* Parse the args */
    cellxy = -1;
@@ -174 +176 @@
       } else if(strncmp(argv[i], "infits=", 7) == 0) { /* infits=fname */
          persistfitsfile = argv[i] + 7;
+
+/* Keep persistence streaks which had a bad slope? */
+      } else if(strncmp(argv[i], "persist=", 8) == 0) {/* persist={t|f} */
+         PERSIST_RETAIN = argv[i][8] == 'y' || argv[i][8] == '1' || argv[i][8] == 't';
 
 /* Output file for PSF gallery */

trunk/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1

@@ -53 +53 @@
         power laws; downward burns as exponentials.
         
+        The fit to downward, persistence burns may have an unreasonable "slope"
+        (meaning exponential sign), increasing away from the burn origin.
+        These are discarded unless "persist=t" is invoked.  It is possible
+        for the fit to be fooled by uncataloged stars, so the persistence
+        can be kept in the output file with "persist=t" (although no fit
+        correction is applied) and they will propagate from input to output
+        until they finally achieve a legal fit with negligible amplitude.
+
         Burntool also identifies really blasted areas which are saturated from
         top to bottom.  These cannot be fitted, but are carried along for
@@ -231 +239 @@
                 'in' takes precedence.
 
+        persist={t|f}
+                Retain persistence streaks whose fit slope turned out to be
+                unreasonable.
+
         out=fname      
                 Output file for burn streaks
@@ -332 +344 @@
 BUGS:
         090224: Still in development
+        090810: Squished a few memory bugs
 
 SEE ALSO:

trunk/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.c

@@ -222 +222 @@
       for(k=0; k<cell[j].npersist; k++) 
       {
-         if(cell[j].persist[k].fiterr) continue;
-         if(cell[j].persist[k].nfit <= 0) continue;
+         if(PERSIST_RETAIN) {
+/* Keep fits which have a dubious slope */
+            if(cell[j].persist[k].fiterr != FIT_SLOPE_ERROR) {
+               if(cell[j].persist[k].fiterr) continue;
+               if(cell[j].persist[k].nfit <= 0) continue;
+            }
+         } else {
+            if(cell[j].persist[k].fiterr) continue;
+            if(cell[j].persist[k].nfit <= 0) continue;
+         }
          num_areas++;
          num_fits += cell[j].persist[k].nfit;
@@ -232 +240 @@
       {
          if(!cell[j].burn[k].burned) continue;
-         if(cell[j].burn[k].fiterr && 
-            cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
-         if(cell[j].burn[k].nfit <= 0) continue;
+         if(PERSIST_RETAIN) {
+/* Keep fits which have a dubious slope */
+            if(cell[j].burn[k].fiterr != FIT_SLOPE_ERROR) {
+               if(cell[j].burn[k].fiterr && 
+                  cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+               if(cell[j].burn[k].nfit <= 0) continue;
+            }
+         } else {
+            if(cell[j].burn[k].fiterr && 
+               cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+            if(cell[j].burn[k].nfit <= 0) continue;
+         }
          num_areas++;
          num_fits += cell[j].burn[k].nfit;
@@ -329 +346 @@
       for(k=0; k<cell[j].npersist; k++) 
       {
-         if(cell[j].persist[k].fiterr) continue;
-         if(cell[j].persist[k].nfit <= 0) continue;     
+         if(PERSIST_RETAIN) {
+/* Keep fits which have a dubious slope */
+            if(cell[j].persist[k].fiterr != FIT_SLOPE_ERROR) {
+               if(cell[j].persist[k].fiterr) continue;
+               if(cell[j].persist[k].nfit <= 0) continue;     
+            }
+         } else {
+            if(cell[j].persist[k].fiterr) continue;
+            if(cell[j].persist[k].nfit <= 0) continue;     
+         }
 
          result = write_area_row(hu, data, table, row++, &(cell[j].persist[k]));
@@ -340 +365 @@
       {
          if(!cell[j].burn[k].burned) continue;
-         if(cell[j].burn[k].fiterr && 
-            cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
-         if(cell[j].burn[k].nfit <= 0) continue;
+         if(PERSIST_RETAIN) {
+/* Keep fits which have a dubious slope */
+            if(cell[j].burn[k].fiterr != FIT_SLOPE_ERROR) {
+               if(cell[j].burn[k].fiterr && 
+                  cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+               if(cell[j].burn[k].nfit <= 0) continue;
+            }
+         } else {
+            if(cell[j].burn[k].fiterr && 
+               cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+            if(cell[j].burn[k].nfit <= 0) continue;
+         }
 
          result = write_area_row(hu, data, table, row++, &(cell[j].burn[k]));
@@ -426 +460 @@
       for(k=0; k<cell[j].npersist; k++) 
       {
-         if(cell[j].persist[k].fiterr) continue;
+         if(PERSIST_RETAIN) {
+/* Keep fits which have a dubious slope */
+            if(cell[j].persist[k].fiterr != FIT_SLOPE_ERROR) {
+               if(cell[j].persist[k].fiterr) continue;
+            }
+         } else {
+            if(cell[j].persist[k].fiterr) continue;
+         }
          for(i=0; i<cell[j].persist[k].nfit; i++) 
          {
@@ -441 +482 @@
       {
          if(!cell[j].burn[k].burned) continue;
-         if(cell[j].burn[k].fiterr && 
-            cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+         if(PERSIST_RETAIN) {
+/* Keep fits which have a dubious slope */
+            if(cell[j].burn[k].fiterr != FIT_SLOPE_ERROR) {
+               if(cell[j].burn[k].fiterr && 
+                  cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+            }
+         } else {
+            if(cell[j].burn[k].fiterr && 
+               cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+         }
 
          for(i=0; i<cell[j].burn[k].nfit; i++) 

trunk/extsrc/gpcsw/gpcsrc/fits/burntool/persistio.c

@@ -51 +51 @@
              &boxbuf[nbox].slope, &boxbuf[nbox].nfit,
              &boxbuf[nbox].sxfit, &boxbuf[nbox].exfit);
-      if(boxbuf[nbox].nfit <= 0) continue;
-      boxbuf[nbox].zero = (double *)calloc(boxbuf[nbox].nfit, sizeof(double));
-      boxbuf[nbox].xfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int));
-      boxbuf[nbox].yfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int));
-      for(i=0; i<boxbuf[nbox].nfit; i++) {
-         if(fgets(line, 1024, fp) == NULL) {
-            fprintf(stderr, "\rerror: short read of burn lines\n");
-            return(-1);
-         }
-         sscanf(line, "%d %d %lf\n", &boxbuf[nbox].xfit[i], 
-                &boxbuf[nbox].yfit[i], &boxbuf[nbox].zero[i]);
+      if(boxbuf[nbox].nfit > 0) {
+         boxbuf[nbox].zero = (double *)calloc(boxbuf[nbox].nfit, sizeof(double));
+         boxbuf[nbox].xfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int));
+         boxbuf[nbox].yfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int));
+         for(i=0; i<boxbuf[nbox].nfit; i++) {
+            if(fgets(line, 1024, fp) == NULL) {
+               fprintf(stderr, "\rerror: short read of burn lines\n");
+               return(-1);
+            }
+            sscanf(line, "%d %d %lf\n", &boxbuf[nbox].xfit[i], 
+                   &boxbuf[nbox].yfit[i], &boxbuf[nbox].zero[i]);
+         }
       }
       boxbuf[nbox].fiterr = 0;
@@ -147 +148 @@
       for(kp=0; kp<n; kp++) {
          k = boxid[kp];
-         zk = box[k].zero[box[k].nfit/2];
+         zk = 0.0;
+         if(box[k].nfit > 0) zk = box[k].zero[box[k].nfit/2];
          for(jp=kp+1; jp<n; jp++) {
             j = boxid[jp];
-            zj = box[j].zero[box[j].nfit/2];
+            zj = 0.0;
+            if(box[j].nfit > 0) zj = box[j].zero[box[j].nfit/2];
             if(ABS(yctr[jp]-yctr[kp]) > DIFFERENT_STREAK) {
 /* Trim back the feebler streak */
@@ -238 +241 @@
 /* First: patched up persists */
       for(k=0; k<cell[j].npersist; k++) {
-         if(cell[j].persist[k].fiterr) continue;
-         if(cell[j].persist[k].nfit <= 0) continue;
+
+         if(PERSIST_RETAIN) {
+/* Keep fits which have a dubious slope */
+            if(cell[j].persist[k].fiterr != FIT_SLOPE_ERROR) {
+               if(cell[j].persist[k].fiterr) continue;
+               if(cell[j].persist[k].nfit <= 0) continue;
+            }
+         } else {
+            if(cell[j].persist[k].fiterr) continue;
+            if(cell[j].persist[k].nfit <= 0) continue;
+         }
          fprintf(fp, "%3d %7d  %3d %3d %5d %3d  %3d %3d  %3d %3d  %3d %3d %3d %3d %3d %3d %3d %3d  %1d %1d %9.6f %3d %3d %3d\n",
                  j, cell[j].persist[k].time, 
@@ -262 +274 @@
       for(k=0; k<cell[j].nburn; k++) {
          if(!cell[j].burn[k].burned) continue;
-         if(cell[j].burn[k].fiterr && 
-            cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
-         if(cell[j].burn[k].nfit <= 0) continue;
+         if(PERSIST_RETAIN) {
+/* Keep fits which have a dubious slope */
+            if(cell[j].burn[k].fiterr != FIT_SLOPE_ERROR) {
+               if(cell[j].burn[k].fiterr && 
+                  cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+               if(cell[j].burn[k].nfit <= 0) continue;
+            }
+         } else {
+            if(cell[j].burn[k].fiterr && 
+               cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+            if(cell[j].burn[k].nfit <= 0) continue;
+         }
 
          i = (cell[j].burn[k].ex - cell[j].burn[k].sx + 1) / 2;

trunk/extsrc/gpcsw/gpcsrc/fits/burntool/trailfit.c

@@ -226 +226 @@
 /* FIXME: sanity check fits */
    if(slope >= FIT_MAX_SLOPE || slope < FIT_MIN_SLOPE) {
+      box->slope = slope;
+      box->nfit = 0;
       box->fiterr = FIT_SLOPE_ERROR;
       return(-1);

trunk/extsrc/gpcsw/gpcsrc/fits/libfh/fh.c

@@ -1039 +1039 @@
          newval[strlen(newval) - 1] = '\0';
          fh_set_str(hu, idx, name, newval, comment);
+         free(newval);
          return;
       }
@@ -1519 +1520 @@
       i++;
    }
+   return FH_SUCCESS;
+}
+
+static void
+pad_header(HeaderUnit hu, int n)
+{
+   int i;
+   double idx = 900000000;
+
+   for (i = 0; i < n; i++)
+   {
+      fh_set_card(hu, idx++, FH_RESERVE);
+   }
+}
+
+fh_result
+fh_copy(HeaderUnit hu, const HeaderUnit source_)
+{
+   fh_result result;
+   int reserve;
+
+   result = fh_merge(hu, source_);
+   if (result != FH_SUCCESS) return result;
+   reserve = fh_get_reserve(source_);
+   if (reserve) pad_header(hu, reserve);
    return FH_SUCCESS;
 }
@@ -1757 +1783 @@
 }
 
+int
+fh_get_reserve(HeaderUnit hu)
+{
+   HeaderUnitStruct* list = FH_HU(hu);
+
+   if (!list)
+   {
+      log_error("invalid argument to fh_get_reserve()");
+      return -1;
+   }
+
+   if (list->reserve)
+      return list->reserve;
+   else
+      return list->reserve_found;
+}
+
 fh_result
 fh_rewrite(HeaderUnit hu)

trunk/extsrc/gpcsw/gpcsrc/fits/libfh/fh.h

@@ -174 +174 @@
  * is used to create a new FITS header.  NOT for use with fh_rewrite().
  */
+int fh_get_reserve(HeaderUnit hu);
+/*
+ * Get the current setting or the number of reserve cards found in
+ * a header unit.  Returns -1 if hu is invalid.
+ */
 fh_result fh_validate(HeaderUnit hu); /* fh_validate.c; for use by fhtool.c */
 
@@ -386 +391 @@
 double fh_idx(HeaderUnit hu); /* idx of the last card returned by fh_next */
 fh_result fh_merge(HeaderUnit hu, const HeaderUnit source); /* source unchanged */
+fh_result fh_copy(HeaderUnit hu, const HeaderUnit source); /* fh_merge+keep reserve */
 
 /* ---------------------------------------------------------
@@ -459 +465 @@
  */
 
-
 #endif /* _INCLUDED_fh */

trunk/ippScripts/Build.PL

@@ -49 +49 @@
         scripts/chip_imfile.pl
         scripts/camera_exp.pl
+        scripts/addstar_run.pl
         scripts/fake_imfile.pl
         scripts/warp_overlap.pl

trunk/ippScripts/scripts/camera_exp.pl

@@ -93 +93 @@
 &my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe2;
 
-my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use
-&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar;
+#my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use
+#&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar;
 
 my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use
@@ -205 +205 @@
 }
 
-my $dtime_addstar = 0;
+#my $dtime_addstar = 0;
 
 unless ($no_op) {
@@ -297 +297 @@
 
         # run addstar on the output fpaObjects (if a DVO database is defined)
-        if (defined $dvodbReal and ($run_state eq 'new')) {
-
-            ## XXX the camera analysis can either save the full set of
-            ## detections, or just the image metadata, in the dvodb
-
-            ## get the addstar recipe for this camera and CAMERA reduction
-            $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -";
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR);
-            }
-            my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
-                &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR);
-
-            ## allow the dvodb to save only images, or the full detection set
-            my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY');
-
-            # XXX this construct requires the user to have a valid .ptolemyrc
-            # XXX which in turn points at ippconfig/dvo.site
-            # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default)
-            # XXX this needs to be converted to addstar_client...
-
-            my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar
-            my $command;
-            $command  = "$addstar -D CAMERA $camdir -update";
-            $command .= " -image" if $imagesOnly;
-            $command .= " -D CATDIR $dvodbReal";
-
-            my $realFile = $ipprc->file_resolve($fpaObjects);
-            $command .= " $realFile";
-
-            my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code);
-            }
-            $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);   # MJD of starting script
-        }
+#         if (defined $dvodbReal and ($run_state eq 'new')) {
+
+#             ## XXX the camera analysis can either save the full set of
+#             ## detections, or just the image metadata, in the dvodb
+
+#             ## get the addstar recipe for this camera and CAMERA reduction
+#             $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -";
+#             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#                 run(command => $command, verbose => $verbose);
+#             unless ($success) {
+#                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+#                 &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR);
+#             }
+#             my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+#                 &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR);
+
+#             ## allow the dvodb to save only images, or the full detection set
+#             my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY');
+
+#             # XXX this construct requires the user to have a valid .ptolemyrc
+#             # XXX which in turn points at ippconfig/dvo.site
+#             # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default)
+#             # XXX this needs to be converted to addstar_client...
+
+#             my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar
+#             my $command;
+#             $command  = "$addstar -D CAMERA $camdir -update";
+#             $command .= " -image" if $imagesOnly;
+#             $command .= " -D CATDIR $dvodbReal";
+
+#             my $realFile = $ipprc->file_resolve($fpaObjects);
+#             $command .= " $realFile";
+
+#             my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
+
+#             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#                 run(command => $command, verbose => $verbose);
+#             unless ($success) {
+#                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+#                 &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code);
+#             }
+#             $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);   # MJD of starting script
+#         }
     }
 }
@@ -353 +353 @@
     $fpaCommand .= " -hostname $host" if defined $host;
     $fpaCommand .= " -dtime_script $dtime_script";
-    $fpaCommand .= " -dtime_addstar $dtime_addstar";
+#    $fpaCommand .= " -dtime_addstar $dtime_addstar";
 } else {
     $fpaCommand .= " -updaterun -state full";

trunk/ippScripts/scripts/ipp_cleanup.pl

@@ -60 +60 @@
 
 
-my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff  => 1,
-               detrend.process.imfile => 1, detrend.process.exp => 1, detrend.stack.imfile => 1,
-               detrend.normstat.imfile => 1, detrend.norm.imfile => 1, detrend.norm.exp => 1,
-               detrend.resid.imfile => 1, detrend.resid.exp => 1 );
+my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff"  => 1,
+               "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1,
+               "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1,
+               "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 );
 unless ($stages{$stage}) {
     die "unknown stage $stage for ipp_cleanup.pl\n";
@@ -119 +119 @@
         # don't clean up unless the data needed to update is available
         # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
+        # goto_scrubbed now requires the config file to not exist.
         if ($mode eq "goto_cleaned") {
             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
-            print STDERR "CHIP: CONFIG_FILE : $config_file\n";
+
             if (!$config_file or ! -e $config_file) {
                 print STDERR "skipping cleanup for chipRun $stage_id $class_id "
@@ -128 +129 @@
             }
         }
+        elsif ($mode eq "goto_scrubbed") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
+
+            if ($config_file and -e $config_file) {
+                print STDERR "skipping scrubbed for chipRun $stage_id $class_id "
+                    . " because config file is present\n";
+                $status = 0;
+            }
+        }
 
         if ($status) {
@@ -160 +170 @@
             if ($mode eq "goto_purged") {
                 $command .= " -topurgedimfile";
-            } else {
+            }
+            elsif ($mode eq "goto_cleaned") {
                 $command .= " -tocleanedimfile";
             }
+            elsif ($mode eq "goto_scrubbed") {
+                $command .= " -toscrubbedimfile";
+            }
+
             $command .= " -dbname $dbname" if defined $dbname;
 
@@ -218 +233 @@
     my $status = 1;
     # don't clean up unless the data needed to update is available
+    # goto_scrubbed now requires the config file to not be present
     if ($mode eq "goto_cleaned") {
         my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
@@ -225 +241 @@
             $status = 0;
         }
+    }
+    elsif ($mode eq "goto_scrubbed") {
+        my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
+
+        if ($config_file and -e $config_file) {
+            print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n";
+            $status = 0;
+        }
     }
     if ($status) {
@@ -246 +270 @@
         }
         if ($mode eq "goto_scrubbed") {
-            $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";
+            $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed";
         }
         if ($mode eq "goto_purged") {
@@ -311 +335 @@
             }
         }
+        elsif ($mode eq "goto_scrubbed") {
+            my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);
+
+            if ($config_file and -e $config_file) {
+                print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" .
+                    " because config file is present\n";
+                $status = 0;
+            }
+        }
         if ($status) {
             # delete the temporary image datafiles
@@ -337 +370 @@
             if ($mode eq "goto_purged") {
                 $command .= " -topurgedskyfile";
-            } else {
+            } 
+            elsif ($mode eq "goto_cleaned") {
                 $command .= " -tocleanedskyfile";
             }
+            elsif ($mode eq "goto_scrubbed") {
+                $command .= " -toscrubbedskyfile";
+            }
             $command .= " -dbname $dbname" if defined $dbname;
 
@@ -395 +432 @@
         if ($mode eq "goto_cleaned") {
             my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id);
-            print STDERR "MY CONFIG FILE = $config_file\n";
-            printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file);
+
             $config_file =~ s%^file://%%;
             if (!$config_file or ! -e $config_file) {
@@ -404 +440 @@
             }
             $config_file = 'file://' . $config_file;
+        }
+        elsif ($mode eq "goto_scrubbed") {
+            my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id);
+            $config_file =~ s%^file://%%;
+            if ($config_file and -e $config_file) {
+                print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" .
+                    " because config file is present\n";
+                $status = 0;
+            }
         }
         if ($status) {
@@ -428 +473 @@
             if ($mode eq "goto_purged") {
                 $command .= " -updaterun -state purged";
-            } else {
+            } 
+            elsif ($mode eq "goto_cleaned") {
                 $command .= " -updaterun -state cleaned";
+            }
+            elsif ($mode eq "goto_scrubbed") {
+                $command .= " -updaterun -state scrubbed";
             }
             $command .= " -dbname $dbname" if defined $dbname;
@@ -485 +534 @@
         if ($mode eq "goto_cleaned") {
             my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
-            print STDERR "MY CONFIG FILE = $config_file\n";
-            printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file);
+
             $config_file =~ s%^file://%%;
             if (!$config_file or ! -e $config_file) {
@@ -494 +542 @@
             }
             $config_file = 'file://' . $config_file;
+        }
+        elsif ($mode eq "goto_scrubbed") {
+            my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
+            $config_file =~ s%^file://%%;
+            if ($config_file and -e $config_file) {
+                print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" .
+                    " because config file ($config_file) is present\n";
+                $status = 0;
+            }
         }
         if ($status) {
@@ -529 +586 @@
         if ($status) {
             my $command = "$difftool -diff_id $stage_id";
-#           my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id";
+
             if ($mode eq "goto_purged") {
                 $command .= " -updaterun -state purged";
-            } else {
+            }
+            elsif ($mode eq "goto_cleaned") {
                 $command .= " -updaterun -state cleaned";
             }
+            elsif ($mode eq "goto_scrubbed") {
+                $command .= " -updaterun -state scrubbed";
+            }
+
             $command .= " -dbname $dbname" if defined $dbname;
             
@@ -545 +607 @@
         } else {
             my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state";
-#           my $command = "$difftool -updaterun -diff_id $stage_id -skycell_id $skycell_id -state $error_state";
+
             $command .= " -dbname $dbname" if defined $dbname;
             
@@ -560 +622 @@
 }
 if ($stage eq 'fake') {
-    die "ipp_cleanup.pl -stage fake not yet implemented. Probably will just mark database cleaned.\n";
+    print STDERR "This does not seem to work at present, as no files exist. Terminating quietly.\n";
+    exit(0);
+    die "--stage_id required for stage fake\n" if !$stage_id;
+    ### select the imfiles for this entry
+
+    # this stage uses 'chiptool'
+    my $faketool = can_run('faketool') or die "Can't find faketool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $imfiles;                      # Array of component files
+    my $command = "$faketool -pendingcleanupimfile -fake_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
+    }
+
+    # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
+    if (@$stdout_buf == 0)  {
+        my $command = "$faketool -fake_id $stage_id -updaterun -set_state new";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
+        }
+        exit 0;
+    }
+
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # extract the metadata for the files into a hash list
+    $imfiles = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # loop over all of the imfiles, determine the path_base and class_id for each
+    foreach my $imfile (@$imfiles) {
+        my $class_id = $imfile->{class_id};
+        my $path_base = $imfile->{path_base};
+        my $status = 1;
+
+        # don't clean up unless the data needed to update is available
+        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
+        # goto_scrubbed now requires the config file to not exist.
+        if ($mode eq "goto_cleaned") {
+            my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id);
+
+            if (!$config_file or ! -e $config_file) {
+                print STDERR "skipping cleanup for fakeRun $stage_id $class_id "
+                    . " because config file is missing\n";
+                $status = 0;
+            }
+        }
+        elsif ($mode eq "goto_scrubbed") {
+            my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id);
+
+            if ($config_file and -e $config_file) {
+                print STDERR "skipping scrubbed for fakeRun $stage_id $class_id "
+                    . " because config file is present\n";
+                $status = 0;
+            }
+        }
+
+        if ($status) {
+            # array of actual filenames to delete
+            my @files = ();
+
+            # delete the temporary image datafiles
+            addFilename (\@files, "PPSIM.OUTPUT.MEF", $path_base, $class_id);
+            addFilename (\@files, "PPSIM.OUTPUT.SPL", $path_base, $class_id);
+            addFilename (\@files, "PPSIM.FAKE.CHIP", $path_base, $class_id);
+            addFilename (\@files, "PPSIM.FORCE.CHIP", $path_base, $class_id);
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+                addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id);
+                addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id);
+                addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id);
+            }
+
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+
+        if ($status)  {
+            my $command = "$faketool -fake_id $stage_id -class_id $class_id";
+            if ($mode eq "goto_purged") {
+                $command .= " -topurgedimfile";
+            }
+            elsif ($mode eq "goto_cleaned") {
+                $command .= " -tocleanedimfile";
+            }
+            elsif ($mode eq "goto_scrubbed") {
+                $command .= " -toscrubbedimfile";
+            }
+
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
+            }
+        } else {
+
+            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
+            my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code);
+            }
+        }
+    }
+    exit 0;
+
 } 
-# fake : faketool : -pendingcleanupimfile (loop over imfiles)
+# Detrend stages
+if ($stage eq "detrend.process.imfile") {
+
+    die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id;
+    ### select the imfiles for this entry
+
+    # this stage uses 'dettool'
+    my $dettool = can_run('dettool') or die "Can't find chiptool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $imfiles;                      # Array of component files
+    my $command = "$dettool -pendingcleanup_processedimfile -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
+    }
+
+    # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
+    if (@$stdout_buf == 0)  {
+        exit 0; # Silently exit if there's nothing to do.  I don't know how we'd ever get here, but let's be safe.
+    }
+
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # extract the metadata for the files into a hash list
+    $imfiles = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # loop over all of the imfiles, determine the path_base and class_id for each
+    foreach my $imfile (@$imfiles) {
+        my $exp_id   = $imfile->{exp_id};
+        my $class_id = $imfile->{class_id};
+        my $path_base = $imfile->{path_base};
+        my $status = 1;
+
+        # don't clean up unless the data needed to update is available
+        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
+        # goto_scrubbed now requires the config file to not exist.
+        
+        # Possibly not the correct config file, but simtest doesn't leave any around to check.
+        if ($mode eq "goto_cleaned") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
+
+            if (!$config_file or ! -e $config_file) {
+                print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id "
+                    . " because config file is missing\n";
+                $status = 0;
+            }
+        }
+        elsif ($mode eq "goto_scrubbed") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
+
+            if ($config_file and -e $config_file) {
+                print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id "
+                    . " because config file is present\n";
+                $status = 0;
+            }
+        }
+
+        if ($status) {
+            # array of actual filenames to delete
+            my @files = ();
+
+            # delete the temporary image datafiles
+            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
+            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
+            }
+
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+
+        if ($status)  {
+            my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile";
+            if ($mode eq "goto_purged") {
+                $command .= " -data_state purged";
+            }
+            elsif ($mode eq "goto_cleaned") {
+                $command .= " -data_state cleaned";
+            }
+            elsif ($mode eq "goto_scrubbed") {
+                $command .= " -data_state scrubbed";
+            }
+
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+
+        } else {
+
+            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
+            my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        }
+    }
+
+    # Check to see if we can mark the whole detRunSummary object as cleaned.
+
+    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
+    $command .= " -dbname $dbname" if defined $dbname;
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+    }
+    if (@$stdout_buf != 0) {
+        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        my $exps = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        
+        foreach my $exp (@$exps) {
+            my $iteration = $exp->{iteration};
+            my $command;
+            if ($mode eq "goto_cleaned") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+            }
+        }
+    }
+    
+    exit 0;
+}
+if ($stage eq "detrend.process.exp") {
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+    # this stage uses 'dettool'
+    my $dettool = can_run('dettool') or die "Can't find dettool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $exps;                      # Array of component files
+    my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    
+    if (@$stdout_buf == 0) {
+        exit 0; #silently abort. I need to fix this for propers
+    }
+
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    $exps = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+
+    foreach my $exp (@$exps) {
+        my $path_base = $exp->{path_base};
+        my $exp_id    = $exp->{exp_id};
+
+        my $status = 1;
+        # don't clean up unless the data needed to update is available
+        # goto_scrubbed now requires the config file to not be present
+        if ($mode eq "goto_cleaned") {
+            my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
+            
+            if (!$config_file or ! -e $config_file) {
+                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
+                $status = 0;
+            }
+        }
+        elsif ($mode eq "goto_scrubbed") {
+            my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
+            
+            if ($config_file and -e $config_file) {
+                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
+                $status = 0;
+            }
+        }
+        if ($status) {
+            my @files = ();
+            # delete the temporary image datafiles
+            # I can't find anything to put here
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
+                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
+            }
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+        
+        if ($status)  {
+            my $command;
+            if ($mode eq "goto_cleaned") {
+                $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        } else {
+            # since 'camera' has only a single imfile, we can just update the run
+            my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+            
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+            exit $PS_EXIT_UNKNOWN_ERROR;
+        }
+    }
+    # Check to see if we can mark the whole detRunSummary object as cleaned.
+
+    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
+    $command .= " -dbname $dbname" if defined $dbname;
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+    }
+    if (@$stdout_buf != 0) {
+        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        $exps = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        
+        foreach my $exp (@$exps) {
+            my $iteration = $exp->{iteration};
+            my $command;
+            if ($mode eq "goto_cleaned") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+            }
+        }
+    }
+
+    exit 0;
+}
+if ($stage eq "detrend.stack.imfile") {
+    
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+
+    # this stage uses 'dettool'
+    my $dettool = can_run('dettool') or die "Can't find dettool";
+
+    # Get list of component imfiles
+    my $stacks;                  # Array reference of component files
+    my $command = "$dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or 
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    $stacks = parse_md_list($metadata) or 
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+    
+    my @files = ();
+    foreach my $stack (@{ $stacks }) {
+        # detStackedImfile does not have a path_base column.  This is inconvenient, as it means we need to calculate it.
+        my $path_base = $stack->{uri};
+        my $iteration = $stack->{iteration};
+        my $class_id  = $stack->{class_id};
+
+        $path_base =~ s/\.fits$//; # That should do it?
+
+        my $status = 1;
+
+#       if ($mode eq "goto_cleaned") {
+#           my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
+
+#           $config_file =~ s%^file://%%;
+#           if (!$config_file or ! -e $config_file) {
+#               print STDERR "skipping cleanup for $stage $stage_id $path_base" .
+#                   " because config file is missing\n";
+#               $status = 0;
+#           }
+#           $config_file = 'file://' . $config_file;
+#       }
+#       elsif ($mode eq "goto_scrubbed") {
+#           my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
+#           $config_file =~ s%^file://%%;
+#           if ($config_file and -e $config_file) {
+#               print STDERR "skipping scrubbed for $stage $stage_id $path_base" .
+#                   " because config file is present\n";
+#               $status = 0;
+#           }
+#       }
+
+        if ($status) {
+            # delete the temporary image datafiles
+            # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion.
+            # I understand that they all point to the same filename right now, but that may not be true in
+            # the future.
+            addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id);
+            addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id);
+            addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id);
+            addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id);
+            addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id);
+            addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id);
+            
+
+            addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id);
+            addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id);
+
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+#               addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id);
+            }
+
+            $status = &delete_files(\@files);
+        }
+
+        if ($status) {
+            my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id";
+            if ($mode eq "goto_purged") {
+                $command .= " -updatestacked -data_state purged";
+            } 
+            elsif ($mode eq "goto_cleaned") {
+                $command .= " -updatestacked -data_state cleaned";
+            }
+            elsif ($mode eq "goto_scrubbed") {
+                $command .= " -updatestacked -data_state scrubbed";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        } else {
+            my $command = "$dettool -updatestacked  -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+            
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+            exit $PS_EXIT_UNKNOWN_ERROR;
+        }
+    }
+    # Check to see if we can mark the whole detRunSummary object as cleaned.
+
+    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
+    $command .= " -dbname $dbname" if defined $dbname;
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+    }
+    if (@$stdout_buf != 0) {
+        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        my $exps = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        
+        foreach my $exp (@$exps) {
+            my $iteration = $exp->{iteration};
+            my $command;
+            if ($mode eq "goto_cleaned") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+            }
+        }
+    }
+    exit 0;
+}
+if ($stage eq "detrend.normstat.imfile") {
+    print STDERR "I'm not convinced there's anything to clean up from stage $stage\n";
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+    # this stage uses 'camtool'
+    my $dettool = can_run('dettool') or die "Can't find dettool";
+
+    my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    my $exps = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    foreach my $exp (@$exps) {
+#       my $path_base = $exp->{path_base};
+        my $iteration = $exp->{iteration};
+        my $class_id  = $exp->{class_id};
+
+        my $status = 1;
+        if ($status)  {
+            my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id";
+            if ($mode eq "goto_cleaned") {
+                $command .= " -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command .= " -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command .= " -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                print STDERR "WTF: $success TTT $error_code QQQ\n";
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        } else {
+            my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+            
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+            exit $PS_EXIT_UNKNOWN_ERROR;
+        }
+    }
+    # Check to see if we can mark the whole detRunSummary object as cleaned.
+
+    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
+    $command .= " -dbname $dbname" if defined $dbname;
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+    }
+    if (@$stdout_buf != 0) {
+        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        $exps = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        
+        foreach my $exp (@$exps) {
+            my $iteration = $exp->{iteration};
+            my $command;
+            if ($mode eq "goto_cleaned") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+            }
+        }
+    }
+
+    exit 0;
+
+}
+if ($stage eq "detrend.norm.imfile") {
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+    # this stage uses 'dettool'
+    my $dettool = can_run('dettool') or die "Can't find dettool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $exps;                      # Array of component files
+    my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    $exps = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    foreach my $exp (@$exps) {
+        my $path_base = $exp->{path_base};
+        my $iteration = $exp->{iteration};
+        my $class_id  = $exp->{class_id};
+
+        my $status = 1;
+        # don't clean up unless the data needed to update is available
+        # goto_scrubbed now requires the config file to not be present
+        if ($mode eq "goto_cleaned") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
+            
+            if (!$config_file or ! -e $config_file) {
+                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
+                $status = 0;
+            }
+        }
+        elsif ($mode eq "goto_scrubbed") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
+            
+            if ($config_file and -e $config_file) {
+                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
+                $status = 0;
+            }
+        }
+        if ($status) {
+            my @files = ();
+
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base);
+                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base);
+
+                addFilename (\@files, "PPIMAGE.OUTPUT", $path_base);
+                addFilename (\@files, "PPIMAGE.STATS", $path_base);
+            }
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+        
+        if ($status)  {
+            my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id";
+            if ($mode eq "goto_cleaned") {
+                $command .= " -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command .= " -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command .= " -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        } else {
+            my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+            
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+            exit $PS_EXIT_UNKNOWN_ERROR;
+        }
+    }
+    # Check to see if we can mark the whole detRunSummary object as cleaned.
+
+    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
+    $command .= " -dbname $dbname" if defined $dbname;
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+    }
+    if (@$stdout_buf != 0) {
+        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        $exps = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        
+        foreach my $exp (@$exps) {
+            my $iteration = $exp->{iteration};
+            my $command;
+            if ($mode eq "goto_cleaned") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+            }
+        }
+    }
+
+    exit 0;
+}
+if ($stage eq "detrend.norm.exp") {
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+    # this stage uses 'dettool'
+    my $dettool = can_run('dettool') or die "Can't find dettool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $exps;                      # Array of component files
+    my $command = "$dettool -pendingcleanup_normalizedexp -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+    }
+
+    if (@$stdout_buf == 0) {
+        exit 0;
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    $exps = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    foreach my $exp (@$exps) {
+        my $exp_id = $exp->{exp_id};
+        my $iteration = $exp->{iteration};
+        my $path_base = $exp->{path_base};
+
+        my $status = 1;
+        # don't clean up unless the data needed to update is available
+        # goto_scrubbed now requires the config file to not be present
+        if ($mode eq "goto_cleaned") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
+            
+            if (!$config_file or ! -e $config_file) {
+                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
+                $status = 0;
+            }
+        }
+        elsif ($mode eq "goto_scrubbed") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
+            
+            if ($config_file and -e $config_file) {
+                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
+                $status = 0;
+            }
+        }
+        if ($status) {
+            my @files = ();
+            # delete the temporary image datafiles
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
+                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
+            }
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+        
+        if ($status)  {
+            my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration";
+            if ($mode eq "goto_cleaned") {
+                $command .= " -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command .= " -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command .= " -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                print STDERR " residexp had an issue setting the state:? $success $error_code\n";
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        } else {
+            my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+            
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+            exit $PS_EXIT_UNKNOWN_ERROR;
+        }
+    }
+    # Check to see if we can mark the whole detRunSummary object as cleaned.
+
+    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
+    $command .= " -dbname $dbname" if defined $dbname;
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+    }
+    if (@$stdout_buf != 0) {
+        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        $exps = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        
+        foreach my $exp (@$exps) {
+            my $iteration = $exp->{iteration};
+            my $command;
+            if ($mode eq "goto_cleaned") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+            }
+        }
+    }
+    exit 0;
+}
+if ($stage eq "detrend.resid.imfile") {
+
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+    ### select the imfiles for this entry
+
+    # this stage uses 'dettool'
+    my $dettool = can_run('dettool') or die "Can't find dettool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $imfiles;                      # Array of component files
+    my $command = "$dettool -pendingcleanup_residimfile -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
+    }
+
+    if (@$stdout_buf == 0) {
+        exit 0;
+    }
+
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # extract the metadata for the files into a hash list
+    $imfiles = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # loop over all of the imfiles, determine the path_base and class_id for each
+    foreach my $imfile (@$imfiles) {
+        my $class_id = $imfile->{class_id};
+        my $iteration = $imfile->{iteration};
+        my $exp_id = $imfile->{exp_id};
+        my $path_base = $imfile->{path_base};
+        my $status = 1;
+
+        # don't clean up unless the data needed to update is available
+        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
+        # goto_scrubbed now requires the config file to not exist.
+        
+        # Possibly not the correct config file, but simtest doesn't leave any around to check.
+        if ($mode eq "goto_cleaned") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
+
+            if (!$config_file or ! -e $config_file) {
+                print STDERR "skipping cleanup for $stage $stage_id $class_id "
+                    . " because config file is missing\n";
+                $status = 0;
+            }
+        }
+        elsif ($mode eq "goto_scrubbed") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
+
+            if ($config_file and -e $config_file) {
+                print STDERR "skipping scrubbed for $stage $stage_id $class_id "
+                    . " because config file is present\n";
+                $status = 0;
+            }
+        }
+
+        if ($status) {
+            # array of actual filenames to delete
+            my @files = ();
+
+            # delete the temporary image datafiles
+            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
+            }
+
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+
+        if ($status)  {
+            my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id";
+            if ($mode eq "goto_purged") {
+                $command .= " -data_state purged";
+            }
+            elsif ($mode eq "goto_cleaned") {
+                $command .= " -data_state cleaned";
+            }
+            elsif ($mode eq "goto_scrubbed") {
+                $command .= " -data_state scrubbed";
+            }
+
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        } else {
+            my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        }
+    }
+    # Check to see if we can mark the whole detRunSummary object as cleaned.
+
+    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
+    $command .= " -dbname $dbname" if defined $dbname;
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+    }
+    if (@$stdout_buf != 0) {
+        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        my $exps = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        
+        foreach my $exp (@$exps) {
+            my $iteration = $exp->{iteration};
+            my $command;
+            if ($mode eq "goto_cleaned") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+            }
+        }
+    }
+
+    exit 0;
+}
+if ($stage eq "detrend.resid.exp") {
+    die "--stage_id required for stage $stage\n" if !$stage_id;
+    # this stage uses 'camtool'
+    my $dettool = can_run('dettool') or die "Can't find dettool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $exps;                      # Array of component files
+    my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this
+    if (@$stdout_buf == 0) {
+        exit 0;
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    $exps = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    foreach my $exp (@$exps) {
+        my $exp_id = $exp->{exp_id};
+        my $iteration = $exp->{iteration};
+        my $path_base = $exp->{path_base};
+#       my $class_id  = $exp->{class_id} 
+        my $status = 1;
+        # don't clean up unless the data needed to update is available
+        # goto_scrubbed now requires the config file to not be present
+        if ($mode eq "goto_cleaned") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
+            
+            if (!$config_file or ! -e $config_file) {
+                print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
+                $status = 0;
+            }
+        }
+        elsif ($mode eq "goto_scrubbed") {
+            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
+            
+            if ($config_file and -e $config_file) {
+                print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
+                $status = 0;
+            }
+        }
+        if ($status) {
+            my @files = ();
+            # delete the temporary image datafiles
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);#, $class_id);
+                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);#, $class_id);
+            }
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+        
+        if ($status)  {
+            my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration";
+            if ($mode eq "goto_cleaned") {
+                $command .= " -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command .= " -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command .= " -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                print STDERR " residexp had an issue setting the state:? $success $error_code\n";
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        } else {
+            my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+            
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+            exit $PS_EXIT_UNKNOWN_ERROR;
+        }
+    }
+    # Check to see if we can mark the whole detRunSummary object as cleaned.
+
+    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
+    $command .= " -dbname $dbname" if defined $dbname;
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+    }
+    if (@$stdout_buf != 0) {
+        $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        $exps = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+        
+        foreach my $exp (@$exps) {
+            my $iteration = $exp->{iteration};
+            my $command;
+            if ($mode eq "goto_cleaned") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
+            }
+            if ($mode eq "goto_scrubbed") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
+            }
+            if ($mode eq "goto_purged") {
+                $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+            }
+        }
+    }
+    exit 0;
+}
+
 
 die "ipp_cleanup.pl -stage $stage not yet implemented\n";
@@ -570 +1833 @@
     my $files = shift; # reference to a list of files to unlink
 
+#     open(TMPLOG,">>/tmp/czw.cleanup.log");
+#     flock(TMPLOG,2);
+
     # this script is, of course, very dangerous.
     foreach my $file (@$files) {
         print STDERR "unlinking $file\n";
+#       print TMPLOG "$stage $stage_id $file";
+#       my $ff = $file;
+#       $ff =~ s%^file://%%;
+#       unless (-e $ff) {
+#           print TMPLOG "\t File not found\n";
+#       }
+#       else {
+#           print TMPLOG "\n";
+#       }
+
         $ipprc->file_delete($file);
     }
+
+#     flock(TMPLOG,8);
+#     close(TMPLOG);
+
     return 1;
 }

trunk/ippScripts/scripts/stack_skycell.pl

@@ -65 +65 @@
 
 my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $stack_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$| = 1;
+print "I've set up: $stack_id\n";
 
 # XXX camera is not known here; cannot use filerules...
@@ -107 +109 @@
 &my_die("Stack list contains less than two elements", $stack_id, $PS_EXIT_SYS_ERROR) unless
     scalar @$files >= 2;
+
+print "I've loaded my inputs: $stack_id\n";
 
 # Parse the list of input files to get the tesselation, skycell identifiers and camera
@@ -146 +150 @@
 }
 
+print "I've configured everything: $stack_id\n";
+
 # Generate MDC file with the inputs
 my $tess_base = basename($tess_id);
@@ -180 +186 @@
     print $listFile "END\n\n";
 }
+
+print "I've checked everything: $stack_id\n";
 
 # Get the output filenames
@@ -200 +208 @@
 
 my $cmdflags;
+
 
 # Perform stacking
@@ -265 +274 @@
 #       &my_die("Couldn't find expected output file: $bin2Name",    $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name);
     }
-
+    
+    print "I've stacked $listName: $stack_id\n";
 }
 
@@ -298 +308 @@
 }
 
+print "I've updated teh database: $stack_id\n";
+
 
 my $my_die_called = 0;
@@ -372 +384 @@
 }
 
+print "I've reached the end of processing with a code of $?: $stack_id\n";
 
 END {

trunk/ippTasks/Makefile.am

@@ -13 +13 @@
         chip.pro \
         camera.pro \
+        addstar.pro \
         fake.pro \
         warp.pro \

trunk/ippTasks/pantasks.pro

@@ -213 +213 @@
   chip.on
   camera.on
+  addstar.on
   fake.on
   warp.on
@@ -225 +226 @@
   chip.off
   camera.off
+  addstar.off
   fake.off
   warp.off
@@ -242 +244 @@
   module chip.pro
   module camera.pro
+  module addstar.pro
   module fake.pro
   module warp.pro

trunk/ippTasks/summit.copy.pro

@@ -449 +449 @@
         end
         if (("$MD5SUM" != "NULL") && ("$MD5SUM" != "0") && (not($COMPRESS)))
+# && (($YEAR > 2008) || (("$YEAR" = "2007") && ($MONTH > 8))))
             $run = $run --md5 $MD5SUM
         end

trunk/ippTools/share/Makefile.am

@@ -4 +4 @@
 
 dist_pkgdata_DATA = \
+     addtool_find_pendingexp.sql \
+     addtool_queue_cam_id.sql \
+     addtool_reset_faulted_runs.sql \
+     addtool_revertprocessedexp.sql \
      camtool_donecleanup.sql \
      camtool_find_chip_id.sql \

trunk/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql

@@ -1 @@
-SELECT
+SELECT DISTINCT
     detNormalizedImfile.*,
     detRunSummary.data_state

trunk/ippTools/share/dettool_pendingcleanup_stacked.sql

@@ -1 @@
-SELECT
+SELECT DISTINCT
     detStackedImfile.*,
     detRunSummary.data_state

trunk/ippTools/share/pxadmin_create_tables.sql

@@ -451 +451 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
+CREATE TABLE addRun (
+    add_id BIGINT AUTO_INCREMENT,
+    cam_id BIGINT,
+    state VARCHAR(64),
+    workdir VARCHAR(255),
+    workdir_state VARCHAR(64),
+    label VARCHAR(64),
+    dvodb VARCHAR(255),
+    magicked BIGINT,
+    PRIMARY KEY(add_id),
+    KEY(add_id),
+    KEY(cam_id),
+    KEY(state),
+    KEY(workdir_state),
+    KEY(label),
+    INDEX(add_id, cam_id),
+    FOREIGN KEY(cam_id) REFERENCES camRun(cam_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE addProcessedExp (
+    add_id BIGINT AUTO_INCREMENT,
+    dtime_addstar FLOAT,
+    n_stars INT,
+    path_base VARCHAR(255),
+    fault SMALLINT NOT NULL,
+    PRIMARY KEY(add_id),
+    FOREIGN KEY(add_id) REFERENCES addRun(add_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE addMask (
+    label VARCHAR(64),
+    PRIMARY KEY(label)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
 CREATE TABLE fakeRun (
     fake_id BIGINT AUTO_INCREMENT,
@@ -1086 +1120 @@
         KEY(magic_id),
         KEY(label),
-        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id)
+        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id),
+        FOREIGN KEY(inv_magic_id) REFERENCES magicRun(magic_id)
         FOREIGN KEY(inv_magic_id) REFERENCES magicRun(magic_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;

trunk/ippTools/share/pxadmin_drop_tables.sql

@@ -19 +19 @@
 DROP TABLE IF EXISTS detRun;
 DROP TABLE IF EXISTS detInputExp;
+DROP TABLE IF EXISTS addRun;
+DROP TABLE IF EXISTS addProcessedExp;
+DROP TABLE IF EXISTS addMask;
 DROP TABLE IF EXISTS fakeRun;
 DROP TABLE IF EXISTS fakeProcessedImfile;

trunk/ippTools/src/Makefile.am

@@ -1 +1 @@
 bin_PROGRAMS = \
+        addtool \
         caltool \
         camtool \
@@ -28 +29 @@
 
 pkginclude_HEADERS = \
+        pxadd.h \
         pxadmin.h \
         pxcam.h \
@@ -42 +44 @@
 
 noinst_HEADERS = \
+        addtool.h \
         caltool.h \
         camtool.h \
@@ -68 +71 @@
 libpxtools_la_LDFLAGS   = -release $(PACKAGE_VERSION)
 libpxtools_la_SOURCES   = \
+        pxadd.c \
         pxcam.c \
         pxchip.c \
@@ -96 +100 @@
     pstamptool.c \
     pstamptoolConfig.c
+
+addtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+addtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+addtool_SOURCES = \
+    addtool.c \
+    addtoolConfig.c
 
 caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)

trunk/ippTools/src/camtool.c

@@ -656 +656 @@
         return false;
     }
+
+    if (!pxaddQueueByCamID(config,
+                           pendingRow->cam_id,
+                           pendingRow->workdir,
+                           pendingRow->label,
+                           pendingRow->reduction,
+                           pendingRow->dvodb
+    )) {
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "failed to queue new addRun");
+        psFree(pendingRow);
+        return false;
+    }
+
     psFree(pendingRow);
 

trunk/ippTools/src/dettoolConfig.c

@@ -70 +70 @@
     psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-sun_angle_min",  0,            "define min solar angle", NAN);
     psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-sun_angle_max",  0,            "define max solar angle", NAN);
+
 
     psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-registered",  0,            "time detrend run was registered", now);
@@ -138 +139 @@
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment",                0,          "search by comment field (LIKE comparison)", NULL);
 
+
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_angle_min",  0,          "define min moon angle", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_angle_max",  0,          "define max moon angle", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_alt_min",  0,            "define min moon alt", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_alt_max",  0,            "define max moon alt", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_min",  0,          "define min moon phase", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_max",  0,          "define max moon phase", NAN);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment",                0,          "search by comment field (LIKE comparison)", NULL);
+
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend",  0,            "print the exposures that would be included in the detrend run and exit", false);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction",  0,            "define reduction class for processing", NULL);

trunk/ippTools/src/pxtools.h

@@ -37 +37 @@
 #include "pxtoolsErrorCodes.h"
 
+#include "pxadd.h"
 #include "pxcam.h"
 #include "pxchip.h"

trunk/ippTools/src/regtool.c

@@ -331 +331 @@
     PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(ordered_by_date, config->args, "-ordered_by_date", false);
 
     psString query = pxDataGet("regtool_processedimfile.sql");
@@ -352 +353 @@
         // don't list faulted rows
         psStringAppend(&query, " %s", "AND rawImfile.fault = 0");
+    }
+
+    // add the ORDER BY statement if desired
+    if (ordered_by_date) {
+        psStringAppend(&query, " ORDER BY dateobs");
     }
 
@@ -923 +929 @@
     PXOPT_COPY_F64(config->args,   where,  "-posang_max", "posang", "<");
     PXOPT_COPY_STR(config->args,   where,  "-object", "object", "==");
-    PXOPT_COPY_F32(config->args,   where,  "-sun_angle_min", "sun_angle", ">=");
-    PXOPT_COPY_F32(config->args,   where,  "-sun_angle_max", "sun_angle", "<");
+    PXOPT_COPY_F32(config->args,   where,  "-solang_min", "solang", ">=");
+    PXOPT_COPY_F32(config->args,   where,  "-solang_max", "solang", "<");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);

trunk/ippTools/src/warptool.c

@@ -234 +234 @@
     PXOPT_COPY_F64(config->args,   where, "-posang_max",         "rawExp.posang",         "<");
     PXOPT_COPY_STR(config->args,   where, "-object",             "rawExp.object",         "==");
-    PXOPT_COPY_F32(config->args,   where, "-sun_angle_min",      "rawExp.sun_angle",      ">=");
-    PXOPT_COPY_F32(config->args,   where, "-sun_angle_max",      "rawExp.sun_angle",      "<");
+    PXOPT_COPY_F32(config->args,   where, "-solang_min",         "rawExp.solang",         ">=");
+    PXOPT_COPY_F32(config->args,   where, "-solang_max",         "rawExp.solang",         "<");
     PXOPT_COPY_STR(config->args,   where, "-reduction",          "fakeRun.reduction",     "==");
     pxAddLabelSearchArgs (config,  where, "-label",             "fakeRun.label",         "==");

trunk/ippconfig/gpc1/format_20090220.config

@@ -185 +185 @@
         FPA.ALT         STR     ALT
         FPA.AZ          STR     AZ
-        FPA.TEMP        STR     DETTEM
+        # FPA.TEMP        STR     DETTEM
         FPA.M1X         STR     M1X
         FPA.M1Y         STR     M1Y   
@@ -501 +501 @@
 # How to translation PS concepts into database lookups
 DATABASE        METADATA
+        # this rule is fragile : does not match the camera
+        FPA.TEMP        STR     SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2
 END
 

trunk/ippconfig/recipes/masks.16bit.config

@@ -20 +20 @@
 
 # Mask values which represent non-astronomical structures
+BURNTOOL        U16     0x0080          # Pixel may contain uncorrected streak.
 CR              U16     0x0100          # Pixel contains a cosmic ray
 SPIKE           U16     0x0200          # Pixel contains a diffraction spike

trunk/ippconfig/recipes/ppImage.config

@@ -16 +16 @@
 MASK.SATURATED     BOOL    TRUE            # Mask the saturated pixels
 MASK.LOW           BOOL    TRUE            # Mask pixels below valid range
+MASK.BURNTOOL      BOOL    TRUE            # Mask potential burntool trails
 VARIANCE.BUILD     BOOL    FALSE           # Build internal variance image
 PATTERN            BOOL    FALSE           # Fit and remove pattern noise?
@@ -54 +55 @@
 ## if we use multithreaded detrending, detrend this number of rows per thread
 SCAN.ROWS        S32     100
+
+BURNTOOL.TRAILS U16 0x01
 
 # Non-linearity correction

trunk/ippconfig/recipes/psastro.config

@@ -3 +3 @@
 PSASTRO.ONLY.REFSTARS      BOOL FALSE  # skip all but refstar matches
 PSASTRO.SAVE.REFMATCH      BOOL FALSE  # save refstar matches as table in output smf file
+
+# select which WCS style to use on output images.
+PSASTRO.WCS.USECDKEYS    BOOL FALSE
 
 # perform single-chip astrometry?

trunk/ppImage/src/Makefile.am

@@ -54 +54 @@
         ppImageDefineFile.c \
         ppImageSetMaskBits.c \
+        ppImageBurntoolMask.c \
         ppImageParityFlip.c \
         ppImageCheckCTE.c \

trunk/ppImage/src/ppImage.h

@@ -26 +26 @@
     bool doMaskBuild;                   // Build internal mask
     bool doVarianceBuild;               // Build internal variance map
+    bool doMaskBurntool;                // mask potential burntool trails
     bool doMaskSat;                     // mask saturated pixels
     bool doMaskLow;                     // mask low pixels
@@ -74 +75 @@
     psImageMaskType darkMask;           // Mask value to give bad dark pixels
     psImageMaskType blankMask;          // Mask value to give blank pixels
-
+    psImageMaskType burntoolMask;       // Suspect pixels that fall where a burntool trail is expected.
     // non-linear correction parameters
     psDataType nonLinearType;
@@ -82 +83 @@
     // options for the analysis
     pmOverscanOptions *overscan;        // Overscan options
-
+    int burntoolTrails;
     // binning parameters
     int xBin1;                          // x-binning, scale 1
@@ -156 +157 @@
 bool ppImageCheckCTE(pmConfig *config, ppImageOptions *options, pmFPAview *view);
 
+bool ppImageBurntoolMask(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *mask);
+
 // Record which detrend file was used for the detrending
 bool ppImageDetrendRecord(

trunk/ppImage/src/ppImageDetrendReadout.c

@@ -26 +26 @@
         pmMaskBadPixels(input, mask, options->maskValue);
     }
+    if (options->doMaskBurntool) {
+      ppImageBurntoolMask(config,options,view,input);
+    }
+
 
 # if 0

trunk/ppImage/src/ppImageOptions.c

@@ -21 +21 @@
     options->doMaskSat       = false;   // mask saturated pixels
     options->doMaskLow       = false;   // mask low pixels
+    options->doMaskBurntool  = false;   // mask potential burntool trails
     options->doVarianceBuild = false;   // Build internal variance
     options->doMask          = false;   // Mask bad pixels
@@ -64 +65 @@
     options->blankMask       = 0x00;    // Blank (no data, cell gap) pixels (supplied to pmChipMosaic, pmFPAMosaic)
     options->markValue       = 0x00;    // A safe bit for internal marking
-
+    options->burntoolMask    = 0x00;    // Suspect pixels that fall where a burntool trail is expected.
+    options->burntoolTrails  = 0x00;    // Which types of burntool areas to mask.
     // crosstalk options
     options->doCrosstalkMeasure = false;   // measure crosstalk
@@ -219 +221 @@
     options->doMaskSat   = psMetadataLookupBool(NULL, recipe, "MASK.SATURATED");
     options->doMaskLow   = psMetadataLookupBool(NULL, recipe, "MASK.LOW");
+    options->doMaskBurntool = psMetadataLookupBool(NULL, recipe, "MASK.BURNTOOL");
     options->doVarianceBuild = psMetadataLookupBool(NULL, recipe, "VARIANCE.BUILD");
 
@@ -245 +248 @@
     options->applyParity = psMetadataLookupBool(NULL, recipe, "APPLY.CELL.PARITY");
 
+    options->burntoolTrails = psMetadataLookupU16(&status, recipe, "BURNTOOL.TRAILS");
+    if (!status) {
+      psWarning("BURNTOOL.TRAILS not found in recipe: setting to default value.\n");
+    }
+    
     // binned image options
     options->xBin1 = psMetadataLookupS32(&status, recipe, "BIN1.XBIN");
     if (!status) {
         psWarning("BIN1.XBIN not found in recipe: setting to default value.\n");
+        options->xBin1 = 4;
     }
     options->yBin1 = psMetadataLookupS32(&status, recipe, "BIN1.YBIN");
     if (!status) {
         psWarning("BIN1.YBIN not found in recipe: setting to default value.\n");
-        options->yBin1 = 16;
+        options->yBin1 = 4;
     }
 

trunk/ppImage/src/ppImageSetMaskBits.c

@@ -38 +38 @@
     psAssert (options->lowMask, "low mask not set");
 
+    // mask for suspect regions due to burntool
+    options->burntoolMask = pmConfigMaskGet("BURNTOOL",config);
+    psAssert (options->burntoolMask, "burntool mask not set");
+    
     // save MASK and MARK on the PSPHOT recipe
     psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE);

trunk/psModules/src/astrom/pmAstrometryWCS.c

@@ -289 +289 @@
     // test the CDELTi varient
     if (pcKeys) {
+        wcs->wcsCDkeys = 0;
+
         wcs->cdelt1 = psMetadataLookupF64 (&status, header, "CDELT1");
         wcs->cdelt2 = psMetadataLookupF64 (&status, header, "CDELT2");
@@ -334 +336 @@
     // test the CDi_j varient
     if (cdKeys) {
+        wcs->wcsCDkeys = 1;
+
         wcs->trans->x->coeff[1][0] = psMetadataLookupF64 (&status, header, "CD1_1"); // == PC1_1
         wcs->trans->x->coeff[0][1] = psMetadataLookupF64 (&status, header, "CD1_2"); // == PC1_2
@@ -375 +379 @@
     // XXX make it optional to write out CDi_j terms, or other versions
     // apply CDELT1,2 (degrees / pixel) to yield PCi,j terms of order unity
-    double cdelt1 = wcs->cdelt1;
-    double cdelt2 = wcs->cdelt2;
-    psMetadataAddF64 (header, PS_LIST_TAIL, "CDELT1", PS_META_REPLACE, "", cdelt1);
-    psMetadataAddF64 (header, PS_LIST_TAIL, "CDELT2", PS_META_REPLACE, "", cdelt2);
-
-    // test the PC00i00j varient:
-    psMetadataAddF64 (header, PS_LIST_TAIL, "PC001001", PS_META_REPLACE, "", wcs->trans->x->coeff[1][0] / cdelt1); // == PC1_1
-    psMetadataAddF64 (header, PS_LIST_TAIL, "PC001002", PS_META_REPLACE, "", wcs->trans->x->coeff[0][1] / cdelt2); // == PC1_2
-    psMetadataAddF64 (header, PS_LIST_TAIL, "PC002001", PS_META_REPLACE, "", wcs->trans->y->coeff[1][0] / cdelt1); // == PC2_1
-    psMetadataAddF64 (header, PS_LIST_TAIL, "PC002002", PS_META_REPLACE, "", wcs->trans->y->coeff[0][1] / cdelt2); // == PC2_2
-
-    // Elixir-style polynomial terms
-    // XXX currently, Elixir/DVO cannot accept mixed orders
-    // XXX need to respect the masks
-    // XXX is wcs->cdelt1,2 always consistent?
-    int fitOrder = wcs->trans->x->nX;
-    if (fitOrder > 1) {
+
+    if (!(wcs->wcsCDkeys)) {
+
+      double cdelt1 = wcs->cdelt1;
+      double cdelt2 = wcs->cdelt2;
+      psMetadataAddF64 (header, PS_LIST_TAIL, "CDELT1", PS_META_REPLACE, "", cdelt1);
+      psMetadataAddF64 (header, PS_LIST_TAIL, "CDELT2", PS_META_REPLACE, "", cdelt2);
+      
+      // test the PC00i00j varient:
+      psMetadataAddF64 (header, PS_LIST_TAIL, "PC001001", PS_META_REPLACE, "", wcs->trans->x->coeff[1][0] / cdelt1); // == PC1_1
+      psMetadataAddF64 (header, PS_LIST_TAIL, "PC001002", PS_META_REPLACE, "", wcs->trans->x->coeff[0][1] / cdelt2); // == PC1_2
+      psMetadataAddF64 (header, PS_LIST_TAIL, "PC002001", PS_META_REPLACE, "", wcs->trans->y->coeff[1][0] / cdelt1); // == PC2_1
+      psMetadataAddF64 (header, PS_LIST_TAIL, "PC002002", PS_META_REPLACE, "", wcs->trans->y->coeff[0][1] / cdelt2); // == PC2_2
+      
+      // Elixir-style polynomial terms
+      // XXX currently, Elixir/DVO cannot accept mixed orders
+      // XXX need to respect the masks
+      // XXX is wcs->cdelt1,2 always consistent?
+      int fitOrder = wcs->trans->x->nX;
+      if (fitOrder > 1) {
         for (int i = 0; i <= fitOrder; i++) {
-            for (int j = 0; j <= fitOrder; j++) {
-                if (i + j < 2)
-                    continue;
-                if (i + j > fitOrder)
-                    continue;
-                sprintf (name, "PCA1X%1dY%1d", i, j);
-                psMetadataAddF64 (header, PS_LIST_TAIL, name, PS_META_REPLACE, "", wcs->trans->x->coeff[i][j] / pow(cdelt1, i) / pow(cdelt2, j));
-                sprintf (name, "PCA2X%1dY%1d", i, j);
-                psMetadataAddF64 (header, PS_LIST_TAIL, name, PS_META_REPLACE, "", wcs->trans->y->coeff[i][j] / pow(cdelt1, i) / pow(cdelt2, j));
-            }
+          for (int j = 0; j <= fitOrder; j++) {
+            if (i + j < 2)
+              continue;
+            if (i + j > fitOrder)
+              continue;
+            sprintf (name, "PCA1X%1dY%1d", i, j);
+            psMetadataAddF64 (header, PS_LIST_TAIL, name, PS_META_REPLACE, "", wcs->trans->x->coeff[i][j] / pow(cdelt1, i) / pow(cdelt2, j));
+            sprintf (name, "PCA2X%1dY%1d", i, j);
+            psMetadataAddF64 (header, PS_LIST_TAIL, name, PS_META_REPLACE, "", wcs->trans->y->coeff[i][j] / pow(cdelt1, i) / pow(cdelt2, j));
+          }
         }
         psMetadataAddS32 (header, PS_LIST_TAIL, "NPLYTERM", PS_META_REPLACE, "", fitOrder);
-    }
-
-    // remove any existing 'CDi_j style' wcs keywords
-    if (psMetadataLookup(header, "CD1_1")) {
+      }
+      
+      // remove any existing 'CDi_j style' wcs keywords
+      if (psMetadataLookup(header, "CD1_1")) {
         psMetadataRemoveKey(header, "CD1_1");
         psMetadataRemoveKey(header, "CD1_2");
         psMetadataRemoveKey(header, "CD2_1");
         psMetadataRemoveKey(header, "CD2_2");
+      }
+    }
+    if (wcs->wcsCDkeys) {
+      psMetadataAddF64 (header, PS_LIST_TAIL, "CD1_1", PS_META_REPLACE, "", wcs->trans->x->coeff[1][0]);
+      psMetadataAddF64 (header, PS_LIST_TAIL, "CD1_2", PS_META_REPLACE, "", wcs->trans->x->coeff[0][1]);
+      psMetadataAddF64 (header, PS_LIST_TAIL, "CD2_1", PS_META_REPLACE, "", wcs->trans->y->coeff[1][0]);
+      psMetadataAddF64 (header, PS_LIST_TAIL, "CD2_2", PS_META_REPLACE, "", wcs->trans->y->coeff[0][1]);
+
+      if (psMetadataLookup(header, "PC001001")) {
+        psMetadataRemoveKey(header, "PC001001");
+        psMetadataRemoveKey(header, "PC001002");
+        psMetadataRemoveKey(header, "PC002001");
+        psMetadataRemoveKey(header, "PC002002");
+      }
     }
 
@@ -535 +556 @@
         fpa->toSky->R -= 2.0*M_PI;
 
+    fpa->wcsCDkeys = wcs->wcsCDkeys;
+
     psTrace ("psastro", 5, "toFPA: %f %f  (%f,%f),(%f,%f)\n",
              chip->toFPA->x->coeff[0][0], chip->toFPA->y->coeff[0][0],
@@ -682 +705 @@
     wcs->cdelt2 = hypot (wcs->trans->y->coeff[1][0], wcs->trans->y->coeff[0][1]);
 
+    wcs->wcsCDkeys = fpa->wcsCDkeys;
     psFree (toTPA);
 
@@ -922 +946 @@
     wcs->trans = psPlaneTransformAlloc (nXorder, nYorder);
     wcs->toSky = NULL;
+    wcs->wcsCDkeys = 0;
 
     memset (wcs->ctype1, 0, PM_ASTROM_WCS_TYPE_SIZE);

trunk/psModules/src/astrom/pmAstrometryWCS.h

@@ -23 +23 @@
     double crpix1, crpix2;
     double cdelt1, cdelt2;
+    bool wcsCDkeys;
     psProjection *toSky;
     psPlaneTransform *trans;

trunk/psModules/src/camera/pmFPA.h

@@ -48 +48 @@
     psPlaneTransform *toTPA;  ///< Transformation from focal plane to tangent plane, or NULL
     psProjection *toSky;         ///< Projection from tangent plane to sky, or NULL
+    bool wcsCDkeys;
     // Information
     psMetadata *concepts;               ///< FPA-level concepts

trunk/psastro/src/psastroChipAstrom.c

@@ -97 +97 @@
                 // write the elapsed time here; this will be updated in psastroMosaicAstrometry, if called
                 psMetadataAddF32 (updates, PS_LIST_TAIL, "DT_ASTR", PS_META_REPLACE, "elapsed psastro time", psTimerMark ("psastroAnalysis"));
-
+                
+                fpa->wcsCDkeys = psMetadataLookupBool(&status, recipe , "PSASTRO.WCS.USECDKEYS");
                 pmAstromWriteWCS (updates, fpa, chip, NONLIN_TOL);
 

trunk/tools/ipp_apply_burntool.pl

@@ -172 +172 @@
 
         if ($RAWTABLES) {
-            $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt", $REALRUN);
+            $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt persist=t", $REALRUN);
         } else {
-            $status = vsystem ("$burntool $tmpImfileReal $prevFileOpt", $REALRUN);
+            $status = vsystem ("$burntool $tmpImfileReal $prevFileOpt persist=t", $REALRUN);
         }
         if ($status) {