Changeset 30812

Timestamp:

Mar 6, 2011, 2:34:55 PM (15 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/eam_branches/ipp-20110213

Files:

• Ohana/src/addstar/src/ReadStarsFITS.c (modified) (9 diffs)
• Ohana/src/dvomerge/src/merge_catalogs_old.c (modified) (1 diff)
• Ohana/src/getstar/src/MatchCoords.c (modified) (6 diffs)
• Ohana/src/libohana/include/ohana_allocate.h (modified) (2 diffs)
• PS-IPP-Config/lib/PS/IPP/Config.pm (modified) (1 diff)
• ippMonitor/raw/czartool_labels.php (modified) (1 diff)
• ippMonitor/raw/site.php.in (modified) (1 diff)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/magic_destreak_cleanup.pl (modified) (3 diffs)
• ippScripts/scripts/magic_destreak_revert.pl (modified) (9 diffs)
• ippScripts/scripts/register_imfile.pl (modified) (1 diff)
• ippTasks/destreak.pro (modified) (2 diffs)
• ippTasks/pstamp.pro (modified) (3 diffs)
• ippTasks/register.pro (modified) (1 diff)
• ippToPsps/src/Batch.h (modified) (1 diff)
• ippToPsps/src/DifferenceBatch.c (modified) (1 diff)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/magicdstool_todestreak_raw.sql (modified) (2 diffs)
• ippTools/share/magicdstool_torevert_diff.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/addtool.c (modified) (1 diff)
• ippTools/src/bgtool.c (modified) (2 diffs)
• ippTools/src/camtool.c (modified) (1 diff)
• ippTools/src/camtoolConfig.c (modified) (2 diffs)
• ippTools/src/chiptool.c (modified) (1 diff)
• ippTools/src/chiptoolConfig.c (modified) (1 diff)
• ippTools/src/diffphottool.c (modified) (1 diff)
• ippTools/src/difftool.c (modified) (1 diff)
• ippTools/src/difftoolConfig.c (modified) (1 diff)
• ippTools/src/faketool.c (modified) (1 diff)
• ippTools/src/magicdstool.c (modified) (1 diff)
• ippTools/src/magictool.c (modified) (1 prop)
• ippTools/src/pubtool.c (modified) (3 diffs)
• ippTools/src/pubtool.h (modified) (1 diff)
• ippTools/src/pubtoolConfig.c (modified) (2 diffs)
• ippTools/src/pxtools.c (modified) (2 diffs)
• ippTools/src/pxtools.h (modified) (1 diff)
• ippTools/src/stacktool.c (modified) (1 diff)
• ippTools/src/staticskytool.c (modified) (1 diff)
• ippTools/src/warptool.c (modified) (1 diff)
• ippTools/src/warptoolConfig.c (modified) (2 diffs)
• ippconfig (modified) (1 prop)
• ippconfig/gpc1/format_20100228.config (copied) (copied from trunk/ippconfig/gpc1/format_20100228.config )
• ippconfig/gpc1/format_20100723.config (copied) (copied from trunk/ippconfig/gpc1/format_20100723.config )
• ippconfig/recipes/nightly_science.config (modified) (1 diff)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (6 diffs)
• pstamp/scripts/pstampparse.pl (modified) (5 diffs)
• pstamp/scripts/pstampstopfaulted (modified) (1 diff)
• tools/czartool/Config.pm (modified) (14 diffs)
• tools/czartool/czarconfig.xml (modified) (5 diffs)
• tools/roboczar.pl (modified) (2 diffs)

branches/eam_branches/ipp-20110213/Ohana/src/addstar/src/ReadStarsFITS.c

@@ -45 +45 @@
   }
   if (!strcmp (type, "PS1_V2")) {
+    if (table.header[0].Naxis[0] == 20) {
+      // skip the invalid DETEFF tables which were mistakenly labeled as PS1_V2
+      return (NULL);
+    }
     stars = Convert_PS1_V2 (&table, &Nstars);
   }
@@ -51 +55 @@
   }
   if (stars == NULL) {
-    fprintf (stderr, "ERROR: invalid table type %s\n", type);
+    fprintf (stderr, "invalid table type %s\n", type);
     return (NULL);
   }
@@ -212 +216 @@
   Stars *stars;
   CMF_PS1_V1 *ps1data;
+
+  // CMF_PS1_V1 was modified 2009.05.26 (r24251) to use doubles for ra & dec.  this was a
+  // mistake in two ways: a new format should have been defined (eg, CMF_PS1_V2), and the
+  // layout used did not have clean byte-boundaries for the corresponding structure.  The
+  // former means we have two varieties of CMF_PS1_V1 out there; the latter means that the
+  // autocode tools do not work to read in the new version, even if we recognize it.  Here we
+  // test for the existence of the broken version (table[0].headers[0].Naxis[0] == 136), and
+  // call a special conversion function if it is found.
+
+  if (table[0].header[0].Naxis[0] == 136) {
+      stars = Convert_PS1_V1_Alt (table, nstars);
+      return (stars);
+  }
+
+  ps1data = gfits_table_get_CMF_PS1_V1 (table, &Nstars, NULL);
+  if (!ps1data) {
+    fprintf (stderr, "skipping inconsistent entry\n");
+    return (NULL);
+  }
+  ZeroPt = GetZeroPoint();
+
+  ALLOCATE (stars, Stars, Nstars);
+  for (i = 0; i < Nstars; i++) {
+    InitStar (&stars[i]);
+    stars[i].measure.Xccd       = ps1data[i].X;
+    stars[i].measure.Yccd       = ps1data[i].Y;
+    stars[i].measure.dXccd      = ToShortPixels(ps1data[i].dX);
+    stars[i].measure.dYccd      = ToShortPixels(ps1data[i].dY);
+
+    stars[i].measure.posangle   = ToShortDegrees(ps1data[i].posangle);
+    stars[i].measure.pltscale   = ps1data[i].pltscale;
+
+    if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) {
+        stars[i].measure.M      = NAN;
+    } else {
+        stars[i].measure.M      = ps1data[i].M + ZeroPt;
+    }
+    stars[i].measure.dM         = ps1data[i].dM;
+    stars[i].measure.dMcal      = ps1data[i].dMcal;
+    stars[i].measure.Map        = ps1data[i].Map + ZeroPt;
+                        
+    stars[i].measure.Sky        = ps1data[i].sky;
+    stars[i].measure.dSky       = ps1data[i].dSky;
+                        
+    stars[i].measure.psfChisq   = ps1data[i].psfChisq;
+    stars[i].measure.psfQual    = ps1data[i].psfQual;
+    stars[i].measure.psfNdof    = ps1data[i].psfNdof;
+    stars[i].measure.psfNpix    = ps1data[i].psfNpix;
+    stars[i].measure.crNsigma   = ps1data[i].crNsigma;
+    stars[i].measure.extNsigma  = ps1data[i].extNsigma;
+
+    stars[i].measure.FWx        = ToShortPixels(ps1data[i].fx);
+    stars[i].measure.FWy        = ToShortPixels(ps1data[i].fy);
+    stars[i].measure.theta      = ToShortDegrees(ps1data[i].df);
+
+    stars[i].measure.Mxx        = ToShortPixels(ps1data[i].Mxx);
+    stars[i].measure.Mxy        = ToShortPixels(ps1data[i].Mxy);
+    stars[i].measure.Myy        = ToShortPixels(ps1data[i].Myy);
+                        
+    stars[i].measure.photFlags  = ps1data[i].flags;
+
+    // this is may optionally be replaced by the internal sequence (see FilterStars.c)
+    stars[i].measure.detID      = ps1data[i].detID; 
+
+    // the Average fields and the following Measure fields are set in FilterStars after
+    // the image metadata is in hand:  dR, dD, Mcal, dt, airmass, az, t, imageID, extID, 
+    // averef is set in find_matches, dbFlags is zero on ingest.
+
+    // the following fields are currently not being set anywhere: t_msec
+  }    
+  *nstars = Nstars;
+  return (stars);
+}
+
+Stars *Convert_PS1_V1_Alt (FTable *table, unsigned int *nstars) {
+
+  off_t Nstars; 
+  unsigned int i;
+  double ZeroPt;
+  Stars *stars;
+  CMF_PS1_V2 *ps1data;
 
   // CMF_PS1_V1 was modified 2009.05.26 (r24251) to use doubles for ra & dec.  this was a
@@ -221 +306 @@
   // call a special conversion function if it is found.
 
-  if (table[0].header[0].Naxis[0] == 136) {
-      stars = Convert_PS1_V1_Alt (table, nstars);
-      return (stars);
-  }
-
-  ps1data = gfits_table_get_CMF_PS1_V1 (table, &Nstars, NULL);
+  ps1data = gfits_table_get_CMF_PS1_V1_Alt (table, &Nstars, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -286 +366 @@
 }
 
-Stars *Convert_PS1_V1_Alt (FTable *table, unsigned int *nstars) {
+Stars *Convert_PS1_V2 (FTable *table, unsigned int *nstars) {
 
   off_t Nstars; 
@@ -294 +374 @@
   CMF_PS1_V2 *ps1data;
 
-  // CMF_PS1_V1 was modified 2009.05.26 (r24251) to use doubles for ra & dec.  this was a
-  // mistake in two ways: a few format should have been defined (eg, CMF_PS1_V2), and the
-  // layout used did not have clean byte-boundaries for the corresponding structure.  The
-  // former means we have two varieties of CMF_PS1_V1 out there; the latter means that the
-  // autocode tools do not work to read in the new version, even if we recognize it.  Here we
-  // test for the existence of the broken version (table[0].headers[0].Naxis[0] == 136), and
-  // call a special conversion function if it is found.
-
-  ps1data = gfits_table_get_CMF_PS1_V1_Alt (table, &Nstars, NULL);
+  ps1data = gfits_table_get_CMF_PS1_V2 (table, &Nstars, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -362 +434 @@
 }
 
-Stars *Convert_PS1_V2 (FTable *table, unsigned int *nstars) {
+Stars *Convert_PS1_V3 (FTable *table, unsigned int *nstars) {
 
   off_t Nstars; 
@@ -368 +440 @@
   double ZeroPt;
   Stars *stars;
-  CMF_PS1_V2 *ps1data;
-
-  ps1data = gfits_table_get_CMF_PS1_V2 (table, &Nstars, NULL);
+  CMF_PS1_V3 *ps1data;
+
+  ps1data = gfits_table_get_CMF_PS1_V3 (table, &Nstars, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -429 +501 @@
   return (stars);
 }
-
-Stars *Convert_PS1_V3 (FTable *table, unsigned int *nstars) {
-
-  off_t Nstars; 
-  unsigned int i;
-  double ZeroPt;
-  Stars *stars;
-  CMF_PS1_V3 *ps1data;
-
-  ps1data = gfits_table_get_CMF_PS1_V3 (table, &Nstars, NULL);
-  if (!ps1data) {
-    fprintf (stderr, "skipping inconsistent entry\n");
-    return (NULL);
-  }
-  ZeroPt = GetZeroPoint();
-
-  ALLOCATE (stars, Stars, Nstars);
-  for (i = 0; i < Nstars; i++) {
-    InitStar (&stars[i]);
-    stars[i].measure.Xccd       = ps1data[i].X;
-    stars[i].measure.Yccd       = ps1data[i].Y;
-    stars[i].measure.dXccd      = ToShortPixels(ps1data[i].dX);
-    stars[i].measure.dYccd      = ToShortPixels(ps1data[i].dY);
-
-    stars[i].measure.posangle   = ToShortDegrees(ps1data[i].posangle);
-    stars[i].measure.pltscale   = ps1data[i].pltscale;
-
-    if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) {
-        stars[i].measure.M      = NAN;
-    } else {
-        stars[i].measure.M      = ps1data[i].M + ZeroPt;
-    }
-    stars[i].measure.dM         = ps1data[i].dM;
-    stars[i].measure.dMcal      = ps1data[i].dMcal;
-    stars[i].measure.Map        = ps1data[i].Map + ZeroPt;
-                        
-    stars[i].measure.Sky        = ps1data[i].sky;
-    stars[i].measure.dSky       = ps1data[i].dSky;
-                        
-    stars[i].measure.psfChisq   = ps1data[i].psfChisq;
-    stars[i].measure.psfQual    = ps1data[i].psfQual;
-    stars[i].measure.psfNdof    = ps1data[i].psfNdof;
-    stars[i].measure.psfNpix    = ps1data[i].psfNpix;
-    stars[i].measure.crNsigma   = ps1data[i].crNsigma;
-    stars[i].measure.extNsigma  = ps1data[i].extNsigma;
-
-    stars[i].measure.FWx        = ToShortPixels(ps1data[i].fx);
-    stars[i].measure.FWy        = ToShortPixels(ps1data[i].fy);
-    stars[i].measure.theta      = ToShortDegrees(ps1data[i].df);
-
-    stars[i].measure.Mxx        = ToShortPixels(ps1data[i].Mxx);
-    stars[i].measure.Mxy        = ToShortPixels(ps1data[i].Mxy);
-    stars[i].measure.Myy        = ToShortPixels(ps1data[i].Myy);
-                        
-    stars[i].measure.photFlags  = ps1data[i].flags;
-
-    // this is may optionally be replaced by the internal sequence (see FilterStars.c)
-    stars[i].measure.detID      = ps1data[i].detID; 
-
-    // the Average fields and the following Measure fields are set in FilterStars after
-    // the image metadata is in hand:  dR, dD, Mcal, dt, airmass, az, t, imageID, extID, 
-    // averef is set in find_matches, dbFlags is zero on ingest.
-
-    // the following fields are currently not being set anywhere: t_msec
-  }    
-  *nstars = Nstars;
-  return (stars);
-}

branches/eam_branches/ipp-20110213/Ohana/src/dvomerge/src/merge_catalogs_old.c

@@ -219 +219 @@
       }
       if (fabs(output[0].measure[Nmeas].dR) > 10*RADIUS) {
-        fprintf (stderr, "error: %10.6f,%10.6f vs %10.6f,%10.6f (%f,%f vs %f,%f)\n", 
-                 output[0].average[n].R, output[0].average[n].D, Rin, Din,
-                 X1[i], X2[Jmin], Y1[i], Y2[Jmin]);
-        // XXX abort on this? -- this is a bad failure...
+        // ok take declination into account and check again.
+        double cosD = cos(RAD_DEG*Din);
+        if (fabs(output[0].measure[Nmeas].dR*cosD) > 10*RADIUS) {
+
+            fprintf (stderr, "error: %10.6f,%10.6f vs %10.6f,%10.6f (%f,%f vs %f,%f)\n", 
+             output[0].average[n].R, output[0].average[n].D, Rin, Din,
+             X1[i], X2[Jmin], Y1[i], Y2[Jmin]);
+
+            // XXX abort on this? -- this is a bad failure...
+        }
       }
       input[0].found[N] = Nmeas;

branches/eam_branches/ipp-20110213/Ohana/src/getstar/src/MatchCoords.c

@@ -3 +3 @@
 # define FLT_MAX 1e32
 # endif
+int ComputeLMLimits(double xmin, double xmax, double ymin, double ymax, 
+            double *Lmin, double *Lmax, double *Mmin, double *Mmax, Coords *coords);
 
 /* given coordinate, find images in list that contain the point */
@@ -12 +14 @@
   double Xi[4], Yi[4];  /* image and original corners */
   double xmin, xmax, ymin, ymax;
+  double Lmin, Lmax, Mmin, Mmax;
 
 
@@ -20 +23 @@
      also define the vtable entries for the images we keep  */
 
+  int haveLimits = 0;
   for (i = 0; i < NdbImages; i++) {
+      haveLimits = 0;
 
     if (!WITH_PHU && !strcmp (&dbImages[i].coords.ctype[4], "-DIS")) continue;
@@ -30 +35 @@
     /* define image corners */
     SetImageCorners (Xi, Yi, &dbImages[i]);
-    // Xi[4] = Xi[0]; Yi[4] = Yi[0];
+    // Xi[3] = Xi[0]; Yi[3] = Yi[0];
 
     ymin = xmin = +FLT_MAX;
@@ -52 +57 @@
             continue;
         }
-
         
-        if (VERBOSE) fprintf(stderr, "%ld %8.1f %8.1f %s\n", i, x, y, dbImages[i].name);
+        if (VERBOSE) fprintf(stderr, OFF_T_FMT" %8.1f %8.1f %s\n", i, x, y, dbImages[i].name);
 
         if ((x >= xmin && x <= xmax) && (y >= ymin && y <= ymax)) {
+            if (VERBOSE) fprintf(stderr, "\tpotential overlap for point %d image %i\n", j, (int) i);
+
+            // We may be using the astrometry far outside it's range of validity.
+            // As a sanity check make sure that the focal plane coords are in bounds.
+            // See ticket 1381
+            if (!haveLimits) {
+                // We defer computing the focal plane limits until we need them
+                status = ComputeLMLimits(xmin, xmax, ymin, ymax, 
+                    &Lmin, &Lmax, &Mmin, &Mmax, &dbImages[i].coords);
+                if (!status) {
+                    continue;
+                }
+                haveLimits = 1;
+            }
+
+            double L, M;
+            status = RD_to_LM(&L, &M, pt->ra, pt->dec, &dbImages[i].coords);
+            if (!status) {
+                // This won't happen because RD_to_XY succeeded above
+                continue;
+            }
+
+            if (VERBOSE) fprintf(stderr, "L: %f M: %f\n", L, M);
+
+            if (L < Lmin || L > Lmax || M < Mmin || M > Mmax) {
+                if (VERBOSE) fprintf(stderr, "\trejecting match for because focal plane coordinates are out of range\n");
+                continue;
+            }
+            if (VERBOSE) fprintf(stderr, "\tmatch confirmed\n");
+
             totalMatches++;
 
@@ -88 +122 @@
   }
 }
+
+// compute bounding box of this image in focal plane coordinates
+int ComputeLMLimits(double xmin, double xmax, double ymin, double ymax, 
+            double *Lmin, double *Lmax, double *Mmin, double *Mmax, Coords *coords)
+{
+    double L0, L1, M0, M1;
+    int status = XY_to_LM(&L0, &M0, xmin, ymin, coords);
+    if (!status) {
+        fprintf(stderr, "failed to transform image corner to LM\n");
+        return 0;
+    }
+    status = XY_to_LM(&L1, &M1, xmax, ymax, coords);
+    if (!status) {
+        fprintf(stderr, "failed to transform image corner to LM\n");
+        return 0;
+    }
+
+    if (L1 > L0) {
+        *Lmin = L0;
+        *Lmax = L1;
+    } else {
+        *Lmin = L1;
+        *Lmax = L0;
+    }
+    if (M1 > M0) {
+        *Mmin = M0;
+        *Mmax = M1;
+    } else {
+        *Mmin = M1;
+        *Mmax = M0;
+    }
+    if (VERBOSE) fprintf(stderr, "Lmin: %f Lmax: %f Mmin: %f Mmax: %f\n", *Lmin, *Lmax, *Mmin, *Mmax);
+    return 1;
+}

branches/eam_branches/ipp-20110213/Ohana/src/libohana/include/ohana_allocate.h

@@ -49 +49 @@
 
 # define REALLOCATE(PTR,TYPE,SIZE) {                                    \
-  PTR = (TYPE *) realloc(PTR,(unsigned)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \
+  PTR = (TYPE *) realloc(PTR,(size_t)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \
   if (PTR == NULL) {                                                    \
     fprintf(stderr,"failed realloc at %d in %s\n", __LINE__, __FILE__); \
@@ -57 +57 @@
   if ((NCURR) >= (SIZE)) {                              \
     SIZE += DELTA;                                                      \
-    PTR = (TYPE *) realloc(PTR,(unsigned)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \
+    PTR = (TYPE *) realloc(PTR,(size_t)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \
     if (PTR == NULL) {                                                  \
       fprintf(stderr,"failed realloc increment at %d in %s\n", __LINE__, __FILE__); \

branches/eam_branches/ipp-20110213/PS-IPP-Config/lib/PS/IPP/Config.pm

@@ -1209 +1209 @@
         if ($@) {
             carp "nebulous delete for $todelete failed. Ignoring.\n";
-            return $PS_EXIT_SYS_ERROR;
         }
     }

branches/eam_branches/ipp-20110213/ippMonitor/raw/czartool_labels.php

@@ -220 +220 @@
                 showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1);
                 showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS);
-
+                showReplicationsStatus($REPL_HOST_PSTAMP, $REPL_USER_PSTAMP, $REPL_PASSWORD_PSTAMP, $REPL_DBNAME_PSTAMP);
               echo "</table>";
             echo "</td>\n";

branches/eam_branches/ipp-20110213/ippMonitor/raw/site.php.in

@@ -32 +32 @@
 $REPL_DBNAME_NEBULOUS = "Nebulous";
 
+$REPL_HOST_PSTAMP = "ippc19.IfA.Hawaii.Edu";
+$REPL_USER_PSTAMP = "ippMonitor";
+$REPL_PASSWORD_PSTAMP = "ippMonitor";
+$REPL_DBNAME_PSTAMP = "ippRequestServer";
+
 ?>

branches/eam_branches/ipp-20110213/ippScripts/scripts/magic_destreak_cleanup.pl

@@ -102 +102 @@
 my $cam_path_base = $run->{cam_path_base};
 my $cam_reduction = $run->{cam_reduction};
+my $replace = $run->{re_place};
 $cam_reduction = 'DEFAULT' if !$cam_reduction or ($cam_reduction eq 'NULL');
 
@@ -111 +112 @@
 
 &my_die("unexpected run state found: $state", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $state ne "goto_cleaned";
-&my_die("clean not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw";
+&my_die("cleanup not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw" and $replace;
 
 my $recipe_psastro;
@@ -217 +218 @@
                 # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
             }
+        } elsif ($stage eq "raw") {
+            if ($backup_path_base) {
+                if ($backup_path_base =~ /fits/) {
+                    $bimage = $backup_path_base;
+                } else {
+                    $bimage = $backup_path_base . ".fits";
+                }
+            }
         }
 

branches/eam_branches/ipp-20110213/ippScripts/scripts/magic_destreak_revert.pl

@@ -78 +78 @@
     defined $outroot;
 
-&my_die("bytes and md5sum are is required for raw stage stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($stage eq 'raw' and (!$bytes or !$md5sum));
+if (defined($replace) and ($replace eq "T")) {
+    $replace = 1;
+} else {
+    $replace = 0;
+}
+
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile;
+
+&my_die("bytes and md5sum are is required for raw stage stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($replace and $stage eq 'raw' and (!$bytes or !$md5sum));
 
 &my_die("cam_path_base is required for chip stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($stage eq 'chip' and !$cam_path_base);
@@ -84 +94 @@
 
 
-my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
-
-$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile;
 
 $cam_reduction = 'DEFAULT' if !defined $cam_reduction or ($cam_reduction eq "NULL");
@@ -118 +125 @@
 my $basename = basename($path_base);
 my $nebulousInput = inNebulous($dirname);
-
-if (defined($replace) and ($replace eq "T")) {
-    $replace = 1;
-} else {
-    $replace = 0;
-}
 
 &my_die("replace not allowed for non-nebulous files", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR)
@@ -162 +163 @@
         $image = $path_base . ".fits";
     }
-    $bimage = $backup_path_base . ".fits";
+    if ($backup_path_base =~ /.*\.fits$/) {
+        $bimage = $backup_path_base;
+    } else {
+        $bimage = $backup_path_base . ".fits";
+    }
 } elsif ($stage eq "chip") {
     # Check to see if we're using dynamic masks
@@ -302 +307 @@
 sub revert_files {
     my $replace = shift;
-    return if !$replace;
+#    return if !$replace;
 
     my $image = shift;
@@ -316 +321 @@
 
     if ($image) {
-        revert_file($image, $bimage) or
+        revert_file($replace, $image, $bimage) or
             &my_die("failed to restore image file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     }
 
     if ($mask) {
-        if (!revert_file($mask, $bmask)) {
+        if (!revert_file($replace, $mask, $bmask)) {
             &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
         }
@@ -327 +332 @@
 
     if ($ch_mask) {
-        if (!revert_file($ch_mask, $bch_mask)) {
+        if (!revert_file($replace, $ch_mask, $bch_mask)) {
             &my_die("failed to restore chip mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
         }
@@ -334 +339 @@
 
     if ($weight) {
-        revert_file($weight, $bweight) or
+        revert_file($replace, $weight, $bweight) or
             &my_die("failed to restore variance image", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     }
 
     if ($sources) {
-        revert_file($sources, $bsources) or
+        revert_file($replace, $sources, $bsources) or
             &my_die("failed to restore sources file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     }
 
     if ($astrom) {
-        revert_file($astrom, $bastrom) or
+        revert_file($replace, $astrom, $bastrom) or
             &my_die("failed to restore astrometry file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
     }
@@ -366 +371 @@
 sub revert_file
 {
+    my $replace = shift;
     my $original = shift;
     my $backup   = shift;
     my $force = 1; # force deletion of backup files in nebulous
+
+    if (!$replace) {
+        # we don't need to do all of this checking unless the destreak run is
+        # replace mode just delete the backup 
+        my $error_code = $ipprc->kill_file($backup);
+        if ($error_code) {
+            print STDERR "failed to kill $backup: error_code $error_code\n";
+            return 0;
+        } else {
+            return 1;
+        }
+    }
 
     my $o_path = $ipprc->file_resolve($original);

branches/eam_branches/ipp-20110213/ippScripts/scripts/register_imfile.pl

@@ -181 +181 @@
 if (uc(&value_for_flag ($cmdflags, "NULL", "-exp_type"))  eq "NULL") { &my_die_for_add ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
 my $dateobs   = &value_for_flag($cmdflags, 0.0, "-dateobs");
+my $exp_type  = &value_for_flag($cmdflags, "NULL", "-exp_type");
 
 my $command = "$regtool -addprocessedimfile";

branches/eam_branches/ipp-20110213/ippTasks/destreak.pro

@@ -204 +204 @@
     book getword magicToDS $pageName dbname -var DBNAME
 
-    sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT
 
     substr $COMPONENT 0 3 COMP_HEAD
     if ("$COMP_HEAD" == "sky")
         set.host.for.skycell $COMPONENT
+        set.workdir.by.skycell $COMPONENT $OUTROOT $default_host WORKDIR
     else 
         # assume component is a class_id, if not we will default to anyhost
         set.host.for.camera $CAMERA $COMPONENT
-    end
+        set.workdir.by.camera $CAMERA $COMPONENT $OUTROOT $default_host WORKDIR
+    end
+
+    sprintf logfile "%s/%s.mds.%s.%s.%s.log" $WORKDIR $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT
 
     # TODO: do not add recoveryroot or replace if they are null or zero
 
-    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED --run-state $RUN_STATE
+    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $WORKDIR --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED --run-state $RUN_STATE
 
     add_standard_args run
@@ -407 +410 @@
     book getword magicDSToRevert $pageName dbname -var DBNAME
 
-    sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT
-
     substr $COMPONENT 0 3 COMP_HEAD
     if ("$COMP_HEAD" == "sky")
         set.host.for.skycell $COMPONENT
+        set.workdir.by.skycell $COMPONENT $OUTROOT $default_host WORKDIR
     else 
         # assume component is a class_id, if not we will default to anyhost
         set.host.for.camera $CAMERA $COMPONENT
-    end
-
-    $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --replace $REPLACE --bothways $BOTHWAYS --magicked $MAGICKED --run-state $RUN_STATE --recovery_path_base $RECOVERY_PATH_BASE
+        set.workdir.by.camera $CAMERA $COMPONENT $OUTROOT $default_host WORKDIR
+    end
+    sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $WORKDIR $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT
+
+    $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $WORKDIR --logfile $logfile --replace $REPLACE --bothways $BOTHWAYS --magicked $MAGICKED --run-state $RUN_STATE --recovery_path_base $RECOVERY_PATH_BASE
 
     add_standard_args run

branches/eam_branches/ipp-20110213/ippTasks/pstamp.pro

@@ -19 +19 @@
 
 # give up on dependents with fault_count >= $PSTAMP_MAX_FAULT_COUNT
-$PSTAMP_MAX_FAULT_COUNT = 5
+$PSTAMP_MAX_FAULT_COUNT = 3
 macro set.max.fault.count
     $PSTAMP_MAX_FAULT_COUNT = $1
@@ -717 +717 @@
 
         $MYLOGFILE = $OUTDIR/checkdep.$DEP_ID.log
-        stdout $MYLOGFILE
+        stdout NULL
         stderr $MYLOGFILE
 
-        $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT
+        $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT --logfile $MYLOGFILE
 
         add_standard_args run
@@ -928 +928 @@
 
     task.exec
+        stdout $LOGSUBDIR/pstamp.stopfaulted.log
         stderr $LOGSUBDIR/pstamp.stopfaulted.log
-        stderr $LOGSUBDIR/pstamp.stopfaulted.log
-        $run = pstampstopfaulted
+        $run = pstampstopfaulted -fault_count $PSTAMP_MAX_FAULT_COUNT
         if ($DB:n == 0)
             option DEFAULT

branches/eam_branches/ipp-20110213/ippTasks/register.pro

@@ -203 +203 @@
 $regRevertImfile_DB = 0
 $regPendingExp_DB = 0
-
+$reg_datePAGE = 0
 # select images ready for register analysis
 # new entries are added to regPendingImfile

branches/eam_branches/ipp-20110213/ippToPsps/src/Batch.h

@@ -14 +14 @@
 #include <psmodules.h>
 #include <dvo_util.h>
-#include "ippToPspsConfig.h"
+#include "Config.h"
 #include <libxml/parser.h>
 #include <libxml/tree.h>

branches/eam_branches/ipp-20110213/ippToPsps/src/DifferenceBatch.c

@@ -8 +8 @@
 
 #include "ippToPsps.h"
-#include "ippToPspsConfig.h"
+#include "Config.h"
 #include "ippToPspsDiffEnums.h"
 

branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_todestreak_raw.sql

@@ -24 +24 @@
     re_place,
     remove,
-    10000 AS priority
+    IFNULL(Label.priority, 10000) AS priority
 FROM magicDSRun
 JOIN magicMask USING (magic_id)
@@ -35 +35 @@
     ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
     AND magicDSFile.component = rawImfile.class_id
+LEFT JOIN Label ON magicDSRun.label = Label.label
 WHERE
     magicDSRun.state = 'new'
     AND magicDSRun.stage = 'raw'
     AND magicDSFile.component IS NULL
+    AND (Label.active OR Label.active IS NULL)
+

branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_diff.sql

@@ -13 +13 @@
     magicDSFile.recovery_path_base,
     "NULL" AS cam_path_base,
-    recovery_path_base,
     CAST(diffRun.bothways AS SIGNED) AS bothways,
     0 AS bytes,

branches/eam_branches/ipp-20110213/ippTools/src/addtool.c

@@ -324 +324 @@
     // pxUpdateRun gets parameters from config->args and runs the update query
     bool result = pxUpdateRun(config, where, &query, "addRun", "add_id",
-        "addProcessedExp", false);
+        "addProcessedExp", false, false);
 
     psFree(query);

branches/eam_branches/ipp-20110213/ippTools/src/bgtool.c

@@ -501 +501 @@
     psString query = psStringCopy("UPDATE chipBackgroundRun");
     bool result = pxUpdateRun(config, where, &query, "chipBackgroundRun", "chip_bg_id",
-                              "chipBackgroundImfile", true);
+                              "chipBackgroundImfile", true, true);
 
     psFree(query);
@@ -1471 +1471 @@
     psString query = psStringCopy("UPDATE warpBackgroundRun");
     bool result = pxUpdateRun(config, where, &query, "warpBackgroundRun", "warp_bg_id",
-                              "warpBackgroundSkyfile", true);
+                              "warpBackgroundSkyfile", true, true);
 
     psFree(query);

branches/eam_branches/ipp-20110213/ippTools/src/camtool.c

@@ -283 +283 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "camRun", "cam_id", "camProcessedExp", true);
+    bool result = pxUpdateRun(config, where, &query, "camRun", "cam_id", "camProcessedExp", true, true);
     if (!result) {
         psError(psErrorCodeLast(), false, "pxUpdateRun failed");

branches/eam_branches/ipp-20110213/ippTools/src/camtoolConfig.c

@@ -86 +86 @@
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0,   "define new dist_group", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0,         "define new note", NULL);
+    pxmagicAddArguments(updaterunArgs);
 
     // -pendingexp
@@ -299 +300 @@
     PXOPT_ADD_MODE("-addprocessedexp",      "add a processed exposure",             CAMTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs);
     PXOPT_ADD_MODE("-processedexp",         "show processed exposures",             CAMTOOL_MODE_PROCESSEDEXP,  processedexpArgs);
-    PXOPT_ADD_MODE("-revertprocessedexp",   "change procesed exp properties",       CAMTOOL_MODE_REVERTPROCESSEDEXP,  revertprocessedexpArgs);
-    PXOPT_ADD_MODE("-updateprocessedexp",   "undo a processed exposure",            CAMTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs);
+    PXOPT_ADD_MODE("-revertprocessedexp",   "undo a processed exposure",       CAMTOOL_MODE_REVERTPROCESSEDEXP,  revertprocessedexpArgs);
+    PXOPT_ADD_MODE("-updateprocessedexp",   "changed processed exp properties",            CAMTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs);
     PXOPT_ADD_MODE("-block",                "set a label block",                    CAMTOOL_MODE_BLOCK,         blockArgs);
     PXOPT_ADD_MODE("-masked",               "show blocked labels",                  CAMTOOL_MODE_MASKED,        maskedArgs);

branches/eam_branches/ipp-20110213/ippTools/src/chiptool.c

@@ -443 +443 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "chipRun", "chip_id", "chipProcessedImfile", true);
+    bool result = pxUpdateRun(config, where, &query, "chipRun", "chip_id", "chipProcessedImfile", true, true);
     if (!result) {
         psError(psErrorCodeLast(), false, "pxUpdateRun failed");

branches/eam_branches/ipp-20110213/ippTools/src/chiptoolConfig.c

@@ -99 +99 @@
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0,   "set dist group", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0,         "set note", NULL);
+    pxmagicAddArguments(updaterunArgs);
 
     // -setimfiletoupdate

branches/eam_branches/ipp-20110213/ippTools/src/diffphottool.c

@@ -228 +228 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "diffPhotRun", "diff_phot_id", "diffPhotSkyfile", true);
+    bool result = pxUpdateRun(config, where, &query, "diffPhotRun", "diff_phot_id", "diffPhotSkyfile", true, false);
 
     psFree(query);

branches/eam_branches/ipp-20110213/ippTools/src/difftool.c

@@ -224 +224 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "diffRun", "diff_id", "diffSkyfile", true);
+    bool result = pxUpdateRun(config, where, &query, "diffRun", "diff_id", "diffSkyfile", true, true);
 
     psFree(query);

branches/eam_branches/ipp-20110213/ippTools/src/difftoolConfig.c

@@ -73 +73 @@
     psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_diff_mode", 0,    "specify type of difference (WW=1,WS=2,SW=3,SS=4)", 0);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0,         "define new note", NULL);
+    pxmagicAddArguments(updaterunArgs);
 
     // -addinputskyfile

branches/eam_branches/ipp-20110213/ippTools/src/faketool.c

@@ -348 +348 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "fakeRun", "fake_id", "fakeProcessedImfile", true);
+    bool result = pxUpdateRun(config, where, &query, "fakeRun", "fake_id", "fakeProcessedImfile", true, false);
     if (!result) {
         psError(psErrorCodeLast(), false, "pxUpdateRun failed");

branches/eam_branches/ipp-20110213/ippTools/src/magicdstool.c

@@ -1395 +1395 @@
           (strcmp(state, "full") == 0) ||
           (strcmp(state, "drop") == 0) ||
+          (strcmp(state, "wait") == 0) ||
           (strcmp(state, "failed_revert") == 0) ||
           (strcmp(state, "error_cleaned") == 0) ||

branches/eam_branches/ipp-20110213/ippTools/src/pubtool.c

@@ -37 +37 @@
 static bool addMode(pxConfig *config);
 static bool revertMode(pxConfig *config);
+static bool updaterunMode(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -63 +64 @@
         MODECASE(PUBTOOL_MODE_ADD, addMode);
         MODECASE(PUBTOOL_MODE_REVERT, revertMode);
+        MODECASE(PUBTOOL_MODE_UPDATERUN, updaterunMode);
       default:
         psAbort("invalid option (this should not happen)");
@@ -459 +461 @@
 }
 
+static bool updaterunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc(); // WHERE conditions
+    PXOPT_COPY_S64(config->args, where, "-pub_id", "pub_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-client_id", "client_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    PXOPT_COPY_STR(config->args, where, "-fault", "fault", "==");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state",  true, false);
+
+    psString query = NULL;              // Query to run
+    psStringAppend(&query, "UPDATE publishRun LEFT JOIN publishDone using(pub_id) SET state = '%s'", state);
+
+    if (psListLength(where->list)) {
+        psString clause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "\n WHERE %s", clause);
+        psFree(clause);
+    } else {
+        psError(PS_ERR_UNKNOWN, false, "select arguments are required");
+        return false;
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    long numUpdated = psDBAffectedRows(config->dbh);
+    psLogMsg("pubtool", PS_LOG_INFO, "%ld rows updated.", numUpdated);
+
+    return true;
+}

branches/eam_branches/ipp-20110213/ippTools/src/pubtool.h

@@ -31 +31 @@
     PUBTOOL_MODE_ADD,
     PUBTOOL_MODE_REVERT,
+    PUBTOOL_MODE_UPDATERUN,
 } pubtoolMode;
 

branches/eam_branches/ipp-20110213/ippTools/src/pubtoolConfig.c

@@ -102 +102 @@
     psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on label", NULL);
 
+    // -updaterun
+    psMetadata *updaterunArgs = psMetadataAlloc();
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL);
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-pub_id", 0, "search on pub_id", 0);
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-client_id", 0, "search on client_id", 0);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "search by fault", NULL);
+
 
     psMetadata *argSets = psMetadataAlloc();
@@ -112 +121 @@
     PXOPT_ADD_MODE("-add", "", PUBTOOL_MODE_ADD, addArgs);
     PXOPT_ADD_MODE("-revert", "", PUBTOOL_MODE_REVERT, revertArgs);
+    PXOPT_ADD_MODE("-updaterun", "", PUBTOOL_MODE_UPDATERUN, updaterunArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/eam_branches/ipp-20110213/ippTools/src/pxtools.c

@@ -278 +278 @@
 
 // shared code for updating the various strings for a Run
-bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group)
+bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group, bool has_magicked)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -360 +360 @@
     if (fileWhere) {
         psStringAppend(pQuery, "%s", fileWhere);
+    }
+    if (has_magicked) {
+        pxmagicAddWhere(config, pQuery, runTable);
     }
 

branches/eam_branches/ipp-20110213/ippTools/src/pxtools.h

@@ -84 +84 @@
 bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets);
 
-bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group);
+bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group, bool has_magicked);
 
 #define PXOPT_ADD_MODE(option, comment, modeval, argset) \

branches/eam_branches/ipp-20110213/ippTools/src/stacktool.c

@@ -750 +750 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true);
+    bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true, false);
 
     psFree(query);

branches/eam_branches/ipp-20110213/ippTools/src/staticskytool.c

@@ -360 +360 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "staticskyRun", "sky_id", "staticskyResult", true);
+    bool result = pxUpdateRun(config, where, &query, "staticskyRun", "sky_id", "staticskyResult", true, false);
     psFree(query);
     psFree(where);

branches/eam_branches/ipp-20110213/ippTools/src/warptool.c

@@ -435 +435 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "warpRun", "warp_id", "warpSkyfile", true);
+    bool result = pxUpdateRun(config, where, &query, "warpRun", "warp_id", "warpSkyfile", true, true);
 
     psFree(query);

branches/eam_branches/ipp-20110213/ippTools/src/warptoolConfig.c

@@ -141 +141 @@
     psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-registered_begin", 0, "search for runs by registration time (>=)", NULL);
     psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-registered_end", 0, "search for runs by registration time (<)", NULL);
-    psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
     psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false);
 
@@ -149 +148 @@
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0,   "define new dist_group", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0,         "define new note", NULL);
+    pxmagicAddArguments(updaterunArgs);
 
     // -exp

branches/eam_branches/ipp-20110213/ippconfig/recipes/nightly_science.config

@@ -2 +2 @@
 CLEAN_MODES METADATA
   MODE           STR CHIP
-  COMMAND        STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@
+  COMMAND        STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked
   RETENTION_TIME U16 21
 END
 CLEAN_MODES METADATA
   MODE           STR WARP
-  COMMAND        STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@
+  COMMAND        STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked
   RETENTION_TIME U16 7
 END
 CLEAN_MODES METADATA
   MODE           STR DIFF
-  COMMAND        STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@
+  COMMAND        STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked
   RETENTION_TIME S16 21
 END

branches/eam_branches/ipp-20110213/pstamp/scripts/pstamp_checkdependent.pl

@@ -27 +27 @@
 my $IPP_DIFF_MODE_STACK_STACK = 4;
 
-my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count);
+my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count, $logfile);
 my ($dbname, $ps_dbserver, $verbose, $save_temps, $no_update);
 
@@ -40 +40 @@
     'fault_count=i' =>  \$fault_count,
     'max_fault_count=i' =>  \$max_fault_count,
+    'logfile=s'     =>  \$logfile,
     'dbname=s'      =>  \$dbname,       # postage stamp server dbname
     'dbserver=s'    =>  \$ps_dbserver,  # postage stamp server dbserver
@@ -68 +69 @@
 my $ipprc = PS::IPP::Config->new();
 
+if ($logfile) {
+   $ipprc->redirect_output($logfile);
+}
+
 if (!$ps_dbserver) {
     $ps_dbserver =  metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
@@ -314 +319 @@
     my $data_state = $metadata->{data_state};
 
+    # The update system currently requires that data that has been magicked be destreaked
+    # at chip stage so it needs magic even if the customer doesn't.
+    if ($metadata->{magicked} != 0) {
+        $need_magic = 1;
+    }
+
     if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
          ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) {
@@ -350 +361 @@
     foreach my $chip (@$data) {
         $chip_id = $chip->{chip_id};
-        if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
+
+        # if chip has been magicked before require it to be magicked again
+        # because the warp pending query requires it.
+        if ($chip->{magicked} < 0) {
+            print "Input has been destreaked so we must destreak before warping\n";
+            $need_magic = 1 
+        }
+
+        if ($need_magic and ($chip->{magicked} eq 0)) {
+            my_die("Client requires magic, but chip never magicked. How did this dependent get queued?", $PS_EXIT_PROG_ERROR);
+        }
+
+        if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} < 0))) {
             $chips_ready = 0;
             $chip->{fault} = $chip->{chip_fault};
@@ -399 +422 @@
     my $skycell = $metadata;
     my $skycell_id = $skycell->{skycell_id};
+
+    # The update system currently requires that data that has been magicked be destreaked
+    # at chip stage so it needs magic even if the customer doesn't.
+    if ($metadata->{magicked} != 0) {
+        $need_magic = 1;
+    }
 
     if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {

branches/eam_branches/ipp-20110213/pstamp/scripts/pstampparse.pl

@@ -31 +31 @@
 my $no_update;
 my $dest_requires_magic;
+
+# set this to true to disable update processing
+my $no_updates_allowed = 0;
 
 GetOptions(
@@ -424 +427 @@
                 # and this user's data store destination is allowed uncensored stamps, so accept the request
                 $need_magic = 0;
+
+                # Since user can get unmagicked data "by coordinate" requests can go back in time
+                # to dredge unusable data from the "dark days"...
+                if ($req_type eq 'bycoord' and $mjd_min eq 0) {
+                    # ... so unless the user sets mjd_min clamp it to 2009-04-01
+                    # XXX: This value should live in the pstampProject table not be hardcoded here
+                    $mjd_min = 54922;
+                }
             } else {
                 print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
@@ -465 +476 @@
     print "Time to locate_images for row $rownum $dtime_locate\n";
 
-    # handle this XXX: what did I mean by this comment
+    # handle this 
+    # XXX: what did I mean by that comment?
     $row->{need_magic} = $need_magic;
 
     $num_jobs += queueJobs($mode, \@rowList, $imageList);
 
-    # if a row slipped through with no jobs add one
+    # if a row slipped through with no jobs add a faulted one
     foreach my $row (@rowList) {
         if ($row->{job_num} == 0) {
@@ -495 +507 @@
     my $option_mask = $row->{OPTION_MASK};
     my $components = $row->{components};
+
+    $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_updates_allowed;
 
     my $roi_string;
@@ -668 +682 @@
     my $rownum = $row->{ROWNUM};
     my $option_mask = $row->{OPTION_MASK};
+
+    $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_updates_allowed;
 
     # For dist_bundle we need

branches/eam_branches/ipp-20110213/pstamp/scripts/pstampstopfaulted

@@ +1 @@
+#!/bin/bash
+#
+# script executed by the task pstamp.stopfaulted to stop proceessing of jobs
+# and dependents that have fault_count >= some maximum value. That value is expected
+# to be passed in as an argument otherwise the default in pstamptool is used
+# All arguments are passed to pstamptool
 
-# script executed by the task pstamp.stopfaulted to stop proceessing of jobs
-# and dependents that have faulted 5 or more times
-# all arguments to this script are passed to pstamptool
-
-# 25 is PSTAMP_NOT_AVAILABLE
+# note: 25 is PSTAMP_NOT_AVAILABLE
 fault_code=25
-fault_count=3
 
 date
 echo stopping faulted dependent jobs
-pstamptool -stopdependentjob -set_fault $fault_code -fault_count $fault_count $*
+cmd="pstamptool -stopdependentjob -set_fault $fault_code $*"
+echo $cmd
+$cmd
 
 echo stopping faulted jobs
-pstamptool -updatejob -state run -set_state stop -set_fault $fault_code -fault_count $fault_count $*
+cmd="pstamptool -updatejob -state run -set_state stop -set_fault $fault_code $*"
+echo $cmd
+$cmd

branches/eam_branches/ipp-20110213/tools/czartool/Config.pm

@@ -66 +66 @@
     $self->{roboczaremail} = $xc->findvalue('//roboczar/email');
     $self->{roboczarserverinterval} = $xc->findvalue('//roboczar/serverinterval');
-
-
+    $self->{roboczarinterestedservers} = $xc->findvalue('//roboczar/interestedservers');
 }
 
@@ -126 +125 @@
 ###########################################################################
 #
+# Returns array of servers we are interested in for roboczar
+#
+###########################################################################
+sub getRoboczarInterestedServers {
+    my ($self) = @_;
+
+    my @servers = split('\s+', $self->{roboczarinterestedservers});
+
+    return @servers;
+}
+
+###########################################################################
+#
 # Returns the email to which roboczar should send its warnings
 #
@@ -136 +148 @@
 ###########################################################################
 #
-# Returns  
+# Returns font for gnuplot
 #
 ###########################################################################
@@ -146 +158 @@
 ###########################################################################
 #
-# Returns  
+# Returns font size for gnuplot
 #
 ###########################################################################
@@ -156 +168 @@
 ###########################################################################
 #
-# Returns  
+# Returns save location for metrics
 #
 ###########################################################################
@@ -166 +178 @@
 ###########################################################################
 #
-# Returns  
+# Returns daily start time for metrics
 #
 ###########################################################################
@@ -176 +188 @@
 ###########################################################################
 #
-# Returns  
+# Returns name for gpc1 Db
 #
 ###########################################################################
@@ -186 +198 @@
 ###########################################################################
 #
-# Returns  
+# Returns host for gpc1 Db
 #
 ###########################################################################
@@ -196 +208 @@
 ###########################################################################
 #
-# Returns  
+# Returns user for gpc1 Db
 #
 ###########################################################################
@@ -206 +218 @@
 ###########################################################################
 #
-# Returns  
+# Returns password for gpc1 Db
 #
 ###########################################################################
@@ -216 +228 @@
 ###########################################################################
 #
-# Returns  
+# Returns czar Db name
 #
 ###########################################################################
@@ -226 +238 @@
 ###########################################################################
 #
-# Returns  
+# Returns host for czar Db
 #
 ###########################################################################
@@ -236 +248 @@
 ###########################################################################
 #
-# Returns  
+# Returns username for czar Db
 #
 ###########################################################################
@@ -246 +258 @@
 ###########################################################################
 #
-# Returns  
+# Returns password for czar Db 
 #
 ###########################################################################

branches/eam_branches/ipp-20110213/tools/czartool/czarconfig.xml

@@ -1 +1 @@
 <?xml version="1.0" encoding="UTF-8"?>
+
+<!-- Global config for all czartool stuff -->
 
 <czarconfig>
 
+  <!-- Gnuplot config section -->
   <gnuplot>
     <font>/usr/share/fonts/corefonts/arial.ttf</font>
@@ -8 +11 @@
   </gnuplot>
 
+  <!-- Metrics section -->
   <metrics>
     <savelocation>/data/ipp004.0/ipp/ippMetrics</savelocation>
@@ -13 +17 @@
   </metrics>
 
+  <!-- gpc1 Db section -->
   <gpc1database>
     <name>gpc1</name>
@@ -20 +25 @@
   </gpc1database>
 
+  <!-- Czar Db section -->
   <czardatabase>
     <name>czardb</name>
@@ -28 +34 @@
   </czardatabase>
 
+  <!-- Roboczar section -->
   <roboczar>
     <email>ps-ipp-ops@ifa.hawaii.edu</email>
+    <!-- How often should ropbczar check status? (MySQL 'interval' format please, eg 20 MINUTE, 1 HOUR etc) check-->
     <serverinterval>20 MINUTE</serverinterval>
+    <!-- whitespace-separated server list for servers we want roboczar to check --> 
+    <!-- <interestedservers>stdscience distribution summitcopy registration pstamp</interestedservers> -->
+    <interestedservers>pstamp</interestedservers>
   </roboczar>
+
 </czarconfig>

branches/eam_branches/ipp-20110213/tools/roboczar.pl

@@ -22 +22 @@
 
 my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO get from Pantasks
-my @serversWeCareAbout = ("stdscience", "distribution", "summitcopy", "registration", "pstamp");
 
+my @interestedServers = $config->getRoboczarInterestedServers();
 
 my $stuckMessage;
@@ -115 +115 @@
     ${$message} = "Pantasks servers:\n";
 
-    foreach $server (@serversWeCareAbout) {
+    foreach $server (@interestedServers) {
 
         # is server alice?