Index: /branches/eam_branches/ipp-20101103/Ohana/src/addstar/src/ReadStarsFITS.c
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/addstar/src/ReadStarsFITS.c	(revision 29713)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/addstar/src/ReadStarsFITS.c	(revision 29714)
@@ -22,5 +22,5 @@
 
   /* load the table data */
-  if (!gfits_fread_ftable_data (f, &table)) {
+  if (!gfits_fread_ftable_data (f, &table, FALSE)) {
     fprintf (stderr, "ERROR: can't read table header\n");
     exit (1);
Index: /branches/eam_branches/ipp-20101103/Ohana/src/addstar/src/ReadStarsSDSS.c
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/addstar/src/ReadStarsSDSS.c	(revision 29713)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/addstar/src/ReadStarsSDSS.c	(revision 29714)
@@ -60,5 +60,5 @@
 
   /* load the table data */
-  if (!gfits_fread_ftable_data (f, &table)) {
+  if (!gfits_fread_ftable_data (f, &table, FALSE)) {
     fprintf (stderr, "ERROR: can't read table header\n");
     exit (1);
Index: /branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/doc/notes.txt
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/doc/notes.txt	(revision 29714)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/doc/notes.txt	(revision 29714)
@@ -0,0 +1,24 @@
+
+2010.11.08 : DVO repairs
+
+  * I have made a dvorepair program that reconstructs cpt files from a valid cpm file
+
+  * the ThreePi.V1 database got busted; I think we have the following issues:
+
+    * cpt files which are short (and can be fixed with the command above)
+
+    * Image.dat has many image that have no matching detections (repeated dvomerge runs after the failure)
+
+    * a number of cpm files which are short : in this case, I believe
+      these files have all detections before the failure, but are
+      missing some of the detections from the failure. 
+
+  * repair step 1 : scan image IDs on the database : count the number
+    of detections for each image ID and find those image IDs which are
+    missing detections
+
+  * remove images that are missing their detections
+
+  * create repaired cpm files excluding the now-orphaned detections
+
+
Index: /branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvorepair.c
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvorepair.c	(revision 29713)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvorepair.c	(revision 29714)
@@ -4,184 +4,173 @@
 # define myAssert(LOGIC,MSG) { if (!LOGIC) { fprintf (stderr, "%s\n", MSG); abort(); } }
 
-// broken cpt file, valid cpm file: we can recover everything in the cpt file from the cpm file:
-// * load the full cpm file
-// * loop over detections
-// * if ave_ref is new: create a new object
-//   * determine RA & DEC from ext_id
-//   * obj_id, cat_id are defined in detection
-//   * 
+// find images which are missing detections:
+// * load the Images.dat file
+// * create an index for imageID (seq = imageIDindex[i])
+// * scan over all catalog files
+// * load the cpm file (pad if short, identify the padded section)
+// * loop over detections: increment detection count for each imageID
 
 int main (int argc, char **argv) {
 
-  off_t Nmeasure, Nimage;
-  int i, Nbytes, NaveMax, Naverage, NAVERAGE, Nave, Nold;
-  int *found;
+  off_t i, j, Nmeasure, Nvalid, Nimage, Nindex, *imageIdx, index, imageSeq;
+  int Nbytes, NbytesPerRow, Nbad, Ncheck, Ntol;
+  int *detCounts;
 
-  Image *image, *thisImage;
-  Average *average;
+  Image *image;
   Measure *measure;
 
-  Matrix matrix;
+  SkyTable *insky;
+  SkyList *inlist;
 
-  char *cpmFilename, *cptFilenameSrc, *cptFilenameTgt, *imageFilename;
+  char *catdir = NULL;
+  char *cpmFilename = NULL;
+  char *imageFilename = NULL;
 
-  Header cptHeaderPHU, cptHeaderTBL;
-  Header cpmHeaderPHU, cpmHeaderTBL;
-  FTable cpmFtable, cptFtable;
+  Header cpmHeaderPHU;
+  Header cpmHeaderTBL;
+  FTable cpmFtable;
 
-  FILE *cptFileSrc = NULL;
-  FILE *cptFileTgt = NULL;
   FILE *cpmFile = NULL;
 
   char catformat;
 
-  if (argc != 5) {
-    fprintf (stderr, "USAGE: dvorepair (images) (cpm) (cptInput) (cptOutput)\n");
+  if (argc != 3) {
+    fprintf (stderr, "USAGE: dvorepair (catdir) (Ntol)\n");
+    fprintf (stderr, "  catdir : database of interest\n");
+    fprintf (stderr, "  Ntol : allow Ntol missing detections\n");
     exit (2);
   }
 
-  imageFilename  = argv[1];
-  cpmFilename    = argv[2];
-  cptFilenameSrc = argv[3];
-  cptFilenameTgt = argv[4];
-
-  cpmFtable.header = &cpmHeaderTBL;
-  cptFtable.header = &cptHeaderTBL;
-
-  // XXX don't bother locking for now: this is totally manual..
+  catdir = argv[1];
+  Ntol = atoi(argv[2]);
 
   // load the image data
+  ALLOCATE(imageFilename, char, strlen(catdir) + 12);
+  sprintf (imageFilename, "%s/Images.dat", catdir);
   if ((image = LoadImages (imageFilename, &Nimage)) == NULL) return (FALSE);
   BuildChipMatch (image, Nimage);
 
-  // open cpm file
-  cpmFile = fopen(cpmFilename, "r");
-  myAssert(cpmFile, "failed to open cpm file");
+  // generate an index for imageIDs:
+  Nindex = 0;
+  for (i = 0; i < Nimage; i++) {
+    Nindex = MAX(image[i].imageID, Nindex);
+  }
+  ALLOCATE (imageIdx, off_t, (Nindex + 1));
+  memset (imageIdx, 0, (Nindex + 1)*sizeof(off_t));
 
-  // load the cpm header
-  if (!gfits_fread_header (cpmFile, &cpmHeaderPHU)) {
-    myAbort("failure to cpm header");
+  for (i = 0; i < Nimage; i++) {
+    index = image[i].imageID;
+    if (index == 0) {
+      fprintf (stderr, "?");
+      continue;
+    }
+    if (imageIdx[index]) {
+      fprintf (stderr, "!");
+      continue;
+    }
+    imageIdx[index] = i;
   }
 
-  // move to TBL header
-  Nbytes = cpmHeaderPHU.datasize + gfits_data_size (&cpmHeaderPHU);
-  fseeko (cpmFile, Nbytes, SEEK_SET);
+  // generate a list of the detection counts:
+  ALLOCATE (detCounts, int, Nimage);
+  memset (detCounts, 0, Nimage*sizeof(int));
 
-  // read cpm TBL header
-  if (!gfits_fread_header (cpmFile, &cpmHeaderTBL)) { 
-    myAbort("can't read header for cpm table");
+  // load the sky table for the existing database
+  insky = SkyTableLoadOptimal (catdir, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
+  myAssert(insky, "can't read SkyTable");
+  SkyTableSetFilenames (insky, catdir, "cpt");
+
+  // XXX apply this...generate the subset matching the user-selected region
+  SkyRegion UserPatch;
+  UserPatch.Rmin = 0.0;
+  UserPatch.Rmax = 360.0;
+  UserPatch.Dmin = -90.0;
+  UserPatch.Dmax = +90.0;
+  inlist = SkyListByPatch (insky, -1, &UserPatch);
+
+  // SkyListPopulatedRange (&Ns, &Ne, inlist, 0);
+  // depth = inlist[0].regions[Ns][0].depth;
+  
+  ALLOCATE(cpmFilename, char, strlen(catdir) + 64);
+
+  // loop over the populated input regions
+  Ncheck = 0;
+  for (i = 0; i < inlist[0].Nregions; i++) {
+    if (!inlist[0].regions[i][0].table) continue;
+
+    if (1) {
+      sprintf (cpmFilename, "%s/%s.cpm", catdir, inlist[0].regions[i][0].name);
+      cpmFtable.header = &cpmHeaderTBL;
+
+      // open cpm file
+      cpmFile = fopen(cpmFilename, "r");
+      if (!cpmFile) continue;
+      // myAssert(cpmFile, "failed to open cpm file");
+    
+      // fprintf (stderr, "input: %s\n", inlist[0].regions[i][0].name);
+
+      // load the cpm header
+      if (!gfits_fread_header (cpmFile, &cpmHeaderPHU)) {
+	myAbort("failure to cpm header");
+      }
+
+      // move to TBL header
+      Nbytes = cpmHeaderPHU.datasize + gfits_data_size (&cpmHeaderPHU);
+      fseeko (cpmFile, Nbytes, SEEK_SET);
+
+      // read cpm TBL header
+      if (!gfits_fread_header (cpmFile, &cpmHeaderTBL)) { 
+	myAbort("can't read header for cpm table");
+      }
+
+      // read Measure table data : format is irrelevant here */
+      if (!gfits_fread_ftable_data (cpmFile, &cpmFtable, TRUE)) { 
+	myAbort("can't read data for cpm table");
+      }
+
+      gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow);
+
+      measure = FtableToMeasure (&cpmFtable, &Nmeasure, &catformat);
+      myAssert(measure, "failed to convert ftable to measure data");
+    
+      Nvalid = (int)(cpmFtable.validsize / NbytesPerRow);
+      Nvalid = MIN(Nmeasure, Nvalid);
+
+      // examine all measurements and new objects as needed
+      for (j = 0; j < Nvalid; j++) {
+	index = measure[j].imageID;
+	if (!index) {
+	  fprintf (stderr, "?");
+	  continue;
+	}
+
+	imageSeq = imageIdx[index];
+	// XXX check the range?
+
+	detCounts[imageSeq] ++;
+      }
+      fclose(cpmFile);
+      gfits_free_header (&cpmHeaderPHU);
+      gfits_free_header (&cpmHeaderTBL);
+      free (measure);
+
+      Ncheck ++;
+      if (Ncheck % 1000 == 0) {
+	fprintf (stderr, "%s...", inlist[0].regions[i][0].name);
+      }
+    }
   }
-  // read Measure table data : format is irrelevant here */
-  if (!gfits_fread_ftable_data (cpmFile, &cpmFtable)) { 
-    myAbort("can't read data for cpm table");
+  fprintf (stderr, "\n");
+
+  Nbad = 0;
+  for (i = 0; i < Nimage; i++) {
+    if (detCounts[i] < image[i].nstar - Ntol) {
+      fprintf (stdout, "image %s : %d vs %d\n", image[i].name, detCounts[i], image[i].nstar);
+      Nbad ++;
+    }
   }
-
-  measure = FtableToMeasure (&cpmFtable, &Nmeasure, &catformat);
-  myAssert(measure, "failed to convert ftable to measure data");
-
-  NaveMax = 0;
-  NAVERAGE = 1000;
-  ALLOCATE (average, Average, NAVERAGE);
-  memset (average, 0, NAVERAGE*sizeof(Average));
-
-  ALLOCATE (found, int, NAVERAGE);
-  memset (found, 0, NAVERAGE*sizeof(int));
-
-  // examine all measurements and new objects as needed
-  for (i = 0; i < Nmeasure; i++) {
-    Nave = measure[i].averef;
-    if (found[Nave]) {
-      average[Nave].Nmeasure ++;
-      myAssert(average[Nave].objID == measure[i].objID, "objIDs do not match!");
-      myAssert(average[Nave].catID == measure[i].catID, "catIDs do not match!");
-      continue;
-    }
-
-    if (Nave >= NAVERAGE) {
-      Nold = NAVERAGE;
-      NAVERAGE = MAX(Nave + 1000, NAVERAGE + 1000);
-      REALLOCATE (average, Average, NAVERAGE);
-      memset (&average[Nold], 0, (NAVERAGE - Nold)*sizeof(Average));
-
-      REALLOCATE (found, int, NAVERAGE);
-      memset (&found[Nold], 0, (NAVERAGE - Nold)*sizeof(int));
-    }
-
-    NaveMax = MAX(Nave, NaveMax);
-
-    found[Nave] = TRUE;
-
-    // we are going to leave most of the elements of average unset: they are the result of 
-    // the relastro analysis for this object and can be recreated with a call to relastro
-
-    // fields we have to set:
-
-    // need to find image so we can use ccd coordinates to determine RA & DEC
-    thisImage = MatchImage (image, Nimage, measure[i].t, measure[i].photcode, measure[i].imageID);
-    XY_to_RD (&average[Nave].R, &average[Nave].D, measure[i].Xccd, measure[i].Yccd, &thisImage[0].coords);
-
-    average[Nave].Nmeasure = 1;
-    average[Nave].Nmissing = 0;
-
-    // assume the resulting table set is unsorted
-    average[Nave].measureOffset = -1;
-    average[Nave].missingOffset = -1;
-    average[Nave].extendOffset = -1;
-
-    average[Nave].objID = measure[i].objID;
-    average[Nave].catID = measure[i].catID;
-    average[Nave].extID = CreatePSPSObjectID(average[Nave].R, average[Nave].D);
+  if (!Nbad) {
+    fprintf (stderr, "no bad images found\n");
   }
-  Naverage = NaveMax + 1;
-
-  // have we created all objects in the range 0 - Naverage?
-  for (i = 0; i < Naverage; i++) {
-    myAssert(found[i], "failed to find one");
-  }
-
-  // open source cpt file
-  cptFileSrc = fopen(cptFilenameSrc, "r");
-  myAssert(cptFileSrc, "failed to open cpt file");
-
-  // load the cpt header (use for CATID, RA,DEC range, filenames)
-  if (!gfits_fread_header (cptFileSrc, &cptHeaderPHU)) {
-    myAbort("failure to cpt header");
-  }
-
-  // update the output header
-  gfits_modify (&cptHeaderPHU, "NSTARS",     "%d",      1,  Naverage);
-  gfits_modify (&cptHeaderPHU, "NMEAS",      OFF_T_FMT, 1,  Nmeasure);
-  gfits_modify (&cptHeaderPHU, "NMISS",      "%d",      1,  0);
-  gfits_modify_alt (&cptHeaderPHU, "SORTED", "%t",      1,  FALSE);
-
-  /* convert internal to external format */
-  if (!AverageToFtable (&cptFtable, average, Naverage, catformat, NULL)) {
-    myAbort("trouble converting format");
-  }
-
-  // create and write the output file
-  cptFileTgt = fopen(cptFilenameTgt, "w");
-  myAssert(cptFileTgt, "failed to open cpt file");
-    
-  // write PHU header
-  if (!gfits_fwrite_header (cptFileTgt, &cptHeaderPHU)) {
-    myAbort("can't write primary header");
-  }
-
-  // write the PHU matrix; this is probably a NOP, do I have to keep it in?
-  gfits_create_matrix (&cptHeaderPHU, &matrix);
-  if (!gfits_fwrite_matrix  (cptFileTgt, &matrix)) {
-    myAbort("can't write primary matrix");
-  }
-  gfits_free_matrix (&matrix);
-
-  // write the table data
-  if (!gfits_fwrite_ftable_range (cptFileTgt, &cptFtable, 0, Naverage, 0, Naverage)) {
-    myAbort("can't write table data");
-  }
-  
-  fclose(cptFileTgt);
-  fclose(cptFileSrc);
-  fclose(cpmFile);
 
   exit (0);
Index: /branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvorepairCPT.c
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvorepairCPT.c	(revision 29714)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvorepairCPT.c	(revision 29714)
@@ -0,0 +1,190 @@
+# include "dvomerge.h"
+
+# define myAbort(MSG) { fprintf (stderr, "%s\n", MSG); abort(); }
+# define myAssert(LOGIC,MSG) { if (!LOGIC) { fprintf (stderr, "%s\n", MSG); abort(); } }
+
+// broken cpt file, valid cpm file: we can recover everything in the cpt file from the cpm file:
+// * load the full cpm file
+// * loop over detections
+// * if ave_ref is new: create a new object
+//   * determine RA & DEC from ext_id
+//   * obj_id, cat_id are defined in detection
+//   * 
+
+// XXX absorb this code into a unified dvorepair program
+
+int main (int argc, char **argv) {
+
+  off_t Nmeasure, Nimage;
+  int i, Nbytes, NaveMax, Naverage, NAVERAGE, Nave, Nold;
+  int *found;
+
+  Image *image, *thisImage;
+  Average *average;
+  Measure *measure;
+
+  Matrix matrix;
+
+  char *cpmFilename, *cptFilenameSrc, *cptFilenameTgt, *imageFilename;
+
+  Header cptHeaderPHU, cptHeaderTBL;
+  Header cpmHeaderPHU, cpmHeaderTBL;
+  FTable cpmFtable, cptFtable;
+
+  FILE *cptFileSrc = NULL;
+  FILE *cptFileTgt = NULL;
+  FILE *cpmFile = NULL;
+
+  char catformat;
+
+  if (argc != 5) {
+    fprintf (stderr, "USAGE: dvorepair (images) (cpm) (cptInput) (cptOutput)\n");
+    exit (2);
+  }
+
+  imageFilename  = argv[1];
+  cpmFilename    = argv[2];
+  cptFilenameSrc = argv[3];
+  cptFilenameTgt = argv[4];
+
+  cpmFtable.header = &cpmHeaderTBL;
+  cptFtable.header = &cptHeaderTBL;
+
+  // XXX don't bother locking for now: this is totally manual..
+
+  // load the image data
+  if ((image = LoadImages (imageFilename, &Nimage)) == NULL) return (FALSE);
+  BuildChipMatch (image, Nimage);
+
+  // open cpm file
+  cpmFile = fopen(cpmFilename, "r");
+  myAssert(cpmFile, "failed to open cpm file");
+
+  // load the cpm header
+  if (!gfits_fread_header (cpmFile, &cpmHeaderPHU)) {
+    myAbort("failure to cpm header");
+  }
+
+  // move to TBL header
+  Nbytes = cpmHeaderPHU.datasize + gfits_data_size (&cpmHeaderPHU);
+  fseeko (cpmFile, Nbytes, SEEK_SET);
+
+  // read cpm TBL header
+  if (!gfits_fread_header (cpmFile, &cpmHeaderTBL)) { 
+    myAbort("can't read header for cpm table");
+  }
+  // read Measure table data : format is irrelevant here */
+  if (!gfits_fread_ftable_data (cpmFile, &cpmFtable, FALSE)) { 
+    myAbort("can't read data for cpm table");
+  }
+
+  measure = FtableToMeasure (&cpmFtable, &Nmeasure, &catformat);
+  myAssert(measure, "failed to convert ftable to measure data");
+
+  NaveMax = 0;
+  NAVERAGE = 1000;
+  ALLOCATE (average, Average, NAVERAGE);
+  memset (average, 0, NAVERAGE*sizeof(Average));
+
+  ALLOCATE (found, int, NAVERAGE);
+  memset (found, 0, NAVERAGE*sizeof(int));
+
+  // examine all measurements and new objects as needed
+  for (i = 0; i < Nmeasure; i++) {
+    Nave = measure[i].averef;
+    if (found[Nave]) {
+      average[Nave].Nmeasure ++;
+      myAssert(average[Nave].objID == measure[i].objID, "objIDs do not match!");
+      myAssert(average[Nave].catID == measure[i].catID, "catIDs do not match!");
+      continue;
+    }
+
+    if (Nave >= NAVERAGE) {
+      Nold = NAVERAGE;
+      NAVERAGE = MAX(Nave + 1000, NAVERAGE + 1000);
+      REALLOCATE (average, Average, NAVERAGE);
+      memset (&average[Nold], 0, (NAVERAGE - Nold)*sizeof(Average));
+
+      REALLOCATE (found, int, NAVERAGE);
+      memset (&found[Nold], 0, (NAVERAGE - Nold)*sizeof(int));
+    }
+
+    NaveMax = MAX(Nave, NaveMax);
+
+    found[Nave] = TRUE;
+
+    // we are going to leave most of the elements of average unset: they are the result of 
+    // the relastro analysis for this object and can be recreated with a call to relastro
+
+    // fields we have to set:
+
+    // need to find image so we can use ccd coordinates to determine RA & DEC
+    thisImage = MatchImage (image, Nimage, measure[i].t, measure[i].photcode, measure[i].imageID);
+    XY_to_RD (&average[Nave].R, &average[Nave].D, measure[i].Xccd, measure[i].Yccd, &thisImage[0].coords);
+
+    average[Nave].Nmeasure = 1;
+    average[Nave].Nmissing = 0;
+
+    // assume the resulting table set is unsorted
+    average[Nave].measureOffset = -1;
+    average[Nave].missingOffset = -1;
+    average[Nave].extendOffset = -1;
+
+    average[Nave].objID = measure[i].objID;
+    average[Nave].catID = measure[i].catID;
+    average[Nave].extID = CreatePSPSObjectID(average[Nave].R, average[Nave].D);
+  }
+  Naverage = NaveMax + 1;
+
+  // have we created all objects in the range 0 - Naverage?
+  for (i = 0; i < Naverage; i++) {
+    myAssert(found[i], "failed to find one");
+  }
+
+  // open source cpt file
+  cptFileSrc = fopen(cptFilenameSrc, "r");
+  myAssert(cptFileSrc, "failed to open cpt file");
+
+  // load the cpt header (use for CATID, RA,DEC range, filenames)
+  if (!gfits_fread_header (cptFileSrc, &cptHeaderPHU)) {
+    myAbort("failure to cpt header");
+  }
+
+  // update the output header
+  gfits_modify (&cptHeaderPHU, "NSTARS",     "%d",      1,  Naverage);
+  gfits_modify (&cptHeaderPHU, "NMEAS",      OFF_T_FMT, 1,  Nmeasure);
+  gfits_modify (&cptHeaderPHU, "NMISS",      "%d",      1,  0);
+  gfits_modify_alt (&cptHeaderPHU, "SORTED", "%t",      1,  FALSE);
+
+  /* convert internal to external format */
+  if (!AverageToFtable (&cptFtable, average, Naverage, catformat, NULL)) {
+    myAbort("trouble converting format");
+  }
+
+  // create and write the output file
+  cptFileTgt = fopen(cptFilenameTgt, "w");
+  myAssert(cptFileTgt, "failed to open cpt file");
+    
+  // write PHU header
+  if (!gfits_fwrite_header (cptFileTgt, &cptHeaderPHU)) {
+    myAbort("can't write primary header");
+  }
+
+  // write the PHU matrix; this is probably a NOP, do I have to keep it in?
+  gfits_create_matrix (&cptHeaderPHU, &matrix);
+  if (!gfits_fwrite_matrix  (cptFileTgt, &matrix)) {
+    myAbort("can't write primary matrix");
+  }
+  gfits_free_matrix (&matrix);
+
+  // write the table data
+  if (!gfits_fwrite_ftable_range (cptFileTgt, &cptFtable, 0, Naverage, 0, Naverage)) {
+    myAbort("can't write table data");
+  }
+  
+  fclose(cptFileTgt);
+  fclose(cptFileSrc);
+  fclose(cpmFile);
+
+  exit (0);
+}
Index: /branches/eam_branches/ipp-20101103/Ohana/src/imregister/imphot/rfits.c
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/imregister/imphot/rfits.c	(revision 29713)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/imregister/imphot/rfits.c	(revision 29714)
@@ -17,5 +17,5 @@
     return (FALSE);
   }
-  if (!gfits_fread_ftable_data (db[0].f, &db[0].ftable)) {
+  if (!gfits_fread_ftable_data (db[0].f, &db[0].ftable, FALSE)) {
     fprintf (stderr, "can't read table data");
     return (FALSE);
Index: /branches/eam_branches/ipp-20101103/Ohana/src/libdvo/src/dvo_catalog_mef.c
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/libdvo/src/dvo_catalog_mef.c	(revision 29713)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/libdvo/src/dvo_catalog_mef.c	(revision 29714)
@@ -66,5 +66,5 @@
   /* read Average table data (or skip) */
   if (catalog[0].catflags & LOAD_AVES) {
-    if (!gfits_fread_ftable_data (f, &ftable)) {
+    if (!gfits_fread_ftable_data (f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table average data");
       return (FALSE);
@@ -94,5 +94,5 @@
   /* read Measure table data */
   if (catalog[0].catflags & LOAD_MEAS) {
-    if (!gfits_fread_ftable_data (f, &ftable)) {
+    if (!gfits_fread_ftable_data (f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table measure data");
       return (FALSE);
@@ -119,5 +119,5 @@
   /* read Missing table data */
   if (catalog[0].catflags & LOAD_MISS) {
-    if (!gfits_fread_ftable_data (f, &ftable)) {
+    if (!gfits_fread_ftable_data (f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table missing data");
       return (FALSE);
@@ -149,5 +149,5 @@
   /* read secfilt table data */
   if (catalog[0].catflags & LOAD_SECF) {
-    if (!gfits_fread_ftable_data (f, &ftable)) {
+    if (!gfits_fread_ftable_data (f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table secfilt data");
       return (FALSE);
Index: /branches/eam_branches/ipp-20101103/Ohana/src/libdvo/src/dvo_catalog_split.c
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/libdvo/src/dvo_catalog_split.c	(revision 29713)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/libdvo/src/dvo_catalog_split.c	(revision 29714)
@@ -218,5 +218,5 @@
     }
     /* read Average table data : format is irrelevant here */
-    if (!gfits_fread_ftable_data (catalog[0].f, &ftable)) { 
+    if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { 
       if (VERBOSE) fprintf (stderr, "can't read table average data");
       return (FALSE);
@@ -249,5 +249,5 @@
     // XXX this allows an empty Measure catalog with non-empty Average catalog : is that OK?
     /* read Measure table data */
-    if (!gfits_fread_ftable_data (catalog[0].measure_catalog[0].f, &ftable)) {
+    if (!gfits_fread_ftable_data (catalog[0].measure_catalog[0].f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table measure data\n");
       return (FALSE);
@@ -279,5 +279,5 @@
   if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MISS)) {
     /* read Missing table data */
-    if (!gfits_fread_ftable_data (catalog[0].missing_catalog[0].f, &ftable)) {
+    if (!gfits_fread_ftable_data (catalog[0].missing_catalog[0].f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table missing data\n");
       return (FALSE);
@@ -313,5 +313,5 @@
   if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_SECF)) {
     /* read secfilt table data */
-    if (!gfits_fread_ftable_data (catalog[0].secfilt_catalog[0].f, &ftable)) {
+    if (!gfits_fread_ftable_data (catalog[0].secfilt_catalog[0].f, &ftable, FALSE)) {
       if (VERBOSE) fprintf (stderr, "can't read table secfilt data\n");
       return (FALSE);
Index: /branches/eam_branches/ipp-20101103/Ohana/src/libdvo/src/fits_db.c
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/libdvo/src/fits_db.c	(revision 29713)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/libdvo/src/fits_db.c	(revision 29714)
@@ -83,5 +83,5 @@
     return (FALSE);
   }
-  if (!gfits_fread_ftable_data (db[0].f, &db[0].ftable)) {
+  if (!gfits_fread_ftable_data (db[0].f, &db[0].ftable, FALSE)) {
     fprintf (stderr, "can't read table data");
     gfits_db_free (db);
Index: /branches/eam_branches/ipp-20101103/Ohana/src/libfits/include/gfitsio.h
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/libfits/include/gfitsio.h	(revision 29713)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/libfits/include/gfitsio.h	(revision 29714)
@@ -64,4 +64,5 @@
   char                   *buffer;
   off_t                   datasize;
+  off_t                   validsize;
 } FTable;
 
@@ -172,5 +173,5 @@
 int     gfits_define_table_column      PROTO((Header *header, char *format, char *label, char *comment, char *unit));
 int   	gfits_fread_ftable             PROTO((FILE *f, FTable *ftable, char *extname)); 
-int   	gfits_fread_ftable_data        PROTO((FILE *f, FTable *ftable));
+int   	gfits_fread_ftable_data        PROTO((FILE *f, FTable *ftable, int padIfShort));
 int   	gfits_fread_ftable_range       PROTO((FILE *f, FTable *ftable, off_t start, off_t Nrows));
 int   	gfits_fread_vtable             PROTO((FILE *f, VTable *vtable, char *extname, off_t Nrow, off_t *row));
Index: /branches/eam_branches/ipp-20101103/Ohana/src/libfits/table/F_read_T.c
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/libfits/table/F_read_T.c	(revision 29713)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/libfits/table/F_read_T.c	(revision 29714)
@@ -36,5 +36,5 @@
     gfits_scan (header, "EXTNAME", "%s", 1, tname);
     if (!strcmp (tname, extname)) {
-      if (gfits_fread_ftable_data (f, table)) return (TRUE);
+      if (gfits_fread_ftable_data (f, table, FALSE)) return (TRUE);
       gfits_free_header (header);
       return (FALSE);
@@ -50,5 +50,5 @@
 
 /*********************** fits read ftable data ***********************************/
-int gfits_fread_ftable_data (FILE *f, FTable *table) {
+int gfits_fread_ftable_data (FILE *f, FTable *table, int padIfShort) {
 
   off_t Nbytes, Nread;
@@ -65,9 +65,18 @@
     if (Nread < gfits_data_min_size (table[0].header)) {
       fprintf (stderr, "error: fits read error in %s, read "OFF_T_FMT", need "OFF_T_FMT"\n", __func__,  Nread,  gfits_data_min_size (table[0].header));
-      gfits_free_table  (table);
-      return (FALSE);
+      if (padIfShort) {
+	memset (&table[0].buffer[Nread], 0, Nbytes - Nread);
+	fprintf (stderr, "warning: file missing data, padding with zeros: USE AT YOUR OWN RISK!\n");
+	table[0].validsize = Nread;
+	table[0].datasize = Nbytes;
+	return (TRUE);
+      } else {
+	gfits_free_table  (table);
+	return (FALSE);
+      }
     }
     fprintf (stderr, "warning: file missing pad\n");
   }
+  table[0].validsize = Nread;
   table[0].datasize = Nbytes;
   return (TRUE);
Index: /branches/eam_branches/ipp-20101103/Ohana/src/opihi/cmd.data/rd.c
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/opihi/cmd.data/rd.c	(revision 29713)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/opihi/cmd.data/rd.c	(revision 29714)
@@ -184,5 +184,5 @@
     ftable.header[0].buffer = NULL;
     gfits_copy_header (&buf[0].header, ftable.header);
-    status = gfits_fread_ftable_data (f, &ftable);  // this just reads the bytes (not even a SWAP)
+    status = gfits_fread_ftable_data (f, &ftable, FALSE);  // this just reads the bytes (not even a SWAP)
     status = gfits_uncompress_image (&buf[0].header, &buf[0].matrix, &ftable);
     // uncompressing the image leaves the format as an extension
Index: /branches/eam_branches/ipp-20101103/Ohana/src/opihi/cmd.data/read_vectors.c
===================================================================
--- /branches/eam_branches/ipp-20101103/Ohana/src/opihi/cmd.data/read_vectors.c	(revision 29713)
+++ /branches/eam_branches/ipp-20101103/Ohana/src/opihi/cmd.data/read_vectors.c	(revision 29714)
@@ -160,5 +160,5 @@
 
   off_t Nbytes;
-  int i, j, k, N, Nextend, Ny, Binary, vecType;
+  int i, j, k, N, Nextend, Ny, Binary, vecType, padIfShort;
   char type[16], ID[80], *CCDKeyword;
   FTable table;
@@ -174,4 +174,10 @@
     CCDKeyword = strcreate (argv[N]);
     remove_argument (N, &argc, argv);
+  }
+
+  padIfShort = FALSE;
+  if ((N = get_argument (argc, argv, "-pad-if-short"))) {
+    remove_argument (N, &argc, argv);
+    padIfShort = TRUE;
   }
 
@@ -205,5 +211,5 @@
     }
     if (!gfits_load_header (f, &header)) ESCAPE ("error reading header for extension");
-    if (!gfits_fread_ftable_data (f, &table)) ESCAPE ("error reading table for extension");
+    if (!gfits_fread_ftable_data (f, &table, padIfShort)) ESCAPE ("error reading table for extension");
 
   } else {
@@ -236,5 +242,5 @@
 	continue;
       }
-      if (!gfits_fread_ftable_data (f, &table)) ESCAPE ("error reading table for extension");
+      if (!gfits_fread_ftable_data (f, &table, padIfShort)) ESCAPE ("error reading table for extension");
       break;
     }