Changeset 35263 for trunk/Ohana/src/opihi

Timestamp:

Mar 7, 2013, 9:51:07 AM (13 years ago)

Author:

eugene

Message:

merge changes from eam_branches/ipp-20130306: addstar now can block against duplicate image IDs; mextract, mmatch now use a subset of image metadata instead of the whole table

Location:

trunk/Ohana

Files:

• . (modified) (1 prop)
• src/opihi (modified) (1 prop)
• src/opihi/cmd.astro (modified) (1 prop)
• src/opihi/cmd.data (modified) (1 prop)
• src/opihi/dvo/dvo_client.c (modified) (2 diffs)
• src/opihi/dvo/mextract.c (modified) (8 diffs)
• src/opihi/dvo/mmatch.c (modified) (7 diffs)
• src/opihi/dvo/mmextract.c (modified) (4 diffs)
• src/opihi/dvo/test/mmatch.sh (modified) (2 diffs)

trunk/Ohana/src/opihi/dvo/dvo_client.c

@@ -39 +39 @@
     remove_argument (N, argc, argv);
   }
+  if (!HOST_ID) {
+    fprintf (stderr, "ERROR: dvo_client requires a -hostID value\n");
+    exit (3);
+  }
 
   HOSTDIR = NULL;
@@ -45 +49 @@
     HOSTDIR = strcreate (argv[N]);;
     remove_argument (N, argc, argv);
+  }
+  if (!HOSTDIR) {
+    fprintf (stderr, "ERROR: dvo_client requires a -hostdir value\n");
+    exit (3);
   }
 

trunk/Ohana/src/opihi/dvo/mextract.c

@@ -1 +1 @@
 # include "dvoshell.h"
+
+int field_needs_images (dbField *field) {
+  // the image subset table requires imageID for all fields
+
+    if (!MEASURE_HAS_XCCD) {
+      // I'm keeping this code because it gives a way of handling dvo dbs that don't have
+      // measure.xccd if we need it
+      if (field->ID == MEAS_XCCD) return TRUE; // full astrometry per chip (120 bytes!)
+      if (field->ID == MEAS_YCCD) return TRUE; // full astrometry per chip (120 bytes!)
+    }
+    if (field->ID == MEAS_XMOSAIC)        return TRUE; // crval1,2 only
+    if (field->ID == MEAS_YMOSAIC)        return TRUE; // crval1,2 only
+    if (field->ID == MEAS_EXTERN_ID)      return TRUE; // externID
+    if (field->ID == MEAS_FLAT)           return TRUE; // Mcal
+    if (field->ID == MEAS_CENTER_OFFSET)  return TRUE; // 0.5*NX, 0.5*NY
+    if (field->ID == MEAS_EXPNAME_AS_INT) return TRUE; // expname (or as int)
+    if (field->ID == MEAS_AIRMASS)        return TRUE; // airmass
+    return FALSE;
+}
 
 int mextract (int argc, char **argv) {
@@ -5 +24 @@
   off_t i, j, k, m; // used for counter averages and measures
   int n, N, Npts, NPTS, last, next, state, Nfields, Nreturn, Ncstack, Nstack;
-  int Nsecfilt, VERBOSE, loadImages, mosaicMode;
+  int Nsecfilt, VERBOSE, loadImages;
   char **cstack, name[1024];
   dbValue *values;
@@ -33 +52 @@
     remove_argument (N, &argc, argv);
     VERBOSE = TRUE;
+  }
+
+  char *imageMetadataFile = FALSE;
+  if ((N = get_argument (argc, argv, "-image-metadata"))) {
+    remove_argument (N, &argc, argv);
+    imageMetadataFile = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
   }
 
@@ -66 +92 @@
 
   // init locally static variables (time refs)
-  dbExtractMeasuresInit();
+  // HOST_ID tells library if operation is on remote client or not
+  dbExtractMeasuresInit(HOST_ID);
 
   // command-line is of the form: mextract field,field, field [where (field op value)...]
@@ -106 +133 @@
   if ((skylist = SelectRegions (selection)) == NULL) goto escape;
 
+  // load image data if needed (for fields listed below)
+  loadImages = FALSE;
+  for (i = 0; !loadImages && (i < Nfields); i++) {
+    loadImages = field_needs_images (&fields[i]);
+  }
+    
   // this does all the work of re-packaging the command, calling it on the remote machines, then loading in the results
   if (PARALLEL && !HOST_ID) {
-    int status = HostTableParallelOps (argc, argv, RESULT_FILE, TRUE, 0, VERBOSE);
+
+    // Image Metadata for remote queries:
+    // 1) figure out if we need any image metadata
+    // 2) load the images and generate a subset table with just the fields of interest
+    // 3) add the input subset filename to the dvo_client command
+
+    // allocate the temp array and copy all but (RA) (DEC)
+    int targc = 0;
+    char **targv = NULL;
+    ALLOCATE (targv, char *, argc);
+    for (i = 0; i < argc; i++) {
+      targv[targc] = strcreate (argv[i]);
+      targc ++;
+    }
+
+    if (loadImages) {
+      Image *image;
+      off_t Nimage;
+      if ((image = LoadImagesDVO (&Nimage)) == NULL) goto escape;
+
+      char *filename = abspath("image.metadata.fits", DVO_MAX_PATH);
+      ImageMetadataSave (filename, image, Nimage);
+
+      REALLOCATE (targv, char *, targc + 2);
+      targv[targc+0] = strcreate ("-image-metadata");
+      targv[targc+1] = strcreate (filename);
+      targc += 2;
+    }
+
+    // call the remote client
+    int status = HostTableParallelOps (targc, targv, RESULT_FILE, TRUE, 0, VERBOSE);
 
     dbFreeFields (fields, Nfields);
@@ -116 +179 @@
     dvo_catalog_free (&catalog);
 
+    // free up targv
+    for (i = 0; i < targc; i++) {
+      free (targv[i]);
+    }
+    free (targv);
+
     return status;
   }
 
-  // load image data if needed (for fields listed below)
-  loadImages = FALSE;
-  mosaicMode = FALSE;
-  for (i = 0; !loadImages && (i < Nfields); i++) {
-    if (!MEASURE_HAS_XCCD) {
-      // I'm keeping this code because it gives a way of handling dvo dbs that don't have
-      // measure.xccd if we need it
-      if (fields[i].ID == MEAS_XCCD) loadImages = TRUE;
-      if (fields[i].ID == MEAS_YCCD) loadImages = TRUE;
-    }
-    if (fields[i].ID == MEAS_XMOSAIC)       loadImages = mosaicMode = TRUE;
-    if (fields[i].ID == MEAS_YMOSAIC)       loadImages = mosaicMode = TRUE;
-    if (fields[i].ID == MEAS_EXTERN_ID)     loadImages = mosaicMode = TRUE;
-    if (fields[i].ID == MEAS_FLAT)          loadImages = mosaicMode = TRUE;
-    if (fields[i].ID == MEAS_CENTER_OFFSET) loadImages = mosaicMode = TRUE;
-  }
-  if (loadImages && !SetImageSelection (mosaicMode, selection)) goto escape;
+  if (loadImages) {
+    if (HOST_ID) {
+      if (!SetImageMetadataSelection (imageMetadataFile)) goto escape;
+    } else {
+      if (!SetImageSelection (TRUE, selection)) goto escape;
+    }
+  }
 
   /* create storage vector */
@@ -251 +310 @@
   free (stack);
   FreeImageSelection ();
+  FreeImageMetadataSelection ();
   SkyListFree (skylist);
   FreeSkyRegionSelection (selection);
@@ -264 +324 @@
   free (stack);
   FreeImageSelection ();
+  FreeImageMetadataSelection ();
   SkyListFree (skylist);
   FreeSkyRegionSelection (selection);

trunk/Ohana/src/opihi/dvo/mmatch.c

@@ -1 +1 @@
 # include "dvoshell.h"
+int field_needs_images (dbField *field);
 
 /* This function uses the 'find_match' algorithm to select the objects of interest.
@@ -48 +49 @@
   }
 
+  // load info about the images from a reduced-size file
+  char *imageMetadataFile = FALSE;
+  if ((N = get_argument (argc, argv, "-image-metadata"))) {
+    remove_argument (N, &argc, argv);
+    imageMetadataFile = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+
   int PARALLEL = FALSE;
   if ((N = get_argument (argc, argv, "-parallel"))) {
@@ -72 +81 @@
 
   // init locally static variables (time refs)
-  dbExtractMeasuresInit();
+  // HOST_ID tells library if operation is on remote client or not
+  dbExtractMeasuresInit(HOST_ID);
+
+  // parse the fields to be extracted and returned
+  int first = 3;
+  if (CoordsFile) {
+    first = 1;
+  }
+  fields = dbCmdlineFields (argc-first, &argv[first], DVO_TABLE_MEASURE, &last, &Nfields);
+  if (fields == NULL) goto help;
+  if ((Nfields == 0) || (last != argc - first)) {
+    dbFreeFields (fields, Nfields);
+    dvo_catalog_free (&catalog);
+    goto help;
+  }
+
+  // load image data if needed (for fields listed below)
+  int loadImages = FALSE;
+  for (i = 0; !loadImages && (i < Nfields); i++) {
+    loadImages = field_needs_images (&fields[i]);
+  }
 
   // this does all the work of re-packaging the command, calling it on the remote machines, then loading in the results
@@ -122 +151 @@
       targc += 2;
     }      
+
+    if (loadImages) {
+      Image *image;
+      off_t Nimage;
+      if ((image = LoadImagesDVO (&Nimage)) == NULL) goto escape;
+
+      char *filename = abspath("image.metadata.fits", DVO_MAX_PATH);
+      ImageMetadataSave (filename, image, Nimage);
+
+      REALLOCATE (targv, char *, targc + 2);
+      targv[targc+0] = strcreate ("-image-metadata");
+      targv[targc+1] = strcreate (filename);
+      targc += 2;
+    }
 
     // call the remote client
@@ -153 +196 @@
   remove_argument (1, &argc, argv);
 
-  // parse the fields to be extracted and returned
-  fields = dbCmdlineFields (argc, argv, DVO_TABLE_MEASURE, &last, &Nfields);
-  if (fields == NULL) goto help;
-  if ((Nfields == 0) || (last != argc)) {
-    dbFreeFields (fields, Nfields);
-    dvo_catalog_free (&catalog);
-    goto help;
-  }
-
-  // load image data if needed (for fields listed below)
-  int loadImages = FALSE;
-  int mosaicMode = FALSE;
-  for (i = 0; !loadImages && (i < Nfields); i++) {
-    if (!MEASURE_HAS_XCCD) {
-      // I'm keeping this code because it gives a way of handling dvo dbs that don't have
-      // measure.xccd if we need it
-      if (fields[i].ID == MEAS_XCCD) loadImages = TRUE;
-      if (fields[i].ID == MEAS_YCCD) loadImages = TRUE;
-    }
-    if (fields[i].ID == MEAS_XMOSAIC)       loadImages = mosaicMode = TRUE;
-    if (fields[i].ID == MEAS_YMOSAIC)       loadImages = mosaicMode = TRUE;
-    if (fields[i].ID == MEAS_EXTERN_ID)     loadImages = mosaicMode = TRUE;
-    if (fields[i].ID == MEAS_FLAT)          loadImages = mosaicMode = TRUE;
-    if (fields[i].ID == MEAS_CENTER_OFFSET) loadImages = mosaicMode = TRUE;
-  }
-
   // use the whole sky (since we select random points around the sky)
   SkyRegionSelection selection;
   selection.useDisplay = FALSE;
   selection.useSkyregion = FALSE;
-  if (loadImages && !SetImageSelection (mosaicMode, &selection)) goto escape;
+
+  if (loadImages) {
+    if (HOST_ID) {
+      if (!SetImageMetadataSelection (imageMetadataFile)) goto escape;
+    } else {
+      if (!SetImageSelection (TRUE, &selection)) goto escape;
+    }
+  }
 
   /* load regions which contain all supplied RA,DEC coordinates */
@@ -318 +342 @@
   if (invec) FreeVectorArray (invec, Ninvec);
   dbFreeFields (fields, Nfields);
+  FreeImageSelection ();
+  FreeImageMetadataSelection ();
   SkyListFree (skylist);
   return (TRUE);
@@ -326 +352 @@
   if (invec) FreeVectorArray (invec, Ninvec);
   dbFreeFields (fields, Nfields);
+  FreeImageSelection ();
+  FreeImageMetadataSelection ();
   SkyListFree (skylist);
   return (FALSE);

trunk/Ohana/src/opihi/dvo/mmextract.c

@@ -1 +1 @@
 # include "dvoshell.h"
+int field_needs_images (dbField *field);
 
 int mmextract (int argc, char **argv) {
@@ -7 +8 @@
   int Nfields, Nreturn, Nreturn_base, Ncstack1, Ncstack2, Nstack1, Nstack2;
   int Nwhere, Iwhere, Nmatch, Imatch, NTABLE, Nt1, Nt2, n1, n2;
-  int Nsecfilt, VERBOSE, loadImages, mosaicMode;
+  int Nsecfilt, VERBOSE, loadImages;
   char **cstack1, **cstack2, name1[1024], name2[1024];
   dbValue *values, **table1, **table2;
@@ -45 +46 @@
   
   // init locally static variables (time refs)
-  dbExtractMeasuresInit();
+  dbExtractMeasuresInit(HOST_ID);
 
   // parse skyregion options
@@ -137 +138 @@
   // load image data if needed (for fields listed below)
   loadImages = FALSE;
-  mosaicMode = FALSE;
   for (i = 0; !loadImages && (i < Nfields); i++) {
-    if (fields[i].ID == MEAS_XCCD) loadImages = TRUE;
-    if (fields[i].ID == MEAS_YCCD) loadImages = TRUE;
-    if (fields[i].ID == MEAS_XMOSAIC) loadImages = mosaicMode = TRUE;
-    if (fields[i].ID == MEAS_YMOSAIC) loadImages = mosaicMode = TRUE;
-  }
-  if (loadImages && !SetImageSelection (mosaicMode, selection)) goto escape;
+    loadImages = field_needs_images (&fields[i]);
+  }
+  if (loadImages && !SetImageSelection (TRUE, selection)) goto escape;
 
   /* create storage vector */

trunk/Ohana/src/opihi/dvo/test/mmatch.sh

@@ -10 +10 @@
   $Ro = 2.59
   $Do = 1.23
-  catdir /data/pikake.0/eugene/src/ipp-dev/Ohana/src/dvomerge/test/catdir.merge
+  catdir catdir.merge
   $Gname = g
   $Rname = r
@@ -21 +21 @@
   subset r_ave = $Rname  if (ra > $Ro - 0.1) && (ra > $Ro + 0.1) && (dec > $Do - 0.1) && (dec < $Do + 0.1)
   vectors 
-  mmatch -v -parallel R D 1.0 RA DEC MAG PHOTCODE -index index
+
+  # mmatch -v -parallel R D 1.0 RA DEC MAG PHOTCODE -index index
+  mmatch -v -parallel R D 1.0 RA DEC MAG PHOTCODE externID mean_airmass -index index
 
   reindex g_ave_match = g_ave using index