Changeset 33963 for trunk

trunk

Property svn:mergeinfo changed

/branches/eam_branches/ipp-20120405 (added)

merged: 33724,33734-33739,33788,33797-33799,33807,33819-33820,33823-33824,33827-33828,33834-33835,33849,33868-33876,33878,33880,33882,33885-33888,33891-33893,33895-33907,33911-33912,33936,33942-33954

trunk/Ohana/src/addstar/Makefile

-              r33719
+              r33963
 mkcmf       : $(BIN)/mkcmf.$(ARCH)
 all: addstar sedstar load2mass skycells mkcmf loadwise loadsupercos dumpskycells
 INSTALL = addstar sedstar load2mass skycells mkcmf loadwise loadsupercos dumpskycells
+all: addstar addstar_client sedstar load2mass skycells mkcmf loadwise loadsupercos dumpskycells
+INSTALL = addstar addstar_client sedstar load2mass skycells mkcmf loadwise loadsupercos dumpskycells
 # I need to fix the client/server version of addstar now that I have dropped Stars
 …
 $(SRC)/resort_threaded.$(ARCH).o \
 $(SRC)/resort_unthreaded.$(ARCH).o \
+$(SRC)/resort_unthreaded_catalogs.$(ARCH).o \
 $(SRC)/StarOps.$(ARCH).o \
 $(SRC)/ReadStarsFITS.$(ARCH).o \
 …
 $(SRC)/update_coords.$(ARCH).o \
 $(SRC)/psps_ids.$(ARCH).o
+# this is the client for parallel operations
+ADDSTAR_CLIENT = \
+$(SRC)/ConfigInit.$(ARCH).o \
+$(SRC)/SetSignals.$(ARCH).o \
+$(SRC)/Shutdown.$(ARCH).o \
+$(SRC)/SkyListForStars.$(ARCH).o \
+$(SRC)/SkyRegionUtils.$(ARCH).o \
+$(SRC)/build_links.$(ARCH).o \
+$(SRC)/addstar_client.$(ARCH).o \
+$(SRC)/args_parallel_client.$(ARCH).o \
+$(SRC)/resort_catalog.$(ARCH).o \
+$(SRC)/resort_threaded.$(ARCH).o \
+$(SRC)/resort_unthreaded.$(ARCH).o \
+$(SRC)/resort_unthreaded_catalogs.$(ARCH).o
 ADDSTARD = \
 …
 $(SRC)/psps_ids.$(ARCH).o
+# this is the client for server / client addstar operations
 ADDSTARC = \
 $(SRC)/addstarc.$(ARCH).o \
 …
 LOAD-WISE = \
 $(SRC)/loadwise.$(ARCH).o \
 $(SRC)/loadwise_prelim_rawdata.$(ARCH).o \
+$(SRC)/loadwise_rawdata.$(ARCH).o \
 $(SRC)/loadwise_ops.$(ARCH).o \
 $(SRC)/loadwise_star_full.$(ARCH).o \
 …
 $(SRC)/SetSignals.$(ARCH).o \
+$(ADDSTARC)   : $(INC)/addstar.h
+$(ADDSTARD)   : $(INC)/addstar.h
+$(ADDSTART)   : $(INC)/addstar.h
+$(ADDSTAR)    : $(INC)/addstar.h
+$(SKYCELLS)   : $(INC)/addstar.h
+$(DUMPSKYCELLS): $(INC)/addstar.h
+$(LOAD-2MASS) : $(INC)/addstar.h $(INC)/2mass.h
+$(LOAD-WISE)  : $(INC)/addstar.h $(INC)/WISE.h
+$(MKCMF)      : $(INC)/mkcmf.h
+$(BIN)/addstar.$(ARCH)     : $(ADDSTAR)
+$(BIN)/addstard.$(ARCH)    : $(ADDSTARD)
+$(BIN)/addstart.$(ARCH)    : $(ADDSTART)
+$(BIN)/addstarc.$(ARCH)    : $(ADDSTARC)
+$(BIN)/mkacc-2mass.$(ARCH) : $(MKACC-2MASS)
+$(BIN)/sedstar.$(ARCH)     : $(SEDSTAR)
+$(BIN)/load2mass.$(ARCH)   : $(LOAD-2MASS)
+$(BIN)/loadwise.$(ARCH)    : $(LOAD-WISE)
+$(BIN)/loadsupercos.$(ARCH): $(LOAD-SUPERCOS)
+$(BIN)/skycells.$(ARCH)    : $(SKYCELLS)
+$(BIN)/dumpskycells.$(ARCH) : $(DUMPSKYCELLS)
+$(BIN)/mkcmf.$(ARCH)       : $(MKCMF)
+$(ADDSTARC)       : $(INC)/addstar.h
+$(ADDSTARD)       : $(INC)/addstar.h
+$(ADDSTART)       : $(INC)/addstar.h
+$(ADDSTAR)        : $(INC)/addstar.h
+$(ADDSTAR_CLIENT) : $(INC)/addstar.h
+$(SKYCELLS)       : $(INC)/addstar.h
+$(DUMPSKYCELLS)   : $(INC)/addstar.h
+$(LOAD-2MASS)     : $(INC)/addstar.h $(INC)/2mass.h
+$(LOAD-WISE)      : $(INC)/addstar.h $(INC)/WISE.h
+$(MKCMF)          : $(INC)/mkcmf.h
+$(BIN)/addstar.$(ARCH)        : $(ADDSTAR)
+$(BIN)/addstar_client.$(ARCH) : $(ADDSTAR_CLIENT)
+$(BIN)/addstard.$(ARCH)       : $(ADDSTARD)
+$(BIN)/addstart.$(ARCH)       : $(ADDSTART)
+$(BIN)/addstarc.$(ARCH)       : $(ADDSTARC)
+$(BIN)/mkacc-2mass.$(ARCH)    : $(MKACC-2MASS)
+$(BIN)/sedstar.$(ARCH)        : $(SEDSTAR)
+$(BIN)/load2mass.$(ARCH)      : $(LOAD-2MASS)
+$(BIN)/loadwise.$(ARCH)       : $(LOAD-WISE)
+$(BIN)/loadsupercos.$(ARCH)   : $(LOAD-SUPERCOS)
+$(BIN)/skycells.$(ARCH)       : $(SKYCELLS)
+$(BIN)/dumpskycells.$(ARCH)   : $(DUMPSKYCELLS)
+$(BIN)/mkcmf.$(ARCH)          : $(MKCMF)
 # I need to fix the client/server version of addstar now that I have dropped Stars

trunk/Ohana/src/addstar/include/WISE.h

-              r33653
+              r33963
 short WISE_W1, WISE_W2, WISE_W3, WISE_W4;
+int ALLSKY;
 AddstarClientOptions args_loadwise (int argc, char **argv, AddstarClientOptions options);
 int loadwise_prelim_rawdata (SkyList *skytable, char *filename, AddstarClientOptions options);
+int loadwise_rawdata (SkyList *skytable, char *filename, AddstarClientOptions options, int ALLSKY);
 int getWISE_setup ();
 …
 int getWISE_sortStars (WISE_Stars *tstars, int Ntstars);
+int loadwise_star_full (Stars **star, char *line, int Nmax);
+int loadwise_star_allsky (Stars **star, char *line, int Nmax);
+int loadwise_star_prelim (Stars **star, char *line, int Nmax);
 char *nextWISEfield (char *line);

trunk/Ohana/src/addstar/include/addstar.h

-              r33653
+              r33963
 typedef struct sockaddr_in SockAddress;
 enum {M_IMAGE, M_REFLIST, M_REFCAT, M_FAKEIMAGE, M_RESORT};
+enum {ADDSTAR_MODE_NONE, ADDSTAR_MODE_IMAGE, ADDSTAR_MODE_REFLIST, ADDSTAR_MODE_REFCAT, ADDSTAR_MODE_FAKEIMAGE, ADDSTAR_MODE_RESORT};
 enum {NONE, SIMPLE_CMP, SIMPLE_CMF, SIMPLE_MEF, MOSAIC_CMP, MOSAIC_CMF, MOSAIC_MEF, MOSAIC_PHU, SDSS_OBJ};
 /* note: MEF implies CMF */
 /* globals which define database info / data sources (KEEP) */
 char   ImageCat[256];
+char   ImageCat[DVO_MAX_PATH];
 char   GSCFILE[256];
 char   CATDIR[256];
+char  *CATDIR;
 char   CATMODE[16];    /* raw, mef, split, mysql */
 char   CATFORMAT[16];  /* internal, elixir, loneos, panstarrs */
 …
 int     OLD_RESORT;
+int    PARALLEL;
+int    PARALLEL_MANUAL;
+int    PARALLEL_SERIAL;
+int    HOST_ID;
+char  *HOSTDIR;
 // carries the mosaic into gstars
 …
 /*** addstar prototypes ***/
+AddstarClientOptions ConfigInit   PROTO((int *argc, char **argv));
+AddstarClientOptions args         PROTO((int argc, char **argv, AddstarClientOptions options));
+AddstarClientOptions ConfigInit           PROTO((int *argc, char **argv));
+AddstarClientOptions args                 PROTO((int argc, char **argv, AddstarClientOptions options));
+AddstarClientOptions args_parallel_client PROTO((int argc, char **argv, AddstarClientOptions options));
 void       AddToCalibration       PROTO((Average *average, SecFilt *secfilt, Measure *measure, Measure *new, off_t *next, off_t Nstar));
 …
 int        replace_match          PROTO((Average *average, Measure *measure, Stars *star));
 int        resort_threaded        PROTO((AddstarClientOptions *options, SkyTable *sky));
+int        resort_unthreaded      PROTO((AddstarClientOptions *options, SkyTable *sky));
+int        resort_unthreaded      PROTO((AddstarClientOptions *options, SkyTable *sky, int hostID, char *hostpath));
+int        resort_unthreaded_catalogs PROTO((AddstarClientOptions *options, SkyList *skylist, int hostID, char *hostpath));
 void       resort_catalog         PROTO((Catalog *catalog));
 void       resort_catalog_old     PROTO((Catalog *catalog));

trunk/Ohana/src/addstar/src/ConfigInit.c

-              r33653
+              r33963
   ScanConfig (config, "TYCHO_DIR",              "%s",  0, TYCHO_DIR);
+  // force CATDIR to be absolute (so parallel mode will work)
+  char tmpcatdir[DVO_MAX_PATH];
+  GetConfig (config, "CATDIR" ,                 "%s",  0, tmpcatdir);
+  CATDIR = abspath (tmpcatdir, DVO_MAX_PATH);
   GetConfig (config, "GSCFILE",                 "%s",  0, GSCFILE);
-  GetConfig (config, "CATDIR",                  "%s",  0, CATDIR);
   GetConfig (config, "PHOTCODE_FILE",           "%s",  0, MasterPhotcodeFile);
   ScanConfig (config, "CATMODE",                "%s",  0, CATMODE);

trunk/Ohana/src/addstar/src/addstar.c

-              r29938
+              r33963
   SkyTableSetFilenames (sky, CATDIR, "cpt");
   if (options.mode == M_RESORT) {
+  if (options.mode == ADDSTAR_MODE_RESORT) {
     if (NTHREADS == 0) {
       resort_unthreaded (&options, sky);
+      resort_unthreaded (&options, sky, 0, NULL);
     } else {
       resort_threaded (&options, sky);
 …
   /*** load in the new data (images, stars) ***/
   switch (options.mode) {
     case M_IMAGE:
+    case ADDSTAR_MODE_IMAGE:
       stars = LoadStars (argv[1], &Nstars, &images, &Nimages, &options);
 …
       ImageOptions (&options, images, Nimages);
       break;
     case M_REFLIST:
+    case ADDSTAR_MODE_REFLIST:
       stars = grefstars (argv[1], options.photcode, &Nstars);
       skylist = SkyListForStars (sky, -1, stars, Nstars);
       break;
     case M_RESORT:
     case M_REFCAT:
+    case ADDSTAR_MODE_RESORT:
+    case ADDSTAR_MODE_REFCAT:
       skylist = SkyListByPatch (sky, -1, &UserPatch);
       break;
     case M_FAKEIMAGE:
+    case ADDSTAR_MODE_FAKEIMAGE:
       images = fakeimage (argv[1], &Nimages, options.photcode);
       ALLOCATE (skylist, SkyList, 1);
 …
     switch (options.mode) {
       case M_IMAGE:
+      case ADDSTAR_MODE_IMAGE:
         Nsubset = Nstars;
         if (options.closest) {
 …
+        }
         break;
       case M_REFCAT:
+      case ADDSTAR_MODE_REFCAT:
         stars = greference (argv[1], skylist[0].regions[i], options.photcode, &Nstars);
       case M_REFLIST:
+      case ADDSTAR_MODE_REFLIST:
         subset = find_subset (skylist[0].regions[i], stars, Nstars, &Nsubset);
         if (options.closest) {
 …
         if (Nsubset) free (subset);
         break;
       case M_RESORT:
+      case ADDSTAR_MODE_RESORT:
         if (options.nosort == 3) catalog.sorted = FALSE;
 …
     dvo_catalog_free (&catalog);
     if (options.mode == M_REFCAT) free (stars);
+    if (options.mode == ADDSTAR_MODE_REFCAT) free (stars);
+  }
 …
   /* add the new images and save */
   if (options.mode == M_IMAGE) {
+  if (options.mode == ADDSTAR_MODE_IMAGE) {
     dvo_image_addrows (&db, images, Nimages);
     SetProtect (TRUE);

trunk/Ohana/src/addstar/src/addstarc.c

-              r21153
+              r33963
   /* send new data to server */
   switch (options.mode) {
     case M_IMAGE:
+    case ADDSTAR_MODE_IMAGE:
       /* load data */
       stars = LoadStars (argv[1], &Nstars, &images, &Nimages, &options);
 …
       break;
     case M_REFLIST:
+    case ADDSTAR_MODE_REFLIST:
       /* load data */
       stars = grefstars (argv[1], options.photcode, &Nstars);
 …
       break;
     case M_REFCAT:
+    case ADDSTAR_MODE_REFCAT:
       /* send data to server */
       SendCommand (BindSocket, 5, "REFCT");

trunk/Ohana/src/addstar/src/addstart.c

-              r25757
+              r33963
     switch (dataset[0].options[0].mode) {
       case M_IMAGE:
+      case ADDSTAR_MODE_IMAGE:
         UpdateDatabase_Image (dataset[0].options, dataset[0].images, dataset[0].Nimages, dataset[0].mosaic, dataset[0].stars, dataset[0].Nstars);
         continue;
       case M_REFLIST:
+      case ADDSTAR_MODE_REFLIST:
         UpdateDatabase_Reflist (dataset[0].options, dataset[0].stars, dataset[0].Nstars);
         continue;
       case M_REFCAT:
+      case ADDSTAR_MODE_REFCAT:
         UpdateDatabase_Refcat (dataset[0].options, dataset[0].patch, dataset[0].refcat);
         continue;

trunk/Ohana/src/addstar/src/args.c

-              r31669
+              r33963
   /* basic mode: image, list, refcat */
   options.mode = M_IMAGE;
+  options.mode = ADDSTAR_MODE_IMAGE;
   if ((N = get_argument (argc, argv, "-ref"))) {
     options.mode = M_REFLIST;
+    options.mode = ADDSTAR_MODE_REFLIST;
     remove_argument (N, &argc, argv);
+  }
   if ((N = get_argument (argc, argv, "-cat"))) {
     options.mode = M_REFCAT;
+    options.mode = ADDSTAR_MODE_REFCAT;
     remove_argument (N, &argc, argv);
+  }
   if ((N = get_argument (argc, argv, "-resort"))) {
     options.mode = M_RESORT;
+    options.mode = ADDSTAR_MODE_RESORT;
     remove_argument (N, &argc, argv);
+  }
   if ((N = get_argument (argc, argv, "-fakeimage"))) {
     options.mode = M_FAKEIMAGE;
+    options.mode = ADDSTAR_MODE_FAKEIMAGE;
     remove_argument (N, &argc, argv);
     FAKE_RA = atof (argv[N]);
 …
     remove_argument (N, &argc, argv);
   } else {
       if (options.mode == M_IMAGE) goto allow;
       if (options.mode == M_FAKEIMAGE) goto allow;
       if (options.mode == M_REFLIST) goto allow;
       if (options.mode == M_REFCAT) {
+      if (options.mode == ADDSTAR_MODE_IMAGE) goto allow;
+      if (options.mode == ADDSTAR_MODE_FAKEIMAGE) goto allow;
+      if (options.mode == ADDSTAR_MODE_REFLIST) goto allow;
+      if (options.mode == ADDSTAR_MODE_REFCAT) {
           fprintf (stderr, "you have requested uploading from a catalog to the entire sky in one pass\n");
+      }
       if (options.mode == M_RESORT) {
+      if (options.mode == ADDSTAR_MODE_RESORT) {
           fprintf (stderr, "you have requested resorting the entire sky in one pass\n");
+      }
 …
+  }
+  if ((options.mode == M_RESORT) && (argc == 1)) return (options);
+  // XXX for the moment, make this selection manual.  it needs to be automatic
+  // based on the state of the SkyTable
+  PARALLEL = FALSE;
+  if ((N = get_argument (argc, argv, "-parallel"))) {
+    PARALLEL = TRUE;
+    remove_argument (N, &argc, argv);
+  }
+  // this is a test mode : rather than launching the remote jobs and waiting for completion,
+  // relphot will simply list the remote command and wait for the user to signal completion
+  PARALLEL_MANUAL = FALSE;
+  if ((N = get_argument (argc, argv, "-parallel-manual"))) {
+    PARALLEL = TRUE; // -parallel-manual implies -parallel
+    PARALLEL_MANUAL = TRUE;
+    remove_argument (N, &argc, argv);
+  }
+  // this is a test mode : rather than launching the relphot_client jobs remotely, they are
+  // run in serial via 'system'
+  PARALLEL_SERIAL = FALSE;
+  if ((N = get_argument (argc, argv, "-parallel-serial"))) {
+    if (PARALLEL_MANUAL) {
+      fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n");
+      exit (1);
+    }
+    PARALLEL = TRUE; // -parallel-serial implies -parallel
+    PARALLEL_SERIAL = TRUE;
+    remove_argument (N, &argc, argv);
+  }
+  if (PARALLEL) {
+    if (options.mode != ADDSTAR_MODE_RESORT) {
+      fprintf (stderr, "parallel mode is only valid for -resort mode\n");
+      exit (2);
+    }
+  }
+  if ((options.mode == ADDSTAR_MODE_RESORT) && (argc == 1)) return (options);
   if (argc == 2) return (options);
   if ((options.mode == M_REFLIST) && (options.photcode == 0)) {
+  if ((options.mode == ADDSTAR_MODE_REFLIST) && (options.photcode == 0)) {
     fprintf (stderr, "photcode must be specified for -ref\n");
     exit (2);

trunk/Ohana/src/addstar/src/args_client.c

-              r18374
+              r33963
   /* basic mode: image, list, refcat */
   options.mode = M_IMAGE;
+  options.mode = ADDSTAR_MODE_IMAGE;
   if ((N = get_argument (argc, argv, "-ref"))) {
     options.mode = M_REFLIST;
+    options.mode = ADDSTAR_MODE_REFLIST;
     remove_argument (N, &argc, argv);
+  }
   if ((N = get_argument (argc, argv, "-cat"))) {
     options.mode = M_REFCAT;
+    options.mode = ADDSTAR_MODE_REFCAT;
     remove_argument (N, &argc, argv);
+  }

trunk/Ohana/src/addstar/src/args_load2mass.c

r24977	r33963
18	18
19	19	/* basic mode: image, list, refcat */
20		options.mode = M_REFCAT;
	20	options.mode = ADDSTAR_MODE_REFCAT;
21	21
22	22	/* provide additional data */

trunk/Ohana/src/addstar/src/args_loadsupercos.c

r33653	r33963
18	18
19	19	/* basic mode: image, list, refcat */
20		options.mode = M_REFCAT;
	20	options.mode = ADDSTAR_MODE_REFCAT;
21	21
22	22	/* we do not allow a subset to be extracted -- all or nothing, babe */

trunk/Ohana/src/addstar/src/args_loadwise.c

-              r33653
+              r33963
 # include "addstar.h"
+# include "WISE.h"
 static void help (void);
 …
+  }
   // a global used by find_matches_refstars.c (value is 1 except for load2mass & loadwise)
+  // a global used by find_matches_refstars.c (value is 1 except for loadwise & loadwise)
   NREFSTAR_GROUP = 4;
 …
   /* basic mode: image, list, refcat */
   options.mode = M_REFCAT;
+  options.mode = ADDSTAR_MODE_REFCAT;
   /* we do not allow a subset to be extracted -- all or nothing, babe */
 …
+  }
+  /* other addstar options which cannot be used in load2mass */
+  /* load the prelim data dump */
+  ALLSKY = TRUE;
+  if ((N = get_argument (argc, argv, "-prelim"))) {
+    ALLSKY = FALSE;
+    remove_argument (N, &argc, argv);
+  }
+  /* other addstar options which cannot be used in loadwise */
   options.photcode = 0;
   options.timeref = 0;
 …
   if (argc < 2) {
     fprintf (stderr, "USAGE: load2mass [options] (wisefile) [..more files]\n");
+    fprintf (stderr, "USAGE: loadwise [options] (wisefile) [..more files]\n");
     exit (2);
+  }

trunk/Ohana/src/addstar/src/args_sedstar.c

r18374	r33963
18	18
19	19	/* basic mode: image, list, refcat */
20		options.mode = M_REFCAT;
	20	options.mode = ADDSTAR_MODE_REFCAT;
21	21
22	22	/* provide additional data */

trunk/Ohana/src/addstar/src/find_matches.c

-              r33653
+              r33963
       catalog[0].secfilt[Nave*Nsecfilt+j].Map         = NAN;
       catalog[0].secfilt[Nave*Nsecfilt+j].dM          = NAN;
+      catalog[0].secfilt[Nave*Nsecfilt+j].Mstdev      = NAN_S_SHORT;
       catalog[0].secfilt[Nave*Nsecfilt+j].Xm          = NAN_S_SHORT;
       catalog[0].secfilt[Nave*Nsecfilt+j].M_20        = NAN_S_SHORT;
 …
       catalog[0].secfilt[Nave*Nsecfilt+j].ubercalDist = 1000;
       catalog[0].secfilt[Nave*Nsecfilt+j].flags       = 0;
-      catalog[0].secfilt[Nave*Nsecfilt+j].dummy[0]    = 0;
-      catalog[0].secfilt[Nave*Nsecfilt+j].dummy[1]    = 0;
+    }

trunk/Ohana/src/addstar/src/find_matches_closest.c

-              r33653
+              r33963
       catalog[0].secfilt[Nave*Nsecfilt+j].Map         = NAN;
       catalog[0].secfilt[Nave*Nsecfilt+j].dM          = NAN;
+      catalog[0].secfilt[Nave*Nsecfilt+j].Mstdev      = NAN_S_SHORT;
       catalog[0].secfilt[Nave*Nsecfilt+j].Xm          = NAN_S_SHORT;
       catalog[0].secfilt[Nave*Nsecfilt+j].M_20        = NAN_S_SHORT;
 …
       catalog[0].secfilt[Nave*Nsecfilt+j].ubercalDist = 1000;
       catalog[0].secfilt[Nave*Nsecfilt+j].flags       = 0;
-      catalog[0].secfilt[Nave*Nsecfilt+j].dummy[0]    = 0;
-      catalog[0].secfilt[Nave*Nsecfilt+j].dummy[1]    = 0;
+    }

trunk/Ohana/src/addstar/src/loadwise.c

r33653	r33963
40	40	for (i = 1; i < argc; i++) {
41	41	fprintf (stderr, "loading %s\n", argv[i]);
42		loadwise_~~prelim_rawdata (skylist, argv[i], options~~);
	42	loadwise_rawdata (skylist, argv[i], options, ALLSKY);
43	43	}
44	44	exit (0);

trunk/Ohana/src/addstar/src/loadwise_star_full.c

-              r33653
+              r33963
 int setWISE_ph_qual (Stars *star, char qual);
+int setWISE_var_flag (Stars *star, char qual);
+int setWISE_ext_flag (Stars **star, char *ptr);
+int setWISE_var_flag_allsky (Stars *star, char qual);
+int setWISE_ext_flag_allsky (Stars **star, char ptr);
+int setWISE_var_flag_prelim (Stars *star, char qual);
+int setWISE_ext_flag_prelim (Stars **star, char ptr);
 int setWISE_cc_flag (Stars *star, char qual);
 int setWISE_sat_flag (Stars *star, char *ptr);
 int setWISE_blend_flag (Stars **star, char *ptr);
+// there are slight format differences between the prelim data dump and the allsky data dump:
+// * after ph_qual & det_bit : new field moon_lev
+// * for each filter, after w?sigp2 : new fields w?dmag, w?ndf, w?mlq, w?mjdmin, w?mjdmax, w?mjdmean
+// * after w4mdjmean : new fields rho12, rho23, rho34, q12, q23, q34
+// * after k_msig_2mass : new fields best_use_cntr, ngrp
 // fill in the data for a WISE quad star.  takes a pointer to the start of the line the
 // RA and DEC have already been set
 int loadwise_star_full (Stars **star, char *line, int Nmax) {
+int loadwise_star_allsky (Stars **star, char *line, int Nmax) {
   int i;
 …
   // W1
   for (i = 0; i < 4; i++) {
+      star[0][0].measure.M  = strtod (ptr, NULL); // w?mpro
+    char *endpoint;
+      star[i][0].measure.M  = strtod (ptr, &endpoint); // w?mpro
+      if (endpoint == ptr) {
+        star[i][0].measure.M  = NAN;
+      }
       ptr = nextWISEfield (ptr);
+      star[0][0].measure.dM = strtod (ptr, NULL); // w?sigmpro
+      star[i][0].measure.dM = strtod (ptr, &endpoint); // w?sigmpro
+      if (endpoint == ptr) {
+        star[i][0].measure.dM  = NAN;
+      }
       ptr = skipNbounds (ptr, '|', 2, Nmax); // skip: w?sigmpro, w?snr
       star[0][0].measure.psfChisq = strtod (ptr, NULL); // w?rchi2
+      star[i][0].measure.psfChisq = strtod (ptr, NULL); // w?rchi2
       ptr = nextWISEfield (ptr); // skip : w1rchi2
 …
   // set ext flags for all 4 measures
   setWISE_ext_flag (star, ptr); // ext_flg
+  setWISE_ext_flag_allsky (star, ptr[0]); // ext_flg
   ptr = nextWISEfield (ptr); // skip ext_flags
   for (i = 0; i < 4; i++) {
+      setWISE_var_flag (star[i], ptr[i]); // var_flg
+      setWISE_var_flag_allsky (star[i], ptr[i]); // var_flg
+  }
+  ptr = nextWISEfield (ptr); // skip var_flags
+  for (i = 0; i < 4; i++) {
+      setWISE_ph_qual (star[i], ptr[i]); // ph_qual
+  }
+  ptr = skipNbounds (ptr, '|', 159, Nmax); // skip: det_bit, moon_lev, w?nm, w?m, w?cov, etc, etc.
+  for (i = 0; i < 4; i++) {
+    ptr = skipNbounds (ptr, '|', 8, Nmax); // skip: det_bit, moon_lev, w?nm, w?m, w?cov, etc, etc.
+    // double mjdmin = strtod (ptr, NULL); // mjd min
+    // ptr = nextWISEfield (ptr); // skip
+    // double mjdmax = strtod (ptr, NULL); // mjd max
+    // ptr = nextWISEfield (ptr); // skip mjd max
+    double mjdmean = strtod (ptr, NULL); // mjd mean
+    ptr = nextWISEfield (ptr); // skip mjd mean
+    // fprintf (stderr, "w%d mjd: %f\n", i, mjdmean);
+    // the release is based on data taken in the period 14 January 2010 to 29 April 2010
+    if (mjdmean == 0.0) {
+      star[i][0].measure.t = 0;
+    } else {
+      star[i][0].measure.t = ohana_mjd_to_sec (mjdmean);
+    }
+  }
+  star[0][0].measure.photcode  = WISE_W1;
+  star[0][0].measure.detID   = 0;
+  star[0][0].measure.imageID = 0;
+  star[1][0].measure.photcode  = WISE_W2;
+  star[1][0].measure.detID   = 0;
+  star[1][0].measure.imageID = 0;
+  star[2][0].measure.photcode  = WISE_W3;
+  star[2][0].measure.detID   = 0;
+  star[2][0].measure.imageID = 0;
+  star[3][0].measure.photcode  = WISE_W4;
+  star[3][0].measure.detID   = 0;
+  star[3][0].measure.imageID = 0;
+  return TRUE;
+}
+// fill in the data for a WISE quad star.  takes a pointer to the start of the line the
+// RA and DEC have already been set
+int loadwise_star_prelim (Stars **star, char *line, int Nmax) {
+  int i;
+  char *ptr;
+  if (line == NULL) Shutdown ("format error in WISE");
+  ptr = line;
+  // I can assign dRA and dDEC to dX and dY if I can consistently set posangle and pltscale
+  star[0][0].measure.posangle = 0.0;
+  star[0][0].measure.pltscale = 1.0;
+  ptr = skipNbounds (ptr, '|', 3, Nmax); // skip: desig, ra, dec,
+  star[0][0].measure.dXccd = ToShortPixels(strtod (ptr, NULL)); // sig_ra
+  ptr = nextWISEfield (ptr);
+  star[0][0].measure.dYccd = ToShortPixels(strtod (ptr, NULL)); // sig_dec
+  ptr = nextWISEfield (ptr);
+  // we only know a single set of values for all 4 bands
+  for (i = 1; i < 4; i++) {
+      star[i][0].measure.dXccd    = star[0][0].measure.dXccd;
+      star[i][0].measure.dYccd    = star[0][0].measure.dYccd;
+      star[i][0].measure.posangle = star[0][0].measure.posangle;
+      star[i][0].measure.pltscale = star[0][0].measure.pltscale;
+  }
+  ptr = skipNbounds (ptr, '|', 5, Nmax); // skip: sig_radec, glon, glat, elon, elat
+  star[0][0].measure.Xccd = strtod (ptr, NULL); // wx
+  ptr = nextWISEfield (ptr); // skip wx
+  star[0][0].measure.Yccd = strtod (ptr, NULL); // wy
+  ptr = nextWISEfield (ptr); // skip wy
+  for (i = 1; i < 4; i++) {
+      star[i][0].measure.Xccd    = star[0][0].measure.Xccd;
+      star[i][0].measure.Yccd    = star[0][0].measure.Yccd;
+  }
+  ptr = skipNbounds (ptr, '|', 4, Nmax); // skip: cntr, source_id, coadd_id, src
+  // W1
+  for (i = 0; i < 4; i++) {
+      star[i][0].measure.M  = strtod (ptr, NULL); // w?mpro
+      ptr = nextWISEfield (ptr);
+      star[i][0].measure.dM = strtod (ptr, NULL); // w?sigmpro
+      ptr = skipNbounds (ptr, '|', 2, Nmax); // skip: w?sigmpro, w?snr
+      star[i][0].measure.psfChisq = strtod (ptr, NULL); // w?rchi2
+      ptr = nextWISEfield (ptr); // skip : w1rchi2
+      // init the photFlags field
+      star[i][0].measure.photFlags = 0;
+  }
+  ptr = nextWISEfield (ptr); // skip: rchi2
+  // set blend flags for all 4 measures
+  setWISE_blend_flag (star, ptr); // nb & na both used here
+  ptr = skipNbounds (ptr, '|', 2, Nmax); // skip: nb, na
+  for (i = 0; i < 4; i++) {
+      setWISE_sat_flag (star[i], ptr); // w1sat
+      ptr = nextWISEfield (ptr);
+  }
+  ptr = nextWISEfield (ptr); // skip satnum
+  for (i = 0; i < 4; i++) {
+      setWISE_cc_flag (star[i], ptr[i]); // cc_flg
+  }
+  ptr = nextWISEfield (ptr); // skip cc_flags
+  // set ext flags for all 4 measures
+  setWISE_ext_flag_prelim (star, *ptr); // ext_flg
+  ptr = nextWISEfield (ptr); // skip ext_flags
+  for (i = 0; i < 4; i++) {
+      setWISE_var_flag_prelim (star[i], ptr[i]); // var_flg
+  }
   ptr = nextWISEfield (ptr); // skip var_flags
 …
 # define FLAG_EXTENDED        0x01000000 // ext == 1
+# define FLAG_EXT_IN_XSC      0x02000000 // ext == 1
+# define FLAG_EXT_BY_XSC      0x04000000 // ext == 1
 # define FLAG_VARIABLE_LEVEL1 0x10000000 // var_flg == 1 to 4
 …
     case '0': break;
     default:
+      fprintf (stderr, "error!\n");
+      exit (2);
+      fprintf (stderr, "error in cc_flag: %c\n", qual);
+  }
   return (TRUE);
+}
+int setWISE_ext_flag (Stars **star, char *ptr) {
+    int ext = atoi (ptr);
+    if (ext == 1) {
+        star[0][0].measure.photFlags |= FLAG_EXTENDED;
+        star[1][0].measure.photFlags |= FLAG_EXTENDED;
+        star[2][0].measure.photFlags |= FLAG_EXTENDED;
+        star[3][0].measure.photFlags |= FLAG_EXTENDED;
+    }
+    return TRUE;
+}
+int setWISE_var_flag (Stars *star, char qual) {
+    int value = qual - '0';
+    assert (value >= 0);
+    assert (value <= 9);
+    if (value == 0) return TRUE;
+    if (value < 5) {
+int setWISE_ext_flag_allsky (Stars **star, char value) {
+  switch (value) {
+    case '0':
+      return TRUE;
+    case '1':
+      star[0][0].measure.photFlags |= FLAG_EXTENDED;
+      star[1][0].measure.photFlags |= FLAG_EXTENDED;
+      star[2][0].measure.photFlags |= FLAG_EXTENDED;
+      star[3][0].measure.photFlags |= FLAG_EXTENDED;
+      return TRUE;
+    case '2':
+      star[0][0].measure.photFlags |= FLAG_EXT_IN_XSC;
+      star[1][0].measure.photFlags |= FLAG_EXT_IN_XSC;
+      star[2][0].measure.photFlags |= FLAG_EXT_IN_XSC;
+      star[3][0].measure.photFlags |= FLAG_EXT_IN_XSC;
+      return TRUE;
+    case '3':
+      star[0][0].measure.photFlags |= FLAG_EXTENDED | FLAG_EXT_IN_XSC;
+      star[1][0].measure.photFlags |= FLAG_EXTENDED | FLAG_EXT_IN_XSC;
+      star[2][0].measure.photFlags |= FLAG_EXTENDED | FLAG_EXT_IN_XSC;
+      star[3][0].measure.photFlags |= FLAG_EXTENDED | FLAG_EXT_IN_XSC;
+      return TRUE;
+    case '4':
+      star[0][0].measure.photFlags |= FLAG_EXT_BY_XSC;
+      star[1][0].measure.photFlags |= FLAG_EXT_BY_XSC;
+      star[2][0].measure.photFlags |= FLAG_EXT_BY_XSC;
+      star[3][0].measure.photFlags |= FLAG_EXT_BY_XSC;
+      return TRUE;
+    case '5':
+      star[0][0].measure.photFlags |= FLAG_EXTENDED | FLAG_EXT_BY_XSC;
+      star[1][0].measure.photFlags |= FLAG_EXTENDED | FLAG_EXT_BY_XSC;
+      star[2][0].measure.photFlags |= FLAG_EXTENDED | FLAG_EXT_BY_XSC;
+      star[3][0].measure.photFlags |= FLAG_EXTENDED | FLAG_EXT_BY_XSC;
+      return TRUE;
+    default:
+      fprintf (stderr, "programming error\n");
+      abort();
+  }
+  return TRUE;
+}
+int setWISE_ext_flag_prelim (Stars **star, char value) {
+  switch (value) {
+    case '0':
+      return TRUE;
+    case '1':
+      star[0][0].measure.photFlags |= FLAG_EXTENDED;
+      star[1][0].measure.photFlags |= FLAG_EXTENDED;
+      star[2][0].measure.photFlags |= FLAG_EXTENDED;
+      star[3][0].measure.photFlags |= FLAG_EXTENDED;
+      return TRUE;
+    default:
+      fprintf (stderr, "programming error\n");
+      abort();
+  }
+  return TRUE;
+}
+// NOTE: var flag definition changed slightly between prelim & allsky
+int setWISE_var_flag_allsky (Stars *star, char qual) {
+  switch (qual) {
+    case 'n':
+        return TRUE;
+    case '0':
+    case '1':
+    case '2':
+    case '3':
+    case '4':
+    case '5':
         star[0].measure.photFlags |= FLAG_VARIABLE_LEVEL1;
         return TRUE;
+    }
     if (value < 8) {
+    case '6':
+    case '7':
         star[0].measure.photFlags |= FLAG_VARIABLE_LEVEL2;
         return TRUE;
+    }
+    star[0].measure.photFlags |= FLAG_VARIABLE_LEVEL3;
+    return TRUE;
+    case '8':
+    case '9':
+      star[0].measure.photFlags |= FLAG_VARIABLE_LEVEL3;
+      return TRUE;
+    default:
+      fprintf (stderr, "programming error\n");
+      abort();
+  }
+}
+int setWISE_var_flag_prelim (Stars *star, char qual) {
+  switch (qual) {
+    case '0':
+      return TRUE;
+    case '1':
+    case '2':
+    case '3':
+    case '4':
+        star[0].measure.photFlags |= FLAG_VARIABLE_LEVEL1;
+        return TRUE;
+    case '5':
+    case '6':
+    case '7':
+        star[0].measure.photFlags |= FLAG_VARIABLE_LEVEL2;
+        return TRUE;
+    case '8':
+    case '9':
+      star[0].measure.photFlags |= FLAG_VARIABLE_LEVEL3;
+      return TRUE;
+    default:
+      fprintf (stderr, "programming error\n");
+      abort();
+  }
+}
 …
     case 'X': star[0].measure.photFlags |= FLAG_PH_X; break;
     default:
+      fprintf (stderr, "error!\n");
+      exit (2);
+      fprintf (stderr, "error in ph_flag: %c\n", qual);
+  }
   return (TRUE);

trunk/Ohana/src/addstar/src/resort_unthreaded.c

-              r29938
+              r33963
 # include "addstar.h"
+int resort_unthreaded (AddstarClientOptions *options, SkyTable *sky) {
+  off_t i;
+  off_t Naverage, Nmeasure;
+  Catalog catalog;
+int resort_unthreaded (AddstarClientOptions *options, SkyTable *sky, int hostID, char *hostpath) {
   double dtime;
 …
+  }
+  // XXX ALLOCATE catalog[Nthreads]
+  /* match stars to existing catalog data (or otherwise manipulate catalog data) */
+  Naverage = Nmeasure = 0;
+  for (i = 0; i < skylist[0].Nregions; i++) {
+    // XXX BLOCK here for an empty thread
+    // set the parameters which guide catalog open/load/create
+    catalog.filename  = skylist[0].filename[i];
+    catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
+    catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
+    catalog.catflags  = LOAD_AVES | LOAD_MEAS;
+    catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
+    // an error exit status here is a significant error (disk I/O or file access)
+    if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE, "w")) {
+      fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", catalog.filename);
+      exit (2);
+    }
+    // Naves_disk == 0 implies an empty catalog file, skip empty catalogs
+    if (catalog.Naves_disk == 0) {
+      dvo_catalog_unlock (&catalog);
+      dvo_catalog_free (&catalog);
+      continue;
+    }
+    // this is an overloaded value to mean 'force sort'
+    if (options->nosort == 3) catalog.sorted = FALSE;
+    // XXX send these to a free thread
+    if (OLD_RESORT) {
+      resort_catalog_old (&catalog);
+    } else {
+      resort_catalog (&catalog);
+    }
+    // XXX wait for completed threads
+    /* report total updated values */
+    Naverage += catalog.Naverage;
+    Nmeasure += catalog.Nmeasure;
+    // write out catalog, if appropriate
+    SetProtect (TRUE);
+    dvo_catalog_save (&catalog, VERBOSE);
+    SetProtect (FALSE);
+    dvo_catalog_unlock (&catalog);
+    dvo_catalog_free (&catalog);
+  }
+  resort_unthreaded_catalogs (options, skylist, 0, NULL);
   gettimeofday (&stop, NULL);
   dtime = DTIME (stop, start);
   fprintf (stderr, "SUCCESS: elapsed time %9.4f sec for, "OFF_T_FMT" average, "OFF_T_FMT" measure\n", dtime, Naverage, Nmeasure);
+  fprintf (stderr, "SUCCESS: elapsed time %9.4f sec\n", dtime);
   return TRUE;

trunk/Ohana/src/addstar/test

Property svn:ignore

old	new
16	16	relphot.Mag.png
17	17	relphot.Mcal.dMcal.png
	18	catdir.wise.test

trunk/Ohana/src/dvomerge/Makefile

-              r31496
+              r33963
 default: all
 help:
         @echo "make options: dvomerge (default)"
+        @echo "make options: dvomerge dvomerge_client dvoconvert (default)"
 include ../../Makefile.System
 …
 FULL_LDFLAGS  = -lkapa -ldvo -lFITS -lohana $(BASE_LDFLAGS)
+dvomerge     : $(BIN)/dvomerge.$(ARCH)
+dvomerge     : $(BIN)/dvoconvert.$(ARCH)
+dvomerge        : $(BIN)/dvomerge.$(ARCH)
+dvomerge_client : $(BIN)/dvomerge_client.$(ARCH)
+dvoconvert      : $(BIN)/dvoconvert.$(ARCH)
 all: dvomerge dvoconvert dvosecfilt
+all: dvomerge dvomerge_client dvoconvert dvosecfilt
 #  $(SRC)/dvomergeContinue.$(ARCH).o
 …
 $(SRC)/dvomergeUpdate.$(ARCH).o \
 $(SRC)/dvomergeCreate.$(ARCH).o \
+$(SRC)/dvomergeUpdate_catalogs.$(ARCH).o \
 $(SRC)/dvomergeUpdate_threaded.$(ARCH).o \
 $(SRC)/dvomergeFromList.$(ARCH).o \
 $(SRC)/dvomergeImageIDs.$(ARCH).o \
 $(SRC)/dvo_image_merge_dbs.$(ARCH).o \
+$(SRC)/IDmapIO.$(ARCH).o \
 $(SRC)/SetSignals.$(ARCH).o \
 $(SRC)/ConfigInit.$(ARCH).o \
 …
 $(SRC)/LoadCatalog.$(ARCH).o \
 $(SRC)/build_links.$(ARCH).o \
+$(SRC)/replace_match.$(ARCH).o \
 $(SRC)/merge_catalogs_new.$(ARCH).o \
 $(SRC)/merge_catalogs_old.$(ARCH).o
 …
 $(BIN)/dvomerge.$(ARCH) : $(DVOMERGE)
+DVOMERGE_CLIENT = \
+$(SRC)/dvomerge_client.$(ARCH).o \
+$(SRC)/dvomergeUpdate_catalogs.$(ARCH).o \
+$(SRC)/dvomergeImageIDs.$(ARCH).o \
+$(SRC)/dvo_image_merge_dbs.$(ARCH).o \
+$(SRC)/IDmapIO.$(ARCH).o \
+$(SRC)/SetSignals.$(ARCH).o \
+$(SRC)/ConfigInit.$(ARCH).o \
+$(SRC)/Shutdown.$(ARCH).o \
+$(SRC)/help.$(ARCH).o \
+$(SRC)/args.$(ARCH).o \
+$(SRC)/psps_ids.$(ARCH).o \
+$(SRC)/SkyTablePopulatedRange.$(ARCH).o \
+$(SRC)/LoadCatalog.$(ARCH).o \
+$(SRC)/build_links.$(ARCH).o \
+$(SRC)/replace_match.$(ARCH).o \
+$(SRC)/merge_catalogs_new.$(ARCH).o \
+$(SRC)/merge_catalogs_old.$(ARCH).o
+$(DVOMERGE_CLIENT) : $(INC)/dvomerge.h
+$(BIN)/dvomerge_client.$(ARCH) : $(DVOMERGE_CLIENT)
 DVOCONVERT = \
 …
 $(BIN)/dvoverify.$(ARCH) : $(DVOVERIFY)
 INSTALL = dvomerge dvoconvert dvosecfilt dvorepair dvoverify
+INSTALL = dvomerge dvomerge_client dvoconvert dvosecfilt dvorepair dvoverify
 # dependancy rules for binary code #########################

trunk/Ohana/src/dvomerge/include/dvomerge.h

-              r33657
+              r33963
 # include <glob.h>
+# define DVO_MAX_PATH 1024
+int    PARALLEL;
+int    PARALLEL_MANUAL;
+int    PARALLEL_SERIAL;
+int    HOST_ID;
+char  *HOSTDIR;
 int    VERBOSE;
 char   CATDIR[256];
 …
 int    SKY_DEPTH;
 int    NTHREADS;
+char   *ALTERNATE_PHOTCODE_FILE;
+char  *ALTERNATE_PHOTCODE_FILE;
+int    REPLACE_BY_PHOTCODE;
 SkyRegion UserPatch;  // used by MODE CAT
 typedef struct {
   off_t Nmap;
   off_t *old;
   off_t *new;
+  unsigned int Nmap;
+  unsigned int *old;
+  unsigned int *new;
 } IDmapType;
 …
 void       dvomerge_help          PROTO((int argc, char **argv));
 int        dvomerge_args          PROTO((int *argc, char **argv));
+void       dvomerge_client_usage  PROTO((void));
+void       dvomerge_client_help   PROTO((int argc, char **argv));
+int        dvomerge_client_args   PROTO((int *argc, char **argv));
 void       dvoconvert_usage       PROTO((void));
 …
 int        dvomergeFromList       PROTO((int argc, char **argv));
 int        dvomergeUpdate_threaded PROTO((int argc, char **argv));
+int        dvomergeUpdate_catalogs PROTO((char *input, char *output, SkyTable *outsky, SkyList *inlist, int NsecfiltInput, int NsecfiltOutput, IDmapType *IDmap, int *secfiltMap));
+int        replace_match           PROTO((Average *average_out, Measure *measure_out, Average *average_in, Measure *measure_in));
+int        IDmapSave               PROTO((char *filename, IDmapType *IDmap));
+IDmapType *IDmapLoad               PROTO((char *filename));

trunk/Ohana/src/dvomerge/src/args.c

-              r33657
+              r33963
     remove_argument (N, argc, argv);
     ALTERNATE_PHOTCODE_FILE = strcreate(argv[N]);
+    remove_argument (N, argc, argv);
+  }
+  /* replace measurement, don't duplicate */
+  REPLACE_BY_PHOTCODE = FALSE;
+  if ((N = get_argument (*argc, argv, "-replace"))) {
+    REPLACE_BY_PHOTCODE = TRUE;
     remove_argument (N, argc, argv);
+  }
 …
+  }
+  // XXX for the moment, make this selection manual.  it needs to be automatic
+  // based on the state of the SkyTable
+  PARALLEL = FALSE;
+  if ((N = get_argument (*argc, argv, "-parallel"))) {
+    PARALLEL = TRUE;
+    remove_argument (N, argc, argv);
+  }
+  // this is a test mode : rather than launching the remote jobs and waiting for completion,
+  // relphot will simply list the remote command and wait for the user to signal completion
+  PARALLEL_MANUAL = FALSE;
+  if ((N = get_argument (*argc, argv, "-parallel-manual"))) {
+    PARALLEL = TRUE; // -parallel-manual implies -parallel
+    PARALLEL_MANUAL = TRUE;
+    remove_argument (N, argc, argv);
+  }
+  // this is a test mode : rather than launching the relphot_client jobs remotely, they are
+  // run in serial via 'system'
+  PARALLEL_SERIAL = FALSE;
+  if ((N = get_argument (*argc, argv, "-parallel-serial"))) {
+    if (PARALLEL_MANUAL) {
+      fprintf (stderr, "ERROR: cannot mix -parallel-manual and -parallel-serial\n");
+      exit (1);
+    }
+    PARALLEL = TRUE; // -parallel-serial implies -parallel
+    PARALLEL_SERIAL = TRUE;
+    remove_argument (N, argc, argv);
+  }
   if ((*argc < 4) || (*argc > 6)) dvomerge_usage();
+  return TRUE;
+}
+/*** check for command line options ***/
+int dvomerge_client_args (int *argc, char **argv) {
+  int N;
+  // by definition, the client is not parallel
+  PARALLEL = FALSE;
+  PARALLEL_MANUAL = FALSE;
+  PARALLEL_SERIAL = FALSE;
+  HOST_ID = 0;
+  if ((N = get_argument (*argc, argv, "-hostID"))) {
+    remove_argument (N, argc, argv);
+    HOST_ID = atoi (argv[N]);
+    remove_argument (N, argc, argv);
+  }
+  if (!HOST_ID) dvomerge_client_usage();
+  HOSTDIR = NULL;
+  if ((N = get_argument (*argc, argv, "-hostdir"))) {
+    remove_argument (N, argc, argv);
+    HOSTDIR = strcreate (argv[N]);
+    remove_argument (N, argc, argv);
+  }
+  if (!HOSTDIR) dvomerge_client_usage();
+  /* extra error messages */
+  VERBOSE = FALSE;
+  if ((N = get_argument (*argc, argv, "-v"))) {
+    VERBOSE = TRUE;
+    remove_argument (N, argc, argv);
+  }
+  /* replace measurement, don't duplicate */
+  REPLACE_BY_PHOTCODE = FALSE;
+  if ((N = get_argument (*argc, argv, "-replace"))) {
+    REPLACE_BY_PHOTCODE = TRUE;
+    remove_argument (N, argc, argv);
+  }
+  NTHREADS = 0;
+  if ((N = get_argument (*argc, argv, "-threads"))) {
+    remove_argument (N, argc, argv);
+    NTHREADS = MAX(0, atoi(argv[N]));
+    remove_argument (N, argc, argv);
+  }
+  /*** provide additional data ***/
+  /* restrict to a portion of the sky? */
+  UserPatch.Rmin = 0;
+  UserPatch.Rmax= 360;
+  UserPatch.Dmin = -90;
+  UserPatch.Dmax = +90;
+  if ((N = get_argument (*argc, argv, "-region"))) {
+    remove_argument (N, argc, argv);
+    UserPatch.Rmin = atof (argv[N]);
+    remove_argument (N, argc, argv);
+    UserPatch.Rmax = atof (argv[N]);
+    remove_argument (N, argc, argv);
+    UserPatch.Dmin = atof (argv[N]);
+    remove_argument (N, argc, argv);
+    UserPatch.Dmax = atof (argv[N]);
+    remove_argument (N, argc, argv);
+  }
+  if ((*argc < 4) || (*argc > 6)) dvomerge_client_usage();
   return TRUE;
+}

trunk/Ohana/src/dvomerge/src/dvo_image_merge_dbs.c

-              r31450
+              r33963
 # include "dvomerge.h"
+// utility functions
 void sort_IDmap (IDmapType *IDmap);
+off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt) {
+  // use bisection to find the starting entry by time
+  off_t Nlo, Nhi, N;
+  // find the last TGT before start
+  Nlo = 0; Nhi = NimagesTgt;
+  while (Nhi - Nlo > 10) {
+    N = 0.5*(Nlo + Nhi);
+    if (TgtTimes[N] < start) {
+      Nlo = MAX(N, 0);
+    } else {
+      Nhi = MIN(N + 1, NimagesTgt);
+    }
+  }
+  // check for the matched mosaic starting from Nlo
+  // we may have to go much beyond Nlo since stop is not sorted
+  // can we use a sorted version of stop to check when we are beyond the valid range??
+  for (N = Nlo; N < NimagesTgt; N++) {
+    if (TgtTimes[N] < start) continue;
+    if (TgtTimes[N] > stop) return (-1);
+    if (TgtCodes[N] != photcode) continue;
+    return (TgtIndex[N]);
+  }
+  return (-1);
+}
+// sort two times vectors and an index by first time vector
+void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N) {
+# define SWAPFUNC(A,B){ e_time tmp_t; off_t tmp_i; short tmp_c;         \
+  tmp_t = S[A]; S[A] = S[B]; S[B] = tmp_t; \
+  tmp_i = I[A]; I[A] = I[B]; I[B] = tmp_i; \
+  tmp_c = C[A]; C[A] = C[B]; C[B] = tmp_c; \
+}
+# define COMPARE(A,B)(S[A] < S[B])
+  OHANA_SORT (N, COMPARE, SWAPFUNC);
+# undef SWAPFUNC
+# undef COMPARE
+}
+void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N);
+off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt);
 // we have two tables; 'tgt' contains some exposures from 'src' : find them and match a map
 …
+  }
   ALLOCATE (IDmap->old, off_t, NimagesSrc);
   ALLOCATE (IDmap->new, off_t, NimagesSrc);
+  ALLOCATE (IDmap->old, unsigned int, NimagesSrc);
+  ALLOCATE (IDmap->new, unsigned int, NimagesSrc);
   IDmap->Nmap = NimagesSrc;
 …
   int status;
+  IDmap->old = NULL;
+  IDmap->new = NULL;
+  // it is OK if there are no images in the database, but there should be no imageIDs to map...
+  if (in->dbstate == LCK_EMPTY) {
+    return TRUE;
+  }
   images = gfits_table_get_Image (&in[0].ftable, &Nimages, &in[0].swapped);
   if (!images) {
 …
+  }
   ALLOCATE (IDmap->old, off_t, Nimages);
   ALLOCATE (IDmap->new, off_t, Nimages);
+  ALLOCATE (IDmap->old, unsigned int, Nimages);
+  ALLOCATE (IDmap->new, unsigned int, Nimages);
   IDmap->Nmap = Nimages;
 …
+}
+// XXX isn't the map just ID_new = ID_old + offset ??
+// map this ID to the new table
+// XXX isn't the map just ID_new = ID_old + offset ?? (probably not)
 off_t dvo_map_image_ID (IDmapType *IDmap, off_t oldID) {
 …
+}
+// given a table of detections, update their image IDs based on the new map
 int dvo_update_image_IDs (IDmapType *IDmap, Catalog *catalog) {
 …
     oldID = catalog[0].measure[i].imageID;
     if (oldID == 0) continue;
+    if (!IDmap->old) {
+      fprintf (stderr, "input database has image IDs, but no Image table\n");
+    }
     newID = dvo_map_image_ID (IDmap, oldID);
 …
+}
+off_t getTgtIndex (e_time start, e_time stop, short photcode, off_t *TgtIndex, e_time *TgtTimes, short *TgtCodes, off_t NimagesTgt) {
+  // use bisection to find the starting entry by time
+  off_t Nlo, Nhi, N;
+  // find the last TGT before start
+  Nlo = 0; Nhi = NimagesTgt;
+  while (Nhi - Nlo > 10) {
+    N = 0.5*(Nlo + Nhi);
+    if (TgtTimes[N] < start) {
+      Nlo = MAX(N, 0);
+    } else {
+      Nhi = MIN(N + 1, NimagesTgt);
+    }
+  }
+  // check for the matched mosaic starting from Nlo
+  // we may have to go much beyond Nlo since stop is not sorted
+  // can we use a sorted version of stop to check when we are beyond the valid range??
+  for (N = Nlo; N < NimagesTgt; N++) {
+    if (TgtTimes[N] < start) continue;
+    if (TgtTimes[N] > stop) return (-1);
+    if (TgtCodes[N] != photcode) continue;
+    return (TgtIndex[N]);
+  }
+  return (-1);
+}
+// sort two times vectors and an index by first time vector
+void SortTgtByTimes (e_time *S, off_t *I, short *C, off_t N) {
+# define SWAPFUNC(A,B){ e_time tmp_t; off_t tmp_i; short tmp_c;         \
+  tmp_t = S[A]; S[A] = S[B]; S[B] = tmp_t; \
+  tmp_i = I[A]; I[A] = I[B]; I[B] = tmp_i; \
+  tmp_c = C[A]; C[A] = C[B]; C[B] = tmp_c; \
+}
+# define COMPARE(A,B)(S[A] < S[B])
+  OHANA_SORT (N, COMPARE, SWAPFUNC);
+# undef SWAPFUNC
+# undef COMPARE
+}

trunk/Ohana/src/dvomerge/src/dvomergeCreate.c

-              r33657
+              r33963
   depth2 = insky2[0].regions[Ns].depth;
+  SkyList *outlist = SkyListByPatch (outsky, -1, &UserPatch);
   // loop over the populatable output regions
   for (i = 0; i < outsky[0].Nregions; i++) {
     if (!outsky[0].regions[i].table) continue;
     if (VERBOSE) fprintf (stderr, "output: %s\n", outsky[0].regions[i].name);
+  for (i = 0; i < outlist[0].Nregions; i++) {
+    if (!outlist[0].regions[i][0].table) continue;
+    if (VERBOSE) fprintf (stderr, "output: %s\n", outlist[0].regions[i][0].name);
     // load / create output catalog
     LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w", NsecfiltOutput);
+    LoadCatalog (&outcatalog, outlist[0].regions[i], outlist[0].filename[i], "w", NsecfiltOutput);
     // combine only tables at equal or larger depth
     // load in all of the tables from input1 for this region
     inlist = SkyListByBounds (insky1, depth1, outsky[0].regions[i].Rmin, outsky[0].regions[i].Rmax, outsky[0].regions[i].Dmin, outsky[0].regions[i].Dmax);
+    inlist = SkyListByBounds (insky1, depth1, outlist[0].regions[i][0].Rmin + 0.01, outlist[0].regions[i][0].Rmax - 0.01, outlist[0].regions[i][0].Dmin + 0.01, outlist[0].regions[i][0].Dmax - 0.01);
     for (j = 0; j < inlist[0].Nregions; j++) {
       if (VERBOSE) fprintf (stderr, "input 1: %s\n", inlist[0].regions[j][0].name);
 …
+      }
       dvo_update_image_IDs (&IDmap1, &incatalog);
       merge_catalogs_new (&outsky[0].regions[i], &outcatalog, &incatalog, secfiltMap1);
+      merge_catalogs_new (outlist[0].regions[i], &outcatalog, &incatalog, secfiltMap1);
       dvo_catalog_unlock (&incatalog);
       dvo_catalog_free (&incatalog);
 …
     // load in all of the tables from input2 for this region
     inlist = SkyListByBounds (insky2, depth2, outsky[0].regions[i].Rmin, outsky[0].regions[i].Rmax, outsky[0].regions[i].Dmin, outsky[0].regions[i].Dmax);
+    inlist = SkyListByBounds (insky2, depth2, outlist[0].regions[i][0].Rmin + 0.01, outlist[0].regions[i][0].Rmax - 0.01, outlist[0].regions[i][0].Dmin + 0.01, outlist[0].regions[i][0].Dmax - 0.01);
     for (j = 0; j < inlist[0].Nregions; j++) {
       if (VERBOSE) fprintf (stderr, "input 2: %s\n", inlist[0].regions[j][0].name);
 …
+      }
       dvo_update_image_IDs (&IDmap2, &incatalog);
       merge_catalogs_old (&outsky[0].regions[i], &outcatalog, &incatalog, RADIUS, secfiltMap2);
+      merge_catalogs_old (outlist[0].regions[i], &outcatalog, &incatalog, RADIUS, secfiltMap2);
       dvo_catalog_unlock (&incatalog);
       dvo_catalog_free (&incatalog);
 …
     in1DB.mode   = dvo_catalog_catmode (CATMODE);
     in1DB.format = dvo_catalog_catformat (CATFORMAT);
     status       = dvo_image_lock (&in1DB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
+    status       = dvo_image_lock (&in1DB, ImageCat, 3600.0, LCK_SOFT);  // shorter timeout?
     if (!status) Shutdown ("ERROR: failure to lock image catalog %s", in1DB.filename);
     // load the image table
+    if (in1DB.dbstate == LCK_EMPTY) {
+      Shutdown ("can't find input (1) image catalog %s", in1DB.filename);
+    }
+    if (!dvo_image_load (&in1DB, VERBOSE, TRUE)) {
+      Shutdown ("can't read input (1) image catalog %s", in1DB.filename);
+    }
+    if (in1DB.dbstate != LCK_EMPTY) {
+      if (!dvo_image_load (&in1DB, VERBOSE, TRUE)) {
+        Shutdown ("can't read input (1) image catalog %s", in1DB.filename);
+      }
+    }
     // convert database table to internal structure & add to output image db
+    // if in1DB has no images, we will (later) insist that there are no image IDs to map
     dvo_image_merge_dbs(IDmap1, &outDB, &in1DB);
     dvo_image_unlock (&in1DB); // unlock input1
     /*** load input2/Images.dat ***/
 …
     in2DB.mode   = dvo_catalog_catmode (CATMODE);
     in2DB.format = dvo_catalog_catformat (CATFORMAT);
     status       = dvo_image_lock (&in2DB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
+    status       = dvo_image_lock (&in2DB, ImageCat, 3600.0, LCK_SOFT);  // shorter timeout?
     if (!status) Shutdown ("ERROR: failure to lock image catalog %s", in2DB.filename);
     /* load the image table */
+    if (in2DB.dbstate == LCK_EMPTY) {
+      Shutdown ("can't find input (2) image catalog %s", in2DB.filename);
+    }
+    if (!dvo_image_load (&in2DB, VERBOSE, TRUE)) {
+      Shutdown ("can't read input (2) image catalog %s", in2DB.filename);
+    if (in2DB.dbstate != LCK_EMPTY) {
+      if (!dvo_image_load (&in2DB, VERBOSE, TRUE)) {
+        Shutdown ("can't read input (2) image catalog %s", in2DB.filename);
+      }
+    }
     // convert database table to internal structure & add to output image db
+    // if in1DB has no images, we will (later) insist that there are no image IDs to map
     dvo_image_merge_dbs(IDmap2, &outDB, &in2DB);
     dvo_image_unlock (&in2DB); // unlock input2

trunk/Ohana/src/dvomerge/src/dvomergeImageIDs.c

-              r29938
+              r33963
 # include "dvomerge.h"
 /*** update the image table ***/
+/*** generate the IDmap for the given databases, and update the output image table ***/
 int dvomergeImagesUpdate (IDmapType *IDmap, char *input, char *output) {
 …
   inDB.mode   = dvo_catalog_catmode (CATMODE);
   inDB.format = dvo_catalog_catformat (CATFORMAT);
   status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
+  status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_SOFT);  // shorter timeout?
   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
   // load the image table
   if (inDB.dbstate == LCK_EMPTY) {
+    dvo_image_unlock (&inDB); // unlock input
     IDmap->Nmap = 0;
     return TRUE;
 …
+}
 /*** update the image table ***/
+/*** generate the map for the given 2 databases ***/
 int dvomergeImagesGetMap (IDmapType *IDmap, char *input, char *output) {
 …
   inDB.mode   = dvo_catalog_catmode (CATMODE);
   inDB.format = dvo_catalog_catformat (CATFORMAT);
   status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
+  status       = dvo_image_lock (&inDB, ImageCat, 5400.0, LCK_XCLD);  // shorter timeout?
   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
   // load the image table
   if (inDB.dbstate == LCK_EMPTY) {
+    dvo_image_unlock (&inDB); // unlock input
     IDmap->Nmap = 0;
     return TRUE;
 …
     Shutdown ("can't read input image catalog %s", inDB.filename);
+  }
+  dvo_image_unlock (&inDB); // unlock input
   /*** load output/Images.dat ***/
 …
   outDB.mode   = dvo_catalog_catmode (CATMODE);
   outDB.format = dvo_catalog_catformat (CATFORMAT);
   status       = dvo_image_lock (&outDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
+  status       = dvo_image_lock (&outDB, ImageCat, 5400.0, LCK_XCLD);  // shorter timeout?
   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", outDB.filename);
   /* load the image table */
   if (outDB.dbstate == LCK_EMPTY) {
     Shutdown ("only use -continue for an existing, partially merged database");
+    Shutdown ("for dvomerge -continue or dvomerge_client, the output database must exist");
+  }
   if (!dvo_image_load (&outDB, VERBOSE, TRUE)) {
     Shutdown ("can't read output image catalog %s", outDB.filename);
+  }
+  dvo_image_unlock (&outDB); // unlock output
   // convert database table to internal structure & add to output image db
   dvo_image_match_dbs(IDmap, &outDB, &inDB);
-  dvo_image_unlock (&inDB); // unlock input
-  dvo_image_unlock (&outDB); // unlock output
   return TRUE;

trunk/Ohana/src/dvomerge/src/dvomergeUpdate.c

-              r33657
+              r33963
   int CONTINUE;
-  off_t i, j;
   SkyTable *outsky, *insky;
+  SkyList *outlist, *inlist;
+  Catalog incatalog, outcatalog;
+  SkyList *inlist;
   char filename[256], *input, *output;
   IDmapType IDmap;
 …
   SetPhotcodeTable(NULL);
+  // loop over the populated input regions
+  for (i = 0; i < inlist[0].Nregions; i++) {
+    if (!inlist[0].regions[i][0].table) continue;
+    if (VERBOSE) fprintf (stderr, "input: %s\n", inlist[0].regions[i][0].name);
+    // load / create output catalog (if catalog does not exist, it will be created)
+    LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput);
+    // skip empty input catalogs
+    if (!incatalog.Naves_disk) {
+        dvo_catalog_unlock (&incatalog);
+        dvo_catalog_free (&incatalog);
+        continue;
+    }
+    // combine only tables at equal or larger depth
+    // load in all of the tables from input for this region
+    // SkyListByBounds will return neighbor catalogs if the boundaries exactly match (due to rounding).  Since the regions are not infinitely small,
+    // compare to a slightly reduced footprint
+    float dPos = 2.0/3600.0;
+    outlist = SkyListByBounds (outsky, inlist[0].regions[i][0].depth, inlist[0].regions[i][0].Rmin + dPos, inlist[0].regions[i][0].Rmax - dPos, inlist[0].regions[i][0].Dmin + dPos, inlist[0].regions[i][0].Dmax - dPos);
+    for (j = 0; j < outlist[0].Nregions; j++) {
+      if (VERBOSE) fprintf (stderr, "output : %s\n", outlist[0].regions[j][0].name);
+      // load input catalog
+      LoadCatalog (&outcatalog, outlist[0].regions[j], outlist[0].filename[j], "w", NsecfiltOutput);
+      dvo_update_image_IDs (&IDmap, &incatalog);
+      merge_catalogs_old (&outsky[0].regions[j], &outcatalog, &incatalog, RADIUS, secfiltMap);
+      outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
+      // if we receive a signal which would cause us to exit, wait until the full catalog is written
+      SetProtect (TRUE);
+      if (!dvo_catalog_save (&outcatalog, VERBOSE)) {
+        fprintf (stderr, "ERROR: failed to save catalog %s\n", outlist[0].filename[j]);
+        exit (1);
+      }
+      SetProtect (FALSE);
+      dvo_catalog_unlock (&outcatalog);
+      dvo_catalog_free (&outcatalog);
+      fprintf (stderr, "merged %s into %s\n", inlist[0].regions[i][0].name, outlist[0].regions[j][0].name);
+    }
+    SkyListFree (outlist);
+    dvo_catalog_unlock (&incatalog);
+    dvo_catalog_free (&incatalog);
+  }
+  dvomergeUpdate_catalogs (input, output, outsky, inlist, NsecfiltInput, NsecfiltOutput, &IDmap, secfiltMap);
   // save the output sky table copy

trunk/Ohana/src/dvomerge/src/dvoverify.c

-              r33657
+              r33963
     CHECKSORTED = TRUE;
     remove_argument (N, &argc, argv);
+  }
+  if ((N = get_argument (argc, argv, "-cpt"))) {
+    remove_argument (N, &argc, argv);
+    char *filename = strcreate (argv[N]);
+    remove_argument (N, &argc, argv);
+    int isBad = FALSE;
+    if (!VerifyTableFile (filename)) {
+      fprintf (stderr, "bad average table %s\n", filename);
+      isBad = TRUE;
+    }
+    // change last 't' to 's':
+    int Nlast;
+    Nlast = strlen(filename) - 1;
+    filename[Nlast] = 's';
+    if (!VerifyTableFile (filename)) {
+      fprintf (stderr, "bad secfilt table %s\n", filename);
+      isBad = TRUE;
+    }
+    // change last 't' to 's':
+    Nlast = strlen(filename) - 1;
+    filename[Nlast] = 'm';
+    if (!VerifyTableFile (filename)) {
+      fprintf (stderr, "bad measure table %s\n", filename);
+      isBad = TRUE;
+    }
+    if (isBad) exit (1);
+    exit (0);
+  }

trunk/Ohana/src/dvomerge/src/help.c

-              r30615
+              r33963
   fprintf (stderr, "   OR: dvomerge (input) into (output) continue\n");
   fprintf (stderr, "   OR: dvomerge (input) into (output) from (list)\n");
+  fprintf (stderr, "   [-region Rmin Rmax Dmin Dmax]\n");
+  exit (2);
+}
+void dvomerge_client_usage (void) {
+  fprintf (stderr, "USAGE: dvomerge_client (input) into (output)\n");
   fprintf (stderr, "   [-region Rmin Rmax Dmin Dmax]\n");
   exit (2);
 …
   fprintf (stderr, "  -h                          : this list\n\n");
   fprintf (stderr, "  -region Rmin Rmax Dmin Dmax : region to merge\n\n");
+  fprintf (stderr, "  -replace                    : replace existing detections of the same photcode\n");
+  fprintf (stderr, "                                (this should only be used to merge reference dbs)\n\n");
+  exit (2);
+}
+void dvomerge_client_help (int argc, char **argv) {
+  /* check for help request */
+  if (get_argument (argc, argv, "-help")) goto show_help;
+  if (get_argument (argc, argv, "-h"))    goto show_help;
+  return;
+show_help:
+  fprintf (stderr, "USAGE\n");
+  fprintf (stderr, "  dvomerge_client (input) into (output)\n");
+  fprintf (stderr, "  merge DVO databases\n");
+  fprintf (stderr, "  optional flags:\n");
+  fprintf (stderr, "  -v                          : verbose mode\n");
+  fprintf (stderr, "  -help                       : this list\n");
+  fprintf (stderr, "  -h                          : this list\n\n");
+  fprintf (stderr, "  -region Rmin Rmax Dmin Dmax : region to merge\n\n");
+  fprintf (stderr, "  -replace                    : replace existing detections of the same photcode\n");
+  fprintf (stderr, "                                (this should only be used to merge reference dbs)\n\n");
   exit (2);
+}

trunk/Ohana/src/dvomerge/src/merge_catalogs_old.c

-              r30761
+              r33963
   if (Nave > 1) sort_coords_index (X2, Y2, N2, Nave);
+  if (REPLACE_BY_PHOTCODE && output[0].Nmeasure && !output[0].sorted) {
+    fprintf (stderr, "ERROR: attempt to merge with replace into an unsorted database\n");
+    exit (3);
+  }
   /* set up pointers for linked list of measure */
   if (output[0].sorted && (output[0].Nmeasure >= output[0].Nmeas_disk)) {
 …
       offset = input[0].average[N].measureOffset + Nin;
+      if (REPLACE_BY_PHOTCODE) {
+        // index to first measure for this object
+        int Mout = output[0].average[n].measureOffset;
+        if (replace_match(&output[0].average[n], &output[0].measure[Mout], &input[0].average[N], &input[0].measure[offset])) {
+          input[0].found[N] = Mout;
+          continue;
+        }
+      }
       /* add to end of measurement list */
       add_meas_link (&output[0].average[n], next_meas, Nmeas, NMEAS);

trunk/Ohana/src/dvomerge/test

Property svn:ignore set to
catdir.test

trunk/Ohana/src/dvomerge/test/catdir.2mass

Property svn:ignore set to
s8230
s3730
s6730
n3000
s0000
n1500
n6000
n2230
s3000
n4500
n5230
s6000
s1500
n7500
s2230
n8230
n3730
s4500
s0730
Images.dat
s5230
n6730
s7500

trunk/Ohana/src/dvosplit/src/dvosplit.c

-              r33656
+              r33963
     char line[2048];
     snprintf (line, 2048, "cp %s/Images.dat %s/Images.dat", CATDIR, OUTDIR);
+    system (line);
+    int status = system (line);
+    if (status) {
+      fprintf (stderr, "ERROR: failed to copy Images.dat\n");
+      exit (1);
+    }
+  }

trunk/Ohana/src/libautocode/def/image.d

-              r33647
+              r33963
 FIELD     dMagSys,          MAG_SYS_ERR,          float,          systematic photometry error
 FIELD     nFitAstrom,       N_FIT_ASTROM,         short,          number of stars used for astrometry cal
 FIELD     nFitPhotom,       N_FIT_PHOTOM,         short,          number of stars used for photometry cal
+FIELD     nFitAstrom,       N_FIT_ASTROM,         unsigned short, number of stars used for astrometry cal
+FIELD     nFitPhotom,       N_FIT_PHOTOM,         unsigned short, number of stars used for photometry cal
 FIELD     photom_map_id,    PHOTOM_MAP_ID,        unsigned int,   reference to 2D zero point map

trunk/Ohana/src/libautocode/def/secfilt-ps1-v3.d

r33647	r33963
16	16	FIELD M_80, MAG_80, short, upper 20percent mag, millimags
17	17	FIELD ubercalDist, UBERCAL_DIST, short, number of images from an ubercal-image
18		FIELD ~~dummy, DUMMY, char[2], padding~~
	18	FIELD Mstdev, MAG_STDEV, short, standard deviation of measurements, millimags

trunk/Ohana/src/libautocode/def/secfilt.d

r33647	r33963
16	16	FIELD M_80, MAG_80, short, upper 20percent mag, millimags
17	17	FIELD ubercalDist, UBERCAL_DIST, short, number of images from an ubercal-image
18		FIELD ~~dummy, DUMMY, char[2], padding~~
	18	FIELD Mstdev, MAG_STDEV, short, standard deviation of measurements, millimags
19	19
20	20	# *** 20090206 : new fields : M_20, M_80; dropped dummy

trunk/Ohana/src/libdvo/include/dvo.h

-              r33779
+              r33963
   NAN_U_INT   = (signed int) 0xffffffff,
 } DVO_INT_NAN;
+// max path length
+# define DVO_MAX_PATH 1024
 /* RegImage.flag values */
 …
 float PhotdM (PhotCode *code, Average *average, SecFilt *secfilt);
+float PhotMstdev (PhotCode *code, Average *average, SecFilt *secfilt);
+float PhotM20 (PhotCode *code, Average *average, SecFilt *secfilt);
+float PhotM80 (PhotCode *code, Average *average, SecFilt *secfilt);
+float PhotUCdist (PhotCode *code, Average *average, SecFilt *secfilt);
 float PhotColorForCode (Average *average, SecFilt *secfilt, Measure *measure, PhotCode *code);
 int PhotColor (Average *average, SecFilt *secfilt, Measure *measure, int c1, int c2, double *color);
 …
 void dvo_catalog_test (Catalog *catalog, int halt);
+int dvo_catalog_backup (Catalog *catalog, int primary);
+int dvo_catalog_unlink_backup (Catalog *catalog, int primary);
 /* catmode-specific APIs */
 int dvo_catalog_load_raw (Catalog *catalog, int VERBOSE);

trunk/Ohana/src/libdvo/include/dvodb.h

-              r33649
+              r33963
       MAG_REF,
       MAG_ERR,
+      MAG_STDEV,
       MAG_AVE_ERR,
       MAG_PHOT_FLAGS,
 …
       MAG_NCODE,
       MAG_NPHOT,
+      MAG_20,
+      MAG_80,
+      MAG_UC_DIST,
 };
 …
 int dbExtractAveragesInitTransform (CoordTransformSystem target);
+int dbExtractAveragesInitAve (void);
 int dbExtractAveragesInit (void);

trunk/Ohana/src/libdvo/src/HostTable.c

-              r33779
+              r33963
   for (i = 0; i < table->Nhosts; i++) {
     if (table->index[table->hosts[i].hostID] == -1) {
+    if (table->index[table->hosts[i].hostID] != -1) {
       fprintf (stderr, "error: duplicate hostID %d\n", table->hosts[i].hostID);
       exit (1);
 …
     // find the host which has finished
     int j;
+    int Nout, j;
     int found = FALSE;
     for (j = 0; j < table->Nhosts; j++) {
 …
       EmptyIOBuffer (&buffer, 100, table->hosts[j].stdio[HOST_STDOUT]);
       fprintf (stderr, "--- stdout from %s ---\n", table->hosts[j].hostname);
+      write (STDOUT_FILENO, buffer.buffer, buffer.Nbuffer);
+      Nout = write (STDOUT_FILENO, buffer.buffer, buffer.Nbuffer);
+      if (Nout != buffer.Nbuffer) { fprintf (stderr, "(error writing log?)\n"); }
       fprintf (stderr, "\n");
 …
       EmptyIOBuffer (&buffer, 100, table->hosts[j].stdio[HOST_STDERR]);
       fprintf (stderr, "--- stderr from %s ---\n", table->hosts[j].hostname);
+      write (STDOUT_FILENO, buffer.buffer, buffer.Nbuffer);
+      Nout = write (STDOUT_FILENO, buffer.buffer, buffer.Nbuffer);
+      if (Nout != buffer.Nbuffer) { fprintf (stderr, "(error writing log?)\n"); }
       fprintf (stderr, "\n");
       if (WIFEXITED(status)) {
 …
         // if (table->hosts[i].stdout.Nbuffer > 0x10000) {
         if (table->hosts[i].stdout.Nbuffer > 0x1000) {
+          int printHead = VERBOSE || table->hosts[i].stdout.Nbuffer;
+          if (printHead) fprintf (stdout, "--- stdout from %s ---\n", table->hosts[i].hostname);
+          write (STDOUT_FILENO, table->hosts[i].stdout.buffer, table->hosts[i].stdout.Nbuffer);
+          int printHead = VERBOSE || (table->hosts[i].stdout.Nbuffer > 0);
+          if (printHead) fprintf (stdout, "--- stdout from %s --- (%d bytes, v1)\n", table->hosts[i].hostname, table->hosts[i].stdout.Nbuffer);
+          int Nout = write (STDOUT_FILENO, table->hosts[i].stdout.buffer, table->hosts[i].stdout.Nbuffer);
+          if (Nout != table->hosts[i].stdout.Nbuffer) { fprintf (stderr, "(error writing log?)\n"); }
           FlushIOBuffer (&table->hosts[i].stdout);
           if (printHead) fprintf (stdout, "\n");
 …
         // if (table->hosts[i].stderr.Nbuffer > 0x10000) {
         if (table->hosts[i].stderr.Nbuffer > 0x1000) {
+          int printHead = VERBOSE || table->hosts[i].stderr.Nbuffer;
+          if (printHead) fprintf (stdout, "--- stderr from %s ---\n", table->hosts[i].hostname);
+          write (STDOUT_FILENO, table->hosts[i].stderr.buffer, table->hosts[i].stderr.Nbuffer);
+          int printHead = VERBOSE || (table->hosts[i].stderr.Nbuffer > 0);
+          if (printHead) fprintf (stdout, "--- stderr from %s --- (%d bytes, v1)\n", table->hosts[i].hostname, table->hosts[i].stderr.Nbuffer);
+          int Nout = write (STDOUT_FILENO, table->hosts[i].stderr.buffer, table->hosts[i].stderr.Nbuffer);
+          if (Nout != table->hosts[i].stderr.Nbuffer) { fprintf (stderr, "(error writing log?)\n"); }
           FlushIOBuffer (&table->hosts[i].stderr);
           if (printHead) fprintf (stdout, "\n");
 …
         // read stdout
         int printHead;
         printHead = VERBOSE || host->stdout.Nbuffer;
+        printHead = VERBOSE || (host->stdout.Nbuffer > 0);
         EmptyIOBuffer (&host->stdout, 100, host->stdio[HOST_STDOUT]);
+        if (printHead) fprintf (stdout, "--- stdout from %s ---\n", host->hostname);
+        write (STDOUT_FILENO, host->stdout.buffer, host->stdout.Nbuffer);
+        if (printHead) fprintf (stdout, "--- stdout from %s --- (%d bytes, v2)\n", host->hostname, host->stdout.Nbuffer);
+        int Nout = write (STDOUT_FILENO, host->stdout.buffer, host->stdout.Nbuffer);
+        if (Nout != host->stdout.Nbuffer) { fprintf (stderr, "(error writing log?)\n"); }
         FlushIOBuffer (&host->stdout);
         if (printHead) fprintf (stdout, "\n");
         // read stderr
         printHead = VERBOSE || host->stderr.Nbuffer;
+        printHead = VERBOSE || (host->stderr.Nbuffer > 0);
         EmptyIOBuffer (&host->stderr, 100, host->stdio[HOST_STDERR]);
+        if (printHead) fprintf (stdout, "--- stderr from %s ---\n", host->hostname);
+        write (STDOUT_FILENO, host->stderr.buffer, host->stderr.Nbuffer);
+        if (printHead) fprintf (stdout, "--- stderr from %s --- (%d bytes, v2)\n", host->hostname, host->stderr.Nbuffer);
+        Nout = write (STDOUT_FILENO, host->stderr.buffer, host->stderr.Nbuffer);
+        if (Nout != host->stderr.Nbuffer) { fprintf (stderr, "(error writing log?)\n"); }
         FlushIOBuffer (&host->stderr);
         if (printHead) fprintf (stdout, "\n");

trunk/Ohana/src/libdvo/src/dbExtractAverages.c

-              r33649
+              r33963
 int dbExtractAveragesInit () {
   GetTimeFormat (&TimeReference, &TimeFormat);
+  return (TRUE);
+}
+int dbExtractAveragesInitAve () {
   haveGalactic = FALSE;
   haveEcliptic = FALSE;
 …
+          }
           break;
+        case MAG_STDEV:
+          value.Flt = PhotMstdev (field->photcode, average, secfilt);
+          break;
         case MAG_PHOT_FLAGS:
           if ((field->photcode->type == PHOT_REF) || (field->photcode->type == PHOT_DEP)) {
 …
           break;
+        case MAG_20:
+          value.Flt = PhotM20 (field->photcode, average, secfilt);
+          break;
+        case MAG_80:
+          value.Flt = PhotM80 (field->photcode, average, secfilt);
+          break;
+        case MAG_UC_DIST:
+          value.Flt = PhotUCdist (field->photcode, average, secfilt);
+          break;
+      }
       break;

trunk/Ohana/src/libdvo/src/dbFields.c

-              r33649
+              r33963
   if (!strcasecmp (string, "ncode"))     return (MAG_NCODE);
   if (!strcasecmp (string, "nphot"))     return (MAG_NPHOT);
+  if (!strcasecmp (string, "stdev"))     return (MAG_STDEV);
+  if (!strcasecmp (string, "20"))        return (MAG_20);
+  if (!strcasecmp (string, "80"))        return (MAG_80);
+  if (!strcasecmp (string, "ucdist"))    return (MAG_UC_DIST);
   return (MAG_NONE);
+}

trunk/Ohana/src/libdvo/src/dvo_catalog.c

-              r33649
+              r33963
   if (!check_file_access (catalog[0].filename, BACKUP, READWRITE, VERBOSE)) {
     if (VERBOSE) fprintf (stderr, "no permission to access %s\n", catalog[0].filename);
+    fprintf (stderr, "no permission to access %s\n", catalog[0].filename);
     return (FALSE);
+  }
 …
   switch (dvo_catalog_lock (catalog, catalog[0].lockmode)) {
   case DVO_CAT_OPEN_FAIL:
     if (VERBOSE) fprintf (stderr, "can't lock file %s\n", catalog[0].filename);
+    fprintf (stderr, "can't lock file %s\n", catalog[0].filename);
     return (FALSE);
   case DVO_CAT_OPEN_OK:
     if (!dvo_catalog_load (catalog, VERBOSE)) {
       if (VERBOSE) fprintf (stderr, "failure loading catalog\n");
+      fprintf (stderr, "failure loading catalog\n");
       return (FALSE);
+    }
     if (!dvo_catalog_check (catalog, Nsecfilt, TRUE)) {
       if (VERBOSE) fprintf (stderr, "can't reduce number of secondary filters\n");
+      fprintf (stderr, "can't reduce number of secondary filters\n");
       return (FALSE);
+    }
 …
+}
+// make a backup of this catalog (including the measure, secfilt, and missing tables as needed)
+int dvo_catalog_backup (Catalog *catalog, int primary) {
+  // skip empty cpt files
+  if (primary && !catalog->Naves_disk) {
+    return TRUE;
+  }
+  char tmpfilename[DVO_MAX_PATH];
+  int status = snprintf (tmpfilename, DVO_MAX_PATH, "%s~", catalog->filename);
+  if (status >= DVO_MAX_PATH) {
+    fprintf (stderr, "path name too long: %s\n", catalog->filename);
+    return FALSE;
+  }
+  // unlock the catalog here (closes file as well) (also closes subcat files Measure, Secfilt, Missing)
+  if (primary) {
+    status = dvo_catalog_unlock (catalog);
+    if (!status) {
+      fprintf (stderr, "failed to unlock catalog %s\n", catalog->filename);
+      return FALSE;
+    }
+  }
+  status = rename (catalog->filename, tmpfilename);
+  if (status) {
+    fprintf (stderr, "failed to rename catalog %s\n", catalog->filename);
+    return FALSE;
+  }
+  // keep the same lockmode
+  int lockmode = catalog->lockmode;
+  // lock the new catalog file here (re-opens file as well) (does NOT open/lock the subcat files)
+  status = dvo_catalog_lock (catalog, lockmode);
+  if (!status) {
+    fprintf (stderr, "failed to lock new catalog file %s\n", catalog->filename);
+    return FALSE;
+  }
+  if (catalog[0].catmode == DVO_MODE_SPLIT) {
+    if (catalog[0].measure_catalog != NULL) {
+      if (!dvo_catalog_backup (catalog[0].measure_catalog, FALSE)) {
+        return FALSE;
+      }
+    }
+    if (catalog[0].missing_catalog != NULL) {
+      if (!dvo_catalog_backup (catalog[0].missing_catalog, FALSE)) {
+        return FALSE;
+      }
+    }
+    if (catalog[0].secfilt_catalog != NULL) {
+      if (!dvo_catalog_backup (catalog[0].secfilt_catalog, FALSE)) {
+        return FALSE;
+      }
+    }
+  }
+  return TRUE;
+}
+// make a backup of this catalog (including the measure, secfilt, and missing tables as needed)
+int dvo_catalog_unlink_backup (Catalog *catalog, int primary) {
+  if (primary && !catalog->Naves_disk) {
+    // skip empty files (empty when read, but output may not be empty)
+    return TRUE;
+  }
+  char tmpfilename[DVO_MAX_PATH];
+  int status = snprintf (tmpfilename, DVO_MAX_PATH, "%s~", catalog->filename);
+  if (status >= DVO_MAX_PATH) {
+    fprintf (stderr, "path name too long: %s\n", catalog->filename);
+    return FALSE;
+  }
+  status = unlink (tmpfilename);
+  if (status) {
+    fprintf (stderr, "failed to unlink catalog %s\n", catalog->filename);
+    return FALSE;
+  }
+  if (catalog[0].catmode == DVO_MODE_SPLIT) {
+    if (catalog[0].measure_catalog != NULL) {
+      if (!dvo_catalog_unlink_backup (catalog[0].measure_catalog, FALSE)) {
+        return FALSE;
+      }
+    }
+    if (catalog[0].missing_catalog != NULL) {
+      if (!dvo_catalog_unlink_backup (catalog[0].missing_catalog, FALSE)) {
+        return FALSE;
+      }
+    }
+    if (catalog[0].secfilt_catalog != NULL) {
+      if (!dvo_catalog_unlink_backup (catalog[0].secfilt_catalog, FALSE)) {
+        return FALSE;
+      }
+    }
+  }
+  return TRUE;
+}

trunk/Ohana/src/libdvo/src/dvo_catalog_split.c

-              r33649
+              r33963
   // write PHU header
   if (!gfits_fwrite_header  (catalog->f, &catalog->header)) {
     fprintf (stderr, "can't write primary header");
+    fprintf (stderr, "can't write primary header\n");
     return (FALSE);
+  }
 …
   gfits_create_matrix (&catalog->header, &matrix);
   if (!gfits_fwrite_matrix  (catalog->f, &matrix)) {
     fprintf (stderr, "can't write primary matrix");
+    fprintf (stderr, "can't write primary matrix\n");
     gfits_free_matrix (&matrix);
     return (FALSE);
 …
   // write the table data
   if (!gfits_fwrite_ftable_range (catalog->f, ftable, start, Nrows, Ndisk, Ntotal)) {
     fprintf (stderr, "can't write table data");
+    fprintf (stderr, "can't write table data\n");
     return (FALSE);
+  }

trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_1.c

r33649	r33963
213	213	// added in PS1_V3
214	214	out[i].Map = 0;
	215	out[i].Mstdev = 0;
215	216	out[i].ubercalDist = 0;
216	217	}

trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c

r33649	r33963
207	207	// added in PS1_V3
208	208	out[i].Map = 0;
	209	out[i].Mstdev = 0;
209	210	out[i].ubercalDist = 0;
210	211	}

trunk/Ohana/src/libdvo/src/dvo_convert_PS1_REF.c

r33649	r33963
167	167	// added in PS1_V3
168	168	out[i].Map = 0;
	169	out[i].Mstdev = 0;
169	170	out[i].ubercalDist = 0;
170	171	}

trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c

r33649	r33963
211	211	// added in PS1_V3
212	212	out[i].Map = 0;
	213	out[i].Mstdev = 0;
213	214	out[i].ubercalDist = 0;
214	215	}

trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c

r33649	r33963
214	214	// added in PS1_V3
215	215	out[i].Map = 0;
	216	out[i].Mstdev = 0;
216	217	out[i].ubercalDist = 0;
217	218	}

trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c

-              r33649
+              r33963
     out[i].M_20        = in[i].M_20;
     out[i].M_80        = in[i].M_80;
+    out[i].Mstdev      = in[i].Mstdev;
     out[i].ubercalDist = in[i].ubercalDist;
+ }
 …
     out[i].M_20        = in[i].M_20;
     out[i].M_80        = in[i].M_80;
+    out[i].Mstdev      = in[i].Mstdev;
     out[i].ubercalDist = in[i].ubercalDist;
+  }

trunk/Ohana/src/libdvo/src/dvo_convert_elixir.c

r33649	r33963
222	222	// added in PS1_V3
223	223	out[i].Map = 0;
	224	out[i].Mstdev = 0;
224	225	out[i].ubercalDist = 0;
225	226	}

trunk/Ohana/src/libdvo/src/dvo_convert_loneos.c

r33649	r33963
215	215	// added in PS1_V3
216	216	out[i].Map = 0;
	217	out[i].Mstdev = 0;
217	218	out[i].ubercalDist = 0;
218	219	}

trunk/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_0.c

r33649	r33963
222	222	// added in PS1_V3
223	223	out[i].Map = 0;
	224	out[i].Mstdev = 0;
224	225	out[i].ubercalDist = 0;
225	226	}

trunk/Ohana/src/libdvo/src/dvo_convert_panstarrs_DEV_1.c

r33649	r33963
222	222	// added in PS1_V3
223	223	out[i].Map = 0;
	224	out[i].Mstdev = 0;
224	225	out[i].ubercalDist = 0;
225	226	}

trunk/Ohana/src/libdvo/src/dvo_photcode_ops.c

-              r33649
+              r33963
+}
+float PhotMstdev (PhotCode *code, Average *average, SecFilt *secfilt) {
+  int Ns;
+  float Mstdev;
+  if (code == NULL) return NAN;
+  Ns = photcodes[0].hashNsec[code[0].code];
+  Mstdev  = (Ns == -1) ? NAN : 0.001*secfilt[Ns].Mstdev;
+  return (Mstdev);
+}
+float PhotM20 (PhotCode *code, Average *average, SecFilt *secfilt) {
+  int Ns;
+  float M20;
+  if (code == NULL) return NAN;
+  Ns = photcodes[0].hashNsec[code[0].code];
+  M20  = (Ns == -1) ? NAN : 0.001*secfilt[Ns].M_20;
+  return (M20);
+}
+float PhotM80 (PhotCode *code, Average *average, SecFilt *secfilt) {
+  int Ns;
+  float M80;
+  if (code == NULL) return NAN;
+  Ns = photcodes[0].hashNsec[code[0].code];
+  M80  = (Ns == -1) ? NAN : 0.001*secfilt[Ns].M_80;
+  return (M80);
+}
+float PhotUCdist (PhotCode *code, Average *average, SecFilt *secfilt) {
+  int Ns;
+  float Muc;
+  if (code == NULL) return NAN;
+  Ns = photcodes[0].hashNsec[code[0].code];
+  Muc  = (Ns == -1) ? NAN : secfilt[Ns].ubercalDist;
+  return (Muc);
+}
 // XXX return NAN or NAN_S_SHORT? (secfilt->Xm is short)
 float PhotXm (PhotCode *code, Average *average, SecFilt *secfilt) {

trunk/Ohana/src/libdvo/src/flatcorr_io.c

r33649	r33963
185	185	float FlatCorrectionOffset (FlatCorrectionTable *flatcorr, int ID, int X, int Y) {
186	186
	187	if (!flatcorr) return 0.0;
	188
187	189	// validate the flat_id (not out of range?)
188	190	int seq = flatcorr->IDtoSeq[ID];

trunk/Ohana/src/libohana/include/ohana_sort.h

r27435	r33963
73	73	void fsortthree (float X, float Y, float *Z, int N);
74	74	void dsortthree (double X, double Y, double *Z, int N);
	75	void fsortfour (float X, float Y, float Z, float W, int N);
	76	void dsortfour (double X, double Y, double Z, double W, int N);
75	77
76	78	#endif

trunk/Ohana/src/libohana/src/rconnect.c

-              r33779
+              r33963
   pid = fork ();
+  if (pid < 0) {
+    perror ("fork:");
+    fprintf (stderr, "error forking child process\n");
+    exit (3);
+  }
   if (!pid) { /* must be child process */
     if (DEBUG) fprintf (stderr, "starting remote connection to %s...", hostname);

trunk/Ohana/src/libohana/src/sorts.c

-              r27435
+              r33963
+}
+void fsortfour (float *X, float *Y, float *Z, float *W, int N) {
+# define SWAPFUNC(A,B){ float tmp; \
+  tmp = X[A]; X[A] = X[B]; X[B] = tmp; \
+  tmp = Y[A]; Y[A] = Y[B]; Y[B] = tmp; \
+  tmp = Z[A]; Z[A] = Z[B]; Z[B] = tmp; \
+  tmp = W[A]; W[A] = W[B]; W[B] = tmp; \
+}
+# define COMPARE(A,B)(X[A] < X[B])
+  OHANA_SORT (N, COMPARE, SWAPFUNC);
+# undef SWAPFUNC
+# undef COMPARE
+}
+void dsortfour (double *X, double *Y, double *Z, double *W, int N) {
+# define SWAPFUNC(A,B){ double tmp; \
+  tmp = X[A]; X[A] = X[B]; X[B] = tmp; \
+  tmp = Y[A]; Y[A] = Y[B]; Y[B] = tmp; \
+  tmp = Z[A]; Z[A] = Z[B]; Z[B] = tmp; \
+  tmp = W[A]; W[A] = W[B]; W[B] = tmp; \
+}
+# define COMPARE(A,B)(X[A] < X[B])
+  OHANA_SORT (N, COMPARE, SWAPFUNC);
+# undef SWAPFUNC
+# undef COMPARE
+}

trunk/Ohana/src/libohana/src/string.c

-              r33646
+              r33963
     line[i] = c;
+  }
+  if (ferror(f)) {
+    perror("scan_line: ");
+    return EOF;
+  }
   line[i - 1] = 0;  /* this could make things crash! */

trunk/Ohana/src/opihi/cmd.astro

Property svn:mergeinfo set to

/branches/czw_branch/20101203/Ohana/src/opihi/cmd.astro	merged	eligible
/branches/eam_branches/ipp-20101103/Ohana/src/opihi/cmd.astro	merged	eligible
/branches/eam_branches/ipp-20101205/Ohana/src/opihi/cmd.astro	merged	eligible
/branches/eam_branches/ipp-20110213/Ohana/src/opihi/cmd.astro	merged	eligible
/branches/eam_branches/ipp-20110404/Ohana/src/opihi/cmd.astro	merged	eligible
/branches/eam_branches/ipp-20110505/Ohana/src/opihi/cmd.astro	merged	eligible
/branches/eam_branches/ipp-20110710/Ohana/src/opihi/cmd.astro	merged	eligible
/branches/eam_branches/ipp-20110906/Ohana/src/opihi/cmd.astro	merged	eligible
/branches/eam_branches/ipp-20111110/Ohana/src/opihi/cmd.astro	merged	eligible
/branches/eam_branches/ipp-20111122/Ohana/src/opihi/cmd.astro	merged	eligible
/branches/eam_branches/ipp-20120405/Ohana/src/opihi/cmd.astro	merged	eligible
/branches/haf_add201112/Ohana/src/opihi/cmd.astro	merged	eligible

trunk/Ohana/src/opihi/cmd.astro/Makefile

r33662	r33963
46	46	$(SRC)/ringflux.$(ARCH).o \
47	47	$(SRC)/petrosian.$(ARCH).o \
	48	$(SRC)/kronflux.$(ARCH).o \
48	49	$(SRC)/multifit.$(ARCH).o \
49	50	$(SRC)/objload.$(ARCH).o \

trunk/Ohana/src/opihi/cmd.astro/init.c

-              r33662
+              r33963
 int ringflux                PROTO((int, char **));
 int petrosian               PROTO((int, char **));
+int kronflux                PROTO((int, char **));
 int mkgauss                 PROTO((int, char **));
 int mksersic                PROTO((int, char **));
 …
   {1, "ringflux",    ringflux,     "mean flux in a ring"},
   {1, "petrosian",   petrosian,    "petrosian parameters given radial bins"},
+  {1, "kronflux",    kronflux,     "measure kronflux stats"},
   {1, "multifit",    multifit,     "fit multi-order spectrum"},
   {1, "objload",     objload,      "plot obj data on Ximage "},

trunk/Ohana/src/opihi/cmd.basic/Makefile

r31635	r33963
44	44	$(SRC)/quit.$(ARCH).o \
45	45	$(SRC)/run_for.$(ARCH).o \
	46	$(SRC)/run_foreach.$(ARCH).o \
46	47	$(SRC)/run_if.$(ARCH).o \
47	48	$(SRC)/run_while.$(ARCH).o \

trunk/Ohana/src/opihi/cmd.basic/init.c

-              r27255
+              r33963
 int quit            PROTO((int, char **));
 int run_for         PROTO((int, char **));
+int run_foreach     PROTO((int, char **));
 int run_if          PROTO((int, char **));
 int run_while       PROTO((int, char **));
 …
   {1, "quit",          quit,               "exit program *"},
   {1, "for",           run_for,            "for loop"},
+  {1, "foreach",       run_foreach,        "foreach loop"},
   {1, "if",            run_if,             "logical cases *"},
   {1, "while",         run_while,          "while loop"},

trunk/Ohana/src/opihi/cmd.basic/input.c

r7917	r33963
40	40	fclose (infile);
41	41
	42	if (!inlist.Nlines) gprint (GP_ERR, "WARNING: input file (%s) was empty\n", argv[1]);
	43
42	44	/* process this list */
43	45	status = exec_loop (&inlist);

trunk/Ohana/src/opihi/cmd.basic/run_for.c

-              r31667
+              r33963
 int run_for (int argc, char **argv) {
   int ThisList, depth, i, done, status, found, NLINES, j;
+  int N, ThisList, depth, i, done, status, found, NLINES, j;
   double start, end, delta;
   char *input, *endptr;
   double value, sign;
   Macro loop;
+  int Inclusive = FALSE;
+  if ((N = get_argument (argc, argv, "-incl"))) {
+    remove_argument (N, &argc, argv);
+    Inclusive = TRUE;
+  }
   if ((argc != 4) && (argc != 5)) {
 …
   REALLOCATE (loop.line, char *, MAX (loop.Nlines, 1));
+  if (Inclusive) end += 0.01*delta;
   status = TRUE;
   interrupt = FALSE;

trunk/Ohana/src/opihi/cmd.data/dot.c

-              r14181
+              r33963
 int dot (int argc, char **argv) {
   int kapa;
+  int kapa, N;
   Graphdata graphmode;
   float x, y;
   if (!style_args (&graphmode, &argc, argv, &kapa)) return FALSE;
+  /* FracPositions uses coordinates of 0-1 relative to axis range */
+  int FracPositions = FALSE;
+  if ((N = get_argument (argc, argv, "-frac"))) {
+    remove_argument (N, &argc, argv);
+    FracPositions = TRUE;
+  }
   if (argc != 3) {
 …
   x = atof(argv[1]);
   y = atof(argv[2]);
+  if (FracPositions) {
+    x =  x * (graphmode.xmax - graphmode.xmin) + graphmode.xmin;
+    y =  y * (graphmode.ymax - graphmode.ymin) + graphmode.ymin;
+  }
   /* set point style and errorbar mode (these are NOT sticky) */

trunk/Ohana/src/opihi/cmd.data/gridify.c

-              r31450
+              r33963
   int i, Nx, Ny, Xb, Yb, Normalize, N;
   float Xmin, Xmax, dX, Ymin, Ymax, dY, initValue;
   float *buf, *val;
+  float *buf, *val, *cnt;
   int *Nval;
-  Buffer *bf;
   Vector *vx, *vy, *vz;
   opihi_flt *x, *y, *z;
+  Buffer *bf = NULL;
+  Buffer *ct = NULL;
   Normalize = TRUE;
 …
     remove_argument (N, &argc, argv);
     Normalize = FALSE;
+    if ((ct = SelectBuffer (argv[N], ANYBUFFER, TRUE)) == NULL) return (FALSE);
+    remove_argument (N, &argc, argv);
+  }
 …
+  }
+  if (argc != 11) {
+    gprint (GP_ERR, "USAGE: gridify x y z buffer Xmin Xmax dX Ymin Ymax dY\n");
+  Xmin = Xmax = dX = NAN;
+  if ((N = get_argument (argc, argv, "-x"))) {
+    remove_argument (N, &argc, argv);
+    Xmin = atof (argv[N]);
+    remove_argument (N, &argc, argv);
+    Xmax = atof (argv[N]);
+    remove_argument (N, &argc, argv);
+    dX   = atof (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+  Ymin = Ymax = dY = NAN;
+  if ((N = get_argument (argc, argv, "-y"))) {
+    remove_argument (N, &argc, argv);
+    Ymin = atof (argv[N]);
+    remove_argument (N, &argc, argv);
+    Ymax = atof (argv[N]);
+    remove_argument (N, &argc, argv);
+    dY   = atof (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
+  if (argc != 5) {
+    gprint (GP_ERR, "USAGE: gridify x y z buffer [-x Xmin Xmax dX] [-y Ymin Ymax dY] [-init-value value] [-raw]\n");
     return (FALSE);
+  }
 …
   REQUIRE_VECTOR_FLT (vz, FALSE);
+  Xmin = atof (argv[5]);
+  Xmax = atof (argv[6]);
+  dX   = atof (argv[7]);
+  if (isnan(dX)) {
+    Xmin = 0;
+    Xmax = bf[0].matrix.Naxis[0];
+    dX = 1;
+  }
+  Ymin = atof (argv[8]);
+  Ymax = atof (argv[9]);
+  dY   = atof (argv[10]);
+  if (isnan(dY)) {
+    Ymin = 0;
+    Ymax = bf[0].matrix.Naxis[1];
+    dY = 1;
+  }
   Nx = (Xmax - Xmin) / dX + 1;
   Ny = (Ymax - Ymin) / dY + 1;
+  Nx = (Xmax - Xmin) / dX;
+  Ny = (Ymax - Ymin) / dY;
+  gfits_free_matrix (&bf[0].matrix);
+  gfits_free_header (&bf[0].header);
+  CreateBuffer (bf, Nx, Ny, -32, 0.0, 1.0);
+  strcpy (bf[0].file, "(empty)");
+  if ((Nx != bf[0].matrix.Naxis[0]) || (Ny != bf[0].matrix.Naxis[1])) {
+    gfits_free_matrix (&bf[0].matrix);
+    gfits_free_header (&bf[0].header);
+    CreateBuffer (bf, Nx, Ny, -32, 0.0, 1.0);
+    strcpy (bf[0].file, "(empty)");
+  }
   ALLOCATE (val, float, Nx*Ny);
 …
     Xb = (*x - Xmin) / dX;
     Yb = (*y - Ymin) / dY;
+    if (Xb < 0) continue;
+    if (Yb < 0) continue;
     if (Xb >= Nx) continue;
     if (Yb >= Ny) continue;
 …
+  }
+  if (!Normalize) {
+    gfits_free_matrix (&ct[0].matrix);
+    gfits_free_header (&ct[0].header);
+    CreateBuffer (ct, Nx, Ny, -32, 0.0, 1.0);
+    strcpy (ct[0].file, "(empty)");
+    buf = (float *) bf[0].matrix.buffer;
+    cnt = (float *) ct[0].matrix.buffer;
+    for (i = 0; i < Nx*Ny; i++) {
+      if (Nval[i] == 0) continue;
+      buf[i] = val[i];
+      cnt[i] = Nval[i];
+    }
+    free (val);
+    free (Nval);
+    return TRUE;
+  }
   buf = (float *) bf[0].matrix.buffer;
   for (i = 0; i < Nx*Ny; i++) {
     buf[i] = initValue;
+    if (Normalize) {
+      if (Nval[i] == 0) {
+        continue;
+      }
+      buf[i] = val[i] / Nval[i];
+    } else {
+      buf[i] = val[i];
+    }
+    if (Nval[i] == 0) continue;
+    buf[i] = val[i] / Nval[i];
+  }

trunk/Ohana/src/opihi/cmd.data/match2d.c

r30610	r33963
167	167	NMATCH = X1->Nelements;
168	168	ResetVector (index, OPIHI_INT, NMATCH);
	169
	170	for (i = 0; i < index->Nelements; i++) { index->elements.Int[i] = -1; }
169	171
170	172	ALLOCATE (N1, off_t, X1->Nelements);

trunk/Ohana/src/opihi/cmd.data/threshold.c

-              r33662
+              r33963
     x0 = isFltX ? vecx[0].elements.Flt[Nhi] : vecy[0].elements.Int[Nhi];
     x1 = isFltX ? vecx[0].elements.Flt[Nhi+1]   : vecy[0].elements.Int[Nhi+1];
+    Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
+    if (y0 == y1) {
+      Xvalue = 0.5*(x0 + x1);
+    } else {
+      Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
+    }
   } else {
     // interpolate to value:
 …
     x0 = isFltX ? vecx[0].elements.Flt[Nhi-1] : vecy[0].elements.Int[Nhi-1];
     x1 = isFltX ? vecx[0].elements.Flt[Nhi]   : vecy[0].elements.Int[Nhi];
+    Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
+    if (y0 == y1) {
+      Xvalue = 0.5*(x0 + x1);
+    } else {
+      Xvalue = (value - y0) * (x1 - x0) / (y1 - y0) + x0;
+    }
+  }

trunk/Ohana/src/opihi/cmd.data/vgroup.c

-              r25757
+              r33963
 # include "data.h"
+enum {USE_MEDIAN, USE_COUNT, USE_SUM, USE_MEAN};
 int vgroup (int argc, char **argv) {
 …
   // }
+  int mode = USE_MEDIAN;
+  if ((N = get_argument (argc, argv, "-sum"))) {
+    remove_argument (N, &argc, argv);
+    mode = USE_SUM;
+  }
+  if ((N = get_argument (argc, argv, "-mean"))) {
+    remove_argument (N, &argc, argv);
+    mode = USE_MEAN;
+  }
+  float binsize = NAN;
+  if ((N = get_argument (argc, argv, "-binsize"))) {
+    remove_argument (N, &argc, argv);
+    binsize = atof(argv[N]);
+    remove_argument (N, &argc, argv);
+  }
   if (argc != 5) {
     gprint (GP_ERR, "USAGE: vbin <xin> <yin> <xout> <yout>\n");
+    gprint (GP_ERR, "USAGE: vgroup <xin> <yin> <xout> <yout>\n");
     gprint (GP_ERR, " group x,y values in bins defined by <xout>\n");
+    gprint (GP_ERR, " by default, yout has the median of the associated input values\n");
+    gprint (GP_ERR, " use -sum to add values in the bin\n");
+    gprint (GP_ERR, " use <yin> = histogram count matching values\n");
+    gprint (GP_ERR, " use -binsize to specify a fixed bin width (<xout> will define the bin center)\n");
     return (FALSE);
+  }
   if ((xin  = SelectVector (argv[1], OLDVECTOR, TRUE)) == NULL) return (FALSE);
-  if ((yin  = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE);
   if ((xout = SelectVector (argv[3], OLDVECTOR, TRUE)) == NULL) return (FALSE);
   if ((yout = SelectVector (argv[4], ANYVECTOR, TRUE)) == NULL) return (FALSE);
+  yin = NULL;
+  // this should conflict with the -sum option...
+  if (!strcmp(argv[2], "histogram")) {
+    mode = USE_COUNT;
+  } else {
+    if ((yin  = SelectVector (argv[2], OLDVECTOR, TRUE)) == NULL) return (FALSE);
+  }
   // re-binning creates a float vector
 …
   for (i = 0; i < xout[0].Nelements - 1; i++) {
+    xmin = xout[0].elements.Flt[i];
+    xmax = xout[0].elements.Flt[i+1];
+    if (isnan(binsize)) {
+      xmin = xout[0].elements.Flt[i];
+      xmax = xout[0].elements.Flt[i+1];
+    } else {
+      xmin = xout[0].elements.Flt[i] - 0.5*binsize;
+      xmax = xout[0].elements.Flt[i] + 0.5*binsize;
+    }
     N = 0;
 …
       if (xin[0].elements.Flt[j] < xmin) continue;
       if (xin[0].elements.Flt[j] > xmax) continue;
+      values[N] = yin[0].elements.Flt[j];
+      if (yin) {
+        values[N] = yin[0].elements.Flt[j];
+      }
       N++;
+    }
+    dsort (values, N);
+    if (N > 1) {
+      sum = (N % 2) ? values[(int)(0.5*N)] : 0.5*(values[N/2] + values[N/2 + 1]);
+    } else {
+      sum = values[0];
+    sum = NAN;
+    switch (mode) {
+      case USE_MEDIAN:
+        dsort (values, N);
+        if (N > 1) {
+          sum = (N % 2) ? values[(int)(0.5*N)] : 0.5*(values[N/2] + values[N/2 + 1]);
+        } else {
+          sum = values[0];
+        }
+        break;
+      case USE_SUM:
+        sum = 0.0;
+        for (j = 0; j < N; j++) {
+          sum += values[j];
+        }
+        break;
+      case USE_COUNT:
+        sum = N;
+        break;
+      case USE_MEAN:
+        sum = 0.0;
+        for (j = 0; j < N; j++) {
+          sum += values[j];
+        }
+        sum /= N;
+        break;
+    }
-    // measure the stat for this bin
-    // sum = 0.0;
-    // for (j = 0; j < N; j++) {
-    //   sum += values[j];
-    // }
-    // sum /= N;
     yout[0].elements.Flt[i] = sum;
+  }

trunk/Ohana/src/opihi/cmd.data/vstats.c

r32632	r33963
144	144	}
145	145	}
146		stdev = sqrt (var / N);
	146	stdev = sqrt (var / (N - 1));
147	147
148	148	Nmode = 0;

trunk/Ohana/src/opihi/cmd.data/write_vectors.c

-              r33662
+              r33963
   FITS = NULL;
   if ((N = get_argument (argc, argv, "-fits"))) {
+    if (format) {
+      gprint (GP_ERR, "ERROR: do not mix -fits and -format\n");
+      free (format);
+      return (FALSE);
+    }
     remove_argument (N, &argc, argv);
     FITS = strcreate (argv[N]);
     remove_argument (N, &argc, argv);
+  }
+  /* option generate a FITS output table */
+  int CSV = FALSE;
+  if ((N = get_argument (argc, argv, "-csv"))) {
+    if (format) {
+      gprint (GP_ERR, "ERROR: do not mix -csv and -format\n");
+      free (format);
+      return (FALSE);
+    }
+    if (FITS) {
+      gprint (GP_ERR, "ERROR: do not mix -csv and -fits\n");
+      free (FITS);
+      return (FALSE);
+    }
+    remove_argument (N, &argc, argv);
+    CSV = TRUE;
+  }
 …
     remove_argument (N, &argc, argv);
     append = TRUE;
+  }
+  int ADD_HEADER = FALSE;
+  if ((N = get_argument (argc, argv, "-header"))) {
+    remove_argument (N, &argc, argv);
+    ADD_HEADER = TRUE;
+  }
 …
   /* default output format */
+  if (ADD_HEADER) {
+    for (j = 0; j < Nvec; j++) {
+      if (CSV) {
+        fprintf (f, "%s,", vec[j][0].name);
+      } else {
+        if (j == 0) fprintf (f, "# ");
+        fprintf (f, "%s ", vec[j][0].name);
+      }
+    }
+    fprintf (f, "\n");
+  }
+  /* default output format */
   if (format == (char *) NULL) {
     for (i = 0; i < vec[0][0].Nelements; i++) {
       for (j = 0; j < Nvec; j++) {
         if (vec[j][0].type == OPIHI_FLT) {
+          fprintf (f, "%.12g ", vec[j][0].elements.Flt[i]);
+          if (CSV) {
+            fprintf (f, "%.12g,", vec[j][0].elements.Flt[i]);
+          } else {
+            fprintf (f, "%.12g ", vec[j][0].elements.Flt[i]);
+          }
         } else {
+          fprintf (f, "%d ", vec[j][0].elements.Int[i]);
+          if (CSV) {
+            fprintf (f, "%d,", vec[j][0].elements.Int[i]);
+          } else {
+            fprintf (f, "%d ", vec[j][0].elements.Int[i]);
+          }
+        }
+      }

trunk/Ohana/src/opihi/dvo/avextract.c

-              r33662
+              r33963
+  }
+  int VERBOSE2 = FALSE;
+  if ((N = get_argument (argc, argv, "-vv"))) {
+    remove_argument (N, &argc, argv);
+    VERBOSE = TRUE;
+    VERBOSE2 = TRUE;
+  }
   int PARALLEL = FALSE;
   if ((N = get_argument (argc, argv, "-parallel"))) {
 …
       return FALSE;
+    }
+  }
+  // this is used to NOT save the results in the results file
+  // use this option when mextract is used in a script which does its
+  // own job of packaging the results
+  int SKIP_RESULTS = FALSE;
+  if ((N = get_argument (argc, argv, "-skip-results"))) {
+    remove_argument (N, &argc, argv);
+    SKIP_RESULTS = TRUE;
+  }
 …
     goto escape;
+  }
+  // init locally static variables (time refs)
+  dbExtractAveragesInit ();
   // command-line is of the form: avextract field,field, field [where (field op value)...]
 …
     catalog.filename = (HOST_ID || PARALLEL_LOCAL) ? hostfile : skylist[0].filename[i];
     catalog.catflags = LOAD_AVES | LOAD_SECF;
+    if (needMeasures) {
+      catalog.catflags |= LOAD_MEAS;
+    }
+    catalog.catflags |= needMeasures ? LOAD_MEAS : SKIP_MEAS;
     catalog.Nsecfilt = 0;
 …
     // an error exit status here is a significant error
     if (!dvo_catalog_open (&catalog, NULL, FALSE, "r")) {
+    if (!dvo_catalog_open (&catalog, NULL, VERBOSE2, "r")) {
       gprint (GP_ERR, "ERROR: failure to open catalog file %s\n", catalog.filename);
       exit (2);
 …
       m = catalog.average[j].measureOffset;
       // reset counters for saved fields, extract fields
+      dbExtractAveragesInit ();
+      dbExtractAveragesInitAve ();  // reset counters for saved fields (costs very little)
       for (n = 0; n < Nfields; n++) {
         values[n] = dbExtractAverages (&catalog.average[j], &catalog.secfilt[j*Nsecfilt], &catalog.measure[m], &fields[n]);
 …
   // write vectors to a table (this is used by parallel dvo operations, but can be used elsewhere)
   if (RESULT_FILE) {
+  if (RESULT_FILE && !SKIP_RESULTS) {
     int status = WriteVectorTableFITS (RESULT_FILE, "RESULT", vec, Nreturn, FALSE, NULL);
+    if (!status) goto escape;
+    if (!status) {
+      goto escape;
+    }
+  }

trunk/Ohana/src/opihi/dvo/avmatch.c

-              r33662
+              r33963
   if (!InitPhotcodes ()) goto escape;
   Nsecfilt = GetPhotcodeNsecfilt ();
+  // init locally static variables (time refs)
+  dbExtractAveragesInit ();
   // this does all the work of re-packaging the command, calling it on the remote machines, then loading in the results
 …
+  }
+  // check the requested fields : are all average/secfilt entries, or do we need measures?
+  int needMeasures = FALSE;
+  for (i = 0; !needMeasures && (i < Nfields); i++) {
+    if (fields[i].magMode == MAG_NONE) continue;
+    if (fields[i].photcode == NULL) continue; // assert this?
+    if (fields[i].photcode[0].type == PHOT_REF) needMeasures = TRUE;
+    if (fields[i].photcode[0].type == PHOT_DEP) needMeasures = TRUE;
+  }
   /* load regions which contain all supplied RA,DEC coordinates */
   if ((skylist = SelectRegionsByCoordVectors (RAvec, DECvec)) == NULL) goto escape;
 …
     snprintf (hostfile, 1024, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
     catalog.filename = HOST_ID ? hostfile : skylist[0].filename[i];
+    catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF;
+    catalog.catflags = LOAD_AVES | LOAD_SECF;
+    catalog.catflags |= needMeasures ? LOAD_MEAS : SKIP_MEAS;
     catalog.Nsecfilt = 0;
 …
       // reset counters for saved fields, extract fields
       dbExtractAveragesInit ();
+      dbExtractAveragesInitAve ();
       for (n = 0; n < Nfields; n++) {
         values[n] = dbExtractAverages (&catalog.average[Ncat], &catalog.secfilt[Ncat*Nsecfilt], &catalog.measure[m], &fields[n]);

trunk/Ohana/src/opihi/dvo/dvo_client.c

-              r33662
+              r33963
   int status = input (2, &argv[0]);
+  if (!status) exit (2);
+  if (!status) {
+    fprintf (stderr, "WARNING: exit status 2 %d (%s)\n", HOST_ID, HOSTDIR);
+    exit (2);
+  }
   exit (0);

trunk/Ohana/src/opihi/dvo/dvo_host_utils.c

-              r33662
+              r33963
 # define PARALLEL_MANUAL 0
 # define PARALLEL_SERIAL 0
+# define MAX_PATH_LENGTH 1024
+int HostTableLaunchJobs (HostTable *table, char *basecmd, char *options) {
+# define DVO_MAX_PATH 1024
+# define DIE(WHO,MSG) { perror(WHO); myAbort(MSG); }
+int HostTableLaunchJobs (HostTable *table, char *basecmd, char *options, int VERBOSE) {
   char uniquer[12];
 …
   snprintf (uniquer, 12, "%05d.%05d", PID, TIME % 100000);
+  int top_status = TRUE;
   int i;
   for (i = 0; i < table->Nhosts; i++) {
     // ensure that the paths are absolute path names
     char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH);
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
     free (table->hosts[i].pathname);
     table->hosts[i].pathname = tmppath;
 …
     // need to save the results filename with the uniquer
     // XXX a bit of a waste (but only 1024 * 60 bytes or so
     ALLOCATE (table->hosts[i].results, char, MAX_PATH_LENGTH);
     snprintf (table->hosts[i].results, MAX_PATH_LENGTH, "%s/dvo.results.%s.fits", table->hosts[i].pathname, uniquer);
     char commandBase[MAX_PATH_LENGTH];
     snprintf (commandBase, MAX_PATH_LENGTH, "dvo.command.%s.txt", uniquer);
     char *commandFile = abspath(commandBase, MAX_PATH_LENGTH);
+    ALLOCATE (table->hosts[i].results, char, DVO_MAX_PATH);
+    snprintf (table->hosts[i].results, DVO_MAX_PATH, "%s/dvo.results.%s.fits", table->hosts[i].pathname, uniquer);
+    char commandBase[DVO_MAX_PATH];
+    snprintf (commandBase, DVO_MAX_PATH, "dvo.command.%s.txt", uniquer);
+    char *commandFile = abspath(commandBase, DVO_MAX_PATH);
     FILE *f = fopen (commandFile, "w");
     fprintf (f, "%s\n", basecmd);
+    fclose (f);
+    if (fflush (f)) DIE("flush", "failed to flush");
+    int fd = fileno (f);
+    if (fsync (fd)) DIE("fsync", "failed to fsync");
+    if (fclose (f)) DIE("close", "failed to close");
+    // force NFS to write the file to disk
+    int state;
+    f = fsetlockfile (commandFile, 0.5, LCK_XCLD, &state);
+    fclearlockfile (commandFile, f, LCK_XCLD, &state);
     char command[1024];
 …
     free (commandFile);
     if (DEBUG || PARALLEL_MANUAL) fprintf (stderr, "command: %s\n", command);
+    if (VERBOSE) gprint (GP_ERR, "command: %s\n", command);
     if (PARALLEL_MANUAL) {
 …
       int status = system (command);
       if (status) {
         fprintf (stderr, "ERROR running relphot_client\n");
         exit (2);
+        gprint (GP_ERR, "ERROR running relphot_client\n");
+        top_status = FALSE;
+      }
     } else {
 …
       int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE);
       if (!pid) {
+        if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
+        exit (1);
+        gprint (GP_ERR, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo);
+        top_status = FALSE;
+        continue;
+      }
       table->hosts[i].pid = pid; // save for future reference
+    }
+  }
   return TRUE;
+  return top_status;
+}
 …
+  }
   char *CATDIR = abspath (tmppath, MAX_PATH_LENGTH);
+  char *CATDIR = abspath (tmppath, DVO_MAX_PATH);
   if (!CATDIR) {
     gprint (GP_ERR, "failed to make an absolute path from %s (too long)\n", tmppath);
 …
   // launch this command remotely
   HostTableLaunchJobs (table, basecmd, options);
+  HostTableLaunchJobs (table, basecmd, options, VERBOSE);
   free (options);
   free (basecmd);
 …
   return TRUE;
+}
+// re-gather the remote results files: this can be used in case one of the clients failed,
+// and has since been re-run
+int HostTableReloadResults (char *uniquer, int VERBOSE) {
+  int i;
+  // load the list of hosts
+  SkyTable *sky = GetSkyTable();
+  if (!sky) {
+    gprint (GP_ERR, "failed to load sky table for database\n");
+    return FALSE;
+  }
+  char *CATDIR = GetCATDIR ();
+  if (!CATDIR) {
+    gprint (GP_ERR, "failed to get CATDIR for database\n");
+    return FALSE;
+  }
+  HostTable *table = HostTableLoad (CATDIR, sky->hosts);
+  if (!table) {
+    gprint (GP_ERR, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
+    return FALSE;
+  }
+  // load fields from file
+  int    Nvec = 0;
+  Vector **vec = NULL;
+  for (i = 0; i < table->Nhosts; i++) {
+    // ensure that the paths are absolute path names
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
+    free (table->hosts[i].pathname);
+    table->hosts[i].pathname = tmppath;
+    // need to save the results filename with the uniquer
+    // XXX a bit of a waste (but only 1024 * 60 bytes or so
+    ALLOCATE (table->hosts[i].results, char, DVO_MAX_PATH);
+    snprintf (table->hosts[i].results, DVO_MAX_PATH, "%s/dvo.results.%s.fits", table->hosts[i].pathname, uniquer);
+    int    Ninvec = 0;
+    Vector **invec = ReadVectorTableFITS (table->hosts[i].results, "RESULT", &Ninvec);
+    if (!invec) {
+      // failed to read the file, now what?
+      gprint (GP_ERR, "failed to read remote result file : %s\n", table->hosts[i].results);
+      free (table->hosts[i].results);
+      table->hosts[i].results = NULL;
+      continue;
+    }
+    free (table->hosts[i].results);
+    table->hosts[i].results = NULL;
+    // fprintf (stderr, "%s : %d\n", table->hosts[i].pathname, invec[0]->Nelements);
+    vec = MergeVectors (vec, &Nvec, invec, Ninvec);
+    if (vec != invec) {
+      FreeVectorArray (invec, Ninvec);
+    }
+  }
+  for (i = 0; i < Nvec; i++) {
+    AssignVector (vec[i], vec[i]->name, ANYVECTOR, TRUE);
+  }
+  free (vec);
+  free (table);
+  return TRUE;
+}

trunk/Ohana/src/opihi/dvo/gstar.c

-              r33662
+              r33963
   /* lock, load, unlock catalog */
+  catalog.catflags = GetMeasures ? LOAD_AVES | LOAD_MEAS | LOAD_SECF : LOAD_AVES | LOAD_SECF;
+  catalog.catflags = LOAD_AVES | LOAD_SECF;
+  catalog.catflags |= GetMeasures ? LOAD_MEAS : SKIP_MEAS;
   catalog.Nsecfilt = 0;

trunk/Ohana/src/opihi/dvo/hosts.c

-              r33662
+              r33963
 # include "dvoshell.h"
 # include <glob.h>
 # define MAX_PATH_LENGTH 1024
+# define DVO_MAX_PATH 1024
 // functions to manage the remote hosts
 …
     for (i = 0; i < table->Nhosts; i++) {
       pglob.gl_offs = 0;
       char name[MAX_PATH_LENGTH];
       snprintf (name, MAX_PATH_LENGTH, "%s/dvo.results.%05d.*.fits", table->hosts[i].pathname, PID);
+      char name[DVO_MAX_PATH];
+      snprintf (name, DVO_MAX_PATH, "%s/dvo.results.%05d.*.fits", table->hosts[i].pathname, PID);
       if (VERBOSE) gprint (GP_ERR, "checking %s\n", name);
       glob (name, 0, NULL, &pglob);

trunk/Ohana/src/opihi/dvo/mextract.c

r33662	r33963
189	189	}
190	190
191		dbExtractMeasuresInitAve (); // reset counters for saved fields (costs very little
	191	dbExtractMeasuresInitAve (); // reset counters for saved fields (costs very little)
192	192
193	193	for (k = 0; (k < catalog.average[j].Nmeasure); k++, m++) {

trunk/Ohana/src/opihi/dvo/remote.c

-              r33662
+              r33963
 # include "dvoshell.h"
 # include <glob.h>
 # define MAX_PATH_LENGTH 1024
+# define DVO_MAX_PATH 1024
 // functions to manage the remote hosts
 …
   int N;
+  int VERBOSE = FALSE;
+  if ((N = get_argument (argc, argv, "-v"))) {
+    remove_argument (N, &argc, argv);
+    VERBOSE = TRUE;
+  }
   if (argc < 2) {
     gprint (GP_ERR, "USAGE: remote (command)\n");
 …
     gprint (GP_ERR, "  options:\n");
     gprint (GP_ERR, "  -v : verbose mode:\n");
+    gprint (GP_ERR, "OR:    remote -reload (uniquer)\n");
     return FALSE;
+  }
 …
   // with a set of vectors to load.
+  int VERBOSE = FALSE;
+  if ((N = get_argument (argc, argv, "-v"))) {
+  if ((N = get_argument (argc, argv, "-reload"))) {
     remove_argument (N, &argc, argv);
+    VERBOSE = TRUE;
+    if (argc != 2) {
+      gprint (GP_ERR, "USAGE: remote -reload (uniquer)\n");
+      gprint (GP_ERR, " (uniquer) is the element in the middle of the results file\n");
+      gprint (GP_ERR, " eg: dvo.results.XXXXX.YYYYY.fits\n");
+      return FALSE;
+    }
+    int status = HostTableReloadResults (argv[1], VERBOSE);
+    return status;
+  }

trunk/Ohana/src/opihi/dvo/skycoverage.c

-              r33662
+              r33963
 // enum to define possible modes
 enum {COVERAGE, DENSITY, MIN_UBERCAL, MIN_DMAG_SYS, MIN_MCAL, MAX_MCAL};
+enum {COVERAGE, DENSITY, MIN_UBERCAL, MIN_DMAG_SYS, MIN_MCAL, MAX_MCAL, MIN_TIME, MAX_TIME};
 int wordhash (char *word);
 …
   PhotCode *PhotcodeValue;
+  time_t TimeReference;
+  int TimeFormat;
+  GetTimeFormat (&TimeReference, &TimeFormat);
   WITH_MOSAIC = FALSE;
   if ((N = get_argument (argc, argv, "+mosaic"))) {
 …
     remove_argument (N, &argc, argv);
     mode = MIN_UBERCAL;
+  }
+  if ((N = get_argument (argc, argv, "-min-time"))) {
+    remove_argument (N, &argc, argv);
+    mode = MIN_TIME;
+  }
+  if ((N = get_argument (argc, argv, "-max-time"))) {
+    remove_argument (N, &argc, argv);
+    mode = MAX_TIME;
+  }
   if ((N = get_argument (argc, argv, "-min-dmag-sys"))) {
 …
           case MIN_DMAG_SYS:
           case MIN_MCAL:
+            V[ys*Nx + xs] = 1E6;
+          case MIN_TIME:
+            V[ys*Nx + xs] = 1E9;
             break;
           case MAX_MCAL:
+            V[ys*Nx + xs] = -1E6;
+          case MAX_TIME:
+            V[ys*Nx + xs] = -1E9;
             break;
+        }
 …
           case MIN_MCAL:
           case MAX_MCAL:
+          case MIN_TIME:
+          case MAX_TIME:
             V[ys*Nx + xs] = NAN;
             break;
 …
               V[ys*Nx + xs] = MAX(V[ys*Nx + xs], image[i].Mcal);
               break;
+            case MIN_TIME: {
+              double timeVal = TimeValue (image[i].tzero, TimeReference, TimeFormat);
+              V[ys*Nx + xs] = MIN(V[ys*Nx + xs], timeVal);
+              break; }
+            case MAX_TIME: {
+              double timeVal = TimeValue (image[i].tzero, TimeReference, TimeFormat);
+              V[ys*Nx + xs] = MAX(V[ys*Nx + xs], timeVal);
+              break; }
+          }
+        }

trunk/Ohana/src/opihi/include/dvoshell.h

-              r33662
+              r33963
 dbValue      dbExtractImages        PROTO((Image *image, off_t Nimage, off_t N, dbField *field));
 int          HostTableLaunchJobs    PROTO((HostTable *table, char *basecmd, char *options));
+int          HostTableLaunchJobs    PROTO((HostTable *table, char *basecmd, char *options, int VERBOSE));
 int          HostTableParallelOps   PROTO((int argc, char **argv, char *ResultFile, int Nelements, int VERBOSE));
+# endif
+int          HostTableReloadResults PROTO((char *uniquer, int VERBOSE));
+# endif // DVOSHELL_H

trunk/Ohana/src/opihi/lib.shell/ListOps.c

-              r33662
+              r33963
   status = !strcmp (comm, "for");
+  free (comm);
+  return (status);
+}
+int is_foreach_loop (char *line) {
+  int status;
+  char *comm;
+  comm = thisword (line);
+  if (comm == (char *) NULL) return (FALSE);
+  status = !strcmp (comm, "foreach");
   free (comm);
   return (status);
 …
   status |= is_macro_create (line);
   status |= is_for_loop (line);
+  status |= is_foreach_loop (line);
   status |= is_list_data (line);
   status |= is_loop (line);

trunk/Ohana/src/opihi/lib.shell/VectorIO.c

r33662	r33963
20	20	}
21	21	if (f == (FILE *) NULL) {
22		gprint (GP_ERR, "can't open file for write~~\n"~~);
	22	gprint (GP_ERR, "can't open file for write : %s\n", filename);
23	23	return (FALSE);
24	24	}

trunk/Ohana/src/opihi/lib.shell/startup.c

r27998	r33963
138	138	if (is_script) {
139	139	/* first argument in input script, rest are argv */
	140	set_str_variable ("SCRIPT_NAME", argv[1]);
140	141	list[Nlist] = strcreate (argv[1]);
141	142	Nlist ++;

trunk/Ohana/src/photdbc/include/dvodist.h

r33655	r33963
27	27	char *dstHostname;
28	28
29		# define ~~MAX_PATH_LENG~~TH 1024
	29	# define DVO_MAX_PATH 1024
30	30
31	31	void usage();

trunk/Ohana/src/photdbc/include/photdbc.h

-              r33655
+              r33963
 // need to get RADIUS from Config
 # define MAX_PATH_LENGTH 1024
+# define DVO_MAX_PATH 1024
 /* global variables */
 …
 double UNIQ_RADIUS;
 double DMCAL_MIN;;
 char   ImageCat[MAX_PATH_LENGTH];
 char   GSCFILE[MAX_PATH_LENGTH];
 char   CATDIR[MAX_PATH_LENGTH];
+char   ImageCat[DVO_MAX_PATH];
+char   GSCFILE[DVO_MAX_PATH];
+char   CATDIR[DVO_MAX_PATH];
 char  *CATMODE;    /* raw, mef, split, mysql */
 char  *CATFORMAT;  /* internal, elixir, loneos, panstarrs */
 char   PhotCodeFile[MAX_PATH_LENGTH];
+char   PhotCodeFile[DVO_MAX_PATH];
 double RMIN;

trunk/Ohana/src/photdbc/src/ConfigInit.c

r33655	r33963
56	56
57	57	WarnConfig (config, "CATDIR", "%s", 0, CATDIR);
58		char *tmpcatdir = abspath (CATDIR, ~~MAX_PATH_LENG~~TH);
	58	char *tmpcatdir = abspath (CATDIR, DVO_MAX_PATH);
59	59	strcpy (CATDIR, tmpcatdir);
60	60	free (tmpcatdir);

trunk/Ohana/src/photdbc/src/md5_ops.c

-              r33655
+              r33963
   IOBuffer buffer;
   char line[MAX_PATH_LENGTH];
   int Nline = snprintf (line, MAX_PATH_LENGTH, "md5sum %s\n", filename);
   assert (Nline < MAX_PATH_LENGTH);
+  char line[DVO_MAX_PATH];
+  int Nline = snprintf (line, DVO_MAX_PATH, "md5sum %s\n", filename);
+  assert (Nline < DVO_MAX_PATH);
+  write (host->stdio[0], line, Nline);
+  int Nout = write (host->stdio[0], line, Nline);
+  if (Nout != Nline) {
+    fprintf (stderr, "failed to execute md5sum\n");
+    return FALSE;
+  }
   InitIOBuffer (&buffer, 100);
 …
   fprintf (stderr, "buffer: %s\n", buffer.buffer);
   char result[MAX_PATH_LENGTH], fileout[MAX_PATH_LENGTH];
+  char result[DVO_MAX_PATH], fileout[DVO_MAX_PATH];
   int Nscan = sscanf (buffer.buffer, "%s %s", result, fileout);
   assert (Nscan == 2);
 …
   IOBuffer buffer, stdout_buf, stderr_buf;
   char line[MAX_PATH_LENGTH];
   int Nline = snprintf (line, MAX_PATH_LENGTH, "job md5sum %s\n", filename);
   assert (Nline < MAX_PATH_LENGTH);
+  char line[DVO_MAX_PATH];
+  int Nline = snprintf (line, DVO_MAX_PATH, "job md5sum %s\n", filename);
+  assert (Nline < DVO_MAX_PATH);
   // fprintf (stderr, "command: %s\n", line);
 …
   // fprintf (stderr, "result from md5sum: %s\n", stdout_buf.buffer);
   char result[MAX_PATH_LENGTH], fileout[MAX_PATH_LENGTH];
+  char result[DVO_MAX_PATH], fileout[DVO_MAX_PATH];
   int Nscan = sscanf (stdout_buf.buffer, "%s %s", result, fileout);
   if (Nscan != 2) {

trunk/Ohana/src/photdbc/src/photdbc_catalogs.c

-              r33655
+              r33963
     // set the parameters which guide catalog open/load/create
     char hostfile[MAX_PATH_LENGTH];
     snprintf (hostfile, MAX_PATH_LENGTH, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
+    char hostfile[DVO_MAX_PATH];
+    snprintf (hostfile, DVO_MAX_PATH, "%s/%s.cpt", HOSTDIR, skylist[0].regions[i]->name);
     incatalog.filename  = hostID ? hostfile : skylist[0].filename[i];
     incatalog.Nsecfilt = GetPhotcodeNsecfilt ();
 …
     // create output catalog filename
     char outfile[MAX_PATH_LENGTH];
     snprintf (outfile, MAX_PATH_LENGTH, "%s/%s.cpt", outroot, skylist[0].regions[i]->name);
+    char outfile[DVO_MAX_PATH];
+    snprintf (outfile, DVO_MAX_PATH, "%s/%s.cpt", outroot, skylist[0].regions[i]->name);
     outcatalog.filename = outfile;
     if (outcatalog.filename == NULL) Shutdown ("error with input catalog name");
 …
     // ensure that the paths are absolute path names
     char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH);
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
     free (table->hosts[i].pathname);
     table->hosts[i].pathname = tmppath;
     // ensure that the paths are absolute path names
     char *tmproot = abspath (outroot, MAX_PATH_LENGTH);
+    char *tmproot = abspath (outroot, DVO_MAX_PATH);
     // options / arguments that can affect relastro_client -update-objects:
     char command[MAX_PATH_LENGTH];
     snprintf (command, MAX_PATH_LENGTH, "photdbc_client %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -D NMEAS_MIN %d -D NMEAS_MIN_FILTERED %d -D AVE_SIGMA_LIM %f -D SIGMA_MAX %f",
+    char command[DVO_MAX_PATH];
+    snprintf (command, DVO_MAX_PATH, "photdbc_client %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -D NMEAS_MIN %d -D NMEAS_MIN_FILTERED %d -D AVE_SIGMA_LIM %f -D SIGMA_MAX %f",
               tmproot, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname,
               REGION.Rmin, REGION.Rmax, REGION.Dmin, REGION.Dmax,
 …
       );
     char tmpline[MAX_PATH_LENGTH];
     if (VERBOSE)            { snprintf (tmpline, MAX_PATH_LENGTH, "%s -v",                command);                             strcpy (command, tmpline); }
     if (ExcludeByInstMag)   { snprintf (tmpline, MAX_PATH_LENGTH, "%s -instmag %f %f",    command, INST_MAG_MIN, INST_MAG_MAX); strcpy (command, tmpline); }
     if (ExcludeByMinSigma)  { snprintf (tmpline, MAX_PATH_LENGTH, "%s -min-sigma %f",     command, SIGMA_MIN_KEEP);             strcpy (command, tmpline); }
     if (ExcludeByMaxMinMag) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -maxminmag %f",     command, MAX_MIN_MAG);                strcpy (command, tmpline); }
     if (PHOTCODE_DROP_LIST) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -photcode-drop %s", command, PHOTCODE_DROP_LIST);         strcpy (command, tmpline); }
     if (PHOTCODE_SKIP_LIST) { snprintf (tmpline, MAX_PATH_LENGTH, "%s -photcode-skip %s", command, PHOTCODE_SKIP_LIST);         strcpy (command, tmpline); }
     if (CATFORMAT)          { snprintf (tmpline, MAX_PATH_LENGTH, "%s -set-format %s",    command, CATFORMAT);                  strcpy (command, tmpline); }
     if (CATMODE)            { snprintf (tmpline, MAX_PATH_LENGTH, "%s -set-mode %s",      command, CATMODE);                    strcpy (command, tmpline); }
+    char tmpline[DVO_MAX_PATH];
+    if (VERBOSE)            { snprintf (tmpline, DVO_MAX_PATH, "%s -v",                command);                             strcpy (command, tmpline); }
+    if (ExcludeByInstMag)   { snprintf (tmpline, DVO_MAX_PATH, "%s -instmag %f %f",    command, INST_MAG_MIN, INST_MAG_MAX); strcpy (command, tmpline); }
+    if (ExcludeByMinSigma)  { snprintf (tmpline, DVO_MAX_PATH, "%s -min-sigma %f",     command, SIGMA_MIN_KEEP);                strcpy (command, tmpline); }
+    if (ExcludeByMaxMinMag) { snprintf (tmpline, DVO_MAX_PATH, "%s -maxminmag %f",     command, MAX_MIN_MAG);                   strcpy (command, tmpline); }
+    if (PHOTCODE_DROP_LIST) { snprintf (tmpline, DVO_MAX_PATH, "%s -photcode-drop %s", command, PHOTCODE_DROP_LIST);    strcpy (command, tmpline); }
+    if (PHOTCODE_SKIP_LIST) { snprintf (tmpline, DVO_MAX_PATH, "%s -photcode-skip %s", command, PHOTCODE_SKIP_LIST);    strcpy (command, tmpline); }
+    if (CATFORMAT)          { snprintf (tmpline, DVO_MAX_PATH, "%s -set-format %s",    command, CATFORMAT);                     strcpy (command, tmpline); }
+    if (CATMODE)            { snprintf (tmpline, DVO_MAX_PATH, "%s -set-mode %s",      command, CATMODE);                       strcpy (command, tmpline); }
     fprintf (stderr, "command: %s\n", command);

trunk/Ohana/src/relastro/include/relastro.h

-              r33652
+              r33963
 /* global variables set in parameter file */
 # define MAX_PATH_LENGTH 1024
 char   ImageCat[MAX_PATH_LENGTH];
 char   GSCFILE[MAX_PATH_LENGTH];
 char   CATDIR[MAX_PATH_LENGTH];
+# define DVO_MAX_PATH 1024
+char   ImageCat[DVO_MAX_PATH];
+char   GSCFILE[DVO_MAX_PATH];
+char   CATDIR[DVO_MAX_PATH];
 char   *HIGH_SPEED_DIR;
 char   CATMODE[16];    /* raw, mef, split, mysql */
 char   CATFORMAT[16];  /* internal, elixir, loneos, panstarrs */
 char   SKY_TABLE[MAX_PATH_LENGTH];
+char   SKY_TABLE[DVO_MAX_PATH];
 int    SKY_DEPTH;  /** XXX EAM : depth of catalog tables, fix usage */

trunk/Ohana/src/relastro/src/ConfigInit.c

r33652	r33963
39	39	// force CATDIR to be absolute (so parallel mode will work)
40	40	GetConfig (config, "CATDIR", "%s", 0, CATDIR);
41		char *tmpcatdir = abspath (CATDIR, ~~MAX_PATH_LENG~~TH);
	41	char *tmpcatdir = abspath (CATDIR, DVO_MAX_PATH);
42	42	strcpy (CATDIR, tmpcatdir);
43	43	free (tmpcatdir);

trunk/Ohana/src/relastro/src/UpdateObjectOffsets.c

r33652	r33963
87	87
88	88	// ensure that the paths are absolute path names
89		char *tmppath = abspath (table->hosts[i].pathname, ~~MAX_PATH_LENG~~TH);
	89	char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
90	90	free (table->hosts[i].pathname);
91	91	table->hosts[i].pathname = tmppath;

trunk/Ohana/src/relastro/src/args.c

r33652	r33963
98	98	RADIUS = atof(argv[N]);
99	99	remove_argument (N, &argc, argv);
100		HIGH_SPEED_DIR = abspath(argv[N], ~~MAX_PATH_LENG~~TH);
	100	HIGH_SPEED_DIR = abspath(argv[N], DVO_MAX_PATH);
101	101	remove_argument (N, &argc, argv);
102	102	}

trunk/Ohana/src/relastro/src/high_speed_catalogs.c

-              r33652
+              r33963
     char line[2048];
     snprintf (line, 2048, "cp %s/Images.dat %s/Images.dat", CATDIR, HIGH_SPEED_DIR);
+    system (line);
+    int status = system (line);
+    if (status) {
+      fprintf (stderr, "copy of Images.dat failed\n");
+      exit (3);
+    }
+  }
 …
     // ensure that the paths are absolute path names
     char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH);
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
     free (table->hosts[i].pathname);
     table->hosts[i].pathname = tmppath;
 …
       free (outputDir);
       outputDir = abspath (tableOut->hosts[index].pathname, MAX_PATH_LENGTH);
+      outputDir = abspath (tableOut->hosts[index].pathname, DVO_MAX_PATH);
+    }

trunk/Ohana/src/relastro/src/load_catalogs.c

r33652	r33963
126	126
127	127	// ensure that the paths are absolute path names
128		char *tmppath = abspath (table->hosts[i].pathname, ~~MAX_PATH_LENG~~TH);
	128	char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
129	129	free (table->hosts[i].pathname);
130	130	table->hosts[i].pathname = tmppath;

trunk/Ohana/src/relastro/src/relastro_objects.c

r33652	r33963
109	109
110	110	// ensure that the paths are absolute path names
111		char *tmppath = abspath (table->hosts[i].pathname, ~~MAX_PATH_LENG~~TH);
	111	char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
112	112	free (table->hosts[i].pathname);
113	113	table->hosts[i].pathname = tmppath;

trunk/Ohana/src/relphot/Makefile

-              r33651
+              r33963
 $(SRC)/relphot_objects.$(ARCH).o         \
 $(SRC)/select_images.$(ARCH).o   \
+$(SRC)/client_logger.$(ARCH).o   \
 $(SRC)/setExclusions.$(ARCH).o   \
 $(SRC)/setMrelFinal.$(ARCH).o    \
 …
 $(SRC)/relphot_objects.$(ARCH).o         \
 $(SRC)/relphot_client.$(ARCH).o  \
+$(SRC)/client_logger.$(ARCH).o   \
 $(SRC)/setExclusions.$(ARCH).o   \
 $(SRC)/setMrelFinal.$(ARCH).o    \

trunk/Ohana/src/relphot/include/relphot.h

-              r33651
+              r33963
 # include <kapa.h>
 # include <signal.h>
+# include <pthread.h>
 /* # define GRID_V1 */
 …
   float dMcal;
   float dMsys;
   short nFitPhotom;
+  unsigned short nFitPhotom;
   short Xm;
   float secz;
   float ubercalDist;
   unsigned int flags;
+  char skipCal;               // if TRUE, this mosaic is incomplete and should not be calibrated
   Coords coords;
 } Mosaic;
+typedef enum {
+  STATS_NONE,
+  STATS_MEAN,
+  STATS_MEDIAN,
+  STATS_WT_MEAN,
+  STATS_INNER_MEAN,
+  STATS_INNER_WTMEAN,
+  STATS_CHI_INNER_MEAN,
+  STATS_CHI_INNER_WTMEAN
+} ListStatsMode;
 typedef struct {
 …
   double total;
   int    Nmeas;
+  ListStatsMode statmode;
 } StatType;
 …
 /* global variables set in parameter file */
 # define MAX_PATH_LENGTH 1024
 char   ImageCat[MAX_PATH_LENGTH];
 char   ImageTemplate[MAX_PATH_LENGTH];
 char   CatTemplate[MAX_PATH_LENGTH];
 char   GSCFILE[MAX_PATH_LENGTH];
+# define DVO_MAX_PATH 1024
+char   ImageCat[DVO_MAX_PATH];
+char   ImageTemplate[DVO_MAX_PATH];
+char   CatTemplate[DVO_MAX_PATH];
+char   GSCFILE[DVO_MAX_PATH];
 char  *CATDIR;
 char   CATMODE[16];    /* raw, mef, split, mysql */
 char   CATFORMAT[16];  /* internal, elixir, loneos, panstarrs */
 char   CameraConfig[MAX_PATH_LENGTH];
+char   CameraConfig[DVO_MAX_PATH];
 char   CAMERA[64];    /* eg, gpc1 */
 char   SKY_TABLE[MAX_PATH_LENGTH];
+char   SKY_TABLE[DVO_MAX_PATH];
 int    SKY_DEPTH;  /** XXX EAM : depth of catalog tables, fix usage */
 …
 int    PARALLEL_MANUAL;
 int    PARALLEL_SERIAL;
+int    NTHREADS;
 int    VERBOSE;
 …
 int    PLOTDELAY;
 int    UpdateAverages;
+int    ApplyOffsets;
 char  *PhotcodeList;
 …
 int           args                PROTO((int argc, char **argv));
 int           args_client         PROTO((int argc, char **argv));
 int           bcatalog            PROTO((Catalog *subcatalog, Catalog *catalog));
+int           bcatalog            PROTO((Catalog *subcatalog, Catalog *catalog, int Ncat));
 void          clean_images        PROTO((void));
 void          clean_measures      PROTO((Catalog *catalog, int Ncatalog, int final, FlatCorrectionTable *flatcorr));
 …
 void          initMosaicBins      PROTO((Catalog *catalog, int Ncatalog, int doMosaicList));
 void          initMosaicGrid      PROTO((Image *image, off_t Nimage));
 void          initMosaics         PROTO((Image *image, off_t Nimage));
+void          initMosaics         PROTO((Image *subset, off_t Nsubset, Image *image, char *inSubset, off_t Nimage));
 void          initMrel            PROTO((Catalog *catalog, int Ncatalog));
 void          initialize          PROTO((int argc, char **argv));
 void          initialize_client   PROTO((int argc, char **argv));
 void          initstats           PROTO((char *mode));
 int           liststats           PROTO((double *value, double *dvalue, int N, StatType *stats));
+void          liststats_setmode   PROTO((StatType *stats, char *strmode));
+int           liststats           PROTO((double *value, double *dvalue, double *wvalue, int N, StatType *stats));
 Catalog      *load_catalogs       PROTO((SkyList *skylist, int *Ncatalog, int hostID, char *hostpath));
 Catalog      *load_catalogs_parallel PROTO((SkyList *sky, int *Ncatalog));
 SkyList      *load_images         PROTO((FITS_DB *db, char *regionName, SkyRegion *region));
 Image        *select_images       PROTO((SkyList *skylist, Image *timage, off_t Ntimage, off_t **LineNumber, off_t *Nimage));
+Image        *select_images       PROTO((SkyList *skylist, Image *timage, off_t Ntimage, char *inSubset, off_t **LineNumber, off_t *Nimage, SkyRegion *region));
 int           main                PROTO((int argc, char **argv));
 …
 int           setMmos             PROTO((Catalog *catalog, int Poor, FlatCorrectionTable *flatcorr));
 int           setMrel             PROTO((Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr));
 void          setMrelFinal        PROTO((Catalog *catalog, FlatCorrectionTable *flatcorr));
+void          setMrelFinal        PROTO((Catalog *catalog, FlatCorrectionTable *flatcorr, int simpleAverage));
 int           setMrelOutput       PROTO((Catalog *catalog, int Ncatalog, int pass, FlatCorrectionTable *flatcorr));
 int           setMave             PROTO((Catalog *catalog, int Ncatalog));
 …
 int ImageSubsetSave(char *filename, ImageSubset *image, off_t Nimage);
 ImageSubset *ImageSubsetLoad(char *filename, off_t *nimage);
+int client_logger_init ();
+int client_logger_message (char *format,...);

trunk/Ohana/src/relphot/src/ConfigInit.c

-              r33651
+              r33963
   // force CATDIR to be absolute (so parallel mode will work)
   char *tmpcatdir = NULL;
   ALLOCATE (tmpcatdir, char, MAX_PATH_LENGTH);
+  ALLOCATE (tmpcatdir, char, DVO_MAX_PATH);
   GetConfig (config, "CATDIR",                 "%s",  0, tmpcatdir);
   CATDIR = abspath (tmpcatdir, MAX_PATH_LENGTH);
+  CATDIR = abspath (tmpcatdir, DVO_MAX_PATH);
   free (tmpcatdir);

trunk/Ohana/src/relphot/src/GridOps.c

-              r33651
+              r33963
   double *list, *dlist;
   float Msys, Mrel, Mcal, Mmos;
   StatType stats;
+  liststats_setmode (&stats, STATMODE);
   if (!USE_GRID) return;
 …
+    }
     liststats (list, dlist, N, &stats);
+    liststats (list, dlist, NULL, N, &stats);
     gridM[i] = stats.mean;
     gridS[i] = stats.sigma;

trunk/Ohana/src/relphot/src/ImageOps.c

-              r33651
+              r33963
 // relationships between the measure,catalog set and the images:
 static IDX_T       **MeasureToImage; // image index from measure,catalog   : MeasureToImage[cat][meas] = ImageIndex
 static IDX_T       **ImageToCatalog; // catalog for given measure on image : ImageCatalog[ImageIndex][i] = cat (i : 0 < NonImage[ImageIndex])
 static IDX_T       **ImageToMeasure; // measure for given measure on image : ImageMeasure[ImageIndex][i] = cat (i : 0 < NonImage[ImageIndex])
+static IDX_T       **MeasureToImage = NULL; // image index from measure,catalog   : MeasureToImage[cat][meas] = ImageIndex
+static IDX_T       **ImageToCatalog = NULL; // catalog for given measure on image : ImageCatalog[ImageIndex][i] = cat (i : 0 < NonImage[ImageIndex])
+static IDX_T       **ImageToMeasure = NULL; // measure for given measure on image : ImageMeasure[ImageIndex][i] = cat (i : 0 < NonImage[ImageIndex])
 // MeasureToImage was 'bin'
 …
   off_t i;
+  if (!MeasureToImage) return -1;
   i = MeasureToImage[cat][meas];
   return (i);
 …
   short distance;
+  if (!MeasureToImage) return -1;
   i = MeasureToImage[cat][meas];
   if (i == -1) return (1000);
 …
   float Msys, Mrel, Mmos, Mgrid, Mflat;
   double *list, *dlist, *Mlist, *dMlist;
   StatType stats;
+  liststats_setmode (&stats, STATMODE);
   if (FREEZE_IMAGES) return;
 …
+    }
+    liststats (list, dlist, N, &stats);
+    // no additional weight modification (we treat all stars on an image equally -- note an image is either ubercal-tied or not)
+    liststats (list, dlist, NULL, N, &stats);
     image[i].Mcal  = stats.mean;
     image[i].dMcal = stats.error;
 …
     // bright end scatter
     liststats (Mlist, dMlist, Nbright, &stats);
+    liststats (Mlist, dMlist, NULL, Nbright, &stats);
     image[i].dMagSys = stats.sigma;
 …
   double *mlist, *slist, *dlist;
   double MaxOffset, MaxScatter, MedOffset;
-  StatType stats;
   if (FREEZE_IMAGES) return;
 …
     N++;
+  }
+  initstats ("MEAN");
+  liststats (mlist, dlist, N, &stats);
+  // use a straight mean to find the global image statistics (no weighting)
+  StatType stats;
+  liststats_setmode (&stats, "MEAN");
+  liststats (mlist, dlist, NULL, N, &stats);
   MaxOffset = MAX (IMAGE_OFFSET, 3*stats.sigma);
   MedOffset = stats.median;
+  liststats (slist, dlist, N, &stats);
+  liststats (slist, dlist, NULL, N, &stats);
   MaxScatter = MAX (IMAGE_SCATTER, 2*stats.median);
   fprintf (stderr, "Mrel: %f, dMrel: %f, Max Scatter: %f, Max Offset: %f\n", MedOffset, stats.median, MaxScatter, MaxOffset);
 …
   fprintf (stderr, "%d images marked poor\n", Nmark);
-  initstats (STATMODE);
   free (mlist);
   free (slist);
 …
   double *list, *dlist;
   float Mcal, Mmos, Mgrid;
   StatType stats;
   bzero (&stats, sizeof (StatType));
   if (FREEZE_IMAGES) return (stats);
 …
+  }
+  liststats (list, dlist, n, &stats);
+  liststats_setmode (&stats, "MEAN");
+  liststats (list, dlist, NULL, n, &stats);
   free (list);
   free (dlist);
 …
+  }
+  liststats (list, dlist, n, &stats);
+  liststats_setmode (&stats, "MEAN");
+  liststats (list, dlist, NULL, n, &stats);
   free (list);
   free (dlist);
 …
+  }
+  liststats (list, dlist, n, &stats);
+  liststats_setmode (&stats, "MEAN");
+  liststats (list, dlist, NULL, n, &stats);
   free (list);
   free (dlist);
 …
+  }
+  liststats (list, dlist, n, &stats);
+  liststats_setmode (&stats, "MEAN");
+  liststats (list, dlist, NULL, n, &stats);
   free (list);
   free (dlist);

trunk/Ohana/src/relphot/src/ImageSubset.c

-              r33651
+              r33963
+}
+// STATUS is value expected for success
+# define CHECK_STATUS(STATUS,MSG,...)                                   \
+  if (!(STATUS)) {                                                      \
+    fprintf (stderr, MSG, __VA_ARGS__);                                 \
+    return FALSE;                                                       \
+  }
 int ImageSubsetSave(char *filename, ImageSubset *image, off_t Nimage) {
 …
+  }
+  gfits_fwrite_header  (f, &header);
+  gfits_fwrite_matrix  (f, &matrix);
+  gfits_fwrite_Theader (f, &theader);
+  gfits_fwrite_table  (f, &ftable);
+  int status;
+  status = gfits_fwrite_header  (f, &header);
+  CHECK_STATUS (status, "ERROR: cannot write header for image subset %s\n", filename);
+  status = gfits_fwrite_matrix  (f, &matrix);
+  CHECK_STATUS (status, "ERROR: cannot write matrix for image subset %s\n", filename);
+  status = gfits_fwrite_Theader (f, &theader);
+  CHECK_STATUS (status, "ERROR: cannot write table header for image subset %s\n", filename);
+  status = gfits_fwrite_table  (f, &ftable);
+  CHECK_STATUS (status, "ERROR: cannot write table data for image subset %s\n", filename);
   gfits_free_header (&header);
 …
   gfits_free_table (&ftable);
+  fclose (f);
+  fflush (f);
+  int fd = fileno (f);
+  status = fflush (f);
+  CHECK_STATUS (!status, "ERROR: cannot flush file image subset %s\n", filename);
+  status = fsync (fd);
+  CHECK_STATUS (!status, "ERROR: cannot flush file image subset %s\n", filename);
+  status = fclose (f);
+  CHECK_STATUS (!status, "ERROR: problem closing image subset file %s\n", filename);
   return TRUE;

trunk/Ohana/src/relphot/src/MosaicOps.c

-              r33651
+              r33963
   ALLOCATE (mosaic, Mosaic, NMOSAIC);
   ALLOCATE (MosaicToImage,  off_t *, NMOSAIC);
+  ALLOCATE (MosaicToImage, off_t *, NMOSAIC);
   ALLOCATE (MosaicN_Image, off_t,   NMOSAIC);
   ALLOCATE (MosaicN_IMAGE, off_t,   NMOSAIC);
 …
     if (found) continue;
     /* a new mosaic, define ranges */
+    /* a new mosaic, define ranges -- preserve the original values incase this image is not used */
     mosaic[Nmosaic].start = start;
     mosaic[Nmosaic].stop  = stop;
     mosaic[Nmosaic].Mcal  = 0.0;
     mosaic[Nmosaic].dMcal = 0.0;
+    mosaic[Nmosaic].Mcal  = 0.0; // note : mosaic stores only offsets relative to the original image values
+    mosaic[Nmosaic].dMcal = 0.0; // note : at the end, mosaic.Mcal is added back to the input images
     mosaic[Nmosaic].dMsys = 0.0;
     mosaic[Nmosaic].Xm    = 0.0;
 …
+}
+/* find mosaic frames (unique time periods & photcode name matches mosaic) */
+void initMosaics (Image *image, off_t Nimage) {
+# define MARKTIME(MSG,...) { \
+  float dtime; \
+  gettimeofday (&stopTimer, (void *) NULL); \
+  dtime = DTIME (stopTimer, startTimer); \
+  fprintf (stderr, MSG, __VA_ARGS__); }
+/* find mosaic frames (unique time periods) (NOTE : we do NOT require matching photcodes...)
+   this function will also identify the images NOT in the subset which belong to a selected mosaic
+ */
+void initMosaics (Image *subset, off_t Nsubset, Image *image, char *inSubset, off_t Nimage) {
   off_t i, j, status, found, NMOSAIC, *MosaicN_IMAGE;
 …
   char *pname;
+  struct timeval startTimer, stopTimer;
   if (!MOSAIC_ZEROPT) return;
+  gettimeofday (&startTimer, (void *) NULL);
   /* a 'mosaic' in relphot is (unlike relastro) a virtual concept: there is no
 …
    */
+  // generate a list of unique start times (these define the mosaics)
+  ALLOCATE (startTimes, unsigned int, Nimage);
+  for (i = 0; i < Nimage; i++) {
+    startTimes[i] = image[i].tzero;
+  }
+  sort_times (startTimes, Nimage);
+  // generate a list of all subset image start times
+  ALLOCATE (startTimes, unsigned int, Nsubset);
+  for (i = 0; i < Nsubset; i++) {
+    startTimes[i] = subset[i].tzero;
+  }
+  sort_times (startTimes, Nsubset);
+  MARKTIME("create array of all image obstimes: %f sec\n", dtime);
   Nmosaic = 0;
 …
   startTimesMosaic[0] = startTimes[0];
+  for (i = 0; i < Nimage; i++) {
+  // generate a list of the unique start times (these define the mosaics)
+  for (i = 0; i < Nsubset; i++) {
     if (startTimes[i] < startTimesMosaic[Nmosaic]) {
       fprintf (stderr, "error?\n");
 …
+  }
   Nmosaic ++;
+  MARKTIME("create subset array of mosaic obstimes: %f sec\n", dtime);
   // now I have a list of uniq start times, and they are in order
 …
     mosaic[i].secz  = NAN;
     mosaic[i].photcode = 0;
+    mosaic[i].skipCal = FALSE;
+    memset (&mosaic[i].coords, 0, sizeof(Coords));
     MosaicN_IMAGE[i] = 10;
 …
+  }
   ALLOCATE (ImageToMosaic, off_t, Nimage); // mosaic to which image belongs
   // assign each image to a mosaic
+  int Nskip, Nmark;
+  Nskip = Nmark = 0;
+  // find any mosaics (startTimesMosaic) which match unselected images
   for (i = 0; i < Nimage; i++) {
+    ImageToMosaic[i] = -1;
+    if (inSubset[i]) {
+      Nskip ++;
+      continue;
+    }
     /* select valid mosaic images by photcode */
     pname = GetPhotcodeNamebyCode (image[i].photcode);
+    if (!pname) continue;
     status = strncmp (pname, MOSAICNAME, strlen (MOSAICNAME));
     if (status) continue;
     /* set image time range */
-    // start = image[i].tzero - MAX(0.01*image[i].trate*image[i].NY, 1);
-    // stop  = image[i].tzero + MAX(1.01*image[i].trate*image[i].NY, 1);
     start = image[i].tzero;
     stop  = image[i].tzero + MAX(1.01*image[i].trate*image[i].NY, 1);
+    /* find a matching mosaic */
+    j = findMosaic(startTimesMosaic, Nmosaic, start);
+    if (j != -1) {
+      // mark this mosaic as bad
+      mosaic[j].skipCal = TRUE;
+      Nmark ++;
+    }
+  }
+  fprintf (stderr, "%d total images, %d overlap skyregion, %d do not overlap, but match overlapping mosaics\n", (int) Nimage, (int) Nskip, (int) Nmark);
+  MARKTIME("find unselected images matching selected mosaics: %f sec\n", dtime);
+  ALLOCATE (ImageToMosaic, off_t, Nsubset); // mosaic to which image belongs
+  // assign each image to a mosaic
+  for (i = 0; i < Nsubset; i++) {
+    ImageToMosaic[i] = -1;
+    /* select valid mosaic images by photcode */
+    pname = GetPhotcodeNamebyCode (subset[i].photcode);
+    status = strncmp (pname, MOSAICNAME, strlen (MOSAICNAME));
+    if (status) continue;
+    start = subset[i].tzero;
+    stop  = subset[i].tzero + MAX(1.01*subset[i].trate*subset[i].NY, 1);
     j = findMosaic(startTimesMosaic, Nmosaic, start);
     if (j == -1) {
       fprintf (stderr, "error?\n");
+      fprintf (stderr, "programming error? all subset images should belong to a mosaic\n");
       abort();
+    }
 …
     mosaic[j].dMsys = 0.0;
     mosaic[j].Xm    = 0.0;
     mosaic[j].flags  = image[i].flags;
     mosaic[j].secz  = image[i].secz;
     mosaic[j].photcode = GetPhotcodeEquivCodebyCode (image[i].photcode);
+    mosaic[j].flags  = subset[i].flags;
+    mosaic[j].secz  = subset[i].secz;
+    mosaic[j].photcode = GetPhotcodeEquivCodebyCode (subset[i].photcode);
+  }
 …
   free (startTimesMosaic);
   initMosaicGrid (image, Nimage);
   fprintf (stderr, "matched %d images to %d mosaics\n", (int) Nimage, (int) Nmosaic);
+  initMosaicGrid (subset, Nsubset);
+  fprintf (stderr, "matched %d images to %d mosaics\n", (int) Nsubset, (int) Nmosaic);
   return;
+}
+// use bisection to find the overlapping mosaic
+// use bisection to find the overlapping mosaic (returns exact match)
+// startTimes is a sorted, unique list of times
+// start might not be in the list
 off_t findMosaic (unsigned int *startTimes, off_t Nmosaic, unsigned int start) {
 …
   // find the last mosaic before start
+  Nlo = 0; // startTimes[Nlo] guaranteed to be <= start
+  Nhi = Nmosaic - 1; // startTimes[Nhi] guaranteed to be >= start
+  while (Nhi - Nlo > 10) {
+  Nlo = 0; // first valid startTimes value
+  Nhi = Nmosaic - 1; // last valid startTimes value
+  // if start is not in this range, return -1
+  if (start < startTimes[Nlo]) return (-1);
+  if (start > startTimes[Nhi]) return (-1);
+  while (Nhi - Nlo > 4) {
     N = 0.5*(Nlo + Nhi);
     if (startTimes[N] < start) {
 …
+    }
+  }
+  // check for the matched mosaic starting from Nlo
   for (N = Nlo; N < Nmosaic; N++) {
     if (start > startTimes[N])  continue;
     return (N);
+  // we now have : startTimes[Nlo] < start <= starTimes[Nhi]
+  // find a matched mosaic starting from Nlo, or return -1
+  for (N = Nlo; N <= Nhi; N++) {
+    if (startTimes[N] == start) return N;
+  }
   return (-1);
 …
+}
+// XXX : what about mosaics with skipCal == TRUE?
 void setMcalFinal () {
 …
+}
+typedef struct {
+  int Nfew;
+  int Nbad;
+  int Ncal;
+  int Ngrid;
+  int Nrel;
+  int Nsys;
+  int Nskip;
+  off_t Nmax;
+  int PoorImages;
+  double *list;
+  double *dlist;
+  double *Mlist;
+  double *dMlist;
+} SetMmosInfo;
+enum {THREAD_RUN, THREAD_DONE};
+typedef struct {
+  int entry;
+  int state;
+  Catalog *catalog;
+  Image *image;
+  FlatCorrectionTable *flatcorr;
+  SetMmosInfo info;
+} ThreadInfo;
+int setMmos_mosaic (Mosaic *mosaic, off_t Nmos, Image *image, Catalog *catalog, SetMmosInfo *info, FlatCorrectionTable *flatcorr);
+void *setMmos_worker (void *data);
+int setMmos_threaded (Catalog *catalog, int PoorImages, FlatCorrectionTable *flatcorr);
+void SetMmosInfoInit (SetMmosInfo *info, off_t Nmax, int allocLists, int PoorImages) {
+  info->Nfew = 0;
+  info->Nbad = 0;
+  info->Ncal = 0;
+  info->Nrel = 0;
+  info->Ngrid = 0;
+  info->Nskip = 0;
+  info->Nsys = 0;
+  info->Nmax = Nmax;
+  info->PoorImages = PoorImages;
+  if (allocLists) {
+    ALLOCATE (info->list, double, Nmax);
+    ALLOCATE (info->dlist, double, Nmax);
+    ALLOCATE (info->Mlist, double, Nmax);
+    ALLOCATE (info->dMlist, double, Nmax);
+  }
+}
+void SetMmosInfoFree (SetMmosInfo *info) {
+  free (info->list);
+  free (info->dlist);
+  free (info->Mlist);
+  free (info->dMlist);
+}
+void SetMmosInfoAccum (SetMmosInfo *summary, SetMmosInfo *results) {
+  summary->Nfew  += results->Nfew ;
+  summary->Nsys  += results->Nsys ;
+  summary->Nbad  += results->Nbad ;
+  summary->Ncal  += results->Ncal ;
+  summary->Nrel  += results->Nrel ;
+  summary->Ngrid += results->Ngrid;
+  summary->Nskip += results->Nskip;
+}
+static int npass_output = 0;
+// mutex to lock setMmos_worker operations
+static pthread_mutex_t setMmos_mutex = PTHREAD_MUTEX_INITIALIZER;
+static int nextMosaic = 0;
+// we have an array of mosaics (mosaic, Nmosaic).  we need to hand out mosaics one at a time to
+// the worker threads as they need
+off_t getNextMosaicForThread () {
+  pthread_mutex_lock (&setMmos_mutex);
+  if (nextMosaic >= Nmosaic) {
+    pthread_mutex_unlock (&setMmos_mutex);
+    return (-1);
+  }
+  int thisMosaic = nextMosaic;
+  nextMosaic ++;
+  pthread_mutex_unlock (&setMmos_mutex);
+  return (thisMosaic);
+}
 int setMmos (Catalog *catalog, int PoorImages, FlatCorrectionTable *flatcorr) {
+  off_t i, j, m, c, n, N, Nmax;
+  int mark, bad, Nfew, Nbad, Ncal, Nrel, Ngrid, Nsys, Nbright;
+  float Msys, Mrel, Mcal, Mgrid, Mflat;
+  double *list, *dlist, *Mlist, *dMlist;
+  StatType stats;
+  off_t i, N, Nmax;
   Image *image;
 …
   if (FREEZE_MOSAICS) return (FALSE);
+  if (NTHREADS) {
+    int status = setMmos_threaded (catalog, PoorImages, flatcorr);
+    return status;
+  }
   image = getimages (&N, NULL);
   fprintf (stderr, "limiting negative clouds to %f\n", CLOUD_TOLERANCE);
-  int Nsecfilt = GetPhotcodeNsecfilt ();
   if (PoorImages) {
 …
     Nmax = MAX (Nmax, N_onMosaic[i]);
+  }
+  ALLOCATE (list, double, Nmax);
+  ALLOCATE (dlist, double, Nmax);
+  ALLOCATE (Mlist, double, Nmax);
+  ALLOCATE (dMlist, double, Nmax);
+  Nfew = Nbad = Ncal = Nrel = Ngrid = Nsys = 0;
+  for (i = 0; i < Nmosaic; i++) {
+    /* on PoorImages run, skip good images */
+    if (PoorImages) {
+      bad = mosaic[i].flags & (ID_IMAGE_PHOTOM_FEW | ID_IMAGE_PHOTOM_POOR | ID_IMAGE_PHOTOM_SKIP);
+      if (!bad) continue;
+    }
+    // UBERCAL image: if this is an ubercal image, set minUbercalDist to 0:
+    // we optionally do not recalibrate images with UBERCAL zero points
+    if (mosaic[i].flags & ID_IMAGE_PHOTOM_UBERCAL) {
+      mosaic[i].ubercalDist = 0;
+      // propagate ubercalDist to the images
+      for (j = 0; j < MosaicN_Image[i]; j++) {
+        off_t im = MosaicToImage[i][j];
+        image[im].ubercalDist = mosaic[i].ubercalDist;
+        // fprintf (stderr, "%d %d %d\n", (int) i, (int) im, image[im].ubercalDist);
+      }
+      if (KEEP_UBERCAL) continue;
+    }
+    int minUbercalDist = 1000;
+    // number of stars to measure the bright-end scatter
+    Nbright = 0;
+    N = 0;
+    for (j = 0; j < N_onMosaic[i]; j++) {
+      m = MosaicToMeasure[i][j];
+      c = MosaicToCatalog[i][j];
+      if (catalog[c].measureT[m].dbFlags & MEAS_BAD) {
+          Nbad ++;
+          continue;
+      }
+      Mcal  = getMcal  (m, c, flatcorr, catalog);
+      if (isnan(Mcal)) {
+          Ncal++;
+          continue;
+      }
+      Mgrid = getMgrid (m, c);
+      if (isnan(Mgrid)) {
+          Ngrid ++;
+          continue;
+      }
+      Mrel  = getMrel  (catalog, m, c);
+      if (isnan(Mrel)) {
+          Nrel ++;
+          continue;
+      }
+      // image.Mcal is not supposed to include the flat-field correction, so we need to
+      // apply that offset as well here for this image (in other words, each detection is
+      // being compared to the model, excluding the zero point, Mcal.  The model includes
+      // the flat-correction.  NOTE the sign of Mflat (Image.Mcal = Measure.Mcal - Mflat)
+      Mflat = getMflat (m, c, flatcorr, catalog);
+      n = catalog[c].measureT[m].averef;
+      Msys = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt]);
+      if (isnan(Msys)) {
+        Nsys++;
+        continue;
+      }
+      PhotCode *code = GetPhotcodebyCode (catalog[c].measureT[m].photcode);
+      if (!code) goto skip;
+      if (code->equiv < 1) goto skip;
+      int Nsec = GetPhotcodeNsec (code->equiv);
+      if (Nsec == -1) goto skip;
+      minUbercalDist = MIN (catalog[c].secfilt[n*Nsecfilt + Nsec].ubercalDist, minUbercalDist);
+    skip:
+      list[N]  = Msys - Mrel - Mcal - Mgrid + Mflat;
+      dlist[N] = MAX (catalog[c].measureT[m].dM, MIN_ERROR);
+      if (catalog[c].measureT[m].dM < IMFIT_SYS_SIGMA_LIM) {
+        Mlist[Nbright] = list[N];
+        dMlist[Nbright] = dlist[N];
+        Nbright ++;
+      }
+      N++;
+    }
+    /* N_onMosaic[i] is all measurements, N is good measurements */
+    /* too few good measurements or too many bad measurements (skip in PoorImages run) */
+    if (!PoorImages) {
+      mark = (N < IMAGE_TOOFEW) || (N < IMAGE_GOOD_FRACTION*N_onMosaic[i]);
+      if (mark) {
+        if (VERBOSE2) { fprintf (stderr, "marked mosaic %s ("OFF_T_FMT"), ("OFF_T_FMT" < %d) || ("OFF_T_FMT" < %f*"OFF_T_FMT")\n", image[MosaicToImage[i][0]].name,  i,  N, IMAGE_TOOFEW,  N, IMAGE_GOOD_FRACTION,  N_onMosaic[i]); }
+        mosaic[i].flags |= ID_IMAGE_PHOTOM_FEW;
+        Nfew ++;
+      } else {
+        mosaic[i].flags &= ~ID_IMAGE_PHOTOM_FEW;
+      }
+    }
+    liststats (list, dlist, N, &stats);
+    if (VERBOSE2 && PoorImages) fprintf (stderr, "Mmos: %f %f %d "OFF_T_FMT"\n", stats.mean, stats.sigma, stats.Nmeas,  N);
+    mosaic[i].Mcal  = stats.mean;
+    mosaic[i].dMcal = stats.error;
+    mosaic[i].nFitPhotom = N;
+    mosaic[i].Xm    = 100.0*log10(stats.chisq);
+    plot_setMcal (list, N, &stats, CLOUD_TOLERANCE);
+    // bright end scatter
+    liststats (Mlist, dMlist, Nbright, &stats);
+    mosaic[i].dMsys = stats.sigma;
+    if (mosaic[i].Mcal < -CLOUD_TOLERANCE) {
+      mosaic[i].Mcal = 0.0;
+    }
+    // minUbercalDist calculated here is the min value for any star owned by this image
+    // since this particular image is tied to that star, bump its distance by 1
+    mosaic[i].ubercalDist = minUbercalDist + 1;
+    // propagate ubercalDist to the images
+    for (j = 0; j < MosaicN_Image[i]; j++) {
+      off_t im = MosaicToImage[i][j];
+      image[im].ubercalDist = mosaic[i].ubercalDist;
+      // fprintf (stderr, "%d %d %d\n", (int) i, (int) im, image[im].ubercalDist);
+    }
+  }
+  free (list);
+  free (dlist);
+  free (Mlist);
+  free (dMlist);
+  fprintf (stderr, "%d mosaics marked having too few measurements (Nbad: %d, Ncal: %d, Ngrid: %d, Nrel: %d, Nsys: %d)\n", Nfew, Nbad, Ncal, Ngrid, Nrel, Nsys);
+  SetMmosInfo info;
+  SetMmosInfoInit (&info, Nmax, TRUE, PoorImages);
+  for (i = 0; i < Nmosaic; i++) {
+    setMmos_mosaic (&mosaic[i], i, image, catalog, &info, flatcorr);
+  }
+  SetMmosInfoFree (&info);
+  npass_output ++;
+  fprintf (stderr, "%d mosaics marked having too few measurements (Nbad: %d, Ncal: %d, Ngrid: %d, Nrel: %d, Nsys: %d)\n", info.Nfew, info.Nbad, info.Ncal, info.Ngrid, info.Nrel, info.Nsys);
   if (PoorImages) {
 …
+}
+// 'mosaic' is a pointer to the current mosaic of interest (Nmos)
+int setMmos_mosaic (Mosaic *myMosaic, off_t Nmos, Image *image, Catalog *catalog, SetMmosInfo *info, FlatCorrectionTable *flatcorr) {
+  Image *imageReal;
+  off_t j, NimageReal;
+  StatType stats;
+  liststats_setmode (&stats, STATMODE);
+  double *list   = info->list;
+  double *dlist  = info->dlist;
+  double *Mlist  = info->Mlist;
+  double *dMlist = info->dMlist;
+  assert (Nmos >= 0);
+  assert (Nmos < Nmosaic);
+  imageReal = getimages (&NimageReal, NULL);
+  /* on PoorImages run, skip good images */
+  if (info->PoorImages) {
+    int bad = myMosaic[0].flags & (ID_IMAGE_PHOTOM_FEW | ID_IMAGE_PHOTOM_POOR | ID_IMAGE_PHOTOM_SKIP);
+    if (!bad) return TRUE;
+  }
+  // UBERCAL image: if this is an ubercal image, set minUbercalDist to 0:
+  // we optionally do not recalibrate images with UBERCAL zero points
+  if (myMosaic[0].flags & ID_IMAGE_PHOTOM_UBERCAL) {
+    myMosaic[0].ubercalDist = 0;
+    // propagate ubercalDist to the images
+    for (j = 0; j < MosaicN_Image[Nmos]; j++) {
+      off_t im = MosaicToImage[Nmos][j];
+      assert (im < NimageReal);
+      assert (im >= 0);
+      image[im].ubercalDist = myMosaic[0].ubercalDist;
+      // fprintf (stderr, "%d %d %d\n", (int) i, (int) im, image[im].ubercalDist);
+    }
+    if (KEEP_UBERCAL) return TRUE;
+  }
+  int Nsecfilt = GetPhotcodeNsecfilt ();
+  int minUbercalDist = 1000;
+  // do not modify the calibration for mosaics with partial images loaded (skipCal TRUE)
+  if (myMosaic[0].skipCal) {
+    info->Nskip ++;
+    return TRUE;
+  }
+  int testImage = FALSE;
+  // testImage |= (abs(myMosaic[0].start - 1324104046) < 10);
+  // testImage |= (abs(myMosaic[0].start - 1324103823) < 10);
+  // testImage |= (abs(myMosaic[0].start - 1323003245) < 10);
+  // testImage |= (abs(myMosaic[0].start - 1323003069) < 10);
+  // testImage |= (abs(myMosaic[0].start - 1323003125) < 10);
+  // testImage |= (abs(myMosaic[0].start - 1323003300) < 10);
+  // testImage |= (abs(myMosaic[0].start - 1323003365) < 10);
+  // testImage |= (abs(myMosaic[0].start - 1323003191) < 10);
+  // testImage |= (abs(myMosaic[0].start - 1323003014) < 10);
+  // testImage |= (abs(myMosaic[0].start - 1323003484) < 10);
+  // testImage |= (abs(myMosaic[0].start - 1323003419) < 10);
+  // testImage |= (abs(myMosaic[0].start - 1323002949) < 10);
+  FILE *fout = NULL;
+  if (testImage) {
+    char filename[64];
+    snprintf (filename, 64, "test.%05d.%02d.dat", (int) Nmos, npass_output);
+    fout = fopen (filename, "w");
+  }
+  // number of stars to measure the bright-end scatter
+  int Nbright = 0;
+  int N = 0;
+  for (j = 0; j < N_onMosaic[Nmos]; j++) {
+    float Msys, Mrel, Mcal, Mgrid, Mflat;
+    off_t m = MosaicToMeasure[Nmos][j];
+    off_t c = MosaicToCatalog[Nmos][j];
+    if (fout) {
+      Mcal  = getMcal  (m, c, flatcorr, catalog);
+      Mgrid = getMgrid (m, c);
+      Mrel  = getMrel  (catalog, m, c);
+      Mflat = getMflat (m, c, flatcorr, catalog);
+      off_t n = catalog[c].measureT[m].averef;
+      Msys = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt]);
+      float delta = Msys - Mrel - Mcal - Mgrid + Mflat;
+      int isBad = (catalog[c].measureT[m].dbFlags & MEAS_BAD);
+      fprintf (fout, "%f %f : %f %f %f %f %f  : %f %d\n", catalog[c].averageT[n].R, catalog[c].averageT[n].D, Msys, Mrel, Mcal, Mgrid, Mflat, delta, isBad);
+    }
+    if (catalog[c].measureT[m].dbFlags & MEAS_BAD) {
+      info->Nbad ++;
+      continue;
+    }
+    Mcal  = getMcal  (m, c, flatcorr, catalog);
+    if (isnan(Mcal)) {
+      info->Ncal++;
+      continue;
+    }
+    Mgrid = getMgrid (m, c);
+    if (isnan(Mgrid)) {
+      info->Ngrid ++;
+      continue;
+    }
+    Mrel  = getMrel  (catalog, m, c);
+    if (isnan(Mrel)) {
+      info->Nrel ++;
+      continue;
+    }
+    // image.Mcal is not supposed to include the flat-field correction, so we need to
+    // apply that offset as well here for this image (in other words, each detection is
+    // being compared to the model, excluding the zero point, Mcal.  The model includes
+    // the flat-correction.  NOTE the sign of Mflat (Image.Mcal = Measure.Mcal - Mflat)
+    Mflat = getMflat (m, c, flatcorr, catalog);
+    off_t n = catalog[c].measureT[m].averef;
+    Msys = PhotSysTiny (&catalog[c].measureT[m], &catalog[c].averageT[n], &catalog[c].secfilt[n*Nsecfilt]);
+    if (isnan(Msys)) {
+      info->Nsys++;
+      continue;
+    }
+    PhotCode *code = GetPhotcodebyCode (catalog[c].measureT[m].photcode);
+    if (!code) goto skip;
+    if (code->equiv < 1) goto skip;
+    int Nsec = GetPhotcodeNsec (code->equiv);
+    if (Nsec == -1) goto skip;
+    minUbercalDist = MIN (catalog[c].secfilt[n*Nsecfilt + Nsec].ubercalDist, minUbercalDist);
+  skip:
+    assert (N < info->Nmax);
+    assert (N >= 0);
+    assert (Nbright < info->Nmax);
+    assert (Nbright >= 0);
+    list[N]  = Msys - Mrel - Mcal - Mgrid + Mflat;
+    dlist[N] = MAX (catalog[c].measureT[m].dM, MIN_ERROR);
+    if (catalog[c].measureT[m].dM < IMFIT_SYS_SIGMA_LIM) {
+      Mlist[Nbright] = list[N];
+      dMlist[Nbright] = dlist[N];
+      Nbright ++;
+    }
+    N++;
+  }
+  /* N_onMosaic[Nmos] is all measurements, N is good measurements */
+  if (fout) {
+    fclose (fout);
+  }
+  /* too few good measurements or too many bad measurements (skip in PoorImages run) */
+  if (!info->PoorImages) {
+    int mark = (N < IMAGE_TOOFEW) || (N < IMAGE_GOOD_FRACTION*N_onMosaic[Nmos]);
+    if (mark) {
+      if (VERBOSE2) { fprintf (stderr, "marked mosaic %s ("OFF_T_FMT"), (%d < %d) || (%d < %f*"OFF_T_FMT")\n", image[MosaicToImage[Nmos][0]].name,  Nmos,  N, IMAGE_TOOFEW,  N, IMAGE_GOOD_FRACTION,  N_onMosaic[Nmos]); }
+      myMosaic[0].flags |= ID_IMAGE_PHOTOM_FEW;
+      info->Nfew ++;
+      if (testImage) {
+        fprintf (stderr, "NOTE: *** marked test image poor : %d %d %d***\n", (int) N, (int) IMAGE_TOOFEW, (int) (IMAGE_GOOD_FRACTION*N_onMosaic[Nmos]));
+      }
+    } else {
+      myMosaic[0].flags &= ~ID_IMAGE_PHOTOM_FEW;
+    }
+  }
+  liststats (list, dlist, NULL, N, &stats);
+  if (VERBOSE2 && info->PoorImages) fprintf (stderr, "Mmos: %f %f %d %d\n", stats.mean, stats.sigma, stats.Nmeas, N);
+  myMosaic[0].Mcal  = stats.mean;
+  myMosaic[0].dMcal = stats.error;
+  myMosaic[0].nFitPhotom = N;
+  myMosaic[0].Xm    = 100.0*log10(stats.chisq);
+  if (testImage) {
+    fprintf (stderr, "test image %d (%d) %f %f %d ... ", (int) Nmos, myMosaic[0].start, stats.mean, stats.error, myMosaic[0].nFitPhotom);
+  }
+  plot_setMcal (list, N, &stats, CLOUD_TOLERANCE);
+  // bright end scatter
+  liststats (Mlist, dMlist, NULL, Nbright, &stats);
+  myMosaic[0].dMsys = stats.sigma;
+  if (myMosaic[0].Mcal < -CLOUD_TOLERANCE) {
+    myMosaic[0].Mcal = 0.0;
+  }
+  if (testImage) {
+    fprintf (stderr, "%f %f  :  %d %f\n", myMosaic[0].Mcal, myMosaic[0].dMsys, myMosaic[0].Xm, pow(10.0, 0.01*myMosaic[0].Xm));
+  }
+  // minUbercalDist calculated here is the min value for any star owned by this image
+  // since this particular image is tied to that star, bump its distance by 1
+  myMosaic[0].ubercalDist = minUbercalDist + 1;
+  // propagate ubercalDist to the images
+  for (j = 0; j < MosaicN_Image[Nmos]; j++) {
+    off_t im = MosaicToImage[Nmos][j];
+    image[im].ubercalDist = myMosaic[0].ubercalDist;
+    // fprintf (stderr, "%d %d %d\n", (int) i, (int) im, image[im].ubercalDist);
+  }
+  return TRUE;
+}
+int setMmos_threaded (Catalog *catalog, int PoorImages, FlatCorrectionTable *flatcorr) {
+  int i;
+  off_t N;
+  Image *image = getimages (&N, NULL);
+  fprintf (stderr, "limiting negative clouds to %f\n", CLOUD_TOLERANCE);
+  if (PoorImages) {
+    // XXX use bad stars and measurements for PoorImages? or not?
+    // IMAGE_BAD = STAR_BAD = MEAS_BAD = 0;
+    IMAGE_BAD = 0;
+  }
+  off_t Nmax = 0;
+  for (i = 0; i < Nmosaic; i++) {
+    Nmax = MAX (Nmax, N_onMosaic[i]);
+  }
+  SetMmosInfo summary;
+  SetMmosInfoInit (&summary, Nmax, FALSE, PoorImages);
+  pthread_attr_t attr;
+  pthread_attr_init (&attr);
+  pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
+  pthread_t *threads;
+  ALLOCATE (threads, pthread_t, NTHREADS);
+  ThreadInfo *threadinfo;
+  ALLOCATE (threadinfo, ThreadInfo, NTHREADS);
+  // each time this function is called, we cycle through the available mosaics
+  // make sure we start at 0
+  nextMosaic = 0;;
+  // launch N worker threads
+  for (i = 0; i < NTHREADS; i++) {
+    threadinfo[i].entry = i;
+    threadinfo[i].state = THREAD_RUN;
+    threadinfo[i].catalog  =  catalog;
+    threadinfo[i].image    =    image;
+    threadinfo[i].flatcorr = flatcorr;
+    SetMmosInfoInit (&threadinfo[i].info, Nmax, FALSE, PoorImages);
+    pthread_create (&threads[i], NULL, setMmos_worker, &threadinfo[i]);
+  }
+  pthread_attr_destroy (&attr);
+  // wait until all threads have finished
+  while (1) {
+    int allDone = TRUE;
+    for (i = 0; i < NTHREADS; i++) {
+      if (threadinfo[i].state == THREAD_RUN) allDone = FALSE;
+    }
+    if (allDone) {
+      break;
+    }
+    usleep (500000);
+  }
+  // all threads are done, free the threads array and grab the info
+  free (threads);
+  // report stats & summary from the threads
+  for (i = 0; i < NTHREADS; i++) {
+    fprintf (stderr, "setMmos thread %d : %d mosaics marked having too few measurements (Nbad: %d, Ncal: %d, Ngrid: %d, Nrel: %d, Nsys: %d), %d partials skipped\n",
+             i,
+             threadinfo[i].info.Nfew,
+             threadinfo[i].info.Nbad,
+             threadinfo[i].info.Ncal,
+             threadinfo[i].info.Ngrid,
+             threadinfo[i].info.Nrel,
+             threadinfo[i].info.Nsys,
+             threadinfo[i].info.Nskip);
+    SetMmosInfoAccum (&summary, &threadinfo[i].info);
+  }
+  fprintf (stderr, "total : %d mosaics marked having too few measurements (Nbad: %d, Ncal: %d, Ngrid: %d, Nrel: %d, Nsys: %d), %d partials skipped\n",
+           summary.Nfew,
+           summary.Nbad,
+           summary.Ncal,
+           summary.Ngrid,
+           summary.Nrel,
+           summary.Nsys,
+           summary.Nskip);
+  free (threadinfo);
+  npass_output ++;
+  if (PoorImages) {
+    IMAGE_BAD = ID_IMAGE_PHOTOM_POOR | ID_IMAGE_PHOTOM_FEW | ID_IMAGE_PHOTOM_SKIP;
+    STAR_BAD  = ID_STAR_POOR | ID_STAR_FEW;
+    MEAS_BAD  = ID_MEAS_NOCAL | ID_MEAS_POOR_PHOTOM | ID_MEAS_SKIP_PHOTOM | ID_MEAS_AREA;
+  }
+  return TRUE;
+}
+void *setMmos_worker (void *data) {
+  ThreadInfo *threadinfo = data;
+  SetMmosInfo results;
+  SetMmosInfoInit (&results, threadinfo->info.Nmax, TRUE, threadinfo->info.PoorImages); // allocate list, dlist arrays here
+  while (1) {
+    off_t i = getNextMosaicForThread();
+    if (i == -1) {
+      threadinfo->state = THREAD_DONE;
+      return NULL;
+    }
+    Catalog *catalog = threadinfo->catalog;
+    FlatCorrectionTable *flatcorr = threadinfo->flatcorr;
+    Image *image = threadinfo->image;
+    setMmos_mosaic (&mosaic[i], i, image, catalog, &results, flatcorr);
+    SetMmosInfoAccum (&threadinfo->info, &results);
+  }
+  SetMmosInfoFree (&results);
+  return NULL;
+}
 // When we rationalize the images/mosaics, we are driving the negative cloud images back
 // to 0.0.  At the same time, we make a guess to the effective impact on all other images,
 …
   for (i = 0; i < Nmosaic; i++) {
     if (mosaic[i].flags & IMAGE_BAD) continue;
+    if (mosaic[i].skipCal) continue;
     list[n] = mosaic[i].Mcal;
     dlist[n] = 1;
 …
+  }
+  liststats (list, dlist, n, &stats);
+  liststats_setmode (&stats, "MEAN");
+  liststats (list, dlist, NULL, n, &stats);
   free (list);
   free (dlist);
 …
   n = 0;
   for (i = 0; i < Nmosaic; i++) {
     if (mosaic[i].flags & IMAGE_BAD) continue;
+    if (mosaic[i].skipCal) continue;
     list[n] = mosaic[i].dMcal;
     dlist[n] = 1;
 …
+  }
+  liststats (list, dlist, n, &stats);
+  liststats_setmode (&stats, "MEAN");
+  liststats (list, dlist, NULL, n, &stats);
   free (list);
   free (dlist);
 …
   for (i = 0; i < Nmosaic; i++) {
     if (mosaic[i].flags & IMAGE_BAD)  continue;
+    if (mosaic[i].skipCal) continue;
     N = 0;
 …
   // fprintf (stderr, "Nmosaic: "OFF_T_FMT", n: "OFF_T_FMT"\n",  Nmosaic,  n);
+  liststats (list, dlist, n, &stats);
+  liststats_setmode (&stats, "MEAN");
+  liststats (list, dlist, NULL, n, &stats);
   free (list);
   free (dlist);
 …
   n = 0;
   for (i = 0; i < Nmosaic; i++) {
     if (mosaic[i].flags & IMAGE_BAD) continue;
+    if (mosaic[i].skipCal) continue;
     list[n] = pow(10.0, 0.01*mosaic[i].Xm);
     dlist[n] = 1;
 …
+  }
+  liststats (list, dlist, n, &stats);
+  liststats_setmode (&stats, "MEAN");
+  liststats (list, dlist, NULL, n, &stats);
   free (list);
   free (dlist);
 …
   double *mlist, *slist, *dlist;
   double MaxOffset, MedOffset, MaxScatter;
-  StatType stats;
   if (!MOSAIC_ZEROPT) return;
 …
   for (i = N = 0; i < Nmosaic; i++) {
     if (mosaic[i].flags & IMAGE_BAD) continue;
+    if (mosaic[i].skipCal) continue;
     mlist[N] = mosaic[i].Mcal;
     slist[N] = mosaic[i].dMcal;
 …
     N++;
+  }
+  initstats ("MEAN");
+  liststats (mlist, dlist, N, &stats);
+  StatType stats;
+  liststats_setmode (&stats, "MEAN");
+  liststats (mlist, dlist, NULL, N, &stats);
   MaxOffset = MAX (IMAGE_OFFSET, 2*stats.sigma);
   MedOffset = stats.median;
+  liststats (slist, dlist, N, &stats);
+  liststats (slist, dlist, NULL, N, &stats);
   MaxScatter = MAX (IMAGE_SCATTER, 2*stats.median);
   fprintf (stderr, "Mrel: %f, dMrel: %f, Max Scatter: %f, Max Offset: %f\n", MedOffset, stats.median, MaxScatter, MaxOffset);
 …
     // if we are keeping ubercal sacrosanct, then we should not be allowed to break them...
     if (KEEP_UBERCAL && (mosaic[i].flags & ID_IMAGE_PHOTOM_UBERCAL)) continue;
+    if (mosaic[i].flags & (ID_IMAGE_PHOTOM_FEW | ID_IMAGE_PHOTOM_SKIP)) continue;
+    if (mosaic[i].skipCal) continue;
     mark = FALSE;
 …
   fprintf (stderr, OFF_T_FMT" mosaics marked poor ("OFF_T_FMT" scatter, "OFF_T_FMT" offset)\n",  Nmark, Nscatter, Noffset);
+  initstats (STATMODE);
   free (mlist);
   free (slist);

trunk/Ohana/src/relphot/src/StarOps.c

-              r33651
+              r33963
 static int Nmax;
+static double *list;
+static double *dlist;
+// When we rationalize the images/mosaics, we are driving the negative cloud images back
+// to 0.0.  At the same time, we make a guess to the effective impact on all other images,
+// driven by the coupling of common stars.  This array carries the impact of those offsets
+// on each star
+static double *Moffset;
+typedef struct {
+  int Nfew;
+  int Ncode;
+  int Nsys;
+  int Nbad;
+  int Ncal;
+  int Nmos;
+  int Ngrid;
+  double *list;
+  double *dlist;
+  double *wlist;
+  double *aplist;
+  double *daplist;
+} SetMrelInfo;
+enum {THREAD_RUN, THREAD_DONE};
+typedef struct {
+  int entry;
+  int state;
+  Catalog *catalog;
+  int Ncatalog;
+  FlatCorrectionTable *flatcorr;
+  SetMrelInfo summary;
+} ThreadInfo;
+// ** internal functions:
+void *setMrel_worker (void *data);
+int setMrel_threaded (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr);
+int setMrel_catalog (Catalog *catalog, int Nc, int pass, FlatCorrectionTable *flatcorr, SetMrelInfo *results, int Nsecfilt);
+int print_measure_set (Average *average, SecFilt *secfilt, Measure *measure);
+// we want to allocate the stats list,dlist arrays only once (or once per thread).
+// this function finds the largest array so we can allocate that max size when needed
 void initMrel (Catalog *catalog, int Ncatalog) {
 …
+    }
+  }
-  ALLOCATE (list,    double, MAX (1, Nmax));
-  ALLOCATE (dlist,   double, MAX (1, Nmax));
-  ALLOCATE (Moffset, double, MAX (1, Nmax));
+}
 …
+}
+int setMrel (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr) {
+  off_t j, k, m;
+  int i, N, Nfew, Nsys, Nbad, Ncal, Nmos, Ngrid;
+  float Msys, Mcal, Mmos, Mgrid;
+  StatType stats;
+  int Nsecfilt = GetPhotcodeNsecfilt ();
+  Nfew = Nsys = Nbad = Ncal = Nmos = Ngrid = 0;
+  for (i = 0; i < Ncatalog; i++) {
+    for (j = 0; j < catalog[i].Naverage; j++) {
+      // XXX accumulate all secfilt values in a single pass?
+      int minUbercalDist = 1000;
+      int Ns;
+      for (Ns = 0; Ns < Nphotcodes; Ns++) {
+        int thisCode = photcodes[Ns][0].code;
+        int Nsec = GetPhotcodeNsec(thisCode);
+        /* calculate the average mag in this SEC photcode for a single star */
+        // skip bad stars
+        if (catalog[i].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD) continue;
+        N = 0;
+        m = catalog[i].averageT[j].measureOffset;
+        for (k = 0; k < catalog[i].averageT[j].Nmeasure; k++, m++) {
+          // skip measurements that do not match the current photcode
+          PhotCode *code = GetPhotcodebyCode (catalog[i].measureT[m].photcode);
+          if (!code) continue;
+          if (code->equiv != thisCode) { continue; }
+          if (catalog[i].measureT[m].dbFlags & MEAS_BAD) { Nbad ++; continue; }
+          if (getImageEntry (m, i) < 0) {
+            Mcal = Mmos = Mgrid = 0;
+          } else {
+            Mcal  = getMcal  (m, i, flatcorr, catalog);
+            if (isnan(Mcal)) { Ncal ++; continue; }
+            Mmos  = getMmos  (m, i);
+            if (isnan(Mmos)) { Nmos ++; continue; }
+            Mgrid = getMgrid (m, i);
+            if (isnan(Mgrid)) { Ngrid++; continue; }
+          }
+          Msys = PhotSysTiny (&catalog[i].measureT[m], &catalog[i].averageT[j], &catalog[i].secfilt[j*Nsecfilt]);
+          if (isnan(Msys)) { Nsys++; continue; }
+          list[N] = Msys - Mcal - Mmos - Mgrid;
+          int myUbercalDist = getUbercalDist(m,i);
+          minUbercalDist = MIN(minUbercalDist, myUbercalDist);
+          // dlist gives the error, which is used as the weight in WT_MEAN.
+          // we can modify the resulting weight in a few ways:
+          // 1) MIN_ERROR guarantees a floor
+          // 2) photomErrSys is added in quadrature as a sytematic error, set per photcode
+          // 3) UBERCAL measurements can have their weight increased by a big factor to help tie down the averages
+          // 4) some reference photcode of some kind can be specified as fixed and have a high weight
+          dlist[N] = MAX (hypot(catalog[i].measureT[m].dM, code->photomErrSys), MIN_ERROR);
+          // up-weight the ubercal values (or convergence can take a long time...)
+          if (catalog[i].measureT[m].dbFlags & ID_MEAS_PHOTOM_UBERCAL) {
+            dlist[N] = MAX (0.1*catalog[i].measureT[m].dM, MIN_ERROR);
+          }
+          // tie down reference photometry if the -refcode (code) option is selected
+          // eg, -refcode g_SDSS
+          // this probably makes no sense in the context of multifilter analysis
+          if (refPhotcode) {
+            if (code->code == refPhotcode->code) {
+              // tiny error -> large weight
+              // dlist[N] = MAX (0.01*catalog[i].measureT[m].dM, MIN_ERROR);
+              dlist[N] = 0.0001;
+            }
+          }
+          N++;
+        }
+        // when performing the grid analysis, STAR_TOOFEW will be set to 1;
+        if (N <= STAR_TOOFEW) { /* too few measurements */
+          // fprintf (f, "%10.6f %10.6f %d %d %d\n", catalog[i].averageT[j].R, catalog[i].averageT[j].D, catalog[i].measureT[catalog[i].averageT[j].measureOffset].imageID, N, STAR_TOOFEW);
+          catalog[i].secfilt[Nsecfilt*j+Nsec].flags |= ID_STAR_FEW;
+          Nfew ++;
+        } else {
+          catalog[i].secfilt[Nsecfilt*j+Nsec].flags &= ~ID_STAR_FEW;
+        }
+        liststats (list, dlist, N, &stats);
+        catalog[i].secfilt[Nsecfilt*j+Nsec].M  = stats.mean;
+        catalog[i].secfilt[Nsecfilt*j+Nsec].dM = stats.sigma;
+        catalog[i].secfilt[Nsecfilt*j+Nsec].Xm = (stats.Nmeas > 1) ? 100.0*log10(stats.chisq + 1e-4) : NAN_S_SHORT;
+        catalog[i].secfilt[Nsecfilt*j+Nsec].ubercalDist = minUbercalDist;
+      }
+    }
+  }
+  fprintf (stderr, "%d stars marked having too few measurements (Nbad: %d, Ncal: %d, Nmos: %d, Ngrid: %d, Nsys: %d)\n", Nfew, Nbad, Ncal, Nmos, Ngrid, Nsys);
+  return (TRUE);
+}
+int print_measure_set (Average *average, SecFilt *secfilt, Measure *measure) {
+  off_t k;
+  int Nsecfilt = GetPhotcodeNsecfilt ();
+  off_t m = average[0].measureOffset;
+  for (k = 0; k < average[0].Nmeasure; k++, m++) {
+    fprintf (stderr, "meas: %08x\n", measure[m].dbFlags);
+  }
+  int Ns;
+  for (Ns = 0; Ns < Nsecfilt; Ns++) {
+    fprintf (stderr, "secf: %08x\n", secfilt[Ns].flags);
+  }
+  return 1;
+}
+# define MARK_SKIP_MEAS \
+  catalog[i].measureT[m].dbFlags |= ID_MEAS_SKIP_PHOTOM; \
+  catalog[i].measure [m].dbFlags |= ID_MEAS_SKIP_PHOTOM;
+void SetMrelInfoInit (SetMrelInfo *results, int allocLists) {
+  results->Nfew  = 0;
+  results->Ncode  = 0;
+  results->Nsys  = 0;
+  results->Nbad  = 0;
+  results->Ncal  = 0;
+  results->Nmos  = 0;
+  results->Ngrid = 0;
+  if (allocLists) {
+    ALLOCATE (results->list,  double, Nmax);
+    ALLOCATE (results->dlist, double, Nmax);
+    ALLOCATE (results->wlist, double, Nmax);
+  }
+}
+void SetMrelInfoFree (SetMrelInfo *results) {
+  free (results->list);
+  free (results->dlist);
+  free (results->wlist);
+}
+void SetMrelInfoAccum (SetMrelInfo *summary, SetMrelInfo *results) {
+  summary->Nfew  += results->Nfew ;
+  summary->Ncode  += results->Ncode ;
+  summary->Nsys  += results->Nsys ;
+  summary->Nbad  += results->Nbad ;
+  summary->Ncal  += results->Ncal ;
+  summary->Nmos  += results->Nmos ;
+  summary->Ngrid += results->Ngrid;
+}
+// mutex to lock setMrel_worker operations
+static pthread_mutex_t setMrel_mutex = PTHREAD_MUTEX_INITIALIZER;
+static int nextCatalog = 0;
+// we have an array of catalogs (catalog, Ncatalog).  we need to hand out catalogs one at a time to
+// the worker threads as they need
+off_t getNextCatalogForThread (int Ncatalog) {
+  pthread_mutex_lock (&setMrel_mutex);
+  if (nextCatalog >= Ncatalog) {
+    pthread_mutex_unlock (&setMrel_mutex);
+    return (-1);
+  }
+  int thisCatalog = nextCatalog;
+  nextCatalog ++;
+  pthread_mutex_unlock (&setMrel_mutex);
+  return (thisCatalog);
+}
 // setMrel and setMrelOutput are extremely similar, but have slightly different implications:
 …
 // * setMrelOutput skips stars for which there are too few good measurements
 // * setMrelOutput is meant to be called repeatedly, relaxing the criteria for 'good' on each pass
+// setMave is also similar to the above.  but:
+// * setMave is called by relphot -update-objects
+// * setMave excludes all detections with (PSF_QF < 0.85), setMrelOutput allows these for PASS > 2
+// * setMave updates 2MASS average flags
+// * setMave updates average EXT flags (PS1 and 2MASS)
+// * setMrelOutput updates 2MASS average flags
+// * setMrelOutput updates average EXT flags (PS1 and 2MASS)
+int setMrel (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr) {
+  int i;
+  if (NTHREADS) {
+    int status = setMrel_threaded (catalog, Ncatalog, flatcorr);
+    return status;
+  }
+  int Nsecfilt = GetPhotcodeNsecfilt ();
+  SetMrelInfo summary, results;
+  SetMrelInfoInit (&summary, FALSE);
+  SetMrelInfoInit (&results, TRUE);
+  for (i = 0; i < Ncatalog; i++) {
+    // pass == -1 for anything other than the final pass
+    setMrel_catalog (catalog, i, -1, flatcorr, &results, Nsecfilt);
+    SetMrelInfoAccum (&summary, &results);
+  }
+  if (VERBOSE2) fprintf (stderr, "%d stars with no data in photcode, %d stars marked having too few measurements (Nbad: %d, Ncal: %d, Nmos: %d, Ngrid: %d, Nsys: %d)\n", summary.Ncode, summary.Nfew, summary.Nbad, summary.Ncal, summary.Nmos, summary.Ngrid, summary.Nsys);
+  SetMrelInfoFree (&results);
+  return (TRUE);
+}
 int setMrelOutput (Catalog *catalog, int Ncatalog, int pass, FlatCorrectionTable *flatcorr) {
+  int i, N;
+  off_t j, k, m, Nmax;
+  float Msys, Mcal, Mmos, Mgrid, Map;
+  double *list, *dlist, *aplist, *daplist;
+  int i;
+  int Nsecfilt = GetPhotcodeNsecfilt ();
+  SetMrelInfo summary, results;
+  SetMrelInfoInit (&summary, FALSE);
+  SetMrelInfoInit (&results, TRUE); // allocates results->list,dlist,wlist
+  ALLOCATE (results.aplist, double, Nmax);
+  ALLOCATE (results.daplist, double, Nmax);
+  for (i = 0; i < Ncatalog; i++) {
+    setMrel_catalog  (catalog, i, pass, flatcorr, &results, Nsecfilt); // XXX add arguments as needed for options
+    SetMrelInfoAccum (&summary, &results);
+  }
+  if (VERBOSE2) fprintf (stderr, "%d stars with no data in photcode, %d stars marked having too few measurements (Nbad: %d, Ncal: %d, Nmos: %d, Ngrid: %d, Nsys: %d)\n", summary.Ncode, summary.Nfew, summary.Nbad, summary.Ncal, summary.Nmos, summary.Ngrid, summary.Nsys);
+  SetMrelInfoFree (&results);
+  free (results.aplist);
+  free (results.daplist);
+  return (TRUE);
+}
+int setMrel_threaded (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr) {
+  int i;
+  SetMrelInfo summary;
+  SetMrelInfoInit (&summary, FALSE);
+  pthread_attr_t attr;
+  pthread_attr_init (&attr);
+  pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
+  pthread_t *threads;
+  ALLOCATE (threads, pthread_t, NTHREADS);
+  ThreadInfo *threadinfo;
+  ALLOCATE (threadinfo, ThreadInfo, NTHREADS);
+  // each time this function is called, we cycle through the available catalogs.
+  // make sure we start at 0
+  nextCatalog = 0;;
+  // launch N worker threads
+  for (i = 0; i < NTHREADS; i++) {
+    threadinfo[i].entry = i;
+    threadinfo[i].state = THREAD_RUN;
+    threadinfo[i].catalog  =  catalog;
+    threadinfo[i].Ncatalog = Ncatalog;
+    threadinfo[i].flatcorr = flatcorr;
+    SetMrelInfoInit (&threadinfo[i].summary, FALSE);
+    pthread_create (&threads[i], NULL, setMrel_worker, &threadinfo[i]);
+  }
+  pthread_attr_destroy (&attr);
+  // wait until all threads have finished
+  while (1) {
+    int allDone = TRUE;
+    for (i = 0; i < NTHREADS; i++) {
+      if (threadinfo[i].state == THREAD_RUN) allDone = FALSE;
+    }
+    if (allDone) {
+      break;
+    }
+    usleep (500000);
+  }
+  // all threads are done, free the threads array and grab the info
+  free (threads);
+  // report stats & summary from the threads
+  for (i = 0; i < NTHREADS; i++) {
+    if (VERBOSE2) fprintf (stderr, "setMrel thread %d : %d stars with no data in photcode, %d stars marked having too few measurements (Nbad: %d, Ncal: %d, Nmos: %d, Ngrid: %d, Nsys: %d)\n",
+             i,
+             threadinfo[i].summary.Ncode,
+             threadinfo[i].summary.Nfew,
+             threadinfo[i].summary.Nbad,
+             threadinfo[i].summary.Ncal,
+             threadinfo[i].summary.Nmos,
+             threadinfo[i].summary.Ngrid,
+             threadinfo[i].summary.Nsys);
+    SetMrelInfoAccum (&summary, &threadinfo[i].summary);
+  }
+  if (VERBOSE2) fprintf (stderr, "total : %d stars with no data in photcode, %d stars marked having too few measurements (Nbad: %d, Ncal: %d, Nmos: %d, Ngrid: %d, Nsys: %d)\n", summary.Ncode, summary.Nfew, summary.Nbad, summary.Ncal, summary.Nmos, summary.Ngrid, summary.Nsys);
+  free (threadinfo);
+  return TRUE;
+}
+void *setMrel_worker (void *data) {
+  ThreadInfo *threadinfo = data;
+  int Nsecfilt = GetPhotcodeNsecfilt ();
+  SetMrelInfo results;
+  SetMrelInfoInit (&results, TRUE); // allocate list, dlist arrays here
+  while (1) {
+    off_t i = getNextCatalogForThread(threadinfo->Ncatalog);
+    if (i == -1) {
+      threadinfo->state = THREAD_DONE;
+      return NULL;
+    }
+    Catalog *catalog = threadinfo->catalog;
+    FlatCorrectionTable *flatcorr = threadinfo->flatcorr;
+    // pass == -1 for anything other than the final pass
+    setMrel_catalog (catalog, i, -1, flatcorr, &results, Nsecfilt);
+    SetMrelInfoAccum (&threadinfo->summary, &results);
+  }
+  SetMrelInfoFree (&results);
+  return NULL;
+}
+# define SKIP_THIS_MEAS(REASON) {                               \
+    catalog[Nc].measureT[m].dbFlags |= ID_MEAS_SKIP_PHOTOM;     \
+    if (catalog[Nc].measure) {                                  \
+      catalog[Nc].measure [m].dbFlags |= ID_MEAS_SKIP_PHOTOM;   \
+    }                                                           \
+    results->REASON ++;                                         \
+    continue; }
+// setMrel_catalog is used in 3 different contexts:
+// * during the main relphot iterations:
+// ** operations only apply to measureT / averageT
+// ** we are applying the analysis to the bright subset catalog
+// ** we skip any stars found to be bad (STAR_BAD)
+// * during the final pass
+// ** operations are applied to all objects
+// ** we have special tests for PS1 extended data, 2MASS extended & good data, & synthetic photometry
+// * during the stand-alone '-averages' mode (relphot_objects)
+// ** no image data is loaded
+// ** we only use the provided calibration (measure.Mcal)
+// ** we skip outlier rejection of measurements
+// set the Mrel values for the specified catalog
+// NOTE: here 'catalog' is a pointer to a specific catalog, not the root of the array
+int setMrel_catalog (Catalog *catalog, int Nc, int pass, FlatCorrectionTable *flatcorr, SetMrelInfo *results, int Nsecfilt) {
+  off_t j, k, m;
+  int N;
+  float Msys, Mcal, Mmos, Mgrid;
   StatType stats, apstats;
+  int Nsec;
+  int Nsecfilt = GetPhotcodeNsecfilt ();
+  /* Nmeasure is now different, need to reallocate */
+  Nmax = 0;
+  for (i = 0; i < Ncatalog; i++) {
+    for (j = 0; j < catalog[i].Naverage; j++) {
+      Nmax = MAX (Nmax, catalog[i].averageT[j].Nmeasure);
+    }
+  }
+  ALLOCATE (list, double, MAX (1, Nmax));
+  ALLOCATE (dlist, double, MAX (1, Nmax));
+  ALLOCATE (aplist, double, MAX (1, Nmax));
+  ALLOCATE (daplist, double, MAX (1, Nmax));
+  for (i = 0; i < Ncatalog; i++) {
+    for (j = 0; j < catalog[i].Naverage; j++) {
+      if (FALSE && (catalog[i].average[j].objID == 0x46a4) && (catalog[i].average[j].catID == 0xf40e)) {
+        fprintf (stderr, "test obj\n");
+        print_measure_set (&catalog[i].average[j], &catalog[i].secfilt[j*Nsecfilt], catalog[i].measure);
+      }
+  liststats_setmode (&stats, STATMODE);
+  liststats_setmode (&apstats, STATMODE);
+  double *list    = results->list;
+  double *dlist   = results->dlist;
+  double *wlist   = results->wlist;
+  double *aplist  = results->aplist;
+  double *daplist = results->daplist;
+  SetMrelInfoInit (results, FALSE); // do not allocate list,dlist,wlist arrays
+  int isSetMrelFinal = (pass >= 0);
+  for (j = 0; j < catalog[Nc].Naverage; j++) {
+    // XXX accumulate all secfilt values in a single pass?
+    // option for a test print
+    if (FALSE && (catalog[Nc].average[j].objID == 0x46a4) && (catalog[Nc].average[j].catID == 0xf40e)) {
+      fprintf (stderr, "test obj\n");
+      print_measure_set (&catalog[Nc].average[j], &catalog[Nc].secfilt[j*Nsecfilt], catalog[Nc].measure);
+    }
+    int GoodPS1 = FALSE;
+    int Good2MASS = FALSE;
+    int Galaxy2MASS = FALSE;
+    int NextPS1 = 0;
+    int NpsfPS1 = 0;
+    int Ns;
+    for (Ns = 0; Ns < Nphotcodes; Ns++) {
+      int thisCode = photcodes[Ns][0].code;
+      int Nsec = GetPhotcodeNsec(thisCode);
+      /* calculate the average mag in this SEC photcode for a single star */
+      /* star/photcodes already calibrated */
+      if ( isSetMrelFinal && catalog[Nc].found[Nsecfilt*j+Nsec]) continue;
+      // skip bad stars
+      if (!isSetMrelFinal && (catalog[Nc].secfilt[Nsecfilt*j+Nsec].flags & STAR_BAD)) continue;
+      int Ncode = 0;
+      int Next = 0;
+      int haveSynth = FALSE;
+      int forceSynth = FALSE;
+      int forceSynthEntry = -1;
+      int haveUbercal = FALSE;
       int minUbercalDist = 1000;
+      int Ns;
+      for (Ns = 0; Ns < Nphotcodes; Ns++) {
+        int thisCode = photcodes[Ns][0].code;
+        Nsec = GetPhotcodeNsec(thisCode);
+        /* star/photcodes already calibrated */
+        if (catalog[i].found[Nsecfilt*j+Nsec]) continue;
+        int Ncode = 0;
+        int Next = 0;
+        int haveSynth = FALSE;
+        int haveUbercal = FALSE;
+        int isRefPhot = FALSE;
+        N = 0;
+        m = catalog[i].averageT[j].measureOffset;
+        for (k = 0; k < catalog[i].averageT[j].Nmeasure; k++, m++) {
+          // skip measurements that do not match the current photcode
+          PhotCode *code = GetPhotcodebyCode (catalog[i].measureT[m].photcode);
+          if (!code) continue;
+          if (code->equiv != thisCode) { continue; }
+          Ncode ++;
+          if (catalog[i].measureT[m].dbFlags & MEAS_BAD) continue;
+          if (getImageEntry (m, i) < 0) {
+            // these detetions have no image (eg, ref values such as 2MASS)
+            Mcal = Mmos = Mgrid = 0;
+            isRefPhot = TRUE;
+          } else {
+            Mcal  = getMcal  (m, i, flatcorr, catalog);
+            if (isnan(Mcal)) {
+              MARK_SKIP_MEAS;
+              continue;
+            }
+            Mmos  = getMmos  (m, i);
+            if (isnan(Mmos)) {
+              MARK_SKIP_MEAS;
+              continue;
+            }
+            Mgrid = getMgrid (m, i);
+            if (isnan(Mgrid)) {
+              MARK_SKIP_MEAS;
+              continue;
+            }
+          }
+          Msys = PhotSysTiny (&catalog[i].measureT[m], &catalog[i].averageT[j], &catalog[i].secfilt[j*Nsecfilt]);
+          if (isnan(Msys)) {
+            if (!isRefPhot) {
+              MARK_SKIP_MEAS;
+            }
+            continue;
+          }
+          list[N] = Msys - Mcal - Mmos - Mgrid;
+          Map = PhotAper (&catalog[i].measure[m]);
+      N = 0;
+      m = catalog[Nc].averageT[j].measureOffset;
+      for (k = 0; k < catalog[Nc].averageT[j].Nmeasure; k++, m++) {
+        // skip measurements that do not match the current photcode
+        PhotCode *code = GetPhotcodebyCode (catalog[Nc].measureT[m].photcode);
+        if (!code) continue;
+        if (code->equiv != thisCode) { continue; }
+        Ncode ++;
+        if (catalog[Nc].measureT[m].dbFlags & MEAS_BAD) SKIP_THIS_MEAS(Nbad);
+        if (getImageEntry (m, Nc) < 0) {
+          // measurements without an image are either external reference photometry or
+          // data for which the associated image has not been loaded (probably because of
+          // overlaps).  Msys + measure.Mcal is our best guess of the true magnitude
+          Mmos = Mgrid = 0;
+          Mcal = catalog[Nc].measureT[m].Mcal; // check that this is zero for loaded REF value
+        } else {
+          Mcal  = getMcal  (m, Nc, flatcorr, catalog);
+          if (isnan(Mcal))  SKIP_THIS_MEAS(Ncal);
+          Mmos  = getMmos  (m, Nc);
+          if (isnan(Mmos))  SKIP_THIS_MEAS(Nmos);
+          Mgrid = getMgrid (m, Nc);
+          if (isnan(Mgrid)) SKIP_THIS_MEAS(Ngrid);
+        }
+        Msys = PhotSysTiny (&catalog[Nc].measureT[m], &catalog[Nc].averageT[j], &catalog[Nc].secfilt[j*Nsecfilt]);
+        if (isnan(Msys)) SKIP_THIS_MEAS(Nsys);
+        list[N] = Msys - Mcal - Mmos - Mgrid;
+        int myUbercalDist = getUbercalDist(m,Nc);
+        minUbercalDist = MIN(minUbercalDist, myUbercalDist);
+        if (isSetMrelFinal) {
+          float Map = PhotAper (&catalog[Nc].measure[m]);
           aplist[N] = Map - Mcal - Mmos - Mgrid;
+          int myUbercalDist = getUbercalDist(m,i);
+          minUbercalDist = MIN(minUbercalDist, myUbercalDist);
+          // count the extended detections
+          if ((catalog[i].measure[m].photcode >= 10000) && (catalog[i].measure[m].photcode <= 10500)) {
+            if (!isnan(catalog[i].measure[m].Map)) {
+              if (catalog[i].measure[m].M - catalog[i].measure[m].Map > 0.5) {
+          // special options for PS1 data
+          if ((catalog[Nc].measure[m].photcode >= 10000) && (catalog[Nc].measure[m].photcode <= 10500)) {
+            // count the extended detections
+            if (!isnan(catalog[Nc].measure[m].Map)) {
+              if (catalog[Nc].measure[m].M - catalog[Nc].measure[m].Map > 0.5) {
                 Next ++;
+                NextPS1 ++;
+              } else {
+                NpsfPS1 ++;
+              }
+            }
+          }
           // count extended detections for 2MASS (XXX NOTE hardwired photcodes 2011, 2012, 2013)
           if ((catalog[i].measure[m].photcode >= 2011) && (catalog[i].measure[m].photcode <= 2013)) {
             if (catalog[i].measure[m].photFlags & 0x00c00000) {
+          if ((catalog[Nc].measure[m].photcode >= 2011) && (catalog[Nc].measure[m].photcode <= 2013)) {
+            if (catalog[Nc].measure[m].photFlags & 0x00c00000) {
               Next ++;
+              Galaxy2MASS = TRUE;
+            }
+            if ((pass == 0) && !(catalog[i].measure[m].photFlags & 0x00000007)) {
+              // detections without one of these bits should only be used in PASS_1
+              MARK_SKIP_MEAS;
+              continue;
+            }
+          }
+          // ignore SYNTH photocdes until PASS == 4 (where we also accept saturated stars)
+          if ((catalog[i].measure[m].photcode >= 3001) && (catalog[i].measure[m].photcode <= 3005)) {
+            if (pass < 4) {
+              MARK_SKIP_MEAS;
+              continue;
+            }
+            haveSynth = TRUE;
+          }
+          // dlist gives the error, which is used as the weight in WT_MEAN.
+          // we can modify the resulting weight in a few ways:
+          // 1) MIN_ERROR guarantees a floor
+          // 2) photomErrSys is added in quadrature as a sytematic error, set per photcode
+          // 3) UBERCAL measurements can have their weight increased by a big factor to help tie down the averages
+          // 4) some reference photcode of some kind can be specified as fixed and have a high weight
+          dlist[N] = MAX (hypot(catalog[i].measureT[m].dM, code->photomErrSys), MIN_ERROR);
+          // up-weight the ubercal values (or convergence can take a long time...) (XXX make this optional?)
+          if (catalog[i].measureT[m].dbFlags & ID_MEAS_PHOTOM_UBERCAL) {
+            dlist[N] = MAX (0.1*catalog[i].measureT[m].dM, MIN_ERROR);
+            haveUbercal = TRUE;
+          }
+          // tie down reference photometry if the -refcode (code) option is selected
+          // eg, -refcode g_SDSS
+          // this probably makes no sense in the context of multifilter analysis
+          if (refPhotcode) {
+            if (code->code == refPhotcode->code) {
+              // tiny error -> large weight
+              // dlist[N] = MAX (0.01*catalog[i].measureT[m].dM, MIN_ERROR);
+              dlist[N] = 0.0001;
+            }
+          }
+          N++;
+        }
+        if (N < 1) continue;
+        for (m = 0; m < N; m++) {
+          daplist[m] = dlist[m];
+        }
+        // XXX force WT_MEAN or MEAN here?
+        liststats (list, dlist, N, &stats);
+        liststats (aplist, daplist, N, &apstats);
+        catalog[i].found[Nsecfilt*j+Nsec] = TRUE;
+        switch (pass) {
+          case 0:
+            catalog[i].secfilt[Nsecfilt*j+Nsec].flags |= ID_PHOTOM_PASS_0;
+            break;
+          case 1:
+            catalog[i].secfilt[Nsecfilt*j+Nsec].flags |= ID_PHOTOM_PASS_1;
+            break;
+          case 2:
+            catalog[i].secfilt[Nsecfilt*j+Nsec].flags |= ID_PHOTOM_PASS_2;
+            break;
+          case 3:
+            catalog[i].secfilt[Nsecfilt*j+Nsec].flags |= ID_PHOTOM_PASS_3;
+            break;
+          case 4:
+            catalog[i].secfilt[Nsecfilt*j+Nsec].flags |= ID_PHOTOM_PASS_4;
+            break;
+        }
+        if (haveSynth) {
+          catalog[i].secfilt[Nsecfilt*j+Nsec].flags |= ID_SECF_USE_SYNTH;
+        }
+        if (haveUbercal) {
+          catalog[i].secfilt[Nsecfilt*j+Nsec].flags |= ID_SECF_USE_UBERCAL;
+        }
+        /* use sigma or error in dM for output? */
+        catalog[i].secfilt[Nsecfilt*j+Nsec].M  = stats.mean;
+        catalog[i].secfilt[Nsecfilt*j+Nsec].Map = apstats.mean;
+        catalog[i].secfilt[Nsecfilt*j+Nsec].dM = MAX (stats.error, stats.sigma);
+        catalog[i].secfilt[Nsecfilt*j+Nsec].Xm = (stats.Nmeas > 1) ? 100.0*log10(stats.chisq) : NAN_S_SHORT;
+        catalog[i].secfilt[Nsecfilt*j+Nsec].Ncode = Ncode;
+        catalog[i].secfilt[Nsecfilt*j+Nsec].Nused = stats.Nmeas;
+        catalog[i].secfilt[Nsecfilt*j+Nsec].M_80 = 1000 * stats.Upper80;
+        catalog[i].secfilt[Nsecfilt*j+Nsec].M_20 = 1000 * stats.Lower20;
+        catalog[i].secfilt[Nsecfilt*j+Nsec].ubercalDist = minUbercalDist;
+        if ((Next > 0) && (Next > 0.5*N)) {
+          catalog[i].secfilt[Nsecfilt*j+Nsec].flags |= ID_SECF_OBJ_EXT;
+        }
+      }
+    }
+  }
+  free (list);
+  free (dlist);
+  free (aplist);
+  free (daplist);
+  return (TRUE);
+}
+/* grab Nsec for named photcode */
+# define NAMED_PHOTCODE_NSEC(MY_NSEC,NAME)      \
+  short MY_NSEC = -1;                           \
+  {                                             \
+  PhotCode *code = GetPhotcodebyName (NAME);    \
+  if (code) {                                   \
+    MY_NSEC = GetPhotcodeNsec (code->equiv);    \
+  } }
+// For each average object, set the average mags based on existing equiv photometry.
+// NOTE: this function operates on the real Measure & Average structures, not the
+// MeasureTiny & AverageTiny structures
+// NOTE: this function is called on the remote machine -- make sure any (global) options
+// are passed to the relphot_client program
+int setMave (Catalog *catalog, int Ncatalog) {
+  off_t j, k, m, Nmax;
+  int i, Ns, Nsecfilt, N, Nc;
+  float Msys;
+  double *list, *dlist;
+  StatType stats;
+  DVOAverageFlags flagBits;
+  flagBits = ID_OBJ_EXT | ID_OBJ_EXT_ALT | ID_OBJ_GOOD | ID_OBJ_GOOD_ALT;
+  // pre-allocate a list for stats purposes
+  Nmax = 0;
+  for (i = 0; i < Ncatalog; i++) {
+    for (j = 0; j < catalog[i].Naverage; j++) {
+      Nmax = MAX (Nmax, catalog[i].average[j].Nmeasure);
+    }
+  }
+  ALLOCATE (list, double, MAX (1, Nmax));
+  ALLOCATE (dlist, double, MAX (1, Nmax));
+  Nsecfilt = GetPhotcodeNsecfilt ();
+  // we want to raise some bits on the 2MASS (JHK) secfilt flags, if we have 2MASS data
+  NAMED_PHOTCODE_NSEC (Nsec_J, "2MASS_J");
+  NAMED_PHOTCODE_NSEC (Nsec_H, "2MASS_H");
+  NAMED_PHOTCODE_NSEC (Nsec_K, "2MASS_K");
+  for (i = 0; i < Ncatalog; i++) {
+    for (j = 0; j < catalog[i].Naverage; j++) {
+      // update average photometry for each of the average filters
+      // XXX Note that this would be faster if we had an array of results and accumulated
+      // them in a single pass
+      int Next = 0;
+      for (Ns = 0; Ns < Nsecfilt; Ns++) {
+        PhotCode *code = GetPhotcodebyNsec (Ns);
+        Nc = code[0].code;
+        N = 0;
+        m = catalog[i].average[j].measureOffset;
+        for (k = 0; k < catalog[i].average[j].Nmeasure; k++, m++) {
+          if (catalog[i].measure[m].dbFlags & MEAS_BAD) continue;
+          if (GetPhotcodeEquivCodebyCode (catalog[i].measure[m].photcode) != Nc) continue;
+          Msys = PhotSys (&catalog[i].measure[m], &catalog[i].average[j], &catalog[i].secfilt[j*Nsecfilt]);
+          if (isnan(Msys)) continue;
+          // reject POOR detections (PSF_QF < 0.85) and count extended detections
+          // XXX only apply this filter for psphot data from GPC1 for now...
+          if ((catalog[i].measure[m].photcode > 10000) && (catalog[i].measure[m].photcode < 10500)) {
+            if (catalog[i].measure[m].psfQual < 0.85) continue;
+            if (!isnan(catalog[i].measure[m].Map)) {
+              if (catalog[i].measure[m].M - catalog[i].measure[m].Map > 0.5) {
+                Next ++;
+            if (pass == 0) {
+              if (catalog[Nc].measure[m].photFlags & 0x00000007) {
+                Good2MASS = TRUE;
+              } else {
+                // detections without one of these bits should only be used in PASS_1
+                SKIP_THIS_MEAS(Nbad);
+                continue;
+              }
+            }
+          }
+          // XXX make it optional to apply Mcal (do I need an extra field for advisory Mcal?)
+          list[N] = Msys - catalog[i].measure[m].Mcal;
+          dlist[N] = MAX (catalog[i].measure[m].dM, MIN_ERROR);
+          N++;
+        }
+        if (N < 1) continue;
+        liststats (list, dlist, N, &stats);
+        /* use sigma or error in dM for output? */
+        catalog[i].secfilt[Nsecfilt*j+Ns].M  = stats.mean;
+        catalog[i].secfilt[Nsecfilt*j+Ns].dM = MAX (stats.error, stats.sigma);
+        catalog[i].secfilt[Nsecfilt*j+Ns].Xm = (stats.Nmeas > 1) ? 100.0*log10(stats.chisq) : NAN_S_SHORT;
+        catalog[i].secfilt[Nsecfilt*j+Ns].Ncode = N;
+        catalog[i].secfilt[Nsecfilt*j+Ns].Nused = stats.Nmeas;
+        catalog[i].secfilt[Nsecfilt*j+Ns].M_80 = 1000 * stats.Upper80;
+        catalog[i].secfilt[Nsecfilt*j+Ns].M_20 = 1000 * stats.Lower20;
+        if ((Next > 0) && (Next > 0.5*N)) {
+          catalog[i].secfilt[Nsecfilt*j+Ns].flags |= ID_SECF_OBJ_EXT;
+        }
+      }
+      // update average flags based on the detection stats.
+      int Galaxy2MASS = FALSE; // best guess for galaxy based on 2MASS J measurements (gal_contam == measure.flags[0x00400000 | 0x00800000])
+      int goodPS1 = FALSE;     // true if any PS1 measurements have psfQual > 0.85
+      int good2MASS = FALSE;   // true if 2MASS J measurements have significant detections
+      int nEXT = 0;
+      int nPSF = 0;     // number of PS1 PSF vs EXT measurements
+      int have2MASS = FALSE;
+      // count, flag good and extended detections
+      m = catalog[i].average[j].measureOffset;
+      for (k = 0; k < catalog[i].average[j].Nmeasure; k++, m++) {
+        // PS1 data :
+        if ((catalog[i].measure[m].photcode >= 10000) && (catalog[i].measure[m].photcode <= 10500)) {
+          if (catalog[i].measure[m].psfQual > 0.85) {
+            goodPS1 = TRUE;
+            if (!isnan(catalog[i].measure[m].Map)) {
+              if (catalog[i].measure[m].M - catalog[i].measure[m].Map > 0.5) {
+                nEXT ++;
+              } else {
+                nPSF ++;
+          // Blindly accepth the SYNTH mags if we are above saturation, otherwise,
+          // ignore SYNTH photcodes until PASS == 4 (where we also accept saturated stars)
+          if ((catalog[Nc].measure[m].photcode >= 3001) && (catalog[Nc].measure[m].photcode <= 3005)) {
+            // something of a hack: force object to use synth values if synth mags >>
+            // saturation (3pi instrumental mags < -15)
+            float MaxMagForceSynth = NAN;
+            switch (catalog[Nc].measure[m].photcode) {
+              case 3001:
+                MaxMagForceSynth = 13.64;
+                break;
+              case 3002:
+                MaxMagForceSynth = 13.76;
+                break;
+              case 3003:
+                MaxMagForceSynth = 13.74;
+                break;
+              case 3004:
+                MaxMagForceSynth = 12.94;
+                break;
+              case 3005:
+                MaxMagForceSynth = 12.01;
+                break;
+            }
+            if (catalog[Nc].measureT[m].M < MaxMagForceSynth) {
+              forceSynth = TRUE;
+              forceSynthEntry = N;
+            } else {
+              if (pass < 4) {
+                SKIP_THIS_MEAS(Nbad);
+                continue;
+              }
+              haveSynth = TRUE;
+            }
+          }
+        }
+        // 2MASS data J-band flags
+        if (catalog[i].measure[m].photcode == 2011) {
+          // only need to do this once (always have JHK triplet; galaxy flag is same for all 3)
+          have2MASS = TRUE;
+          if (catalog[i].measure[m].photFlags & 0x00c00000) {
+            Galaxy2MASS = TRUE;
+        // dlist gives the error per measurement, wlist gives the weight
+        // we can modify the error and weight in a few ways:
+        // 1) MIN_ERROR guarantees a floor
+        // 2) photomErrSys is added in quadrature as a sytematic error, set per photcode
+        // 3) UBERCAL measurements can have their weight increased by a big factor to help tie down the averages
+        // 4) some reference photcode of some kind can be specified as fixed and have a high weight
+        dlist[N] = MAX (hypot(catalog[Nc].measureT[m].dM, code->photomErrSys), MIN_ERROR);
+        wlist[N] = 1.0;
+        // up-weight the ubercal values (or convergence can take a long time...)
+        if (catalog[Nc].measureT[m].dbFlags & ID_MEAS_PHOTOM_UBERCAL) {
+          wlist[N] = 10.0;
+        }
+        // tie down reference photometry if the -refcode (code) option is selected
+        // eg, -refcode g_SDSS
+        // this probably makes no sense in the context of multifilter analysis
+        if (refPhotcode) {
+          if (code->code == refPhotcode->code) {
+            wlist[N] = 100.0;
+          }
+          if (catalog[i].measure[m].photFlags & 0x00000007) {
+            good2MASS = TRUE;
+            if (Nsec_J > -1) catalog[i].secfilt[j*Nsecfilt + Nsec_J].flags |= ID_PHOTOM_PASS_0;
+          } else {
+            if (Nsec_J > -1) catalog[i].secfilt[j*Nsecfilt + Nsec_J].flags |= ID_PHOTOM_PASS_1;
+          }
+        }
+        // 2MASS data H-band flags
+        if (catalog[i].measure[m].photcode == 2012) {
+          if (catalog[i].measure[m].photFlags & 0x00000007) {
+            good2MASS = TRUE;
+            if (Nsec_H > -1) catalog[i].secfilt[j*Nsecfilt + Nsec_H].flags |= ID_PHOTOM_PASS_0;
+          } else {
+            if (Nsec_H > -1) catalog[i].secfilt[j*Nsecfilt + Nsec_H].flags |= ID_PHOTOM_PASS_1;
+          }
+        }
+        // 2MASS data K-band flags
+        if (catalog[i].measure[m].photcode == 2013) {
+          if (catalog[i].measure[m].photFlags & 0x00000007) {
+            good2MASS = TRUE;
+            if (Nsec_K > -1) catalog[i].secfilt[j*Nsecfilt + Nsec_K].flags |= ID_PHOTOM_PASS_0;
+          } else {
+            if (Nsec_K > -1) catalog[i].secfilt[j*Nsecfilt + Nsec_K].flags |= ID_PHOTOM_PASS_1;
+          }
+        }
+      }
+        }
+        N++;
+      }
+      int Nminmeas = isSetMrelFinal ? 1 : STAR_TOOFEW + 1;
+      // when performing the grid analysis, STAR_TOOFEW should be set to 1;
+      if (N < Nminmeas) { /* too few measurements */
+        // fprintf (f, "%10.6f %10.6f %d %d %d\n", catalog[Nc].averageT[j].R, catalog[Nc].averageT[j].D, catalog[Nc].measureT[catalog[Nc].averageT[j].measureOffset].imageID, N, STAR_TOOFEW);
+        catalog[Nc].secfilt[Nsecfilt*j+Nsec].flags |= ID_STAR_FEW;
+        if (Ncode == 0) {
+          results->Ncode ++;
+        } else {
+          results->Nfew ++;
+        }
+        continue;
+      } else {
+        catalog[Nc].secfilt[Nsecfilt*j+Nsec].flags &= ~ID_STAR_FEW;
+      }
+      if (forceSynth) {
+        // use the single SYNTH value instead of the other mags here
+        myAssert ((forceSynthEntry < N) && (forceSynthEntry >= 0), "programming error");
+        list[0]  = list[forceSynthEntry];
+        dlist[0] = dlist[forceSynthEntry];
+        wlist[0] = wlist[forceSynthEntry];
+        N = 1;
+      }
+      liststats (list, dlist, wlist, N, &stats);
+      catalog[Nc].secfilt[Nsecfilt*j+Nsec].M  = stats.mean;
+      catalog[Nc].secfilt[Nsecfilt*j+Nsec].dM = stats.error;
+      catalog[Nc].secfilt[Nsecfilt*j+Nsec].Xm = (stats.Nmeas > 1) ? 100.0*log10(stats.chisq + 1e-4) : NAN_S_SHORT;
+      // when running -averages, we have no information about the images, so we cannot set this
+      if (minUbercalDist > -1) {
+        catalog[Nc].secfilt[Nsecfilt*j+Nsec].ubercalDist = minUbercalDist;
+      }
+      if (isSetMrelFinal) {
+        catalog[Nc].found[Nsecfilt*j+Nsec] = TRUE;
+        catalog[Nc].secfilt[Nsecfilt*j+Nsec].Mstdev = 1000.0*stats.sigma; // Mstdev is in millimags (not enough space for more precision)
+        catalog[Nc].secfilt[Nsecfilt*j+Nsec].Ncode = Ncode;
+        catalog[Nc].secfilt[Nsecfilt*j+Nsec].Nused = stats.Nmeas;
+        catalog[Nc].secfilt[Nsecfilt*j+Nsec].M_80 = 1000 * stats.Upper80;
+        catalog[Nc].secfilt[Nsecfilt*j+Nsec].M_20 = 1000 * stats.Lower20;
+        // NOTE : use the modified weight for apmags as well as psf mags
+        liststats (aplist, daplist, wlist, N, &apstats);
+        catalog[Nc].secfilt[Nsecfilt*j+Nsec].Map  = apstats.mean;
+        // NOTE: for 2MASS measurements, Next should be 1, as should N
+        if ((Next > 0) && (Next > 0.5*N)) {
+          catalog[Nc].secfilt[Nsecfilt*j+Nsec].flags |= ID_SECF_OBJ_EXT;
+        }
+        switch (pass) {
+          case 0:
+            catalog[Nc].secfilt[Nsecfilt*j+Nsec].flags |= ID_PHOTOM_PASS_0;
+            GoodPS1 = TRUE;
+            break;
+          case 1:
+            catalog[Nc].secfilt[Nsecfilt*j+Nsec].flags |= ID_PHOTOM_PASS_1;
+            GoodPS1 = TRUE;
+            break;
+          case 2:
+            catalog[Nc].secfilt[Nsecfilt*j+Nsec].flags |= ID_PHOTOM_PASS_2;
+            GoodPS1 = TRUE;
+            break;
+          case 3:
+            catalog[Nc].secfilt[Nsecfilt*j+Nsec].flags |= ID_PHOTOM_PASS_3;
+            break;
+          case 4:
+            catalog[Nc].secfilt[Nsecfilt*j+Nsec].flags |= ID_PHOTOM_PASS_4;
+            break;
+        }
+        if (haveSynth) {
+          catalog[Nc].secfilt[Nsecfilt*j+Nsec].flags |= ID_SECF_USE_SYNTH;
+        }
+        if (haveUbercal) {
+          catalog[Nc].secfilt[Nsecfilt*j+Nsec].flags |= ID_SECF_USE_UBERCAL;
+        }
+      }
+    }
+    if (isSetMrelFinal) {
+      DVOAverageFlags flagBits = ID_OBJ_EXT | ID_OBJ_EXT_ALT | ID_OBJ_GOOD | ID_OBJ_GOOD_ALT;
       // we attempt to set a few flags here; reset those bits before trying:
+      catalog[i].average[j].flags &= ~flagBits;
+      // XXX set the secfilt bits?
+      if (nEXT && (nEXT > nPSF)) {
+        catalog[i].average[j].flags |= ID_OBJ_EXT;
+      }
+      if (goodPS1) {
+        catalog[i].average[j].flags |= ID_OBJ_GOOD;
+      catalog[Nc].average[j].flags &= ~flagBits;
+      if (NextPS1 && (NextPS1 > NpsfPS1)) {
+        catalog[Nc].average[j].flags |= ID_OBJ_EXT;
+      }
+      if (GoodPS1) {
+        catalog[Nc].average[j].flags |= ID_OBJ_GOOD;
+      }
       if (Galaxy2MASS) {
+        catalog[i].average[j].flags |= ID_OBJ_EXT_ALT;
+        if (Nsec_J > -1) catalog[i].secfilt[j*Nsecfilt + Nsec_J].flags |= ID_SECF_OBJ_EXT;
+        if (Nsec_H > -1) catalog[i].secfilt[j*Nsecfilt + Nsec_H].flags |= ID_SECF_OBJ_EXT;
+        if (Nsec_K > -1) catalog[i].secfilt[j*Nsecfilt + Nsec_K].flags |= ID_SECF_OBJ_EXT;
+      }
+      if (good2MASS) {
+        catalog[i].average[j].flags |= ID_OBJ_GOOD_ALT;
+      }
+    }
+  }
+  free (list);
+  free (dlist);
+        catalog[Nc].average[j].flags |= ID_OBJ_EXT_ALT;
+      }
+      if (Good2MASS) {
+        catalog[Nc].average[j].flags |= ID_OBJ_GOOD_ALT;
+      }
+    }
+  }
   return (TRUE);
+}
+int print_measure_set (Average *average, SecFilt *secfilt, Measure *measure) {
+  off_t k;
+  int Nsecfilt = GetPhotcodeNsecfilt ();
+  off_t m = average[0].measureOffset;
+  for (k = 0; k < average[0].Nmeasure; k++, m++) {
+    fprintf (stderr, "meas: %08x\n", measure[m].dbFlags);
+  }
+  int Ns;
+  for (Ns = 0; Ns < Nsecfilt; Ns++) {
+    fprintf (stderr, "secf: %08x\n", secfilt[Ns].flags);
+  }
+  return 1;
+}
 …
         if (isnan(Mgrid)) continue;
+        // note that measurements for which the image is not selected will not be modified
+        // (that is a good thing! -- we keep a prior calibration)
         // set the output calibration
         catalog[i].measure[m].Mcal = Mcal + Mmos + Mgrid;
 …
   double Chisq, MaxScatter, MaxChisq;
   double *xlist, *slist, *dlist;
   StatType stats;
+  liststats_setmode (&stats, "MEAN");
   if (VERBOSE) fprintf (stderr, "marking poor stars\n");
 …
   int Nsecfilt = GetPhotcodeNsecfilt ();
-  // XX int oldPLOTSTUFF = PLOTSTUFF;
-  // XX PLOTSTUFF = TRUE;
-  // XX plot_chisq (catalog, Ncatalog);
-  // XX PLOTSTUFF = oldPLOTSTUFF;
   // eliminate bad stars using the stats for a single secfilt at a time
-  // XXX DEP replace average.flags with secfilt flags
   for (Ns = 0; Ns < Nphotcodes; Ns ++) {
 …
+    }
+    initstats ("MEAN");
+    liststats (xlist, dlist, Ntot, &stats);
+    liststats (xlist, dlist, NULL, Ntot, &stats);
     MaxChisq = MAX (STAR_CHISQ, 2*stats.median);
+    liststats (slist, dlist, Ntot, &stats);
+    liststats (slist, dlist, NULL, Ntot, &stats);
     MaxScatter = MAX (STAR_SCATTER, 2*stats.median);
     fprintf (stderr, "Max Scatter: %f, Max Chisq: %f\n", MaxScatter, MaxChisq);
 …
+    }
     fprintf (stderr, "%d stars marked variable (%d scat, %d nan, %d chi), %d total\n", Ndel, Nscat, Nnan, Nchi, Nave);
-    initstats (STATMODE);
+  }
   free (xlist);
 …
   free (dlist);
+}
+// clean_measures examines the stats for a single star.  It measures the INNER 50% mean
+// and sigma, it then re-measures the mean and sigma using all stars with NSIGMA_CLIP (3)
+// sigma of the INNER 50% mean.  it then flags any measurements which are more than
+// NSIGMA_REJECT (5) sigma of the mean
 # define NSIGMA_CLIP 3.0
 …
   double *tlist, *list, *dlist;
   float Msys, Mcal, Mmos, Mgrid;
-  StatType stats;
   int Ncal, Nmos, Ngrid, Nfew;
 …
   /* it makes no sense to mark 3-sigma outliers with <5 measurements */
   TOOFEW = MAX (5, STAR_TOOFEW);
+  // stats structures for inner and full stats
+  StatType instats, stats;
+  liststats_setmode (&instats, "INNER_MEAN");
+  liststats_setmode (&stats, "MEAN");
   Ndel = Nave = 0;
 …
         // calculated mean of inner 50%
+        initstats ("INNER_MEAN");
+        liststats (list, dlist, N, &stats);
+        stats.sigma = MAX (MIN_ERROR, stats.sigma); /* if measurements agree too well, sigma -> 0.0 */
+        liststats (list, dlist, NULL, N, &instats);
+        instats.sigma = MAX (MIN_ERROR, instats.sigma); /* if measurements agree too well, sigma -> 0.0 */
         // ignore entries > 3sigma from inner mean
         for (k = m = 0; k < N; k++) {
           if (fabs (list[k] - stats.median) < NSIGMA_CLIP*stats.sigma) {
+          if (fabs (list[k] - instats.median) < NSIGMA_CLIP*instats.sigma) {
             list[m] = list[k];
             m++;
 …
+        }
         // recalculate the mean & sigma of the accepted measurements
+        initstats ("MEAN");
+        liststats (list, dlist, m, &stats);
+        liststats (list, dlist, NULL, m, &stats);
         stats.sigma = MAX (MIN_ERROR, stats.sigma);
 …
+    }
+  }
-  initstats (STATMODE);
   if (VERBOSE) fprintf (stderr, OFF_T_FMT" measures marked poor, "OFF_T_FMT" total\n", Ndel, Nave);
   free (list);
   free (dlist);
   free (ilist);
   free (tlist);
+}
 …
+  }
+  // fprintf (stderr, "N1: %d, N2: %d, N3: %d, N4: %d, N0: %d\n", N1, N2, N3, N4, N0);
+  liststats (list, dlist, n, &stats);
+  liststats_setmode (&stats, "MEAN");
+  liststats (list, dlist, NULL, n, &stats);
   free (list);
   free (dlist);
 …
+  }
+  liststats (list, dlist, n, &stats);
+  liststats_setmode (&stats, "MEAN");
+  liststats (list, dlist, NULL, n, &stats);
   free (list);
   free (dlist);
 …
+  }
+  liststats (list, dlist, n, &stats);
+  liststats_setmode (&stats, "MEAN");
+  liststats (list, dlist, NULL, n, &stats);
   free (list);
   free (dlist);

trunk/Ohana/src/relphot/src/args.c

-              r33651
+              r33963
+  }
+  NTHREADS = 0;
+  if ((N = get_argument (argc, argv, "-threads"))) {
+    remove_argument (N, &argc, argv);
+    NTHREADS = atof (argv[N]);
+    remove_argument (N, &argc, argv);
+  }
   // XXX for the moment, make this selection manual.  it needs to be automatic
   // based on the state of the SkyTable
 …
+  }
+  ApplyOffsets = FALSE;
+  if ((N = get_argument (argc, argv, "-apply-offsets"))) {
+    remove_argument (N, &argc, argv);
+    ApplyOffsets = TRUE;
+  }
   if (UpdateAverages && (argc == 1)) return TRUE;
   if (argc != 2) relphot_usage ();
 …
+  }
   if (!HOSTDIR) relphot_client_usage();
-  // CATDIR[0] = 0;
-  // if ((N = get_argument (argc, argv, "-catdir"))) {
-  //   remove_argument (N, &argc, argv);
-  //   strcpy (CATDIR, argv[N]);
-  //   remove_argument (N, &argc, argv);
-  // }
-  // if (!CATDIR[0]) relphot_client_usage();
   IMAGES = NULL; // used in -update mode

trunk/Ohana/src/relphot/src/bcatalog.c

-              r33651
+              r33963
 int LimitDensityCatalog_ByNmeasure (Catalog *subcatalog, Catalog *oldcatalog);
 int bcatalog (Catalog *subcatalog, Catalog *catalog) {
+int bcatalog (Catalog *subcatalog, Catalog *catalog, int Ncat) {
   off_t i, j, offset;
 …
         subcatalog[0].secfilt[Nsecfilt*Naverage+Nsec].Map    = NAN;
         subcatalog[0].secfilt[Nsecfilt*Naverage+Nsec].dM     = NAN;
+        subcatalog[0].secfilt[Nsecfilt*Naverage+Nsec].Mstdev = NAN_S_SHORT;
         subcatalog[0].secfilt[Nsecfilt*Naverage+Nsec].Xm     = NAN;
         subcatalog[0].secfilt[Nsecfilt*Naverage+Nsec].M_20   = NAN_S_SHORT;
 …
       subcatalog[0].measureT[Nmeasure].averef = Naverage;
       if (RESET) {
+        if (!KEEP_UBERCAL) {
+          subcatalog[0].measureT[Nmeasure].Mcal = 0;
+          subcatalog[0].measureT[Nmeasure].dbFlags &= ~ID_MEAS_PHOTOM_UBERCAL;
+        // only reset Mcal for measures with a matching image
+        // do not reset Mcal for ubercal images unless explicitly requested
+        if (subcatalog[0].measureT[Nmeasure].dbFlags & ID_MEAS_PHOTOM_UBERCAL) {
+          if (!KEEP_UBERCAL) {
+            subcatalog[0].measureT[Nmeasure].Mcal = 0.0;
+            subcatalog[0].measureT[Nmeasure].dbFlags &= ~ID_MEAS_PHOTOM_UBERCAL;
+          }
+        } else {
+          if (getImageEntry (Nmeasure, Ncat) >= 0) {
+            subcatalog[0].measureT[Nmeasure].Mcal = 0.0;
+          }
+        }
         subcatalog[0].measureT[Nmeasure].dbFlags &= ~PHOTOM_FLAGS;

trunk/Ohana/src/relphot/src/global_stats.c

-              r33651
+              r33963
 # include "relphot.h"
+# define MARKTIME(MSG,...) { \
+  float dtime; \
+  gettimeofday (&stopTimer, (void *) NULL); \
+  dtime = DTIME (stopTimer, startTimer); \
+  fprintf (stderr, MSG, __VA_ARGS__); }
 void global_stats (Catalog *catalog, int Ncatalog, FlatCorrectionTable *flatcorr) {
 …
   StatType stN, stX, stS, imN, imX, imM, imD, msM, msX, msN, msD;
   initstats ("MEAN");
+  // struct timeval startTimer, stopTimer;
   fprintf (stderr, "\n");
   fprintf (stderr, "STATS            median     mean    sigma      min      max   Nmeas\n");
+  // gettimeofday (&startTimer, (void *) NULL);
   int Ns;
 …
     int seccode = photcodes[Ns][0].code;
+    // MARKTIME("start star stats: %f sec\n", dtime);
     stN = statsStarN (catalog, Ncatalog, Nsec, seccode, flatcorr);
+    // MARKTIME("done star N: %f sec\n", dtime);
     stX = statsStarX (catalog, Ncatalog, Nsec);
+    // MARKTIME("done star X: %f sec\n", dtime);
     stS = statsStarS (catalog, Ncatalog, Nsec);
+    // MARKTIME("done star S: %f sec\n", dtime);
     fprintf (stderr, "   --- stats for %s ---\n", photcodes[Ns][0].name);
 …
   fprintf (stderr, "dMcal mosaic:   %7.4f  %7.4f  %7.4f  %7.4f  %7.4f  %6d\n",   msD.median, msD.mean, msD.sigma, msD.min, msD.max, msD.Nmeas);
   fprintf (stderr, "chisq mosaic:   %7.1f  %7.1f  %7.1f  %7.1f  %7.1f  %6d\n",   msX.median, msX.mean, msX.sigma, msX.min, msX.max, msX.Nmeas);
-  initstats (STATMODE);
+}

trunk/Ohana/src/relphot/src/initialize.c

-              r33651
+              r33963
     PhotcodeList = strcreate (argv[1]);
     photcodes = ParsePhotcodeList (PhotcodeList, &Nphotcodes, TRUE); // require SEC photcodes
+  }
+  } else {
+    char tmpline1[256];
+    int Ns;
+    Nphotcodes = GetPhotcodeNsecfilt ();
+    ALLOCATE (photcodes, PhotCode *, Nphotcodes);
+    ALLOCATE (PhotcodeList, char, 256);
+    for (Ns = 0; Ns < Nphotcodes; Ns++) {
+      photcodes[Ns] = GetPhotcodebyNsec (Ns);
+      if (Ns > 0) {
+        snprintf (tmpline1, 256, "%s,%s", PhotcodeList, photcodes[Ns][0].name);
+      } else {
+        snprintf (tmpline1, 256, "%s", photcodes[Ns][0].name);
+      }
+      strcpy (PhotcodeList, tmpline1);
+    }
+  }
   if (USE_GRID && (Nphotcodes > 1)) {
     fprintf (stderr, "grid correction analysis currently can only operate on a single photcode\n");
     exit (1);
+  }
-  initstats (STATMODE);
   IMAGE_BAD = ID_IMAGE_PHOTOM_POOR | ID_IMAGE_PHOTOM_FEW | ID_IMAGE_PHOTOM_SKIP;

trunk/Ohana/src/relphot/src/liststats.c

-              r33651
+              r33963
 # include "relphot.h"
+enum {M_MEAN, M_MEDIAN, M_WT_MEAN, M_INNER_MEAN,
+      M_INNER_WTMEAN, M_CHI_INNER_MEAN, M_CHI_INNER_WTMEAN};
+void liststats_setmode (StatType *stats, char *strmode) {
+static int statmode;
+  stats->statmode = -1;
+  if (!strcmp (strmode, "MEAN"))             { stats->statmode = STATS_MEAN; return; }
+  if (!strcmp (strmode, "MEDIAN"))           { stats->statmode = STATS_MEDIAN; return; }
+  if (!strcmp (strmode, "WT_MEAN"))          { stats->statmode = STATS_WT_MEAN; return; }
+  if (!strcmp (strmode, "INNER_MEAN"))       { stats->statmode = STATS_INNER_MEAN; return; }
+  if (!strcmp (strmode, "INNER_WTMEAN"))     { stats->statmode = STATS_INNER_WTMEAN; return; }
+  if (!strcmp (strmode, "CHI_INNER_MEAN"))   { stats->statmode = STATS_CHI_INNER_MEAN; return; }
+  if (!strcmp (strmode, "CHI_INNER_WTMEAN")) { stats->statmode = STATS_CHI_INNER_WTMEAN; return; }
+void initstats (char *mode) {
+  statmode = -1;
+  if (!strcmp (mode, "MEAN")) statmode = M_MEAN;
+  if (!strcmp (mode, "MEDIAN")) statmode = M_MEDIAN;
+  if (!strcmp (mode, "WT_MEAN")) statmode = M_WT_MEAN;
+  if (!strcmp (mode, "INNER_MEAN")) statmode = M_INNER_MEAN;
+  if (!strcmp (mode, "INNER_WTMEAN")) statmode = M_INNER_WTMEAN;
+  if (!strcmp (mode, "CHI_INNER_MEAN")) statmode = M_CHI_INNER_MEAN;
+  if (!strcmp (mode, "CHI_INNER_WTMEAN")) statmode = M_CHI_INNER_WTMEAN;
+  if (statmode == -1) {
+    fprintf (stderr, "ERROR: invalid stats mode: %s\n", mode);
+    exit (1);
+  }
+  fprintf (stderr, "ERROR: invalid stats mode: %s\n", strmode);
+  exit (1);
+}
 int liststats (double *value, double *dvalue, int N, StatType *stats) {
+int liststats (double *value, double *dvalue, double *weight, int N, StatType *stats) {
+  int i, ks, ke, Nm;
+  double Mo, dMo, M, dM, X2, dS, *chi;
+  int i, ks, ke;
+  double Mo, dMo, M, dM, Nm, X2, dS, R, W, *chi;
+  myAssert (stats->statmode != STATS_NONE, "programming error, liststats mode not set");
   ke = ks = dMo = 0;
 …
   if (N < 1) return (FALSE);
-  dsortpair (value, dvalue, N);
-  stats[0].median = value[(int)(0.5*N)];
-  stats[0].min    = value[0];
-  stats[0].max    = value[N-1];
   int N80 = MIN (N-1, 0.8*N);
   int N20 = MAX (0, 0.2*N);
+  if (weight) {
+    dsortthree (value, dvalue, weight, N);
+  } else {
+    dsortpair (value, dvalue, N);
+  }
+  // these values do not depend on the errors or weighting scheme
+  stats[0].median  = value[(int)(0.5*N)];
+  stats[0].min     = value[0];
+  stats[0].max     = value[N-1];
   stats[0].Upper80 = value[N80];
   stats[0].Lower20 = value[N20];
+  switch (statmode) {
+  case M_MEDIAN:
+    ks = 0;
+    ke = N;
+    Mo = stats[0].median;
+    Nm = N;
+    goto chisq;
+    break;
+  case M_MEAN:
+  case M_WT_MEAN:
+    ks = 0;
+    ke = N;
+    break;
+  case M_INNER_MEAN:
+  case M_INNER_WTMEAN:
+  case M_CHI_INNER_MEAN:
+  case M_CHI_INNER_WTMEAN:
+    ks = 0.25*N + 0.50;
+    ke = 0.75*N + 0.25;
+    if (N <= 3) {
+  switch (stats->statmode) {
+    case STATS_MEDIAN:
       ks = 0;
       ke = N;
+    }
+    break;
+      Mo = stats[0].median;
+      Nm = N;
+      goto chisq;
+      break;
+    case STATS_MEAN:
+    case STATS_WT_MEAN:
+      ks = 0;
+      ke = N;
+      break;
+    case STATS_INNER_MEAN:
+    case STATS_INNER_WTMEAN:
+    case STATS_CHI_INNER_MEAN:
+    case STATS_CHI_INNER_WTMEAN:
+      ks = 0.25*N + 0.50;
+      ke = 0.75*N + 0.25;
+      if (N <= 3) {
+        ks = 0;
+        ke = N;
+      }
+      break;
+    case STATS_NONE:
+      myAbort ("undefined stats");
+  }
+  if ((statmode == M_CHI_INNER_MEAN) || (statmode == M_CHI_INNER_WTMEAN)) {
+  // for these two modes, I need a vector of the chi-square contribution
+  // I'm actually just using chisq to get the correct sorting order
+  if ((stats->statmode == STATS_CHI_INNER_MEAN) || (stats->statmode == STATS_CHI_INNER_WTMEAN)) {
     ALLOCATE (chi, double, N);
     for (i = 0; i < N; i++) {
       chi[i] = (value[i] - stats[0].median) / dvalue[i];
+    }
+    dsortthree (chi, value, dvalue, N);
+    if (weight) {
+      dsortfour (chi, value, dvalue, weight, N);
+    } else {
+      dsortthree (chi, value, dvalue, N);
+    }
     free (chi);
+  }
+  /* calculating the per-star offset based on the weighted average */
+  M = dM = Nm = 0;
+  if ((statmode == M_WT_MEAN) || (statmode == M_INNER_WTMEAN) || (statmode == M_CHI_INNER_WTMEAN)) {
+    for (i = ks; i < ke; i++) {
+      M   += value[i] / SQ (dvalue[i]);
+      dM  += 1.0 / SQ (dvalue[i]);
+      Nm  ++;
+    }
+    Mo = M / dM;
+    dMo = sqrt (1.0 / dM);
+  int WeightedMean = FALSE;
+  WeightedMean |= (stats->statmode == STATS_WT_MEAN);
+  WeightedMean |= (stats->statmode == STATS_INNER_WTMEAN);
+  WeightedMean |= (stats->statmode == STATS_CHI_INNER_WTMEAN);
+  /* calculating the per-star offset based on the desired weighting scheme */
+  M = dM = Nm = W = R = 0;
+  if (weight) {
+    // the weight value is multiplied by whichever nominal weighting scheme is provided
+    // thus the user should set weight to 1.0 for nominal weight, and 100 for heavy weight (or so)
+    // and 0.01 for under-weight
+    if (WeightedMean) {
+      for (i = ks; i < ke; i++) {
+        M  += value[i] * weight[i] / SQ(dvalue[i]);
+        W  +=            weight[i] / SQ(dvalue[i]);
+        dM += SQ (weight[i] / dvalue[i]);
+        R  += weight[i] / SQ(dvalue[i]);
+        Nm += 1.0;
+      }
+      Mo  = M / W;
+      dMo = sqrt (dM) / R;
+    } else {
+      for (i = ks; i < ke; i++) {
+        M  += value[i] * weight[i];
+        W  +=            weight[i];
+        dM += SQ (weight[i] * dvalue[i]);
+        R  += weight[i];
+        Nm += 1.0;
+      }
+      Mo  = M / W;
+      dMo = sqrt (dM) / R;
+    }
   } else {
+    for (i = ks; i < ke; i++) {
+      M   += value[i];
+      dM  += SQ (dvalue[i]);
+      Nm  ++;
+    }
+    Mo = M / (double) Nm;
+    dMo = sqrt (dM / (double) Nm);
+    // NULL weight vector is supplied, revert to standard form (above reverts if weight[i] == 1)
+    if (WeightedMean) {
+      // weighted by inverse-variance
+      for (i = ks; i < ke; i++) {
+        M   += value[i] / SQ (dvalue[i]);
+        dM  += 1.0 / SQ (dvalue[i]);
+        Nm  += 1.0;
+      }
+      Mo = M / dM;
+      dMo = sqrt (1.0 / dM);
+    } else {
+      // pure un-weighted
+      for (i = ks; i < ke; i++) {
+        M   += value[i];
+        dM  += SQ (dvalue[i]);
+        Nm  += 1.0;
+      }
+      Mo = M / Nm;
+      dMo = sqrt (dM) / Nm;
+    }
+  }
 …
+  }
   X2 = X2 / (Nm - 1);
   dS = sqrt (dS / Nm);
+  dS = sqrt (dS / (Nm - 1));
   stats[0].mean  = Mo;
 …
+}
+// we could define the weight to be the only scale factor:
+// \mu      = \sum (value_i * weight_i) / \sum (weight_i)
+// \sigma^2 = (1/R) \sum (weight_i^2 \sigma_i^2)
+// R = \sum (weight_i^2)
+// or, we could define the weight to be a scale factor times the inverse error:
+// \mu      = \sum (value_i * weight_i / sigma_i) / \sum (weight_i)
+// \sigma^2 = (1/R) \sum (weight_i^2 \sigma_i^2)
+// R = \sum (weight_i^2)

trunk/Ohana/src/relphot/src/load_catalogs.c

-              r33651
+              r33963
     // select only the brighter stars
     bcatalog (&catalog[i], &tcatalog);
+    bcatalog (&catalog[i], &tcatalog, i);
     dvo_catalog_unlock (&tcatalog);
     dvo_catalog_free (&tcatalog);
 …
     // ensure that the paths are absolute path names
     char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH);
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
     free (table->hosts[i].pathname);
     table->hosts[i].pathname = tmppath;
 …
+  }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
+    // if any one of the remote jobs fails, we should fail and exit
+    int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
+    if (!status) {
+      fprintf (stderr, "at least one remote client job failed to load data, exiting\n");
+      exit (3);
+    }
+  }

trunk/Ohana/src/relphot/src/load_images.c

-              r33651
+              r33963
   off_t     *LineNumber;
   struct timeval start, stop;
+  char *inSubset;
   SkyTable *sky = NULL;
 …
   MARKTIME("read image table: %f sec\n", dtime);
+  // allocate and init an array to identify the images included in the subset
+  ALLOCATE (inSubset, char, Nimage);
+  memset (inSubset, 0, Nimage);
   // determine the populated SkyRegions overlapping the requested area
 …
     // select the images which overlap the selected sky regions
     // 'subset' points to a new copy of the data (different from 'image')
     subset = select_images (skylist, image, Nimage, &LineNumber, &Nsubset);
+    subset = select_images (skylist, image, Nimage, inSubset, &LineNumber, &Nsubset, NULL);
     MARKTIME("selected %d overlapping images: %f sec\n", (int) Nsubset, dtime);
   } else {
 …
       skylist = SkyListByPatch (sky, -1, &subregion);
-      subregion.Rmin = region[0].Rmin; subregion.Rmax = 360.0;
-      SkyList *extraList = SkyListByPatch (sky, -1, &subregion);
       // select the images which overlap the selected sky regions
       // 'subset' points to a new copy of the data (different from 'image')
       subset = select_images (skylist, image, Nimage, &LineNumber, &Nsubset);
+      subset = select_images (skylist, image, Nimage, inSubset, &LineNumber, &Nsubset, &subregion);
       MARKTIME("selected %d overlapping images: %f sec\n", (int) Nsubset, dtime);
 …
       off_t *LineNumberExtra, i;
+      subsetExtra = select_images (extraList, image, Nimage, &LineNumberExtra, &NsubsetExtra);
+      subregion.Rmin = region[0].Rmin; subregion.Rmax = 360.0;
+      SkyList *extraList = SkyListByPatch (sky, -1, &subregion);
+      subsetExtra = select_images (extraList, image, Nimage, inSubset, &LineNumberExtra, &NsubsetExtra, &subregion);
       MARKTIME("selected %d overlapping images: %f sec\n", (int) Nsubset, dtime);
 …
       // select the images which overlap the selected sky regions
       // 'subset' points to a new copy of the data (different from 'image')
       subset = select_images (skylist, image, Nimage, &LineNumber, &Nsubset);
+      subset = select_images (skylist, image, Nimage, inSubset, &LineNumber, &Nsubset, region);
       MARKTIME("selected %d overlapping images: %f sec\n", (int) Nsubset, dtime);
+    }
+  }
+# if (1)
+  FILE *ftest = fopen ("skydump.dat", "w");
+  int i;
+  for (i = 0; i < skylist[0].Nregions; i++) {
+    fprintf (ftest, "%s : %f %f : %f %f\n", skylist[0].regions[i][0].name, skylist[0].regions[i][0].Rmin, skylist[0].regions[i][0].Rmax, skylist[0].regions[i][0].Dmin, skylist[0].regions[i][0].Dmax);
+  }
+  fclose (ftest);
+# endif
+  // match chips to mosaics (if applicable)
+  initMosaics (subset, Nsubset, image, inSubset, Nimage);
+  MARKTIME("init mosaics: %f sec\n", dtime);
   // save the subset of images in the static reference in ImageOps, set up indexes
   initImages (subset, LineNumber, Nsubset);
   MARKTIME("init images: %f sec\n", dtime);
-  // match chips to mosaics (if applicable)
-  initMosaics (subset, Nsubset);
-  MARKTIME("init mosaics: %f sec\n", dtime);
   return (skylist);
+}

trunk/Ohana/src/relphot/src/reload_catalogs.c

-              r33651
+              r33963
     TIMESTAMP(time4);
+    setMrelFinal (&catalog, flatcorr);
+    initMrel (&catalog, 1);
+    setMrelFinal (&catalog, flatcorr, FALSE);
     TIMESTAMP(time5);
 …
     freeGridBins (1);
     TIMESTAMP(time7);
+    if (hostID) {
+      client_logger_message ("updated catalog file %s\n", catalog.filename);
+    }
+  }
 …
 int reload_catalogs_parallel (SkyList *sky) {
+  off_t Nimage;
+  ImageSubset *image = getimages_subset (&Nimage);
+  // write out the subset table of image information
+  // name of image subset file:
   char imageFile[512];
   snprintf (imageFile, 512, "%s/Images.subset.dat", CATDIR);
+  if (!ImageSubsetSave (imageFile, image, Nimage)) {
+    fprintf (stderr, "failed to write image subset\n");
+    exit (1);
+  }
+  free (image);
+  // the ApplyOffsets option re-uses an existing Images.subset.dat file
+  if (!ApplyOffsets) {
+    off_t Nimage;
+    ImageSubset *image = getimages_subset (&Nimage);
+    if (!ImageSubsetSave (imageFile, image, Nimage)) {
+      fprintf (stderr, "failed to write image subset\n");
+      exit (1);
+    }
+    free (image);
+  } else {
+    // load the current sky table (layout of all SkyRegions)
+    SkyTable *skytable = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE);
+    // XXX kind of hackish...
+    ALLOCATE (sky, SkyList, 1);
+    strcpy (sky->hosts, skytable->hosts);
+  }
   // now launch the relphot_client jobs to the parallel hosts
 …
     // ensure that the paths are absolute path names
     char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH);
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
     free (table->hosts[i].pathname);
     table->hosts[i].pathname = tmppath;
 …
+  }
   if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) {
+    HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
+    int status = HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE);
+    if (!status) {
+      fprintf (stderr, "at least one remote client job failed to load data, exiting\n");
+      exit (3);
+    }
+  }
   return (TRUE);

trunk/Ohana/src/relphot/src/relphot.c

-              r33651
+              r33963
   if (UpdateAverages) {
     relphot_objects (0, NULL);
+    exit (0);
+  }
+  if (ApplyOffsets) {
+    // re-run this step from a previous attempt (assumes an existing Images.subset.dat file)
+    if (!PARALLEL) {
+      fprintf (stderr, "-apply-offsets only makes sense in an parallel context\n");
+      exit (2);
+    }
+    reload_catalogs (NULL, NULL, 0, NULL);
     exit (0);
+  }
 …
         plot_scatter (catalog, Ncatalog, flatcorr);
+      }
       setMrel  (catalog, Ncatalog, flatcorr);
+      setMrel  (catalog, Ncatalog, flatcorr); // threaded
       if (PLOTSTUFF) {
         plot_scatter (catalog, Ncatalog, flatcorr);
 …
       setMmos  (catalog, FALSE, flatcorr);
       setMgrid (catalog, flatcorr);
+      MARKTIME("-- set Mrel, Mcal, Mmos, Mgrid : %f sec\n", dtime);
       if (PLOTSTUFF) {
 …
       // if (i < NLOOP - 1) rationalize_mosaics (catalog, Ncatalog);
       // if (i % 6 == 1) rationalize_images ();
+      if (i % 6 == 2) clean_measures (catalog, Ncatalog, FALSE, flatcorr);
+      if (i % 6 == 3) clean_stars (catalog, Ncatalog);
+      if (i % 6 == 5) clean_mosaics ();
+      if (i % 6 == 5) clean_images ();
+      // NOTE : in the past, I was not iterating enough before cleaning.  make sure we do
+      // at least 8 loops first -- that should get the systematic errors down to the ~1%
+      // level, even in cases where we have an even split between photometric data and
+      // data with 1 mag of extinction.
+      if ((i > 8) && (i % 8 == 2)) clean_measures (catalog, Ncatalog, FALSE, flatcorr);
+      if ((i > 8) && (i % 8 == 3)) clean_stars (catalog, Ncatalog);
+      if ((i > 8) && (i % 8 == 5)) clean_mosaics ();
+      if ((i > 8) && (i % 8 == 5)) clean_images ();
       // if ((i == 1) || (i == 5) || (i ==  9) || (i == 13)) clean_measures (catalog, Ncatalog, FALSE);
 …
       // if ((i == 4) || (i == 8) || (i == 12) || (i == 16)) clean_mosaics ();
       // if ((i == 4) || (i == 8) || (i == 12) || (i == 16)) clean_images ();
       global_stats (catalog, Ncatalog, flatcorr);
+      if (i % 3 == 2) global_stats (catalog, Ncatalog, flatcorr);
       MARKTIME("-- finished loop %d: %f sec\n", i, dtime);
+    }

trunk/Ohana/src/relphot/src/relphot_client.c

-              r33651
+              r33963
   // get configuration info, args, lockfile (set CATDIR, HOST_ID, HOSTDIR, etc)
   initialize_client (argc, argv);
+  client_logger_init ();
   // load the current sky table (layout of all SkyRegions)
 …
       exit (2);
+  }
+  client_logger_message ("loaded sky table and defined patch\n");
   switch (MODE) {
 …
           exit (2);
+      }
+      client_logger_message ("loaded catalogs\n");
       BrightCatalog *bcatalog = BrightCatalogMerge (catalog, Ncatalog);
+      client_logger_message ("generated subset data\n");
       if (!BrightCatalogSave (BCATALOG, bcatalog)) {
           fprintf (stderr, "ERROR saving bright catalog from %s\n", CATDIR);
           exit (2);
+      }
+      client_logger_message ("generated subset table\n");
       break;
+    }
 …
           exit (2);
+      }
+      client_logger_message ("loaded image subset data\n");
       // save the available image information in the static array in ImageOps.c
 …
       snprintf (flatcorrFile, 1024, "%s/flatcorr.fits", CATDIR);
       FlatCorrectionTable *flatcorr = FlatCorrectionLoad (flatcorrFile, VERBOSE);
+      client_logger_message ("loaded flat-field correction data\n");
       reload_catalogs (skylist, flatcorr, HOST_ID, HOSTDIR);
+      client_logger_message ("updated catalogs\n");
       break;
+    }
 …
     case MODE_UPDATE_OBJECTS: {
       relphot_objects (HOST_ID, HOSTDIR);
+      client_logger_message ("updated objects\n");
       break;
+    }
 …
       abort();
+  }
+  client_logger_message ("done with relphot_client\n");
   exit (0);
+}

trunk/Ohana/src/relphot/src/relphot_objects.c

-              r33651
+              r33963
           catalog.secfilt[j*Nsecfilt + k].Map    = NAN;
           catalog.secfilt[j*Nsecfilt + k].dM     = NAN;
+          catalog.secfilt[j*Nsecfilt + k].Mstdev = NAN_S_SHORT;
           catalog.secfilt[j*Nsecfilt + k].Xm     = NAN;
           catalog.secfilt[j*Nsecfilt + k].M_20   = NAN_S_SHORT;
 …
+    }
+    setMave (&catalog, 1);
+    populate_tiny_values(&catalog, DVO_TV_MEASURE | DVO_TV_AVERAGE);
+    initMrel (&catalog, 1);
+    setMrelFinal (&catalog, NULL, TRUE);
     if (!UPDATE) {
       dvo_catalog_unlock (&catalog);
+      free_tiny_values(&catalog);
       dvo_catalog_free (&catalog);
       continue;
 …
     dvo_catalog_save (&catalog, VERBOSE);
     dvo_catalog_unlock (&catalog);
+    free_tiny_values(&catalog);
     dvo_catalog_free (&catalog);
+  }
 …
     // ensure that the paths are absolute path names
     char *tmppath = abspath (table->hosts[i].pathname, MAX_PATH_LENGTH);
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
     free (table->hosts[i].pathname);
     table->hosts[i].pathname = tmppath;

trunk/Ohana/src/relphot/src/select_images.c

-              r33651
+              r33963
 } SkyRegionCoords;
+void MakeSkyCoordIndex (SkyRegionCoords **skycoords_out, double **RmaxSky_out, off_t **index_out, SkyList *skylist);
 void dsortindex (double *X, off_t *Y, int N);
 off_t getRegionStartByRA (double R, double *Rref, off_t Nregions);
 …
   fprintf (stderr, MSG, __VA_ARGS__); }
 Image *select_images (SkyList *skylist, Image *timage, off_t Ntimage, off_t **LineNumber, off_t *Nimage) {
+Image *select_images (SkyList *skylist, Image *timage, off_t Ntimage, char *inSubset, off_t **LineNumber, off_t *Nimage, SkyRegion *region) {
   Image *image;
 …
   off_t *line_number;
   int InRange, ecode, found;
   double Ri[5], Di[5], Xi[5], Yi[5], dx, dy;
+  double Ri[5], Di[5], Xi[5], Yi[5];
   Coords tcoords;
-  SkyRegionCoords *skycoords;
   struct timeval start, stop;
-  double RmaxSkyRegion, RminSkyRegion, RmidSkyRegion, DminSkyRegion, DmaxSkyRegion;
-  double *RmaxSky;
-  off_t *index;
   if (skylist[0].Nregions < 1) {
     *Nimage = 0;
 …
   strcpy (tcoords.ctype, "RA---TAN");
+  ALLOCATE (skycoords, SkyRegionCoords, skylist[0].Nregions);
+  ALLOCATE (RmaxSky, double, skylist[0].Nregions);
+  ALLOCATE (index, off_t, skylist[0].Nregions);
+  RminSkyRegion = +360.0;
+  RmaxSkyRegion = -360.0;
+  DminSkyRegion = +90.0;
+  DmaxSkyRegion = -90.0;
+  D_NIMAGE = 6000;
+  // FILE *ftest = fopen ("relphot.dump.dat", "w");
+  /* compare with each region file */
+  double RminSkyRegion = region[0].Rmin;
+  double RmaxSkyRegion = region[0].Rmax;
+  double DminSkyRegion = region[0].Dmin;
+  double DmaxSkyRegion = region[0].Dmax;
+  double RmidSkyRegion = 0.5*(RminSkyRegion + RmaxSkyRegion);
+  // generate a set of Rmin/Rmax values using this RmidSkyRegion for DEC bands of width
+  // dDecBand (0.1 deg).  DecIndex = (Dec + 90) / dDecBand
+  double *RminBand, *RmaxBand;
+  float dDecBand = 0.1;
+  int NDecBands = 180 / dDecBand + 1;
+  ALLOCATE (RminBand, double, NDecBands);
+  ALLOCATE (RmaxBand, double, NDecBands);
+  for (i = 0; i < NDecBands; i++) {
+    RminBand[i] = +360.0;
+    RmaxBand[i] = -360.0;
+  }
+   /* compare with each region file */
   for (i = 0; i < skylist[0].Nregions; i++) {
+    /* we make positional comparisons in the projection of catalog */
+    skycoords[i].Rc = 0.5*(skylist[0].regions[i][0].Rmax + skylist[0].regions[i][0].Rmin);
+    skycoords[i].Dc = 0.5*(skylist[0].regions[i][0].Dmax + skylist[0].regions[i][0].Dmin);
+    tcoords.crval1 = skycoords[i].Rc;
+    tcoords.crval2 = skycoords[i].Dc;
+    /* define catalog corners */
+    RD_to_XY (&skycoords[i].Xc[0], &skycoords[i].Yc[0], skylist[0].regions[i][0].Rmin, skylist[0].regions[i][0].Dmin, &tcoords);
+    RD_to_XY (&skycoords[i].Xc[1], &skycoords[i].Yc[1], skylist[0].regions[i][0].Rmax, skylist[0].regions[i][0].Dmin, &tcoords);
+    RD_to_XY (&skycoords[i].Xc[2], &skycoords[i].Yc[2], skylist[0].regions[i][0].Rmax, skylist[0].regions[i][0].Dmax, &tcoords);
+    RD_to_XY (&skycoords[i].Xc[3], &skycoords[i].Yc[3], skylist[0].regions[i][0].Rmin, skylist[0].regions[i][0].Dmax, &tcoords);
+    skycoords[i].Xc[4] = skycoords[i].Xc[0];
+    skycoords[i].Yc[4] = skycoords[i].Yc[0];
+    RmaxSky[i] = skylist[0].regions[i][0].Rmax;
+    index[i] = i;
+    dx = 0.02*(skycoords[i].Xc[2] - skycoords[i].Xc[0]);
+    dy = 0.02*(skycoords[i].Yc[2] - skycoords[i].Yc[0]);
+    skycoords[i].Xc[0] -= dx; skycoords[i].Yc[0] -= dy;
+    skycoords[i].Xc[1] += dx; skycoords[i].Yc[1] -= dy;
+    skycoords[i].Xc[2] += dx; skycoords[i].Yc[2] += dy;
+    skycoords[i].Xc[3] -= dx; skycoords[i].Yc[3] += dy;
+    skycoords[i].Xc[4] -= dx; skycoords[i].Yc[4] -= dy;
+    RminSkyRegion = MIN(RminSkyRegion, skylist[0].regions[i][0].Rmin);
+    RmaxSkyRegion = MAX(RmaxSkyRegion, skylist[0].regions[i][0].Rmax);
+    DminSkyRegion = MIN(DminSkyRegion, skylist[0].regions[i][0].Dmin);
+    DmaxSkyRegion = MAX(DmaxSkyRegion, skylist[0].regions[i][0].Dmax);
+  }
+  RmidSkyRegion = 0.5*(RminSkyRegion + RmaxSkyRegion);
+  MARKTIME("create sky region coords: %f sec\n", dtime);
+  dsortindex (RmaxSky, index, skylist[0].Nregions);
+  MARKTIME("sort sky coords: %f sec\n", dtime);
+    int iDecBandMin = (skylist[0].regions[i][0].Dmin + 90.0) / dDecBand;
+    int iDecBandMax = (skylist[0].regions[i][0].Dmax + 90.0) / dDecBand;
+    double RminAlt = ohana_normalize_angle_to_midpoint (skylist[0].regions[i][0].Rmin, RmidSkyRegion);
+    double RmaxAlt = ohana_normalize_angle_to_midpoint (skylist[0].regions[i][0].Rmax, RmidSkyRegion);
+    for (j = iDecBandMin; j <= iDecBandMax; j++) {
+      RminBand[j] = MIN(RminBand[j], RminAlt);
+      RmaxBand[j] = MAX(RmaxBand[j], RmaxAlt);
+    }
+  }
+# if (1)
+  // XXX quick test of the dec bands:
+  FILE *fout = fopen ("dec.bands.dat", "w");
+  for (i = 0; i < NDecBands; i++) {
+    fprintf (fout, "%d %f : %f - %f\n", (int) i, dDecBand * i - 90.0, RminBand[i], RmaxBand[i]);
+  }
+  fclose (fout);
+# endif
+  SkyRegionCoords *skycoords = NULL;
+  double *RmaxSky = NULL;
+  off_t *index = NULL;
+  if (!USE_BASIC_CHECK) {
+    MakeSkyCoordIndex (&skycoords, &RmaxSky, &index, skylist);
+  }
   if (VERBOSE) fprintf (stderr, "finding images\n");
   BuildChipMatch (timage, Ntimage);
+  MARKTIME("build chip match: %f sec\n", dtime);
+  MARKTIME("build chip match for %d images: %f sec\n", (int) Ntimage, dtime);
+  D_NIMAGE = 6000;
   nimage = 0;
 …
     if (RminImage > RmaxSkyRegion) continue;
     if (RmaxImage < RminSkyRegion) continue;
+    // the above checks are only valid for the outermost region.  however, at a given
+    // declination, the range of RA is limited by the actual catalog boundaries
+    // check if this image is in range for the Dmax and Dmin locations
+    int iDecBandMin = (DminImage + 90.0) / dDecBand;
+    int iDecBandMax = (DmaxImage + 90.0) / dDecBand;
+    // the sky region RA is defined to be 0 - 360.0
+    if (RminImage < RminBand[iDecBandMin]) {
+      if (VERBOSE2) fprintf (stderr, "skip image %s (%f,%f) on boundary\n", timage[i].name, 0.5*(RminImage + RmaxImage), 0.5*(DminImage + DmaxImage));
+      continue;
+    }
+    if (RminImage < RminBand[iDecBandMax]) {
+      if (VERBOSE2) fprintf (stderr, "skip image %s (%f,%f) on boundary\n", timage[i].name, 0.5*(RminImage + RmaxImage), 0.5*(DminImage + DmaxImage));
+      continue;
+    }
+    if (RmaxImage > RmaxBand[iDecBandMin]) {
+      if (VERBOSE2) fprintf (stderr, "skip image %s (%f,%f) on boundary\n", timage[i].name, 0.5*(RminImage + RmaxImage), 0.5*(DminImage + DmaxImage));
+      continue;
+    }
+    if (RmaxImage > RmaxBand[iDecBandMax]) {
+      if (VERBOSE2) fprintf (stderr, "skip image %s (%f,%f) on boundary\n", timage[i].name, 0.5*(RminImage + RmaxImage), 0.5*(DminImage + DmaxImage));
+      continue;
+    }
     // image overlaps region, keep it
 …
   found_it:
     image[nimage] = timage[i];
+    inSubset[i] = TRUE;
     /* always allow 'few' images to succeed, if possible (new images / detections may have
      * been added) */
 …
   MARKTIME("finish image selection: %f sec\n", dtime);
-  // fclose (ftest);
   if (VERBOSE) fprintf (stderr, "found "OFF_T_FMT" images\n", nimage);
   REALLOCATE (image, Image, MAX (nimage, 1));
   REALLOCATE (line_number, off_t, MAX (nimage, 1));
+  free (skycoords);
+  free (RmaxSky);
+  free (index);
+  if (!USE_BASIC_CHECK) {
+    free (skycoords);
+    free (RmaxSky);
+    free (index);
+  }
   *Nimage  = nimage;
 …
   return (Nlo);
+}
+void MakeSkyCoordIndex (SkyRegionCoords **skycoords_out, double **RmaxSky_out, off_t **index_out, SkyList *skylist) {
+  off_t i;
+  SkyRegionCoords *skycoords;
+  Coords tcoords;
+  double *RmaxSky;
+  off_t *index;
+  double dx, dy;
+  // the comparison is made in the catalog local projection. below we set crval1,2
+  tcoords.crpix1 = tcoords.crpix2 = 0.0;
+  tcoords.cdelt1 = tcoords.cdelt2 = 1.0 / 3600.0;
+  tcoords.pc1_1  = tcoords.pc2_2 = 1.0;
+  tcoords.pc1_2  = tcoords.pc2_1 = 0.0;
+  strcpy (tcoords.ctype, "RA---TAN");
+  /* compare with each region file */
+  ALLOCATE (skycoords, SkyRegionCoords, skylist[0].Nregions);
+  ALLOCATE (RmaxSky, double, skylist[0].Nregions);
+  ALLOCATE (index, off_t, skylist[0].Nregions);
+  for (i = 0; i < skylist[0].Nregions; i++) {
+    /* we make positional comparisons in the projection of catalog */
+    skycoords[i].Rc = 0.5*(skylist[0].regions[i][0].Rmax + skylist[0].regions[i][0].Rmin);
+    skycoords[i].Dc = 0.5*(skylist[0].regions[i][0].Dmax + skylist[0].regions[i][0].Dmin);
+    tcoords.crval1 = skycoords[i].Rc;
+    tcoords.crval2 = skycoords[i].Dc;
+    /* define catalog corners */
+    RD_to_XY (&skycoords[i].Xc[0], &skycoords[i].Yc[0], skylist[0].regions[i][0].Rmin, skylist[0].regions[i][0].Dmin, &tcoords);
+    RD_to_XY (&skycoords[i].Xc[1], &skycoords[i].Yc[1], skylist[0].regions[i][0].Rmax, skylist[0].regions[i][0].Dmin, &tcoords);
+    RD_to_XY (&skycoords[i].Xc[2], &skycoords[i].Yc[2], skylist[0].regions[i][0].Rmax, skylist[0].regions[i][0].Dmax, &tcoords);
+    RD_to_XY (&skycoords[i].Xc[3], &skycoords[i].Yc[3], skylist[0].regions[i][0].Rmin, skylist[0].regions[i][0].Dmax, &tcoords);
+    skycoords[i].Xc[4] = skycoords[i].Xc[0];
+    skycoords[i].Yc[4] = skycoords[i].Yc[0];
+    RmaxSky[i] = skylist[0].regions[i][0].Rmax;
+    index[i] = i;
+    dx = 0.02*(skycoords[i].Xc[2] - skycoords[i].Xc[0]);
+    dy = 0.02*(skycoords[i].Yc[2] - skycoords[i].Yc[0]);
+    skycoords[i].Xc[0] -= dx; skycoords[i].Yc[0] -= dy;
+    skycoords[i].Xc[1] += dx; skycoords[i].Yc[1] -= dy;
+    skycoords[i].Xc[2] += dx; skycoords[i].Yc[2] += dy;
+    skycoords[i].Xc[3] -= dx; skycoords[i].Yc[3] += dy;
+    skycoords[i].Xc[4] -= dx; skycoords[i].Yc[4] -= dy;
+  }
+  dsortindex (RmaxSky, index, skylist[0].Nregions);
+  *RmaxSky_out = RmaxSky;
+  *index_out = index;
+  *skycoords_out = skycoords;
+  return;
+}

trunk/Ohana/src/relphot/src/setMrelFinal.c

-              r33651
+              r33963
 // output dbFlags values
 void setMrelFinal (Catalog *catalog, FlatCorrectionTable *flatcorr) {
+void setMrelFinal (Catalog *catalog, FlatCorrectionTable *flatcorr, int simpleAverage) {
   off_t i;
 …
         catalog[0].secfilt[N].Map    = NAN;
         catalog[0].secfilt[N].dM     = NAN;
+        catalog[0].secfilt[N].Mstdev = NAN_S_SHORT;
         catalog[0].secfilt[N].Xm     = NAN;
         catalog[0].secfilt[N].M_20   = NAN_S_SHORT;
 …
+          }
+          if (!KEEP_UBERCAL) {
+            catalog[0].measure[m].Mcal = 0;
+            catalog[0].measure[m].dbFlags &= ~ID_MEAS_PHOTOM_UBERCAL;
+          // only reset Mcal for measures with a matching image
+          // do not reset Mcal for ubercal images unless explicitly requested
+          if (catalog[0].measure[m].dbFlags & ID_MEAS_PHOTOM_UBERCAL) {
+            if (!KEEP_UBERCAL) {
+              catalog[0].measure[m].Mcal = 0.0;
+              catalog[0].measure[m].dbFlags &= ~ID_MEAS_PHOTOM_UBERCAL;
+            }
+          } else {
+            if (getImageEntry (m, 0) >= 0) {
+              catalog[0].measure[m].Mcal = 0.0;
+            }
+          }
           catalog[0].measure[m].dbFlags &= ~PHOTOM_FLAGS;
 …
+  }
+  clean_measures (catalog, 1, TRUE, flatcorr);    /* mark outliers ID_MEAS_POOR_PHOTOM */
+  // XXX make this optional? (do not clean for -averages?)
+  if (!simpleAverage) clean_measures (catalog, 1, TRUE, flatcorr);    /* mark outliers ID_MEAS_POOR_PHOTOM */
   for (i = 0; i < 5; i++) {
     skip_measurements (catalog, i, flatcorr);       /* set ID_MEAS_SKIP for measures to be skipped */
     setMrelOutput  (catalog, 1, i, flatcorr);    /* set Mrel using remaining measures */
+  }
   setMcalOutput (catalog, 1, flatcorr);
+  if (!simpleAverage) setMcalOutput (catalog, 1, flatcorr);
   /* clear ID_STAR_POOR, ID_STAR_FEW values before writing ??? */
 …
+    }
+  }
   if (VERBOSE2) fprintf (stderr, "pass %d, Ntot: "OFF_T_FMT", Ntry: "OFF_T_FMT", Nskip: "OFF_T_FMT", Nkeep: "OFF_T_FMT"\n",
+  if (VERBOSE) fprintf (stderr, "pass %d, Ntot: "OFF_T_FMT", Ntry: "OFF_T_FMT", Nskip: "OFF_T_FMT", Nkeep: "OFF_T_FMT"\n",
                         pass, Ntot, Ntry, Nskip, Nkeep);
+}

trunk/Ohana/src/uniphot/include/setphot.h

-              r33654
+              r33963
 /* global variables set in parameter file */
 # define MAX_PATH_LENGTH 1024
 char         ImageCat[MAX_PATH_LENGTH];
+# define DVO_MAX_PATH 1024
+char         ImageCat[DVO_MAX_PATH];
 char        *CATDIR;
 int          HOST_ID;

trunk/Ohana/src/uniphot/src/initialize_setphot.c

-              r33654
+              r33963
   // force CATDIR to be absolute (so parallel mode will work)
   char *tmpcatdir = NULL;
   ALLOCATE (tmpcatdir, char, MAX_PATH_LENGTH);
+  ALLOCATE (tmpcatdir, char, DVO_MAX_PATH);
   GetConfig (config, "CATDIR",                 "%s",  0, tmpcatdir);
   CATDIR = abspath (tmpcatdir, MAX_PATH_LENGTH);
+  CATDIR = abspath (tmpcatdir, DVO_MAX_PATH);
   free (tmpcatdir);

trunk/Ohana/src/uniphot/src/setphot.c

-              r33654
+              r33963
     zpts = load_zpt_ubercal (argv[1], &Nzpts, &flatcorrTable);
     char flatcorrfile[MAX_PATH_LENGTH];
     int size = snprintf (flatcorrfile, MAX_PATH_LENGTH, "%s/flatcorr.fits", CATDIR);
     assert (size < MAX_PATH_LENGTH);
+    char flatcorrfile[DVO_MAX_PATH];
+    int size = snprintf (flatcorrfile, DVO_MAX_PATH, "%s/flatcorr.fits", CATDIR);
+    assert (size < DVO_MAX_PATH);
     FlatCorrectionSave(&flatcorrTable, flatcorrfile);
     // XXX should this program save any old copy of this file?

trunk/Ohana/src/uniphot/src/update_dvo_setphot.c

r33654	r33963
105	105
106	106	// ensure that the paths are absolute path names
107		char *tmppath = abspath (table->hosts[i].pathname, ~~MAX_PATH_LENG~~TH);
	107	char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
108	108	free (table->hosts[i].pathname);
109	109	table->hosts[i].pathname = tmppath;

trunk/ippMonitor

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippMonitor (added) merged: 33949

trunk/ippScripts/scripts/destreak_restore_camera.pl

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippScripts/scripts/destreak_restore_camera.pl (added) merged: 33949

trunk/ippScripts/scripts/ipp_apply_burntool_single.pl

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippScripts/scripts/ipp_apply_burntool_single.pl (added) merged: 33949

trunk/ippScripts/scripts/magic_destreak.pl

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippScripts/scripts/magic_destreak.pl (added) merged: 33949

trunk/ippTools/share/camtool_find_pendingimfile.sql

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippTools/share/camtool_find_pendingimfile.sql (added) merged: 33949

trunk/ippTools/share/chiptool_setimfiletoupdate.sql

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippTools/share/chiptool_setimfiletoupdate.sql (added) merged: 33949

trunk/ippTools/share/pxadmin_create_tables.sql

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippTools/share/pxadmin_create_tables.sql (added) merged: 33949

trunk/ippTools/share/warptool_towarped.sql

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippTools/share/warptool_towarped.sql (added) merged: 33949

trunk/ippTools/src

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippTools/src (added) merged: 33949

trunk/ippTools/src/magictool.c

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippTools/src/magictool.c (added) merged: 33949

trunk/ippconfig

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippconfig (added) merged: 33948

trunk/ippconfig/recipes/reductionClasses.mdc

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ippconfig/recipes/reductionClasses.mdc (added) merged: 33948

trunk/ppImage/src

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/ppImage/src (added) merged: 33949

trunk/psModules

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20111122/psModules merged: 33044,33085,33087,33096,33638
/branches/eam_branches/ipp-20120405/psModules (added) merged: 33947,33951

trunk/psModules/src/objects/pmModelFuncs.h

-              r29004
+              r33963
 typedef enum {
+    PM_MODEL_OP_NONE    = 0x00,
+    PM_MODEL_OP_FUNC    = 0x01,
+    PM_MODEL_OP_RES0    = 0x02,
+    PM_MODEL_OP_RES1    = 0x04,
+    PM_MODEL_OP_FULL    = 0x07,
+    PM_MODEL_OP_SKY     = 0x08,
+    PM_MODEL_OP_CENTER  = 0x10,
+    PM_MODEL_OP_NORM    = 0x20,
+    PM_MODEL_OP_NOISE   = 0x40,
+    PM_MODEL_OP_NONE     = 0x00,
+    PM_MODEL_OP_FUNC     = 0x01,
+    PM_MODEL_OP_RES0     = 0x02,
+    PM_MODEL_OP_RES1     = 0x04,
+    PM_MODEL_OP_FULL     = 0x07,
+    PM_MODEL_OP_SKY      = 0x08,
+    PM_MODEL_OP_CENTER   = 0x10,
+    PM_MODEL_OP_NORM     = 0x20,
+    PM_MODEL_OP_NOISE    = 0x40,
+    PM_MODEL_OP_MODELVAR = 0x80,
 } pmModelOpMode;

trunk/psModules/src/objects/pmSource.c

-              r33879
+              r33963
     psFree(tmp->pixels);
     psFree(tmp->variance);
+    psFree(tmp->modelVar);
     psFree(tmp->maskObj);
     psFree(tmp->maskView);
 …
     psFree (source->pixels);
     psFree (source->variance);
+    psFree (source->modelVar);
     psFree (source->maskObj);
     psFree (source->maskView);
 …
     source->pixels = NULL;
     source->variance = NULL;
+    source->modelVar = NULL;
     source->maskObj = NULL;
     source->maskView = NULL;
 …
     source->pixels = NULL;
     source->variance = NULL;
+    source->modelVar = NULL;
     source->maskObj = NULL;
     source->maskView = NULL;
 …
     source->pixels   = in->pixels   ? psImageCopyView(NULL, in->pixels)   : NULL;
     source->variance = in->variance ? psImageCopyView(NULL, in->variance) : NULL;
+    source->modelVar = NULL;
     source->maskView = in->maskView ? psImageCopyView(NULL, in->maskView) : NULL;
 …
     bool addNoise = mode & PM_MODEL_OP_NOISE;
+    bool addModelVar = mode & PM_MODEL_OP_MODELVAR;
+    if (addModelVar) psAssert (source->modelVar, "programming error");
     // require the use of pmModelAddWithOffset if we are adding noise (because the model size and norm are rescaled)
 …
+        }
         psF32 **target = source->pixels->data.F32;
+        psF32 **target = addModelVar ? source->modelVar->data.F32 : source->pixels->data.F32;
         for (int iy = 0; iy < source->modelFlux->numRows; iy++) {
 …
+            }
+        }
+        if (add) {
+            source->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED;
+        } else {
+            source->tmpFlags |= PM_SOURCE_TMPF_SUBTRACTED;
+        if (!addModelVar) {
+            if (add) {
+                source->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED;
+            } else {
+                source->tmpFlags |= PM_SOURCE_TMPF_SUBTRACTED;
+            }
+        }
         return true;
 …
         target = source->variance;
+    }
+    if (addModelVar) {
+        target = source->modelVar;
+    }
     if (add) {
         status = pmModelAddWithOffset (target, source->maskObj, model, PM_MODEL_OP_FULL, maskVal, dx, dy);
         source->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED;
+        if (!addNoise && !addModelVar) source->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED;
     } else {
         status = pmModelSubWithOffset (target, source->maskObj, model, PM_MODEL_OP_FULL, maskVal, dx, dy);
         source->tmpFlags |= PM_SOURCE_TMPF_SUBTRACTED;
+        if (!addNoise && !addModelVar) source->tmpFlags |= PM_SOURCE_TMPF_SUBTRACTED;
+    }

trunk/psModules/src/objects/pmSource.h

r33879	r33963
72	72	psImage *pixels; ///< Rectangular region including object pixels.
73	73	psImage *variance; ///< Image variance.
	74	psImage *modelVar; ///< variance based on current models
74	75	psImage *maskObj; ///< unique mask for this object which marks included pixels associated with objects.
75	76	psImage *maskView; ///< view into global image mask for this object region

trunk/psModules/src/objects/pmSourcePhotometry.c

-              r32998
+              r33963
+}
 double pmSourceModelWeight(const pmSource *Mi, int term, const bool unweighted_sum, const float covarFactor, psImageMaskType maskVal)
+double pmSourceModelWeight(const pmSource *Mi, int term, const pmSourceFitVarMode fitVarMode, const float covarFactor, psImageMaskType maskVal)
+{
     PS_ASSERT_PTR_NON_NULL(Mi, NAN);
     double flux = 0, wt = 0, factor = 0;
+    double flux = 0, wt = 1.0, factor = 0;
     const psImage *Pi = Mi->modelFlux;
     assert (Pi != NULL);
+    const psImage *Wi = Mi->variance;
+    if (!unweighted_sum) {
+        assert (Wi != NULL);
+    }
+    const psImage *Wi = NULL;
+    switch (fitVarMode) {
+      case PM_SOURCE_PHOTFIT_CONST:
+        break;
+      case PM_SOURCE_PHOTFIT_IMAGE_VAR:
+        Wi = Mi->variance;
+        psAssert (Wi, "programming error");
+        break;
+      case PM_SOURCE_PHOTFIT_MODEL_VAR:
+        Wi = Mi->modelVar;
+        psAssert (Wi, "programming error");
+        break;
+      case PM_SOURCE_PHOTFIT_NONE:
+        psAbort("programming error");
+    }
     const psImage *Ti = Mi->maskObj;
     assert (Ti != NULL);
 …
     for (int yi = 0; yi < Pi->numRows; yi++) {
         for (int xi = 0; xi < Pi->numCols; xi++) {
+            if (Ti->data.PS_TYPE_IMAGE_MASK_DATA[yi][xi] & maskVal)
+                continue;
+            if (!unweighted_sum) {
+            if (Ti->data.PS_TYPE_IMAGE_MASK_DATA[yi][xi] & maskVal) continue;
+            if (fitVarMode != PM_SOURCE_PHOTFIT_CONST) {
                 wt = covarFactor * Wi->data.F32[yi][xi];
+                if (wt == 0)
+                    continue;
+                if (wt == 0) continue;
+            }
 …
+            }
+            if (unweighted_sum) {
+                flux += (factor * Pi->data.F32[yi][xi]);
+            } else {
+                flux += (factor * Pi->data.F32[yi][xi]) / wt;
+            }
+            // wt is 1.0 for CONST
+            flux += (factor * Pi->data.F32[yi][xi]) / wt;
+        }
+    }
 …
+}
 double pmSourceModelDotModel (const pmSource *Mi, const pmSource *Mj, const bool unweighted_sum, const float covarFactor, psImageMaskType maskVal)
+double pmSourceModelDotModel (const pmSource *Mi, const pmSource *Mj, const pmSourceFitVarMode fitVarMode, const float covarFactor, psImageMaskType maskVal)
+{
     PS_ASSERT_PTR_NON_NULL(Mi, NAN);
 …
     int xIs, xJs, yIs, yJs;
     int xIe, yIe;
+    double flux, wt;
+    double flux;
+    double wt = 1.0;
     const psImage *Pi = Mi->modelFlux;
 …
     assert (Pj != NULL);
+    const psImage *Wi = Mi->variance;
+    if (!unweighted_sum) {
+        assert (Wi != NULL);
+    }
+    const psImage *Wi = NULL;
+    switch (fitVarMode) {
+      case PM_SOURCE_PHOTFIT_CONST:
+        break;
+      case PM_SOURCE_PHOTFIT_IMAGE_VAR:
+        Wi = Mi->variance;
+        psAssert (Wi, "programming error");
+        break;
+      case PM_SOURCE_PHOTFIT_MODEL_VAR:
+        Wi = Mi->modelVar;
+        psAssert (Wi, "programming error");
+        break;
+      case PM_SOURCE_PHOTFIT_NONE:
+        psAbort("programming error");
+    }
     const psImage *Ti = Mi->maskObj;
 …
                 continue;
+            // XXX skip the nonsense weight pixels?
+            if (unweighted_sum) {
+                flux += (Pi->data.F32[yi][xi] * Pj->data.F32[yj][xj]);
+            } else {
+            float value = (Pi->data.F32[yi][xi] * Pj->data.F32[yj][xj]);
+            switch (fitVarMode) {
+              case PM_SOURCE_PHOTFIT_CONST:
+                wt = 1.0;
+                break;
+              case PM_SOURCE_PHOTFIT_IMAGE_VAR:
+              case PM_SOURCE_PHOTFIT_MODEL_VAR:
                 wt = covarFactor * Wi->data.F32[yi][xi];
+                if (wt > 0) {
+                    flux += (Pi->data.F32[yi][xi] * Pj->data.F32[yj][xj]) / wt;
+                }
+            }
+                break;
+              case PM_SOURCE_PHOTFIT_NONE:
+                psAbort("programming error");
+            }
+            // skip pixels with nonsense weight values
+            if (wt <= 0) continue;
+            flux += value / wt;
+        }
+    }
 …
+}
 double pmSourceDataDotModel (const pmSource *Mi, const pmSource *Mj, const bool unweighted_sum, const float covarFactor, psImageMaskType maskVal)
+double pmSourceDataDotModel (const pmSource *Mi, const pmSource *Mj, const pmSourceFitVarMode fitVarMode, const float covarFactor, psImageMaskType maskVal)
+{
     PS_ASSERT_PTR_NON_NULL(Mi, NAN);
 …
     int xIs, xJs, yIs, yJs;
     int xIe, yIe;
+    double flux, wt;
+    double flux;
+    double wt = 1.0;
     const psImage *Pi = Mi->pixels;
 …
     assert (Pj != NULL);
+    const psImage *Wi = Mi->variance;
+    if (!unweighted_sum) {
+        assert (Wi != NULL);
+    }
+    const psImage *Wi = NULL;
+    switch (fitVarMode) {
+      case PM_SOURCE_PHOTFIT_CONST:
+        break;
+      case PM_SOURCE_PHOTFIT_IMAGE_VAR:
+        Wi = Mi->variance;
+        psAssert (Wi, "programming error");
+        break;
+      case PM_SOURCE_PHOTFIT_MODEL_VAR:
+        Wi = Mi->modelVar;
+        psAssert (Wi, "programming error");
+        break;
+      case PM_SOURCE_PHOTFIT_NONE:
+        psAbort("programming error");
+    }
     const psImage *Ti = Mi->maskObj;
 …
                 continue;
+            // XXX skip the nonsense weight pixels?
+            if (unweighted_sum) {
+                flux += (Pi->data.F32[yi][xi] * Pj->data.F32[yj][xj]);
+            } else {
+            float value = (Pi->data.F32[yi][xi] * Pj->data.F32[yj][xj]);
+            switch (fitVarMode) {
+              case PM_SOURCE_PHOTFIT_CONST:
+                wt = 1.0;
+                break;
+              case PM_SOURCE_PHOTFIT_IMAGE_VAR:
+              case PM_SOURCE_PHOTFIT_MODEL_VAR:
                 wt = covarFactor * Wi->data.F32[yi][xi];
+                if (wt > 0) {
+                    flux += (Pi->data.F32[yi][xi] * Pj->data.F32[yj][xj]) / wt;
+                }
+            }
+                break;
+              case PM_SOURCE_PHOTFIT_NONE:
+                psAbort("programming error");
+            }
+            // skip pixels with nonsense weight values
+            if (wt <= 0) continue;
+            flux += value / wt;
+        }
+    }

trunk/psModules/src/objects/pmSourcePhotometry.h

-              r31670
+              r33963
 } pmSourcePhotometryMode;
+typedef enum {
+    PM_SOURCE_PHOTFIT_NONE       = 0,
+    PM_SOURCE_PHOTFIT_CONST      = 1,
+    PM_SOURCE_PHOTFIT_IMAGE_VAR  = 2,
+    PM_SOURCE_PHOTFIT_MODEL_VAR  = 3,
+} pmSourceFitVarMode;
 bool pmSourcePhotometryModel(
     float *fitMag,                      ///< integrated fit magnitude
 …
 bool pmSourceMeasureDiffStats (pmSource *source, psImageMaskType maskVal, psImageMaskType markVal);
 double pmSourceDataDotModel (const pmSource *Mi, const pmSource *Mj, const bool unweighted_sum, const float covarFactor, psImageMaskType maskVal);
 double pmSourceModelDotModel (const pmSource *Mi, const pmSource *Mj, const bool unweighted_sum, const float covarFactor, psImageMaskType maskVal);
 double pmSourceModelWeight(const pmSource *Mi, int term, const bool unweighted_sum, const float covarFactor, psImageMaskType maskVal);
+double pmSourceDataDotModel (const pmSource *Mi, const pmSource *Mj, const pmSourceFitVarMode fitVarMode, const float covarFactor, psImageMaskType maskVal);
+double pmSourceModelDotModel (const pmSource *Mi, const pmSource *Mj, const pmSourceFitVarMode fitVarMode, const float covarFactor, psImageMaskType maskVal);
+double pmSourceModelWeight(const pmSource *Mi, int term, const pmSourceFitVarMode fitVarMode, const float covarFactor, psImageMaskType maskVal);
 bool pmSourceNeighborFlags (pmSource *source);

trunk/psphot

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20111122/psphot merged: 33070-33071,33086,33088,33094-33095,33612,33640
/branches/eam_branches/ipp-20120405/psphot (added) merged: 33946,33953

trunk/psphot/src

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20111122/psphot/src merged: 33070-33071,33086,33094,33612,33640
/branches/eam_branches/ipp-20120405/psphot/src (added) merged: 33946,33953

trunk/psphot/src/psphot.h

r33883	r33963
99	99
100	100	bool psphotFitSourcesLinear (pmConfig config, const pmFPAview view, const char *filerule, bool final);
101		bool psphotFitSourcesLinearReadout (psMetadata recipe, pmReadout readout, psArray sources, pmPSF psf, bool final);
	101	bool psphotFitSourcesLinearReadout (psMetadata recipe, pmReadout readout, psArray sources, pmPSF psf, bool final, pmSourceFitVarMode fitVarMode);
102	102
103	103	bool psphotSourceSize (pmConfig config, const pmFPAview view, const char *filerule, bool getPSFsize);

trunk/psphot/src/psphotDefineFiles.c

-              r25983
+              r33963
         output->save = true;
+    }
+//    // optionally save the smoothed variance model (small FITS image)
+//    if (psMetadataLookupBool(NULL, recipe, "SAVE.BACKMDL")) {
+//        int DX = psMetadataLookupS32 (&status, recipe, "BACKGROUND.XBIN");
+//        int DY = psMetadataLookupS32 (&status, recipe, "BACKGROUND.YBIN");
+//        pmFPAfile *output = pmFPAfileDefineFromFile (config, input, DX, DY, "PSPHOT.VARMDL");
+//        if (!output) {
+//            psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.VARMDL");
+//            return false;
+//        }
+//        output->save = true;
+//    }
+//    // optionally save the smoothed variance model's standard deviation (small FITS image)
+//    if (psMetadataLookupBool(NULL, recipe, "SAVE.VARMDL.STDEV")) {
+//        int DX = psMetadataLookupS32 (&status, recipe, "BACKGROUND.XBIN");
+//        int DY = psMetadataLookupS32 (&status, recipe, "BACKGROUND.YBIN");
+//        pmFPAfile *output = pmFPAfileDefineFromFile (config, input, DX, DY, "PSPHOT.VARMDL.STDEV");
+//        if (!output) {
+//            psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.VARMDL.STDEV");
+//            return false;
+//        }
+//        output->save = true;
+//    }
     // optionally save the PSF Model
     if (psMetadataLookupBool(NULL, recipe, "SAVE.PSF")) {

trunk/psphot/src/psphotEfficiency.c

r32695	r33963
420	420
421	421	// psphotFitSourcesLinearReadout subtracts the model fits
422		if (!psphotFitSourcesLinearReadout(recipe, readout, fakeSourcesAll, psf, true)) {
	422	if (!psphotFitSourcesLinearReadout(recipe, readout, fakeSourcesAll, psf, true, PM_SOURCE_PHOTFIT_CONST)) {
423	423	psError(PS_ERR_UNKNOWN, false, "Unable to perform linear fit on fake sources.");
424	424	psFree(fakeSources);

trunk/psphot/src/psphotFitSourcesLinear.c

-              r32996
+              r33963
 static bool SetBorderMatrixElements (psSparseBorder *border, pmReadout *readout, psArray *sources, bool constant_weights, int SKY_FIT_ORDER, psImageMaskType markVal);
+bool psphotGenerateModelVariance (pmConfig *config, const pmFPAview *view, pmFPAfile *file, int index, psMetadata *recipe, pmReadout *readout, psArray *sources);
+pmSourceFitVarMode psphotGetFitVarMode (psMetadata *recipe);
+bool psphotFreeModelVariance (pmReadout *readout, psArray *sources);
 // for now, let's store the detections on the readout->analysis for each readout
 bool psphotFitSourcesLinear (pmConfig *config, const pmFPAview *view, const char *filerule, bool final)
 …
     assert (recipe);
+    pmSourceFitVarMode fitVarMode = psphotGetFitVarMode (recipe);
+    if (!fitVarMode) {
+        psError (PSPHOT_ERR_CONFIG, true, "failed to get LINEAR_FIT_VARIANCE_MODE");
+        return false;
+    }
+    // MODEL_VAR requires 2 passes -- in the first, we get the rough fluxes; in the second, we
+    // use the flux to define the model variance before fitting the objects.  Other modes only
+    // do a single pass.
+    pmSourceFitVarMode fitVarModePass1 = (fitVarMode == PM_SOURCE_PHOTFIT_MODEL_VAR) ? PM_SOURCE_PHOTFIT_CONST : fitVarMode;
     int num = psphotFileruleCount(config, filerule);
 …
         psAssert (psf, "missing psf?");
         if (!psphotFitSourcesLinearReadout (recipe, readout, sources, psf, final)) {
+        if (!psphotFitSourcesLinearReadout (recipe, readout, sources, psf, final, fitVarModePass1)) {
             psError (PSPHOT_ERR_CONFIG, false, "failed to fit sources (linear) for %s entry %d", filerule, i);
             return false;
+        }
+        // the MODEL_VAR weighting scheme requires knowledge of the model fluxes to generate the variance
+        // after we have determined the initial set of fits, then we can generate the variance image and
+        // re-run the fit against that variance.
+        if (fitVarMode == PM_SOURCE_PHOTFIT_MODEL_VAR) {
+            // generate the model variance image & source pointers
+            if (!psphotGenerateModelVariance (config, view, file, i, recipe, readout, sources)) {
+                psError (PSPHOT_ERR_CONFIG, false, "failed to fit sources (linear) for %s entry %d", filerule, i);
+                return false;
+            }
+            // replace all sources (use TMPF_SUBTRACTED as test)
+            psphotReplaceAllSourcesReadout (config, view, filerule, i, recipe, false);
+            // rerun fit with correct fitVarMode
+            if (!psphotFitSourcesLinearReadout (recipe, readout, sources, psf, final, fitVarMode)) {
+                psError (PSPHOT_ERR_CONFIG, false, "failed to fit sources (linear) for %s entry %d", filerule, i);
+                return false;
+            }
+            // free the model variance image & source pointers
+            if (!psphotFreeModelVariance (readout, sources)) {
+                psError (PSPHOT_ERR_CONFIG, false, "failed to fit sources (linear) for %s entry %d", filerule, i);
+                return false;
+            }
+        }
         psphotVisualShowResidualImage (readout, (num > 0));
 …
+}
+bool psphotFitSourcesLinearReadout (psMetadata *recipe, pmReadout *readout, psArray *sources, pmPSF *psf, bool final) {
+// look up the fit variance mode from the recipe; older recipes do not have the value
+// 'LINEAR_FIT_VARIANCE_MODE'; in those cases, look for 'CONSTANT_PHOTOMETRIC_WEIGHTS' as a boolean and
+// set the value to either CONST or IMAGE_VAR
+pmSourceFitVarMode psphotGetFitVarMode (psMetadata *recipe) {
+    bool status = false;
+    char *fitVarModeString = psMetadataLookupStr(&status, recipe, "LINEAR_FIT_VARIANCE_MODE");
+    if (!status) {
+        bool CONSTANT_PHOTOMETRIC_WEIGHTS = psMetadataLookupBool(&status, recipe, "CONSTANT_PHOTOMETRIC_WEIGHTS");
+        if (!status) {
+            psAbort("You must provide a value for LINEAR_FIT_VARIANCE_MODE or CONSTANT_PHOTOMETRIC_WEIGHTS");
+        }
+        pmSourceFitVarMode fitVarMode = CONSTANT_PHOTOMETRIC_WEIGHTS ? PM_SOURCE_PHOTFIT_CONST : PM_SOURCE_PHOTFIT_IMAGE_VAR;
+        return fitVarMode;
+    }
+    if (!strcasecmp(fitVarModeString, "CONSTANT") || !strcasecmp(fitVarModeString, "CONST")) {
+        return PM_SOURCE_PHOTFIT_CONST;
+    }
+    if (!strcasecmp(fitVarModeString, "IMAGE") || !strcasecmp(fitVarModeString, "IMAGE_VAR")) {
+        return PM_SOURCE_PHOTFIT_IMAGE_VAR;
+    }
+    if (!strcasecmp(fitVarModeString, "MODEL") || !strcasecmp(fitVarModeString, "MODEL_VAR")) {
+        return PM_SOURCE_PHOTFIT_MODEL_VAR;
+    }
+    psError (PSPHOT_ERR_CONFIG, false, "Invalid value for LINEAR_FIT_VARIANCE_MODE (%s)", fitVarModeString);
+    return PM_SOURCE_PHOTFIT_NONE;
+}
+bool psphotFitSourcesLinearReadout (psMetadata *recipe, pmReadout *readout, psArray *sources, pmPSF *psf, bool final, pmSourceFitVarMode fitVarMode) {
     bool status;
 …
     if (psRegionIsNaN (AnalysisRegion)) psAbort("analysis region mis-defined");
-    bool CONSTANT_PHOTOMETRIC_WEIGHTS =
-        psMetadataLookupBool(&status, recipe, "CONSTANT_PHOTOMETRIC_WEIGHTS");
-    if (!status) {
-        psAbort("You must provide a value for the BOOL recipe CONSTANT_PHOTOMETRIC_WEIGHTS");
+    }
     int SKY_FIT_ORDER = psMetadataLookupS32(&status, recipe, "SKY_FIT_ORDER");
     if (!status) {
 …
     psSparseBorder *border = psSparseBorderAlloc (sparse, nBorder);
+    // if fitVarMode is MODEL_VAR, then we need to generate the model image variance
+    // XXX we have two possibilities here:
+    // 1) do 2 passes, where in the first case we use the CONST weighting, and in the second
+    // use the fitted model values to define the model
+    // 2) do a single pass, and use the model guess to define the model variance (but do I trust the Model Guess?)
     // fill out the sparse matrix elements and border elements (B)
     // SRCi is the current source of interest
 …
         // diagonal elements of the sparse matrix (auto-cross-product)
         f = pmSourceModelDotModel (SRCi, SRCi, CONSTANT_PHOTOMETRIC_WEIGHTS, covarFactor, maskVal);
+        f = pmSourceModelDotModel (SRCi, SRCi, fitVarMode, covarFactor, maskVal);
         psSparseMatrixElement (sparse, i, i, f);
         // the formal error depends on the weighting scheme
         if (CONSTANT_PHOTOMETRIC_WEIGHTS) {
             float var = pmSourceModelDotModel (SRCi, SRCi, false, covarFactor, maskVal);
+        // if we have used CONSTANT errors, then we need to calculate the value of the parameter error
+        if (fitVarMode != PM_SOURCE_PHOTFIT_IMAGE_VAR) {
+            float var = pmSourceModelDotModel (SRCi, SRCi, PM_SOURCE_PHOTFIT_IMAGE_VAR, covarFactor, maskVal);
             errors->data.F32[i] = 1.0 / sqrt(var);
         } else {
 …
         // find the image x model value
         f = pmSourceDataDotModel (SRCi, SRCi, CONSTANT_PHOTOMETRIC_WEIGHTS, covarFactor, maskVal);
+        f = pmSourceDataDotModel (SRCi, SRCi, fitVarMode, covarFactor, maskVal);
         psSparseVectorElement (sparse, i, f);
 …
         switch (SKY_FIT_ORDER) {
           case 1:
             f = pmSourceModelWeight (SRCi, 1, CONSTANT_PHOTOMETRIC_WEIGHTS, covarFactor, maskVal);
+            f = pmSourceModelWeight (SRCi, 1, fitVarMode, covarFactor, maskVal);
             psSparseBorderElementB (border, i, 1, f);
             f = pmSourceModelWeight (SRCi, 2, CONSTANT_PHOTOMETRIC_WEIGHTS, covarFactor, maskVal);
+            f = pmSourceModelWeight (SRCi, 2, fitVarMode, covarFactor, maskVal);
             psSparseBorderElementB (border, i, 2, f);
           case 0:
             f = pmSourceModelWeight (SRCi, 0, CONSTANT_PHOTOMETRIC_WEIGHTS, covarFactor, maskVal);
+            f = pmSourceModelWeight (SRCi, 0, fitVarMode, covarFactor, maskVal);
             psSparseBorderElementB (border, i, 0, f);
             break;
 …
             // got an overlap; calculate cross-product and add to output array
             f = pmSourceModelDotModel (SRCi, SRCj, CONSTANT_PHOTOMETRIC_WEIGHTS, covarFactor, maskVal);
+            f = pmSourceModelDotModel (SRCi, SRCj, fitVarMode, covarFactor, maskVal);
             psSparseMatrixElement (sparse, j, i, f);
+        }
 …
     // set the sky, sky_x, sky_y components of border matrix
     SetBorderMatrixElements (border, readout, fitSources, CONSTANT_PHOTOMETRIC_WEIGHTS, SKY_FIT_ORDER, markVal);
+    SetBorderMatrixElements (border, readout, fitSources, (fitVarMode == PM_SOURCE_PHOTFIT_CONST), SKY_FIT_ORDER, markVal);
     psSparseConstraint constraint;
 …
     return true;
+}
+bool psphotModelBackgroundReadout(psImage *model,  // Model image
+                                  psImage *modelStdev, // Model stdev image
+                                  psMetadata *analysis, // Analysis metadata for outputs
+                                  pmReadout *readout, // Readout for which to generate a background model
+                                  psImageBinning *binning, // Binning parameters
+                                  const pmConfig *config,// Configuration
+                                  bool useVarianceImage
+    );
+bool psphotGenerateModelVariance (pmConfig *config, const pmFPAview *view, pmFPAfile *file, int index, psMetadata *recipe, pmReadout *readout, psArray *sources) {
+    bool status = false;
+    psRegion fullRegion = psRegionSet (0, 0, 0, 0);
+    // bit-masks to test for good/bad pixels
+    psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT");
+    assert (maskVal);
+    // bit-mask to mark pixels not used in analysis
+    psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT");
+    assert (markVal);
+    // maskVal is used to test for rejected pixels, and must include markVal
+    maskVal |= markVal;
+    // create a model variance image
+    psImage *modelVar = psImageCopy (NULL, readout->variance, PS_TYPE_F32);
+    // find the binning information
+    psImageBinning *backBinning = psphotBackgroundBinning (modelVar, config);
+    assert (backBinning);
+    psImage *varModel = psImageAlloc(backBinning->nXruff, backBinning->nYruff, PS_TYPE_F32); // Background model
+    psImage *varModelStdev = psImageAlloc(backBinning->nXruff, backBinning->nYruff, PS_TYPE_F32); // Background model
+    if (!psphotModelBackgroundReadout(varModel, varModelStdev, NULL, readout, backBinning, config, true)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to generate background model");
+        psFree (varModel);
+        psFree (varModelStdev);
+        return false;
+    }
+    // linear interpolation to full-scale
+    if (!psImageUnbin (modelVar, varModel, backBinning)) {
+        psError (PSPHOT_ERR_PROG, true, "inconsistent sizes for unbinning");
+        psFree (varModel);
+        psFree (varModelStdev);
+        return false;
+    }
+    psFree (varModel);
+    psFree (varModelStdev);
+    // XXX for a test:
+    psphotSaveImage (NULL, modelVar, "model.bck.fits");
+    // insert all of the source models
+    for (int i = 0; i < sources->n; i++) {
+        // source of interest
+        pmSource *source = sources->data[i];
+        // skip sources which were not fitted already
+        if (!(source->mode & PM_SOURCE_MODE_LINEAR_FIT)) continue;
+        // pixel region appropriate for the source
+        psRegion region = psRegionForImage (source->pixels, fullRegion);
+        // define the source->modelVar pixels (view on modelVar image)
+        psAssert (!source->modelVar, "programming error : modelVar should be NULL here");
+        psAssert (source->modelFlux, "programming error : modelFlux should not be NULL here");
+        psAssert (source->modelFlux->data.F32, "programming error : modelFlux should not be NULL here");
+        source->modelVar = psImageSubset(modelVar, region);
+        // add the source model to the model variance image
+        pmSourceAdd (source, PM_MODEL_OP_MODELVAR, maskVal);
+    }
+    // XXX for a test:
+    psphotSaveImage (NULL, modelVar, "model.var.fits");
+    psphotSaveImage (NULL, readout->variance, "image.var.fits");
+    // we save the model variance for future reference
+    psMetadataAddImage(readout->analysis, PS_LIST_TAIL, "PSPHOT.MODEL.VAR", PS_META_REPLACE, "model variance", modelVar);
+    psFree (modelVar);
+    return true;
+}
+bool psphotFreeModelVariance (pmReadout *readout, psArray *sources) {
+    bool status = false;
+    // find the binning information
+    psImage *modelVar = psMetadataLookupPtr(&status, readout->analysis, "PSPHOT.MODEL.VAR");
+    assert (modelVar);
+    psMetadataRemoveKey (readout->analysis, "PSPHOT.MODEL.VAR");
+    // clear modelVar pointers for all of the source models
+    for (int i = 0; i < sources->n; i++) {
+        // source of interest
+        pmSource *source = sources->data[i];
+        psFree (source->modelVar);
+    }
+    return true;
+}

trunk/psphot/src/psphotModelBackground.c

-              r32348
+              r33963
 // corresponding to the model.  Other information about the background model is saved on the
 // readout->analysis
+static bool psphotModelBackgroundReadout(psImage *model,  // Model image
+                                  psImage *modelStdev, // Model stdev image
+                                  psMetadata *analysis, // Analysis metadata for outputs
+                                  pmReadout *readout, // Readout for which to generate a background model
+                                  psImageBinning *binning, // Binning parameters
+                                  const pmConfig *config // Configuration
+bool psphotModelBackgroundReadout(psImage *model,  // Model image
+                                  psImage *modelStdev, // Model stdev image
+                                  psMetadata *analysis, // Analysis metadata for outputs
+                                  pmReadout *readout, // Readout for which to generate a background model
+                                  psImageBinning *binning, // Binning parameters
+                                  const pmConfig *config,// Configuration
+                                  bool useVarianceImage
+    )
+{
 …
     bool status = true;
+    psImage *image = readout->image, *mask = readout->mask; // Image and mask for readout
+    psImage *image = useVarianceImage ? readout->variance : readout->image;
+    psImage *mask = readout->mask; // Image and mask for readout
     // select the appropriate recipe information
 …
     // we save the binning structure for use in psphotMagnitudes
+    psMetadataAddPtr(analysis, PS_LIST_TAIL, "PSPHOT.BACKGROUND.BINNING", PS_DATA_UNKNOWN | PS_META_REPLACE, "Background binning", binning);
+    if (!useVarianceImage) {
+        psMetadataAddPtr(analysis, PS_LIST_TAIL, "PSPHOT.BACKGROUND.BINNING", PS_DATA_UNKNOWN | PS_META_REPLACE, "Background binning", binning);
+    }
     psVector *dQ = psVectorAllocEmpty (100, PS_TYPE_F32);
 …
     // measure clipped median for subimages
     psRegion ruffRegion = {0,0,0,0};
     psRegion fineRegion = {0,0,0,0};
+    psRegion ruffRegion = psRegionSet (0,0,0,0);
+    psRegion fineRegion = psRegionSet (0,0,0,0);
     for (int iy = 0; iy < model->numRows; iy++) {
         for (int ix = 0; ix < model->numCols; ix++) {
 …
     psVectorStats (statsDQ, dQ, NULL, NULL, 0);
+    psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SKY_MEAN",  PS_META_REPLACE, "sky mean", Value);
+    psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SKY_STDEV", PS_META_REPLACE, "sky stdev", ValueStdev);
+    psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SKY_DQ",    PS_META_REPLACE, "sky quartile slope", statsDQ->sampleMedian);
+    psLogMsg ("psphot", PS_LOG_INFO, "image sky : mean %f, stdev %f, dQ %f", Value, ValueStdev, statsDQ->sampleMedian);
+    // measure image and background stats and save for later output
+    psStats *statsBck = psStatsAlloc (PS_STAT_SAMPLE_MEAN |
+                                      PS_STAT_SAMPLE_STDEV |
+                                      PS_STAT_MIN |
+                                      PS_STAT_MAX);
+    psImageStats (statsBck, model, NULL, 0);
+    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_MN",  PS_META_REPLACE, "sky model mean",          statsBck->sampleMean);
+    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_SIG", PS_META_REPLACE, "sky model stdev",         statsBck->sampleStdev);
+    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_DEV", PS_META_REPLACE, "sky stdev",               ValueStdev);
+    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_DQ",  PS_META_REPLACE, "sky quartile slope",      statsDQ->sampleMedian);
+    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_MAX", PS_META_REPLACE, "sky model maximum value", statsBck->max);
+    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_MIN", PS_META_REPLACE, "sky model minimum value", statsBck->min);
+    psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "MSKY_NX", PS_META_REPLACE, "sky model size (x)",      model->numCols);
+    psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "MSKY_NY", PS_META_REPLACE, "sky model size (y)",      model->numRows);
+    psLogMsg ("psphot", PS_LOG_INFO, "background sky : min %f mean %f max %f stdev %f",
+              statsBck->min, statsBck->sampleMean, statsBck->max, statsBck->sampleStdev);
+    if (!useVarianceImage) {
+        psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SKY_MEAN",  PS_META_REPLACE, "sky mean", Value);
+        psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SKY_STDEV", PS_META_REPLACE, "sky stdev", ValueStdev);
+        psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SKY_DQ",    PS_META_REPLACE, "sky quartile slope", statsDQ->sampleMedian);
+        psLogMsg ("psphot", PS_LOG_INFO, "image sky : mean %f, stdev %f, dQ %f", Value, ValueStdev, statsDQ->sampleMedian);
+        // measure image and background stats and save for later output
+        psStats *statsBck = psStatsAlloc (PS_STAT_SAMPLE_MEAN |
+                                          PS_STAT_SAMPLE_STDEV |
+                                          PS_STAT_MIN |
+                                          PS_STAT_MAX);
+        psImageStats (statsBck, model, NULL, 0);
+        psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_MN",  PS_META_REPLACE, "sky model mean",          statsBck->sampleMean);
+        psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_SIG", PS_META_REPLACE, "sky model stdev",         statsBck->sampleStdev);
+        psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_DEV", PS_META_REPLACE, "sky stdev",               ValueStdev);
+        psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_DQ",  PS_META_REPLACE, "sky quartile slope",      statsDQ->sampleMedian);
+        psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_MAX", PS_META_REPLACE, "sky model maximum value", statsBck->max);
+        psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_MIN", PS_META_REPLACE, "sky model minimum value", statsBck->min);
+        psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "MSKY_NX", PS_META_REPLACE, "sky model size (x)",      model->numCols);
+        psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "MSKY_NY", PS_META_REPLACE, "sky model size (y)",      model->numRows);
+        psLogMsg ("psphot", PS_LOG_INFO, "background sky : min %f mean %f max %f stdev %f",
+                  statsBck->min, statsBck->sampleMean, statsBck->max, statsBck->sampleStdev);
+        psFree(statsBck);
+    } else {
+        psLogMsg ("psphot", PS_LOG_INFO, "variance data : mean %f, stdev %f, dQ %f", Value, ValueStdev, statsDQ->sampleMedian);
+    }
     psFree(statsDQ);
 …
     psFree(stats);
-    psFree(statsBck);
     psFree(statsDefaults);
     psFree(binning);
 …
     psImage *modelStdev = psImageAlloc(binning->nXruff, binning->nYruff, PS_TYPE_F32); // Standard deviation
     if (!psphotModelBackgroundReadout(model, modelStdev, readout->analysis, readout, binning, config)) {
+    if (!psphotModelBackgroundReadout(model, modelStdev, readout->analysis, readout, binning, config, false)) {
         psFree(model);
         psFree(modelStdev);
 …
     pmReadout *modelStdev = pmFPAGenerateReadout(config, view, "PSPHOT.BACKMDL.STDEV", inFPA, binning, index);
     if (!psphotModelBackgroundReadout(model->image, modelStdev->image, model->analysis, readout, binning, config)) {
+    if (!psphotModelBackgroundReadout(model->image, modelStdev->image, model->analysis, readout, binning, config, false)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to generate background model");
         return false;

trunk/psphot/src/psphotStackImageLoop.c

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20111122/psphot/src/psphotStackImageLoop.c merged: 33640
/branches/eam_branches/ipp-20120405/psphot/src/psphotStackImageLoop.c (added) merged: 33946,33953

trunk/psphot/src/psphotTest.c

r33089	r33963
103	103	# if (0)
104	104
105		~~psRegion region = {0,0,0,0}~~; // a region representing the entire array
	105	psRegion region = psRegionSet (0,0,0,0); // a region representing the entire array
106	106	psphotTestArguments (&argc, argv);
107	107

trunk/psphot/src/psphotTestSourceOutput.c

r21183	r33963
136	136
137	137	// generate working image for this source
138		psRegion region = ~~{ix - dx, ix + dx, iy - dy, iy + dy}~~;
	138	psRegion region = psRegionSet(ix - dx, ix + dx, iy - dy, iy + dy);
139	139
140	140	psImage *vM = psImageSubset (imMo, region);

trunk/pstamp/scripts

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/pstamp/scripts (added) merged: 33948

trunk/psvideophot

Property svn:mergeinfo changed
/branches/eam_branches/ipp-20120405/psvideophot (added) merged: 33948

/branches/eam_branches/ipp-20111122/psModules	merged: 33044,33085,33087,33096,33638
/branches/eam_branches/ipp-20120405/psModules (added)	merged: 33947,33951

/branches/eam_branches/ipp-20111122/psphot	merged: 33070-33071,33086,33088,33094-33095,33612,33640
/branches/eam_branches/ipp-20120405/psphot (added)	merged: 33946,33953

/branches/eam_branches/ipp-20111122/psphot/src	merged: 33070-33071,33086,33094,33612,33640
/branches/eam_branches/ipp-20120405/psphot/src (added)	merged: 33946,33953

/branches/eam_branches/ipp-20111122/psphot/src/psphotStackImageLoop.c	merged: 33640
/branches/eam_branches/ipp-20120405/psphot/src/psphotStackImageLoop.c (added)	merged: 33946,33953

Context Navigation

Legend: