Index: /branches/eam_branches/ipp-20140904/Ohana/src/addstar/Makefile =================================================================== --- /branches/eam_branches/ipp-20140904/Ohana/src/addstar/Makefile (revision 37401) +++ /branches/eam_branches/ipp-20140904/Ohana/src/addstar/Makefile (revision 37402) @@ -24,4 +24,5 @@ sedstar : $(BIN)/sedstar.$(ARCH) load2mass : $(BIN)/load2mass.$(ARCH) +loadstarpar : $(BIN)/loadstarpar.$(ARCH) loadwise : $(BIN)/loadwise.$(ARCH) dumpskycells : $(BIN)/dumpskycells.$(ARCH) @@ -31,7 +32,7 @@ mkcmf : $(BIN)/mkcmf.$(ARCH) -all: addstar addstar_client sedstar load2mass skycells mkcmf loadwise loadsupercos dumpskycells findskycell - -INSTALL = addstar addstar_client sedstar load2mass skycells mkcmf loadwise loadsupercos dumpskycells findskycell +all: addstar addstar_client sedstar load2mass loadstarpar skycells mkcmf loadwise loadsupercos dumpskycells findskycell + +INSTALL = addstar addstar_client sedstar load2mass loadstarpar skycells mkcmf loadwise loadsupercos dumpskycells findskycell # I need to fix the client/server version of addstar now that I have dropped Stars @@ -241,4 +242,24 @@ $(SRC)/psps_ids.$(ARCH).o +LOAD-STARPAR = \ +$(SRC)/loadstarpar.$(ARCH).o \ +$(SRC)/loadstarpar_rawdata.$(ARCH).o \ +$(SRC)/loadstarpar_catalog.$(ARCH).o \ +$(SRC)/loadstarpar_readstars.$(ARCH).o \ +$(SRC)/loadstarpar_make_subset.$(ARCH).o \ +$(SRC)/loadstarpar_save_remote.$(ARCH).o \ +$(SRC)/loadstarpar_io.$(ARCH).o \ +$(SRC)/loadstarpar_remote_hosts.$(ARCH).o \ +$(SRC)/find_matches_refstars.$(ARCH).o \ +$(SRC)/args_loadstarpar.$(ARCH).o \ +$(SRC)/replace_match.$(ARCH).o \ +$(SRC)/update_coords.$(ARCH).o \ +$(SRC)/SkyRegionUtils.$(ARCH).o \ +$(SRC)/StarOps.$(ARCH).o \ +$(SRC)/ConfigInit.$(ARCH).o \ +$(SRC)/Shutdown.$(ARCH).o \ +$(SRC)/SetSignals.$(ARCH).o \ +$(SRC)/psps_ids.$(ARCH).o + LOAD-WISE = \ $(SRC)/loadwise.$(ARCH).o \ Index: /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadstarpar_client.c =================================================================== --- /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadstarpar_client.c (revision 37401) +++ /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadstarpar_client.c (revision 37402) @@ -15,5 +15,5 @@ StarPar_Stars *stars = loadstarpar_load_stars (INPUT, &Nstars); - loadstarpar_catalog (stars, Nstars, region); + loadstarpar_catalog (stars, Nstars, region, CPT_FILE); free (stars); Index: /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadstarpar_rawdata.c =================================================================== --- /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadstarpar_rawdata.c (revision 37401) +++ /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadstarpar_rawdata.c (revision 37402) @@ -39,5 +39,5 @@ // In parallel mode, write out the subset to a disk file. Block until a remote host // is available. In serial mode, just match against the appropriate region and save - loadstarpar_save_remote (subset, Nsubset, hosts, region); + loadstarpar_save_remote (subset, Nsubset, hosts, region, skylist[0].filename[0]); } Index: /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadstarpar_save_remote.c =================================================================== --- /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadstarpar_save_remote.c (revision 37401) +++ /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/loadstarpar_save_remote.c (revision 37402) @@ -2,5 +2,5 @@ # include "loadstarpar.h" -int loadstarpar_save_remote (StarPar_Stars *stars, int Nstars, HostTable *hosts, SkyRegion *region) { +int loadstarpar_save_remote (StarPar_Stars *stars, int Nstars, HostTable *hosts, SkyRegion *region, char *fullname) { char uniquer[12]; @@ -10,5 +10,7 @@ // if this region is a parallel thing, save and launch remote - if (region->hostID) { + if (!region->hostID) { + loadstarpar_catalog (stars, Nstars, region, fullname); + } else { int N = hosts->index[region->hostID]; HostInfo *hostMach = hosts->hosts[N]; Index: /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/resort_catalog.c =================================================================== --- /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/resort_catalog.c (revision 37401) +++ /branches/eam_branches/ipp-20140904/Ohana/src/addstar/src/resort_catalog.c (revision 37402) @@ -38,4 +38,5 @@ resort_catalog_measure (catalog); resort_catalog_lensing (catalog); + resort_catalog_starpar (catalog); } @@ -335,4 +336,141 @@ } +void resort_catalog_starpar (Catalog *catalog) { + + off_t Naverage, Nstarpar; + Starpar *starpar; + Average *average; + off_t i, j, N, currentAve; + + off_t *starparSeq = NULL; + off_t *averageSeq = NULL; + Starpar *starparTMP = NULL; + + // struct timeval start, stop; + // gettimeofday (&start, NULL); + + /* internal counters */ + Nstarpar = catalog[0].Nstarpar; + Naverage = catalog[0].Naverage; + + if (!Nstarpar) return; + + starpar = catalog[0].starpar; + average = catalog[0].average; + + // we have a table of average objects and an unsorted table of measurements. each measurement + // has a reference to the average object sequence (as well as an ID) + // starpar[i].averef -> average[averef] + // starpar[i].objID = average[averef].objID + // starpar[i].catID = average[averef].catID + + // we want a sorted starpar array with all averef entries in sequence + + ALLOCATE (starparSeq, off_t, Nstarpar); + ALLOCATE (averageSeq, off_t, Nstarpar); + + for (i = 0; i < Nstarpar; i++) { + starparSeq[i] = i; + averageSeq[i] = starpar[i].averef; + + if (catalog[0].catformat >= DVO_FORMAT_PS1_V1) { + // earlier formats did not carry the objID or catID, so they are not available (we could assign on load, but we don't) + myAssert(average[averageSeq[i]].catID == starpar[starparSeq[i]].catID, "object / detection mismatch"); +# if (1) + myAssert(average[averageSeq[i]].objID == starpar[starparSeq[i]].objID, "object / detection mismatch"); +# else + // for reasons I do not understand, the mini dvodbs generated on stsci1X had a handful of detections with an inconsistency between averef and objID. + // this happened for 28 detections in the dbs on /data/stsci1?.0/eugene/dvo3pi.20130616, but not at all (as far as I know) in the rest of LAP DVO + if (average[averageSeq[i]].objID != starpar[starparSeq[i]].objID) { + fprintf (stderr, "R"); + starpar[starparSeq[i]].objID = average[averageSeq[i]].objID; // XXX I don't really like this... + } +# endif + } + } + + // check that averageSeq is now in order + // for (i = 1; i < Nstarpar; i++) { + // if (averageSeq[i] < averageSeq[i-1]) { + // fprintf (stderr, "%d ", (int) i); + // } + // } + // fprintf (stderr, "\n"); + + SortAveMatch(starparSeq, averageSeq, Nstarpar); + // MARKTIME("sort : %f sec\n", dtime); + + // check that averageSeq is now in order + // for (i = 1; i < Nstarpar; i++) { + // if (averageSeq[i] < averageSeq[i-1]) { + // fprintf (stderr, "%d ", (int) i); + // } + // } + // fprintf (stderr, "\n"); + + // copy the starpar entries in the sorted order + ALLOCATE (starparTMP, Starpar, Nstarpar); + for (i = 0; i < Nstarpar; i++) { + j = starparSeq[i]; + starparTMP[i] = starpar[j]; + } + // MARKTIME("assign starpar : %f sec\n", dtime); + + // update the values of average.starparOffset and average.Nstarpar + FREE(starpar); + catalog[0].starpar = starparTMP; + + N = 0; + currentAve = averageSeq[0]; + average[currentAve].starparOffset = 0; + for (i = 0; i < Nstarpar; i++) { + if (averageSeq[i] != currentAve) { + // we have hit the next entry in the list + average[currentAve].Nstarpar = N; + N = 0; + currentAve = averageSeq[i]; + average[currentAve].starparOffset = i; + } + N++; + } + // N++; + average[currentAve].Nstarpar = N; + // MARKTIME("update Nstarpar : %f sec\n", dtime); + + int NstarparTotal = 0; + int starparOffsetOK = TRUE; + for (i = 0; i < Naverage; i++) { + NstarparTotal += catalog[0].average[i].Nstarpar; + if (VERBOSE && !(NstarparTotal <= catalog[0].Nstarpar)) { + fprintf (stderr, "too few starpar: %d %d %d\n", (int) i, NstarparTotal, (int) catalog[0].Nstarpar); + } + starparOffsetOK &= (catalog[0].average[i].starparOffset < catalog[0].Nstarpar); + if (VERBOSE && !(catalog[0].average[i].starparOffset < catalog[0].Nstarpar)) { + fprintf (stderr, "offset too large: %d %d %d\n", (int) i, catalog[0].average[i].Nstarpar, (int) catalog[0].Nstarpar); + } + starparOffsetOK &= (catalog[0].average[i].starparOffset + catalog[0].average[i].Nstarpar <= catalog[0].Nstarpar); + if (VERBOSE && !(catalog[0].average[i].starparOffset + catalog[0].average[i].Nstarpar <= catalog[0].Nstarpar)) { + fprintf (stderr, "orrset + Nstarpar too large: %d + %d > %d %d\n", (int) i, catalog[0].average[i].starparOffset, catalog[0].average[i].Nstarpar, (int) catalog[0].Nstarpar); + } + } + + if (!starparOffsetOK) { + fprintf (stderr, "ERROR: catalog %s has an invalid starparOffset\n", catalog[0].filename); + } + + if (NstarparTotal != catalog[0].Nstarpar) { + fprintf (stderr, "ERROR: catalog %s has an invalid Nstarpar\n", catalog[0].filename); + } + + // MARKTIME(" match time %9.4f sec for %7lld starpars, %6lld average\n", dtime, (long long) Nstarpar, (long long) Naverage); + + catalog[0].sorted = TRUE; + + FREE (starparSeq); + FREE (averageSeq); + + return; +} + // sort the measure or lensing Sequence based on the average Sequence entries void SortAveMatch (off_t *MEAS, off_t *AVE, off_t N) { Index: /branches/eam_branches/ipp-20140904/Ohana/src/libautocode/def/starpar-ps1-v5.d =================================================================== --- /branches/eam_branches/ipp-20140904/Ohana/src/libautocode/def/starpar-ps1-v5.d (revision 37401) +++ /branches/eam_branches/ipp-20140904/Ohana/src/libautocode/def/starpar-ps1-v5.d (revision 37402) @@ -2,12 +2,17 @@ EXTNAME DVO_STELLAR_PARAMS_PS1_V5 TYPE BINTABLE -SIZE 48 +SIZE 72 DESCRIPTION DVO Table of Stellar Properties FIELD galLat, GAL_LAT, double, galactic latitude FIELD galLon, GAL_LON, double, galactic longitude -FIELD MKtype, MK_TYPE, float, O0 = 0.0? -FIELD Par, PARALLAX, float, 1/pc -FIELD dPar, PARALLAX_ERR, float, 1/pc +FIELD Ebv, E_BV, float, extinction +FIELD dEbv, E_BV_ERR, float, extinction error +FIELD DistMag, DISTANCE_MOD, float, mag +FIELD dDistMag, DISTANCE_MOD_ERR, float, mag +FIELD M_r, M_R_ABS, float, r-band abs magnitude +FIELD dM_r, M_R_ABS_ERR, float, r-band abs magnitude error +FIELD FeH, F_E_H, float, metallicity +FIELD dFeH, F_E_H_ERR, float, metallicity error FIELD uRA, U_RA, float, model guess for proper motion FIELD uDEC, U_DEC, float, model guess for proper motion @@ -15,4 +20,3 @@ FIELD objID, OBJ_ID, unsigned int, unique ID for object in table FIELD catID, CAT_ID, unsigned int, unique ID for table in which object was first realized - -# this is the list of stellar parameters loaded from Greg Green's tables: +FIELD dummy, DUMMY, unsigned int, dummy Index: /branches/eam_branches/ipp-20140904/Ohana/src/libautocode/def/starpar.d =================================================================== --- /branches/eam_branches/ipp-20140904/Ohana/src/libautocode/def/starpar.d (revision 37401) +++ /branches/eam_branches/ipp-20140904/Ohana/src/libautocode/def/starpar.d (revision 37402) @@ -2,5 +2,5 @@ EXTNAME DVO_STELLAR_PARAMS TYPE BINTABLE -SIZE 48 +SIZE 72 DESCRIPTION DVO Table of Stellar Properties @@ -20,4 +20,5 @@ FIELD objID, OBJ_ID, unsigned int, unique ID for object in table FIELD catID, CAT_ID, unsigned int, unique ID for table in which object was first realized +FIELD dummy, DUMMY, unsigned int, dummy # this is the list of stellar parameters loaded from Greg Green's tables: Index: /branches/eam_branches/ipp-20140904/Ohana/src/libdvo/src/coordops.c =================================================================== --- /branches/eam_branches/ipp-20140904/Ohana/src/libdvo/src/coordops.c (revision 37401) +++ /branches/eam_branches/ipp-20140904/Ohana/src/libdvo/src/coordops.c (revision 37402) @@ -656,5 +656,5 @@ if (status) { char equinoxString[80]; - int haveEquinox = gfits_scan (header, "EQUINOX", "%s", 1, &equinoxString); + int haveEquinox = gfits_scan (header, "EQUINOX", "%s", 1, equinoxString); if (haveEquinox) { // is the string a valid number (it is bad to interpret an error message as year 0.0) @@ -664,5 +664,5 @@ } if (!haveEquinox) { - haveEquinox = gfits_scan (header, "EPOCH", "%s", 1, &equinoxString); + haveEquinox = gfits_scan (header, "EPOCH", "%s", 1, equinoxString); if (haveEquinox) { // is the string a valid number (it is bad to interpret an error message as year 0.0) Index: /branches/eam_branches/ipp-20140904/Ohana/src/libdvo/src/dvo_catalog.c =================================================================== --- /branches/eam_branches/ipp-20140904/Ohana/src/libdvo/src/dvo_catalog.c (revision 37401) +++ /branches/eam_branches/ipp-20140904/Ohana/src/libdvo/src/dvo_catalog.c (revision 37402) @@ -440,14 +440,19 @@ // init all data, or just catalog data void dvo_starpar_init (StarPar *starpar) { - starpar->galLat = NAN; - starpar->galLon = NAN; - starpar->MKtype = NAN; - starpar->Par = NAN; - starpar->dPar = NAN; - starpar->uRA = NAN; - starpar->uDEC = NAN; - starpar->averef = -1; - starpar->objID = -1; - starpar->catID = -1; + starpar->galLat = NAN; + starpar->galLon = NAN; + starpar->Ebv = NAN; + starpar->dEbv = NAN; + starpar->DistMag = NAN; + starpar->dDistMag = NAN; + starpar->M_r = NAN; + starpar->dM_r = NAN; + starpar->FeH = NAN; + starpar->dFeH = NAN; + starpar->uRA = NAN; + starpar->uDEC = NAN; + starpar->averef = -1; + starpar->objID = -1; + starpar->catID = -1; } Index: /branches/eam_branches/ipp-20140904/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c =================================================================== --- /branches/eam_branches/ipp-20140904/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c (revision 37401) +++ /branches/eam_branches/ipp-20140904/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c (revision 37402) @@ -678,9 +678,14 @@ out[i].galLon = in[i].galLon; - out[i].MKtype = in[i].MKtype; - out[i].Par = in[i].Par ; - out[i].dPar = in[i].dPar ; - out[i].uRA = in[i].uRA ; - out[i].uDEC = in[i].uDEC ; + out[i].Ebv = in[i].Ebv ; + out[i].dEbv = in[i].dEbv ; + out[i].DistMag = in[i].DistMag ; + out[i].dDistMag = in[i].dDistMag; + out[i].M_r = in[i].M_r ; + out[i].dM_r = in[i].dM_r ; + out[i].FeH = in[i].FeH ; + out[i].dFeH = in[i].dFeH ; + out[i].uRA = in[i].uRA ; + out[i].uDEC = in[i].uDEC ; out[i].averef = in[i].averef; @@ -703,9 +708,14 @@ out[i].galLon = in[i].galLon; - out[i].MKtype = in[i].MKtype; - out[i].Par = in[i].Par ; - out[i].dPar = in[i].dPar ; - out[i].uRA = in[i].uRA ; - out[i].uDEC = in[i].uDEC ; + out[i].Ebv = in[i].Ebv ; + out[i].dEbv = in[i].dEbv ; + out[i].DistMag = in[i].DistMag ; + out[i].dDistMag = in[i].dDistMag; + out[i].M_r = in[i].M_r ; + out[i].dM_r = in[i].dM_r ; + out[i].FeH = in[i].FeH ; + out[i].dFeH = in[i].dFeH ; + out[i].uRA = in[i].uRA ; + out[i].uDEC = in[i].uDEC ; out[i].averef = in[i].averef;