Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/include/relastro.h =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/include/relastro.h (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/include/relastro.h (revision 38695) @@ -276,4 +276,6 @@ int USE_ICRF_SHFIT; int USE_ICRF_POLE; + +int USE_ALL_IMAGES; int RESET; @@ -413,5 +415,5 @@ int liststats_pos PROTO((double *value, double *dvalue, int N, StatType *stats, int XVERB)); Catalog *load_catalogs PROTO((SkyList *skylist, int *Ncatalog, int subselect, int hostID, char *hostpath, char *syncfile)); -int load_images PROTO((FITS_DB *db, SkyList *skylist, int UseFullOverlap)); +int load_images PROTO((FITS_DB *db, SkyList *skylist, int UseFullOverlap, int UseAllImages)); Image *select_images PROTO((SkyList *skylist, Image *timage, off_t Ntimage, off_t **LineNumber, off_t *Nimage, int UseFullOverlap)); Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/FrameCorrectionUtils.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/FrameCorrectionUtils.c (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/FrameCorrectionUtils.c (revision 38695) @@ -240,10 +240,10 @@ float *buffer = (float *)matrix->buffer; - float **value = (raDirection) ? map->dXv : map->dYv; + float *value = (raDirection) ? map->dXv : map->dYv; int ix, iy; for (ix = 0; ix < map->Nx; ix++) { for (iy = 0; iy < map->Ny; iy++) { - buffer[ix + map->Nx*iy] = value[ix][iy]; + buffer[ix + map->Nx*iy] = value[ix + map->Nx*iy]; } } @@ -265,10 +265,10 @@ float *buffer = (float *) matrix->buffer; - float **value = (raDirection) ? map->dXv : map->dYv; + float *value = (raDirection) ? map->dXv : map->dYv; int ix, iy; for (ix = 0; ix < map->Nx; ix++) { for (iy = 0; iy < map->Ny; iy++) { - value[ix][iy] = buffer[ix + map->Nx*iy]; + value[ix + map->Nx*iy] = buffer[ix + map->Nx*iy]; } } Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/ImageOps.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/ImageOps.c (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/ImageOps.c (revision 38695) @@ -994,7 +994,7 @@ if (VERBOSE) fprintf (stderr, "%d measures marked poor, %d total\n", Ndel, Nave); free (R); - free(dR); - free(D); - free(dD); + free (dR); + free (D); + free (dD); } @@ -1123,9 +1123,9 @@ if (VERBOSE) fprintf (stderr, "%d measures marked poor, %d total\n", Ndel, Nave); free (R); - free(dR); - free(D); - free(dD); - free(d2); - free(index); + free (dR); + free (D); + free (dD); + free (d2); + free (index); } Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/UpdateObjectOffsets.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 38695) @@ -155,6 +155,11 @@ char *command = NULL; - strextend (&command, "relastro_client -update-offsets -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -minerror %f", - group->hosts[i][0].hostID, CATDIR, group->hosts[i][0].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE, MIN_ERROR); + strextend (&command, "relastro_client -update-offsets"); + strextend (&command, "-hostID %d", group->hosts[i][0].hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-hostdir %s", group->hosts[i][0].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + strextend (&command, "-statmode %s", STATMODE); + strextend (&command, "-minerror %f", MIN_ERROR); if (FIT_MODE == FIT_PM_ONLY) strextend (&command, "-pm"); @@ -176,4 +181,5 @@ if (ExcludeBogus) strextend (&command, "-exclude-bogus %f", ExcludeBogusRadius); + if (USE_ALL_IMAGES) strextend (&command, "-use-all-images"); if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/args.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/args.c (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/args.c (revision 38695) @@ -251,4 +251,13 @@ PARALLEL_SERIAL = TRUE; remove_argument (N, &argc, argv); + } + + // If we are looking at the whole sky (or whole relevant sky), use the full image table + // -- this save substantial memory. this could be automatic if the skyregion covers + // more than 2pi. + USE_ALL_IMAGES = FALSE; + if ((N = get_argument (argc, argv, "-use-all-images"))) { + remove_argument (N, &argc, argv); + USE_ALL_IMAGES = TRUE; } @@ -694,4 +703,13 @@ UserPatch.Dmax = UserPatch.Dmin + 0.001; remove_argument (N, &argc, argv); + } + + // If we are looking at the whole sky (or whole relevant sky), use the full image table + // -- this save substantial memory. this could be automatic if the skyregion covers + // more than 2pi. + USE_ALL_IMAGES = FALSE; + if ((N = get_argument (argc, argv, "-use-all-images"))) { + remove_argument (N, &argc, argv); + USE_ALL_IMAGES = TRUE; } Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/high_speed_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/high_speed_catalogs.c (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/high_speed_catalogs.c (revision 38695) @@ -153,4 +153,10 @@ PHOTCODE_A_LIST, PHOTCODE_B_LIST, RADIUS, outputDir, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + strextend (&command, "relastro_client -high-speed %s %s %f %s", PHOTCODE_A_LIST, PHOTCODE_B_LIST, RADIUS, outputDir); + strextend (&command, "-hostID %d", table->hosts[i].hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-hostdir %s", table->hosts[i].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + free (outputDir); @@ -166,4 +172,5 @@ if (MaxDensityUse) { strextend (&command, "-max-density %f", MaxDensityValue); } + if (USE_ALL_IMAGES) { strextend (&command, "-use-all-images"); } if (USE_BASIC_CHECK) { strextend (&command, "-basic-image-search"); } if (FlagOutlier) { strextend (&command, "-clip %d", CLIP_THRESH); } Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/hpm_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/hpm_catalogs.c (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/hpm_catalogs.c (revision 38695) @@ -148,6 +148,9 @@ char *command = NULL; - strextend (&command, "relastro_client -hpm %f %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f", - RADIUS, outputDir, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + strextend (&command, "relastro_client -hpm %f %s", RADIUS, outputDir); + strextend (&command, " -hostID %d", table->hosts[i].hostID); + strextend (&command, " -D CATDIR %s", CATDIR); + strextend (&command, " -hostdir %s", table->hosts[i].pathname); + strextend (&command, " -region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); free (outputDir); @@ -164,4 +167,5 @@ if (MaxDensityUse) { strextend (&command, "-max-density %f", MaxDensityValue); } + if (USE_ALL_IMAGES) { strextend (&command, "-use-all-images"); } if (USE_BASIC_CHECK) { strextend (&command, "-basic-image-search"); } if (FlagOutlier) { strextend (&command, "-clip %d", CLIP_THRESH); } Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/load_catalogs.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/load_catalogs.c (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/load_catalogs.c (revision 38695) @@ -163,6 +163,12 @@ char *command = NULL; - strextend (&command, "relastro_client -load-objects %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -minerror %f -D RELASTRO_SIGMA_LIM %f", - table->hosts[i].results, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE, MIN_ERROR, SIGMA_LIM); + strextend (&command, "relastro_client -load-objects %s", table->hosts[i].results); + strextend (&command, " -hostID %d", table->hosts[i].hostID); + strextend (&command, " -D CATDIR %s", CATDIR); + strextend (&command, " -hostdir %s", table->hosts[i].pathname); + strextend (&command, " -region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + strextend (&command, " -statmode %s", STATMODE); + strextend (&command, " -minerror %f", MIN_ERROR); + strextend (&command, " -D RELASTRO_SIGMA_LIM %f", SIGMA_LIM); if (FIT_MODE == FIT_PM_ONLY) strextend (&command, "-pm"); @@ -188,4 +194,5 @@ } + if (USE_ALL_IMAGES) strextend (&command, "-use-all-images"); if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); if (PHOTCODE_KEEP_LIST) strextend (&command, "+photcode %s", PHOTCODE_KEEP_LIST); Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/load_images.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/load_images.c (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/load_images.c (revision 38695) @@ -7,9 +7,9 @@ */ -int load_images (FITS_DB *db, SkyList *skylist, int UseFullOverlap) { +int load_images (FITS_DB *db, SkyList *skylist, int UseFullOverlap, int UseAllImages) { Image *image, *subset; off_t Nimage, Nsubset; - off_t *LineNumber; + off_t *LineNumber, i; INITTIME; @@ -39,15 +39,22 @@ // select the images which overlap the selected sky regions - subset = select_images (skylist, image, Nimage, &LineNumber, &Nsubset, UseFullOverlap); - MARKTIME(" select images: %f sec\n", dtime); + if (UseAllImages) { + ALLOCATE (LineNumber, off_t, Nimage); + for (i = 0; i < Nimage; i++) LineNumber[i] = i; + subset = image; + Nsubset = Nimage; + } else { + subset = select_images (skylist, image, Nimage, &LineNumber, &Nsubset, UseFullOverlap); + MARKTIME(" select images: %f sec\n", dtime); - if (Nsubset == Nimage) { - free (subset); - subset = image; + if (Nsubset == Nimage) { + free (subset); + subset = image; + } } - + initImages (subset, LineNumber, Nsubset); MARKTIME(" init images: %f sec\n", dtime); - + initMosaics (subset, Nsubset); MARKTIME(" init mosaics: %f sec\n", dtime); Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_client.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_client.c (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_client.c (revision 38695) @@ -79,5 +79,5 @@ /* load regions and images based on specified sky patch (default depth) */ - load_images (&db, skylist, FALSE); + load_images (&db, skylist, FALSE, USE_ALL_IMAGES); // I can free the database after I have loaded the images... Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_images.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_images.c (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_images.c (revision 38695) @@ -21,5 +21,5 @@ /* load regions and images based on specified sky patch (default depth) (require full overlap) */ - load_images (&db, skylist, TRUE); + load_images (&db, skylist, TRUE, USE_ALL_IMAGES); MARKTIME("load images: %f sec\n", dtime); @@ -98,6 +98,11 @@ freeImageBins (1); - // XXX why do I do this exactly? - reload_images (&db); + // If we did NOT use all images, then we applied the measured corrections to a subset of + // images. we now need to set the images associated with db to have those values so + // they will be written out by the dvo_image_update() operations. If we USE_ALL_IMAGES, + // then we do not need to do this, and we should call dvo_image_save (not dvo_image_update) + if (!USE_ALL_IMAGES) { + reload_images (&db); + } if (PARALLEL) { @@ -105,15 +110,34 @@ // need to also save the image map table... save_astrom_table (); - dvo_image_update (&db, VERBOSE); + if (USE_ALL_IMAGES) { + dvo_image_save (&db, VERBOSE); + } else { + dvo_image_update (&db, VERBOSE); + } dvo_image_unlock (&db); } + // NOTE: if we have parallel partitions, then we need to save the Images.dat and + // AstroMap.fits tables BEFORE the remote relastro_clients are launched (they load + // Images.dat and AstroMap.fits) + + // if we do NOT have parallel partitions, we must NOT write them out yet: the act of + // writing out the files byte-swaps the data and makes the values invalide for the + // following calls to UpdateObjectOffsets (which attempt to apply the image values from + // the structure in memory) + // iterate over catalogs to make detection coordinates consistant - UpdateObjectOffsets (skylist, 0, NULL); + if (APPLY_OFFSETS) { + UpdateObjectOffsets (skylist, 0, NULL); + } if (!PARALLEL) { // save the updated image parameters save_astrom_table (); - dvo_image_update (&db, VERBOSE); + if (USE_ALL_IMAGES) { + dvo_image_save (&db, VERBOSE); + } else { + dvo_image_update (&db, VERBOSE); + } dvo_image_unlock (&db); } Index: /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_objects.c =================================================================== --- /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_objects.c (revision 38694) +++ /branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_objects.c (revision 38695) @@ -150,6 +150,10 @@ char *command = NULL; - strextend (&command, "relastro_client -update-objects -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s", - table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE); + strextend (&command, "relastro_client -update-objects"); + strextend (&command, "-hostID %d", table->hosts[i].hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-hostdir %s", table->hosts[i].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + strextend (&command, "-statmode %s", STATMODE); if (FIT_MODE == FIT_PM_ONLY) { strextend (&command, "-pm"); } @@ -164,4 +168,6 @@ if (MaxDensityUse) { strextend (&command, "-max-density %f", MaxDensityValue); } if (FlagOutlier) { strextend (&command, "-clip %d", CLIP_THRESH); } + + if (USE_ALL_IMAGES) { strextend (&command, "-use-all-images"); } if (USE_FIXED_PIXCOORDS) { strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); } if (PHOTCODE_KEEP_LIST) { strextend (&command, "+photcode %s", PHOTCODE_KEEP_LIST); }