IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 38695


Ignore:
Timestamp:
Aug 31, 2015, 2:43:11 PM (11 years ago)
Author:
eugene
Message:

three major memory-saving modifications: load the disk version of the AstromOffset table in small chunks (these are converted to the AstromOffsetMap entries anyway); convert AstromOffsetMap to use (float *) not (float ); do not make a subset image array if -use-all-images is selected

Location:
branches/eam_branches/ipp-20150625/Ohana/src/relastro
Files:
12 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/include/relastro.h

    r38675 r38695  
    276276int    USE_ICRF_SHFIT;
    277277int    USE_ICRF_POLE;
     278
     279int    USE_ALL_IMAGES;
    278280
    279281int    RESET;
    … …  
    413415int           liststats_pos       PROTO((double *value, double *dvalue, int N, StatType *stats, int XVERB));
    414416Catalog      *load_catalogs       PROTO((SkyList *skylist, int *Ncatalog, int subselect, int hostID, char *hostpath, char *syncfile));
    415 int           load_images         PROTO((FITS_DB *db, SkyList *skylist, int UseFullOverlap));
     417int           load_images         PROTO((FITS_DB *db, SkyList *skylist, int UseFullOverlap, int UseAllImages));
    416418Image        *select_images       PROTO((SkyList *skylist, Image *timage, off_t Ntimage, off_t **LineNumber, off_t *Nimage, int UseFullOverlap));
    417419

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/FrameCorrectionUtils.c

    r38553 r38695  
    240240  float *buffer = (float *)matrix->buffer;
    241241
    242   float **value = (raDirection) ? map->dXv : map->dYv;
     242  float *value = (raDirection) ? map->dXv : map->dYv;
    243243
    244244  int ix, iy;
    245245  for (ix = 0; ix < map->Nx; ix++) {
    246246    for (iy = 0; iy < map->Ny; iy++) {
    247       buffer[ix + map->Nx*iy] = value[ix][iy];
     247      buffer[ix + map->Nx*iy] = value[ix + map->Nx*iy];
    248248    }
    249249  }
    … …  
    265265  float *buffer = (float *) matrix->buffer;
    266266
    267   float **value = (raDirection) ? map->dXv : map->dYv;
     267  float *value = (raDirection) ? map->dXv : map->dYv;
    268268
    269269  int ix, iy;
    270270  for (ix = 0; ix < map->Nx; ix++) {
    271271    for (iy = 0; iy < map->Ny; iy++) {
    272       value[ix][iy] = buffer[ix + map->Nx*iy];
     272      value[ix + map->Nx*iy] = buffer[ix + map->Nx*iy];
    273273    }
    274274  }

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/ImageOps.c

    r38675 r38695  
    994994  if (VERBOSE) fprintf (stderr, "%d measures marked poor, %d total\n", Ndel, Nave);
    995995  free (R);
    996   free(dR);
    997   free(D);
    998   free(dD);
     996  free (dR);
     997  free (D);
     998  free (dD);
    999999}
    10001000
    … …  
    11231123  if (VERBOSE) fprintf (stderr, "%d measures marked poor, %d total\n", Ndel, Nave);
    11241124  free (R);
    1125   free(dR);
    1126   free(D);
    1127   free(dD);
    1128   free(d2);
    1129   free(index);
     1125  free (dR);
     1126  free (D);
     1127  free (dD);
     1128  free (d2);
     1129  free (index);
    11301130}
    11311131

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/UpdateObjectOffsets.c

    r38671 r38695  
    155155
    156156    char *command = NULL;
    157     strextend (&command, "relastro_client -update-offsets -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -minerror %f",
    158               group->hosts[i][0].hostID, CATDIR, group->hosts[i][0].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE, MIN_ERROR);
     157    strextend (&command, "relastro_client -update-offsets");
     158    strextend (&command, "-hostID %d", group->hosts[i][0].hostID);
     159    strextend (&command, "-D CATDIR %s", CATDIR);
     160    strextend (&command, "-hostdir %s", group->hosts[i][0].pathname);
     161    strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
     162    strextend (&command, "-statmode %s", STATMODE);
     163    strextend (&command, "-minerror %f", MIN_ERROR);
    159164
    160165    if (FIT_MODE == FIT_PM_ONLY)         strextend (&command, "-pm");
    … …  
    176181    if (ExcludeBogus)    strextend (&command, "-exclude-bogus %f", ExcludeBogusRadius);
    177182   
     183    if (USE_ALL_IMAGES)      strextend (&command, "-use-all-images");
    178184    if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1");
    179185

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/args.c

    r38655 r38695  
    251251    PARALLEL_SERIAL = TRUE;
    252252    remove_argument (N, &argc, argv);
     253  }
     254
     255  // If we are looking at the whole sky (or whole relevant sky), use the full image table
     256  // -- this save substantial memory.  this could be automatic if the skyregion covers
     257  // more than 2pi.
     258  USE_ALL_IMAGES = FALSE;
     259  if ((N = get_argument (argc, argv, "-use-all-images"))) {
     260    remove_argument (N, &argc, argv);
     261    USE_ALL_IMAGES = TRUE;
    253262  }
    254263
    … …  
    694703    UserPatch.Dmax = UserPatch.Dmin + 0.001;
    695704    remove_argument (N, &argc, argv);
     705  }
     706
     707  // If we are looking at the whole sky (or whole relevant sky), use the full image table
     708  // -- this save substantial memory.  this could be automatic if the skyregion covers
     709  // more than 2pi.
     710  USE_ALL_IMAGES = FALSE;
     711  if ((N = get_argument (argc, argv, "-use-all-images"))) {
     712    remove_argument (N, &argc, argv);
     713    USE_ALL_IMAGES = TRUE;
    696714  }
    697715

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/high_speed_catalogs.c

    r38655 r38695  
    153153              PHOTCODE_A_LIST, PHOTCODE_B_LIST, RADIUS, outputDir, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
    154154
     155    strextend (&command, "relastro_client -high-speed %s %s %f %s", PHOTCODE_A_LIST, PHOTCODE_B_LIST, RADIUS, outputDir);
     156    strextend (&command, "-hostID %d", table->hosts[i].hostID);
     157    strextend (&command, "-D CATDIR %s", CATDIR);
     158    strextend (&command, "-hostdir %s", table->hosts[i].pathname);
     159    strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
     160
    155161    free (outputDir);
    156162
    … …  
    166172    if (MaxDensityUse) { strextend (&command, "-max-density %f", MaxDensityValue); }
    167173   
     174    if (USE_ALL_IMAGES)      { strextend (&command, "-use-all-images"); }
    168175    if (USE_BASIC_CHECK)     { strextend (&command, "-basic-image-search"); }
    169176    if (FlagOutlier)         { strextend (&command, "-clip %d", CLIP_THRESH); }

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/hpm_catalogs.c

    r38655 r38695  
    148148
    149149    char *command = NULL;
    150     strextend (&command, "relastro_client -hpm %f %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f",
    151               RADIUS, outputDir, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
     150    strextend (&command, "relastro_client -hpm %f %s", RADIUS, outputDir);
     151    strextend (&command, " -hostID %d", table->hosts[i].hostID);
     152    strextend (&command, " -D CATDIR %s", CATDIR);
     153    strextend (&command, " -hostdir %s", table->hosts[i].pathname);
     154    strextend (&command, " -region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
    152155
    153156    free (outputDir);
    … …  
    164167    if (MaxDensityUse) { strextend (&command, "-max-density %f", MaxDensityValue); }
    165168   
     169    if (USE_ALL_IMAGES)      { strextend (&command, "-use-all-images"); }
    166170    if (USE_BASIC_CHECK)     { strextend (&command, "-basic-image-search"); }
    167171    if (FlagOutlier)         { strextend (&command, "-clip %d", CLIP_THRESH); }

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/load_catalogs.c

    r38471 r38695  
    163163
    164164    char *command = NULL;
    165     strextend (&command, "relastro_client -load-objects %s -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s -minerror %f -D RELASTRO_SIGMA_LIM %f",
    166               table->hosts[i].results, table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE, MIN_ERROR, SIGMA_LIM);
     165    strextend (&command, "relastro_client -load-objects %s", table->hosts[i].results);
     166    strextend (&command, " -hostID %d", table->hosts[i].hostID);
     167    strextend (&command, " -D CATDIR %s", CATDIR);
     168    strextend (&command, " -hostdir %s", table->hosts[i].pathname);
     169    strextend (&command, " -region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
     170    strextend (&command, " -statmode %s", STATMODE);
     171    strextend (&command, " -minerror %f", MIN_ERROR);
     172    strextend (&command, " -D RELASTRO_SIGMA_LIM %f", SIGMA_LIM);
    167173
    168174    if (FIT_MODE == FIT_PM_ONLY)         strextend (&command, "-pm");
    … …  
    188194    }
    189195
     196    if (USE_ALL_IMAGES)      strextend (&command, "-use-all-images");
    190197    if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1");
    191198    if (PHOTCODE_KEEP_LIST)  strextend (&command, "+photcode %s", PHOTCODE_KEEP_LIST);

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/load_images.c

    r38688 r38695  
    77 */
    88
    9 int load_images (FITS_DB *db, SkyList *skylist, int UseFullOverlap) {
     9int load_images (FITS_DB *db, SkyList *skylist, int UseFullOverlap, int UseAllImages) {
    1010
    1111  Image     *image, *subset;
    1212  off_t      Nimage, Nsubset;
    13   off_t     *LineNumber;
     13  off_t     *LineNumber, i;
    1414
    1515  INITTIME;
    … …  
    3939
    4040  // select the images which overlap the selected sky regions
    41   subset = select_images (skylist, image, Nimage, &LineNumber, &Nsubset, UseFullOverlap);
    42   MARKTIME("  select images: %f sec\n", dtime);
     41  if (UseAllImages) {
     42    ALLOCATE (LineNumber, off_t, Nimage);
     43    for (i = 0; i < Nimage; i++) LineNumber[i] = i;
     44    subset  = image;
     45    Nsubset = Nimage;
     46  } else {
     47    subset = select_images (skylist, image, Nimage, &LineNumber, &Nsubset, UseFullOverlap);
     48    MARKTIME("  select images: %f sec\n", dtime);
    4349
    44   if (Nsubset == Nimage) {
    45     free (subset);
    46     subset = image;
     50    if (Nsubset == Nimage) {
     51      free (subset);
     52      subset = image;
     53    }
    4754  }
    48 
     55   
    4956  initImages (subset, LineNumber, Nsubset);
    5057  MARKTIME("  init images: %f sec\n", dtime);
    51 
     58 
    5259  initMosaics (subset, Nsubset);
    5360  MARKTIME("  init mosaics: %f sec\n", dtime);

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_client.c

    r38620 r38695  
    7979
    8080      /* load regions and images based on specified sky patch (default depth) */
    81       load_images (&db, skylist, FALSE);
     81      load_images (&db, skylist, FALSE, USE_ALL_IMAGES);
    8282     
    8383      // I can free the database after I have loaded the images...

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_images.c

    r38664 r38695  
    2121
    2222  /* load regions and images based on specified sky patch (default depth) (require full overlap) */
    23   load_images (&db, skylist, TRUE);
     23  load_images (&db, skylist, TRUE, USE_ALL_IMAGES);
    2424  MARKTIME("load images: %f sec\n", dtime);
    2525
    … …  
    9898  freeImageBins (1);
    9999
    100   // XXX why do I do this exactly?
    101   reload_images (&db);
     100  // If we did NOT use all images, then we applied the measured corrections to a subset of
     101  // images.  we now need to set the images associated with db to have those values so
     102  // they will be written out by the dvo_image_update() operations.  If we USE_ALL_IMAGES,
     103  // then we do not need to do this, and we should call dvo_image_save (not dvo_image_update)
     104  if (!USE_ALL_IMAGES) {
     105    reload_images (&db);
     106  }
    102107   
    103108  if (PARALLEL) {
    … …  
    105110    // need to also save the image map table...
    106111    save_astrom_table ();
    107     dvo_image_update (&db, VERBOSE);
     112    if (USE_ALL_IMAGES) {
     113      dvo_image_save (&db, VERBOSE);
     114    } else {
     115      dvo_image_update (&db, VERBOSE);
     116    }
    108117    dvo_image_unlock (&db);
    109118  }
    110119
     120  // NOTE: if we have parallel partitions, then we need to save the Images.dat and
     121  // AstroMap.fits tables BEFORE the remote relastro_clients are launched (they load
     122  // Images.dat and AstroMap.fits)
     123
     124  // if we do NOT have parallel partitions, we must NOT write them out yet: the act of
     125  // writing out the files byte-swaps the data and makes the values invalide for the
     126  // following calls to UpdateObjectOffsets (which attempt to apply the image values from
     127  // the structure in memory)
     128
    111129  // iterate over catalogs to make detection coordinates consistant
    112   UpdateObjectOffsets (skylist, 0, NULL);
     130  if (APPLY_OFFSETS) {
     131    UpdateObjectOffsets (skylist, 0, NULL);
     132  }
    113133
    114134  if (!PARALLEL) {
    115135    // save the updated image parameters
    116136    save_astrom_table ();
    117     dvo_image_update (&db, VERBOSE);
     137    if (USE_ALL_IMAGES) {
     138      dvo_image_save (&db, VERBOSE);
     139    } else {
     140      dvo_image_update (&db, VERBOSE);
     141    }
    118142    dvo_image_unlock (&db);
    119143  }

  • branches/eam_branches/ipp-20150625/Ohana/src/relastro/src/relastro_objects.c

    r38655 r38695  
    150150
    151151    char *command = NULL;
    152     strextend (&command, "relastro_client -update-objects -hostID %d -D CATDIR %s -hostdir %s -region %f %f %f %f -statmode %s",
    153               table->hosts[i].hostID, CATDIR, table->hosts[i].pathname, UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax, STATMODE);
     152    strextend (&command, "relastro_client -update-objects");
     153    strextend (&command, "-hostID %d", table->hosts[i].hostID);
     154    strextend (&command, "-D CATDIR %s", CATDIR);
     155    strextend (&command, "-hostdir %s", table->hosts[i].pathname);
     156    strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
     157    strextend (&command, "-statmode %s", STATMODE);
    154158
    155159    if (FIT_MODE == FIT_PM_ONLY)         { strextend (&command, "-pm"); }
    … …  
    164168    if (MaxDensityUse) { strextend (&command, "-max-density %f", MaxDensityValue); }
    165169    if (FlagOutlier)   { strextend (&command, "-clip %d", CLIP_THRESH); }
     170
     171    if (USE_ALL_IMAGES)      { strextend (&command, "-use-all-images"); }
    166172    if (USE_FIXED_PIXCOORDS) { strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); }
    167173    if (PHOTCODE_KEEP_LIST)  { strextend (&command, "+photcode %s", PHOTCODE_KEEP_LIST); }
Note: See TracChangeset for help on using the changeset viewer.