IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29797


Ignore:
Timestamp:
Nov 18, 2010, 2:28:52 PM (15 years ago)
Author:
eugene
Message:

unify some of the code; setup threading for basic Update as well as Continue

Location:
branches/eam_branches/ipp-20101103/Ohana/src/dvomerge
Files:
2 added
6 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/Makefile

    r29794 r29797  
    2222all: dvomerge dvoconvert dvosecfilt
    2323
     24#  $(SRC)/dvomergeContinue.$(ARCH).o
     25
    2426DVOMERGE = \
    2527$(SRC)/dvomerge.$(ARCH).o \
    2628$(SRC)/dvomergeUpdate.$(ARCH).o \
    2729$(SRC)/dvomergeCreate.$(ARCH).o \
    28 $(SRC)/dvomergeContinue.$(ARCH).o \
    29 $(SRC)/dvomergeContinue_threaded.$(ARCH).o \
     30$(SRC)/dvomergeUpdate_threaded.$(ARCH).o \
    3031$(SRC)/dvomergeFromList.$(ARCH).o \
     32$(SRC)/dvomergeImageIDs.$(ARCH).o \
    3133$(SRC)/dvo_image_merge_dbs.$(ARCH).o \
    3234$(SRC)/SetSignals.$(ARCH).o \

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/include/dvomerge.h

    r29794 r29797  
    111111int        dvo_image_match_dbs    PROTO((IDmapType *IDmap, FITS_DB *tgt, FITS_DB *src));
    112112int        dvomergeImagesGetMap   PROTO((IDmapType *IDmap, char *input, char *output));
    113 int        dvomergeContinue       PROTO((int argc, char **argv));
    114113int        dvomergeFromList       PROTO((int argc, char **argv));
    115 int        dvomergeContinue_threaded PROTO((int argc, char **argv));
     114int        dvomergeUpdate_threaded PROTO((int argc, char **argv));

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvomerge.c

    r29794 r29797  
    88  dvomerge_args (&argc, argv);
    99
    10   // XXX require both inputs to be sorted?
    11 
    1210  if (argc == 6) {
    1311    if (!strcasecmp (argv[2], "and")) {
    … …  
    1715    }
    1816  }
    19   if (argc == 4) dvomergeUpdate (argc, argv);
    20   if (argc == 5) {
     17  if ((argc == 4) || (argc == 5)) {
    2118    if (NTHREADS) {
    22       dvomergeContinue_threaded (argc, argv);
     19      dvomergeUpdate_threaded (argc, argv);
    2320    } else {
    24       dvomergeContinue (argc, argv);
     21      dvomergeUpdate (argc, argv);
    2522    }
    2623  }

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvomergeContinue.c

    r29794 r29797  
    7171  }
    7272
    73   // XXX need to determine the mapping from the input to the output images
     73  // need to determine the mapping from the input to the output images
    7474  dvomergeImagesGetMap (&IDmap, input, output);
    7575
    … …  
    9696  depth = inlist[0].regions[Ns][0].depth;
    9797 
    98   // XXX test: are we using the table (if so, if it is NULL, we should get errors)
    9998  SetPhotcodeTable(NULL);
    10099
    … …  
    164163  exit (0);
    165164}
    166 
    167 /*** update the image table ***/
    168 int dvomergeImagesGetMap (IDmapType *IDmap, char *input, char *output) {
    169 
    170   FITS_DB inDB;
    171   FITS_DB outDB;
    172   int    status;
    173 
    174   /*** load input1/Images.dat ***/
    175   sprintf (ImageCat, "%s/Images.dat", input);
    176   inDB.mode   = dvo_catalog_catmode (CATMODE);
    177   inDB.format = dvo_catalog_catformat (CATFORMAT);
    178   status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    179   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
    180 
    181   // load the image table
    182   if (inDB.dbstate == LCK_EMPTY) {
    183     Shutdown ("only use -continue for an existing, partially merged database");
    184   }
    185   if (!dvo_image_load (&inDB, VERBOSE, TRUE)) {
    186     Shutdown ("can't read input image catalog %s", inDB.filename);
    187   }
    188 
    189   /*** load output/Images.dat ***/
    190   sprintf (ImageCat, "%s/Images.dat", output);
    191   outDB.mode   = dvo_catalog_catmode (CATMODE);
    192   outDB.format = dvo_catalog_catformat (CATFORMAT);
    193   status       = dvo_image_lock (&outDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    194   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", outDB.filename);
    195 
    196   /* load the image table */
    197   if (outDB.dbstate == LCK_EMPTY) {
    198     Shutdown ("only use -continue for an existing, partially merged database");
    199   }
    200   if (!dvo_image_load (&outDB, VERBOSE, TRUE)) {
    201     Shutdown ("can't read output image catalog %s", outDB.filename);
    202   }
    203 
    204   // convert database table to internal structure & add to output image db
    205   dvo_image_match_dbs(IDmap, &outDB, &inDB);
    206   dvo_image_unlock (&inDB); // unlock input
    207   dvo_image_unlock (&outDB); // unlock output
    208 
    209   return TRUE;
    210 }

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvomergeContinue_threaded.c

    r29795 r29797  
    2525} ThreadData;
    2626
    27 void *dvomergeContinue_threadjob (void *inputData) {
     27void *dvomergeUpdate_threadjob (void *inputData) {
    2828
    2929  off_t j;
    … …  
    9696}
    9797
    98 int dvomergeContinue_threaded (int argc, char **argv) {
     98int dvomergeUpdate_threaded (int argc, char **argv) {
    9999
    100100  int j, launched, done, Nwait, Nrun;
    … …  
    118118  gettimeofday (&start, NULL);
    119119
     120  CONTINUE = FALSE;
    120121  if (strcasecmp (argv[2], "into")) dvomerge_usage();
    121   if (strcasecmp (argv[4], "continue")) dvomerge_usage();
     122  if (argc == 5) {
     123    if (strcasecmp (argv[4], "continue")) dvomerge_usage();
     124    CONTINUE = TRUE;
     125  }
    122126
    123127  input  = argv[1];
    … …  
    170174  }
    171175
    172   // XXX need to determine the mapping from the input to the output images
    173   dvomergeImagesGetMap (&IDmap, input, output);
     176  if (CONTINUE) {
     177    // need to determine the mapping from the input to the output images
     178    dvomergeImagesGetMap (&IDmap, input, output);
     179  } else {
     180    dvomergeImagesUpdate (&IDmap, input, output);
     181  }
    174182
    175183  // load the sky table for the existing database
    … …  
    192200  // loop over the populatable output tables; check for data in input in the corresponding regions
    193201
    194   // XXX test: are we using the table (if so, if it is NULL, we should get errors)
    195   SetPhotcodeTable(NULL);
    196 
    197202  SkyListPopulatedRange (&Ns, &Ne, inlist, 0);
    198203  depth = inlist[0].regions[Ns][0].depth;
    199204 
     205  SetPhotcodeTable(NULL);
     206
    200207  ALLOCATE(threads, pthread_t, NTHREADS);
    201208  ALLOCATE(threadData, ThreadData, NTHREADS);
    … …  
    205212  }
    206213  for (i = 0; i < NTHREADS; i++) {
    207     pthread_create (&threads[i], NULL, &dvomergeContinue_threadjob, (void *) &threadData[i]);
     214    pthread_create (&threads[i], NULL, &dvomergeUpdate_threadjob, (void *) &threadData[i]);
    208215  }
    209216

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvomergeUpdate.c

    r29794 r29797  
    33int dvomergeUpdate (int argc, char **argv) {
    44
    5   int depth;
     5  int depth, CONTINUE;
    66  off_t i, j, Ns, Ne;
    77  SkyTable *outsky, *insky;
    … …  
    1515  int NsecfiltInput, NsecfiltOutput;
    1616
     17  double dtime;
     18  struct timeval start, stop;
     19  gettimeofday (&start, NULL);
     20
     21  CONTINUE = FALSE;
    1722  if (strcasecmp (argv[2], "into")) dvomerge_usage();
     23  if (argc == 5) {
     24    if (strcasecmp (argv[4], "continue")) dvomerge_usage();
     25    CONTINUE = TRUE;
     26  }
    1827
    1928  input  = argv[1];
    … …  
    6675  }
    6776
    68   dvomergeImagesUpdate (&IDmap, input, output);
     77  if (CONTINUE) {
     78    // need to determine the mapping from the input to the output images
     79    dvomergeImagesGetMap (&IDmap, input, output);
     80  } else {
     81    dvomergeImagesUpdate (&IDmap, input, output);
     82  }
    6983
    7084  // load the sky table for the existing database
    … …  
    97111    if (VERBOSE) fprintf (stderr, "input: %s\n", inlist[0].regions[i][0].name);
    98112
    99     // SetPhotcodeTable(inputPhotcodes);
    100113    // load / create output catalog (if catalog does not exist, it will be created)
    101114    LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput);
    … …  
    118131
    119132      // load input catalog
    120       // SetPhotcodeTable(outputPhotcodes);
    121133      LoadCatalog (&outcatalog, outlist[0].regions[j], outlist[0].filename[j], "w", NsecfiltOutput);
    122134
    … …  
    153165  }
    154166
     167  gettimeofday (&stop, NULL);
     168  dtime = DTIME (stop, start);
     169  fprintf (stderr, "SUCCESS: elapsed time %9.4f sec\n", dtime);
     170
    155171  exit (0);
    156172}
    157 
    158 /*** update the image table ***/
    159 int dvomergeImagesUpdate (IDmapType *IDmap, char *input, char *output) {
    160 
    161   FITS_DB inDB;
    162   FITS_DB outDB;
    163   int    status;
    164 
    165   /*** load input1/Images.dat ***/
    166   sprintf (ImageCat, "%s/Images.dat", input);
    167   inDB.mode   = dvo_catalog_catmode (CATMODE);
    168   inDB.format = dvo_catalog_catformat (CATFORMAT);
    169   status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    170   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
    171 
    172   // load the image table
    173   if (inDB.dbstate == LCK_EMPTY) {
    174     // Shutdown ("can't find input image catalog %s", inDB.filename);
    175     IDmap->Nmap = 0;
    176     return TRUE;
    177   }
    178   if (!dvo_image_load (&inDB, VERBOSE, TRUE)) {
    179     Shutdown ("can't read input image catalog %s", inDB.filename);
    180   }
    181 
    182   /*** load output/Images.dat ***/
    183   sprintf (ImageCat, "%s/Images.dat", output);
    184   outDB.mode   = dvo_catalog_catmode (CATMODE);
    185   outDB.format = dvo_catalog_catformat (CATFORMAT);
    186   status       = dvo_image_lock (&outDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    187   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", outDB.filename);
    188 
    189   /* load the image table */
    190   if (outDB.dbstate == LCK_EMPTY) {
    191     dvo_image_create (&outDB, GetZeroPoint());
    192   } else {
    193     if (!dvo_image_load (&outDB, VERBOSE, TRUE)) {
    194       Shutdown ("can't read output image catalog %s", outDB.filename);
    195     }
    196   }
    197 
    198   // convert database table to internal structure & add to output image db
    199   dvo_image_merge_dbs(IDmap, &outDB, &inDB);
    200   dvo_image_unlock (&inDB); // unlock input
    201 
    202   SetProtect (TRUE);
    203   dvo_image_save (&outDB, VERBOSE);
    204   SetProtect (FALSE);
    205   dvo_image_unlock (&outDB); // unlock output
    206 
    207   return TRUE;
    208 }
Note: See TracChangeset for help on using the changeset viewer.