IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29794


Ignore:
Timestamp:
Nov 18, 2010, 11:37:50 AM (15 years ago)
Author:
eugene
Message:

add threading to dvomergeContinue : requires changing LoadCatalog to avoid the global photcode table

Location:
branches/eam_branches/ipp-20101103/Ohana/src/dvomerge
Files:
11 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/Makefile

    r29789 r29794  
    2727$(SRC)/dvomergeCreate.$(ARCH).o \
    2828$(SRC)/dvomergeContinue.$(ARCH).o \
     29$(SRC)/dvomergeContinue_threaded.$(ARCH).o \
    2930$(SRC)/dvomergeFromList.$(ARCH).o \
    3031$(SRC)/dvo_image_merge_dbs.$(ARCH).o \

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/include/dvomerge.h

    r29789 r29794  
    2828float  RADIUS;
    2929int    SKY_DEPTH;
     30int    NTHREADS;
    3031char   *ALTERNATE_PHOTCODE_FILE;
    3132
    … …  
    6869SkyList   *SkyTablePopulatedList_old  PROTO((SkyTable *sky, off_t Ns, off_t Ne));
    6970
    70 int        LoadCatalog            PROTO((Catalog *catalog, SkyRegion *region, char *filename, char *mode));
     71int        LoadCatalog            PROTO((Catalog *catalog, SkyRegion *region, char *filename, char *mode, int Nsecfilt));
    7172
    7273int        merge_catalogs_new     PROTO((SkyRegion *region, Catalog *output, Catalog *input, int *secflitMap));
    … …  
    112113int        dvomergeContinue       PROTO((int argc, char **argv));
    113114int        dvomergeFromList       PROTO((int argc, char **argv));
     115int        dvomergeContinue_threaded PROTO((int argc, char **argv));

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/LoadCatalog.c

    r28329 r29794  
    11# include "dvomerge.h"
    22
    3 int LoadCatalog (Catalog *catalog, SkyRegion *region, char *filename, char *mode) {
     3int LoadCatalog (Catalog *catalog, SkyRegion *region, char *filename, char *mode, int Nsecfilt) {
    44
    55    // set the parameters which guide catalog open/load/create
    66    catalog[0].filename  = filename;
    7     catalog[0].Nsecfilt  = GetPhotcodeNsecfilt ();
     7    catalog[0].Nsecfilt  = Nsecfilt;
    88
    99    // always load all of the data (if any exists)

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/args.c

    r29788 r29794  
    1515    remove_argument (N, argc, argv);
    1616    ALTERNATE_PHOTCODE_FILE = strdup(argv[N]);
     17    remove_argument (N, argc, argv);
     18  }
     19
     20  NTHREADS = 0;
     21  if ((N = get_argument (*argc, argv, "-threads"))) {
     22    remove_argument (N, argc, argv);
     23    NTHREADS = MAX(0, atoi(argv[N]));
    1724    remove_argument (N, argc, argv);
    1825  }

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvoconvert.c

    r29001 r29794  
    66  char filename[256], *input, *output;
    77
    8   int depth;
     8  int depth, Nsecfilt;
    99  off_t i, j, Ns, Ne;
    1010  SkyTable *outsky, *insky;
    … …  
    3232      exit (1);
    3333    }
     34    Nsecfilt = GetPhotcodeNsecfilt();
     35
    3436    // save the photcodes in the output catdir
    3537    sprintf (filename, "%s/Photcodes.dat", output);
    … …  
    6769
    6870    // load / create output catalog
    69     LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w");
     71    LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w", Nsecfilt);
    7072
    7173    // combine only tables at equal or larger depth
    … …  
    7779
    7880      // load input catalog
    79       LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r");
     81      LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r", Nsecfilt);
    8082
    8183      // skip empty input catalogs

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvomerge.c

    r29789 r29794  
    1818  }
    1919  if (argc == 4) dvomergeUpdate (argc, argv);
    20   if (argc == 5) dvomergeContinue (argc, argv);
     20  if (argc == 5) {
     21    if (NTHREADS) {
     22      dvomergeContinue_threaded (argc, argv);
     23    } else {
     24      dvomergeContinue (argc, argv);
     25    }
     26  }
    2127  dvomerge_usage();
    2228  exit (2); // cannot reach here.

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvomergeContinue.c

    r29789 r29794  
    1313  PhotCodeData *outputPhotcodes;
    1414  int *secfiltMap = NULL;
     15  int NsecfiltInput, NsecfiltOutput;
     16
     17  double dtime;
     18  struct timeval start, stop;
     19  gettimeofday (&start, NULL);
    1520
    1621  if (strcasecmp (argv[2], "into")) dvomerge_usage();
    … …  
    3237  }     
    3338  inputPhotcodes = GetPhotcodeTable();
     39  NsecfiltInput = GetPhotcodeNsecfilt();
    3440
    3541  // since we are merging the input db into the output db, the output defines the photcode
    … …  
    4248
    4349    outputPhotcodes = GetPhotcodeTable();
     50    NsecfiltOutput = GetPhotcodeNsecfilt();
    4451
    4552    secfiltMap = GetSecFiltMap(outputPhotcodes, inputPhotcodes);
    … …  
    5158    fprintf(stderr, "%s not found using photcodes from %s/Photcodes.dat\n", filename, input);
    5259    outputPhotcodes = inputPhotcodes;
     60    NsecfiltOutput = NsecfiltInput;
     61
    5362    if (!check_dir_access (output, VERBOSE)) {
    5463      fprintf (stderr, "error creating output database directory %s\n", output);
    … …  
    8796  depth = inlist[0].regions[Ns][0].depth;
    8897 
     98  // XXX test: are we using the table (if so, if it is NULL, we should get errors)
     99  SetPhotcodeTable(NULL);
     100
    89101  // loop over the populated input regions
    90102  for (i = 0; i < inlist[0].Nregions; i++) {
    … …  
    92104    if (VERBOSE) fprintf (stderr, "input: %s\n", inlist[0].regions[i][0].name);
    93105
    94     SetPhotcodeTable(inputPhotcodes);
    95106    // load / create output catalog (if catalog does not exist, it will be created)
    96     LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r");
     107    LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput);
    97108    // skip empty input catalogs
    98109    if (!incatalog.Naves_disk) {
    … …  
    113124
    114125      // load input catalog
    115       SetPhotcodeTable(outputPhotcodes);
    116       LoadCatalog (&outcatalog, outlist[0].regions[j], outlist[0].filename[j], "w");
     126      LoadCatalog (&outcatalog, outlist[0].regions[j], outlist[0].filename[j], "w", NsecfiltOutput);
    117127
    118128      dvo_update_image_IDs (&IDmap, &incatalog);
    … …  
    148158  }
    149159
     160  gettimeofday (&stop, NULL);
     161  dtime = DTIME (stop, start);
     162  fprintf (stderr, "SUCCESS: elapsed time %9.4f sec\n", dtime);
     163
    150164  exit (0);
    151165}

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvomergeCreate.c

    r28855 r29794  
    1515  char *input1, *input2, *output;
    1616  IDmapType IDmap1, IDmap2;
     17  int NsecfiltInput1, NsecfiltInput2, NsecfiltOutput;
    1718
    1819  if (strcasecmp (argv[2], "and")) dvomerge_usage();
    … …  
    3940  }
    4041  input1Photcodes = GetPhotcodeTable();
     42  NsecfiltInput1 = GetPhotcodeNsecfilt();
    4143
    4244  // Read the input2 photcodes
    … …  
    4850  }
    4951  input2Photcodes = GetPhotcodeTable();
     52  NsecfiltInput2 = GetPhotcodeNsecfilt();
    5053
    5154  if (ALTERNATE_PHOTCODE_FILE) {
    … …  
    6063    }
    6164    outputPhotcodes = GetPhotcodeTable();
     65    NsecfiltOutput = GetPhotcodeNsecfilt();
    6266
    6367    secfiltMap1 = GetSecFiltMap(outputPhotcodes, input1Photcodes);
    … …  
    7478    outputPhotcodes = input1Photcodes;
    7579    codesFilename = filename1;
     80    NsecfiltOutput = NsecfiltInput1;
    7681    // secfitMap1 can remain NULL since input1 defines the set of codes
    7782  }
    … …  
    8489  }
    8590
     91  // trigger any dependencies, if any
     92  SetPhotcodeTable(NULL);
     93
    8694  // save the output photcodes in the output catdir
    87   SetPhotcodeTable(outputPhotcodes);
     95  // SetPhotcodeTable(outputPhotcodes);
    8896  sprintf (filename, "%s/Photcodes.dat", output);
    8997  if (!check_file_access (filename, TRUE, TRUE, VERBOSE)) {
    … …  
    121129    if (VERBOSE) fprintf (stderr, "output: %s\n", outsky[0].regions[i].name);
    122130
    123     if (outputPhotcodes) {
    124         SetPhotcodeTable(outputPhotcodes);
    125     }
     131    // if (outputPhotcodes) {
     132    //     SetPhotcodeTable(outputPhotcodes);
     133    // }
    126134    // load / create output catalog
    127     LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w");
    128 
    129     if (input1Photcodes) {
    130         SetPhotcodeTable(input1Photcodes);
    131     }
     135    LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w", NsecfiltOutput);
     136
     137    // if (input1Photcodes) {
     138    // SetPhotcodeTable(input1Photcodes);
     139    // }
    132140    // combine only tables at equal or larger depth
    133141     
    … …  
    138146
    139147      // load input catalog (1)
    140       LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r");
     148      LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r", NsecfiltInput1);
    141149
    142150      // skip empty input catalogs
    … …  
    153161    SkyListFree (inlist);
    154162
    155     if (input2Photcodes) {
    156       SetPhotcodeTable(input2Photcodes);
    157     }
     163    // if (input2Photcodes) {
     164    //   SetPhotcodeTable(input2Photcodes);
     165    // }
    158166
    159167    // load in all of the tables from input2 for this region
    … …  
    163171
    164172      // load input catalog (2)
    165       LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r");
     173      LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r", NsecfiltInput2);
    166174
    167175      // skip empty input catalogs

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvomergeFromList.c

    r29789 r29794  
    1212  char *listname, **list, inputfile[256], outputfile[256];
    1313  int Nlist, NLIST;
     14  int NsecfiltInput, NsecfiltOutput;
    1415
    1516  if (strcasecmp (argv[2], "into")) dvomerge_usage();
    … …  
    5657  }     
    5758  inputPhotcodes = GetPhotcodeTable();
     59  NsecfiltInput = GetPhotcodeNsecfilt();
    5860
    5961  // since we are merging the input db into the output db, the output defines the photcode
    … …  
    6668
    6769    outputPhotcodes = GetPhotcodeTable();
     70    NsecfiltOutput = GetPhotcodeNsecfilt();
    6871
    6972    secfiltMap = GetSecFiltMap(outputPhotcodes, inputPhotcodes);
    … …  
    7578    fprintf(stderr, "%s not found using photcodes from %s/Photcodes.dat\n", filename, input);
    7679    outputPhotcodes = inputPhotcodes;
     80    NsecfiltOutput = NsecfiltInput;
     81
    7782    if (!check_dir_access (output, VERBOSE)) {
    7883      fprintf (stderr, "error creating output database directory %s\n", output);
    … …  
    8994  dvomergeImagesGetMap (&IDmap, input, output);
    9095
     96  SetPhotcodeTable(NULL);
     97
    9198  // loop over the populated input regions
    9299  for (i = 0; i < Nlist; i++) {
    … …  
    96103    sprintf (outputfile, "%s/%s", output, list[i]);
    97104
    98     SetPhotcodeTable(inputPhotcodes);
    99     LoadCatalog (&incatalog, NULL, inputfile, "r");
     105    LoadCatalog (&incatalog, NULL, inputfile, "r", NsecfiltInput);
    100106
    101107    // skip empty input catalogs
    … …  
    115121
    116122    // load input catalog
    117     SetPhotcodeTable(outputPhotcodes);
    118     LoadCatalog (&outcatalog, NULL, outputfile, "w");
     123    // SetPhotcodeTable(outputPhotcodes);
     124    LoadCatalog (&outcatalog, NULL, outputfile, "w", NsecfiltOutput);
    119125
    120126    if (outcatalog.Naverage == 0) {

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvomergeUpdate.c

    r29693 r29794  
    1313  PhotCodeData *outputPhotcodes;
    1414  int *secfiltMap = NULL;
     15  int NsecfiltInput, NsecfiltOutput;
    1516
    1617  if (strcasecmp (argv[2], "into")) dvomerge_usage();
    … …  
    3132  }     
    3233  inputPhotcodes = GetPhotcodeTable();
     34  NsecfiltInput = GetPhotcodeNsecfilt();
    3335
    3436  // since we are merging the input db into the output db, the output defines the photcode
    … …  
    4143
    4244    outputPhotcodes = GetPhotcodeTable();
     45    NsecfiltOutput = GetPhotcodeNsecfilt();
    4346
    4447    secfiltMap = GetSecFiltMap(outputPhotcodes, inputPhotcodes);
    … …  
    5053    fprintf(stderr, "%s not found using photcodes from %s/Photcodes.dat\n", filename, input);
    5154    outputPhotcodes = inputPhotcodes;
     55    NsecfiltOutput = NsecfiltInput;
     56
    5257    if (!check_dir_access (output, VERBOSE)) {
    5358      fprintf (stderr, "error creating output database directory %s\n", output);
    … …  
    8590  depth = inlist[0].regions[Ns][0].depth;
    8691 
     92  SetPhotcodeTable(NULL);
     93
    8794  // loop over the populated input regions
    8895  for (i = 0; i < inlist[0].Nregions; i++) {
    … …  
    9097    if (VERBOSE) fprintf (stderr, "input: %s\n", inlist[0].regions[i][0].name);
    9198
    92     SetPhotcodeTable(inputPhotcodes);
     99    // SetPhotcodeTable(inputPhotcodes);
    93100    // load / create output catalog (if catalog does not exist, it will be created)
    94     LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r");
     101    LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput);
    95102    // skip empty input catalogs
    96103    if (!incatalog.Naves_disk) {
    … …  
    111118
    112119      // load input catalog
    113       SetPhotcodeTable(outputPhotcodes);
    114       LoadCatalog (&outcatalog, outlist[0].regions[j], outlist[0].filename[j], "w");
     120      // SetPhotcodeTable(outputPhotcodes);
     121      LoadCatalog (&outcatalog, outlist[0].regions[j], outlist[0].filename[j], "w", NsecfiltOutput);
    115122
    116123      dvo_update_image_IDs (&IDmap, &incatalog);

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/help.c

    r29789 r29794  
    1919void dvosecfilt_usage(void) {
    2020
    21   fprintf (stderr, "USAGE: dvosecfilt (input) -photcodes (photcodes.txt)\n");
     21  fprintf (stderr, "USAGE: dvosecfilt (catdir) (Nsecfilt)\n");
    2222
    2323  exit (2);
    … …  
    8080
    8181  fprintf (stderr, "USAGE\n");
    82   fprintf (stderr, "  dvosecfilt (input) (Nsecfilt)\n\n");
     82  fprintf (stderr, "  dvosecfilt (catdir) (Nsecfilt)\n\n");
    8383
    84   fprintf (stderr, "  change number of secfilt entries in photcode table (updates secfilt tables only)\n");
     84  fprintf (stderr, "  change number of secfilt entries in photcode table (updates secfilt tables (cps), NSECFILT in cpt files)\n");
     85  fprintf (stderr, "  NOTE: the user must change the photcode table to reflect the change (use photcode-table -export / -import)\n");
    8586 
    8687  fprintf (stderr, "  optional flags:\n");
Note: See TracChangeset for help on using the changeset viewer.