IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 35765


Ignore:
Timestamp:
Jul 3, 2013, 2:24:26 PM (13 years ago)
Author:
eugene
Message:

add new options : -images-only, -verify-catalogs, -force-merge, -parallel-input, -verify-catalogs

Location:
trunk/Ohana/src/dvomerge
Files:
11 edited
2 copied

Legend:

Unmodified
Added
Removed

  • trunk/Ohana/src/dvomerge

  • trunk/Ohana/src/dvomerge/include/dvomerge.h

    r35590 r35765  
    2424char  *HOSTDIR;
    2525
     26int    PARALLEL_INPUT;
     27
    2628int    VERBOSE;
    2729int    VERIFY;
     30int    VERIFY_CATALOG_ONLY;
     31int    IMAGES_ONLY;
    2832char   CATDIR[256];
    2933char   GSCFILE[256];
    … …  
    3640char  *ALTERNATE_PHOTCODE_FILE;
    3741int    REPLACE_BY_PHOTCODE;
     42int    FORCE_MERGE;
    3843
    3944char *SINGLE_CPT;

  • trunk/Ohana/src/dvomerge/src/args.c

    r35590 r35765  
    1515  /* verify merge status of output tables, but do not modify */
    1616  VERIFY = FALSE;
     17  VERIFY_CATALOG_ONLY = FALSE;
    1718  if ((N = get_argument (*argc, argv, "-verify"))) {
    1819    VERIFY = TRUE;
    … …  
    2122  if ((N = get_argument (*argc, argv, "-check-only"))) {
    2223    VERIFY = TRUE;
     24    remove_argument (N, argc, argv);
     25  }
     26  if ((N = get_argument (*argc, argv, "-verify-catalogs"))) {
     27    VERIFY = TRUE;
     28    VERIFY_CATALOG_ONLY = TRUE;
     29    remove_argument (N, argc, argv);
     30  }
     31
     32  /* extra error messages */
     33  IMAGES_ONLY = FALSE;
     34  if ((N = get_argument (*argc, argv, "-images-only"))) {
     35    IMAGES_ONLY = TRUE;
    2336    remove_argument (N, argc, argv);
    2437  }
    … …  
    2841    remove_argument (N, argc, argv);
    2942    ALTERNATE_PHOTCODE_FILE = strcreate(argv[N]);
     43    remove_argument (N, argc, argv);
     44  }
     45
     46  // merge even if header claims db has already been merged
     47  FORCE_MERGE = FALSE;
     48  if ((N = get_argument (*argc, argv, "-force-merge"))) {
     49    FORCE_MERGE = TRUE;
    3050    remove_argument (N, argc, argv);
    3151  }
    … …  
    6383  }
    6484
     85  // is the input database a parallel db?
     86  PARALLEL_INPUT = FALSE;
     87  if ((N = get_argument (*argc, argv, "-parallel-input"))) {
     88    PARALLEL_INPUT = TRUE;
     89    remove_argument (N, argc, argv);
     90  }
     91
     92  // is the output database a parallel db?
    6593  PARALLEL = FALSE;
    6694  if ((N = get_argument (*argc, argv, "-parallel"))) {
    … …  
    102130  PARALLEL_MANUAL = FALSE;
    103131  PARALLEL_SERIAL = FALSE;
     132  IMAGES_ONLY = FALSE;
     133
     134  // is the input database a parallel db?
     135  PARALLEL_INPUT = FALSE;
     136  if ((N = get_argument (*argc, argv, "-parallel-input"))) {
     137    PARALLEL_INPUT = TRUE;
     138    remove_argument (N, argc, argv);
     139  }
    104140
    105141  HOST_ID = 0;
    … …  
    134170  if ((N = get_argument (*argc, argv, "-check-only"))) {
    135171    VERIFY = TRUE;
     172    remove_argument (N, argc, argv);
     173  }
     174  if ((N = get_argument (*argc, argv, "-verify-catalogs"))) {
     175    VERIFY = TRUE;
     176    VERIFY_CATALOG_ONLY = TRUE;
     177    remove_argument (N, argc, argv);
     178  }
     179
     180  // merge even if header claims db has already been merged
     181  FORCE_MERGE = FALSE;
     182  if ((N = get_argument (*argc, argv, "-force-merge"))) {
     183    FORCE_MERGE = TRUE;
    136184    remove_argument (N, argc, argv);
    137185  }
    … …  
    227275  }
    228276
     277  // is the output database a parallel db?
    229278  PARALLEL = FALSE;
    230279  if ((N = get_argument (*argc, argv, "-parallel"))) {

  • trunk/Ohana/src/dvomerge/src/dvomerge.c

    r35590 r35765  
    1414        exit (3);
    1515      }
     16      if (IMAGES_ONLY) {
     17        fprintf (stderr, "ERROR : dvomerge (input1) and (input2) into (output) : not compatible with '-images-only'\n");
     18        exit (5);
     19      }
    1620      dvomergeCreate (argc, argv);
    1721    } else {
    … …  
    1923        fprintf (stderr, "WARNING / ERROR : dvomerge (input) into (output) from (list) : VERIFY mode not implemented\n");
    2024        exit (3);
     25      }
     26      if (IMAGES_ONLY) {
     27        fprintf (stderr, "ERROR : dvomerge (input) input (output) from (list) : not compatible with '-images-only'\n");
     28        exit (5);
    2129      }
    2230      dvomergeFromList (argc, argv);

  • trunk/Ohana/src/dvomerge/src/dvomergeImageIDs.c

    r34405 r35765  
    8787  int    status;
    8888
     89  if (VERIFY && VERIFY_CATALOG_ONLY) return TRUE;
     90
    8991  /*** load input1/Images.dat ***/
    9092  sprintf (ImageCat, "%s/Images.dat", input);

  • trunk/Ohana/src/dvomerge/src/dvomergeUpdate.c

    r35590 r35765  
    2222    if (strcasecmp (argv[4], "continue")) dvomerge_usage();
    2323    CONTINUE = TRUE;
     24    if (IMAGES_ONLY) {
     25      fprintf (stderr, "-images-only is not compatible with the 'continue' option\n");
     26      exit (5);
     27    }
    2428  }
    2529  if (VERIFY) CONTINUE = TRUE;
    … …  
    7983  } else {
    8084    dvomergeImagesUpdate (&IDmap, input, output);
     85    if (IMAGES_ONLY) exit (0);
    8186  }
    8287

  • trunk/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c

    r35590 r35765  
    1919    int status = dvomergeUpdate_parallel (input, output, outsky, IDmap);
    2020    return status;
     21  }
     22
     23  // load the list of hosts
     24  HostTable *table_input = NULL;
     25  if (PARALLEL_INPUT) {
     26    table_input = HostTableLoad (input, inlist->hosts);
     27    if (!table_input) {
     28      fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", inlist->hosts, input);
     29      exit (1);
     30    }   
     31
     32    // ensure we have absolute paths for hostdirs
     33    for (i = 0; i < table_input->Nhosts; i++) {
     34      char *tmppath = abspath (table_input->hosts[i].pathname, DVO_MAX_PATH);
     35      free (table_input->hosts[i].pathname);
     36      table_input->hosts[i].pathname = tmppath;
     37    }
    2138  }
    2239
    … …  
    2542    if (!inlist[0].regions[i][0].table) continue;
    2643    if (VERBOSE) fprintf (stderr, "input: %s\n", inlist[0].regions[i][0].name);
     44
     45    // in a parallel context, we need to re-map the input filename
     46    char *filename_input = strcreate (inlist[0].filename[i]);
     47    if (PARALLEL_INPUT) {
     48
     49      int hostID_input = inlist[0].regions[i]->hostID;
     50      int table_index = table_input->index[hostID_input];
     51      char *hostdir_input = table_input->hosts[table_index].pathname;
     52
     53      // set the parameters which guide catalog open/load/create
     54      char hostfile_input[DVO_MAX_PATH];
     55      snprintf (hostfile_input, DVO_MAX_PATH, "%s/%s.cpt", hostdir_input, inlist[0].regions[i]->name);
     56      free (filename_input);
     57      filename_input = strcreate (hostfile_input);
     58    }
    2759
    2860    // SkyListByBounds will return neighbor catalogs if the boundaries exactly match (due to rounding).  Since the regions are not infinitely small,
    … …  
    4577      OutputStatusFree (outstat, outlist->Nregions);
    4678      SkyListFree (outlist);
     79      free (filename_input);
    4780      continue;
    4881    }
    4982
    5083    // get stats for history check, skip input catalog if file not found (NULL inStats)
    51     dmhObjectStats *inStats = dmhObjectStatsRead (inlist[0].filename[i]);
     84    dmhObjectStats *inStats = dmhObjectStatsRead (filename_input);
    5285    if (!inStats) {
    53       if (VERBOSE) fprintf (stderr, "skipping %s, empty \n", inlist[0].filename[i]);
    54       OutputStatusFree (outstat, outlist->Nregions);
    55       SkyListFree (outlist);
     86      if (VERBOSE) fprintf (stderr, "skipping %s, empty \n", filename_input);
     87      OutputStatusFree (outstat, outlist->Nregions);
     88      SkyListFree (outlist);
     89      free (filename_input);
    5690      continue;
    5791    }
    … …  
    75109      missed = (missed || outstat[j].missed);
    76110    }
    77     if (!missed) {
    78       if (VERBOSE || VERIFY) fprintf (stderr, "skipping %s, already merged\n", inlist[0].filename[i]);
     111    if (!FORCE_MERGE && !missed) {
     112      if (VERBOSE || VERIFY) fprintf (stderr, "skipping %s, already merged\n", filename_input);
    79113      OutputStatusFree (outstat, outlist->Nregions);
    80114      dmhObjectStatsFree (inStats);
    81115      SkyListFree (outlist);
     116      free (filename_input);
    82117      continue;
    83118    }
    84119    if (VERIFY) {
    85       fprintf (stderr, "%s NOT merged\n", inlist[0].filename[i]);
     120      fprintf (stderr, "%s NOT merged\n", filename_input);
     121      OutputStatusFree (outstat, outlist->Nregions);
     122      dmhObjectStatsFree (inStats);
     123      SkyListFree (outlist);
     124      free (filename_input);
    86125      continue;
    87126    }
    88127
    89128    // read the input catalog
    90     LoadCatalog (&incatalog, &inlist[0].regions[i][0], inlist[0].filename[i], "r", NsecfiltInput);
     129    LoadCatalog (&incatalog, &inlist[0].regions[i][0], filename_input, "r", NsecfiltInput);
    91130
    92131    // skip empty input catalogs
    … …  
    97136        dmhObjectStatsFree (inStats);
    98137        SkyListFree (outlist);
     138        free (filename_input);
    99139        continue;
    100140    }
    … …  
    106146
    107147      // skip if we have already done the merge
    108       if (!outstat[j].missed) continue;
     148      if (!FORCE_MERGE && !outstat[j].missed) continue;
    109149
    110150      if (VERBOSE) fprintf (stderr, "output : %s\n", outlist[0].regions[j][0].name);
    … …  
    121161      outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
    122162
    123       dmhObjectAdd (outstat[j].history, &outcatalog.header, inStats);
     163      if (outstat[i].missed) {
     164        dmhObjectAdd (outstat[j].history, &outcatalog.header, inStats);
     165      }
    124166
    125167      if (!dvo_catalog_backup (&outcatalog, TRUE)) {
    … …  
    150192    dmhObjectStatsFree (inStats);
    151193
     194    free (filename_input);
    152195    dvo_catalog_unlock (&incatalog);
    153196    dvo_catalog_free (&incatalog);
    … …  
    167210  snprintf (IDmapFilename, DVO_MAX_PATH, "%s/IDmap.fits", absoutput);
    168211
    169   // save IDmap information
    170   if (!IDmapSave (IDmapFilename, IDmap)) {
    171     fprintf (stderr, "ERROR: failure to save the image ID map\n");
    172     exit (1);
     212  if (!VERIFY_CATALOG_ONLY) {
     213    // save IDmap information
     214    if (!IDmapSave (IDmapFilename, IDmap)) {
     215      fprintf (stderr, "ERROR: failure to save the image ID map\n");
     216      exit (1);
     217    }
    173218  }
    174219
    … …  
    196241
    197242    char tmpline[DVO_MAX_PATH];
    198     if (VERBOSE)             { snprintf (tmpline, DVO_MAX_PATH, "%s -v",       command); strcpy (command, tmpline); }
    199     if (VERIFY)              { snprintf (tmpline, DVO_MAX_PATH, "%s -verify",  command); strcpy (command, tmpline); }
    200     if (REPLACE_BY_PHOTCODE) { snprintf (tmpline, DVO_MAX_PATH, "%s -replace", command); strcpy (command, tmpline); }
     243    if (VERBOSE)             { snprintf (tmpline, DVO_MAX_PATH, "%s -v",               command); strcpy (command, tmpline); }
     244    if (VERIFY)              { snprintf (tmpline, DVO_MAX_PATH, "%s -verify",          command); strcpy (command, tmpline); }
     245    if (VERIFY_CATALOG_ONLY) { snprintf (tmpline, DVO_MAX_PATH, "%s -verify-catalogs", command); strcpy (command, tmpline); }
     246    if (REPLACE_BY_PHOTCODE) { snprintf (tmpline, DVO_MAX_PATH, "%s -replace",         command); strcpy (command, tmpline); }
     247    if (PARALLEL_INPUT)      { snprintf (tmpline, DVO_MAX_PATH, "%s -parallel-input",  command); strcpy (command, tmpline); }
     248    if (FORCE_MERGE)         { snprintf (tmpline, DVO_MAX_PATH, "%s -force-merge",     command); strcpy (command, tmpline); }
    201249
    202250    fprintf (stderr, "command: %s\n", command);

  • trunk/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c

    r35590 r35765  
    124124    if (strcasecmp (argv[4], "continue")) dvomerge_usage();
    125125    CONTINUE = TRUE;
     126    if (IMAGES_ONLY) {
     127      fprintf (stderr, "-images-only is not compatible with the 'continue' option\n");
     128      exit (5);
     129    }
    126130  }
    127131
    … …  
    183187  } else {
    184188    dvomergeImagesUpdate (&IDmap, input, output);
     189    if (IMAGES_ONLY) exit (0);
    185190  }
    186191

  • trunk/Ohana/src/dvomerge/src/dvomerge_client.c

    r33963 r35765  
    5555  snprintf (IDmapFilename, DVO_MAX_PATH, "%s/IDmap.fits", absoutput);
    5656
    57   // save IDmap information
    58   IDmap = IDmapLoad (IDmapFilename);
    59   if (!IDmap) {
    60     fprintf (stderr, "ERROR: failure to save the image ID map\n");
    61     exit (1);
     57  if (!VERIFY_CATALOG_ONLY) {
     58    // save IDmap information
     59    IDmap = IDmapLoad (IDmapFilename);
     60    if (!IDmap) {
     61      fprintf (stderr, "ERROR: failure to save the image ID map\n");
     62      exit (1);
     63    }
    6264  }
    6365

  • trunk/Ohana/src/dvomerge/test

    • Property svn:ignore

      •  

        old new  
        44catdir.merge.p2
        55catdir.merge.p3
         6catdir.2mass.input
         7catdir.2mass.input.p1
         8catdir.2mass.input.p2
         9catdir.2mass.input.p3
         10catdir.grizy.output
         11catdir.grizy.output.p1
         12catdir.grizy.output.p2
         13catdir.grizy.output.p3

  • trunk/Ohana/src/dvomerge/test/dvomerge.parallel.sh

    r33963 r35765  
    11#!/bin/csh -f
    22
    3  rm -rf catdir.merge
    4  rsync -auv catdir.2mass/ catdir.merge/
     3 rm -rf catdir.merge catdir.merge.p?
     4 rsync --exclude=.svn -auv catdir.2mass/ catdir.merge/
    55 cp HostTable.dat catdir.merge/
    66 dvodist -out catdir.merge
     7 rm -rf catdir.merge/n????
     8
    79 dvomerge -parallel catdir.grizy into catdir.merge
     10 dvomerge -parallel catdir.grizy into catdir.merge -verify
Note: See TracChangeset for help on using the changeset viewer.