IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 27397


Ignore:
Timestamp:
Mar 22, 2010, 4:54:49 PM (16 years ago)
Author:
eugene
Message:

correctly handle image IDs

Location:
branches/eam_branches/largefiles.20100314/Ohana/src
Files:
1 added
6 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/largefiles.20100314/Ohana/src/addstar/test/dvomerge.dvo

    r27390 r27397  
    7373  tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)"
    7474
    75   exec rm test.in.txt test.cmf
    76   exec rm -rf catdir.test1
    77   exec rm -rf catdir.test2
    78   exec rm -rf catdir.test3
     75  # check on updates to imageID
     76  catdir catdir.test3
     77  imextract imageID
     78  sort imageID
     79  tapOK {imageID[]  == 4} "image IDs exist"
     80  tapOK {imageID[0] == 1} "updated image IDs"
     81  tapOK {imageID[1] == 2} "updated image IDs"
     82  tapOK {imageID[2] == 3} "updated image IDs"
     83  tapOK {imageID[3] == 4} "updated image IDs"
     84
     85  catdir catdir.test3
     86  mextract imageID, time
     87  set id = imageID
     88  set t = time
     89  imextract imageID, time
     90
     91  for i 0 time[]
     92    subset T = t if (id == imageID[$i])
     93    set dT = T - time[$i]
     94    vstat -q dT
     95    tapOK {abs($MEAN)  < 0.00001} "time for measure ID $i (MEAN)"
     96    tapOK {abs($SIGMA) < 0.00001} "time for measure ID $i (SIGMA)"
     97  end
     98
     99  # exec rm test.in.txt test.cmf
     100  # exec rm -rf catdir.test1
     101  # exec rm -rf catdir.test2
     102  # exec rm -rf catdir.test3
    79103
    80104  tapDONE
    … …  
    84108macro test.dvomerge.create
    85109
    86   tapPLAN 51
     110  tapPLAN 21
    87111
    88112  exec rm -rf catdir.test1
    … …  
    148172  tapOK {abs($MEAN)  < 0.001} "ra (in - 0) vs ra (out - 0) (MEAN)"
    149173  tapOK {abs($SIGMA) < 0.001} "ra (in - 0) vs ra (out - 0) (SIGMA)"
     174
     175  # check on updates to imageID
     176  catdir catdir.test3
     177  imextract imageID
     178  sort imageID
     179  tapOK {imageID[]  == 4} "image IDs exist"
     180  tapOK {imageID[0] == 1} "updated image IDs"
     181  tapOK {imageID[1] == 2} "updated image IDs"
     182  tapOK {imageID[2] == 3} "updated image IDs"
     183  tapOK {imageID[3] == 4} "updated image IDs"
     184
     185  catdir catdir.test3
     186  mextract imageID, time
     187  set id = imageID
     188  set t = time
     189  imextract imageID, time
     190
     191  for i 0 time[]
     192    subset T = t if (id == imageID[$i])
     193    set dT = T - time[$i]
     194    vstat -q dT
     195    tapOK {abs($MEAN)  < 0.00001} "time for measure ID $i (MEAN)"
     196    tapOK {abs($SIGMA) < 0.00001} "time for measure ID $i (SIGMA)"
     197  end
     198
     199  # exec rm test.in.txt test.cmf
     200  # exec rm -rf catdir.test1
     201  # exec rm -rf catdir.test2
     202  # exec rm -rf catdir.test3
    150203
    151204  tapDONE

  • branches/eam_branches/largefiles.20100314/Ohana/src/dvomerge/Makefile

    r27384 r27397  
    2424$(SRC)/dvomergeUpdate.$(ARCH).o \
    2525$(SRC)/dvomergeCreate.$(ARCH).o \
     26$(SRC)/dvo_image_merge_dbs.$(ARCH).o \
    2627$(SRC)/SetSignals.$(ARCH).o \
    2728$(SRC)/ConfigInit.$(ARCH).o \

  • branches/eam_branches/largefiles.20100314/Ohana/src/dvomerge/include/dvomerge.h

    r27384 r27397  
    2727
    2828SkyRegion UserPatch;  // used by MODE CAT
     29
     30typedef struct {
     31  off_t Nmap;
     32  off_t *old;
     33  off_t *new;
     34} IDmapType;
    2935
    3036int        main                   PROTO((int argc, char **argv));
    … …  
    6571uint64_t   CreatePSPSObjectID     PROTO((double ra, double dec));
    6672
    67 int        dvo_image_merge_dbs    PROTO((FITS_DB *out, FITS_DB *in));
    68 
     73int        dvomergeImagesCreate   PROTO((IDmapType *IDmap1, char *input1, IDmapType *IDmap2, char *input2, char *output));
     74int        dvomergeImagesUpdate   PROTO((IDmapType *IDmap, char *input, char *output));
     75int        dvo_image_merge_dbs    PROTO((IDmapType *IDmap, FITS_DB *out, FITS_DB *in));
     76off_t      dvo_map_image_ID       PROTO((IDmapType *IDmap, off_t oldID));
     77int        dvo_update_image_IDs   PROTO((IDmapType *IDmap, Catalog *catalog));

  • branches/eam_branches/largefiles.20100314/Ohana/src/dvomerge/src/dvomergeCreate.c

    r27384 r27397  
    1111  Catalog incatalog, outcatalog;
    1212  char filename[256], *input1, *input2, *output;
     13  IDmapType IDmap1, IDmap2;
    1314
    1415  if (strcasecmp (argv[2], "and")) usage();
    … …  
    1819  input2 = argv[3];
    1920  output = argv[5];
     21
     22  dvomergeImagesCreate (&IDmap1, input1, &IDmap2, input2, output);
    2023
    2124  // the first input defines the photcode table & db layout
    … …  
    8083        continue;
    8184      }
     85      dvo_update_image_IDs (&IDmap1, &incatalog);
    8286      merge_catalogs_new (&outsky[0].regions[i], &outcatalog, &incatalog);
    8387      dvo_catalog_unlock (&incatalog);
    … …  
    100104        continue;
    101105      }
     106      dvo_update_image_IDs (&IDmap2, &incatalog);
    102107      merge_catalogs_old (&outsky[0].regions[i], &outcatalog, &incatalog, RADIUS);
    103108      dvo_catalog_unlock (&incatalog);
    … …  
    120125  }
    121126
     127  exit (0);
     128}
     129
    122130  /*** update the image table ***/
    123   {
     131int dvomergeImagesCreate (IDmapType *IDmap1, char *input1, IDmapType *IDmap2, char *input2, char *output) {
     132
    124133    FITS_DB in1DB;
    125134    FITS_DB in2DB;
    … …  
    157166
    158167    // convert database table to internal structure & add to output image db
    159     dvo_image_merge_dbs(&outDB, &in1DB);
     168    dvo_image_merge_dbs(IDmap1, &outDB, &in1DB);
    160169    dvo_image_unlock (&in1DB); // unlock input1
    161170
    … …  
    177186
    178187    // convert database table to internal structure & add to output image db
    179     dvo_image_merge_dbs(&outDB, &in2DB);
     188    dvo_image_merge_dbs(IDmap2, &outDB, &in2DB);
    180189    dvo_image_unlock (&in2DB); // unlock input2
    181190
    … …  
    185194    SetProtect (FALSE);
    186195    dvo_image_unlock (&outDB); // unlock output
     196
     197    return TRUE;
    187198  }
    188199
    189   exit (0);
    190 }
    191 
    192 // merge db2 into db1
    193 int dvo_image_merge_dbs (FITS_DB *out, FITS_DB *in) {
    194 
    195   Image *images;
    196   off_t Nimages;
    197   off_t Nout;
    198  
    199   images = gfits_table_get_Image (&in[0].ftable, &Nimages, &in[0].swapped);
    200 
    201  /* adjust header */
    202   Nout = 0;
    203   gfits_scan (&out[0].header, "NIMAGES", "%lld", 1, (long long *) &Nout);
    204 
    205   if (!out[0].swapped) {
    206     gfits_convert_Image ((Image *) out[0].ftable.buffer, sizeof(Image), Nout);
    207     out[0].swapped = TRUE;
    208   }
    209 
    210   Nout += Nimages;
    211   gfits_modify (&out[0].header, "NIMAGES", "%lld", 1, (long long) Nout);
    212 
    213   gfits_add_rows (&out[0].ftable, (char *) images, Nimages, sizeof(Image));
    214   return TRUE;
    215 }

  • branches/eam_branches/largefiles.20100314/Ohana/src/dvomerge/src/dvomergeUpdate.c

    r27384 r27397  
    99  Catalog incatalog, outcatalog;
    1010  char filename[256], *input, *output;
     11  IDmapType IDmap;
    1112
    1213  if (strcasecmp (argv[2], "into")) usage();
    … …  
    2122    exit (1);
    2223  }
     24
     25  dvomergeImagesUpdate (&IDmap, input, output);
    2326
    2427  // load the sky table for the existing database
    … …  
    5962        continue;
    6063      }
     64      dvo_update_image_IDs (&IDmap, &incatalog);
    6165      merge_catalogs_old (&outsky[0].regions[i], &outcatalog, &incatalog, RADIUS);
    6266      dvo_catalog_unlock (&incatalog);
    … …  
    7983  }
    8084
    81   /*** update the image table ***/
    82   {
    83     FITS_DB inDB;
    84     FITS_DB outDB;
    85     int    status;
    86 
    87     /*** load input1/Images.dat ***/
    88     sprintf (ImageCat, "%s/Images.dat", input);
    89     inDB.mode   = dvo_catalog_catmode (CATMODE);
    90     inDB.format = dvo_catalog_catformat (CATFORMAT);
    91     status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    92     if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
    93 
    94     // load the image table
    95     if (inDB.dbstate == LCK_EMPTY) {
    96       Shutdown ("can't find input (1) image catalog %s", inDB.filename);
    97     }
    98     if (!dvo_image_load (&inDB, VERBOSE, TRUE)) {
    99       Shutdown ("can't read input (1) image catalog %s", inDB.filename);
    100     }
    101 
    102     /*** load output/Images.dat ***/
    103     sprintf (ImageCat, "%s/Images.dat", output);
    104     outDB.mode   = dvo_catalog_catmode (CATMODE);
    105     outDB.format = dvo_catalog_catformat (CATFORMAT);
    106     status       = dvo_image_lock (&outDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    107     if (!status) Shutdown ("ERROR: failure to lock image catalog %s", outDB.filename);
    108 
    109     /* load the image table */
    110     if (outDB.dbstate == LCK_EMPTY) {
    111       Shutdown ("can't find input (2) image catalog %s", outDB.filename);
    112     }
    113     if (!dvo_image_load (&outDB, VERBOSE, TRUE)) {
    114       Shutdown ("can't read input (2) image catalog %s", outDB.filename);
    115     }
    116 
    117     // convert database table to internal structure & add to output image db
    118     dvo_image_merge_dbs(&outDB, &inDB);
    119     dvo_image_unlock (&inDB); // unlock input
    120 
    121     SetProtect (TRUE);
    122     dvo_image_save (&outDB, VERBOSE);
    123     SetProtect (FALSE);
    124     dvo_image_unlock (&outDB); // unlock output
    125   }
    126 
    12785  exit (0);
    12886}
    12987
    130 /*
     88/*** update the image table ***/
     89int dvomergeImagesUpdate (IDmapType *IDmap, char *input, char *output) {
    13190
    132    for the upgrade of the synth database, at a minimum I need to use an output format that uses
    133    doubles for the astrometry.  Optionally I should merge in the 2MASS photometry measurements, since that
    134    would be quite useful for comparison. 
     91  FITS_DB inDB;
     92  FITS_DB outDB;
     93  int    status;
    13594
    136    generality questions:
    137    * optional
     95  /*** load input1/Images.dat ***/
     96  sprintf (ImageCat, "%s/Images.dat", input);
     97  inDB.mode   = dvo_catalog_catmode (CATMODE);
     98  inDB.format = dvo_catalog_catformat (CATFORMAT);
     99  status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
     100  if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
    138101
     102  // load the image table
     103  if (inDB.dbstate == LCK_EMPTY) {
     104    Shutdown ("can't find input (1) image catalog %s", inDB.filename);
     105  }
     106  if (!dvo_image_load (&inDB, VERBOSE, TRUE)) {
     107    Shutdown ("can't read input (1) image catalog %s", inDB.filename);
     108  }
    139109
     110  /*** load output/Images.dat ***/
     111  sprintf (ImageCat, "%s/Images.dat", output);
     112  outDB.mode   = dvo_catalog_catmode (CATMODE);
     113  outDB.format = dvo_catalog_catformat (CATFORMAT);
     114  status       = dvo_image_lock (&outDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
     115  if (!status) Shutdown ("ERROR: failure to lock image catalog %s", outDB.filename);
    140116
    141    */
     117  /* load the image table */
     118  if (outDB.dbstate == LCK_EMPTY) {
     119    Shutdown ("can't find input (2) image catalog %s", outDB.filename);
     120  }
     121  if (!dvo_image_load (&outDB, VERBOSE, TRUE)) {
     122    Shutdown ("can't read input (2) image catalog %s", outDB.filename);
     123  }
    142124
     125  // convert database table to internal structure & add to output image db
     126  dvo_image_merge_dbs(IDmap, &outDB, &inDB);
     127  dvo_image_unlock (&inDB); // unlock input
     128
     129  SetProtect (TRUE);
     130  dvo_image_save (&outDB, VERBOSE);
     131  SetProtect (FALSE);
     132  dvo_image_unlock (&outDB); // unlock output
     133
     134  return TRUE;
     135}

  • branches/eam_branches/largefiles.20100314/Ohana/src/dvomerge/src/merge_catalogs_new.c

    r27384 r27397  
    6464  output[0].Nmeasure = Nmeasure;
    6565  output[0].Nsecf_mem = Naverage * Nsecfilt;
     66
     67  // If we are using dvomergeCreate to split an existing catalog, then the max objID in
     68  // each of the new subdivisions of the original catalog is the max objID of the original
     69  // catalog.  It is not allowed to use dvomerge to unify catalogs to a lower-resolution
     70  // partition.
     71  output[0].objID = input[0].objID; // new max value, save on catalog close
    6672 
    6773  if (VERBOSE) {
Note: See TracChangeset for help on using the changeset viewer.