IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29785


Ignore:
Timestamp:
Nov 16, 2010, 12:41:50 PM (15 years ago)
Author:
eugene
Message:

created dvoverify; add dvorepairFixTables

Location:
branches/eam_branches/ipp-20101103/Ohana/src/dvomerge
Files:
1 added
3 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/Makefile

    r29779 r29785  
    8787$(BIN)/dvorepair.$(ARCH) : $(DVOREPAIR)
    8888
    89 INSTALL = dvomerge dvoconvert dvosecfilt dvorepair
     89DVOVERIFY = \
     90$(SRC)/dvoverify.$(ARCH).o
     91
     92$(DVOVERIFY)  : $(INC)/dvomerge.h
     93
     94$(BIN)/dvoverify.$(ARCH) : $(DVOVERIFY)
     95
     96INSTALL = dvomerge dvoconvert dvosecfilt dvorepair dvoverify
    9097
    9198# dependancy rules for binary code #########################

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvorepair.c

    r29779 r29785  
    66 
    77  if (!strcmp(argv[1], "-fix-cpt")) dvorepairFixCPT(argc, argv);
     8  // if (!strcmp(argv[1], "-fix-tables")) dvorepairFixTables(argc, argv);
    89  if (!strcmp(argv[1], "-images-vs-measures")) dvorepairImagesVsMeasures(argc, argv);
    910  if (!strcmp(argv[1], "-delete-image-list")) dvorepairDeleteImageList(argc, argv);

  • branches/eam_branches/ipp-20101103/Ohana/src/dvomerge/src/dvoverify.c

    r29001 r29785  
    88*/
    99
     10int VerifyTableFile (char *filename);
     11
    1012int main (int argc, char **argv) {
    1113
    12   char filename[256], *input, *output;
    13 
    14   int depth;
    15   off_t i, j, Ns, Ne;
    16   SkyTable *outsky, *insky;
     14  char filename[1024];
     15
     16  int N, Nbad;
     17  off_t i;
     18  SkyTable *insky;
    1719  SkyList *inlist;
    18   Catalog incatalog, outcatalog;
    19 
    20   SetSignals ();
    21   dvoconvert_help (argc, argv);
    22   ConfigInit (&argc, argv);
    23   dvoconvert_args (&argc, argv);
    24 
    25   if (strcasecmp (argv[2], "to")) dvoconvert_usage();
    26   input = argv[1];
    27   output = argv[3];
    28 
    29   // load input images, save to output images
    30   dvoConvert_copy_images (input, output);
    31 
    32   // copy photcode table
    33   {
    34     // the first input defines the photcode table & db layout
    35     sprintf (filename, "%s/Photcodes.dat", input);
    36     if (!LoadPhotcodes (filename, NULL, FALSE)) {
    37       fprintf (stderr, "error loading photcode table %s\n", filename);
    38       exit (1);
    39     }
    40     // save the photcodes in the output catdir
    41     sprintf (filename, "%s/Photcodes.dat", output);
    42     if (!check_file_access (filename, TRUE, TRUE, VERBOSE)) {
    43       fprintf (stderr, "error creating output catdir %s\n", output);
    44       exit (1);
    45     }
    46     if (!SavePhotcodesFITS (filename)) {
    47       fprintf (stderr, "error saving photcode table %s\n", filename);
    48       exit (1);
    49     }
    50   }
    51 
    52   // copy skytable
    53   {
    54     // load the sky table for the existing database
    55     insky = SkyTableLoadOptimal (input, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
    56     SkyTableSetFilenames (insky, input, "cpt");
    57 
    58     // generate an output table populated at the desired depth
    59     // XXX force this to match the input?
    60     outsky = SkyTableLoadOptimal (output, NULL, GSCFILE, TRUE, SKY_DEPTH, VERBOSE);
    61     SkyTableSetFilenames (outsky, output, "cpt");
    62 
    63     SkyTablePopulatedRange (&Ns, &Ne, insky, 0);
    64     depth = insky[0].regions[Ns].depth;
    65     // XXX this seems to imply that insky is a uniform depth...
    66   }
    67 
    68   // loop over all input catalogs, save to output catalogs
    69   // loop over the populatable output regions
    70   for (i = 0; i < outsky[0].Nregions; i++) {
    71     if (!outsky[0].regions[i].table) continue;
    72     if (VERBOSE) fprintf (stderr, "output: %s\n", outsky[0].regions[i].name);
    73 
    74     // load / create output catalog
    75     LoadCatalog (&outcatalog, &outsky[0].regions[i], outsky[0].filename[i], "w");
    76 
    77     // combine only tables at equal or larger depth
    78      
    79     // load in all of the tables from input for this region
    80     inlist = SkyListByBounds (insky, depth, outsky[0].regions[i].Rmin, outsky[0].regions[i].Rmax, outsky[0].regions[i].Dmin, outsky[0].regions[i].Dmax);
    81     for (j = 0; j < inlist[0].Nregions; j++) {
    82       if (VERBOSE) fprintf (stderr, "input : %s\n", inlist[0].regions[j][0].name);
    83 
    84       // load input catalog
    85       LoadCatalog (&incatalog, inlist[0].regions[j], inlist[0].filename[j], "r");
    86 
    87       // skip empty input catalogs
    88       if (!incatalog.Naves_disk) {
    89         dvo_catalog_unlock (&incatalog);
    90         dvo_catalog_free (&incatalog);
    91         continue;
    92       }
    93       merge_catalogs_new (&outsky[0].regions[i], &outcatalog, &incatalog);
    94       dvo_catalog_unlock (&incatalog);
    95       dvo_catalog_free (&incatalog);
    96     }
    97     SkyListFree (inlist);
    98 
    99     outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
    100     dvo_catalog_save (&outcatalog, VERBOSE);
    101     dvo_catalog_unlock (&outcatalog);
    102     dvo_catalog_free (&outcatalog);
    103   }
    104 
    105   // save the output sky table copy
    106   sprintf (filename, "%s/SkyTable.fits", output);
    107   check_file_access (filename, TRUE, TRUE, VERBOSE);
    108   if (!SkyTableSave (outsky, filename)) {
    109     fprintf (stderr, "ERROR: failed to save sky table for %s\n", output);
     20  SkyRegion UserPatch;
     21  // Catalog catalog;
     22
     23  // restrict to a portion of the sky
     24  UserPatch.Rmin = 0;
     25  UserPatch.Rmax = 360;
     26  UserPatch.Dmin = -90;
     27  UserPatch.Dmax = +90;
     28  if ((N = get_argument (argc, argv, "-region"))) {
     29    remove_argument (N, &argc, argv);
     30    UserPatch.Rmin = atof (argv[N]);
     31    remove_argument (N, &argc, argv);
     32    UserPatch.Rmax = atof (argv[N]);
     33    remove_argument (N, &argc, argv);
     34    UserPatch.Dmin = atof (argv[N]);
     35    remove_argument (N, &argc, argv);
     36    UserPatch.Dmax = atof (argv[N]);
     37    remove_argument (N, &argc, argv);
     38  }
     39
     40  if (argc != 2) {
     41    fprintf (stderr, "USAGE: dvoverify (catdir) [-region Rmin Rmax Dmin Dmax]\n");
     42    fprintf (stderr, "  catdir : database of interest\n");
     43    exit (2);
     44  }
     45
     46  char *catdir = argv[1];
     47
     48  Nbad = 0;
     49
     50  // XXX make this step optional
     51  if (1) {
     52    // check the photcode table
     53    sprintf (filename, "%s/Photcodes.dat", catdir);
     54    if (!VerifyTableFile (filename)) {
     55      Nbad ++;
     56    }
     57
     58    // check the skytable
     59    sprintf (filename, "%s/SkyTable.fits", catdir);
     60    if (!VerifyTableFile (filename)) {
     61      Nbad ++;
     62    }
     63
     64    // check the image table
     65    sprintf (filename, "%s/Images.dat", catdir);
     66    if (!VerifyTableFile (filename)) {
     67      Nbad ++;
     68    }
     69  }
     70
     71  // load the sky table for the existing database
     72  insky = SkyTableLoadOptimal (catdir, NULL, NULL, FALSE, SKY_DEPTH_HST, VERBOSE);
     73  myAssert(insky, "can't read SkyTable");
     74  SkyTableSetFilenames (insky, catdir, "cpt");
     75  inlist = SkyListByPatch (insky, -1, &UserPatch);
     76 
     77  // loop over all catalogs, save to output catalogs
     78  for (i = 0; i < inlist[0].Nregions; i++) {
     79    if (!inlist[0].regions[i][0].table) continue;
     80
     81    sprintf (filename, "%s/%s.cpm", catdir, inlist[0].regions[i][0].name);
     82    if (!VerifyTableFile (filename)) {
     83      Nbad ++;
     84    }
     85
     86    sprintf (filename, "%s/%s.cpm", catdir, inlist[0].regions[i][0].name);
     87    if (!VerifyTableFile (filename)) {
     88      Nbad ++;
     89    }
     90
     91    sprintf (filename, "%s/%s.cpm", catdir, inlist[0].regions[i][0].name);
     92    if (!VerifyTableFile (filename)) {
     93      Nbad ++;
     94    }
     95  }
     96
     97  if (Nbad > 0) {
     98    fprintf (stderr, "ERROR: %d files are bad\n", Nbad);
    11099    exit (1);
    111100  }
    112101
     102  fprintf (stderr, "SUCCESS: no files are bad\n");
    113103  exit (0);
    114104}
    115105
    116 int dvoConvert_copy_images (char *input, char *output) {
    117 
    118   FITS_DB inDB;
    119   FITS_DB outDB;
    120   int    status;
    121 
    122   Image *images;
    123   off_t Nimages;
    124   off_t ID;
    125 
    126   /*** load output/Images.dat ***/
    127   sprintf (ImageCat, "%s/Images.dat", output);
    128   outDB.mode   = dvo_catalog_catmode (CATMODE);
    129   outDB.format = dvo_catalog_catformat (CATFORMAT);
    130   status       = dvo_image_lock (&outDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    131   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", outDB.filename);
    132 
    133   // output image table should not already exist
    134   if (outDB.dbstate != LCK_EMPTY) {
    135     Shutdown ("ERROR: image table %s already exists", outDB.filename);
    136   }
    137   dvo_image_create (&outDB, GetZeroPoint());
    138 
    139   /*** load input/Images.dat ***/
    140   sprintf (ImageCat, "%s/Images.dat", input);
    141   // inDB.mode   = dvo_catalog_catmode (CATMODE);
    142   // inDB.format = dvo_catalog_catformat (CATFORMAT);
    143   status       = dvo_image_lock (&inDB, ImageCat, 3600.0, LCK_XCLD);  // shorter timeout?
    144   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", inDB.filename);
    145 
    146   // load the image table
    147   if (inDB.dbstate == LCK_EMPTY) {
    148     Shutdown ("can't find input image catalog %s", inDB.filename);
    149   }
    150   if (!dvo_image_load (&inDB, VERBOSE, TRUE)) {
    151     Shutdown ("can't read input image catalog %s", inDB.filename);
    152   }
    153 
    154   // convert the raw image table to Image type (byteswap if needed)
    155   images = gfits_table_get_Image (&inDB.ftable, &Nimages, &inDB.swapped);
    156   if (!images) {
    157     fprintf (stderr, "ERROR: failed to read images\n");
    158     exit (2);
    159   }
    160 
    161   // update additional metadata
    162   gfits_scan (&inDB.header, "IMAGEID", OFF_T_FMT, 1,  &ID);
    163   gfits_modify (&outDB.header, "NIMAGES", OFF_T_FMT, 1,  Nimages);
    164   gfits_modify (&outDB.header, "IMAGEID", OFF_T_FMT, 1,  ID);
    165 
    166   // copy input rows to output table
    167   gfits_add_rows (&outDB.ftable, (char *) images, Nimages, sizeof(Image));
    168 
    169   // write out the image table to disk
    170   SetProtect (TRUE);
    171   dvo_image_save (&outDB, VERBOSE);
    172   SetProtect (FALSE);
    173   dvo_image_unlock (&outDB); // unlock output
    174   dvo_image_unlock (&inDB); // unlock input1
    175 
     106// is this file a consistent FITS file?
     107int VerifyTableFile (char *filename) {
     108
     109  int status;
     110  off_t Nbytes;
     111  Header header;
     112
     113  struct stat fileStats;
     114  FILE *file;
     115
     116  // does the file exist?
     117  status = stat (filename, &fileStats);
     118  if (status) {
     119    // some error accessing the file.  there is only one acceptable error: file not found
     120    switch (errno) {
     121      case ENOENT:
     122        // fprintf (stderr, "file does not exist, skipping %s\n", filename);
     123        return TRUE;
     124      case ENOMEM:
     125        fprintf (stderr, "Out of memory: %s\n", filename);
     126        return TRUE;
     127      case EACCES:
     128        fprintf (stderr, "Permission error on %s\n", filename);
     129        return FALSE;
     130      case EFAULT:
     131        fprintf (stderr, "Bad address: %s\n", filename);
     132        return FALSE;
     133      case ELOOP:
     134        fprintf (stderr, "Too many symbolic links encountered while traversing the path: %s\n", filename);
     135        return FALSE;
     136      case ENAMETOOLONG:
     137        fprintf (stderr, "File name too long: %s\n", filename);
     138        return FALSE;
     139      case ENOTDIR:
     140        fprintf (stderr, "A component of the path is not a directory: %s\n", filename);
     141        return FALSE;
     142      case EOVERFLOW:
     143        fprintf (stderr, "file too large for program version: %s\n", filename);
     144        return FALSE;
     145      default:
     146        fprintf (stderr, "unknown error: %s\n", filename);
     147        return FALSE;
     148    }
     149  }
     150
     151  // does it have any data?
     152  if (fileStats.st_size == 0) {
     153    fprintf (stderr, "file is empty: %s\n", filename);
     154    return FALSE;
     155  }
     156
     157  // can we open it?
     158  file = fopen(filename, "r");
     159  if (!file) {
     160    fprintf (stderr, "unable to open valid file: %s\n", filename);
     161    return FALSE;
     162  }
     163
     164  // scan all extentions
     165  Nbytes = 0;
     166  while (Nbytes > fileStats.st_size) {
     167
     168    // Check on the PHU
     169    if (!gfits_fread_header (file, &header)) {
     170      fprintf (stderr, "unable to read PHU header for %s\n", filename);
     171      fclose(file);
     172      return (FALSE);
     173    }
     174
     175    // move to TBL header
     176    Nbytes += header.datasize + gfits_data_size (&header);
     177    if (Nbytes > fileStats.st_size) {
     178      fprintf (stderr, "file is short ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nbytes, fileStats.st_size);
     179      gfits_free_header(&header);
     180      fclose (file);
     181      return FALSE;
     182    }
     183    gfits_free_header(&header);
     184
     185    status = fseeko (file, Nbytes, SEEK_SET);
     186    if (status) {
     187      switch (errno) {
     188        case EBADF:
     189          fprintf (stderr, "something wrong with file handle: %s\n", filename);
     190          fclose (file);
     191          return FALSE;
     192        case EINVAL:
     193          fprintf (stderr, "invalid offset: %s\n", filename);
     194          fclose (file);
     195          return FALSE;
     196        default:
     197          fprintf (stderr, "other error in fseeko: %s\n", filename);
     198          fclose (file);
     199          return FALSE;
     200      }
     201    }
     202  }
     203  // fprintf (stderr, "file is good: %s\n", filename);
     204  fclose (file);
    176205  return TRUE;
    177206}
     207
     208  // gfits_scan(&cpmHeaderTBL, "NAXIS1", "%d", 1, &NbytesPerRow);
     209  // gfits_scan(&cpmHeaderTBL, "NAXIS2", "%d", 1, &Nrows);
     210   
     211 
Note: See TracChangeset for help on using the changeset viewer.