Index: branches/eam_branches/ipp-20130509/Ohana/src/addstar/include/addstar.h =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/addstar/include/addstar.h (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/addstar/include/addstar.h (revision 35594) @@ -119,4 +119,6 @@ int ACCEPT_TIME; // accept time stamp (or 0) int NO_STARS; // ignore the stars +int NO_DUPLICATE_IMAGES; // allow / skip duplicate images +int IMAGE_ID_OVERRIDE; // allow / skip duplicate images int TEXTMODE; // force input file to be loaded as RAW int SUBPIX; // apply a subpix correction Index: branches/eam_branches/ipp-20130509/Ohana/src/addstar/src/ImageIndex.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/addstar/src/ImageIndex.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/addstar/src/ImageIndex.c (revision 35594) @@ -159,5 +159,5 @@ if (!index) { fprintf (stderr, "image index file is not found, cannot check for image duplicates\n"); - exit (2); + exit (1); } Index: branches/eam_branches/ipp-20130509/Ohana/src/addstar/src/UpdateImageIDs.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/addstar/src/UpdateImageIDs.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/addstar/src/UpdateImageIDs.c (revision 35594) @@ -35,4 +35,9 @@ } + // TEST IMAGE_ID OVER-RIDE: + if (IMAGE_ID_OVERRIDE) { + imageID = IMAGE_ID_OVERRIDE; + } + // XXX should the first image ID be 1, not 0? if (imageID == 0) imageID = 1; @@ -54,5 +59,7 @@ // set and update the imageID sequence // the file holding the index is created above if this is an empty db - CheckDuplicateImageIDs (images, Nimages); + if (NO_DUPLICATE_IMAGES) { + CheckDuplicateImageIDs (images, Nimages); + } imageID += Nimages; Index: branches/eam_branches/ipp-20130509/Ohana/src/addstar/src/args.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/addstar/src/args.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/addstar/src/args.c (revision 35594) @@ -308,4 +308,17 @@ if ((N = get_argument (argc, argv, "-no-stars"))) { NO_STARS = TRUE; + remove_argument (N, &argc, argv); + } + /* skip the stars */ + NO_DUPLICATE_IMAGES = TRUE; + if ((N = get_argument (argc, argv, "-dup-images"))) { + NO_DUPLICATE_IMAGES = FALSE; + remove_argument (N, &argc, argv); + } + /* skip the stars */ + IMAGE_ID_OVERRIDE = 0; + if ((N = get_argument (argc, argv, "-image-id-override"))) { + remove_argument (N, &argc, argv); + IMAGE_ID_OVERRIDE = atoi(argv[N]); remove_argument (N, &argc, argv); } @@ -438,4 +451,5 @@ fprintf (stderr, " -use-name : use the given name instead of the filename as the filename\n"); fprintf (stderr, " -no-stars : skip the stars\n"); + fprintf (stderr, " -dup-images : skip the test for duplicate image IDs (test only!)\n"); fprintf (stderr, " -force : force read of database with inconsistent info\n"); fprintf (stderr, " -threads (N) : use N threads for resort option\n"); Index: branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/include/dvomerge.h =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/include/dvomerge.h (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/include/dvomerge.h (revision 35594) @@ -25,4 +25,6 @@ int VERBOSE; +int VERIFY; +int IMAGES_ONLY; char CATDIR[256]; char GSCFILE[256]; Index: branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/args.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/args.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/args.c (revision 35594) @@ -12,4 +12,17 @@ remove_argument (N, argc, argv); } + + /* verify merge status of output tables, but do not modify */ + VERIFY = FALSE; + if ((N = get_argument (*argc, argv, "-verify"))) { + VERIFY = TRUE; + remove_argument (N, argc, argv); + } + if ((N = get_argument (*argc, argv, "-check-only"))) { + VERIFY = TRUE; + remove_argument (N, argc, argv); + } + + /* use a different photcode file to define mean values */ if ((N = get_argument (*argc, argv, "-photcode-file"))) { remove_argument (N, argc, argv); @@ -113,4 +126,15 @@ } + /* verify merge status of output tables, but do not modify */ + VERIFY = FALSE; + if ((N = get_argument (*argc, argv, "-verify"))) { + VERIFY = TRUE; + remove_argument (N, argc, argv); + } + if ((N = get_argument (*argc, argv, "-check-only"))) { + VERIFY = TRUE; + remove_argument (N, argc, argv); + } + /* replace measurement, don't duplicate */ REPLACE_BY_PHOTCODE = FALSE; Index: branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomerge.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomerge.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomerge.c (revision 35594) @@ -10,6 +10,14 @@ if (argc == 6) { if (!strcasecmp (argv[2], "and")) { + if (VERIFY) { + fprintf (stderr, "WARNING / ERROR : dvomerge (input1) and (input2) into (output) : VERIFY mode not implemented\n"); + exit (3); + } dvomergeCreate (argc, argv); } else { + if (VERIFY) { + fprintf (stderr, "WARNING / ERROR : dvomerge (input) into (output) from (list) : VERIFY mode not implemented\n"); + exit (3); + } dvomergeFromList (argc, argv); } @@ -26,9 +34,13 @@ } -/* we have two possible modes of operation: +/* we have two major modes of operation: - Create : dvomerge (in1) and (in2) to (out) -- create a new db from two input dbs - Update : dvomerge (in) into (out) -- merge a new db into an existing db - Continue : dvomerge (in) into (out) continue -- merge a new db into an existing db + Create : dvomerge (in1) and (in2) to (out) -- create a new db from two input dbs + Update : dvomerge (in) into (out) -- merge a new db into an existing db + we also have varients on Update: + Continue : dvomerge (in) into (out) continue -- merge only unmerged tables into the existing db + From List : dvomerge (in) into (out) from (list) -- merge only specified tables into the existing db + + neither of the 2 above modes update the image table */ Index: branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeFromList.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeFromList.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeFromList.c (revision 35594) @@ -108,4 +108,6 @@ dmhObjectStats *inStats = dmhObjectStatsRead (inputfile); + // XXX : we are not checking for already-merged entries + LoadCatalog (&incatalog, NULL, inputfile, "r", NsecfiltInput); Index: branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeUpdate.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeUpdate.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeUpdate.c (revision 35594) @@ -23,4 +23,5 @@ CONTINUE = TRUE; } + if (VERIFY) CONTINUE = TRUE; input = argv[1]; @@ -78,4 +79,5 @@ } else { dvomergeImagesUpdate (&IDmap, input, output); + if (IMAGES_ONLY) exit (0); } Index: branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeUpdate_catalogs.c (revision 35594) @@ -76,8 +76,12 @@ } if (!missed) { - if (VERBOSE) fprintf (stderr, "skipping %s, empty or already merged\n", inlist[0].filename[i]); + if (VERBOSE || VERIFY) fprintf (stderr, "skipping %s, already merged\n", inlist[0].filename[i]); OutputStatusFree (outstat, outlist->Nregions); dmhObjectStatsFree (inStats); SkyListFree (outlist); + continue; + } + if (VERIFY) { + fprintf (stderr, "%s NOT merged\n", inlist[0].filename[i]); continue; } @@ -193,4 +197,5 @@ char tmpline[DVO_MAX_PATH]; if (VERBOSE) { snprintf (tmpline, DVO_MAX_PATH, "%s -v", command); strcpy (command, tmpline); } + if (VERIFY) { snprintf (tmpline, DVO_MAX_PATH, "%s -verify", command); strcpy (command, tmpline); } if (REPLACE_BY_PHOTCODE) { snprintf (tmpline, DVO_MAX_PATH, "%s -replace", command); strcpy (command, tmpline); } Index: branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/dvomergeUpdate_threaded.c (revision 35594) @@ -129,4 +129,7 @@ output = argv[3]; + fprintf (stderr, "WARNING / ERROR : multi-threaded dvomerge does not handle merge tracking yet\n"); + exit (2); + if (ALTERNATE_PHOTCODE_FILE) { fprintf (stderr, "cannot specify photcodes when merging into an existing catdir\n"); Index: branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/help.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/help.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/dvomerge/src/help.c (revision 35594) @@ -59,4 +59,6 @@ fprintf (stderr, " optional flags:\n"); fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -verify : verify merge status of output tables, but do not modify\n"); + fprintf (stderr, " -check-only : verify merge status of output tables, but do not modify [same as -verify]\n"); fprintf (stderr, " -help : this list\n"); fprintf (stderr, " -h : this list\n\n"); @@ -81,4 +83,6 @@ fprintf (stderr, " optional flags:\n"); fprintf (stderr, " -v : verbose mode\n"); + fprintf (stderr, " -verify : verify merge status of output tables, but do not modify\n"); + fprintf (stderr, " -check-only : verify merge status of output tables, but do not modify [same as -verify]\n"); fprintf (stderr, " -help : this list\n"); fprintf (stderr, " -h : this list\n\n"); Index: branches/eam_branches/ipp-20130509/Ohana/src/dvopsps/include/dvopsps.h =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/dvopsps/include/dvopsps.h (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/dvopsps/include/dvopsps.h (revision 35594) @@ -13,4 +13,5 @@ uint64_t ippObjID; uint64_t objID; + int photcode; unsigned int flags; float zp; Index: branches/eam_branches/ipp-20130509/Ohana/src/dvopsps/src/DetectionOps.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/dvopsps/src/DetectionOps.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/dvopsps/src/DetectionOps.c (revision 35594) @@ -54,4 +54,5 @@ GET_COLUMN(ippObjID , "ippObjID", int64_t); GET_COLUMN(objID , "objID", int64_t); + GET_COLUMN(photcode , "photcode", int); GET_COLUMN(flags , "flags", int); GET_COLUMN(zp , "zp", float); @@ -73,4 +74,5 @@ detections[i].ippObjID = ippObjID[i]; detections[i].objID = objID[i]; + detections[i].photcode = photcode[i]; detections[i].flags = flags[i]; detections[i].zp = zp[i]; @@ -90,4 +92,5 @@ free (ippObjID ); free (objID ); + free (photcode ); free (flags ); free (zp ); @@ -154,4 +157,5 @@ // gfits_define_bintable_column (&theader, "K", "ippObjID", NULL, NULL, 1.0, BZERO_INT64); // gfits_define_bintable_column (&theader, "K", "objID", NULL, NULL, 1.0, BZERO_INT64); + gfits_define_bintable_column (&theader, "J", "photcode", NULL, NULL, 1.0, 0.0); gfits_define_bintable_column (&theader, "J", "flags", NULL, NULL, 1.0, BZERO_INT32); gfits_define_bintable_column (&theader, "E", "zp", NULL, NULL, 1.0, 0.0); @@ -173,4 +177,5 @@ uint64_t *ippObjID ; ALLOCATE (ippObjID , uint64_t, Ndetections); uint64_t *objID ; ALLOCATE (objID , uint64_t, Ndetections); + int *photcode ; ALLOCATE (photcode , int, Ndetections); uint32_t *flags ; ALLOCATE (flags , uint32_t, Ndetections); float *zp ; ALLOCATE (zp , float, Ndetections); @@ -190,4 +195,5 @@ ippObjID[i] = detections[i].ippObjID ; objID[i] = detections[i].objID ; + photcode[i] = detections[i].photcode ; flags[i] = detections[i].flags ; zp[i] = detections[i].zp ; @@ -207,4 +213,5 @@ gfits_set_bintable_column (&theader, &ftable, "ippObjID", ippObjID , Ndetections); gfits_set_bintable_column (&theader, &ftable, "objID", objID , Ndetections); + gfits_set_bintable_column (&theader, &ftable, "photcode", photcode , Ndetections); gfits_set_bintable_column (&theader, &ftable, "flags", flags , Ndetections); gfits_set_bintable_column (&theader, &ftable, "zp", zp , Ndetections); @@ -222,4 +229,5 @@ free (ippObjID ); free (objID ); + free (photcode ); free (flags ); free (zp ); Index: branches/eam_branches/ipp-20130509/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/dvopsps/src/insert_detections_dvopsps_catalog.c (revision 35594) @@ -49,4 +49,5 @@ // detections[Ndetections]. ((uint64_t)average->catID << 32) + (uint64_t)average->objID; // ippObjID detections[Ndetections].objID = average->extID; // objID + detections[Ndetections].photcode = measure->photcode; // photcode detections[Ndetections].flags = measure->dbFlags; // flags detections[Ndetections].zp = code->C * 0.001 + code->K * (measure->airmass - 1) - measure->Mcal; // zp @@ -168,5 +169,5 @@ } - PrintIOBuffer (buffer, "INSERT INTO dvoDetectionFull (imageID, ippDetectID, detectID, ippObjID, objID, flags, zp, zpErr, airMass, expTime, ra, dec_, raErr, decErr) VALUES \n"); + PrintIOBuffer (buffer, "INSERT INTO dvoDetectionFull (imageID, ippDetectID, detectID, ippObjID, objID, photcode, flags, zp, zpErr, airMass, expTime, ra, dec_, raErr, decErr) VALUES \n"); return TRUE; @@ -179,25 +180,5 @@ int insert_detections_mysql_value (IOBuffer *buffer, Average *average, Measure *measure) { - // XXX I am changing the def of ippObjID if I use the code below PhotCode *code = GetPhotcodebyCode(measure->photcode); - // PrintIOBuffer (buffer, " (%d, %u, %lu, %lu, %lu, %u, %f, %f, %f, %f, %lf, %lf, %f, %f),\n", - // measure->imageID, // imageID - // measure->detID, // ippDetectID - // measure->extID, // detectID - // ((uint64_t)average->catID * 1000000000) + (uint64_t)average->objID, // ippObjID - // // NOTE: this is better, but the above is the current ippToPsps value - // // ((uint64_t)average->catID << 32) + (uint64_t)average->objID, // ippObjID - // average->extID, // objID - // measure->dbFlags, // flags - // code->C * 0.001 + code->K * (measure->airmass - 1) - measure->Mcal, // zp - // measure->dMcal, // zpErr - // measure->airmass, - // pow(10.0, 0.4 * measure->dt), // expTime - // average->R - measure->dR / 3600., // ra - // average->D - measure->dR / 3600., // dec - // measure->dXccd * 0.01 * fabs(measure->pltscale), // estimate of raErr - // measure->dYccd * 0.01 * fabs(measure->pltscale) // estimate of decErr - // ); - PrintIOBuffer (buffer, "(%d, ", measure->imageID); // imageID PrintIOBuffer (buffer, "%u, ", measure->detID); // ippDetectID @@ -207,4 +188,5 @@ // ((uint64_t)average->catID << 32) + (uint64_t)average->objID, // ippObjID PrintIOBuffer (buffer, "%lu, ", average->extID); // objID + PrintIOBuffer (buffer, "%d, ", measure->photcode); // photcode PrintIOBuffer (buffer, "%u, ", measure->dbFlags); // flags Index: branches/eam_branches/ipp-20130509/Ohana/src/libfits/include/gfitsio.h =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/libfits/include/gfitsio.h (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/libfits/include/gfitsio.h (revision 35594) @@ -26,4 +26,7 @@ # define FT_LINE_LENGTH 80 /* FITS header line length */ # define FT_RECORD_SIZE 2880 /* FITS block size */ + +# define FT_BZERO_INT16 1.0*0x8000 +# define FT_BZERO_INT32 1.0*0x80000000 /* this structure defines the buffer which contains a header Index: branches/eam_branches/ipp-20130509/Ohana/src/libfits/table/F_define_column.c =================================================================== --- branches/eam_branches/ipp-20130509/Ohana/src/libfits/table/F_define_column.c (revision 35593) +++ branches/eam_branches/ipp-20130509/Ohana/src/libfits/table/F_define_column.c (revision 35594) @@ -1,4 +1,48 @@ # include # include + +# if (0) +/*** special version of this function to define unsigned int columns (special values for BZERO, BSCALE) ********************/ +int gfits_define_bintable_column_unsigned_int (Header *header, char *format, char *label, char *comment, char *unit, int inttype) { + + assert (label); + assert (format); + + off_t Naxis1; + int Nfields, Nbytes, Nval; + char type[16], field[16]; + + if (!gfits_bintable_format (format, type, &Nval, &Nbytes)) return (FALSE); + + Nfields = 0; + gfits_scan (header, "TFIELDS", "%d", 1, &Nfields); + gfits_scan (header, "NAXIS1", OFF_T_FMT, 1, &Naxis1); + Nfields ++; + Naxis1 += Nbytes*Nval; + + sprintf (field, "TTYPE%d", Nfields); + gfits_modify (header, field, "%s", 1, label); + gfits_modify_alt (header, field, "%C", 1, comment); + sprintf (field, "TUNIT%d", Nfields); + gfits_modify (header, field, "%s", 1, unit); + sprintf (field, "TFORM%d", Nfields); + gfits_modify (header, field, "%s", 1, format); + + // add scaling parameters unless they amount to a noop + if ((bscale != 1.0) || (bzero != 0.0)) { + sprintf (field, "TSCAL%d", Nfields); + gfits_modify (header, field, "%lf", 1, bscale); + sprintf (field, "TZERO%d", Nfields); + gfits_modify (header, field, "%lf", 1, bzero); + } + + /* update TFIELDS & NAXIS1 */ + gfits_modify (header, "TFIELDS", "%d", 1, Nfields); + gfits_modify (header, "NAXIS1", OFF_T_FMT, 1, Naxis1); + header[0].Naxis[0] = Naxis1; + + return (TRUE); +} +# endif /***********************/