Changeset 24512 for trunk

Timestamp:

Jun 19, 2009, 12:38:16 PM (17 years ago)

Author:

Paul Price

Message:

Done with development of mechanism to automatically publish detections: merging branches/pap_mops (r24510) into trunk

Location:

trunk

Files:

• . (modified) (1 prop)
• Nebulous (modified) (1 prop)
• Nebulous-Server (modified) (1 prop)
• Nebulous/bin (modified) (1 prop)
• Nebulous/examples (modified) (1 prop)
• Nebulous/lib (modified) (1 prop)
• Nebulous/nebclient (modified) (1 prop)
• Nebulous/nebclient/tests/tap (modified) (1 prop)
• Nebulous/t (modified) (1 prop)
• Ohana (modified) (1 prop)
• Ohana/src/relastro (modified) (1 prop)
• dbconfig/changes.txt (modified) (1 diff)
• dbconfig/ipp.m4 (modified) (1 diff)
• dbconfig/publish.md (copied) (copied from branches/pap_mops/dbconfig/publish.md )
• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/scripts/publish_file.pl (copied) (copied from branches/pap_mops/ippScripts/scripts/publish_file.pl )
• ippTasks/Makefile.am (modified) (1 diff)
• ippTasks/publish.pro (copied) (copied from branches/pap_mops/ippTasks/publish.pro )
• ippTools/doc/publish_flow.txt (copied) (copied from branches/pap_mops/ippTools/doc/publish_flow.txt )
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/camtool_find_processedexp.sql (modified) (1 diff)
• ippTools/share/chiptool_export_imfile.sql (modified) (1 prop)
• ippTools/share/chiptool_export_processed_imfile.sql (modified) (1 prop)
• ippTools/share/difftool_definewarpstack.sql (modified) (1 prop)
• ippTools/share/difftool_definewarpstack_part1.sql (modified) (1 prop)
• ippTools/share/difftool_definewarpstack_part2.sql (modified) (1 prop)
• ippTools/share/difftool_definewarpstack_temp_create.sql (modified) (1 prop)
• ippTools/share/difftool_skyfile.sql (modified) (2 diffs)
• ippTools/share/pubtool_definerun.sql (copied) (copied from branches/pap_mops/ippTools/share/pubtool_definerun.sql )
• ippTools/share/pubtool_pending.sql (copied) (copied from branches/pap_mops/ippTools/share/pubtool_pending.sql )
• ippTools/share/pubtool_revert.sql (copied) (copied from branches/pap_mops/ippTools/share/pubtool_revert.sql )
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff)
• ippTools/share/warptool_revertoverlap.sql (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/Makefile.am (modified) (3 diffs)
• ippTools/src/pubtool.c (copied) (copied from branches/pap_mops/ippTools/src/pubtool.c )
• ippTools/src/pubtool.h (copied) (copied from branches/pap_mops/ippTools/src/pubtool.h )
• ippTools/src/pubtoolConfig.c (copied) (copied from branches/pap_mops/ippTools/src/pubtoolConfig.c )
• ppImage (modified) (1 prop)
• ppStack (modified) (1 prop)
• ppSub (modified) (1 prop)
• psLib/src/fits/psFitsTable.c (modified) (6 diffs)
• psLib/src/fits/psFitsTable.h (modified) (2 diffs)
• psModules (modified) (1 prop)
• psastro (modified) (1 prop)
• psastro/src/psastroFindChip.c (modified) (1 prop)
• psconfig (modified) (1 prop)
• psconfig/tagsets/ipp-2.8.dist (modified) (1 diff)
• psphot (modified) (1 prop)

trunk/dbconfig/changes.txt

@@ -1128 +1128 @@
 
 ALTER TABLE warpRun ADD COLUMN reduction VARCHAR(64) AFTER tess_id;
+
+-- Tables to support publishing of detections to a Science Client
+-- Clients to which we send stuff
+CREATE TABLE publishClient (
+    client_id BIGINT AUTO_INCREMENT, -- unique identifier
+    product VARCHAR(64),             -- product name
+    stage VARCHAR(64) NOT NULL, -- stage of interest (chip, camera, diff, etc.)
+    workdir VARCHAR(255) NOT NULL, -- working directory
+    comment VARCHAR(255),            -- for human memory
+    PRIMARY KEY(client_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+-- Publishing a set of data (e.g., a specific diffRun)
+CREATE TABLE publishRun (
+    pub_id BIGINT AUTO_INCREMENT, -- unique identifier
+    client_id BIGINT NOT NULL,  -- link to publishClient
+    stage_id BIGINT NOT NULL,   -- link to various stage tables
+    label VARCHAR(64),          -- label for run
+    state VARCHAR(64),          -- state of run (new, full, etc.)
+    PRIMARY KEY(pub_id),
+    KEY(client_id),
+    KEY(stage_id),
+    KEY(label),
+    KEY(state),
+    FOREIGN KEY(client_id) REFERENCES publishClient(client_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+-- Publishing a file within a set
+CREATE TABLE publishDone (
+    pub_id BIGINT AUTO_INCREMENT, -- link to publishRun
+    path_base VARCHAR(255),     -- base path of output
+    fault SMALLINT NOT NULL DEFAULT 0, -- Fault code
+    PRIMARY KEY(pub_id),
+    KEY(fault),
+    FOREIGN KEY(pub_id) REFERENCES publishRun(pub_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;

trunk/dbconfig/ipp.m4

@@ -26 +26 @@
 include(rc.md)
 include(receive.md)
+include(publish.md)

trunk/ippScripts/Build.PL

@@ -91 +91 @@
         scripts/receive_setstatus.pl
         scripts/rcserver_checkstatus.pl
+        scripts/publish_file.pl
     )],
     dist_abstract => 'Scripts for running the Pan-STARRS IPP',

trunk/ippTasks/Makefile.am

@@ -23 +23 @@
         rcserver.pro \
         pstamp.pro \
-        receive.pro
+        receive.pro \
+        publish.pro
 
 other_files = \

trunk/ippTools/share/Makefile.am

@@ -168 +168 @@
      pxadmin_create_mirror_tables.sql \
      pxadmin_drop_tables.sql \
+     pubtool_definerun.sql \
+     pubtool_pending.sql \
+     pubtool_revert.sql \
      pztool_find_completed_exp.sql \
      pztool_pendingimfile.sql \

trunk/ippTools/share/camtool_find_processedexp.sql

@@ -5 +5 @@
     rawExp.exp_tag,
     rawExp.exp_name,
+    rawExp.exp_time,
     rawExp.camera,
     rawExp.telescope,

trunk/ippTools/share/difftool_skyfile.sql

@@ -5 +5 @@
     diffRun.state,
     diffRun.workdir,
+    diffRun.bothways,
+    camProcessedExp.zpt_obs,
+    camProcessedExp.zpt_stdev,
+    camProcessedExp.zpt_lq,
+    camProcessedExp.zpt_uq,
+    rawExp.exp_time,
+    rawExp.camera,
     warp1,
     stack1,
@@ -13 +20 @@
 JOIN diffInputSkyfile USING(diff_id, skycell_id)
 JOIN warpRun
+-- NOTE: joining input only!
+-- This is so that we can get the correct zero point
+-- XXX This needs to be more clever to handle diffs between stacks
     ON warpRun.warp_id = diffInputSkyfile.warp1
 JOIN fakeRun USING(fake_id)
 JOIN camRun USING(cam_id)
+JOIN camProcessedExp USING(cam_id)
 JOIN chipRun USING(chip_id)
 JOIN rawExp USING(exp_id)

trunk/ippTools/share/pxadmin_create_tables.sql

@@ -1369 +1369 @@
 
 
+
+-- Tables to support publishing of detections to a Science Client
+
+-- Clients to which we send stuff
+CREATE TABLE publishClient (
+    client_id BIGINT AUTO_INCREMENT, -- unique identifier
+    product VARCHAR(64),             -- product name
+    stage VARCHAR(64) NOT NULL, -- stage of interest (chip, camera, diff, etc.)
+    workdir VARCHAR(255) NOT NULL, -- working directory
+    comment VARCHAR(255),            -- for human memory
+    PRIMARY KEY(client_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Publishing a set of data (e.g., a specific diffRun)
+CREATE TABLE publishRun (
+    pub_id BIGINT AUTO_INCREMENT, -- unique identifier
+    client_id BIGINT NOT NULL,  -- link to publishClient
+    stage_id BIGINT NOT NULL,   -- link to various stage tables
+    label VARCHAR(64),          -- label for run
+    state VARCHAR(64),          -- state of run (new, full, etc.)
+    PRIMARY KEY(pub_id),
+    KEY(client_id),
+    KEY(stage_id),
+    KEY(label),
+    KEY(state),
+    FOREIGN KEY(client_id) REFERENCES publishClient(client_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Publishing a file within a set
+CREATE TABLE publishDone (
+    pub_id BIGINT AUTO_INCREMENT, -- link to publishRun
+    path_base VARCHAR(255),     -- base path of output
+    fault SMALLINT NOT NULL DEFAULT 0, -- Fault code
+    PRIMARY KEY(pub_id),
+    KEY(fault),
+    FOREIGN KEY(pub_id) REFERENCES publishRun(pub_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+
 -- This comment line is here to avoid empty query error.
 -- Another way to avoid that problem is to omit the semicolon above but I think that is untidy.

trunk/ippTools/src

@@ -34 +34 @@
 disttool
 receivetool
+
+pubtool

@@ -34 +34 @@
 disttool
 receivetool
+
+pubtool

trunk/ippTools/src/Makefile.am

@@ -20 +20 @@
         stacktool \
         warptool \
-        receivetool
+        receivetool \
+        pubtool
 
 
@@ -59 +60 @@
         regtool.h \
         stacktool.h \
-        warptool.h
+        warptool.h \
+        pubtool.h
 
 lib_LTLIBRARIES = libpxtools.la
@@ -220 +222 @@
     receivetoolConfig.c
 
+pubtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+pubtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+pubtool_SOURCES = \
+    pubtool.c \
+    pubtoolConfig.c
+
 clean-local:
         -rm -f TAGS

trunk/psLib/src/fits/psFitsTable.c

@@ -390 +390 @@
 
 
-bool psFitsInsertTable(psFits* fits,
-                       const psMetadata* header,
-                       const psArray* table,
-                       const char *extname,
-                       bool after)
+static bool fitsInsertTable(psFits* fits,             // FITS file
+                            const psMetadata* header, // FITS header to write
+                            const psArray* table,     // Table to write
+                            const char *extname,      // Extension name to give table
+                            bool after,               // Write table after current extension?
+                            bool writeData            // Write data?
+                            )
 {
     PS_ASSERT_FITS_NON_NULL(fits, false);
@@ -403 +405 @@
 
     long numRows = table->n;
-    if (numRows < 1) {
+    if (writeData && numRows < 1) {
         // no table data, what can I do?
         psError(PS_ERR_BAD_PARAMETER_SIZE, true,
@@ -504 +506 @@
         fits_create_tbl(fits->fd,
                         BINARY_TBL,
-                        table->n, // number of rows in table
+                        writeData ? numRows : 0, // number of rows in table
                         numColumns, // number of columns in table
                         (char**)columnNames->data, // names of the columns
@@ -524 +526 @@
         // Insert the table
         fits_insert_btbl(fits->fd,
-                         table->n, // number of rows in table
+                         writeData ? numRows : 0, // number of rows in table
                          numColumns, // number of columns in table
                          (char**)columnNames->data, // names of the columns
@@ -562 +564 @@
 
     // cfitsio requires that we write the data by columns --- urgh!
-    psMetadataIteratorSet(colSpecsIter, PS_LIST_HEAD);
-    for (long colNum = 1; (colSpecItem = psMetadataGetAndIncrement(colSpecsIter)); colNum++) {
-        // Note: colNum is unit-indexed, because it's for cfitsio
-        colSpec *spec = colSpecItem->data.V; // The specification
-        if (PS_DATA_IS_PRIMITIVE(spec->type)) {
-            size_t dataSize = PSELEMTYPE_SIZEOF(spec->type); // Size (in bytes) of this type
-            psVector *columnData = psVectorAlloc(table->n, spec->type); // The raw row data, to be written
-            psVectorInit(columnData, 0);
-            for (long i = 0; i < table->n; i++) {
-                psMetadata *row = table->data[i]; // The row of interest
-                psMetadataItem *dataItem = psMetadataLookup(row, colSpecItem->name); // The value of interest
-                memcpy(&columnData->data.U8[i * dataSize], &dataItem->data, dataSize);
+    if (writeData) {
+        psMetadataIteratorSet(colSpecsIter, PS_LIST_HEAD);
+        for (long colNum = 1; (colSpecItem = psMetadataGetAndIncrement(colSpecsIter)); colNum++) {
+            // Note: colNum is unit-indexed, because it's for cfitsio
+            colSpec *spec = colSpecItem->data.V; // The specification
+            if (PS_DATA_IS_PRIMITIVE(spec->type)) {
+                size_t dataSize = PSELEMTYPE_SIZEOF(spec->type); // Size (in bytes) of this type
+                psVector *columnData = psVectorAlloc(table->n, spec->type); // The raw row data, to be written
+                psVectorInit(columnData, 0);
+                for (long i = 0; i < table->n; i++) {
+                    psMetadata *row = table->data[i]; // The row of interest
+                    psMetadataItem *dataItem = psMetadataLookup(row, colSpecItem->name); // Value of interest
+                    memcpy(&columnData->data.U8[i * dataSize], &dataItem->data, dataSize);
+                }
+
+                int fitsDataType;           // Data type for cfitsio
+                p_psFitsTypeToCfitsio(spec->type, NULL, NULL, &fitsDataType);
+                fits_write_col(fits->fd,
+                               fitsDataType,
+                               colNum, // column number
+                               1, // first row
+                               1, // first element
+                               table->n, // number of rows
+                               columnData->data.U8, // the data
+                               &status);
+                psFree(columnData);
+            } else {
+                switch (spec->type) {
+                  case PS_DATA_STRING: {
+                      psArray *strings = psArrayAlloc(table->n); // Array of strings
+                      for (long i = 0; i < table->n; i++) {
+                          psMetadata *row = table->data[i]; // The row of interest
+                          strings->data[i] = psMemIncrRefCounter(psMetadataLookupStr(NULL, row,
+                                                                                     colSpecItem->name));
+                      }
+                      fits_write_col_str(fits->fd, colNum, 1, 1, table->n, (char**)strings->data, &status);
+                      psFree(strings);
+                      break;
+                  }
+                  case PS_DATA_VECTOR: {
+                      size_t dataSize = PSELEMTYPE_SIZEOF(spec->vectorType); // Size of data, in bytes
+                      psVector *columnData = psVectorAlloc(spec->size * table->n * dataSize, PS_TYPE_U8);
+                      psVectorInit(columnData, 0);
+                      for (long i = 0; i < table->n; i++) {
+                          psMetadata *row = table->data[i]; // The row of interest
+                          psMetadataItem* dataItem = psMetadataLookup(row, colSpecItem->name);
+                          if (dataItem->type != PS_DATA_VECTOR) {
+                              // Just in case --- get a zero instead of some weird result
+                              continue;
+                          }
+                          psVector *vector = dataItem->data.V;
+                          memcpy(&columnData->data.U8[i * dataSize * spec->size], vector->data.U8,
+                                 vector->n * dataSize);
+                      }
+
+                      int fitsDataType;           // Data type for cfitsio
+                      p_psFitsTypeToCfitsio(spec->vectorType, NULL, NULL, &fitsDataType);
+                      fits_write_col(fits->fd, fitsDataType, colNum, 1, 1, table->n * spec->size,
+                                     columnData->data.U8, &status);
+                      psFree(columnData);
+                      break;
+                  }
+                  default:
+                    psAbort("Should never get here.\n");
+                }
             }
 
-            int fitsDataType;           // Data type for cfitsio
-            p_psFitsTypeToCfitsio(spec->type, NULL, NULL, &fitsDataType);
-            fits_write_col(fits->fd,
-                           fitsDataType,
-                           colNum, // column number
-                           1, // first row
-                           1, // first element
-                           table->n, // number of rows
-                           columnData->data.U8, // the data
-                           &status);
-            psFree(columnData);
-        } else {
-            switch (spec->type) {
-            case PS_DATA_STRING: {
-                    psArray *strings = psArrayAlloc(table->n); // Array of strings
-                    for (long i = 0; i < table->n; i++) {
-                        psMetadata *row = table->data[i]; // The row of interest
-                        strings->data[i] = psMemIncrRefCounter(psMetadataLookupStr(NULL, row,
-                                                               colSpecItem->name));
-                    }
-                    fits_write_col_str(fits->fd, colNum, 1, 1, table->n, (char**)strings->data, &status);
-                    psFree(strings);
-                    break;
-                }
-            case PS_DATA_VECTOR: {
-                    size_t dataSize = PSELEMTYPE_SIZEOF(spec->vectorType); // Size of data, in bytes
-                    psVector *columnData = psVectorAlloc(spec->size * table->n * dataSize, PS_TYPE_U8);
-                    psVectorInit(columnData, 0);
-                    for (long i = 0; i < table->n; i++) {
-                        psMetadata *row = table->data[i]; // The row of interest
-                        psMetadataItem* dataItem = psMetadataLookup(row, colSpecItem->name);
-                        if (dataItem->type != PS_DATA_VECTOR) {
-                            // Just in case --- get a zero instead of some weird result
-                            continue;
-                        }
-                        psVector *vector = dataItem->data.V;
-                        memcpy(&columnData->data.U8[i * dataSize * spec->size], vector->data.U8,
-                               vector->n * dataSize);
-                    }
-
-                    int fitsDataType;           // Data type for cfitsio
-                    p_psFitsTypeToCfitsio(spec->vectorType, NULL, NULL, &fitsDataType);
-                    fits_write_col(fits->fd, fitsDataType, colNum, 1, 1, table->n * spec->size,
-                                   columnData->data.U8, &status);
-                    psFree(columnData);
-                    break;
-                }
-            default:
-                psAbort("Should never get here.\n");
+            // Check error status from writing column
+            if (status != 0) {
+                psFitsError(status, true, "Unable to write column %ld of FITS table", colNum);
+                psFree(colSpecsIter);
+                psFree(colSpecs);
+                return false;
             }
-        }
-
-        // Check error status from writing column
-        if (status != 0) {
-            psFitsError(status, true, "Unable to write column %ld of FITS table", colNum);
-            psFree(colSpecsIter);
-            psFree(colSpecs);
-            return false;
         }
     }
@@ -650 +654 @@
     return true;
 }
+
+
+bool psFitsInsertTable(psFits* fits, const psMetadata* header, const psArray* table, const char *extname,
+                       bool after)
+{
+    PS_ASSERT_FITS_NON_NULL(fits, false);
+    PS_ASSERT_FITS_WRITABLE(fits, false);
+    return fitsInsertTable(fits, header, table, extname, after, true);
+}
+
+bool psFitsWriteTableEmpty(psFits *fits, const psMetadata *header, const psMetadata *columns,
+                           const char *extname)
+{
+    PS_ASSERT_FITS_NON_NULL(fits, false);
+    PS_ASSERT_FITS_WRITABLE(fits, false);
+    if (!psFitsMoveLast(fits)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to move to last extension to write table");
+        return false;
+    }
+    psArray *table = psArrayAlloc(1);   // Dummy table carrying column definitions
+    table->data[0] = psMemIncrRefCounter((psPtr)columns); // Casting away const
+    bool status = fitsInsertTable(fits, header, table, extname, true, false); // Status of insertion
+    psFree(table);
+    return status;
+}
+
+bool psFitsInsertTableEmpty(psFits *fits, const psMetadata *header, const psMetadata *columns,
+                            const char *extname, bool after)
+{
+    PS_ASSERT_FITS_NON_NULL(fits, false);
+    PS_ASSERT_FITS_WRITABLE(fits, false);
+    psArray *table = psArrayAlloc(1);   // Dummy table carrying column definitions
+    table->data[0] = psMemIncrRefCounter((psPtr)columns); // Casting away const
+    bool status = fitsInsertTable(fits, header, table, extname, after, false); // Status of insertion
+    psFree(table);
+    return status;
+}
+
 
 bool psFitsUpdateTable(psFits* fits,

trunk/psLib/src/fits/psFitsTable.h

@@ -90 +90 @@
 );
 
+/// Write an empty table
+bool psFitsWriteTableEmpty(
+    psFits *fits,                       ///< FITS file pointer
+    const psMetadata *header,           ///< Header to write
+    const psMetadata *columns,          ///< Column definitions; no data used except name,type
+    const char *extname                 ///< Extension name for table
+    );
+
 /** Inserts a whole FITS table. A new HDU of the type BINTABLE is inserted either
  *  before or after, depending on the AFTER parameter, the current HDU.
@@ -104 +112 @@
     bool after    ///< TRUE if insert is done after CHDU, otherwise table is inserted before CHDU
 );
+
+/// Insert an empty table
+bool psFitsInsertTableEmpty(
+    psFits *fits,              ///< FITS file pointer
+    const psMetadata *header,  ///< Header to write
+    const psMetadata *columns, ///< Column definitions; no data used except name,type
+    const char *extname,       ///< Extension name for table
+    bool after                 ///< Insert after current HDU?
+    );
+
 
 /** Updates a FITS table.  The current HDU type must be either

trunk/psconfig/tagsets/ipp-2.8.dist

@@ -69 +69 @@
   YYYYY  DataStore              ipp-2-8          -0
 
+  YYYYY  ppMops                 ipp-2-8          -0
+
   YNNNN  extsrc/gpcsw           ipp-2-8          -0
 

@@ -34 +34 @@
 disttool
 receivetool
+
+pubtool