Changeset 35546 for branches/eam_branches/ipp-20130419/ippTools

Timestamp:

May 9, 2013, 11:26:59 AM (13 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20130419/ippTools

Files:

• share (modified) (1 prop)
• share/camtool_find_pendingimfile.sql (modified) (1 prop)
• share/chiptool_pendingcleanupimfile.sql (modified) (1 diff)
• share/chiptool_setimfiletoupdate.sql (modified) (1 prop)
• share/pxadmin_create_tables.sql (modified) (4 diffs, 1 prop)
• share/releasetool_definerelstack.sql (modified) (2 diffs)
• share/releasetool_definerelstack_with_skycal.sql (modified) (1 diff)
• share/releasetool_listrelstack.sql (modified) (1 diff)
• share/warptool_pendingcleanupskyfile.sql (modified) (1 diff)
• share/warptool_towarped.sql (modified) (1 prop)
• src (modified) (1 prop)
• src/camtool.c (modified) (1 prop)
• src/camtoolConfig.c (modified) (1 prop)
• src/magictool.c (modified) (1 prop)
• src/releasetool.c (modified) (11 diffs)
• src/releasetoolConfig.c (modified) (2 diffs)

branches/eam_branches/ipp-20130419/ippTools/share/chiptool_pendingcleanupimfile.sql

@@ -13 +13 @@
     USING(chip_id)
 WHERE
-    ((chipRun.state = 'goto_cleaned' AND (chipProcessedImfile.data_state = 'full'
-                                       OR chipProcessedImfile.data_state = 'update') AND chipProcessedImfile.quality = 0)
+    ((chipRun.state = 'goto_cleaned'
+        AND (chipProcessedImfile.data_state = 'full'
+          OR chipProcessedImfile.data_state = 'update'
+          OR chipProcessedImfile.data_state = 'error_cleaned')
+        AND chipProcessedImfile.quality = 0)
 OR 
     (chipRun.state = 'goto_scrubbed' AND chipProcessedImfile.data_state != 'scrubbed')

branches/eam_branches/ipp-20130419/ippTools/share/pxadmin_create_tables.sql

@@ -1 @@
-CREATE TABLE dbversion (
+eREATE TABLE dbversion (
     schema_version VARCHAR(64),
     updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
@@ -2172 +2172 @@
     priority    INT,
     PRIMARY KEY(rel_id),
+    UNIQUE KEY(surveyID, release_name),
     KEY(release_name),
     KEY(release_state),
@@ -2196 +2197 @@
     PRIMARY KEY (relexp_id),
     FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
+    UNIQUE KEY(rel_id, exp_id),
     KEY (state),
     KEY (fault),
@@ -2224 +2226 @@
     time_stamp  DATETIME,
     PRIMARY KEY (relstack_id),
+    UNIQUE KEY (rel_id, tess_id, skycell_id, filter, stack_type, mjd_obs),
     KEY (tess_id, skycell_id),
-    KEY (rel_id, tess_id, skycell_id, filter, mjd_obs),
     KEY (stack_type),
     KEY (mjd_obs),

branches/eam_branches/ipp-20130419/ippTools/share/releasetool_definerelstack.sql

@@ -6 +6 @@
     stackRun.tess_id,
     stackRun.filter,
-    0 AS mjd_obs,
+    stackSumSkyfile.mjd_obs,
     0 AS zpt_obs,
     0 AS zpt_stdev
@@ -12 +12 @@
 FROM ippRelease
 JOIN stackRun
-LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter)
-
+JOIN stackSumSkyfile using(stack_id)
+LEFT JOIN relStack AS previousRelStack 
+    ON previousRelStack.rel_id = ippRelease.rel_id
+    AND previousRelStack.tess_id = stackRun.tess_id 
+    AND previousRelStack.skycell_id = stackRun.skycell_id
+    AND previousRelStack.filter = stackRun.filter
+    -- JOIN hook %s
 WHERE previousRelStack.relstack_id IS NULL
+    AND stackRun.state ='full'
+    AND stackSumSkyfile.quality = 0
+    AND stackSumSkyfile.fault = 0

branches/eam_branches/ipp-20130419/ippTools/share/releasetool_definerelstack_with_skycal.sql

@@ -13 +13 @@
 JOIN stackRun
 JOIN stackSumSkyfile USING(stack_id)
-LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter)
 JOIN skycalRun ON skycalRun.stack_id = stackRun.stack_id
 JOIN skycalResult ON skycalRun.skycal_id = skycalResult.skycal_id
+-- LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter)
+LEFT JOIN relStack AS previousRelStack 
+    ON ippRelease.rel_id = previousRelStack.rel_id 
+    AND stackRun.tess_id = previousRelStack.tess_id 
+    AND stackRun.skycell_id = previousRelStack.skycell_id 
+    AND stackRun.filter = previousRelStack.filter
+-- JOIN Hook %s
 
 WHERE previousRelStack.relstack_id IS NULL
+    AND skycalRun.state = 'full'

branches/eam_branches/ipp-20130419/ippTools/share/releasetool_listrelstack.sql

@@ -7 +7 @@
     stackRun.state AS stack_state,
     stackRun.data_group AS stack_data_group,
+    skycalResult.fwhm_major,
     skycalResult.path_base AS skycal_path_base,
     skycalRun.data_group AS skycal_data_group,

branches/eam_branches/ipp-20130419/ippTools/share/warptool_pendingcleanupskyfile.sql

@@ -10 +10 @@
     USING(warp_id)
 WHERE
-   ((warpRun.state = 'goto_cleaned'  AND (warpSkyfile.data_state = 'full'
-                                      OR  warpSkyfile.data_state = 'update')
-    AND warpSkyfile.quality = 0)
+   ((warpRun.state = 'goto_cleaned'  
+       AND (warpSkyfile.data_state = 'full'
+          OR  warpSkyfile.data_state = 'error_cleaned'
+          OR  warpSkyfile.data_state = 'update')
+       AND warpSkyfile.quality = 0)
     OR
     (warpRun.state = 'goto_scrubbed' AND warpSkyfile.data_state != 'scrubbed')

branches/eam_branches/ipp-20130419/ippTools/src/releasetool.c

@@ -702 +702 @@
     PXOPT_COPY_STR(config->args, where, "-cam_data_group",  "camRun.data_group", "LIKE");
     PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-tess_id",      "warpRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id",   "warpSkyfile.skycell_id", "==");
+    
+    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false);
 
     PXOPT_COPY_STR(config->args, where, "-surveyName",  "survey.surveyName", "LIKE");
@@ -715 +719 @@
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
+    }
+
+    if (skycell_id) {
+        psStringAppend(&query, "\nJOIN warpSkyfile ON warpRun.warp_id = warpSkyfile.warp_id AND warpRun.tess_id = warpSkyfile.tess_id");
     }
 
@@ -741 +749 @@
 
     if (priority_order) {
-        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority");
+        psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority DESC");
     }
 
@@ -787 +795 @@
     bool includeSkycal = false;
 
+    PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE");
     PXOPT_COPY_S64(config->args, where, "-skycal_id",  "skycalRun.skycal_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "==");
     if (psListLength(where->list)) {
         includeSkycal = true;
@@ -795 +803 @@
 
     PXOPT_COPY_STR(config->args, where, "-label",      "stackRun.label", "==");
-    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE");
     PXOPT_COPY_S64(config->args, where, "-stack_id",   "stackRun.stack_id", "==");
 
@@ -848 +856 @@
     }
 
+    psString joinConditions = psStringCopy("");
+    // Finish up the selectors for the JOIN to previousRelStack
+    // join to rows with our stack_type
+    psStringAppend(&joinConditions, "\nAND previousRelStack.stack_type = '%s'", stack_type);
+
+    if (!strcmp(stack_type, "nightly")) {
+        // detect nightly stack entries that already exist for this day
+        psStringAppend(&joinConditions, "\nAND previousRelStack.mjd_obs = floor(stackSumSkyfile.mjd_obs)");
+    }
+
+    // Add in the where conditions
     psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-    psStringAppend(&query, " AND %s", whereClause);
+    psStringAppend(&query, "\nAND %s", whereClause);
     psFree(whereClause);
+
+    if (!strcmp(stack_type, "nightly")) {
+        // avoid stacks with NAN mjd_obs. These are old skycells that were lost due to system failure
+        // prior to the time that the mjd_obs was extracted from the headers and added to stackSumSkyfile
+        psStringAppend(&query, "\nAND stackSumSkyfile.mjd_obs IS NOT NULL");
+    }
 
     psFree(where);
@@ -860 +885 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
+    if (!p_psDBRunQueryF(config->dbh, query, joinConditions)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
+        psFree(joinConditions);
         return false;
     }
     psFree(query);
+    psFree(joinConditions);
 
     psArray *output = p_psDBFetchResult(config->dbh);
@@ -985 +1012 @@
     PXOPT_COPY_F32(config->args, where, "-mjd_min",    "stackSumSkyfile.mjd_obs", ">=");
     PXOPT_COPY_F32(config->args, where, "-mjd_max",    "stackSumSkyfile.mjd_obs", "<=");
-    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "skycalResult.fwhm_minor", ">=");
-    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalresult.fwhm_major", "<=");
     PXOPT_COPY_STR(config->args, where, "-tess_id",     "relStack.tess_id", "==");
     PXOPT_COPY_STR(config->args, where, "-skycell_id",  "relStack.skycell_id", "LIKE");
+//    PXOPT_COPY_STR(config->args, where, "-stack_type",  "relStack.stack_type", "==");
     PXOPT_COPY_STR(config->args, where, "-stack_data_group",  "stackRun.data_group", "LIKE");
     PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE");
@@ -996 +1022 @@
     pxskycellAddWhere(config, where);
 
+    PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "IFNULL(skycalResult.fwhm_major, 999)", ">=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "IFNULL(skycalResult.fwhm_major, 0)", "<=");
+
     PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    pxAddLabelSearchArgs (config, where, "-stack_type", "relStack.stack_type", "==");
 
     psString query = pxDataGet("releasetool_listrelstack.sql");
@@ -1018 +1049 @@
 
     if (priority_order) {
-        psStringAppend(&query, "\nAND priority > 0 order by stack_id, priority");
+        psStringAppend(&query, "\nAND priority > 0 order by stack_id, priority DESC");
     }
 
@@ -1487 +1518 @@
     PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<=");
     PXOPT_COPY_F32(config->args, where, "-fwhm_min",    "skycalResult.fwhm_major", ">=");
-    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalresult.fwhm_major", "<=");
+    PXOPT_COPY_F32(config->args, where, "-fwhm_max",    "skycalResult.fwhm_major", "<=");
     PXOPT_COPY_STR(config->args, where, "-exp_name",    "rawExp.exp_ame", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id",      "relExp.exp_id", "==");

branches/eam_branches/ipp-20130419/ippTools/src/releasetoolConfig.c

@@ -164 +164 @@
     psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-cam_data_group", 0, "camRun.data_group (LIKE comparison)", NULL);
     psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-warp_data_group", 0, "warpRun.data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-skycell_id", 0, "select by skycell", NULL);
+    psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-tess_id", 0,    "warpRun.tess_id", NULL);
 
     psMetadataAddStr(listrelexpArgs,  PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL);
@@ -220 +222 @@
     psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-fwhm_max", 0, "search by seeing (<=)", NAN);
 
+    psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-mjd_min", 0, "search by MJD seeing (>=)", NAN);
+    psMetadataAddF32(listrelstackArgs,  PS_LIST_TAIL, "-mjd_max", 0, "search by MJD (<=)", NAN);
+
     psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-tess_id", 0, "select by tess_id", NULL);
     psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-skycell_id", 0, "select by skycell_id (LIKE comparision)", NULL);
+    psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-stack_type", PS_META_DUPLICATE_OK, "select by stack_type", NULL);
 
     psMetadataAddStr(listrelstackArgs,  PS_LIST_TAIL, "-stack_data_group", 0, "select by stackRun.data_group (LIKE comparison)", NULL);