Changeset 8406

Timestamp:

Aug 16, 2006, 5:31:39 PM (20 years ago)

Author:

jhoblitt

Message:

stub out -toresidexp
reimplement -addresidimfile

Location:

trunk/ippTools/src

Files:

• dettool.c (modified) (5 diffs)
• dettoolConfig.c (modified) (5 diffs)

trunk/ippTools/src/dettool.c

@@ -43 +43 @@
 static detNormalizedStatImfileRow *detStackedToDetNormalizedStatImfile(pxConfig *config, detStackedImfileRow *stackedImfile);
 static  detNormalizedImfileRow *detNormalizedStatToDetNormalizedmfile(pxConfig *config, detNormalizedStatImfileRow *statImfile);
+static  detResidImfileRow *detNormalizedToDetResidImfile(pxConfig *config, detNormalizedImfileRow *normalizedImfile);
 static psArray *validDetInputClassIds(pxConfig *config, const char *det_id);
 static psArray *searchInputImfiles(pxConfig *config, const char *det_id);
@@ -2357 +2358 @@
 }
 
-static bool toresidexpMode(pxConfig *config)
-{
-    return true;
-}
 
 static bool normalizedframeMode(pxConfig *config)
@@ -2449 +2446 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    // det_id, exp_id, class_id, & recipe are required
+    // make sure that there is a coresponding entry in detNormalizedImfile
+    // and the exp_id specified is valid
+    // select * from detNormalizedImfile
+    // by det_id, iteration, class_id
+    // where det_id, iteration, class_id is not in detResidImfile
+    psString query = psStringCopy(
+        "SELECT"
+        "   detNormalizedImfile.*"
+        " FROM detNormalizedImfile"
+        " LEFT JOIN detInputExp"
+        "   USING(det_id, iteration)"
+        " LEFT JOIN detResidImfile"
+        "   ON detNormalizedImfile.det_id = detResidImfile.det_id"
+        "   AND detNormalizedImfile.iteration= detResidImfile.iteration"
+        "   AND detInputExp.exp_id = detResidImfile.exp_id"
+        "   AND detNormalizedImfile.class_id = detResidImfile.class_id"
+        " WHERE"
+        "  detInputExp.exp_id = '%s'" 
+        "  AND detResidImfile.det_id IS NULL" 
+        "  AND detResidImfile.iteration IS NULL" 
+        "  AND detResidImfile.exp_id IS NULL" 
+        "  AND detResidImfile.class_id IS NULL" 
+        );
+
+    {
+        // build a query to search by det_id, iteration, class_id
+        psMetadata *where = psMetadataAlloc();
+        bool status = false;
+        psString det_id = psMetadataLookupStr(&status, config->args, "-det_id");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id");
+            psFree(where);
+            psFree(query);
+            return false;
+        }
+        if (det_id) {
+            if (!psMetadataAddStr(where, PS_LIST_TAIL, "det_id", 0, "==", det_id)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to add item det_id");
+                psFree(where);
+                psFree(query);
+                return false;
+            }
+        }
+        psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration");
+            psFree(where);
+            psFree(query);
+            return false;
+        }
+        // always set iteration
+        if (!psMetadataAddS32(where, PS_LIST_TAIL, "iteration", 0, "==", iteration)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to add item iteration");
+            psFree(where);
+            psFree(query);
+            return false;
+        }
+        psString class_id = psMetadataLookupStr(&status, config->args, "-class_id");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id");
+            psFree(where);
+            psFree(query);
+            return false;
+        }
+        if (class_id) {
+            if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to add item class_id");
+                psFree(where);
+                psFree(query);
+                return false;
+            }
+        }
+
+        // there's not
+        psString whereClaus = psDBGenerateWhereConditionSQL(where, "detNormalizedImfile");
+        psFree(where);
+        if (whereClaus) {
+            psStringAppend(&query, " AND %s", whereClaus);
+            psFree(whereClaus);
+        }
+    }
+
+    // exp_id is manadatory to cross check that this is a exp_id for this
+    // detRUn
     bool status = false;
-    psString det_id = psMetadataLookupStr(&status, config->args, "-det_id");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -det_id");
-        return false;
-    }
-    if (!det_id) {
-        psError(PS_ERR_UNKNOWN, true, "-det_id is required");
-        return false;
-    }
     psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id");
     if (!status) {
         psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id");
+        psFree(query);
         return false;
     }
     if (!exp_id) {
         psError(PS_ERR_UNKNOWN, true, "-exp_id is required");
-        return false;
-    }
-    psString class_id = psMetadataLookupStr(&status, config->args, "-class_id");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id");
-        return false;
-    }
-    if (!class_id) {
-        psError(PS_ERR_UNKNOWN, true, "-class_id is required");
+        psFree(query);
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query, exp_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    } 
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        // XXX check psError here
+        psError(PS_ERR_UNKNOWN, false, "no pending detNormalizedImfile rows found");
+        return false;
+    }
+
+    // start a transaction so it's all rows or nothing
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(output);
+        return false;
+    }
+
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+        // convert metadata into a detNormalizedImfile object
+        detNormalizedImfileRow *normalizedImfile = detNormalizedImfileObjectFromMetadata(row);
+        // convert detNormalizedImfile object into a detResidImfile
+        detResidImfileRow *residImfile  = detNormalizedToDetResidImfile(config, normalizedImfile);
+        psFree(normalizedImfile);
+        if (!residImfile) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "failed to convert detNormalizedImfile to detResidImfile");
+            psFree(output);
+            return false;
+        }
+        // insert detResidImfile object into the database
+        if (!detResidImfileInsertObject(config->dbh, residImfile)) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(residImfile);
+            psFree(output);
+        }
+        psFree(residImfile);
+    }
+
+    psFree(output);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static  detResidImfileRow *detNormalizedToDetResidImfile(pxConfig *config, detNormalizedImfileRow *normalizedImfile)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+    PS_ASSERT_PTR_NON_NULL(normalizedImfile, NULL);
+
+    bool status = false;
+    psString exp_id = psMetadataLookupStr(&status, config->args, "-exp_id");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_id");
+        return false;
+    }
+    if (!exp_id) {
+        psError(PS_ERR_UNKNOWN, true, "-exp_id is required");
         return false;
     }
@@ -2508 +2646 @@
     }
 
-    // iteration has a default value
-    psS32 iteration = psMetadataLookupS32(&status, config->args, "-iteration");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -iteration");
-        return false;
-    }
-
-    // check det_id, exp_id, & class_id against the input rawImfiles
-    
-    // XXX what we really want here is to do a join but psDB doesn't support
-    // that type of operation 
-    // we have to generate our own where claus as we want to exclude -stats
-    // from the search
-    psMetadata *where = psMetadataAlloc();
-    if (!psMetadataAddS32(where, PS_LIST_TAIL, "det_id", 0, "==",
-            (psS32)atoi(det_id))) {
-        psError(PS_ERR_UNKNOWN, false, "failed to add item det_id");
-        psFree(where);
-        return false;
-    }
-    if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_id", 0, "==", exp_id)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to add item exp_id");
-        psFree(where);
-        return false;
-    }
-
-    psArray *rawImfiles = searchRawImfiles(config, where);
-    psFree(where);
-    if (!rawImfiles) {
-        psError(PS_ERR_UNKNOWN, true,
-    "det_id %s, exp_id %s, class_id %s does not corespond to an input imfile",
-            det_id, exp_id, class_id);
-        return false;
-    }
-
-    // iterate through rawImfiles looking for a match to our class_id
-    bool found = false;
-    for (long i = 0; i < psArrayLength(rawImfiles); i++) {
-        if (strcmp(class_id,
-                ((rawImfileRow *)rawImfiles->data[i])->class_id) == 0) {
-            found = true; 
-            break;
-        }
-    }
-
-    psFree(rawImfiles);
-
-    if (!found) {
-        psError(PS_ERR_UNKNOWN, true,
-    "det_id %s, exp_id %s, class_id %s does not corespond to an input imfile",
-            det_id, exp_id, class_id);
-        return false;
-    }
-
     // create a new detResidImfileRow and insert it
-    detResidImfileRow *residImfile = detResidImfileRowAlloc(
-                (psS32)atol(det_id),
-                iteration,
-                exp_id,
-                class_id,
-                recipe,
-                uri,
-                b1_uri,
-                b2_uri
-            );
-
-    if (!detResidImfileInsertObject(config->dbh, residImfile)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(residImfile);
-        return false;
-    }
-
-    psFree(residImfile);
-
-    return true;
+    return detResidImfileRowAlloc(
+            normalizedImfile->det_id,
+            normalizedImfile->iteration,
+            exp_id,
+            normalizedImfile->class_id,
+            recipe,
+            uri,
+            b1_uri,
+            b2_uri
+        );
 }
 
@@ -2617 +2691 @@
 
     psFree(residImfiles);
+
+    return true;
+}
+
+static bool toresidexpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    /*
+which returns a list of exposures for which all component class ids
+have had residuals created.  This list includes the detrend id,
+iteration, exposure id, detrend type, and whether the exposure was
+included in the stack for this iteration.
+    */
+
+
+    // select detRun.position
+    // select detRun.iteration
+    // select detRun.det_type
+    // select detInputExp.exp_id
+    // select detInputExp.include
+    // by:
+    // find the current iteration bassed on det_id
+    // find all exp_ids in the current det_id/iteration from detInputExp
+    // compare to detInputExp.imfiles to derResidImfile by class_id
+    // and: 
+    // detResidImfile.{det_id, iteration, exp_id} is not in detResidExp
+
+    psString query = psStringCopy(
+        "SELECT DISTINCT"
+        "   detRun.position AS det_id,"
+        "   detRun.iteration,"
+        "   detRun.det_type,"
+        "   detInputExp.exp_id,"
+        "   detInputExp.include"
+        " FROM detRun"
+        " LEFT JOIN detInputExp"
+        "   ON detRun.position = detInputExp.det_id"
+        "   AND detRun.iteration = detInputExp.iteration"
+        " LEFT JOIN rawDetrendExp"
+        "   ON detInputExp.exp_id = rawDetrendExp.exp_id"
+        " LEFT JOIN detResidImfile"
+        "   ON detRun.position = detResidImfile.det_id"
+        "   AND detRun.iteration = detResidImfile.iteration"
+        "   AND detInputExp.exp_id = detResidImfile.exp_id"
+        " LEFT JOIN detResidExp"
+        "   ON detRun.position = detResidExp.det_id"
+        "   AND detRun.iteration = detResidExp.iteration"
+        "   AND detInputExp.exp_id = detResidExp.exp_id"
+        " GROUP BY"
+        "   detInputExp.exp_id"
+        " WHERE"
+        "   detResidExp.det_id IS NULL"
+        "   AND detResidExp.iteration IS NULL"
+        "   AND detResidExp.exp_id IS NULL"
+        " HAVING rawDetrendExp.imfiles = COUNT(detStackedImfile.class_id)"
+        ); 
+
+    // XXX does it make sens to accept any search params?
+#if 0
+    if (config->where) {
+        psString whereClaus = psDBGenerateWhereConditionSQL(config->where);
+        psStringAppend(&query, " AND %s", whereClaus);
+        psFree(whereClaus);
+    }
+#endif
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    } 
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        // XXX check psError here
+        psError(PS_ERR_UNKNOWN, false, "no pending rawDetrendExp rows found");
+        return false;
+    }
+
+    bool simple = false;
+    {
+        bool status = false;
+        simple = psMetadataLookupBool(&status, config->args, "-simple");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
+            return false;
+        }
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "rawDetrendImfile", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
 
     return true;

trunk/ippTools/src/dettoolConfig.c

@@ -249 +249 @@
     psMetadataAddBool(normalizedArgs, PS_LIST_TAIL, "-simple",  0,
         "use the simple output format", false);
+ 
+    // -toresidexp
+    psMetadata *toresidexpArgs = psMetadataAlloc();
+    psMetadataAddBool(toresidexpArgs, PS_LIST_TAIL, "-simple",  0,
+        "use the simple output format", false);
 
     // -normalizedframe
@@ -391 +396 @@
     PXTOOL_MODE("-toresid",         DETTOOL_MODE_TORESID,       toresidArgs);
     PXTOOL_MODE("-normalized",      DETTOOL_MODE_NORMALIZED,    normalizedArgs);
+    PXTOOL_MODE("-toresidexp",      DETTOOL_MODE_TORESIDEXP,    toresidexpArgs);
     PXTOOL_MODE("-normalizedframe", DETTOOL_MODE_NORMALIZEDFRAME,normalizedframeArgs);
     PXTOOL_MODE("-addresidimfile",  DETTOOL_MODE_ADDRESIDIMFILE,  addresidimfileArgs);
@@ -413 +419 @@
         printf("\nPan-STARRS Detrend Tool\n");
         printf("Usage: %s <mode> [<options>]\n\n", argv[0]);
-        printf(" <mode> : [-pending|-definebyexp|-definebyquery|-runs|-selectrun|-select|-raw|-addprocessed|-tostack|-processed|-addstacked|-tonormalize|-addnormstat|-normstat|-stacked|-stackedframe|-addnormalizedimfile|-toresid|-normalized|-normalizedframe|-addresidimfile|-residimfile|-addresidexp|-residexp|-declaremasterframe|-masterframe|-master|-rerun]\n\n");
+        printf(" <mode> : [-pending|-definebyexp|-definebyquery|-runs|-selectrun|-select|-raw|-addprocessed|-tostack|-processed|-addstacked|-tonormalize|-addnormstat|-normstat|-stacked|-stackedframe|-addnormalizedimfile|-toresid|-normalized|-toresidexp|-normalizedframe|-addresidimfile|-residimfile|-addresidexp|-residexp|-declaremasterframe|-masterframe|-master|-rerun]\n\n");
         fprintf (stdout, "-pending ");
         psArgumentHelp(pendingArgs);
@@ -492 +498 @@
         psArgumentHelp(normalizedArgs);
         psFree(normalizedArgs);
+
+        fprintf (stdout, "-toresidexp");
+        psArgumentHelp(toresidexpArgs);
+        psFree(toresidexpArgs);
 
         fprintf (stdout, "-normalizedframe ");
@@ -552 +562 @@
     psFree(toresidArgs);
     psFree(normalizedArgs);
+    psFree(toresidexpArgs);
     psFree(normalizedframeArgs);
     psFree(addresidimfileArgs);