Changeset 30118 for branches/czw_branch/20101203/ippTools

Timestamp:

Dec 20, 2010, 2:30:45 PM (16 years ago)

Author:

watersc1

Message:

Attempting to bring branch in sync with trunk

Location:

branches/czw_branch/20101203

Files:

• . (modified) (1 prop)
• ippTools (modified) (1 prop)
• ippTools/configure.ac (modified) (2 diffs)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/pubtool_definerun.sql (modified) (4 diffs)
• ippTools/share/pubtool_pending.sql (modified) (3 diffs)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff, 1 prop)
• ippTools/share/regtool_pendingexp.sql (modified) (1 diff)
• ippTools/src (modified) (1 prop)
• ippTools/src/dettool.c (modified) (1 diff)
• ippTools/src/dettoolConfig.c (modified) (1 diff)
• ippTools/src/difftool.c (modified) (4 diffs)
• ippTools/src/difftoolConfig.c (modified) (1 diff)
• ippTools/src/magicdstool.c (modified) (1 diff)
• ippTools/src/magicdstoolConfig.c (modified) (1 diff)
• ippTools/src/pstamptool.c (modified) (1 diff)
• ippTools/src/pstamptoolConfig.c (modified) (1 diff)
• ippTools/src/pubtool.c (modified) (2 diffs)
• ippTools/src/pubtoolConfig.c (modified) (1 diff)
• ippTools/src/regtool.c (modified) (4 diffs)

branches/czw_branch/20101203/ippTools/configure.ac

@@ -1 +1 @@
 AC_PREREQ(2.61)
 
-AC_INIT([ipptools], [1.1.65], [ipp-support@ifa.hawaii.edu])
+AC_INIT([ipptools], [1.1.66], [ipp-support@ifa.hawaii.edu])
 AC_CONFIG_SRCDIR([autogen.sh])
 
@@ -18 +18 @@
 PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0])
 PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
-PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.65]) 
+PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.66]) 
 PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1]) 
 

branches/czw_branch/20101203/ippTools/share/pubtool_definerun.sql

@@ -3 +3 @@
     client_id,
     stage_id,
-    src_label
+    src_label,
+    output_format
 FROM (
     -- Get diffs to publish
@@ -9 +10 @@
         client_id,
         diff_id AS stage_id,
-        diffRun.label AS src_label
+        diffRun.label AS src_label,
+        output_format
     FROM publishClient
     JOIN diffRun
@@ -28 +30 @@
         client_id,
         cam_id AS stage_id,
-        camRun.label AS src_label
+        camRun.label AS src_label,
+        output_format
     FROM publishClient
     JOIN camRun
@@ -43 +46 @@
         client_id,
         diff_phot_id AS stage_id,
-        diffPhotRun.label AS src_label
+        diffPhotRun.label AS src_label,
+        output_format
     FROM publishClient
     JOIN diffPhotRun

branches/czw_branch/20101203/ippTools/share/pubtool_pending.sql

@@ -9 +9 @@
         publishClient.stage,
         publishClient.workdir,
+        publishClient.output_format,
         diffRun.diff_id AS stage_id,
         rawExp.camera,
@@ -35 +36 @@
         publishClient.stage,
         publishClient.workdir,
+        publishClient.output_format,
         camRun.cam_id AS stage_id,
         rawExp.camera,
@@ -56 +58 @@
         publishClient.stage,
         publishClient.workdir,
+        publishClient.output_format,
         diffPhotRun.diff_phot_id AS stage_id,
         rawExp.camera,

branches/czw_branch/20101203/ippTools/share/pxadmin_create_tables.sql

@@ -1645 +1645 @@
     workdir VARCHAR(255) NOT NULL, -- working directory
     comment VARCHAR(255),            -- for human memory
-    PRIMARY KEY(client_id)
+    name varchar(64) default NULL, -- unique client_id verbose identifier
+    output_format SMALLINT NOT NULL default 1, -- format output versioning
+    PRIMARY KEY(client_id),
+    UNIQUE KEY name (name)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 

branches/czw_branch/20101203/ippTools/share/regtool_pendingexp.sql

@@ -1 +1 @@
 SELECT DISTINCT
-    exp_id,
-    tmp_exp_name,
-    tmp_camera,
-    tmp_telescope,
-    state,
-    workdir,
-    workdir_state,
-    reduction,
-    dvodb,
-    tess_id,
-    end_stage,
-    label,
-    camera,
-    filter,
-    obs_mode,
-    obs_group,
-    epoch,
-    dateobs
-FROM
-    (SELECT
-       newExp.*,
-       newImfile.tmp_class_id,
-       rawImfile.tmp_class_id as raw_tmp_class_id,
-       rawImfile.camera,
-       rawImfile.filter,
-       rawImfile.dateobs,
-       rawImfile.obs_mode,
-       rawImfile.obs_group
-    FROM newExp
-    JOIN newImfile
-       USING(exp_id)
-    LEFT JOIN rawExp
-       USING(exp_id)
-    LEFT JOIN rawImfile
-        ON newImfile.exp_id = rawImfile.exp_id
-        AND newImfile.tmp_class_id = rawImfile.tmp_class_id
-    WHERE
-        newExp.state = 'run'
-        AND rawExp.exp_id IS NULL
-        AND rawImfile.data_state = 'full'
--- where hook %s
-    GROUP BY
-        newExp.exp_id
-    HAVING
-        COUNT(newImfile.tmp_class_id) = COUNT(rawImfile.tmp_class_id)
-        AND SUM(rawImfile.fault) = 0
+     exp_id,
+     tmp_exp_name,
+     tmp_camera,
+     tmp_telescope,
+     state,
+     workdir,
+     workdir_state,
+     reduction,
+     dvodb,
+     tess_id,
+     end_stage,
+     label,
+     camera,
+     filter,
+     obs_mode,
+     obs_group,
+     epoch,
+     dateobs
+FROM 
+        (
+        select DISTINCT * from 
+         (
+          select newExp.*,count(newImfile.tmp_class_id) AS new_count
+          from newExp
+          LEFT JOIN newImfile USING (exp_id)
+          LEFT JOIN rawExp USING (exp_id) 
+          where 
+           newExp.state = 'run' AND rawExp.exp_id IS NULL
+          GROUP BY newExp.exp_id
+         ) AS NEWEXPOSURES
+         JOIN 
+         (
+          select DISTINCT newExp.exp_id,
+          rawImfile.camera,
+          rawImfile.filter,
+          rawImfile.dateobs,
+          rawImfile.obs_mode,
+          rawImfile.obs_group,
+          count(rawImfile.tmp_class_id) AS raw_count
+          from newExp
+          LEFT JOIN rawExp USING (exp_id) 
+          LEFT JOIN rawImfile USING (exp_id)
+          where 
+           newExp.state = 'run' AND rawExp.exp_id IS NULL
+           AND rawImfile.data_state = 'full'
+           -- where hook %s
+          GROUP BY newExp.exp_id
+          HAVING SUM(rawImfile.fault) = 0
+         ) AS RAWEXPOSURES
+         USING (exp_id)
+        WHERE raw_count = new_count 
+    ) AS Foo
 -- limit hook %s
-    ) as Foo

branches/czw_branch/20101203/ippTools/src/dettool.c

@@ -572 +572 @@
     PXOPT_COPY_F64(config->args, where, "-select_moon_phase_min", "moon_phase", ">=");
     PXOPT_COPY_F64(config->args, where, "-select_moon_phase_max", "moon_phase", "<=");
+
+    PXOPT_COPY_STR(config->args, where, "-select_state", "state", "=");
     PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE");
 

branches/czw_branch/20101203/ippTools/src/dettoolConfig.c

@@ -138 +138 @@
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_min",  0,          "define min moon phase", NAN);
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_max",  0,          "define max moon phase", NAN);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_state",           0,          "select by state (default is 'full')", "full");
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment",                0,          "search by comment field (LIKE comparison)", NULL);
 

branches/czw_branch/20101203/ippTools/src/difftool.c

@@ -483 +483 @@
     psFree(where);
 
-     psStringAppend(&query, "\nORDER by priority DESC, diff_id");
+     psStringAppend(&query, "\nORDER by priority DESC, diff_id, skycell_id");
 
     // treat limit == 0 as "no limit"
@@ -1978 +1978 @@
     PXOPT_COPY_STR(config->args, stack2Where, "-skycell_id", "stackRun.skycell_id", "==");
     PXOPT_COPY_STR(config->args, stack1Where, "-input_label", "stackRun.label","==");
+    PXOPT_COPY_STR(config->args, stack1Where, "-input_data_group", "stackRun.data_group","==");
     PXOPT_COPY_STR(config->args, stack2Where, "-template_label", "stackRun.label","==");
     PXOPT_COPY_F32(config->args, stack1Where, "-good_frac", "stackSumSkyfile.good_frac", ">=");
@@ -2042 +2043 @@
     }
 
-    psString queryCopy = psStringCopy(query);
-    psFree(query);
-    query = queryCopy;
-    
     psStringSubstitute(&query, stack1Query, "@STACK1_QUERY@");
     psStringSubstitute(&query, stack2Query, "@STACK2_QUERY@");
@@ -2120 +2117 @@
         psStringAppend(&this_stack1Query,"AND %s", thisWhere);
         psFree(thisWhere);
-
-        psString queryCopy = psStringCopy(query);
-        psFree(query);
-        query = queryCopy;
-    
 
         psStringSubstitute(&query, this_stack1Query, "@STACK1_QUERY@");

branches/czw_branch/20101203/ippTools/src/difftoolConfig.c

@@ -336 +336 @@
     psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-input_label", 0, "search by stack label for input", NULL);
     psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-template_label", 0, "search by stack label for template", NULL);
+    psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-input_data_group", 0, "search by stack data_group for input", NULL);
     psMetadataAddF32(definestackstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
     psMetadataAddStr(definestackstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL);

branches/czw_branch/20101203/ippTools/src/magicdstool.c

@@ -1155 +1155 @@
 
     psMetadata *where = psMetadataAlloc();
-    // new state
-    PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", false, false);
+    // new state (required
+    PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", true, false);
     // old state (required)
     PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);

branches/czw_branch/20101203/ippTools/src/magicdstoolConfig.c

@@ -163 +163 @@
     // -clearstatefaults
     psMetadata *clearstatefaultsArgs = psMetadataAlloc();
-    psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL);
+    psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL);
     psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL);
     psMetadataAddS64(clearstatefaultsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0);

branches/czw_branch/20101203/ippTools/src/pstamptool.c

@@ -950 +950 @@
     PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
     PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
+
+    PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-component", "component", "==");
+
     PXOPT_COPY_S32(config->args, where, "-fault",  "pstampDependent.fault", "==");
+
     pxAddLabelSearchArgs(config, where, "-label", "pstampRequest.label", "LIKE");
-    // if (fault_count) {
-        PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampDependent.fault_count", ">=");
-    // }
+    PXOPT_COPY_S32(config->args, where, "-fault_count", "pstampDependent.fault_count", ">=");
 
     // XXX: How about selecting by pstampRequest.label? No. That is too dangerous by itself.

branches/czw_branch/20101203/ippTools/src/pstamptoolConfig.c

@@ -178 +178 @@
     psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL,  "-job_id", 0,       "job_id of jobs to update", 0);
     psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL,  "-dep_id", 0,       "dep_id of jobs to update", 0);
+    psMetadataAddS64(stopdependentjobArgs, PS_LIST_TAIL,  "-stage_id", 0,     "stage_id of jobs to update", 0);
+    psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL,  "-component", 0,    "component of jobs to update", NULL);
     psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL,  "-fault", 0,        "current value for dependent fault", 0);
     psMetadataAddS32(stopdependentjobArgs, PS_LIST_TAIL,  "-fault_count", 0,   "select by fault_count (>=)", 0);

branches/czw_branch/20101203/ippTools/src/pubtool.c

@@ -96 +96 @@
     PXOPT_LOOKUP_STR(comment, config->args, "-comment",  false, false);
     PXOPT_LOOKUP_BOOL(unmagicked, config->args, "-unmagicked",  false);
-
-    if (!publishClientInsert(config->dbh, 0, 0, product, stage, !unmagicked, workdir, comment)) {
+    PXOPT_LOOKUP_STR(name, config->args, "-name",  false, false);
+    PXOPT_LOOKUP_S16(output_format, config->args, "-output_format",  false, false);
+
+    if (!publishClientInsert(config->dbh, 0, 0, product, stage, !unmagicked, workdir, comment, name, output_format)) {
         psError(PS_ERR_UNKNOWN, false, "Database error");
         return false;
@@ -313 +315 @@
     PXOPT_COPY_STR(config->args, where, "-stage", "publishClient.stage", "==");
     PXOPT_COPY_STR(config->args, where, "-comment", "publishClient.comment", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-name", "publishClient.name", "LIKE");
     pxAddLabelSearchArgs(config, where, "-label", "publishRun.label", "==");
 

branches/czw_branch/20101203/ippTools/src/pubtoolConfig.c

@@ -83 +83 @@
     psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple",  0, "use simple output format?", false);
     psMetadataAddU64(pendingArgs, PS_LIST_TAIL, "-limit",  0, "limit result set", 0);
+    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-name", 0, "search on client name", NULL);
 
     // -add

branches/czw_branch/20101203/ippTools/src/regtool.c

@@ -300 +300 @@
   psStringSubstitute(&query,date,"@DATE@");
 
-  fprintf(stderr,"%s",query);
+  //  fprintf(stderr,"%s",query);
 
   if (!p_psDBRunQuery(config->dbh, query)) {
@@ -349 +349 @@
       psMetadataAddStr(row,PS_LIST_TAIL,"previous_class_id",PS_META_REPLACE,"",previous_class_id);
     }
-    
+    // class_id = NULL sorts to the top of the list, so skip those (incomplete downloads)
+    if (!this_class_id) {
+      continue;
+    }
     // Determine if we've crossed a class_id boundary, as this resets the bits.
     if (previous_class_id) {
@@ -370 +373 @@
     if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
         (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
+      //      printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id);
       ok_to_burn = false;
     }
     if (already_burned == false) {
+      //      printf("already_burned looks false %s %s\n",this_uri,this_class_id);
       ok_to_burn = false;
     }
@@ -398 +403 @@
     }      
 
-    //    printf("STATUS: %s %s %s %s (%d %d)\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned);
+    //    printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded"));
 
     // If the state of this imfile is not "pending_burntool" then we can't burn it.