Changeset 36622

Timestamp:

Mar 31, 2014, 11:50:56 AM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20140226

Files:

• . (modified) (1 prop)
• Ohana/src/libohana/src (modified) (1 prop)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/publish_file.pl (modified) (1 prop)
• ippScripts/scripts/sc_prepare_chip.pl (copied) (copied from trunk/ippScripts/scripts/sc_prepare_chip.pl )
• ippScripts/scripts/sc_remote_exec.pl (copied) (copied from trunk/ippScripts/scripts/sc_remote_exec.pl )
• ippScripts/scripts/sc_validate_files.pl (copied) (copied from trunk/ippScripts/scripts/sc_validate_files.pl )
• ippScripts/scripts/sc_validate_processing.pl (copied) (copied from trunk/ippScripts/scripts/sc_validate_processing.pl )
• ippToPsps/jython/cleanup.py (modified) (2 diffs)
• ippToPsps/jython/dxlayer.py (modified) (4 diffs)
• ippToPsps/jython/odm.py (modified) (2 diffs)
• ippToPsps/perl/deleteFromDXLayer.pl (modified) (1 diff)
• ippToPsps/perl/deleteFromDXLayer2.pl (modified) (1 diff)
• ippTools/src/difftool.c (modified) (2 diffs)
• ippconfig/recipes/filerules-split.mdc (modified) (2 diffs)
• ippconfig/recipes/ppSub.config (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 prop)
• ppBackground/src/Makefile.am (modified) (3 diffs)
• ppBackground/src/ppBackground.c (modified) (1 diff)
• ppBackground/src/ppBackgroundStack.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStack.c )
• ppBackground/src/ppBackgroundStack.h (copied) (copied from trunk/ppBackground/src/ppBackgroundStack.h )
• ppBackground/src/ppBackgroundStackArguments.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStackArguments.c )
• ppBackground/src/ppBackgroundStackCamera.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStackCamera.c )
• ppBackground/src/ppBackgroundStackData.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStackData.c )
• ppBackground/src/ppBackgroundStackLoop.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStackLoop.c )
• ppBackground/src/ppBackgroundStackMath.c (copied) (copied from trunk/ppBackground/src/ppBackgroundStackMath.c )
• ppStack/src/ppStackReadout.c (modified) (2 diffs)
• ppSub/src/ppSubMatchPSFs.c (modified) (1 diff)
• psphot/src (modified) (1 prop)
• psphot/src/psphotStackImageLoop.c (modified) (1 prop)
• pstamp/scripts/pstamp_insert_request.pl (modified) (3 diffs)
• pstamp/scripts/pstampparse.pl (modified) (5 diffs)

branches/eam_branches/ipp-20140226/ippToPsps/jython/cleanup.py

@@ -87 +87 @@
         self.logger.infoPair("To delete from DXLayer", "%d" % len(deleteFromDxLayerIDs))
         self.logger.infoPair("To delete from local disk", "%d" % len(deleteFromLocalIDs))
-    
+    #    self.logger.infoPair("Did I get here", "yes?")
         # delete stuff from local disk
         if self.skychunk.deleteLocal:
@@ -110 +110 @@
     
             self.reportResults("datastore", deleteFromDatastoreIDs, count) 
-        
+       # self.logger.infoPair("here","here")
         # remove stuff from DXLayer
         if self.skychunk.deleteDxLayer:
+#            self.logger.infoPair ("skip remove from dxlayer", "HAF")
             count = 0
             for id in deleteFromDxLayerIDs:
-        
+                self.logger.infoPair("delete from dxlayer", id);
                 if self.dxlayer.deleteBatch(id):
+                    self.logger.infoPair("delete dxlayer",id);
                     self.ippToPspsDb.updateDeletedDxlayer(id, 1)
                     count = count + 1

branches/eam_branches/ipp-20140226/ippToPsps/jython/dxlayer.py

@@ -37 +37 @@
        
         if p.returncode != 0:
-           self.logger.debugPair("DXLayer webservice", "failed")
+           self.logger.infoPair("DXLayer webservice", "failed")
            return 0
         else:
-           self.logger.debugPair("DXLayer webservice", "successful")
+           self.logger.infoPair("DXLayer webservice", "successful")
 
         ret = self.decodeDXLayerResponse(tempFile.name)
@@ -48 +48 @@
 
         if ret: self.logger.infoPair("Deleted batch from DXLayer 1", batch) 
-        else: self.logger.errorPair("Unable to delete batch from DXLayer 1" , batch)
+        else: self.logger.infoPair("Unable to delete batch from DXLayer 1" , batch)
 
 
@@ -61 +61 @@
        
         if p.returncode != 0:
-           self.logger.debugPair("DXLayer webservice", "failed")
+           self.logger.infoPair("DXLayer webservice", "failed")
            return 0
         else:
@@ -72 +72 @@
 
         if ret: self.logger.infoPair("Deleted batch from DXLayer 2", batch) 
-        else: self.logger.errorPair("Unable to delete batch from DXLayer 2" , batch)
+        else: self.logger.infoPair("Unable to delete batch from DXLayer 2" , batch)
 
 

branches/eam_branches/ipp-20140226/ippToPsps/jython/odm.py

@@ -78 +78 @@
                 DETAILS = doc.find("{%s}OdmBatchState/{%s}Details" % (NAMESPACE, NAMESPACE)).text
                 results['DETAILS'] = DETAILS
-            elif re.search("processing", STATE): 
+            elif re.search("processing", STATE):
                 loadedToODM = 1
             if re.search("MergeWorthy", MESSAGE): 
@@ -87 +87 @@
                 mergeWorthy = 1
                 merged = 1
+#            if re.search("released", STATE):
+#                loadedToODM = 1
+#                mergeWorthy = 1
+#                merged = 1
+#            if re.search("released", MESSAGE):
+#                loadedToODM = 1
+#                mergeWorthy = 1
+#                merged = 1
         except:
             pass

branches/eam_branches/ipp-20140226/ippToPsps/perl/deleteFromDXLayer.pl

@@ -18 +18 @@
         );
 
-my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
+#my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
+
+#my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb01/cgi-bin/dxwebservice.cgi";
+my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb01/cgi-bin/dxlayer/dxwebservice.cgi";
+
+
 #my $dxWebServiceUrl2 = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
 

branches/eam_branches/ipp-20140226/ippToPsps/perl/deleteFromDXLayer2.pl

@@ -18 +18 @@
         );
 
-my $dxWebServiceUrl = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
+#my $dxWebServiceUrl = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
+
+#my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb02/cgi-bin/dxwebservice.cgi";
+my $dxWebServiceUrl = "http://web02.psps.ifa.hawaii.edu/lsb02/cgi-bin/dxlayer/dxwebservice.cgi";
 #my $dxWebServiceUrl2 = "http://lsb02.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi";
 

branches/eam_branches/ipp-20140226/ippTools/src/difftool.c

@@ -1349 +1349 @@
     if (!mdok) {
         psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psString warpDataGroup = psMetadataLookupStr(&mdok, row, "warpDataGroup");
+    if (!mdok) {
+        psError(PXTOOLS_ERR_PROG, false, "warpDataGroup not found");
         if (!psDBRollback(config->dbh)) {
           psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1391 +1399 @@
                                         workdir,
                                         label,
-                                        data_group ? data_group : label,
+                                        data_group ? data_group : warpDataGroup,
                                         dist_group,
                                         reduction,

branches/eam_branches/ipp-20140226/ippconfig/recipes/filerules-split.mdc

@@ -303 +303 @@
 SKYCELL.TEMPLATE             OUTPUT {OUTPUT}.skycell                  SKYCELL         NONE       FPA        TRUE      NONE
                                                                                                               
-PPSUB.OUTPUT                 OUTPUT {OUTPUT}.fits                     IMAGE           COMP_SUB   FPA        TRUE      NONE
+PPSUB.OUTPUT                 OUTPUT {OUTPUT}.fits                     IMAGE           NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.MASK            OUTPUT {OUTPUT}.mask.fits                MASK            COMP_MASK  FPA        TRUE      NONE
 PPSUB.OUTPUT.VARIANCE        OUTPUT {OUTPUT}.wt.fits                  VARIANCE        COMP_WT    FPA        TRUE      NONE
@@ -311 +311 @@
 PPSUB.OUTPUT.JPEG2           OUTPUT {OUTPUT}.b2.jpg                   JPEG            NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.RESID.JPEG      OUTPUT {OUTPUT}.res.jpg                  JPEG            NONE       FPA        TRUE      NONE
-PPSUB.INVERSE                OUTPUT {OUTPUT}.inv.fits                 IMAGE           COMP_SUB   FPA        TRUE      NONE
+PPSUB.INVERSE                OUTPUT {OUTPUT}.inv.fits                 IMAGE           NONE       FPA        TRUE      NONE
 PPSUB.INVERSE.MASK           OUTPUT {OUTPUT}.inv.mask.fits            MASK            COMP_MASK  FPA        TRUE      NONE
 PPSUB.INVERSE.VARIANCE       OUTPUT {OUTPUT}.inv.wt.fits              VARIANCE        COMP_WT    FPA        TRUE      NONE

branches/eam_branches/ipp-20140226/ppBackground/src/Makefile.am

@@ -1 @@
-bin_PROGRAMS = ppBackground
+bin_PROGRAMS = ppBackground ppBackgroundStack
 
 if HAVE_SVNVERSION
@@ -28 +28 @@
 ppBackground_LDFLAGS  = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PPSTATS_LIBS)   $(PSPHOT_LIBS)   $(PPBACKGROUND_LIBS)
 
+ppBackgroundStack_CPPFLAGS = $(ppBackground_CPPFLAGS)
+ppBackgroundStack_LDFLAGS = $(ppBackground_LDFLAGS)
+
 ppBackground_SOURCES =          \
         ppBackground.c          \
@@ -38 +41 @@
         ppBackgroundErrorCodes.c        \
         ppBackgroundExit.c
+
+ppBackgroundStack_SOURCES = \
+        ppBackgroundStack.c \
+        ppBackgroundStackArguments.c \
+        ppBackgroundStackCamera.c \
+        ppBackgroundStackData.c \
+        ppBackgroundStackLoop.c \
+        ppBackgroundStackMath.c \
+        ppBackgroundVersion.c \
+        ppBackgroundErrorCodes.c \
+        ppBackgroundExit.c
+
 
 noinst_HEADERS = \

branches/eam_branches/ipp-20140226/ppBackground/src/ppBackground.c

@@ -18 +18 @@
     ppBackgroundErrorRegister();
 
+    printf("Initializing data\n");
     ppBackgroundData *data = ppBackgroundDataInit(&argc, argv);
     if (!data) {
         goto DIE;
     }
-
+    printf("Reading Arguments\n");
     if (!ppBackgroundArguments(data, argc, argv)) {
         goto DIE;
     }
-
+    printf("Setting up Camera\n");
     if (!ppBackgroundCamera(data)) {
         goto DIE;
     }
 
+    printf("Looping over data!\n");
     if (!ppBackgroundLoop(data)) {
         goto DIE;

branches/eam_branches/ipp-20140226/ppStack/src/ppStackReadout.c

@@ -184 +184 @@
     psFree(stack);
 
+    psLogMsg("ppStack", PS_LOG_INFO, "initial stack image sectionNum %d", sectionNum);
+
     sectionNum++;
 
@@ -293 +295 @@
     psFree(stack);
 
+    psLogMsg("ppStack", PS_LOG_INFO, "final stack image sectionNum %d", sectionNum);
+
     sectionNum++;
 

branches/eam_branches/ipp-20140226/ppSub/src/ppSubMatchPSFs.c

@@ -156 +156 @@
     psLogMsg("ppSub", PS_LOG_INFO, "Input FWHM: %f\nReference FWHM: %f\n", inFWHM, refFWHM);
     if (!isfinite(inFWHM) || !isfinite(refFWHM)) {
+#ifdef SET_QUALITY_INSTEAD_OF_ERROR
         psErrorStackPrint(stderr, "Cannot determine FHWM for images, giving up.");
         int error = psErrorCodeLast(); // Error code
         ppSubDataQuality(data, error, PPSUB_FILES_ALL);
         return true;
+#else
+        psError(PPSUB_ERR_DATA, false, "Cannot determine FHWM for images, giving up.");
+        return false;
+#endif
     }
 

branches/eam_branches/ipp-20140226/pstamp/scripts/pstamp_insert_request.pl

@@ -45 +45 @@
 my $pstamptool = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $fields = can_run('fields')  or (warn "Can't find fields"  and $missing_tools = 1);
+my $fhead = can_run('fhead')  or (warn "Can't find fhead"  and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -56 +57 @@
     # Note that if it's a pstamp request then REQ_NAME should be defined.
     # if it's a detectability query it will not have a REQ_NAME but will have a QUERY_ID
-    my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID";
+    # my $command = "echo $tmp_req_file | $fields -x 0 EXTNAME EXTVER REQ_NAME QUERY_ID";
+    my $command = "$fhead -x 0 $tmp_req_file";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-if (0) {
-    # stoopid fields doesn't set exit status to zero when it works
     unless ($success) {
         print STDERR @$stderr_buf;
         exit $error_code >> 8;
     }
-}
     my $output = join "", @$stdout_buf;
-    (undef, $extname, $extver, $req_name) = split " ", $output;
+
+
+    my $makehash = 0;
+    my %hash;
+    foreach my $line (split "\n", $output) {
+        chomp $line;
+        # split lines inte left and right using equals sign. left is the keyword
+        my ($key, $right) = split "=", $line;
+            # skip if there was no '='
+        next if !$right;
+
+        $key =~ s/ //g;
+
+        # separate value from comment
+        my ($value, $comment) = split "/", $right;
+        # remove ' and space characters from key and value
+        $value =~ s/'//g;
+        $value =~ s/ //g;
+
+        #    print "$key $value\n";
+
+        # extract the values that we are looking for
+        $extname  = $value if ($key eq "EXTNAME");
+        $extver   = $value if ($key eq "EXTVER");
+        $req_name = $value if ($key eq "REQ_NAME");
+        $req_name = $value if ($key eq "QUERY_ID");
+
+        # optionally build hash. Duplicate keys get the last value seen
+        if ($makehash) {
+            $hash{$key} = $value;
+        }
+    }
+
+    # (undef, $extname, $extver, $req_name) = split " ", $output;
+
     if (!$extname or ! (($extname eq "PS1_PS_REQUEST") or ($extname eq "MOPS_DETECTABILITY_QUERY"))) {
         print STDERR "invalid request file\n";
+        print "invalid request file\n";
         exit $PS_EXIT_DATA_ERROR;
     }
+
     if (!defined $req_name) {
         print STDERR "invalid request file no REQ_NAME or QUERY_ID\n";
+        print "invalid request file no REQ_NAME or QUERY_ID\n";
+        exit $PS_EXIT_DATA_ERROR;
+    }
+    if (!defined $extver) {
+        print STDERR "invalid request file no EXTVER found\n";
+        print "invalid request file no EXTVER found\n";
         exit $PS_EXIT_DATA_ERROR;
     }
@@ -119 +160 @@
 
 exit 0;
+
 # Ask the database for the next web request number
 sub get_webreq_num

branches/eam_branches/ipp-20140226/pstamp/scripts/pstampparse.pl

@@ -243 +243 @@
 my $watch_for_big_requests = (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/));
 
-my $big_limit = 100;    # XXX: this should be in a configuration file somewhere not hard coded
+# XXX: these should be in a configuration file somewhere not hard coded
+my $big_limit = 100;
+my $job_big_limit = 400;
 
 if ($watch_for_big_requests and $nRows > $big_limit) {
@@ -267 +269 @@
 
     # see whether number of jobs limit for high priority request was exceeded
-    if ($watch_for_big_requests and $num_jobs > $big_limit) {
+    if ($watch_for_big_requests and $num_jobs > $job_big_limit) {
         $label = change_to_lower_priority_label($label);
         $watch_for_big_requests = 0;
@@ -311 +313 @@
     my $rownum   = $row->{ROWNUM};
     if (!validID($rownum)) {
+        $rownum = 'NULL' if !defined $rownum;
         print STDERR "$rownum is not a valid ROWNUM\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -316 +319 @@
     }
     my $job_type = $row->{JOB_TYPE};
-    if (($job_type ne "stamp") and ($job_type ne "get_image")) {
+    if (!defined $job_type || (($job_type ne "stamp") and ($job_type ne "get_image"))) {
+        $job_type = 'NULL' if !defined $job_type;
         print STDERR "$job_type is not a valid JOB_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
@@ -325 +329 @@
     if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and
         ($req_type ne "byskycell") and ($req_type ne "bydiff")) {
+        $req_type = 'NULL' if !defined $req_type;
         print STDERR "$req_type is not a valid REQ_TYPE\n";
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);