Changeset 28003 for branches/pap/ippToPsps/scripts/ippToPsps_run.pl

Timestamp:

May 18, 2010, 12:49:05 PM (16 years ago)

Author:

eugene

Message:

merging changes from trunk into branches/pap

Location:

branches/pap

Files:

• . (modified) (1 prop)
• ippToPsps/scripts/ippToPsps_run.pl (modified) (40 diffs)

branches/pap/ippToPsps/scripts/ippToPsps_run.pl

@@ -12 +12 @@
 
 # globals
-my ($verbose, $save_temps, $no_op, $no_update, $camera, $product, $output, $dvodb, $label, $singleExpId, $no_publish);
+my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $label, $singleExpId, $no_publish);
 
 # TODO temporary until we use database to store current jobid
@@ -26 +26 @@
 $output = undef;
 $singleExpId = undef;
+$datastoreProduct = "PSPS_test";
 
 # get user args
 GetOptions(
         'output|o=s' => \$output,
-        'product|p=s' => \$product,
+        'batch|b=s' => \$batchType,
         'dvodb|d=s' => \$dvodb,
         'label|l=s' => \$label,
         'expid|e=s' => \$singleExpId,
+        'product|p=s' => \$datastoreProduct,
         'no_publish' => \$no_publish,
         'verbose|v' => \$verbose,
@@ -46 +48 @@
 pod2usage(
         -msg => "\n   Required options:\n\n".
-        "--product <init|det|diff|stack>\n".
+        "--batch <init|det|diff|stack>\n".
         "--output <path>\n".
-        "--label <label> | --expid <expid>\n" .
-        "--dvodb <path>\n\n",
+        "--label <label> or --expid <expid>\n" .
+        "--dvodb <path>\n".
+        "\n   Optional:\n\n".
+        "--product <datastoreProduct>\n".
+        "--no_publish - won't publish to datastore\n".
+        "--save_temps - will save temporary files\n".
+        "--no-op - ?\n".
+        "--no-update - will not update database\n",
         -exitval => 3
         ) unless 
-defined $product and
+defined $batchType and
 defined $output and
 defined $dvodb and
@@ -68 +76 @@
 
 close $resultsFile;
+
+#######################################################################################
+# 
+# Generates a PSPS-suitable survey 'type' from the IPP distribution group
+#
+#######################################################################################
+sub getSurveyTypeFromDistGroup {
+    my ($distGroup) = @_;
+
+    if ($distGroup eq "MD01") {return $distGroup;}
+    if ($distGroup eq "MD02") {return $distGroup;}
+    if ($distGroup eq "MD03") {return $distGroup;}
+    if ($distGroup eq "MD04") {return $distGroup;}
+    if ($distGroup eq "MD05") {return $distGroup;}
+    if ($distGroup eq "MD06") {return $distGroup;}
+    if ($distGroup eq "MD07") {return $distGroup;}
+    if ($distGroup eq "MD08") {return $distGroup;}
+    if ($distGroup eq "MD09") {return $distGroup;}
+    if ($distGroup eq "MD10") {return $distGroup;}
+    if ($distGroup eq "M31") {return $distGroup;}
+    if ($distGroup eq "sts") {return "STS";}
+    if ($distGroup eq "SweetSpot") {return "SS";}
+    if ($distGroup eq "3PI") {return $distGroup;}
+    if ($distGroup eq "ThreePi") {return "3PI";}
+    if ($distGroup eq "SAS") {return "3PI";}
+
+    print "* Do not understand distribution group: '$distGroup'\n";
+    return undef;
+}
 
 #######################################################################################
@@ -91 +128 @@
 
         # query to retrieve all exposures with the provided label
-        $query =
-            "SELECT DISTINCT rawExp.exp_id ".
-            "FROM camRun, chipRun, rawExp ".
-            "WHERE camRun.chip_id = chipRun.chip_id ".
-            "AND chipRun.exp_id = rawExp.exp_id ".
-            "AND camRun.label like '%$label%' ".
-            #    "AND rawExp.decl >= '-0.157079633' AND rawExp.decl <= '0.157079633' ". # TODO Jim's decl range
-            "ORDER BY rawExp.exp_id ASC";
+        $query = $db->prepare(<<SQL);
+
+        SELECT DISTINCT rawExp.exp_id, camRun.dist_group
+            FROM camRun, chipRun, rawExp
+            WHERE camRun.chip_id = chipRun.chip_id
+            AND chipRun.exp_id = rawExp.exp_id
+            AND camRun.label LIKE '%$label%'
+            ORDER BY rawExp.exp_id ASC
+SQL
+            #AND rawExp.exp_id > 136561
+            #AND camRun.dist_group LIKE 'sas'
+            #AND rawExp.decl >= '-0.157079633' AND rawExp.decl <= '0.157079633' Jims Dec range
     }
     else {
 
-        $query = "SELECT DISTINCT exp_id FROM rawExp WHERE rawExp.exp_id = $singleExpId"
-
-    }
-
-    if ($verbose) { print"$query\n";}
+        $query = $db->prepare(<<SQL);
+
+        SELECT DISTINCT rawExp.exp_id, dist_group FROM rawExp, chipRun WHERE chipRun.exp_id = rawExp.exp_id AND rawExp.exp_id = $singleExpId
+SQL
+    }
 
     my $batchId = 0;
@@ -114 +155 @@
     my $jobId = getJobId();
     $jobId++;
-    if ($jobId < 0) { print "Cannot find sensible jobId ($jobId)\n";
+    if ($jobId < 0) { print "* Cannot find sensible jobId ($jobId)\n";
         return 0;
     }
 
-    print "------------------------------------------------------------------------------------------\n\n";
+    print "*******************************************************************************\n*\n";
 
     my $anyBatches = 0;
 
+
+    $query->execute;
+
     # loop round exposures
-    my $expResults = $db->selectall_arrayref( $query );
-
-    for my $expRow (@$expResults) {
-
-        my ($expId) = @$expRow;
-
-        print "      *** Exposure ID = $expId ***\n";
+    while (my @row = $query->fetchrow_array()) {
+        my ($expId, $distGroup) = @row;
+
+        print "* Exposure ID = $expId with dist group = '$distGroup'\n";
+
+        my $surveyType = getSurveyTypeFromDistGroup($distGroup);
+
+        if (!$surveyType) {next;}
+        print "* Using survey type of $surveyType\n";
 
         # generate batch and job paths from job and batch IDs
@@ -137 +183 @@
         my $batchOutputPath = sprintf("$jobOutputPath/%s", $batchDir);
 
-        print "Preparing job '$job'...\n";
+        print "* Preparing exposure $expId as job '$job'...\n";
 
         # make directories
         mkdir($jobOutputPath, 0777);
         mkdir($batchOutputPath, 0777);
+        setJobId($jobId);
 
         # run IppToPsps program
-        if (runIppToPsps($db, $expId, $batchOutputPath, $product)) {
+        if (runIppToPsps($db, $expId, $batchOutputPath, $batchType)) {
 
             # write batch manifest and tar and zip up
-            if (writeBatchManifest($batchOutputPath, $batchDir, $product)) {
+            if (writeBatchManifest($batchOutputPath, $batchDir, $batchType)) {
 
                 if (tarAndZipDirectory($jobOutputPath, $batchDir)) {
@@ -161 +208 @@
             elsif ($anyBatches) {
 
-                if (writeJobManifest($output, $job)) {
+                if (writeJobManifest($output, $job, $surveyType)) {
 
                     if (!$no_publish && !publishToDatastore($output, $job)) { 
@@ -168 +215 @@
                 }
 
-                setJobId($jobId);
                 $jobId++;
                 $batchId = 0;
@@ -175 +221 @@
             else {
 
-                print "No batches to publish\n"
+                print "* No batches to publish\n"
             }
         }
 
-        if ($product eq "init" ) {print "Wrote initialisation batch\n"; last;}
-
-
-        print "------------------------------------------------------------------------------------------\n\n";
-    }
+        if ($batchType eq "init" ) {print "* Wrote initialisation batch\n"; last;}
+
+
+        print "*\n*******************************************************************************\n\n";
+    }
+
+    $db->disconnect();
 
     return 1;
@@ -204 +252 @@
     close (MYFILE);
 
-    if ($verbose) { print "Current jobId = $jobId\n"; }
+    if ($verbose) { print "* Current jobId = $jobId\n"; }
 
     return $jobId;
@@ -221 +269 @@
     close (MYFILE);
 
-    if ($verbose) { print "New jobId = $jobId\n"; }
+    if ($verbose) { print "* New jobId = $jobId\n"; }
 }
 
@@ -230 +278 @@
 #######################################################################################
 sub runIppToPsps {
-    my ($db, $expId, $batchOutputPath, $product) = @_;
+    my ($db, $expId, $batchOutputPath, $batchType) = @_;
 
     my $success = 0;
 
     # before anything else, we can publish the init batch
-    if ($product =~ /init/) {
+    if ($batchType =~ /init/) {
         $success = produceInit($batchOutputPath);
     }
-    elsif ($product =~ /det/) {
+    elsif ($batchType =~ /det/) {
         $success = produceDetections($db, $expId, $batchOutputPath);
     }
-    elsif ($product =~ /diff/) {
+    elsif ($batchType =~ /diff/) {
         $success = produceDiffs($db, $expId, $batchOutputPath);
     }
@@ -265 +313 @@
         run(command => $command, verbose => $verbose);
 
-    if (!$success) { print "Unable to tar up dir: $batch\n" and return 0 };
+    if (!$success) { print "* Unable to tar up dir: $batch\n" and return 0 };
 
     # zip
@@ -272 +320 @@
         run(command => $command, verbose => $verbose);
 
-    if (!$success) { print "Unable to gzip: $batchTar\n" and return 0 };
+    if (!$success) { print "* Unable to gzip: $batchTar\n" and return 0 };
 
     $command = "rm $batchTar";
@@ -278 +326 @@
         run(command => $command, verbose => $verbose);
 
-    if (!$success) { print "Unable to remove: $batchTar\n" };
+    if (!$success) { print "* Unable to remove: $batchTar\n" };
 
     $command = "rm -r $batchDir";
@@ -284 +332 @@
         run(command => $command, verbose => $verbose);
 
-    if (!$success) { print "Unable to remove: $batch\n"};
+    if (!$success) { print "* Unable to remove: $batch\n"};
 
     return 1;
@@ -296 +344 @@
 sub writeBatchManifest {
 
-    my ($path,$batch,$product) = @_;
+    my ($path,$batch,$batchType) = @_;
 
     use XML::Writer;
@@ -311 +359 @@
     # determine batch 'type'
     my $type;
-    if($product eq 'init') {$type = "IN";}
-    elsif($product eq 'det') {$type = "P2";}
-    elsif ($product eq 'stack') {$type = "ST";}
-    elsif ($product eq 'diff') {$type = "OB";}
+    if($batchType eq 'init') {$type = "IN";}
+    elsif($batchType eq 'det') {$type = "P2";}
+    elsif ($batchType eq 'stack') {$type = "ST";}
+    elsif ($batchType eq 'diff') {$type = "OB";}
     else {$type = "UNKNOWN";}
 
@@ -325 +373 @@
     my $maxObjId;
     my $filename;
-    if($product eq 'det') {
+    if($batchType eq 'det') {
 
         while (<MYFILE>) {
@@ -343 +391 @@
 
     # open directory to hunt for FITS files
-    opendir(DIR, $path) or print "Cannot open '$path' to write batch manifest\n" and return 0;
+    opendir(DIR, $path) or print "* Cannot open '$path' to write batch manifest\n" and return 0;
     my @thefiles= readdir(DIR);
     closedir(DIR);
@@ -353 +401 @@
         if ($f =~ /\.FITS/) {
 
-            if ($foundFits) { print "More than one FITS file found for this batch\n"; return 0;}
+            if ($foundFits) { print "* More than one FITS file found for this batch\n"; return 0;}
 
             # get md5sum of file
@@ -362 +410 @@
 
             # write manifest element TODO untidy
-            if($product eq 'det' && $filename eq $f) {
+            if($batchType eq 'det' && $filename eq $f) {
 
                 $writer->startTag('manifest',
@@ -389 +437 @@
             #$writer->startTag('md5'); $writer->characters("$md5sum"); $writer->endTag();
 
-            #if($product eq 'det' && $filename eq $f) {
+            #if($batchType eq 'det' && $filename eq $f) {
 
                 #    $writer->startTag('minObjId'); $writer->characters("$minObjId"); $writer->endTag();
@@ -404 +452 @@
 
 
-    if (!$foundFits) { print "Could not find any FITS files for this batch\n"; return 0;}
+    if (!$foundFits) { print "* Could not find any FITS files for this batch\n"; return 0;}
     return 1;
 }
@@ -414 +462 @@
 #######################################################################################
 sub writeJobManifest {
-    my ($path, $job) = @_;
+    my ($path, $job, $surveyType) = @_;
 
     use XML::Writer;
@@ -433 +481 @@
     $writer->startTag('manifest',
             "name" => "$job",
-            "type" => "3PI", # TODO survey ID
+            "type" => $surveyType,
             "timestamp" => "$timeStamp");
 
@@ -439 +487 @@
 
     # open directory to hunt for FITS files
-    opendir(DIR, $jobPath) or print "Cannot open '$path' to write job manifest\n" and return 0;
+    opendir(DIR, $jobPath) or print "* Cannot open '$path' to write job manifest\n" and return 0;
     my @thefiles= readdir(DIR);
     closedir(DIR);
@@ -472 +520 @@
     $writer->end();
 
-    if (!$foundBatch) { print "Could not find any batches for this job\n"; return 0;}
+    if (!$foundBatch) { print "* Could not find any batches for this job\n"; return 0;}
     return 1;
 }
@@ -489 +537 @@
 
     # loop through all batch files in this job directory
-    opendir(DIR, $fullJobPath) or print "Cannot open '$fullJobPath' in order to publish to datastore\n" and return 0;
+    opendir(DIR, $fullJobPath) or print "* Cannot open '$fullJobPath' in order to publish to datastore\n" and return 0;
     my @thefiles= readdir(DIR);
     closedir(DIR);
@@ -512 +560 @@
     $command .= " --datapath $fullJobPath";
     $command .= " --type IPP_PSPS";
-    $command .= " --product PSPS_test";
+    $command .= " --product $datastoreProduct";
     $command .= " --list $tempName";
 
@@ -519 +567 @@
         run(command => $command, verbose => $verbose);
 
-    if (!$success) { print "Unable to publish $job to datastore\n" and return 0 };
-
-    print "Successfully published $job to datastore\n";
+    if (!$success) { print "* Unable to publish $job to datastore\n" and return 0 };
+
+    print "* Successfully published $job to datastore\n";
 
     close($tempFile);
@@ -536 +584 @@
     my ($outputPath) = @_;
 
-    my $success = runProgram( "NULL", $outputPath, 0, $product);
+    my $success = runProgram( "NULL", $outputPath, 0, $batchType);
     return $success;
 }
@@ -558 +606 @@
         "WHERE rawExp.exp_id = $expId AND camRun.magicked";
 
-    if ($verbose) { print"$query\n";}
+    if ($verbose) { print"* $query\n";}
 
 
@@ -564 +612 @@
     my $results = $db->selectall_arrayref( $query );
     my $size = @$results;
-    if ($size < 1) {print "No smf files found for this exposure\n"; return 0;}
+    if ($size < 1) {print "* No smf files found for this exposure\n"; return 0;}
 
     # loop round db results TODO should only be one result
@@ -580 +628 @@
         close $tempFile;
 
-        $success = runProgram($tempName, $outputPath, $expId, $product);
+        $success = runProgram($tempName, $outputPath, $expId, $batchType);
         return $success;
     }
@@ -604 +652 @@
         "WHERE rawExp.exp_id = $expId AND camRun.magicked";
 
-    if ($verbose) { print"$query\n";}
+    if ($verbose) { print"* $query\n";}
 
     # execute query and check results
     my $results = $db->selectall_arrayref( $query );
     my $size = @$results;
-    if ($size < 1) {print "No cmf files found for this exposure\n"; return 0;}
+    if ($size < 1) {print "* No cmf files found for this exposure\n"; return 0;}
 
     my ($tempFile, $tempName) = tempfile( "inputFileList.XXXX", UNLINK => !$save_temps);
@@ -629 +677 @@
     }
 
-    my $ret = runProgram($tempName, $outputPath, $expId, $product);
+    my $ret = runProgram($tempName, $outputPath, $expId, $batchType);
 
     close $tempFile;
@@ -642 +690 @@
 #######################################################################################
 sub runProgram {
-    my ($input, $output, $expid, $product) = @_;
+    my ($input, $output, $expid, $batchType) = @_;
 
     # build command
@@ -651 +699 @@
     $command .= " -config config"; # TODO
     $command .= " -expid $expid";
-    $command .= " -product $product";
+    $command .= " -batch $batchType";
     $command .= " -results $resultsFileName";