Changeset 28661

Timestamp:

Jul 12, 2010, 4:46:07 PM (16 years ago)

Author:

rhenders

Message:

Big changes to support Conrad's new DXLayer, among other things

File:

• trunk/ippToPsps/scripts/ippToPsps_run.pl (modified) (31 diffs)

trunk/ippToPsps/scripts/ippToPsps_run.pl

@@ -13 +13 @@
 
 # globals
-my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $survey, $singleExpId, $no_publish, $force, $initBatch);
+my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $survey, $singleExpId, $force, $initBatch, $help);
 
 # default values for certain globals
@@ -20 +20 @@
 $no_op = 0;
 $no_update = 0;
-$no_publish = 0;
 $camera = 'GPC1';
 $output = undef;
 $singleExpId = undef;
-$datastoreProduct = "PSPS_test";
+$datastoreProduct = undef;
 $force = 0;
 $initBatch = 0;
+$help = 0;
 
 # get user args
@@ -36 +36 @@
         'expid|e=s' => \$singleExpId,
         'product|p=s' => \$datastoreProduct,
-        'no_publish' => \$no_publish,
         'verbose|v' => \$verbose,
         'save_temps' => \$save_temps,
@@ -42 +41 @@
         'no-update' => \$no_update,
         'force|f' => \$force,
-        ) or pod2usage( 2 );
-
-pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-
-# tell off user for not providing proper args
-pod2usage(
-        -msg => "\n   Required options:\n\n".
-        "--batch <init|det|diff|stack>\n".
-        "--output <path>\n".
-        "--survey <ThreePi|MD01,2 etc|STS|SAS|SweetSpot> or --expid <expid>\n" .
-        "--dvodb <path>\n".
-        "\n   Optional:\n\n".
-        "--product <datastoreProduct>\n".
-        "--no_publish - won't publish to datastore\n".
-        "--save_temps - will save temporary files\n".
-        "--no-op - ?\n".
-        "--no-update - will not update database\n",
-        -exitval => 3
-        ) unless 
-defined $batchType and
-defined $output and
-defined $dvodb and
-(( defined $survey or defined $singleExpId ) or ( $batchType eq "init"));
-
-if ($batchType eq "init") {$initBatch = 1;}
+        'help|h' => \$help,
+        );
+
+if ($help) {printUsage(); exit;}
+if (@ARGV) {print "* Unknown option: @ARGV\n"; printUsage(); exit;}
+
+my $quit = 0;
+print "\n";
+if (!defined $batchType) {print "* ERROR: need to define a batch type (-b)\n"; $quit=1;}
+if (!defined $output) {print "* ERROR: need to provide an output path (-o)\n"; $quit=1;}
+if (!defined $dvodb) {print "* ERROR: need to provide a DVO Db path (-d)\n"; $quit=1;}
+if (!defined $survey and !defined $singleExpId) {print "* ERROR: need to provide a survey type (-s) or an exposure ID (-e)\n"; $quit=1;}
+print "\n\n";
+
+if ($quit) { exit; }
+
 
 # check we can run programs and get camera config
@@ -74 +64 @@
 
 # make a temporary file for saving results
-my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps);
-
+
+if ($batchType eq "init") {$initBatch = 1;}
 process();
 
-close $resultsFile;
+
+#######################################################################################
+# 
+# Generates a PSPS-suitable survey 'type' from the IPP distribution group
+#
+#######################################################################################
+sub printUsage {
+
+    print "\n" . 
+        "  *** Required options:\n".
+        "\n" .
+        " --batch         <init|det|diff|stack>\n".
+        " --output        <path>\n".
+        " --survey        <ThreePi|STS|SAS|SweetSpot|MD01|MD02|MD03 etc>\n" .
+        " --expid         <expid>\n" .
+        " --dvodb         <path>\n".
+        "\n" .
+        "    *** Optional:\n".
+        "\n" .
+        " --product       <datastoreProduct> if not provided, data will NOT be published\n".
+        " --save_temps    will save temporary files\n".
+        " --no-op         ?\n".
+        " --no-update     will not update database\n" .
+        " --force         will force an already processed exposure\n" .
+        " --help          show this message\n\n";
+
+}
 
 #######################################################################################
@@ -128 +144 @@
     print "*******************************************************************************\n*\n";
 
+    my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps);
     my $gpc1Db = connectToDb("gpc1", "ippdb01", "ippuser", "ippuser") or return 0;
     my $ippToPspsDb = connectToDb("ippToPsps", "ippdb01", "ipp", "ipp") or return 0;
@@ -160 +177 @@
             AND chipRun.exp_id = rawExp.exp_id 
             AND camRun.dist_group = '$survey'   
-            AND rawExp.dateobs >= '2010-02-20'
             GROUP BY  rawExp.exp_id 
             ORDER BY rawExp.exp_id ASC;
 SQL
 
+            #AND rawExp.exp_id > 133887
+            #AND filter = 'r.00000'
             #AND rawExp.dateobs <= '2010-03-12'
             #AND rawExp.exp_id > $lastExpId
@@ -173 +191 @@
     }
 
+# TODO check if there are no exposures and give a warning
+
     my $batchId = 0;
     my $batchesPerJob = 1;
-    my $anyBatches = 0;
     my $published = 0;
 
     $query->execute;
 
-    my $jobId = -1;
+    #my $batchId = -1;
 
     # loop round exposures
@@ -195 +214 @@
         if (!$surveyType) {next;}
 
-        $jobId = getNewBatchId($ippToPspsDb, $expId, $surveyType);
-
-        # TODO quit here if no sensible job ID
-
-        # generate batch and job paths from job and batch IDs
-        my $job = sprintf("J%06d", $jobId);
-        my $jobOutputPath = sprintf("$output/$job");
-        my $batchDir = sprintf("B%03d", $batchId);
-        my $batchOutputPath = sprintf("$jobOutputPath/%s", $batchDir);
-        mkdir($jobOutputPath, 0777);
-        mkdir($batchOutputPath, 0777);
-
-        print "* Preparing exposure $expId as job '$job'...\n";
+        $batchId = getNewBatchId($ippToPspsDb, $expId, $surveyType);
+
+        # TODO quit here if no sensible batch ID
+
+        # generate batch path from batch IDs
+        my $batch = sprintf("B%08d", $batchId);
+        my $batchDir = sprintf("$output/$batch");
+        mkdir($batchDir, 0777);
+        $published = 0;
+
+        print "* Preparing exposure $expId as batch '$batch'...\n";
 
         # run IppToPsps program
-        if (runIppToPsps($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath, $batchType)) {
+        if (runIppToPsps($gpc1Db, $expId, $expName, $surveyType, $batchDir, $batchType, $resultsFilePath)) {
+
+            my ($filename, $minObjId, $maxObjId, $totalDetections);
+            parseResults($resultsFilePath, \$filename, \$minObjId, \$maxObjId, \$totalDetections);
 
             # write batch manifest and tar and zip up
-            if (writeBatchManifest($batchOutputPath, $batchDir, $batchType)) {
-
-                if (tarAndZipDirectory($jobOutputPath, $batchDir)) {
-
-                    $anyBatches = 1
-                }
+            if (writeBatchManifest($batchDir, $batch, $batchType, $surveyType, $filename, $minObjId, $maxObjId)) {
+
+                my $tarball = tarAndZipBatch($output, $batch);
+                if ($tarball && $datastoreProduct && publishToDatastore($batch, $output, $tarball)) {$published = 1;}
             }
 
-            if (($batchId+1) < $batchesPerJob) {
-                $batchId++;
-            }
-            # job is complete, now publish
-            elsif ($anyBatches) {
-
-                $published = 0;
-                if (writeJobManifest($output, $job, $surveyType)) {
-
-                    if (!$no_publish && publishToDatastore($output, $job)) { 
-                        $published = 1;
-                    }
-                }
-
-                updateDb($ippToPspsDb, $jobId, $expId, 1, $published);
-
-                $batchId = 0;
-                $anyBatches = 0;
-            }
-            else {
-
-                print "* No batches to publish\n";
-            }
+            updateDb($ippToPspsDb, $batchId, $expId, 1, $published, $totalDetections);
         }
 
@@ -253 +249 @@
     $gpc1Db->disconnect();
     $ippToPspsDb->disconnect();
+    close $resultsFile;
 
     return 1;
@@ -289 +286 @@
 ########################################################################################
 sub updateDb {
-    my ($db, $batchId, $expId, $processed, $published) = @_;
+    my ($db, $batchId, $expId, $processed, $published, $totalDetections) = @_;
 
     my $query = $db->prepare(<<SQL);
 
     UPDATE batches 
-        SET processed = $processed, on_datastore = $published 
+        SET processed = $processed, on_datastore = $published, total_detections = $totalDetections 
         WHERE batch_id = $batchId 
         AND exp_id = $expId;
@@ -330 +327 @@
 #######################################################################################
 #
-# Generates a new record in the database with a new jobId 
+# Generates a new record in the database with a new batch ID 
 #
 ########################################################################################
@@ -371 +368 @@
 #######################################################################################
 sub runIppToPsps {
-    my ($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath, $batchType) = @_;
+    my ($gpc1Db, $expId, $expName, $surveyType, $batchDir, $batchType, $resultsFilePath) = @_;
 
     my $success = 0;
 
     if ($batchType =~ /init/) {
-        $success = produceInit($batchOutputPath);
+        $success = produceInit($batchDir, $resultsFilePath);
     }
     elsif ($batchType =~ /det/) {
-        $success = produceDetections($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath);
+        $success = produceDetections($gpc1Db, $expId, $expName, $surveyType, $batchDir, $resultsFilePath);
     }
     elsif ($batchType =~ /diff/) {
-        $success = produceDiffs($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath);
+        $success = produceDiffs($gpc1Db, $expId, $expName, $surveyType, $batchDir, $resultsFilePath);
     }
 
@@ -390 +387 @@
 #######################################################################################
 #
-# tar 'n zip up a directory TODO handle errors
-#
-#######################################################################################
-sub tarAndZipDirectory {
-    my ($jobOutputPath,$batch) = @_;
-
-    my $batchDir = "$jobOutputPath/$batch";
-    my $batchTar = "$batchDir.tar";
-    my $batchZip = "$batchTar.gz";
+# tar 'n zip a batch TODO handle errors
+#
+#######################################################################################
+sub tarAndZipBatch {
+    my ($path,$batch) = @_;
+
+    my $batchDir = "$path/$batch";
+    my $tar = "$batch.tar";
+    my $tarball = "$tar.gz";
+    my $batchTar = "$path/$tar";
+    my $batchZip = "$path/$tarball";
 
     # tar
-    my $command = "tar -cf $batchTar -C $jobOutputPath $batch";
+    my $command = "tar -cvf $batchTar -C $path $batch";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-
-    if (!$success) { print "* Unable to tar up dir: $batch\n" and return 0 };
+    if (!$success) { print "* Unable to tar up contents of dir: $batchDir\n" and return undef; }
 
     # zip
@@ -411 +409 @@
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-
-    if (!$success) { print "* Unable to gzip: $batchTar\n" and return 0 };
-
+    if (!$success) { print "* Unable to gzip: $batchTar\n" and return undef; }
+
+    # remove tar file
     $command = "rm $batchTar";
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-
     if (!$success) { print "* Unable to remove: $batchTar\n" };
 
+    # remove batch dir
     $command = "rm -r $batchDir";
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-
     if (!$success) { print "* Unable to remove: $batch\n"};
 
-    return 1;
-}
+    return $tarball;
+}
+
+#######################################################################################
+#
+# zip a file then delete it 
+#
+#######################################################################################
+sub zipFile {
+    my ($path,$origFile) = @_;
+
+    my $file;
+
+    # zip
+    my $zippedName =  "$origFile.gz";
+    my $command = "gzip -c $path/$origFile  > $path/$zippedName";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
+        run(command => $command, verbose => $verbose);
+
+    if (!$success) { 
+
+        $file = $origFile;
+        print "* Unable to gzip: $path/$origFile\n"; 
+    }
+    else {
+
+        $file = $zippedName;
+
+        # now delete original
+        $command = "rm -r $path/$origFile";
+        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+
+        if (!$success) { print "* Unable to remove: $path/$origFile\n"};
+    }
+
+    return $file;
+}
+
+
+#######################################################################################
+#
+# parse results XML
+#
+#######################################################################################
+sub parseResults {
+    my ($resultsFilePath, $filename, $minObjId, $maxObjId, $totalDetections) = @_;
+
+    my $parser = XML::LibXML->new;
+    my $doc = $parser->parse_file($resultsFilePath);
+    ${$filename} = $doc->findvalue('//filename');
+    ${$minObjId} = $doc->findvalue('//minObjID');
+    ${$maxObjId} = $doc->findvalue('//maxObjID');
+    ${$totalDetections} = $doc->findvalue('//totalDetections');
+}
+
 
 #######################################################################################
@@ -435 +486 @@
 #######################################################################################
 sub writeBatchManifest {
-
-    my ($path,$batch,$batchType) = @_;
+    my ($path,$batch,$batchType,$surveyType,$filename,$minObjId,$maxObjId) = @_;
 
     use XML::Writer;
@@ -451 +501 @@
     # determine batch 'type'
     my $type;
-    if($batchType eq 'init') {$type = "IN";}
-    elsif($batchType eq 'det') {$type = "P2";}
+    if ($batchType eq 'init') {$type = "IN";}
+    elsif ($batchType eq 'det') {$type = "P2";}
     elsif ($batchType eq 'stack') {$type = "ST";}
     elsif ($batchType eq 'diff') {$type = "OB";}
     else {$type = "UNKNOWN";}
-
-    # read results
-    my $parser = XML::LibXML->new;
-    my $doc = $parser->parse_file($resultsFilePath);
-    my $filename = $doc->findvalue('//filename');
-    my $minObjId = $doc->findvalue('//minObjID');
-    my $maxObjId = $doc->findvalue('//maxObjID');
 
     # create XML file
@@ -475 +518 @@
 
     my $foundFits = 0;
-
     foreach my $f (@thefiles) {
 
@@ -481 +523 @@
 
             if ($foundFits) { print "* More than one FITS file found for this batch\n"; return 0;}
+
+            # my $file = zipFile($path, $f); 
 
             # get md5sum of file
@@ -494 +538 @@
                         "name" => "$batch",
                         "type" => $type,
+                        "survey" => $surveyType,
                         "timestamp" => "$timeStamp",
                         "minObjId" => $minObjId,
@@ -511 +556 @@
                     "md5" => $md5sum);
 
-            #$writer->startTag('file' );
-            #$writer->startTag('name'); $writer->characters("$f"); $writer->endTag();
-            #$writer->startTag('bytes'); $writer->characters("$filesize"); $writer->endTag();
-            #$writer->startTag('md5'); $writer->characters("$md5sum"); $writer->endTag();
-
-            #if($batchType eq 'det' && $filename eq $f) {
-
-                #    $writer->startTag('minObjId'); $writer->characters("$minObjId"); $writer->endTag();
-                #    $writer->startTag('maxObjId'); $writer->characters("$maxObjId"); $writer->endTag();
-                #}
             $writer->endTag();
-            # finish up
             $writer->endTag();
             $writer->end();
@@ -530 +564 @@
     }
 
-
     if (!$foundFits) { print "* Could not find any FITS files for this batch\n"; return 0;}
     return 1;
@@ -537 +570 @@
 #######################################################################################
 #
-# write job manifest
-#
-#######################################################################################
-sub writeJobManifest {
-    my ($path, $job, $surveyType) = @_;
-
-    use XML::Writer;
-    use Digest::MD5::File qw( file_md5_hex );
-
-    # sort out paths
-    my $outputPath = "$path/$job/JobManifest.xml";
-    my $output = new IO::File(">$outputPath");
-
-    # create a timestamp
-    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
-    my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec;
-
-    # create XML file
-    my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2);
-    $writer->xmlDecl('UTF-8');
-    $writer->doctype('manifest', "", "psps-manifest.dtd");
-    $writer->startTag('manifest',
-            "name" => "$job",
-            "type" => $surveyType,
-            "timestamp" => "$timeStamp");
-
-    my $jobPath = "$path/$job";
-
-    # open directory to hunt for FITS files
-    opendir(DIR, $jobPath) or print "* Cannot open '$path' to write job manifest\n" and return 0;
-    my @thefiles= readdir(DIR);
-    closedir(DIR);
-
-    my $foundBatch = 0;
-
-    foreach my $f (@thefiles) {
-
-        if (($f =~ /\.tar\.gz/)) {
-
-            my $filePath = "$jobPath/$f";
-
-            # get md5sum of file
-            my $md5sum  = file_md5_hex($filePath);
-
-            # get size of file in bytes
-            my $filesize = -s $filePath;
-
-            # write the tag for this batch
-            $writer->startTag('file',
-                    "name" => $f,
-                    "bytes" => "$filesize",
-                    "md5" => "$md5sum",
-                    );
-            $writer->endTag();
-            $foundBatch = 1;
-        }
-    }
-
-    # finish up
-    $writer->endTag();
-    $writer->end();
-
-    if (!$foundBatch) { print "* Could not find any batches for this job\n"; return 0;}
-    return 1;
-}
-
-#######################################################################################
-#
 # register new job with the datastore
 #
 ########################################################################################
 sub publishToDatastore {
-    my ($path, $job) = @_;
-
-    my $fullJobPath = "$path/$job";
+    my ($batch, $path, $tarball) = @_;
 
     my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_dsregList.XXXX", UNLINK => !$save_temps);
 
-    # loop through all batch files in this job directory
-    opendir(DIR, $fullJobPath) or print "* Cannot open '$fullJobPath' in order to publish to datastore\n" and return 0;
-    my @thefiles= readdir(DIR);
-    closedir(DIR);
-
-    my $foundBatch = 0;
-
-    foreach my $f (@thefiles) {
-
-        # get all batch tgz files
-        if ($f =~ /\.tar.gz/) {print $tempFile $f . '|||tgz' . "\n"; $foundBatch = 1;}
-
-        # get JobManifest XML file
-        elsif ($f =~ /\.xml/) {print $tempFile $f . '|||xml' . "\n";}
-    }
-
-    if (!$foundBatch) { print "Could not find any batches to publish\n"; return 0;}
+    print $tempFile $tarball . '|||tgz' . "\n";
 
     # build dsreg command command
     my $command  = "$dsreg";
-    $command .= " --add $job";
+    $command .= " --add $batch";
     $command .= " --copy";
-    $command .= " --datapath $fullJobPath";
+    $command .= " --datapath $path";
     $command .= " --type IPP_PSPS";
     $command .= " --product $datastoreProduct";
@@ -646 +593 @@
         run(command => $command, verbose => $verbose);
 
-    if (!$success) { print "* Unable to publish $job to datastore\n" and return 0 };
-
-    print "* Successfully published $job to datastore\n";
+    if (!$success) { print "* Unable to publish $tarball to datastore\n" and return 0 };
+
+    print "* Successfully published $tarball to datastore\n";
 
     close($tempFile);
@@ -661 +608 @@
 #######################################################################################
 sub produceInit {
-    my ($outputPath) = @_;
-
-    my $success = runProgram("NULL", $outputPath, 0, "NULL", "NULL", $batchType);
+    my ($outputPath, $resultsFilePath) = @_;
+
+    my $success = runProgram("NULL", $outputPath, 0, "NULL", "NULL", $batchType, $resultsFilePath);
     return $success;
 }
+
 #######################################################################################
 # 
@@ -672 +620 @@
 #######################################################################################
 sub produceDetections {
-    my ($gpc1Db, $expId, $expName, $surveyType, $outputPath) = @_;
+    my ($gpc1Db, $expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_;
 
     # query to retrieve nebulous key of camera smf file for this exposure
@@ -708 +656 @@
         close $tempFile;
 
-        $success = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType);
+        $success = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath);
         return $success;
     }
@@ -719 +667 @@
 #######################################################################################
 sub produceDiffs {
-    my ($gpc1Db, $expId, $expName, $surveyType, $outputPath) = @_;
+    my ($gpc1Db, $expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_;
 
     my $query = 
@@ -757 +705 @@
     }
 
-    my $ret = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType);
+    my $ret = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath);
 
     close $tempFile;
@@ -770 +718 @@
 #######################################################################################
 sub runProgram {
-    my ($input, $output, $expid, $expName, $surveyType, $batchType) = @_;
+    my ($input, $output, $expid, $expName, $surveyType, $batchType, $resultsFilePath) = @_;
 
     # build command
@@ -790 +738 @@
     return $success;
 }
-
-
-