Changeset 29093

Timestamp:

Sep 1, 2010, 12:33:59 PM (16 years ago)

Author:

rhenders

Message:

Neatened code and now deletes from file system when a batch has reached a state of 'merged'

File:

• trunk/ippToPsps/perl/checkOdmStatus.pl (modified) (10 diffs)

trunk/ippToPsps/perl/checkOdmStatus.pl

@@ -5 +5 @@
 
 use LWP::UserAgent;
+use IPC::Cmd 0.36 qw( can_run run );
+use IPC::Cmd 0.36 qw( can_run run );
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use XML::LibXML;
@@ -17 +19 @@
 my $toTime = undef;
 my $product = undef;
+my $filePath = undef;
 
 
@@ -24 +27 @@
         'to|t=s' => \$toTime,
         'product|p=s' => \$product,
+        'location|l=s' => \$filePath,
         'verbose|v' => \$verbose,
         'save_temps|s' => \$save_temps
@@ -41 +45 @@
     $toTime = "2099-12-31";
     print "* OPTIONAL: to time                         -t <dateTime>        (default = $toTime)\n";
+}
+if (!defined $filePath) {
+    print "* OPTIONAL: location for files to be deleted    -l <path>\n";
 }
 if (!defined $verbose) {
@@ -72 +79 @@
     my $numOfBatches;
 
-    if (defined $singleBatch ) { $numOfBatches = $ippToPspsDb->getSingleBatchOnDatastore($singleBatch, \$batches);}
-    else { $numOfBatches = $ippToPspsDb->getListOfBatchesOnDatastore(\$batches, $fromTime, $toTime);}
+    if (defined $singleBatch ) { $numOfBatches = $ippToPspsDb->getBatch($singleBatch, \$batches);}
+    else { $numOfBatches = $ippToPspsDb->getBatches(\$batches, $fromTime, $toTime);}
 
     if ($numOfBatches < 1) {return 0;}
@@ -84 +91 @@
     # loop round batches
     my $batch;
+    my $numBatchesToCheck = 0;
     my $numChecked = 0;
+    my $numRemovedFromDatastore = 0;
+    my $numDeleted = 0;
     foreach $batch ( @{$batches} ) {
-        my ($timestamp, $expId, $batchId, $surveyType, $deleted) =  @{$batch};
-
-        if (checkBatch($timestamp, $expId, $batchId, $surveyType, $deleted)) {$numChecked++;}
-
+        my ($timestamp, $expId, $batchId, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged) =  @{$batch};
+
+        if (!$processed) {next;}
+
+        $numBatchesToCheck++;
+
+        my $batchName = getBatchName($batchId);
+
+        # if not merged then update by polling ODM for status
+        if (!$merged) {
+
+            if (checkODM($batchName, \$loadedToOdm, \$mergeWorthy, \$merged)) {$numChecked++;}
+        }
+
+        # delete from datastore
+        if (defined $product && !$deleted && $loadedToOdm && $mergeWorthy) {
+
+            $deleted = $datastore->remove($batchName);
+            if ($deleted) {
+                $ippToPspsDb->setBatchAsDeleted($batchId, $expId);
+                $numRemovedFromDatastore++;
+            }
+        }
+
+        # if merged and already removed from datastore then delete files
+        if (defined $filePath && $merged && $deleted) {
+
+            print "$command\n";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            if (!$success) {print "* Unable to run: $command\n";}
+            else {$numDeleted++;}
+        }
+
+        # update database
+        $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $mergeWorthy, $merged, $deleted);
+        printf( "| %18s  | %11s  | %10d   |    %6s      |    %6s     | %6s  |  %5s   |\n", 
+                $timestamp, 
+                $batchName, 
+                $expId, 
+                $loadedToOdm ? "yes" : "no", 
+                $mergeWorthy ? "yes" : "no", 
+                $merged ? "yes" : "no",
+                $deleted ? "yes" : "no");
     }
+
     printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n");
 
-    printf( "* Successfully checked %d batch%s out of %d\n", $numChecked, ($numChecked==1) ? "" : "es", $numOfBatches);
-
-}
-
-######################################################################################y
+    printf( "* Successfully checked %d batch%s out of %d\n", $numChecked, ($numChecked==1) ? "" : "es", $numBatchesToCheck);
+    printf( "* Successfully removed %d batch%s from the datastore\n", $numRemovedFromDatastore, ($numRemovedFromDatastore==1) ? "" : "es");
+    printf( "* Successfully deleted %d batch%s from local file system\n", $numDeleted, ($numDeleted==1) ? "" : "es");
+
+}
+
+########################################################################################
+# 
+# Returns batch name from batch_id TODO needs to be a batch class
+# 
+########################################################################################
+sub getBatchName {
+    my ($batchId) = @_;
+
+    return sprintf("B%08d", $batchId);
+}
+
+########################################################################################
 # 
 # Check a single batch
 # 
 ########################################################################################
-sub checkBatch {
-    my ($timestamp, $expId, $batchId, $surveyType, $deleted) = @_;
-
-
-    my $batchFilter = sprintf("B%08d", $batchId);
+sub checkODM {
+    my ($batchName, $loadedToOdm, $mergeWorthy, $merged) = @_;
+
     my $statusFilter = "*";
 
     my $response = $ua->post($odmUrl,
-            [batchNameFilter => "%".$batchFilter,
+            [batchNameFilter => "%".$batchName,
             statusFilter => $statusFilter,
             fromFilter => "2010-01-01",
@@ -119 +181 @@
     if ($response->code != 200) {
 
-        print "Problem connecting to web service for '$batchFilter', HTTP response status: ".$response->status_line."\n";
-        return;
+        print "Problem connecting to web service for '$batchName', HTTP response status: ".$response->status_line."\n";
+        return 0;
     }
     #        print( "HTTP response status: ".$response->content."\n" );
@@ -128 +190 @@
     close($tempFile);
 
-    my $loadedToOdm = 0;
-    my $mergeWorthy = 0;
-    my $mergeCompleted = 0;
-
-    parseXml($tempName, \$loadedToOdm, \$mergeWorthy, \$mergeCompleted);
-
-    # delete from datastore
-    if(defined $product) {
-    
-        if (!$deleted && $loadedToOdm && $mergeWorthy) {
-        
-            $deleted = $datastore->remove($batchFilter);
-            if ($deleted) {
-                $ippToPspsDb->setBatchAsDeleted($batchId, $expId);
-            }
-        }
-    }
-    
-    printf( "| %18s  | %11s  | %10d   |    %6s      |    %6s     | %6s  |  %5s   |\n", 
-            $timestamp, 
-            $batchFilter, 
-            $expId, 
-            $loadedToOdm ? "yes" : "no", 
-            $mergeWorthy ? "yes" : "no", 
-            $mergeCompleted ? "yes" : "no",
-            $deleted ? "yes" : "no");
-
-    $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $mergeWorthy, $mergeCompleted, $deleted);
-
+    ${$loadedToOdm} = 0;
+    ${$mergeWorthy} = 0;
+    ${$merged} = 0;
+
+    parseXml($tempName, $loadedToOdm, $mergeWorthy, $merged);
+
+    return 1;
 }
 
@@ -165 +205 @@
 ########################################################################################
 sub parseXml {
-    my ($xmlFile, $loadedToOdm, $mergeWorthy, $mergeCompleted) = @_;
+    my ($xmlFile, $loadedToOdm, $mergeWorthy, $merged) = @_;
 
     my $parser = XML::LibXML->new;
@@ -174 +214 @@
     ${$loadedToOdm} = 0;
     ${$mergeWorthy} = 0;
-    ${$mergeCompleted} = 0;
+    ${$merged} = 0;
 
     if ($result =~ m/LoadStarted/) { ${$loadedToOdm} = 1;}
     if ($result =~ m/MergeWorthy/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1;}
-    if ($result =~ m/Merge[1-9]Completed/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1; ${$mergeCompleted} = 1;}
-}
-
+    if ($result =~ m/Merge[1-9]Completed/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1; ${$merged} = 1;}
+}
+