Changeset 36221

Timestamp:

Oct 16, 2013, 3:49:07 PM (13 years ago)

Author:

watersc1

Message:

Functional version of the task and script that I'm running on a test pantasks now. The load step is unreasonably slow, but that may just be due to the host I"m running the pantasks_server on (ippc25).

The date based iteration seems to be working correctly.

Location:

trunk

Files:

• ippScripts/scripts/rawcheck.pl (modified) (5 diffs)
• ippTasks/rawcheck.pro (modified) (8 diffs)

trunk/ippScripts/scripts/rawcheck.pl

@@ -28 +28 @@
 GetOptions(
     'server|s=s'     => \$server,
-    'dbname'         => \$dbname,
+    'dbname=s'       => \$dbname,
     'exp_id|x=s'     => \$exp_id,
 ) || pod2usage( 2 );
@@ -36 +36 @@
 pod2usage( -msg => "Required options: --server", -exitval => 2 )
     unless $server;
+pod2usage( -msg => "Required options: --dbname", -exitval => 2 )
+    unless $dbname;
 pod2usage( -msg => "missing key", exitval => 2 )
     unless defined $exp_id;
@@ -217 +219 @@
         $quality[0] = 1;
         print "cp $good_file $files[0]\n";
+        system("cp $good_file $files[0]");
+# Begin my best validation thought
+        {
+            my $fh;
+            open($fh,$files[0]);
+            my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest;
+            close($fh);
+            if ($tmpmd5 ne $md5sum) { 
+                die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+            }
+        }
+# End my best validation thought.
+
         $Ngood = 1;  # We now hand off this single valid instance object to be handled by the Ngood=1 case.
     }
@@ -265 +280 @@
                 @$uris = map {URI->new($_)->file if $_} @$uris;
 
-                print $uris . " " . ${ $uris }[0] . " \n";
+#               print $uris . " " . ${ $uris }[0] . " \n";
                 my $fh;
                 open($fh,${ $uris }[0]);
@@ -287 +302 @@
         }
         for (my $i = 0; $i <= $#files; $i++) {
-            print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
+#           print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
             if ($quality[$i] != 1) {
                 my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];

trunk/ippTasks/rawcheck.pro

@@ -5 +5 @@
 
 # load in the local configuration (NEB_USER, etc.)
-module nebulous.site.pro
+# module nebulous.site.pro
 
 $LOGSUBDIR = $LOGDIR/rawcheck
@@ -42 +42 @@
 
 
-
-$EXP_ID_START = 0
-$EXP_ID_RANGE = 500000
+$DATEOBS_MIN  = 1970-01-01T00:00:00
 
 task           rawcheck.load
@@ -50 +48 @@
 
   # modify these after the tasks are tested
-  periods      -poll 0.5
+  periods      -poll 600
   periods      -exec 5
-  periods      -timeout 1500
+  periods      -timeout 20
   npending     1
 
@@ -66 +64 @@
      end      
      
-     $run = regtool -processedexp -state compressed 
+     $run = regtool -processedexp -state compressed -dateobs_begin $DATEOBS_MIN 
      if ($DB:n == 0)
       option DEFAULT
@@ -77 +75 @@
     end
     add_poll_args run
-    add_poll_labels run
+#    add_poll_labels run
     command $run
   end
@@ -85 +83 @@
     # convert 'stdout' to book format
     ipptool2book stdout rawcheckPending -key exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-        if ($VERBOSE > 2)
+    if ($VERBOSE > 2)
       book listbook rawcheckPending
     end
+
+      book listbook rawcheckPending
+    book npages rawcheckPending -var N
+    book getpage rawcheckPending N -var pageName
+    book getword rawcheckPending $pageName dateobs -var DATEOBS_MIN
 
     # delete existing entries in the appropriate pantaskStates
@@ -112 +115 @@
   periods      -poll 0.5
   periods      -exec 5
-  periods      -timeout 600
+  periods      -timeout 1200
+  host         anyhost
+
 
   task.exec
@@ -136 +141 @@
 
     # these operations do not require a database to be specified
-    $run = neb-insck --dbname $DBNAME --exp_id $EXP_ID
+    $run = rawcheck.pl --dbname $DBNAME --exp_id $EXP_ID
 
     # save the pageName for future reference below