Changeset 29256

Timestamp:

Sep 28, 2010, 6:13:48 PM (16 years ago)

Author:

watersc1

Message:

OSS implementation.

Also: burntool table auto-replicate I thought I already merged.

Location:

tags/ipp-20100823

Files:

• ippScripts/scripts/automate_stacks.pl (modified) (9 diffs)
• ippScripts/scripts/ipp_apply_burntool.pl (modified) (2 diffs)
• ippconfig/recipes/nightly_science.config (modified) (2 diffs)

tags/ipp-20100823/ippScripts/scripts/automate_stacks.pl

@@ -129 +129 @@
 my @target_list = ();
 my @filter_list = ();
+my %distribution_list = ();
 my %tessID_list = ();
 my %obsmode_list = ();
@@ -135 +136 @@
 my %cleanmods_list = ();
 my %stackable_list = ();
+my %extra_processing = ();
 my %reduction_class = ();
 my %macro_formats = ();
@@ -206 +208 @@
                 $this_target = ${ $tentry }{value};
                 push @target_list, $this_target;
+                $distribution_list{$this_target} = $this_target;
+            }
+            elsif (${ $tentry }{name} eq 'DISTRIBUTION') {
+                $distribution_list{$this_target} = ${ $tentry }{value};
             }
             elsif (${ $tentry }{name} eq 'TESS') {
@@ -222 +228 @@
                 $stackable_list{$this_target} = ${ $tentry }{value};
             }
+            elsif (${ $tentry }{name} eq 'EXTRA_PROCESSING') {
+                $extra_processing{$this_target} = ${ $tentry }{value};
+            }
             elsif (${ $tentry }{name} eq 'REDUCTION') {
                 $reduction_class{$this_target} = ${ $tentry }{value};
@@ -1183 +1192 @@
         &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     }
+    
+    
     return(0);
 }
@@ -1190 +1201 @@
     my $pretend = shift;
     foreach my $target (@target_list) {
-        if ($stackable_list{$target} == 1) {
-            foreach my $filter (@filter_list) {
+        foreach my $filter (@filter_list) {
+            if (exists($extra_processing{$target})) {
+                do_extra_processing($date,$target,$filter,$pretend);
+            }
+            if ($stackable_list{$target} == 1) {
                 my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
                 if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
@@ -1231 +1245 @@
                 }
             }
+            else {
+                # print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
+            }
         }
-        else {
-            # print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
-        }
-    }
-
+    }
 }
 
@@ -1291 +1304 @@
 
 #
+# Extra processing
+################################################################################
+
+sub do_extra_processing {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+    my $pretend = shift;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    if ($target eq 'OSS') {
+        my $db = init_gpc_db();
+
+        my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+        $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
+        print STDERR "$obj_sth\n";
+        my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+        foreach my $object_row (@{ $object_ref }) {
+            my $this_object = shift @{ $object_row };
+            my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+            $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+            $input_sth .= " ORDER BY dateobs ";
+
+            my $warps = $db->selectall_arrayref( $input_sth );
+            
+            if (($#{ $warps } + 1) % 2 != 0) {
+                print STDERR "Number of input warps to make OSS diffs is not even! $#{ $warps }\n";
+                die;
+            }
+            
+            while ($#{ $warps } > -1) {
+                my $input_warp = shift @{ $warps };
+                my $template_warp = shift @{ $warps };
+                my $input_exp_id = ${ $input_warp }[0];
+                my $template_exp_id = ${ $template_warp }[0];
+                
+                my $cmd = "$difftool  -definewarpwarp  -input_label $label  -template_label $label  -timediff 3600  -set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $data_group ";
+                $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
+                $cmd .= " -pretend ";
+                if (defined($pretend)) {
+                    $cmd .= ' -pretend ';
+                }
+                if ($debug == 1) {
+                    $cmd .= ' -pretend ';
+                }
+                print STDERR "EXTRA_PROCESSING: $cmd\n";
+                if (($debug == 0)&&(!defined($pretend))) {
+                    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run ( command => $cmd, verbose => $verbose );
+                    unless ($success) {
+                        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                        &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+                    }
+                }
+            }
+        }
+    }
+}
+
+            
+
+
+#
 # Auto-Clean
 ################################################################################
@@ -1380 +1457 @@
     my $object   = $object_list{$target};
     my $comment = $comment_list{$target};
-    my $dist_group = $target;
+    my $dist_group = $distribution_list{$target};
     my $data_group = "${target}.${trunc_date}";
     my $tess_id = $tessID_list{$target};

tags/ipp-20100823/ippScripts/scripts/ipp_apply_burntool.pl

@@ -55 +55 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $nebXattr = can_run('neb-xattr') or (warn "Can't find neb-xattr" and $missing_tools = 1);
+my $nebreplicate = can_run('neb-replicate') or (warn "Can't find neb-replicate" and $missing_tools = 1);
 #my $fpack    = can_run('fpack')    or (warn "Can't find fpack" and $missing_tools = 1);
 if ($missing_tools) {
@@ -214 +215 @@
 
         $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN);
+        $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN);
         $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState", $REALRUN);
         if ($status) {

tags/ipp-20100823/ippconfig/recipes/nightly_science.config

@@ -37 +37 @@
 FILTERS STR z.00000
 FILTERS STR y.00000
+FILTERS STR w.00000
 
 UNRECOVERABLE_QUALITY MULTI
@@ -234 +235 @@
 END
 TARGETS METADATA
+  NAME STR OSS
+  DISTRIBUTION STR SweetSpot
+  TESS STR RINGS.V0
+  OBSMODE STR OSS
+  STACKABLE BOOL TRUE
+  EXTRA_PROCESSING BOOL TRUE
+END
+TARGETS METADATA
   NAME STR ThreePi
   TESS STR RINGS.V0