Changeset 37981

Timestamp:

Mar 17, 2015, 2:55:55 PM (11 years ago)

Author:

watersc1

Message:

changes to enable the LAP W stacks to be constructed at LANL

Location:

tags/ipp-pv3-20140717

Files:

• ippScripts/scripts/lap_science.pl (modified) (2 diffs)
• ippScripts/scripts/sc_prepare_run.pl (modified) (3 diffs)
• ippScripts/scripts/sc_prepare_warp.pl (modified) (1 diff)
• ippTools/share/stacktool_definebyquery_insert_random_part2.sql (modified) (1 diff)

tags/ipp-pv3-20140717/ippScripts/scripts/lap_science.pl

@@ -1049 +1049 @@
     $command .= " -dbname $dbname " if defined $dbname;
     $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter ";
+    $command .= " -select_fwhm_major_max 10.0 -select_zpt_obs_min 25.85 -select_bg_max 425 ";
+    $command .= " -select_good_frac_min 0.20 ";
 #    $command .= " -select_label $label "; # Removed to allow exposure sharing to work
     $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group ";
-    $command .= " -min_num ${minimum_stack_inputs} -max_num ${maximum_stack_inputs} -set_reduction THREEPI_STACK_1DG -set_dist_group $exposure->{dist_group} ";
+    $command .= " -min_num ${minimum_stack_inputs} -random 100 -set_reduction THREEPI_STACK_1DG -set_dist_group $exposure->{dist_group} ";
     $command .= " $warps ";
 
@@ -1064 +1066 @@
 
     ##
-    ## Queue the "deep" stacks that will run locally.
-    my $deep_minimum = $maximum_stack_inputs + 1;
-    if ($deep_minimum < 2) { $deep_minimum = 2; }
-    $command = "$stacktool ";
-    $command .= " -pretend " if defined $debug;
-    $command .= " -dbname $dbname " if defined $dbname;
-    $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter ";
-#    $command .= " -select_label $label "; # Removed to allow exposure sharing to work
-    $command .= " -set_label ${label}.local -set_workdir $workdir -set_data_group $data_group ";
-    $command .= " -min_num ${deep_minimum} -set_reduction THREEPI_STACK_1DG -set_dist_group $exposure->{dist_group} ";
-    $command .= " $warps ";
-
-    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform stacktool -definebyquery: $error_code :: $command", $lap_id);
-    }
-    my $stacks_made_deep = $mdcParser->parse_list(join "", @$stdout_buf) or
-        &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, "");    
-
-    # Append deep stacks to stacks so we can get a complete count later.
-    push @{ $stacks_made }, @{ $stacks_made_deep };
+#     ## Queue the "deep" stacks that will run locally.
+#     my $deep_minimum = $maximum_stack_inputs + 1;
+#     if ($deep_minimum < 2) { $deep_minimum = 2; }
+#     $deep_minimum = 2;
+#     $command = "$stacktool ";
+#     $command .= " -pretend " if defined $debug;
+#     $command .= " -dbname $dbname " if defined $dbname;
+#     $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter ";
+#     $command .= " -select_fwhm_major_max 10.0 -select_zpt_obs_min 25.85 -select_bg_max 425 ";
+#     $command .= " -select_good_frac_min 0.5 -random 100 ";
+# #    $command .= " -select_label $label "; # Removed to allow exposure sharing to work
+#     $command .= " -set_label ${label}.local -set_workdir $workdir -set_data_group $data_group ";
+#     $command .= " -min_num ${deep_minimum} -set_reduction THREEPI_STACK_1DG -set_dist_group $exposure->{dist_group} ";
+#     $command .= " $warps ";
+
+#     ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#       run(command => $command, verbose => $verbose);
+#     unless ($success) {
+#       $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+#       &my_die("Unable to perform stacktool -definebyquery: $error_code :: $command", $lap_id);
+#     }
+#     my $stacks_made_deep = $mdcParser->parse_list(join "", @$stdout_buf) or
+#       &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, "");    
+
+#     # Append deep stacks to stacks so we can get a complete count later.
+#     push @{ $stacks_made }, @{ $stacks_made_deep };
     
     ## Determine the sass_id so we can record it in the database.

tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_run.pl

@@ -35 +35 @@
 $job_cost{"chip"}   = 150 / 60 / 60;
 $job_cost{"warp"}   = 110 / 60 / 60;
-$job_cost{"stack"}  = 1500 / 60 / 60;
+$job_cost{"stack"}  = 2; # 1500 / 60 / 60; ## deep ~100 input stacks seem to take about 2 hours 
 $job_cost{"staticsky"} = 2; # really?  avg time of 5720s?
 
@@ -44 +44 @@
 $job_subscription{"chip"}   = 1;
 $job_subscription{"warp"}   = 1;
-$job_subscription{"stack"}  = 3;
+$job_subscription{"stack"}  = 24; # This should reserve the entire node for stack.
 $job_subscription{"staticsky"} = 3;
 
 my $proc_per_node = 24;                     # processors per node
 my $min_nodes     = 1;                      # smallest allocation to ask for
-my $max_nodes     = 1000;                   # largest allocation to ask for
+my $max_nodes     = 200;                    # largest allocation to ask for
 my $min_time      = 1;                      # shortest allocation to ask for
 my $max_time      = 8;                      # longest allocation to ask for
@@ -218 +218 @@
 # Construct the moab command last, so we can use the job_index counter to estimate resources.  Somehow.
 my $proc_need = $job_index * $threads;       # how many total processors do we need?
+if ($stage eq "stack") { 
+    $proc_need = $job_index;  # Only for large stacks, however.
+}
 my $node_need = $proc_need / $proc_per_node; # this equals how many nodes?
 my $time_need = $job_index * $job_cost{$stage};      # How many seconds will this take?

tags/ipp-pv3-20140717/ippScripts/scripts/sc_prepare_warp.pl

@@ -59 +59 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SKY.SAVE","LOG.EXP");
+# For LANL-only version.
+my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "LOG.EXP", "PSWARP.OUTPUT.SOURCES", "PSPHOT.PSF.SKY.SAVE"); #, "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SKY.SAVE","LOG.EXP");
 
 # STEP 0: Open output files

tags/ipp-pv3-20140717/ippTools/share/stacktool_definebyquery_insert_random_part2.sql

@@ -4 +4 @@
     -- Sort by the random number, and take the first N
     -- to get a random set of N.
-    ORDER BY rnd_num
+    ORDER BY good_frac DESC
     LIMIT @RANDOM_LIMIT@
 ) AS randomWarps