Changeset 38168

Timestamp:

Apr 23, 2015, 3:12:58 PM (11 years ago)

Author:

watersc1

Message:

Updated remote code with ff support and ippconfig recipes.

Location:

trunk

Files:

• ippScripts/scripts/sc_check_diff.pl (modified) (2 diffs)
• ippScripts/scripts/sc_mk_stack_mdc.pl (modified) (1 diff)
• ippScripts/scripts/sc_mk_staticsky_mdc.pl (modified) (1 diff)
• ippScripts/scripts/sc_prepare_camera.pl (modified) (5 diffs)
• ippScripts/scripts/sc_prepare_chip.pl (modified) (5 diffs)
• ippScripts/scripts/sc_prepare_diff.pl (modified) (5 diffs)
• ippScripts/scripts/sc_prepare_ff.pl (modified) (8 diffs)
• ippScripts/scripts/sc_prepare_run.pl (modified) (2 diffs)
• ippScripts/scripts/sc_prepare_stack.pl (modified) (6 diffs)
• ippScripts/scripts/sc_prepare_staticsky.pl (modified) (6 diffs)
• ippScripts/scripts/sc_prepare_warp.pl (modified) (4 diffs)
• ippScripts/scripts/sc_remote_exec.pl (modified) (8 diffs)
• ippScripts/scripts/sc_remote_poll.pl (modified) (10 diffs)
• ippTasks/remote.pro (modified) (10 diffs)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/remotetool_definebyquery_ff.sql (added)
• ippTools/src/remotetool.c (modified) (1 diff)
• ippconfig/recipes/Makefile.am (modified) (1 diff)
• ippconfig/recipes/remote_hosts.config (modified) (3 diffs)
• ippconfig/system.config (modified) (1 diff)

trunk/ippScripts/scripts/sc_check_diff.pl

@@ -8 +8 @@
 
 
-my $remote_root = '/scratch3/watersc1/';
+#my $remote_root = '/scratch3/watersc1/';
 
-my ($diff_id,$skycell_id,$dbname,$out_path_base);
+my ($diff_id,$skycell_id,$dbname,$out_path_base, $remote_root);
 GetOptions(
     'diff_id=s'         => \$diff_id,
@@ -16 +16 @@
     'dbname=s'          => \$dbname,
     'out_path_base=s'   => \$out_path_base,
+    'remote_root=s'     => \$remote_root,
     ) or pod2usage( 2 );
 
-pod2usage( -msg => "Required options: --out_path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --out_path_base --remote_root", -exitval => 3) unless
     defined($out_path_base) &&
     defined($diff_id) &&
     defined($skycell_id) &&
+    defined($remote_root) &&
     defined($dbname);
 

trunk/ippScripts/scripts/sc_mk_stack_mdc.pl

@@ -8 +8 @@
 
 
-my $remote_root = '/scratch3/watersc1/';
+#my $remote_root = '/scratch3/watersc1/';
 
-my ($compmap_file);
+my ($compmap_file,$remote_root);
 GetOptions(
     'compmap=s'   => \$compmap_file,
+    'remote_root=s' => \$remote_root,
     ) or pod2usage( 2 );
 
 pod2usage( -msg => "Required options: --compmap", -exitval => 3) unless
+    defined($remote_root) and
     defined($compmap_file);
 

trunk/ippScripts/scripts/sc_mk_staticsky_mdc.pl

@@ -8 +8 @@
 
 
-my $remote_root = '/scratch3/watersc1/';
+#my $remote_root = '/scratch3/watersc1/';
 
-my ($compmap_file);
+my ($compmap_file, $remote_root);
 GetOptions(
     'compmap=s'   => \$compmap_file,
+    'remote_root=s' => \$remote_root,
     ) or pod2usage( 2 );
 
 pod2usage( -msg => "Required options: --compmap", -exitval => 3) unless
+    defined($remote_root) and
     defined($compmap_file);
 

trunk/ippScripts/scripts/sc_prepare_camera.pl

@@ -19 +19 @@
 use Pod::Usage qw( pod2usage );
 
-# Hard coded values
-# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-my $remote_root     = '/scratch3/watersc1/';  # Far side destination base location
-my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
-my $threads_req     = 0;                      # How many threads are we going to use?
-
-my $fail_state = "prep_fail";
-
 # Look for programs we need
 my $missing_tools;
@@ -32 +24 @@
 my $camtool    = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -38 +31 @@
 }
 
-my ($remote_id,$cam_id,$camera,$dbname,$verbose,$path_base,$no_update);
+my ($remote_id,$cam_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe);
 GetOptions(
     'remote_id=s'    => \$remote_id,
@@ -44 +37 @@
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
+    'recipe=s'       => \$cmd_recipe,
     'path_base=s'    => \$path_base,
     'no_update'      => \$no_update,
@@ -50 +44 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --cam_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id --cam_id --camera --dbname --path_base --recipe", -exitval => 3) unless
     defined($remote_id) and
     defined($cam_id) and
     defined($camera) and
     defined($path_base) and
+    defined($cmd_recipe) and
     defined($dbname);
+
+
+# Hard coded values
+# Now accessible from a recipe
+my %remote_recipe = ();
+{
+    my $verbose = 0;
+    my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $conf_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR);
+    }
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+
+    my $active_recipe = '';
+    my %recipes = ();
+    
+#    print Dumper($metadata);
+    foreach my $entry (@{ $metadata }) {
+        if (${ $entry }{name} eq 'ACTIVE') {
+            $active_recipe = ${ $entry }{value}; # Not actually used
+        }
+        else {
+            if (${ $entry }{class} eq 'metadata') { # A real recipe
+                my $name = ${ $entry }{name};
+                foreach my $tentry (@{ ${ $entry }{value} }) {
+                    if (${ $tentry }{class} eq 'scalar') { # A recipe value
+                        $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value};
+                    }
+                    elsif (${ $tentry }{class} eq 'metadata') { # A recipe array 
+                        foreach my $arr_entry (@{ ${ $tentry }{value} }) {
+                        push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value};
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) };
+#    print Dumper(%recipes);
+    %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe.
+#    print Dumper(\%remote_recipe);
+}
+
+
+# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+my $remote_root     = $remote_recipe{REMOTE_ROOT};
+my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME};         # Name of the remote node.
+my $threads_req     = 0;                      # How many threads are we going to use?
+
+my $fail_state = "prep_fail";
+
 
 my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $cam_id, $PS_EXIT_CONFIG_ERROR, $fail_state);

trunk/ippScripts/scripts/sc_prepare_chip.pl

@@ -18 +18 @@
 use Pod::Usage qw( pod2usage );
 
-# Hard coded values
-# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-my $remote_root     = '/scratch3/watersc1/';  # Far side destination base location
-my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
-my $remote_raw      = "${remote_root}/tmp/";  # Directory to find raw data in.
-my $threads_req     = 4;
-
-my $fail_state = "prep_fail";
-my $hard_fail_state = "fail";
 
 # Look for programs we need
@@ -34 +25 @@
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
 my $ipp_burntool_fix = can_run('ipp_apply_burntool_fix.pl') or (warn "Can't find ipp_apply_burntool_fix.pl" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -41 +33 @@
 
 # Options
-my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose,$dbverbose);
+my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose,$dbverbose,$cmd_recipe);
 GetOptions(
     'remote_id=s'    => \$remote_id,
@@ -47 +39 @@
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
+    'recipe=s'       => \$cmd_recipe,
     'path_base=s'    => \$path_base,
     'no_update'      => \$no_update,
@@ -54 +47 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --chip_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id --chip_id --camera --dbname --path_base --recipe", -exitval => 3) unless
     defined($remote_id) and
     defined($chip_id) and
     defined($camera) and
     defined($path_base) and
+    defined($cmd_recipe) and
     defined($dbname);
+
+
+# Hard coded values
+# Now accessible from a recipe
+my %remote_recipe = ();
+{
+    my $verbose = 0;
+    my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $conf_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR);
+    }
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+
+    my $active_recipe = '';
+    my %recipes = ();
+    
+#    print Dumper($metadata);
+    foreach my $entry (@{ $metadata }) {
+        if (${ $entry }{name} eq 'ACTIVE') {
+            $active_recipe = ${ $entry }{value}; # Not actually used
+        }
+        else {
+            if (${ $entry }{class} eq 'metadata') { # A real recipe
+                my $name = ${ $entry }{name};
+                foreach my $tentry (@{ ${ $entry }{value} }) {
+                    if (${ $tentry }{class} eq 'scalar') { # A recipe value
+                        $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value};
+                    }
+                    elsif (${ $tentry }{class} eq 'metadata') { # A recipe array 
+                        foreach my $arr_entry (@{ ${ $tentry }{value} }) {
+                            push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value};
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) };
+#    print Dumper(%recipes);
+    %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe.
+#    print Dumper(\%remote_recipe);
+}
+
+
+# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+my $remote_root     = $remote_recipe{REMOTE_ROOT};
+my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME};         # Name of the remote node.
+my $remote_raw      = "${remote_root}/tmp/";  # Directory to find raw data in.
+my $threads_req     = 4;
+
+my $fail_state = "prep_fail";
+my $hard_fail_state = "fail";
 
 my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);

trunk/ippScripts/scripts/sc_prepare_diff.pl

@@ -18 +18 @@
 use Pod::Usage qw( pod2usage );
 
-# Hard coded values
-# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-my $remote_root     = '/scratch3/watersc1/';
-my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
-my $threads_req     = 4;                      # How many threads are we going to use?
-
-my $fail_state = "prep_fail";
-
 # Look for programs we need
 my $missing_tools;
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
 my $difftool    = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -36 +29 @@
 }
 
-my ($remote_id,$diff_id,$camera,$dbname,$verbose,$path_base,$no_update);
+my ($remote_id,$diff_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe);
 GetOptions(
     'remote_id=s'    => \$remote_id,
@@ -42 +35 @@
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
+    'recipe=s'       => \$cmd_recipe,
     'path_base=s'    => \$path_base,
     'no_update'      => \$no_update,
@@ -48 +42 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id -diff_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id -diff_id --camera --dbname --path_base --recipe", -exitval => 3) unless
     defined($remote_id) and
     defined($diff_id) and
     defined($camera) and
     defined($path_base) and
+    defined($cmd_recipe) and
     defined($dbname);
+
+
+# Hard coded values
+# Now accessible from a recipe
+my %remote_recipe = ();
+{
+    my $verbose = 0;
+    my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $conf_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR);
+    }
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+
+    my $active_recipe = '';
+    my %recipes = ();
+    
+#    print Dumper($metadata);
+    foreach my $entry (@{ $metadata }) {
+        if (${ $entry }{name} eq 'ACTIVE') {
+            $active_recipe = ${ $entry }{value}; # Not actually used
+        }
+        else {
+            if (${ $entry }{class} eq 'metadata') { # A real recipe
+                my $name = ${ $entry }{name};
+                foreach my $tentry (@{ ${ $entry }{value} }) {
+                    if (${ $tentry }{class} eq 'scalar') { # A recipe value
+                        $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value};
+                    }
+                    elsif (${ $tentry }{class} eq 'metadata') { # A recipe array 
+                        foreach my $arr_entry (@{ ${ $tentry }{value} }) {
+                            push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value};
+                        }
+                    }
+                }
+            }
+        }
+    }
+        
+    unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) };
+#    print Dumper(%recipes);
+    %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe.
+#    print Dumper(\%remote_recipe);
+}
+
+# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+my $remote_root     = $remote_recipe{REMOTE_ROOT};
+my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME};         # Name of the remote node.
+my $threads_req     = 4;                      # How many threads are we going to use?
+
+my $fail_state = "prep_fail";
+
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id, $diff_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
@@ -198 +248 @@
         my $remote_templsourc = &uri_local_to_remote($ipp_templsourc);
         
-        my $pre_command     = " sc_check_diff.pl --diff_id ${diff_id} --skycell_id ${skycell_id} --dbname ${dbname} --out_path_base ${remote_outroot} $remote_inimage $remote_inmask $remote_inwt $remote_insourc $remote_templimage $remote_templmask $remote_templwt $remote_templsourc ";
+        my $pre_command     = " sc_check_diff.pl --diff_id ${diff_id} --remote_root ${remote_root} --skycell_id ${skycell_id} --dbname ${dbname} --out_path_base ${remote_outroot} $remote_inimage $remote_inmask $remote_inwt $remote_insourc $remote_templimage $remote_templmask $remote_templwt $remote_templsourc ";
 
         # make any directory we may need

trunk/ippScripts/scripts/sc_prepare_ff.pl

@@ -18 +18 @@
 use Pod::Usage qw( pod2usage );
 
-# Hard coded values
-# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-my $remote_root     = '/scratch3/watersc1/';
-my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
-my $threads_req     = 4;                      # How many threads are we going to use?
-
-my $fail_state = "prep_fail";
-
 # Look for programs we need
 my $missing_tools;
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
 my $fftool    = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -36 +29 @@
 }
 
-my ($remote_id,$ff_id,$camera,$dbname,$verbose,$path_base,$no_update);
+my ($remote_id,$ff_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe);
 GetOptions(
     'remote_id=s'    => \$remote_id,
@@ -42 +35 @@
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
+    'recipe=s'       => \$cmd_recipe,
     'path_base=s'    => \$path_base,
     'no_update'      => \$no_update,
@@ -48 +42 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id -ff_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id -ff_id --camera --dbname --path_base --recipe", -exitval => 3) unless
     defined($remote_id) and
     defined($ff_id) and
     defined($camera) and
     defined($path_base) and
+    defined($cmd_recipe) and
     defined($dbname);
+
+
+# Hard coded values
+# Now accessible from a recipe
+my %remote_recipe = ();
+{
+    my $verbose = 0;
+    my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $conf_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR);
+    }
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+
+    my $active_recipe = '';
+    my %recipes = ();
+    
+#    print Dumper($metadata);
+    foreach my $entry (@{ $metadata }) {
+        if (${ $entry }{name} eq 'ACTIVE') {
+            $active_recipe = ${ $entry }{value}; # Not actually used
+        }
+        else {
+            if (${ $entry }{class} eq 'metadata') { # A real recipe
+                my $name = ${ $entry }{name};
+                foreach my $tentry (@{ ${ $entry }{value} }) {
+                    if (${ $tentry }{class} eq 'scalar') { # A recipe value
+                        $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value};
+                    }
+                    elsif (${ $tentry }{class} eq 'metadata') { # A recipe array 
+                        foreach my $arr_entry (@{ ${ $tentry }{value} }) {
+                            push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value};
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) };
+#    print Dumper(%recipes);
+    %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe.
+#    print Dumper(\%remote_recipe);
+}
+
+# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+my $remote_root     = $remote_recipe{REMOTE_ROOT};
+my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME};         # Name of the remote node.
+my $remote_raw      = "${remote_root}/tmp/";  # Directory to find raw data in.
+
+my $threads_req     = 4;                      # How many threads are we going to use?
+
+my $fail_state = "prep_fail";
+
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id, $ff_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
@@ -126 +178 @@
         my $ipp_insourc = $ipprc->filename("PSPHOT.OUTPUT.CFF",$source_pb);
 
-        my ($ipp_disk_inimage, $remote_inimage) = &uri_to_outputs($ipp_inimage);
-        my ($ipp_disk_inmask, $remote_inmask)  = &uri_to_outputs($ipp_inmask);
-        my ($ipp_disk_inwt, $remote_inwt)    = &uri_to_outputs($ipp_inwt);
-        my ($ipp_disk_insourc, $remote_insourc) = &uri_to_outputs($ipp_insourc);
+#       my ($ipp_disk_inimage, $remote_inimage) = &uri_to_outputs($ipp_inimage);
+#       my ($ipp_disk_inmask, $remote_inmask)  = &uri_to_outputs($ipp_inmask);
+#       my ($ipp_disk_inwt, $remote_inwt)    = &uri_to_outputs($ipp_inwt);
+#       my ($ipp_disk_insourc, $remote_insourc) = &uri_to_outputs($ipp_insourc);
+
+        my (undef,$remote_inimage) = uri_to_outputs_raw($ipp_inimage);
+        my (undef,$remote_inmask)  = uri_to_outputs_raw($ipp_inmask);
+        my (undef,$remote_inwt)    = uri_to_outputs_raw($ipp_inwt);
+        my (undef,$remote_insourc) = uri_to_outputs_raw($ipp_insourc);
 
         # make any directory we may need
@@ -142 +199 @@
                     $remote_id,$ff_id,  $PS_EXIT_CONFIG_ERROR, $fail_state);
         }
+
 
 
@@ -176 +234 @@
             print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
         }
+        # If we're this far, we succeeded, so delete the input files.
+        print CONFIG " && rm -f $remote_insourc $remote_inimage $remote_inmask $remote_inwt ";
         print CONFIG "\n";
 #       die();
@@ -234 +294 @@
     my $neb_uri = shift;
     my ($ipp_disk, $remote_disk) = &uri_convert( $neb_uri );
+
+    print TRANSFER "$ipp_disk\n";
+    print CHECK    "$remote_disk\n";
+    return($ipp_disk,$remote_disk);
+}
+
+sub uri_to_outputs_raw {
+    my $neb_uri = shift;
+    my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
+    $remote_disk = $remote_raw . $ipp_disk;
 
     print TRANSFER "$ipp_disk\n";

trunk/ippScripts/scripts/sc_prepare_run.pl

@@ -21 +21 @@
 use Pod::Usage qw( pod2usage );
 
+#use Data::Dumper;
+# Look for programs we need
+my $missing_tools;
+my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Options
+my ($remote_id,$cmd_recipe,$stage,$camera,$path_base,$dbname,$verbose,$no_update);
+GetOptions(
+    'remote_id=s'    => \$remote_id,
+    'recipe=s'       => \$cmd_recipe,
+    'stage=s'        => \$stage,
+    'camera|c=s'     => \$camera,
+    'path_base=s'    => \$path_base,
+    'dbname|d=s'     => \$dbname,
+    'verbose'        => \$verbose,
+    'no_update'      => \$no_update,
+    ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --remote_id --stage --camera --dbname --path_base --recipe", -exitval => 3) unless
+    defined($remote_id) and
+    defined($stage) and
+    defined($camera) and
+    defined($path_base) and
+    defined($cmd_recipe) and
+    defined($dbname) and
+    defined($dbname);
+
 # Hard coded values
-# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-my $remote_root = '/scratch3/watersc1/';
+# Now accessible from a recipe
+my %remote_recipe = ();
+{
+    my $verbose = 0;
+    my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $conf_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR);
+    }
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+
+    my $active_recipe = '';
+    my %recipes = ();
+    
+#    print Dumper($metadata);
+    foreach my $entry (@{ $metadata }) {
+        if (${ $entry }{name} eq 'ACTIVE') {
+            $active_recipe = ${ $entry }{value}; # Not actually used
+        }
+        else {
+            if (${ $entry }{class} eq 'metadata') { # A real recipe
+                my $name = ${ $entry }{name};
+                foreach my $tentry (@{ ${ $entry }{value} }) {
+                    if (${ $tentry }{class} eq 'scalar') { # A recipe value
+                        $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value};
+                    }
+                    elsif (${ $tentry }{class} eq 'metadata') { # A recipe array 
+                        foreach my $arr_entry (@{ ${ $tentry }{value} }) {
+                            push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value};
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) };
+#    print Dumper(%recipes);
+    %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe.
+#    print Dumper(\%remote_recipe);
+}
+
+my $remote_root = $remote_recipe{REMOTE_ROOT};
 my $remote_raw  = "${remote_root}/tmp/";  # Directory to find raw data in.
 
@@ -51 +129 @@
 $job_subscription{"ff"} = 1;
 
-my $proc_per_node = 24;                     # processors per node
-my $min_nodes     = 1;                      # smallest allocation to ask for
-my $max_nodes     = 1000;                   # largest allocation to ask for
-my $min_time      = 1;                      # shortest allocation to ask for
-my $max_time      = 8;                      # longest allocation to ask for
+my $proc_per_node = $remote_recipe{PROC_PER_NODE};  # processors per node
+my $min_nodes     = $remote_recipe{MIN_NODES};      # smallest allocation to ask for
+my $max_nodes     = $remote_recipe{MAX_NODES};      # largest allocation to ask for
+my $min_time      = $remote_recipe{MIN_TIME};       # shortest allocation to ask for
+my $max_time      = $remote_recipe{MAX_TIME};       # longest allocation to ask for
 
 # We need to ensure we only ever try to transfer a file once.
 my %file_filter = ();
 
-# Look for programs we need
-my $missing_tools;
-my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
-
-if ($missing_tools) {
-    warn("Can't find required tools.");
-    exit($PS_EXIT_CONFIG_ERROR);
-}
-
-# Options
-my ($remote_id,$stage,$camera,$path_base,$dbname,$verbose,$no_update);
-GetOptions(
-    'remote_id=s'    => \$remote_id,
-    'stage=s'        => \$stage,
-    'camera|c=s'     => \$camera,
-    'path_base=s'    => \$path_base,
-    'dbname|d=s'     => \$dbname,
-    'verbose'        => \$verbose,
-    'no_update'      => \$no_update,
-    ) or pod2usage( 2 );
-
-pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --stage --camera --dbname --path_base", -exitval => 3) unless
-    defined($remote_id) and
-    defined($stage) and
-    defined($camera) and
-    defined($path_base) and
-    defined($dbname);
-
+
+# Finish setup
 my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
 

trunk/ippScripts/scripts/sc_prepare_stack.pl

@@ -18 +18 @@
 use Pod::Usage qw( pod2usage );
 
-# Hard coded values
-# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-my $remote_root     = '/scratch3/watersc1/';  # Far side destination base location
-my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
-my $threads_req     = 4;                      # How many threads are we going to use?
-my $have_warps      = 0;                      # Do we have a copy of the warp if we don't find one there?
-
-my $fail_state = "prep_fail";
-
 # Look for programs we need
 my $missing_tools;
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
 my $stacktool    = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -37 +29 @@
 }
 
-my ($remote_id,$stack_id,$camera,$dbname,$verbose,$path_base,$no_update);
+my ($remote_id,$stack_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe);
 GetOptions(
     'remote_id=s'    => \$remote_id,
@@ -43 +35 @@
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
+    'recipe=s'       => \$cmd_recipe,
     'path_base=s'    => \$path_base,
     'no_update'      => \$no_update,
@@ -49 +42 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --stack_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id --stack_id --camera --dbname --path_base --recipe", -exitval => 3) unless
     defined($remote_id) and
     defined($stack_id) and
     defined($camera) and
-    defined($path_base) and
+    defined($path_base) and    
+    defined($cmd_recipe) and
     defined($dbname);
+
+# Hard coded values
+# Now accessible from a recipe
+my %remote_recipe = ();
+{
+    my $verbose = 0;
+    my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $conf_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR);
+    }
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+
+    my $active_recipe = '';
+    my %recipes = ();
+    
+#    print Dumper($metadata);
+    foreach my $entry (@{ $metadata }) {
+        if (${ $entry }{name} eq 'ACTIVE') {
+            $active_recipe = ${ $entry }{value}; # Not actually used
+        }
+        else {
+            if (${ $entry }{class} eq 'metadata') { # A real recipe
+                my $name = ${ $entry }{name};
+                foreach my $tentry (@{ ${ $entry }{value} }) {
+                    if (${ $tentry }{class} eq 'scalar') { # A recipe value
+                        $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value};
+                    }
+                    elsif (${ $tentry }{class} eq 'metadata') { # A recipe array 
+                        foreach my $arr_entry (@{ ${ $tentry }{value} }) {
+                            push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value};
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) };
+#    print Dumper(%recipes);
+    %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe.
+#    print Dumper(\%remote_recipe);
+}
+
+# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+my $remote_root     = $remote_recipe{REMOTE_ROOT};  # Far side destination base location
+my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME};         # Name of the remote node.
+my $threads_req     = 4;                      # How many threads are we going to use?
+my $have_warps      = $remote_recipe{TRANSFER_WARP_IMAGES};                      # Do we have a copy of the warp if we don't find one there?
+
+my $fail_state = "prep_fail";
+
 
 my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $stack_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
@@ -158 +207 @@
         #  ${remote_root}/tmp/${ipp_disk}
 
-        if ($have_warps != 0) { # Check to see if we have a copy, because if we do, then we can send it over if it's missing.
+        if ($have_warps) { # Check to see if we have a copy, because if we do, then we can send it over if it's missing.
             (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".fits");
             ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".fits");
@@ -201 +250 @@
     close(COMPMAP);
 
-    my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl --compmap ${remote_root}/tmp/${component_map} $warp_path_base_string > ${remote_outroot}.in.mdc";
+    my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl --remote_root ${remote_root} --compmap ${remote_root}/tmp/${component_map} $warp_path_base_string > ${remote_outroot}.in.mdc";
     my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use
     my $ppstack_command  = " ppStack -input ${remote_outroot}.in.mdc ";

trunk/ippScripts/scripts/sc_prepare_staticsky.pl

@@ -18 +18 @@
 use Pod::Usage qw( pod2usage );
 
-# Hard coded values
-# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-my $remote_root     = '/scratch3/watersc1/';  # Far side destination base location
-my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
-my $threads_req     = 4;                      # How many threads are we going to use?
-
-my $fail_state = "prep_fail";
-
 # Look for programs we need
 my $missing_tools;
@@ -31 +23 @@
 my $stacktool    = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 my $staticskytool= can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -37 +30 @@
 }
 
-my ($remote_id,$sky_id,$camera,$dbname,$verbose,$path_base,$no_update);
+my ($remote_id,$sky_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe);
 GetOptions(
     'remote_id=s'    => \$remote_id,
@@ -43 +36 @@
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
+    'recipe=s'       => \$cmd_recipe,
     'path_base=s'    => \$path_base,
     'no_update'      => \$no_update,
@@ -49 +43 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --sky_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id --sky_id --camera --dbname --path_base --recipe", -exitval => 3) unless
     defined($remote_id) and
     defined($sky_id) and
     defined($camera) and
     defined($path_base) and
+    defined($cmd_recipe) and
     defined($dbname);
+
+# Hard coded values
+# Now accessible from a recipe
+my %remote_recipe = ();
+{
+    my $verbose = 0;
+    my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $conf_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR);
+    }
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+
+    my $active_recipe = '';
+    my %recipes = ();
+    
+#    print Dumper($metadata);
+    foreach my $entry (@{ $metadata }) {
+        if (${ $entry }{name} eq 'ACTIVE') {
+            $active_recipe = ${ $entry }{value}; # Not actually used
+        }
+        else {
+            if (${ $entry }{class} eq 'metadata') { # A real recipe
+                my $name = ${ $entry }{name};
+                foreach my $tentry (@{ ${ $entry }{value} }) {
+                    if (${ $tentry }{class} eq 'scalar') { # A recipe value
+                        $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value};
+                    }
+                    elsif (${ $tentry }{class} eq 'metadata') { # A recipe array 
+                        foreach my $arr_entry (@{ ${ $tentry }{value} }) {
+                            push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value};
+                        }
+                    }
+                }
+            }
+        }
+    }
+        
+    unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) };
+#    print Dumper(%recipes);
+    %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe.
+#    print Dumper(\%remote_recipe);
+}
+
+
+my $remote_root     = $remote_recipe{REMOTE_ROOT};  # Far side destination base location
+my $remote_hostname = $remote_recipe{REMOTE_HOSTNAME};         # Name of the remote node.
+my $threads_req     = 4;                      # How many threads are we going to use?
+
+my $fail_state = "prep_fail";
+
 
 my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $sky_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
@@ -186 +235 @@
     close(COMPMAP);
 
-    my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_staticsky_mdc.pl --compmap ${remote_root}/tmp/${component_map} $input_path_base_string > ${remote_outroot}.in.mdc";
+    my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_staticsky_mdc.pl --remote_root ${remote_root} --compmap ${remote_root}/tmp/${component_map} $input_path_base_string > ${remote_outroot}.in.mdc";
     my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_PSPHOT');
     my $recipe_ppsub  = $ipprc->reduction($reduction, 'STACKPHOT_PPSUB');

trunk/ippScripts/scripts/sc_prepare_warp.pl

@@ -18 +18 @@
 use Pod::Usage qw( pod2usage );
 
-# Hard coded values
-# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-my $remote_root     = '/scratch3/watersc1/';
-my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
-my $threads_req     = 4;                      # How many threads are we going to use?
-
-my $fail_state = "prep_fail";
-
 # Look for programs we need
 my $missing_tools;
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
 my $warptool    = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -36 +29 @@
 }
 
-my ($remote_id,$warp_id,$camera,$dbname,$verbose,$path_base,$no_update);
+my ($remote_id,$warp_id,$camera,$dbname,$verbose,$path_base,$no_update,$cmd_recipe);
 GetOptions(
     'remote_id=s'    => \$remote_id,
@@ -42 +35 @@
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
+    'recipe=s'       => \$cmd_recipe,
     'path_base=s'    => \$path_base,
     'no_update'      => \$no_update,
@@ -48 +42 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id -warp_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id -warp_id --camera --dbname --path_base --recipe", -exitval => 3) unless
     defined($remote_id) and
     defined($warp_id) and
     defined($camera) and
     defined($path_base) and
+    defined($cmd_recipe) and
     defined($dbname);
 
+
+# Hard coded values
+# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+# Now accessible from a recipe
+my %remote_recipe = ();
+{
+    my $verbose = 0;
+    my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $conf_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR);
+    }
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+
+    my $active_recipe = '';
+    my %recipes = ();
+    
+#    print Dumper($metadata);
+    foreach my $entry (@{ $metadata }) {
+        if (${ $entry }{name} eq 'ACTIVE') {
+            $active_recipe = ${ $entry }{value}; # Not actually used
+        }
+        else {
+            if (${ $entry }{class} eq 'metadata') { # A real recipe
+                my $name = ${ $entry }{name};
+                foreach my $tentry (@{ ${ $entry }{value} }) {
+                    if (${ $tentry }{class} eq 'scalar') { # A recipe value
+                        $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value};
+                    }
+                    elsif (${ $tentry }{class} eq 'metadata') { # A recipe array 
+                        foreach my $arr_entry (@{ ${ $tentry }{value} }) {
+                            push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value};
+                        }
+                    }
+                }
+            }
+        }
+    }
+    unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) };
+#    print Dumper(%recipes);
+    %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe.
+#    print Dumper(\%remote_recipe);
+
+}
+
+my $remote_root = $remote_recipe{REMOTE_ROOT};
+my $hostname    = $remote_recipe{REMOTE_HOSTNAME};
+my $have_warps  = $remote_recipe{TRANSFER_WARP_IMAGES};
+my $threads_req     = 4;                      # How many threads are we going to use?
+
+my $fail_state = "prep_fail";
+
+
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id, $warp_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SKY.SAVE","LOG.EXP");
+my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SKY.SAVE","LOG.EXP");
+if ($have_warps) {
+    push @return_component_list, ("PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE");
+}
 
 # STEP 0: Open output files

trunk/ippScripts/scripts/sc_remote_exec.pl

@@ -13 +13 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
-
-# Hard coded values
-my $DMZ_HOST = 'wtrw';
-my @SEC_HOSTS= ('mu-fe1','mu-fe2','mu-fe3'
-                # ,'mu-fe4'
-    );
-my $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; # 'mu-fe';
-my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
-my $remote_root  = '/scratch3/watersc1/';
 
 # tools
@@ -28 +19 @@
 my $scp        = can_run('scp')  or (warn "Can't find scp" and $missing_tools = 1);
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -35 +27 @@
 
 # Options
-my ($remote_id,$job_id,$path_base,$dbname,$verbose,$no_update,$camera);
+my ($remote_id,$job_id,$path_base,$dbname,$verbose,$no_update,$camera,$cmd_recipe);
 $verbose = 0;
 GetOptions(
@@ -42 +34 @@
     'camera=s'      => \$camera,
     'dbname=s'      => \$dbname,
+    'recipe=s'       => \$cmd_recipe,
     'verbose'       => \$verbose,
     'no_update'     => \$no_update,
@@ -47 +40 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id --path_base --recipe", -exitval => 3) unless
     defined($path_base) and
+    defined($camera) and
+    defined($cmd_recipe) and
+    defined($dbname) and
     defined($remote_id);
+
+# Hard coded values
+# Now accessible from a recipe
+my %remote_recipe = ();
+{
+    my $verbose = 0;
+    my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $conf_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR);
+    }
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+
+    my $active_recipe = '';
+    my %recipes = ();
+    
+#    print Dumper($metadata);
+    foreach my $entry (@{ $metadata }) {
+        if (${ $entry }{name} eq 'ACTIVE') {
+            $active_recipe = ${ $entry }{value}; # Not actually used
+        }
+        else {
+            if (${ $entry }{class} eq 'metadata') { # A real recipe
+                my $name = ${ $entry }{name};
+                foreach my $tentry (@{ ${ $entry }{value} }) {
+                    if (${ $tentry }{class} eq 'scalar') { # A recipe value
+                        $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value};
+                    }
+                    elsif (${ $tentry }{class} eq 'metadata') { # A recipe array 
+                        foreach my $arr_entry (@{ ${ $tentry }{value} }) {
+                            push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value};
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) };
+#    print Dumper(%recipes);
+    %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe.
+#    print Dumper(\%remote_recipe);
+}
+
+# Hard coded values
+my $DMZ_HOST = $remote_recipe{DMZ_HOST};
+my @SEC_HOSTS= @{ $remote_recipe{SEC_HOST} };
+my $SEC_HOST;
+if ($#SEC_HOSTS != -1) {
+    $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; 
+}
+else {
+    $SEC_HOST = '';
+}
+my $IPP_PATH = $remote_recipe{IPP_PATH};
+my $remote_root  = $remote_recipe{REMOTE_ROOT};
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id);
@@ -141 +196 @@
     my $file = shift;
     my $destination = shift;
-    my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}";
+    my $cmd;
+    if ($SEC_HOST ne '') {
+        $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}";
+    }
+    else {
+        $cmd = "$ssh -n $DMZ_HOST $cmd";
+    }
 
     my $directory = dirname($destination);
@@ -157 +218 @@
     my $destination = shift;
     my $file = shift;
-
-    my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file ";
+    my $cmd;
+    if ($SEC_HOST ne '') {
+        $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file ";
+    }
+    else {
+        $cmd = "$ssh -n $DMZ_HOST $cmd";
+    }
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -170 +236 @@
 sub ssh_exec_command {
     my $cmd = shift;
-    $cmd = "$ssh -n $DMZ_HOST ssh  ${SEC_HOST} $cmd";
+    if ($SEC_HOST ne '') {
+        $cmd = "$ssh -n $DMZ_HOST ssh  ${SEC_HOST} $cmd";
+    }
+    else {
+        $cmd = "$ssh -n $DMZ_HOST $cmd";
+    }
+
     print "EXEC: $cmd\n";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

trunk/ippScripts/scripts/sc_remote_poll.pl

@@ -26 +26 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
-
-# Hard coded values
-my $DMZ_HOST = 'wtrw';
-my @SEC_HOSTS= ('mu-fe1','mu-fe2','mu-fe3'
-                # ,'mu-fe4'
-    );
-my $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; # 'mu-fe';
-my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
-my $remote_root  = '/scratch3/watersc1/';
 
 # tools
@@ -44 +35 @@
 my $camtool    = can_run('camtool')    or (warn "Can't find camtool"    and $missing_tools = 1);
 my $warptool   = can_run('warptool')   or (warn "Can't find warptool"   and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -51 +43 @@
 
 # Options
-my ($remote_id,$path_base,$skip_poll,$skip_transfer,$job_id,$dbname,$retry,$verbose,$no_update,$camera);
+my ($remote_id,$path_base,$skip_poll,$skip_transfer,$job_id,$dbname,$retry,$verbose,$no_update,$camera,$cmd_recipe);
 $verbose = 0;
 GetOptions(
@@ -61 +53 @@
     'camera=s'      => \$camera,
     'dbname=s'      => \$dbname,
+    'recipe=s'       => \$cmd_recipe,
     'retry'         => \$retry,
     'verbose'       => \$verbose,
@@ -67 +60 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --job_id --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id --job_id --path_base --recipe", -exitval => 3) unless
     defined($path_base) and
     defined($remote_id) and
+    defined($cmd_recipe) and
     defined($job_id);
+
+
+
+# Now accessible from a recipe
+my %remote_recipe = ();
+{
+    my $verbose = 0;
+    my $conf_cmd = "$ppConfigDump -dump-recipe REMOTE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $conf_cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", -1, $PS_EXIT_SYS_ERROR);
+    }
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+
+    my $active_recipe = '';
+    my %recipes = ();
+    
+#    print Dumper($metadata);
+    foreach my $entry (@{ $metadata }) {
+        if (${ $entry }{name} eq 'ACTIVE') {
+        $active_recipe = ${ $entry }{value}; # Not actually used
+        }
+        else {
+            if (${ $entry }{class} eq 'metadata') { # A real recipe
+                my $name = ${ $entry }{name};
+                foreach my $tentry (@{ ${ $entry }{value} }) {
+                    if (${ $tentry }{class} eq 'scalar') { # A recipe value
+                        $recipes{$name}{${ $tentry }{name}} = ${ $tentry }{value};
+                    }
+                    elsif (${ $tentry }{class} eq 'metadata') { # A recipe array 
+                        foreach my $arr_entry (@{ ${ $tentry }{value} }) {
+                            push @{ $recipes{$name}{${ $tentry }{name}} }, ${ $arr_entry }{value};
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    unless (exists($recipes{$cmd_recipe})) { &my_die("Cannot find recipe $cmd_recipe", -1, $PS_EXIT_CONFIG_ERROR) };
+#    print Dumper(%recipes);
+    %remote_recipe = %{ $recipes{$cmd_recipe} }; # Select the appropriate recipe.
+#    print Dumper(\%remote_recipe);
+}
+
+
+# Hard coded values
+my $DMZ_HOST = $remote_recipe{DMZ_HOST};
+my @SEC_HOSTS= @{ $remote_recipe{SEC_HOST} };
+my $SEC_HOST;
+if ($#SEC_HOSTS != -1) {
+    $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; 
+}
+else {
+    $SEC_HOST = '';
+}
+my $IPP_PATH = $remote_recipe{IPP_PATH};
+my $remote_root  = $remote_recipe{REMOTE_ROOT};
+
 
 my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id);
@@ -351 +407 @@
     my $file = shift;
     my $destination = shift;
-    my $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}";
+    my $cmd;
+    if ($SEC_HOST ne '') {
+        $cmd = "$scp $file ${DMZ_HOST}:${SEC_HOST}:${destination}";
+    }
+    else {
+        $cmd = "$scp $file ${DMZ_HOST}:${destination}";
+    }
 
     my $directory = dirname($destination);
@@ -367 +429 @@
     my $destination = shift;
     my $file = shift;
-
-    my $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file ";
+    my $cmd;
+    if ($SEC_HOST ne '') {
+        $cmd = "$scp ${DMZ_HOST}:${SEC_HOST}:${destination} $file ";
+    }
+    else {
+        $cmd = "$scp $file ${DMZ_HOST}:${destination}";
+    }
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -380 +447 @@
 sub ssh_exec_command {
     my $cmd = shift;
-    $cmd = "$ssh -n $DMZ_HOST ssh  ${SEC_HOST} $cmd";
+
+    if ($SEC_HOST ne '') {
+        $cmd = "$ssh -n $DMZ_HOST ssh  ${SEC_HOST} $cmd";
+    }
+    else {
+        $cmd = "$ssh -n $DMZ_HOST $cmd";
+    }
+
     print "EXEC: $cmd\n";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -511 +585 @@
     }
     elsif ($stage eq 'staticsky') {
-#       ($class_id, $stage_id) = $file =~ /(/;
+        ($class_id, $stage_id) = $file =~ /(skycell.\d\d\d\d.\d\d\d).sky.(\d*).dbinfo/;
     }
     elsif ($stage eq 'diff') {
-#       ($class_id,$stage_id) = $file =~ /(/;
+        ($class_id,$stage_id) = $file =~ /(skycell.\d\d\d\d.\d\d\d).WS.dif.(\d*).dbinfo/;
     }
     elsif ($stage eq 'ff') {
-#       ($class_id,$stage_id) = $file =~ /(/;
+        ($class_id,$stage_id) = $file =~ /wrp\.(\d*).ff.(\d*).dbinfo/;
     }
     return ($stage_id, $class_id);
@@ -548 +622 @@
         $class_id = 0;
     }
+    elsif ($stage eq 'staticsky') {
+        ($class_id, $stage_id, $component) = $file =~ /(skycell.\d\d\d\d.\d\d\d).sky.(\d*).([\w\.]+)$/;
+    }
+    elsif ($stage eq 'diff') {
+        ($class_id,$stage_id, $component) = $file =~ /(skycell.\d\d\d\d.\d\d\d).WS.dif.(\d*).([\w\.]+)$/;
+    }
+    elsif ($stage eq 'ff') {
+        ($class_id,$stage_id, $component) = $file =~ /wrp\.(\d*).ff.(\d*).([\w\.]+)$/;
+    }
+
     return($stage_id,$class_id,$component);
 }

trunk/ippTasks/remote.pro

@@ -20 +20 @@
 $remoteExec_DB = 0
 $remotePoll_DB = 0
+$REMOTE_RECIPE = 0
+
 
 list STAGES
@@ -94 +96 @@
     active true
   end
+end
+
+macro set.remote.recipe
+  if ($0 != 2)
+     echo "USAGE: set.remote.recipe (recipe_name)"
+     break
+  end
+  $REMOTE_RECIPE = $1
 end
 
@@ -273 +283 @@
     if ($N == 0) break
     if ($NETWORK == 0) break
+    if ($REMOTE_RECIPE == 0) break
 
     book getpage remotePrepCompRuns 0 -var pageName -key pantaskState INIT
@@ -287 +298 @@
 
     if ("$STAGE" == "chip")
-      $command = sc_prepare_chip.pl --camera GPC1 --chip_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME
+      $command = sc_prepare_chip.pl --chip_id $STAGE_ID
     end
     if ("$STAGE" == "camera")
-      $command = sc_prepare_camera.pl --camera GPC1 --cam_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME
+      $command = sc_prepare_camera.pl --cam_id $STAGE_ID
     end
     if ("$STAGE" == "warp")
-      $command = sc_prepare_warp.pl --camera GPC1 --warp_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME
+      $command = sc_prepare_warp.pl --warp_id $STAGE_ID
     end
     if ("$STAGE" == "stack")
-      $command = sc_prepare_stack.pl --camera GPC1 --stack_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME
+      $command = sc_prepare_stack.pl --stack_id $STAGE_ID
     end
     if ("$STAGE" == "staticsky")
-      $command = sc_prepare_staticsky.pl --camera GPC1 --sky_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME
+      $command = sc_prepare_staticsky.pl --sky_id $STAGE_ID
     end
     if ("$STAGE" == "diff")
-      $command = sc_prepare_diff.pl --camera GPC1 --diff_id $STAGE_ID --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME
-    end
+      $command = sc_prepare_diff.pl --diff_id $STAGE_ID
+    end
+    if ("$STAGE" == "ff")
+      $command = sc_prepare_ff.pl --ff_id $STAGE_ID
+    end
+
+# Common elements
+    $command = $command --recipe $REMOTE_RECIPE --camera GPC1 --remote_id $REMOTE_ID --path_base $outroot --dbname $DBNAME
 
     options $pageName
@@ -391 +408 @@
     if ($N == 0) break
     if ($NETWORK == 0) break
+    if ($REMOTE_RECIPE == 0) break
 
     book getpage remotePrepRuns 0 -var pageName -key pantaskState INIT
@@ -404 +422 @@
     sprintf outroot "%s/remote_%s.%s" $PATH_BASE $STAGE $REMOTE_ID
 
-    $command = sc_prepare_run.pl --camera GPC1 --remote_id $REMOTE_ID --stage $STAGE --path_base $outroot --dbname $DBNAME
+    $command = sc_prepare_run.pl --camera GPC1 --remote_id $REMOTE_ID --stage $STAGE --path_base $outroot --dbname $DBNAME --recipe $REMOTE_RECIPE
 
     options $pageName
@@ -491 +509 @@
     if ($N == 0) break
     if ($NETWORK == 0) break
+    if ($REMOTE_RECIPE == 0) break
 
     book getpage remoteExecRuns 0 -var pageName -key pantaskState INIT
@@ -505 +524 @@
     sprintf outroot "%s/remote_%s.%s" $PATH_BASE $STAGE $REMOTE_ID
 
-    $command = sc_remote_exec.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1
+    $command = sc_remote_exec.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 --recipe $REMOTE_RECIPE
 
     options $pageName
@@ -593 +612 @@
     if ($N == 0) break
     if ($NETWORK == 0) break
+    if ($REMOTE_RECIPE == 0) break
 
     book getpage remotePollRuns 0 -var pageName -key pantaskState INIT
@@ -609 +629 @@
     # This can't have an invalid job_id
     if ($JOB_ID == -1) break
-    $command = sc_remote_poll.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 --job_id $JOB_ID
+    $command = sc_remote_poll.pl --remote_id $REMOTE_ID --path_base $outroot --verbose --dbname $DBNAME --camera GPC1 --job_id $JOB_ID --recipe $REMOTE_RECIPE
 
     options $pageName

trunk/ippTools/share/Makefile.am

@@ -528 +528 @@
         remotetool_definebyquery_staticsky.sql \
         remotetool_definebyquery_diff.sql \
+        remotetool_definebyquery_ff.sql \
         remotetool_dropcomponent.sql \
         remotetool_listcomponent.sql \

trunk/ippTools/src/remotetool.c

@@ -166 +166 @@
   else if (!strcmp(stage,"diff")) {
     query = pxDataGet("remotetool_definebyquery_diff.sql");
+    if (!query) {
+      psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement");
+      return(false);
+    }
+
+    if (label) {
+      psStringAppend(&whereOption, "\n AND (label = '%s')", label);
+    }
+  }
+  else if (!strcmp(stage,"ff")) {
+    query = pxDataGet("remotetool_definebyquery_ff.sql");
     if (!query) {
       psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement");

trunk/ippconfig/recipes/Makefile.am

@@ -33 +33 @@
         nightly_science.config \
         ppBackground.mdc \
-        pstamp.config
+        pstamp.config \
+        remote_hosts.config
 
 install_DATA = $(install_files)

trunk/ippconfig/recipes/remote_hosts.config

@@ -5 +5 @@
 
      SEC_HOST MULTI
-       SEC_HOST STR     mu-fe1
-       SEC_HOST STR     mu-fe2
-       SEC_HOST STR     mu-fe3
-       SEC_HOST STR     mu-fe4
+     SEC_HOST METADATA
+       SEC_HOST1        STR     mu-fe1
+       SEC_HOST2        STR     mu-fe2
+       SEC_HOST3        STR     mu-fe3
+       SEC_HOST4        STR     mu-fe4
      END
 
      IPP_PATH        STR     /turquoise/usr/projects/cosmo/mswarren/ipp/
      REMOTE_ROOT     STR     /scratch3/watersc1/
-     REMOTE_HOSTNAME STR     LANL/Mustang
+     REMOTE_HOSTNAME STR     LANL/Mustang       
+
+# This is used to decide if we bring back warp images, and if stack should ask for local copies.
+     TRANSFER_WARP_IMAGES BOOL FALSE
+
+     PROC_PER_NODE   S32     24
+     MIN_NODES       S32     1
+     MAX_NODES       S32     1000
+     MIN_TIME        S32     1
+     MAX_TIME        S32     8
 END
 
@@ -20 +30 @@
 
      SEC_HOST MULTI
+     SEC_HOST METADATA
      END
 
@@ -25 +36 @@
      REMOTE_ROOT     STR     /lus/scratch/watersc/
      REMOTE_HOSTNAME STR     UH/Cray
+
+     TRANSFER_WARP_IMAGES BOOL FALSE
+
+     PROC_PER_NODE   S32     20
+     MIN_NODES       S32     1
+     MAX_NODES       S32     10
+     MIN_TIME        S32     1
+     MAX_TIME        S32     8
 END     

trunk/ippconfig/system.config

@@ -66 +66 @@
         PSVIDEOPHOT     STR             recipes/psvideophot.config # video cell photometry
         PSTAMP          STR             recipes/pstamp.config       # postage stamp server
+        REMOTE          STR             recipes/remote_hosts.config # Remote supercomputer processing
 END