IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 26873


Ignore:
Timestamp:
Feb 10, 2010, 4:16:03 PM (16 years ago)
Author:
eugene
Message:

updates from trunk

Location:
branches/eam_branches/20091201/ippScripts/scripts
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/20091201/ippScripts/scripts/automate_stacks.pl

    r26721 r26873  
    99use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
    1010use Pod::Usage qw( pod2usage );
    11 
    12 
     11use DateTime;
     12
     13#
    1314# Set up
     15################################################################################
     16
    1417my $missing_tools = 0;
    1518my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
     19my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
    1620my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
     21my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
    1722my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
    1823my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1);
    … …  
    2328}
    2429
    25 # global variables.
    26 my @target_list = ('M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10');
    27 my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000');
     30# my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000');
     31# my @target_list = ('CMB','M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10',
     32#                  'STS','SVS','SweetSpot','ThreePi');
     33# my %tessID_list = ('CMB' => 'RINGS.V0', 'M31' => 'M31', 'MD01' => 'MD01', 'MD02' => 'MD02',
     34#                  'MD03' => 'MD03', 'MD04' => 'MD04', 'MD05' => 'MD05', 'MD06' => 'MD06',
     35#                  'MD07' => 'MD07', 'MD08' => 'MD08', 'MD09' => 'MD09', 'MD10' => 'MD10',
     36#                  'STS' => 'STS', 'SVS' => 'RINGS.V0', 'SweetSpot' => 'RINGS.V0', 'ThreePi' => 'RINGS.V0');
     37# my %comment_list = ('CMB' => 'CMB_Cold%', 'M31' => 'M31%', 'MD01' => 'MD01%', 'MD02' => 'MD02%',
     38#                   'MD03' => 'MD03%', 'MD04' => 'MD04%', 'MD05' => 'MD05%', 'MD06' => 'MD06%',
     39#                   'MD07' => 'MD07%', 'MD08' => 'MD08%', 'MD09' => 'MD09%', 'MD10' => 'MD10%',
     40#                   'STS' => 'Stellar Transit%', 'SVS' => 'SVS%', 'SweetSpot' => 'Sweetspot%', 'ThreePi' => 'ThreePi%');
     41# my %stackable_list = ('CMB' => 0, 'M31' => 1, 'MD01' => 1, 'MD02' => 1,
     42#                   'MD03' => 1, 'MD04' => 1, 'MD05' => 1, 'MD06' => 1,
     43#                   'MD07' => 1, 'MD08' => 1, 'MD09' => 1, 'MD10' => 1,
     44#                   'STS' => 1, 'SVS' => 0, 'SweetSpot' => 0, 'ThreePi' => 0);
     45# my $retention_time = 9000;  # days.
     46
     47
    2848my $db;
    2949my $debug = 0;
    … …  
    3353chomp($loghead = `date`);
    3454print STDERR 'Starting: ' . $loghead . ' ' . $0 . ' ' . (join ' ', @ARGV) . "\n";
     55
    3556# Grab options
    3657my ( $date, $camera, $dbname, $logfile, $verbose);
    37 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips, $check_stacks, $queue_stacks);
    38 my ( $help, $this_target_only, $this_filter_only, $isburning, $force_stack_count);
     58my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only);
     59my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
     60my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);
     61
    3962GetOptions(
    4063    'help|h'               => \$help,
    … …  
    4467    'logfile=s'            => \$logfile,
    4568    'verbose'              => \$verbose,
     69    'isburning'            => \$isburning,
     70    'force_stack_count'    => \$force_stack_count,
     71    'test_mode'            => \$test_mode,
     72    'this_target_only=s'   => \$this_target_only,
     73    'this_filter_only=s'   => \$this_filter_only,
     74    'check_registration'   => \$check_registration,
     75    'define_burntool'      => \$define_burntool,
     76    'queue_burntool'       => \$queue_burntool,
    4677    'check_chips'          => \$check_chips,
    4778    'queue_chips'          => \$queue_chips,
    4879    'check_stacks'         => \$check_stacks,
    4980    'queue_stacks'         => \$queue_stacks,
    50     'queue_burntool'       => \$queue_burntool,
    51     'define_burntool'      => \$define_burntool,
    52     'check_registration|R' => \$check_registration,
    53     'this_target_only=s'   => \$this_target_only,
    54     'this_filter_only=s'   => \$this_filter_only,
    55     'isburning'            => \$isburning,
    56     'force_stack_count'    => \$force_stack_count,
     81    'check_diffs'          => \$check_diffs,
     82    'queue_diffs'          => \$queue_diffs,
     83    'clean_old'            => \$clean_old,
    5784    ) or pod2usage ( 2 );
    5885pod2usage( -msg =>
    … …  
    6491           --dbname <db>          Default gpc1.
    6592           --verbose             
     93           --isburning            Signal that we are currently burntooling.
     94           --force_stack_count    Force the chip/warp counts.
    6695           --this_target_only     Process only a single target.
    6796           --this_filter_only     Process only a single filter.
    68            --isburning            Signal that we are currently burntooling.
    69            --force_stack_count    Force the chip/warp counts.
    7097        Modes:
    7198           --check_registration   Confirm the data downloaded correctly.
    … …  
    75102           --queue_chips          Issue chiptool commands to queue chips.
    76103           --check_stacks         Confirm that stacks can be built.
    77            --queue_stacks         Issue stacktool commands to queue stacks.\n",
     104           --queue_stacks         Issue stacktool commands to queue stacks.
     105           --check_diffs          Confirm that diffs can be done.
     106           --queue_diffs          Issue difftool commands to queue diffs.
     107           --clean_old            Mark old data 'goto_cleanup'.\n",
    78108           -exitval => 2, ) if (defined($help));
    79109pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    … …  
    84114    defined $check_registration or defined $define_burntool or defined $queue_burntool or
    85115    defined $queue_chips or defined $queue_stacks or
    86     defined $check_chips or defined $check_stacks;
     116    defined $check_chips or defined $check_stacks or
     117    defined $test_mode or defined $clean_old;
     118
     119# Configurable parameters from our config file.
     120my @target_list = ();
     121my @filter_list = ();
     122my %tessID_list = ();
     123my %comment_list= ();
     124my %stackable_list = ();
     125my $retention_time = 9000;
     126
     127# Grab the configuration data.
     128my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -";
     129my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     130    run(command => $conf_cmd, verbose => $verbose);
     131unless ($success) {
     132    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     133    &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR);
     134}
     135
     136my $mdcParser = PS::IPP::Metadata::Config->new;
     137my $metadata = $mdcParser->parse(join "", @$stdout_buf);
     138foreach my $entry (@{ $metadata }) {
     139    if (${ $entry }{name} eq 'RETENTION_TIME') {
     140        $retention_time = ${ $entry }{value};
     141    }
     142    elsif (${ $entry }{name} eq 'FILTERS') {
     143        push @filter_list, ${ $entry }{value};
     144    }
     145    elsif (${ $entry }{name} eq 'TARGETS') {
     146        my @target_data = @{ ${ $entry }{value} };
     147        my $this_target = '';
     148        foreach my $tentry (@target_data) {
     149            if (${ $tentry }{name} eq 'NAME') {
     150                $this_target = ${ $tentry }{value};
     151                push @target_list, $this_target;
     152            }
     153            elsif (${ $tentry }{name} eq 'TESS') {
     154                $tessID_list{$this_target} = ${ $tentry }{value};
     155            }
     156            elsif (${ $tentry }{name} eq 'COMMENT') {
     157                $comment_list{$this_target} = ${ $tentry }{value};
     158            }
     159            elsif (${ $tentry }{name} eq 'STACKABLE') {
     160                $stackable_list{$this_target} = ${ $tentry }{value};
     161            }
     162        }
     163    }
     164}
     165
     166
    87167
    88168
    … …  
    120200}
    121201
    122 if (defined($check_registration)) {
     202#
     203# Mode selection
     204################################################################################
     205if (defined($test_mode)) {
     206    $debug = 1;
     207}
     208
     209if (defined($check_registration) || defined($test_mode)) {
     210    $metadata_out{nsState} = 'NEW';
    123211    my ($Nsummit_exp,$Nfaults) = check_summit_copy($date);
    124    
    125     if ($Nsummit_exp == 0) {
    126         print STDERR "No exposures were found on the summit for $date.\n";
    127         $metadata_out{nsState} = 'DROP';
    128     }
    129     elsif ($Nfaults != 0) {
    130         print STDERR "There were faults while downloading the exposures for $date.\n";
    131         $metadata_out{nsState} = 'NEW';
    132     }
    133     else {
    134         print STDERR "Summit copy and Registration have succeeded for $date.\n";
    135         $metadata_out{nsState} = 'REGISTERED';
    136     }
    137212    return_metadata($date);
    138     exit(0);
    139 }
    140 if (defined($define_burntool)) {
     213    unless (defined($test_mode)) { exit(0); }
     214}
     215if (defined($define_burntool) || defined($test_mode)) {
    141216    $metadata_out{nsState} = 'QUEUEBURNING';
    142217    &find_burntool_ranges($date);
    143218    return_metadata($date);
    144     exit(0);
    145 }
    146 if (defined($queue_burntool)) {
     219    unless (defined($test_mode)) { exit(0); }
     220}
     221if (defined($queue_burntool) || defined($test_mode)) {
    147222    $metadata_out{nsState} = 'BURNING';
    148223    return_metadata($date);
    149     exit(0);
    150 }
    151 if (defined($check_chips)) {
     224    unless (defined($test_mode)) { exit(0); }
     225}
     226if (defined($check_chips) || defined($test_mode)) {
    152227    $metadata_out{nsState} = 'QUEUECHIPS';
    153228    &execute_chips($date,"pretend");
    … …  
    156231    }
    157232    return_metadata($date);
    158     exit(0);
    159 }
    160 if (defined($check_stacks)) {
    161     $metadata_out{nsState} = 'TOWARP';
    162     &execute_stacks($date,"pretend");
    163     if ($metadata_out{nsState} eq 'FORCETOWARP') {
    164         $metadata_out{nsState} = 'TOWARP';
    165     }
    166     return_metadata($date);
    167     exit(0);
     233    unless (defined($test_mode)) { exit(0); }
    168234}
    169235if (defined($queue_chips)) {
    … …  
    173239    exit(0);
    174240}
     241if (defined($check_stacks) || defined($test_mode)) {
     242    $metadata_out{nsState} = 'TOWARP';
     243    &execute_stacks($date,"pretend");
     244    if ($metadata_out{nsState} eq 'FORCETOWARP') {
     245        $metadata_out{nsState} = 'TOWARP';
     246    }
     247    return_metadata($date);
     248    unless (defined($test_mode)) { exit(0); }
     249}
    175250if (defined($queue_stacks)) {
    176251    $metadata_out{nsState} = 'STACKING';
    … …  
    179254    exit(0);
    180255}
    181 
     256if (defined($check_diffs) || defined($queue_diffs)) {
     257    die("Diffs are currently not implemented.");
     258}
     259if (defined($clean_old) || defined($test_mode)) {
     260    if (defined($test_mode)) {
     261        &execute_cleans($date,"pretend");
     262    }
     263    else {
     264        &execute_cleans($date);
     265    }
     266    unless (defined($test_mode)) { exit(0); }
     267}
     268exit(0);
     269#
    182270# Registration
     271################################################################################
    183272
    184273sub check_summit_copy {
    … …  
    240329        }
    241330    }
    242    
    243     return($Nsummit_exps,$Nsummit_faults + $Ndownload_faults + $Nregister_faults);
     331
     332    my $Nfaults = $Nsummit_faults + $Ndownload_faults + $Nregister_faults;
     333    if ($Nsummit_exps == 0) {
     334        print STDERR "No exposures were found on the summit for $date.\n";
     335        $metadata_out{nsState} = 'DROP';
     336    }
     337    elsif ($Nfaults != 0) {
     338        print STDERR "There were faults while downloading the exposures for $date.\n";
     339        $metadata_out{nsState} = 'NEW';
     340    }
     341    else {
     342        print STDERR "Summit copy and Registration have succeeded for $date.\n";
     343        $metadata_out{nsState} = 'REGISTERED';
     344    }
     345   
     346    return($Nsummit_exps,$Nfaults);
    244347}       
    245348
     349#
    246350# Burntool
     351################################################################################
    247352
    248353sub get_goodBTvalue {
    … …  
    325430}
    326431   
    327 
     432#
    328433# Chips
     434################################################################################
    329435
    330436sub construct_chiptool_cmd {
    331437    my $date = shift;
    332438    my $target = shift;
     439
     440    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target);
    333441   
    334442    my $select =  "-dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 ";
    … …  
    337445    my $cmd = "$chiptool";
    338446    $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp ';
    339     $cmd .= " -set_label ${target}.nightlystack ";
    340     $cmd .= " -set_workdir neb://\@HOST\@.0/gpc1/${target}.nightlystack -set_dist_group $target ";
    341     $cmd .= " -set_tess_id $target -comment ${target}% -set_data_group ${target}.${date} ";
     447    $cmd .= " -set_label $label ";
     448    $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
     449    $cmd .= " -set_tess_id $tess_id -comment '$comment' -set_data_group $data_group ";
    342450    $cmd .= " $select ";
    343451    if ($debug == 1) {
    … …  
    355463    my $db = init_gpc_db();
    356464    $date =~ s/-//g;
    357     my $data_group = "${target}.${date}";
     465    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target);
    358466
    359467    my $sth = "SELECT exp_id from chipRun where data_group = '$data_group' AND exp_id = $exp_id";
    … …  
    431539}
    432540
     541#
    433542# Stacking
     543################################################################################
    434544
    435545sub construct_stacktool_cmd {
    … …  
    437547    my $target = shift;
    438548    my $filter = shift;
     549
     550    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target);
    439551
    440552    my $select =  "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
    … …  
    444556#    $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 ';  # Probably silly, but I want to be safe and not overwrite
    445557    $cmd .= ' -simple -dbname gpc1 -definebyquery '; 
    446     $cmd .= " -set_label ${target}.nightlystack -select_label ${target}.nightlystack ";
    447     $cmd .= " -set_workdir neb://\@HOST\@.0/gpc1/${target}.nightlystack -set_dist_group $target ";
    448     $cmd .= " -select_filter $filter -set_data_group ${target}.${date} ";
     558    $cmd .= " -set_label $label -select_label $label ";
     559    $cmd .= " -set_workdir $workdir -set_dist_group $dist_group ";
     560    $cmd .= " -select_filter $filter -set_data_group $data_group ";
    449561    $cmd .= " $select ";
    450562    if ($debug == 1) {
    … …  
    463575    my $db = init_gpc_db();
    464576    $date =~ s/-//g;
    465     my $data_group = "${target}.${date}";
     577    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target);
    466578
    467579    my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter'";
    … …  
    476588    my $filter = shift;
    477589
     590    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target);
    478591    # check warp stage == chip stage
    479592    my $db = init_gpc_db();
    … …  
    481594    my $trunc_date = $date; $trunc_date =~ s/-//g;
    482595
    483     my $where = " label = '${target}.nightlystack' AND data_group = '${target}.${trunc_date}' ";
     596    my $where = " label = '$label' AND data_group = '$data_group' ";
    484597    my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where ";
    485598    my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where ";
    … …  
    528641    my $pretend = shift;
    529642    foreach my $target (@target_list) {
    530         foreach my $filter (@filter_list) {
    531             my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
    532             if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
    533                 print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
    534                 $metadata_out{nsState} = 'FORCETOWARP';
    535                 next;
    536             }
    537             if ($Nexposures == 0) {
    538                 print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n";
    539                 next;
    540             }
    541             if ($Nalready != 0) {
    542                 print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
    543                 next;
    544             }
    545             unless ($metadata_out{nsState} eq 'FORCETOWARP') {
    546                 $metadata_out{nsState} = 'QUEUESTACKS';
    547             }
    548             unless(defined($pretend)) {
    549                 print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n";
    550                 stack_queue($date,$target,$filter);
    551             }
    552         }
    553     }
    554    
    555 }
    556 
     643        if ($stackable_list{$target} == 1) {
     644            foreach my $filter (@filter_list) {
     645                my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
     646                if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
     647                    print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
     648                    $metadata_out{nsState} = 'FORCETOWARP';
     649                    next;
     650                }
     651                if ($Nexposures == 0) {
     652                    print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n";
     653                    next;
     654                }
     655                if ($Nalready != 0) {
     656                    print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
     657                    next;
     658                }
     659                unless ($metadata_out{nsState} eq 'FORCETOWARP') {
     660                    $metadata_out{nsState} = 'QUEUESTACKS';
     661                }
     662                unless(defined($pretend)) {
     663                    print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n";
     664                    stack_queue($date,$target,$filter);
     665                }
     666            }
     667        }
     668        else {
     669            print STDERR "execute_stacks: Target $target is not auto-stackable.\n";
     670        }
     671    }
     672   
     673}
     674
     675#
     676# Auto-Clean
     677################################################################################
     678
     679sub construct_cleantool_args {
     680    my $date = shift;
     681    my $target = shift;
     682   
     683    my ($year,$month,$day) = split /-/,$date;
     684    my $dt = DateTime->new(year => $year, month => $month, day => $day,
     685                               hour => 0, minute => 0, second => 0, nanosecond => 0,
     686                               time_zone => 'Pacific/Honolulu');
     687    $dt->subtract(days => $retention_time);
     688    my $cleaning_date = $dt->ymd;
     689
     690    my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($cleaning_date,$target);
     691
     692    my $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -data_group $data_group ";
     693    if ($debug == 1) {
     694        $args .= ' -pretend ';
     695    }
     696    return($cleaning_date,$args);
     697}
     698
     699sub execute_cleans {
     700    my $date = shift;
     701    my $pretend = shift;
     702
     703    foreach my $target (@target_list) {
     704        my ($cleaning_date,$args) = construct_cleantool_args($date,$target);
     705
     706        my $command = $chiptool . $args;
     707        print STDERR "$command\n";
     708        if (!(defined($pretend) || $debug == 1)) {
     709#           print STDERR "BEAR IS DRIVING!?\n";
     710            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     711                run ( command => $command, verbose => $verbose );
     712            unless ($success) {
     713                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     714                &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     715            }
     716        }
     717        $command = $warptool . $args;
     718        print STDERR "$command\n";
     719        if (!(defined($pretend) || $debug == 1)) {
     720#           print STDERR "BEAR IS DRIVING $pretend $debug!?\n";
     721            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     722                run ( command => $command, verbose => $verbose );
     723            unless ($success) {
     724                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     725                &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     726            }
     727        }
     728    }
     729    return(0);
     730}
     731       
     732       
     733           
     734
     735#
    557736# Utilities
     737################################################################################
     738
     739sub get_tool_parameters {
     740    my $date = shift;
     741    my $target = shift;
     742    my $workdir_date = $date; $workdir_date =~ s%-%/%g;
     743    my $trunc_date = $date; $trunc_date =~ s/-//g;
     744
     745    my $label = "${target}.nightlyscience";
     746    my $workdir = "neb://\@HOST\@.0/gpc1/${target}.nt/${workdir_date}";
     747    my $comment = $comment_list{$target};
     748    my $dist_group = $target;
     749    my $data_group = "${target}.${trunc_date}";
     750    my $tess_id = $tessID_list{$target};
     751    return($label,$workdir,$comment,$tess_id,$dist_group,$data_group);
     752}
     753
    558754
    559755sub init_gpc_db {

  • branches/eam_branches/20091201/ippScripts/scripts/diff_skycell.pl

    r26753 r26873  
    3535}
    3636
    37 my ($diff_id, $dbname, $threads, $outroot, $reduction, $inverse, $verbose, $no_update, $no_op, $redirect);
     37my ($diff_id, $dbname, $threads, $outroot, $reduction, $inverse, $run_state, $verbose, $no_update, $no_op, $redirect);
    3838my ($skycell_id, $diff_skyfile_id);
    3939GetOptions(
    … …  
    4343    'dbname|d=s'        => \$dbname, # Database name
    4444    'threads=s'         => \$threads,   # Number of threads to use
     45    'run-state=s'       => \$run_state,   # state for run: 'new' or 'update'
    4546    'outroot=s'         => \$outroot, # Output root name
    4647    'inverse'           => \$inverse, # Make inverse subtraction?
    … …  
    5455pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    5556pod2usage(
    56     -msg => "Required options: --diff_id --skycell_id --outroot --diff_skyfile_id",
     57    -msg => "Required options: --diff_id --skycell_id --outroot --run-state --diff_skyfile_id",
    5758    -exitval => 3,
    5859          ) unless defined $diff_id
    5960    and defined $skycell_id
    6061    and defined $diff_skyfile_id
     62    and defined $run_state
    6163    and defined $outroot;
    6264
    … …  
    6668# my $logDest = $ipprc->filename("LOG.EXP", $outroot);
    6769my $logDest = "$outroot.log";
     70$logDest .= ".update" if $run_state eq "update";
    6871$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
    6972
    … …  
    155158# The output file will be considered magicked if the input has been magicked and the
    156159# template is either a stack or a warp that has been magicked.
    157 # note that difftool -inputskyfile outputs the magicked boolean as an int not T or F
    158 # because the output is constructed from a union of two selects
    159 my $magicked = $inputMagic && $templateMagic;
     160my $magicked = $inputMagic && $templateMagic ? $inputMagic : 0;
    160161
    161162# Recipes to use based on reduction class
    … …  
    216217my $traceDest = $ipprc->filename("TRACE.EXP", $outroot);
    217218
     219if ($run_state eq 'update') {
     220    $traceDest .= '.update';
     221    $outputStats .= '.update';
     222}
     223
    218224my ($inverseName, $inverseMask, $inverseVariance, $inverseSources);
    219225if ($inverse) {
    … …  
    227233
    228234# Perform subtraction
    229 unless ($no_op) {
     235{
    230236    my $command = "$ppSub $outroot";
    231237    $command .= " -inimage $input";
    … …  
    239245    $command .= " -stats $outputStats";
    240246    $command .= " -threads $threads" if defined $threads;
     247    if ($run_state eq "new") {
     248        $command .= " -dumpconfig $configuration";
     249    } else {
     250        my $configurationReal = $ipprc->file_resolve($configuration) or &my_die("Couldn't resolve configuration file: $configuration", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR);
     251        $command .= " -ipprc $configurationReal";
     252    }
    241253    $command .= " -recipe PPSUB $recipe_ppSub";
    242254    $command .= " -recipe PSPHOT $recipe_psphot";
    … …  
    245257    $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF";
    246258    $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF";
    247     $command .= " -photometry";
     259    if ($run_state eq "new") {
     260        $command .= " -photometry";
     261    }
    248262    $command .= " -inverse" if $inverse;
    249263    $command .= " -tracedest $traceDest -log $logDest";
    250     $command .= " -dumpconfig $configuration";
    251264    $command .= " -dbname $dbname" if defined $dbname;
    252265    $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id;
    253266    $command .= " -source_id $source_id" if defined $source_id;
    254267
    255     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    256         run(command => $command, verbose => $verbose);
    257     unless ($success) {
    258         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    259         &my_die("Unable to perform ppSub: $error_code", $diff_id, $skycell_id, $error_code);
    260     }
    261 
    262     my $outputStatsReal = $ipprc->file_resolve($outputStats);
    263     &my_die("Couldn't find expected output file: $outputStats", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal;
    264 
    265     # measure chip stats
    266     $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL";
    267     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    268         run(command => $command, verbose => $verbose);
    269     unless ($success) {
    270         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    271         &my_die("Unable to perform ppStatsFromMetadata: $error_code", $diff_id, $skycell_id, $error_code);
    272     }
    273     foreach my $line (@$stdout_buf) {
    274         $cmdflags .= " $line";
    275     }
    276     chomp $cmdflags;
    277 
    278     my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
    279 
    280     if (!$quality) {
    281         &my_die("Couldn't find expected output file: $outputName", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
    282         &my_die("Couldn't find expected output file: $outputMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
    283         &my_die("Couldn't find expected output file: $outputVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance);
    284         &my_die("Couldn't find expected output file: $outputSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
    285         &my_die("Couldn't find expected output file: $jpeg1Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg1Name);
    286         &my_die("Couldn't find expected output file: $jpeg2Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg2Name);
    287         if ($inverse) {
    288             &my_die("Couldn't find expected output file: $inverseName", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseName);
    289             &my_die("Couldn't find expected output file: $inverseMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseMask);
    290             &my_die("Couldn't find expected output file: $inverseVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseVariance);
    291             &my_die("Couldn't find expected output file: $inverseSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseSources);
    292         }
    293     }
    294 }
    295 
    296 unless ($no_update) {
    297 
    298     # Add the subtraction result
    299     {
    300         my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -path_base $outroot";
    301         $command .= " $cmdflags";
    302         $command .= " -magicked" if $magicked;
    303         $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    304         $command .= " -hostname $host" if defined $host;
    305         $command .= " -dbname $dbname" if defined $dbname;
    306 
     268    unless ($no_op) {
    307269        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    308270            run(command => $command, verbose => $verbose);
    309271        unless ($success) {
    310272            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    311             &my_die("Unable to perform difftool -adddiffskyfile: $error_code", $diff_id, $skycell_id, $error_code);
     273            &my_die("Unable to perform ppSub: $error_code", $diff_id, $skycell_id, $error_code);
     274        }
     275
     276        my $outputStatsReal = $ipprc->file_resolve($outputStats);
     277        &my_die("Couldn't find expected output file: $outputStats", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal;
     278
     279        # measure chip stats
     280        $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL";
     281        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     282            run(command => $command, verbose => $verbose);
     283        unless ($success) {
     284            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     285            &my_die("Unable to perform ppStatsFromMetadata: $error_code", $diff_id, $skycell_id, $error_code);
     286        }
     287        foreach my $line (@$stdout_buf) {
     288            $cmdflags .= " $line";
     289        }
     290        chomp $cmdflags;
     291
     292        my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
     293
     294        if (!$quality) {
     295            &my_die("Couldn't find expected output file: $outputName", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
     296            &my_die("Couldn't find expected output file: $outputMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
     297            &my_die("Couldn't find expected output file: $outputVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance);
     298            &my_die("Couldn't find expected output file: $outputSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
     299            &my_die("Couldn't find expected output file: $jpeg1Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg1Name);
     300            &my_die("Couldn't find expected output file: $jpeg2Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg2Name);
     301            if ($inverse) {
     302                &my_die("Couldn't find expected output file: $inverseName", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseName);
     303                &my_die("Couldn't find expected output file: $inverseMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseMask);
     304                &my_die("Couldn't find expected output file: $inverseVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseVariance);
     305                &my_die("Couldn't find expected output file: $inverseSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseSources);
     306            }
     307        }
     308    } else {
     309        print "Not executing: $command\n";
     310    }
     311}
     312
     313unless ($no_update) {
     314
     315    # Add the subtraction result
     316    {
     317        my $command = "$difftool -diff_id $diff_id -skycell_id $skycell_id";
     318        $command .= " -magicked $magicked" if $magicked;
     319        if ($run_state eq 'new') {
     320            $command .= " -adddiffskyfile -path_base $outroot";
     321            $command .= " $cmdflags";
     322            $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
     323            $command .= " -hostname $host" if defined $host;
     324        } else {
     325            $command .= " -tofullskyfile";
     326        }
     327        $command .= " -dbname $dbname" if defined $dbname;
     328
     329        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     330            run(command => $command, verbose => $verbose);
     331        unless ($success) {
     332            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     333            my $err_message = $run_state eq "update" ?
     334                "Unable to perform difftool -adddiffskyfile" :
     335                "Unable to perform difftool -tofullskyfile";
     336            &my_die("$err_message: $error_code", $diff_id, $skycell_id, $error_code);
    312337        }
    313338    }
    … …  
    326351    warn($msg);
    327352    if (defined $diff_id and defined $skycell_id and not $no_update) {
    328         my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -fault $exit_code";
    329         $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    330         $command .= " -hostname $host" if defined $host;
    331         $command .= " -path_base $outroot" if defined $outroot;
    332         $command .= " -dbname $dbname" if defined $dbname;
     353        my $command = "$difftool -diff_id $diff_id -skycell_id $skycell_id -fault $exit_code";
     354        if ($run_state eq 'new') {
     355            $command .= " -adddiffskyfile";
     356            $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
     357            $command .= " -hostname $host" if defined $host;
     358            $command .= " -path_base $outroot" if defined $outroot;
     359            $command .= " -dbname $dbname" if defined $dbname;
     360        } else {
     361            $command .= " -updatediffskyfile";
     362        }
    333363        run(command => $command, verbose => $verbose);
    334364    }

  • branches/eam_branches/20091201/ippScripts/scripts/ipp_image_path.pl

    r25207 r26873  
    1919use PS::IPP::Metadata::Config;
    2020use PS::IPP::Config 1.01 qw( :standard );
     21use Nebulous::Client;
    2122
    2223use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
    2324use Pod::Usage qw( pod2usage );
    2425
    25 my ($exp_name, $class_id, $from_registered, $dbname, $verbose);
     26my ($exp_name, $class_id, $from_registered, $dbname, $verbose, $alternate);
    2627
    2728GetOptions(
    … …  
    3132    'dbname|d=s'    => \$dbname, # Database name   
    3233    'verbose'       => \$verbose,   # Print to stdout
     34    'alternate'     => \$alternate,
    3335) or pod2usage( 2 );
    3436
    … …  
    99101                $path = resolve_ipp008_file($uri);
    100102            } elsif ($use_017_workaround && ($uri =~ /4683/) &&
    101                     ($uri =~ /ipp017/)) {
     103                     ($uri =~ /ipp017/)) {
    102104                $path = resolve_ipp017_file($uri);
    103             } else {
     105            } elsif ($alternate) {
     106                my $neb = $ipprc->nebulous();
     107                my $uris = $neb->find_instances($uri,'any');
     108                my @files = map {URI->new($_)->file if $_} @$uris;
     109
     110                if ($#files > 0) {
     111                    $path = $files[1];
     112                }
     113                else {
     114                    $path = $files[0];
     115                }
     116
     117            }
     118            else {
    104119                $path = $ipprc->file_resolve($uri);
    105120            }
Note: See TracChangeset for help on using the changeset viewer.