IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Feb 24, 2009, 12:00:25 PM (17 years ago)
Author:
Paul Price
Message:

Adding handling for missed exceptions, similar to register_imfile.pl --- the idea is to have exceptions trigger the my_die functions so that errors are recorded in the database, and don't go on forever.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/detrend_norm_calc.pl

    r19374 r22430  
    2525use Pod::Usage qw( pod2usage );
    2626
     27# Look for programs we need
     28my $missing_tools;
     29my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
     30my $ppNormCalc = can_run('ppNormCalc') or (warn "Can't find ppNormCalc" and $missing_tools = 1);
     31if ($missing_tools) {
     32    warn("Can't find required tools.");
     33    exit($PS_EXIT_CONFIG_ERROR);
     34}
     35
    2736# Parse command-line arguments
    2837my ($det_id, $iter, $detType, $outroot, $dbname, $verbose, $no_update, $no_op, $redirect );
    2938GetOptions(
    30     'det_id|d=s'        => \$det_id,    # Detrend id                         
    31     'iteration|i=s'     => \$iter,      # Iteration                         
    32     'det_type|t=s'      => \$detType,   # Detrend type                       
    33     'outroot|w=s'       => \$outroot,   # output file base name
    34     'dbname|d=s'        => \$dbname,    # Database name                     
    35     'verbose'           => \$verbose,   # Print to stdout
    36     'no-update'         => \$no_update, # Don't update the database?         
    37     'no-op'             => \$no_op,     # Don't do operations               
     39    'det_id|d=s'        => \$det_id,    # Detrend id
     40    'iteration|i=s'     => \$iter,      # Iteration
     41    'det_type|t=s'      => \$detType,   # Detrend type
     42    'outroot|w=s'       => \$outroot,   # output file base name
     43    'dbname|d=s'        => \$dbname,    # Database name
     44    'verbose'           => \$verbose,   # Print to stdout
     45    'no-update'         => \$no_update, # Don't update the database?
     46    'no-op'             => \$no_op,     # Don't do operations
    3847    'redirect-output'   => \$redirect,
    3948    ) or pod2usage( 2 );
    … …  
    4150pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    4251pod2usage( -msg => "Required options --det_id --iteration --det_type --outroot",
    43            -exitval => 3,
    44            ) unless
    45     defined $det_id  and 
    46     defined $iter    and 
     52           -exitval => 3,
     53           ) unless
     54    defined $det_id  and
     55    defined $iter    and
    4756    defined $detType and
    4857    defined $outroot;
    4958
     59# Unhandled exceptions should be passed on to my_die so they get pushed into the database
     60$SIG{__DIE__} = sub { die @_ if $^S;
     61                      my_die( $_[0], $det_id, $iter, $PS_EXIT_UNKNOWN_ERROR ); };
     62
    5063my $logfile = $outroot . ".log";
    5164
    … …  
    5669# Define which detrend types we normalise
    5770use constant NORMALIZE => {
    58     'bias'             => 0,
    59     'dark'             => 0,   
     71    'bias'             => 0,
     72    'dark'             => 0,
    6073    'dark_premask'     => 0,
    61     'shutter'          => 0,
     74    'shutter'          => 0,
    6275    'flat_premask'     => 1,
    6376    'domeflat_premask' => 1,
    … …  
    6679    'domeflat_raw'     => 1,
    6780    'skyflat_raw'      => 1,
    68     'flat'             => 1,
    69     'domeflat'         => 1,
    70     'skyflat'          => 1,   
    71     'fringe'           => 0,   
    72     'mask'             => 0,
    73     'darkmask'         => 0,
    74     'flatmask'         => 0,
    75     };
    76 
    77 # Look for programs we need
    78 my $missing_tools;
    79 my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
    80 my $ppNormCalc = can_run('ppNormCalc') or (warn "Can't find ppNormCalc" and $missing_tools = 1);
    81 if ($missing_tools) {
    82     warn("Can't find required tools.");
    83     exit($PS_EXIT_CONFIG_ERROR);
    84 }
     81    'flat'             => 1,
     82    'domeflat'         => 1,
     83    'skyflat'          => 1,
     84    'fringe'           => 0,
     85    'mask'             => 0,
     86    'darkmask'         => 0,
     87    'flatmask'         => 0,
     88    };
    8589
    8690&my_die("Unrecognised detrend type: $detType", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless exists NORMALIZE()->{lc($detType)};
    … …  
    8993
    9094# Get the list of inputs
    91 my @files;                      # The input files
     95my @files;                      # The input files
    9296{
    9397    my $command = "$dettool -processedimfile";
    … …  
    99103    print "Running [$command]...\n" if $verbose;
    100104    if (not run(\@command, \$stdin, \$stdout, \$stderr)) {
    101         &my_die("Unable to perform dettool -processedimfile on detrend $det_id/$iter: $?",
    102                 $det_id, $iter, $PS_EXIT_SYS_ERROR);
     105        &my_die("Unable to perform dettool -processedimfile on detrend $det_id/$iter: $?",
     106                $det_id, $iter, $PS_EXIT_SYS_ERROR);
    103107    }
    104108    print $stdout . "\n" if $verbose;
    105    
     109
    106110    # Because of the length, need to split into individual metadatas --- it parses SO much quicker!
    107111    my @whole = split /\n/, $stdout;
    108112    my @single = ();
    109113    while ( scalar @whole > 0 ) {
    110         my $value = shift @whole;
    111         push @single, $value;
    112         if ($value =~ /^\s*END\s*$/) {
    113             push @single, "\n";
    114 
    115             my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
    116             &my_die("Unable to parse output from dettool", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless
    117                 defined $list;
    118             push @files, @$list;
    119             @single = ();
    120         }
    121     }
    122 }
    123 
    124 my $norms;                      # MDC with normalisations
     114        my $value = shift @whole;
     115        push @single, $value;
     116        if ($value =~ /^\s*END\s*$/) {
     117            push @single, "\n";
     118
     119            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
     120            &my_die("Unable to parse output from dettool", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless
     121                defined $list;
     122            push @files, @$list;
     123            @single = ();
     124        }
     125    }
     126}
     127
     128my $norms;                      # MDC with normalisations
    125129if (NORMALIZE()->{lc($detType)} and not $no_op) {
    126130
    127131    my %matrix; # Matrix of statistics as a function of exposures and classes
    128132    foreach my $file (@files) {
    129         my $exp_id = $file->{'exp_id'}; # Exposure ID
    130         my $class_id = $file->{'class_id'}; # Class ID
    131         my $stat = $file->{STATISTIC()}; # Statistic of interest
    132        
    133         # Create matrix elements
    134         $matrix{$exp_id} = {} if not defined $matrix{$exp_id};
    135         $matrix{$exp_id}->{$class_id} = $stat;
    136     }
    137    
     133        my $exp_id = $file->{'exp_id'}; # Exposure ID
     134        my $class_id = $file->{'class_id'}; # Class ID
     135        my $stat = $file->{STATISTIC()}; # Statistic of interest
     136
     137        # Create matrix elements
     138        $matrix{$exp_id} = {} if not defined $matrix{$exp_id};
     139        $matrix{$exp_id}->{$class_id} = $stat;
     140    }
     141
    138142    # Generate the input for ppNormCalc
    139     my $normData;                       # Normalisation data
     143    my $normData;                       # Normalisation data
    140144    foreach my $exp_id (keys %matrix) {
    141         $normData .= "$exp_id\tMETADATA\n";
    142         foreach my $class_id (keys %{$matrix{$exp_id}}) {
    143             $normData .= "\t" . $class_id . "\tF32\t" . $matrix{$exp_id}->{$class_id} . "\n";
    144         }
    145         $normData .= "END\n\n";
     145        $normData .= "$exp_id\tMETADATA\n";
     146        foreach my $class_id (keys %{$matrix{$exp_id}}) {
     147            $normData .= "\t" . $class_id . "\tF32\t" . $matrix{$exp_id}->{$class_id} . "\n";
     148        }
     149        $normData .= "END\n\n";
    146150    }
    147151
    148152    # Run ppNormCalc
    149153    {
    150         my ( $stdout, $stderr ); # Buffers for running program
    151         my @command = split /\s+/, $ppNormCalc;
    152         print "Running [$ppNormCalc]...\n" if $verbose;
    153         if (not run(\@command, \$normData, \$stdout, \$stderr)) {
    154             &my_die("Unable to perform ppNormCalc: $?", $det_id, $iter, $PS_EXIT_SYS_ERROR);
    155         }
    156         print $stdout . "\n" if $verbose;
    157        
    158         # Parse the output
    159         $norms = $mdcParser->parse($stdout);
    160         &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless $norms;
     154        my ( $stdout, $stderr ); # Buffers for running program
     155        my @command = split /\s+/, $ppNormCalc;
     156        print "Running [$ppNormCalc]...\n" if $verbose;
     157        if (not run(\@command, \$normData, \$stdout, \$stderr)) {
     158            &my_die("Unable to perform ppNormCalc: $?", $det_id, $iter, $PS_EXIT_SYS_ERROR);
     159        }
     160        print $stdout . "\n" if $verbose;
     161
     162        # Parse the output
     163        $norms = $mdcParser->parse($stdout);
     164        &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless $norms;
    161165    }
    162166
    163167} else {
    164168    # It's something that doesn't need normalisation --- just push in a normalisation of 1
    165     my %classes;                # List of unique classes
     169    my %classes;                # List of unique classes
    166170    foreach my $file (@files) {
    167         my $class_id = $file->{'class_id'}; # Class Id
    168         $classes{$class_id} = 1;
    169     }
    170    
     171        my $class_id = $file->{'class_id'}; # Class Id
     172        $classes{$class_id} = 1;
     173    }
     174
    171175    foreach my $class_id (keys %classes) {
    172         my %mdValue;    # Metadata value for this class id
    173         $mdValue{name} = $class_id;
    174         $mdValue{value} = 1.0;
    175         push @$norms, \%mdValue;
     176        my %mdValue;    # Metadata value for this class id
     177        $mdValue{name} = $class_id;
     178        $mdValue{value} = 1.0;
     179        push @$norms, \%mdValue;
    176180    }
    177181}
    … …  
    186190    foreach my $normItem (@$norms) {
    187191
    188         my $className = $normItem->{name}; # Name of component
    189         my $normalisation = $normItem->{value}; # Normalisation for component
    190 
    191         if ($normalisation == 0.0 or lc($normalisation) eq 'nan') {
    192             warn("Class $className has bad normalisation: $normalisation");
    193             exit($PS_EXIT_SYS_ERROR);
    194         }
    195 
    196         my $command = $commandBase;
    197         $command .= " -class_id $className";
    198         $command .= " -norm $normalisation";
    199 
    200         my @command = split /\s+/, $command;
    201 
    202         my ( $stdin, $stdout, $stderr ); # Buffers for running program
    203         print "Running [$command]...\n" if $verbose;
    204         if (not run \@command, \$stdin, \$stdout, \$stderr) {
    205             warn("Unable to perform dettool -addnormstat for $className: $?");
    206             exit($PS_EXIT_SYS_ERROR);
    207         }
    208         print $stdout . "\n" if $verbose;
     192        my $className = $normItem->{name}; # Name of component
     193        my $normalisation = $normItem->{value}; # Normalisation for component
     194
     195        if ($normalisation == 0.0 or lc($normalisation) eq 'nan') {
     196            warn("Class $className has bad normalisation: $normalisation");
     197            exit($PS_EXIT_SYS_ERROR);
     198        }
     199
     200        my $command = $commandBase;
     201        $command .= " -class_id $className";
     202        $command .= " -norm $normalisation";
     203
     204        my @command = split /\s+/, $command;
     205
     206        my ( $stdin, $stdout, $stderr ); # Buffers for running program
     207        print "Running [$command]...\n" if $verbose;
     208        if (not run \@command, \$stdin, \$stdout, \$stderr) {
     209            warn("Unable to perform dettool -addnormstat for $className: $?");
     210            exit($PS_EXIT_SYS_ERROR);
     211        }
     212        print $stdout . "\n" if $verbose;
    209213    }
    210214} else {
    … …  
    212216    foreach my $normItem (@$norms) {
    213217
    214         my $className = $normItem->{name}; # Name of component
    215         my $normalisation = $normItem->{value}; # Normalisation for component
    216 
    217         if ($normalisation == 0.0 or lc($normalisation) eq 'nan') {
    218             warn("Class $className has bad normalisation: $normalisation");
    219             exit($PS_EXIT_SYS_ERROR);
    220         }
    221         print "$className : $normalisation\n";
     218        my $className = $normItem->{name}; # Name of component
     219        my $normalisation = $normItem->{value}; # Normalisation for component
     220
     221        if ($normalisation == 0.0 or lc($normalisation) eq 'nan') {
     222            warn("Class $className has bad normalisation: $normalisation");
     223            exit($PS_EXIT_SYS_ERROR);
     224        }
     225        print "$className : $normalisation\n";
    222226    }
    223227}
    … …  
    225229sub my_die
    226230{
    227     my $msg = shift;            # Warning message on die
    228     my $det_id = shift;         # Detrend identifier
    229     my $iter = shift;           # Iteration
    230     my $exit_code = shift;      # Exit code to add
     231    my $msg = shift;            # Warning message on die
     232    my $det_id = shift;         # Detrend identifier
     233    my $iter = shift;           # Iteration
     234    my $exit_code = shift;      # Exit code to add
    231235
    232236    carp($msg);
    233237    if (defined $det_id and defined $iter and not $no_update) {
    234         my $command = "$dettool -addnormalizedstat";
    235         $command .= " -det_id $det_id";
    236         $command .= " -iteration $iter";
    237         # XXX EAM : we should add this to the db : $command .= " -path_base $outroot";
    238         $command .= " -code $exit_code";
    239         $command .= " -dbname $dbname" if defined $dbname;
     238        my $command = "$dettool -addnormalizedstat";
     239        $command .= " -det_id $det_id";
     240        $command .= " -iteration $iter";
     241        # XXX EAM : we should add this to the db : $command .= " -path_base $outroot";
     242        $command .= " -code $exit_code";
     243        $command .= " -dbname $dbname" if defined $dbname;
    240244        system ($command);
    241245    }
Note: See TracChangeset for help on using the changeset viewer.