IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 11837


Ignore:
Timestamp:
Feb 15, 2007, 2:23:33 PM (19 years ago)
Author:
Paul Price
Message:

Adding --no-op option which turns off processing (so one can track only the database work flow) for everything except the phase 0 scripts (need to have some real data in the database in order to track it).

Location:
trunk/ippScripts/scripts
Files:
14 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/detrend_norm_apply.pl

    r11830 r11837  
    3939    $dbname,                    # Database name
    4040    $workdir,                   # Working directory, for output files
    41     $no_update                  # Don't update the database
     41    $no_update,                 # Don't update the database
     42    $no_op,                     # Don't do any operations
    4243    );
    4344GetOptions(
    … …  
    5152    'dbname|d=s'        => \$dbname,
    5253    'workdir|w=s'       => \$workdir, # Working directory, for output files
    53     'no-update'         => \$no_update
     54    'no-update'         => \$no_update,
     55    'no-op'             => \$no_op,
    5456    ) or pod2usage( 2 );
    5557   
    … …  
    100102
    101103# Run normalisation
    102 {
     104my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
     105unless ($no_op) {
    103106    my $command = "$ppImage -file $input $outputRoot -norm $value -stat $statsName -recipe PPIMAGE " . RECIPE(); # Command to run
    104107    $command .= ' -isfringe' if lc($det_type) eq 'fringe';
    … …  
    113116    &my_die("Can't find expected output file: $b2name", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -e $b2name;
    114117    &my_die("Can't find expected output file: $statsName", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -e $statsName;
    115 }
    116 
    117 # Get the statistics on the normalised image
    118 my $stats;                      # Statistics from ppImage
    119 {
     118   
     119    # Get the statistics on the normalised image
    120120    my $statsFile;              # File handle
    121121    open $statsFile, $statsName or &my_die("Can't open statistics file $statsName: $!\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR);
    … …  
    124124    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
    125125    my $metadata = $mdcParser->parse(join "", @contents)
    126         or &my_die("Unable to parse metadata config", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR);
    127     $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
     126        or &my_die("Unable to parse metadata config", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR);
    128127    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR);
    129 }
     128}   
    130129
    131130# Update the database
    … …  
    133132$b1name = $ipprc->convert_filename_relative( $b1name );
    134133$b2name = $ipprc->convert_filename_relative( $b2name );
     134
     135my $bg = ($stats->bg_mean() or 'NAN');
     136my $bg_stdev = ($stats->bg_stdev() or 'NAN');
     137my $bg_mean_stdev = ($stats->bg_mean_stdev() or 'NAN');
     138
    135139unless ($no_update) {
    136140    my $command = "$dettool -addnormalizedimfile -det_id $det_id -iteration $iter -class_id $class_id ".
    137141        "-uri $output -b1_uri $outputRoot"; # Command to run
    138     # Add the statistics triplet
    139     $command .= " -bg " . $stats->bg_mean();
    140     if (defined($stats->bg_mean_stdev())) {
    141         $command .= " -bg_mean_stdev " . $stats->bg_mean_stdev();
    142     } else {
    143         # May be undefined if there's only a single imfile
    144         $command .= " -bg_mean_stdev 0";
    145     }
    146     $command .= " -bg_stdev " . $stats->bg_stdev();
    147 
     142    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
    148143    $command .= " -dbname $dbname" if defined $dbname;
    149144

  • trunk/ippScripts/scripts/detrend_norm_calc.pl

    r11831 r11837  
    3131
    3232# Parse command-line arguments
    33 my ($detId,                     # Detrend id
     33my ($det_id,                    # Detrend id
    3434    $iter,                      # Iteration
    3535    $detType,                   # Detrend type
    3636    $workdir,                   # Working directory for output files
    3737    $dbname,                    # Database name
    38     $no_update                  # Don't update the database?
     38    $no_update,                 # Don't update the database?
     39    $no_op,                     # Don't do operations
    3940    );
    4041GetOptions(
    41         'det_id|d=s'    => \$detId,
     42        'det_id|d=s'    => \$det_id,
    4243        'iteration|i=s' => \$iter,
    4344        'det_type|t=s'  => \$detType,
    4445        'workdir|w=s'   => \$workdir,
    4546        'dbname|d=s'    => \$dbname,# Database name
    46         'no-update'     => \$no_update
     47        'no-update'     => \$no_update,
     48        'no-op'         => \$no_op,
    4749        ) or pod2usage( 2 );
    4850
    … …  
    5052pod2usage( -msg => "Required options --det_id --iteration --det_type",
    5153           -exitval => 3,
    52            ) unless defined $detId
     54           ) unless defined $det_id
    5355    and defined $iter
    5456    and defined $detType;
    … …  
    8587my @files;                      # The input files
    8688{
    87     my $command = "$dettool -processedimfile -det_id $detId"; # Command to run
     89    my $command = "$dettool -processedimfile -det_id $det_id"; # Command to run
    8890    $command .= " -dbname $dbname" if defined $dbname;
    8991    my @command = split /\s+/, $command;
    … …  
    9193    print "Running [$command]...\n";
    9294    if (not run(\@command, \$stdin, \$stdout, \$stderr)) {
    93         &my_die("Unable to perform dettool -processedimfile on detrend $detId/$iter: $?",
     95        &my_die("Unable to perform dettool -processedimfile on detrend $det_id/$iter: $?",
    9496                $det_id, $iter, $PS_EXIT_SYS_ERROR);
    9597    }
    … …  
    115117
    116118my $norms;                      # MDC with normalisations
    117 if (NORMALIZE()->{lc($detType)}) {
     119if (NORMALIZE()->{lc($detType)} and not $no_op) {
    118120
    119121    my %matrix; # Matrix of statistics as a function of exposures and classes
    120122    foreach my $file (@files) {
    121         my $expTag = $file->{'exp_tag'}; # Exposure ID
    122         my $classId = $file->{'class_id'}; # Class ID
     123        my $exp_tag = $file->{'exp_tag'}; # Exposure ID
     124        my $class_id = $file->{'class_id'}; # Class ID
    123125        my $stat = $file->{STATISTIC()}; # Statistic of interest
    124126       
    125127        # Create matrix elements
    126         $matrix{$expTag} = {} if not defined $matrix{$expTag};
    127         $matrix{$expTag}->{$classId} = $stat;
     128        $matrix{$exp_tag} = {} if not defined $matrix{$exp_tag};
     129        $matrix{$exp_tag}->{$class_id} = $stat;
    128130    }
    129131   
    130132    # Generate the input for ppNormCalc
    131133    my $normData;                       # Normalisation data
    132     foreach my $expTag (keys %matrix) {
    133         $normData .= "$expTag\tMETADATA\n";
    134         foreach my $classId (keys %{$matrix{$expTag}}) {
    135             $normData .= "\t" . $classId . "\tF32\t" . $matrix{$expTag}->{$classId} . "\n";
     134    foreach my $exp_tag (keys %matrix) {
     135        $normData .= "$exp_tag\tMETADATA\n";
     136        foreach my $class_id (keys %{$matrix{$exp_tag}}) {
     137            $normData .= "\t" . $class_id . "\tF32\t" . $matrix{$exp_tag}->{$class_id} . "\n";
    136138        }
    137139        $normData .= "END\n\n";
    … …  
    157159    my %classes;                # List of unique classes
    158160    foreach my $file (@files) {
    159         my $classId = $file->{'class_id'}; # Class Id
    160         $classes{$classId} = 1;
     161        my $class_id = $file->{'class_id'}; # Class Id
     162        $classes{$class_id} = 1;
    161163    }
    162164   
    163     foreach my $classId (keys %classes) {
     165    foreach my $class_id (keys %classes) {
    164166        my %mdValue;    # Metadata value for this class id
    165         $mdValue{name} = $classId;
     167        $mdValue{name} = $class_id;
    166168        $mdValue{value} = 1.0;
    167169        push @$norms, \%mdValue;
    … …  
    176178        my $normalisation = $normItem->{value}; # Normalisation for component
    177179
    178         my $command = "$dettool -addnormalizedstat -det_id $detId -iteration $iter -class_id $className ".
     180        my $command = "$dettool -addnormalizedstat -det_id $det_id -iteration $iter -class_id $className ".
    179181            "-norm $normalisation"; # Command to run
    180182        $command .= " -dbname $dbname" if defined $dbname;
    … …  
    199201
    200202    warn($msg);
    201     if ($det_id and $iter and $class_id and not $no_update) {
     203    if ($det_id and $iter and not $no_update) {
    202204        my $command = "$dettool -addnormalizedstat -det_id $det_id -iteration $iter -code $exit_code";
    203205        $command .= " -dbname $dbname" if defined $dbname;

  • trunk/ippScripts/scripts/detrend_norm_exp.pl

    r11829 r11837  
    3333use Pod::Usage qw( pod2usage );
    3434
    35 my ($det_id, $iter, $det_type, $camera, $dbname, $workdir, $no_update);
     35my ($det_id, $iter, $det_type, $camera, $dbname, $workdir, $no_update, $no_op);
    3636GetOptions(
    3737    'det_id|d=s'        => \$det_id,
    … …  
    4141    'dbname|d=s'        => \$dbname, # Database name
    4242    'workdir|w=s'       => \$workdir,   # Working directory, for output files
    43     'no-update'         => \$no_update
     43    'no-update'         => \$no_update,
     44    'no-op'             => \$no_op,
    4445) or pod2usage( 2 );
    4546
    … …  
    104105        my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
    105106        $stats->add_data(@backgrounds);
    106         $bg = $stats->mean();
    107         $bg_mean_stdev = $stats->standard_deviation() || 0.0;
     107        $bg = ($stats->mean() or 'NAN');
     108        $bg_mean_stdev = ($stats->standard_deviation() or 'NAN');
    108109    }
    109110    {
    110111        my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
    111112        $stats->add_data(@variances);
    112         $bg_stdev = sqrt( $stats->mean() );
     113        $bg_stdev = (sqrt( $stats->mean() ) or 'NAN');
    113114    }
    114115}
    … …  
    147148my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2
    148149
    149 # Make the jpeg for binning 1
    150 {
    151     my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE " . RECIPE1; # Command to run
    152     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    153         run(command => $command, verbose => 1);
    154     &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $jpeg1Name;
    155 }
    156 
    157 # Make the jpeg for binning 2
    158 {
    159     my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE " . RECIPE2; # Command to run
    160     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    161         run(command => $command, verbose => 1);
    162     &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $jpeg2Name;
     150unless ($no_op) {
     151    # Make the jpeg for binning 1
     152    {
     153        my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE " . RECIPE1; # Command to run
     154        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     155            run(command => $command, verbose => 1);
     156        &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $jpeg1Name;
     157    }
     158   
     159    # Make the jpeg for binning 2
     160    {
     161        my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE " . RECIPE2; # Command to run
     162        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     163            run(command => $command, verbose => 1);
     164        &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $jpeg2Name;
     165    }
    163166}
    164167

  • trunk/ippScripts/scripts/detrend_process_exp.pl

    r11829 r11837  
    3333use Pod::Usage qw( pod2usage );
    3434
    35 my ($det_id, $exp_tag, $camera, $dbname, $workdir, $no_update);
     35my ($det_id, $exp_tag, $camera, $dbname, $workdir, $no_update, $no_op);
    3636GetOptions(
    3737    'det_id|d=s'        => \$det_id,
    … …  
    4040    'dbname|d=s'        => \$dbname, # Database name
    4141    'workdir|w=s'       => \$workdir,   # Working directory, for output files
    42     'no-update'         => \$no_update
     42    'no-update'         => \$no_update,
     43    'no-op'             => \$no_op,
    4344) or pod2usage( 2 );
    4445
    … …  
    102103        my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
    103104        $stats->add_data(@backgrounds);
    104         $bg = $stats->mean();
    105         $bg_mean_stdev = $stats->standard_deviation() || 0.0;
     105        $bg = ($stats->mean() or 'NAN');
     106        $bg_mean_stdev = ($stats->standard_deviation() or 'NAN');
    106107    }
    107108    {
    108109        my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
    109110        $stats->add_data(@variances);
    110         $bg_stdev = sqrt( $stats->mean() );
     111        $bg_stdev = (sqrt( $stats->mean() ) or 'NAN');
    111112    }
    112113}
    … …  
    143144my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2
    144145
    145 # Make the jpeg for binning 1
    146 {
    147     my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE " . RECIPE1; # Command to run
    148     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    149         run(command => $command, verbose => 1);
    150     unless ($success) {
    151         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    152         &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_tag, $error_code);
    153     }
    154     &my_die("Unable to find expected output file: $jpeg1", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $jpeg1;
    155 }
    156 
    157 # Make the jpeg for binning 2
    158 {
    159     my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE " . RECIPE2; # Command to run
    160     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    161         run(command => $command, verbose => 1);
    162     unless ($success) {
    163         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    164         &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_tag, $error_code);
    165     }
    166     &my_die("Unable to find expected output file: $jpeg2", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $jpeg2;
    167 }
    168 
     146unless ($no_op) {
     147    # Make the jpeg for binning 1
     148    {
     149        my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE " . RECIPE1; # Command to run
     150        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     151            run(command => $command, verbose => 1);
     152        unless ($success) {
     153            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     154            &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_tag, $error_code);
     155        }
     156        &my_die("Unable to find expected output file: $jpeg1", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $jpeg1;
     157    }
     158   
     159    # Make the jpeg for binning 2
     160    {
     161        my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE " . RECIPE2; # Command to run
     162        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     163            run(command => $command, verbose => 1);
     164        unless ($success) {
     165            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     166            &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_tag, $error_code);
     167        }
     168        &my_die("Unable to find expected output file: $jpeg2", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $jpeg2;
     169    }
     170}
    169171
    170172# Add the result into the database

  • trunk/ippScripts/scripts/detrend_process_imfile.pl

    r11830 r11837  
    3232use Pod::Usage qw( pod2usage );
    3333
    34 my ($det_id, $exp_tag, $class_id, $det_type, $input_uri, $camera, $dbname, $workdir, $no_update);
     34my ($det_id, $exp_tag, $class_id, $det_type, $input_uri, $camera, $dbname, $workdir, $no_update, $no_op);
    3535GetOptions(
    3636    'det_id|d=s'        => \$det_id,
    … …  
    4242    'dbname|d=s'        => \$dbname, # Database name
    4343    'workdir|w=s'       => \$workdir, # Working directory, for output files
    44     'no-update'         => \$no_update
     44    'no-update'         => \$no_update,
     45    'no-op'             => \$no_op,
    4546) or pod2usage( 2 );
    4647
    … …  
    105106
    106107# Run ppImage
    107 {
     108my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
     109unless ($no_op) {
    108110    my $command = "$ppImage -file $input_uri $outputRoot -recipe PPIMAGE $recipe" .
    109111        " -stat $outputStats"; # Command to run ppImage
    … …  
    118120    &my_die("Couldn't find expected output file: $outputBin1", $det_id, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin1;
    119121    &my_die("Couldn't find expected output file: $outputBin2", $det_id, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin2;
    120 }
    121122
    122 # Get the statistics on the processed image
    123 my $stats;                      # Statistics from ppImage
    124 {
     123    # Get the statistics on the processed image
    125124    my $statsFile;              # File handle
    126125    open $statsFile, "$outputStats" or die "Can't open statistics file $outputStats: $!\n";
    … …  
    130129    my $metadata = $mdcParser->parse(join "", @contents)
    131130        or &my_die("Unable to parse metadata config", $det_id, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
    132     $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
    133131    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
    134132}
    … …  
    139137$outputBin2  = $ipprc->convert_filename_relative( $outputBin2  );
    140138
     139my $bg = ($stats->bg_mean() or 'NAN');
     140my $bg_stdev = ($stats->bg_stdev() or 'NAN');
     141my $bg_mean_stdev = ($stats->bg_mean_stdev() or 'NAN');
     142
    141143# Add the processed file to the database
    142144unless ($no_update) {
    143145    my $command = "$dettool -addprocessedimfile -det_id $det_id -exp_tag $exp_tag " .
    144146        "-class_id $class_id -recip $recipe -uri $outputImage -b1_uri $outputRoot";  # Command to run dettool
    145     $command .= " -bg " . $stats->bg_mean();
    146     if (defined($stats->bg_mean_stdev())) {
    147         $command .= " -bg_mean_stdev " . $stats->bg_mean_stdev();
    148     } else {
    149         # May be undefined if there's only a single imfile
    150         $command .= " -bg_mean_stdev 0";
    151     }
    152     $command .= " -bg_stdev " . $stats->bg_stdev();
     147    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
    153148    $command .= " -dbname $dbname" if defined $dbname;
    154149

  • trunk/ippScripts/scripts/detrend_reject_exp.pl

    r11829 r11837  
    3232use Pod::Usage qw( pod2usage );
    3333
    34 my ($det_id, $iter, $det_type, $camera, $filter, $workdir, $dbname, $no_update);
     34my ($det_id, $iter, $det_type, $camera, $filter, $workdir, $dbname, $no_update, $no_op);
    3535GetOptions(
    3636           'det_id|d=s'        => \$det_id,
    … …  
    4141           'workdir|w=s'       => \$workdir, # Working directory for output files
    4242           'dbname|d=s'        => \$dbname, # Database name
    43            'no-update'         => \$no_update
     43           'no-update'         => \$no_update,
     44           'no-op'             => \$no_op,
    4445) or pod2usage( 2 );
    4546
    … …  
    133134    $command = "$dettool -updateresidexp -det_id $det_id -iteration $iter -exp_tag $expTag"; # Command to run
    134135    $command .= " -dbname $dbname" if defined $dbname;
    135 
     136   
    136137    $reject = 0;                # Reject this exposure?
    137138
    … …  
    143144    }
    144145
     146    # Cop-out if we're not doing any operations
     147    if ($no_op) {
     148        # Make sure something gets rejected (just once!), just so that
     149        # we can trace the full range of the workflow
     150        if ($i == 0 and $iter == 0) {
     151            $reject = 1;
     152        }
     153        goto UPDATE;
     154    }
     155   
    145156    if ($reject_mean and defined $meanStats->standard_deviation() ) {
    146157        my $dMean = abs($means[$i] - $meanStats->mean()) ;
    … …  
    201212    }
    202213}
    203    
     214
    204215# Decide if the current is sufficient to use as a master, and if we can stop iterating
    205216my $master = 1;                 # This is good enough for a master
    … …  
    220231print "Stop: $stop\n";
    221232
    222 # Put the result into the database
     233# Put results into the database
    223234unless ($no_update) {
    224     my $varianceStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
    225     $varianceStats->add_data(@variances);
    226 
    227     my $command = "$dettool -adddetrunsummary -det_id $det_id -iteration $iter" .
    228         " -bg " . $meanStats->mean() . " -bg_stdev " . sqrt($varianceStats->mean()) .
    229         " -bg_mean_stdev " . $meanStats->standard_deviation();
    230     $command .= " -accept" if $master;
    231     $command .= " -dbname $dbname" if defined $dbname;
    232 
    233     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    234         run(command => $command, verbose => 1);
    235     unless ($success) {
    236         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    237         warn("Unable to perform dettool -adddetrunsummary: $error_code");
    238         exit($error_code);
    239     }
    240 }
    241 
    242 # Re-run processing if required
    243 unless ($no_update) {
    244     my $command = "$dettool -updatedetrun -det_id $det_id";
    245     if ($stop) {
    246         $command .= ' -state stop';
    247     } else {
    248         $command .= ' -again';
    249     }
    250    
    251     $command .= " -dbname $dbname" if defined $dbname;
    252    
    253     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    254         run(command => $command, verbose => 1);
    255     unless ($success) {
    256         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    257         warn("Unable to perform dettool -updatedetrun: $error_code");
    258         exit($error_code);
     235
     236    {
     237        # Add the summary
     238        my $varianceStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
     239        $varianceStats->add_data(@variances);
     240       
     241        my $bg = ($meanStats->mean() or 'NAN');
     242        my $bg_stdev = (sqrt( $varianceStats->mean() ) or 'NAN');
     243        my $bg_mean_stdev = ($meanStats->standard_deviation() or 'NAN');
     244       
     245        my $command = "$dettool -adddetrunsummary -det_id $det_id -iteration $iter";
     246        $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
     247        $command .= " -accept" if $master;
     248        $command .= " -dbname $dbname" if defined $dbname;
     249       
     250        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     251            run(command => $command, verbose => 1);
     252        unless ($success) {
     253            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     254            warn("Unable to perform dettool -adddetrunsummary: $error_code");
     255            exit($error_code);
     256        }
     257    }
     258
     259    # Re-run processing if required
     260    {
     261        my $command = "$dettool -updatedetrun -det_id $det_id";
     262        if ($stop) {
     263            $command .= ' -state stop';
     264        } else {
     265            $command .= ' -again';
     266        }
     267       
     268        $command .= " -dbname $dbname" if defined $dbname;
     269       
     270        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     271            run(command => $command, verbose => 1);
     272        unless ($success) {
     273            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     274            warn("Unable to perform dettool -updatedetrun: $error_code");
     275            exit($error_code);
     276        }
    259277    }
    260278}

  • trunk/ippScripts/scripts/detrend_reject_imfile.pl

    r11829 r11837  
    3333use Pod::Usage qw( pod2usage );
    3434
    35 my ($det_id, $iter, $exp_tag, $det_type, $camera, $filter, $reject, $dbname, $workdir, $no_update);
     35my ($det_id, $iter, $exp_tag, $det_type, $camera, $filter, $reject, $dbname, $workdir, $no_update, $no_op);
    3636GetOptions(
    3737           'det_id|d=s'        => \$det_id,
    … …  
    4545           'workdir|w=s'       => \$workdir, # Working directory, for output files
    4646           'no-update'         => \$no_update,
     47           'no-op'             => \$no_op,
    4748           ) or pod2usage( 2 );
    4849
    … …  
    129130my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2
    130131
    131 # Make the jpeg for binning 1
    132 {
    133     my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE " . RECIPE1; # Command to run
    134     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    135         run(command => $command, verbose => 1);
    136     unless ($success) {
    137         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    138         &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_tag, $error_code);
    139     }
    140     &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $jpeg1Name;
    141 }
    142 
    143 # Make the jpeg for binning 2
    144 {
    145     my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE " . RECIPE2; # Command to run
    146     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    147         run(command => $command, verbose => 1);
    148     unless ($success) {
    149         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    150         &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_tag, $error_code);
    151     }
    152     &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $jpeg2Name;
     132unless ($no_op) {
     133
     134    # Make the jpeg for binning 1
     135    {
     136        my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE " . RECIPE1; # Command to run
     137        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     138            run(command => $command, verbose => 1);
     139        unless ($success) {
     140            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     141            &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_tag, $error_code);
     142        }
     143        &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $jpeg1Name;
     144    }
     145   
     146    # Make the jpeg for binning 2
     147    {
     148        my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE " . RECIPE2; # Command to run
     149        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     150            run(command => $command, verbose => 1);
     151        unless ($success) {
     152            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     153            &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_tag, $error_code);
     154        }
     155        &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $jpeg2Name;
     156    }
    153157}
    154158
    … …  
    172176    my $stdev = sqrt($variances[$i]);   # Stdev for this imfile
    173177
     178    last if $no_op;
     179
    174180    if ($reject_imfile_mean) {
    175181        if (abs($mean) > $reject_imfile_mean * $stdev) {
    … …  
    212218        print "no rejection for imfile S/N\n";
    213219    }
    214    
    215220}
    216221
    … …  
    235240## Reject based on the exposure ensemble stats
    236241# reject if the exposure ensemble mean is deviant
    237 if ($reject_exp_mean) {
    238     if (abs($mean) > $reject_exp_mean * $stdev) {
    239         print "Rejecting exposure based on bad mean: \n";
    240         print "mean: $mean, stdev: $stdev (limit is: % $reject_exp_mean\n";
    241         $reject = 1;
    242     }
    243 } else {
    244     print "no rejection for exp mean\n";
    245 }
    246 # reject if the exposure ensemble stdev is deviant
    247 if ($reject_exp_stdev) {
    248     if ($stdev > $reject_exp_stdev) {
    249         print "Rejecting exposure based on bad mean stdev: " . $stdev . " vs " .
    250             $reject_exp_stdev . "\n";
    251         $reject = 1;
    252     }
    253 } else {
    254     print "no rejection for exp stdev\n";
    255 }
    256 # reject if the exposure ensemble mean stdev is deviant
    257 if ($reject_exp_meanstdev) {
    258     if ($meanStdev > $reject_exp_meanstdev) {
    259         print "Rejecting exposure based on bad mean stdev: " . $meanStdev . " vs " .
    260             $reject_exp_meanstdev . "\n";
    261         $reject = 1;
    262     }
    263 } else {
    264     print "no rejection for exp mean stdev\n";
    265 }
    266 # reject if the signal-to-noise is insufficient
    267 if ($reject_exp_sn) {
    268     if (abs($mean) < abs($stdev * $reject_exp_sn)) {
    269         print "Rejecting exposure based on poor S/N: \n";
    270         print "signal: $mean vs noise: $stdev (s/n limit is: $reject_exp_sn)\n";
    271         $reject = 1;
    272     }
    273 } else {
    274     print "no rejection for exp S/N\n";
     242unless ($no_op) {
     243    if ($reject_exp_mean) {
     244        if (abs($mean) > $reject_exp_mean * $stdev) {
     245            print "Rejecting exposure based on bad mean: \n";
     246            print "mean: $mean, stdev: $stdev (limit is: % $reject_exp_mean\n";
     247            $reject = 1;
     248        }
     249    } else {
     250        print "no rejection for exp mean\n";
     251    }
     252    # reject if the exposure ensemble stdev is deviant
     253    if ($reject_exp_stdev) {
     254        if ($stdev > $reject_exp_stdev) {
     255            print "Rejecting exposure based on bad mean stdev: " . $stdev . " vs " .
     256                $reject_exp_stdev . "\n";
     257            $reject = 1;
     258        }
     259    } else {
     260        print "no rejection for exp stdev\n";
     261    }
     262    # reject if the exposure ensemble mean stdev is deviant
     263    if ($reject_exp_meanstdev) {
     264        if ($meanStdev > $reject_exp_meanstdev) {
     265            print "Rejecting exposure based on bad mean stdev: " . $meanStdev . " vs " .
     266                $reject_exp_meanstdev . "\n";
     267            $reject = 1;
     268        }
     269    } else {
     270        print "no rejection for exp mean stdev\n";
     271    }
     272    # reject if the signal-to-noise is insufficient
     273    if ($reject_exp_sn) {
     274        if (abs($mean) < abs($stdev * $reject_exp_sn)) {
     275            print "Rejecting exposure based on poor S/N: \n";
     276            print "signal: $mean vs noise: $stdev (s/n limit is: $reject_exp_sn)\n";
     277            $reject = 1;
     278        }
     279    } else {
     280        print "no rejection for exp S/N\n";
     281    }
    275282}
    276283
    … …  
    278285$jpeg1Name = $ipprc->convert_filename_relative( $jpeg1Name );
    279286$jpeg2Name = $ipprc->convert_filename_relative( $jpeg2Name );
     287
     288my $bg = $mean;
     289my $bg_stdev = $stdev;
     290my $bg_mean_stdev = $meanStdev;
     291
    280292unless ($no_update) {
    281293    my $command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_tag $exp_tag " .
    282         "-recip " . RECIPE1() . "," . RECIPE2() . " -b1_uri $outputRoot " .
    283         "-bg $mean -bg_stdev $stdev -bg_mean_stdev $meanStdev"; # Command to run
     294        "-recip " . RECIPE1() . "," . RECIPE2() . " -b1_uri $outputRoot "; # Command to run
     295    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
    284296    $command .= ' -reject' if $reject;
    285297    $command .= " -dbname $dbname" if defined $dbname;

  • trunk/ippScripts/scripts/detrend_resid.pl

    r11829 r11837  
    3434
    3535my ($det_id, $iter, $exp_tag, $class_id, $det_type, $detrend,
    36         $input_uri, $camera, $mode, $dbname, $workdir, $no_update);
     36        $input_uri, $camera, $mode, $dbname, $workdir, $no_update, $no_op);
    3737GetOptions(
    3838    'det_id|d=s'        => \$det_id,
    … …  
    4848    'workdir|w=s'       => \$workdir,   # Working directory, for output files
    4949    'no-update'         => \$no_update,
     50    'no-op'             => \$no_op,
    5051) or pod2usage( 2 );
    5152
    … …  
    138139
    139140# Run ppImage
    140 {
     141my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
     142unless ($no_op) {
    141143    my $command = "$ppImage -file $input_uri $outputRoot -recipe PPIMAGE $recipe" .
    142144        " -stat $outputStats "; # Command to run ppImage
    … …  
    159161    &my_die("Couldn't find expected output file: $bin1Name", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $bin1Name;
    160162    &my_die("Couldn't find expected output file: $bin2Name", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $bin2Name;
    161 }
    162 
    163 # Get the statistics on the residual image
    164 my $stats;                      # Statistics from ppImage
    165 {
     163
     164    # Get the statistics on the residual image
    166165    my $statsFile;              # File handle
    167166    open $statsFile, $outputStats or &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR);
    … …  
    179178$bin1Name = $ipprc->convert_filename_relative( $bin1Name );
    180179$bin2Name = $ipprc->convert_filename_relative( $bin2Name );
     180
     181my $bg = ($stats->bg_mean() or 'NAN');
     182my $bg_stdev = ($stats->bg_stdev() or 'NAN');
     183my $bg_mean_stdev = ($stats->bg_mean_stdev() or 'NAN');
     184
    181185unless ($no_update) {
    182186    my $command = "$dettool -addresidimfile -det_id $det_id -iteration $iter -exp_tag $exp_tag " .
    183187        "-class_id $class_id -recip $recipe -uri $outputName -b1_uri $outputRoot"; # Command to run dettool
    184     $command .= " -bg " . $stats->bg_mean();
     188    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
    185189    $command .= " -dbname $dbname" if defined $dbname;
    186 
    187     if (defined($stats->bg_mean_stdev())) {
    188         $command .= " -bg_mean_stdev " . $stats->bg_mean_stdev();
    189     } else {
    190         # May be undefined if there is only a single imfile
    191         $command .= ' -bg_mean_stdev 0';
    192     }
    193 
    194     $command .= " -bg_stdev " . $stats->bg_stdev();
    195190
    196191    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

  • trunk/ippScripts/scripts/detrend_stack.pl

    r11830 r11837  
    3333use Pod::Usage qw( pod2usage );
    3434
    35 my ($det_id, $iter, $class_id, $det_type, $camera, $dbname, $workdir, $no_update);
     35my ($det_id, $iter, $class_id, $det_type, $camera, $dbname, $workdir, $no_update, $no_op);
    3636GetOptions(
    3737    'det_id|d=s'        => \$det_id,
    … …  
    4242    'dbname|d=s'        => \$dbname, # Database name
    4343    'workdir|w=s'       => \$workdir,   # Working directory, for output files
    44     'no-update'         => \$no_update
     44    'no-update'         => \$no_update,
     45    'no-op'             => \$no_op,
    4546) or pod2usage( 2 );
    4647
    … …  
    112113my $outputStats = $outputRoot . '.stats'; # Statistics name
    113114
    114 {
     115my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
     116unless ($no_op) {
    115117    my $command = "$ppMerge $outputStack"; # Command to run
    116118    foreach my $file (@$files) {
    … …  
    122124    $command .= ' -type ' . uc($det_type); # Type of stacking to perform
    123125    $command .= " -stats $outputStats"; # Statistics output filename
    124 
     126   
    125127    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    126128        run(command => $command, verbose => 1);
    … …  
    131133    &my_die("Unable to find expected output file: $outputStack\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStack;
    132134    &my_die("Unable to find expected output file: $outputStats\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStats;
    133 }
    134135
    135 # Get the statistics on the stacked image
    136 my $stats;                      # Statistics from ppMerge
    137 {
     136    # Get the statistics on the stacked image
    138137    open(my $statsFile, "$outputStats") or
    139138        &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR);
    140139    my $contents = do { local $/; <$statsFile> }; # Contents of file
    141140    close($statsFile);
    142 
     141   
    143142    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
    144143    my $metadata = $mdcParser->parse($contents) or
    … …  
    149148}
    150149
     150my $bg = ($stats->bg_mean() or 'NAN');
     151my $bg_stdev = ($stats->bg_stdev() or 'NAN');
     152my $bg_mean_stdev = ($stats->bg_mean_stdev() or 'NAN');
     153
    151154# Add the resultant into the database
    152155$outputStack = $ipprc->convert_filename_relative($outputStack);
    … …  
    154157    my $command = "$dettool -addstacked -det_id $det_id -iteration $iter -class_id $class_id" .
    155158        " -uri $outputStack -recip $recipe"; # Command to run
    156     $command .= ' -bg ' . $stats->bg_mean();
    157     if (defined($stats->bg_mean_stdev())) {
    158     $command .= ' -bg_mean_stdev ' . $stats->bg_mean_stdev();
    159     } else {
    160         # May be undefined if there is only a single imfile
    161         $command .= ' -bg_mean_stdev 0';
    162     }
    163     $command .= ' -bg_stdev ' . $stats->bg_stdev();
     159    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
    164160    $command .= " -dbname $dbname" if defined $dbname;
     161
    165162    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    166163        run(command => $command, verbose => 1);

  • trunk/ippScripts/scripts/phase2.pl

    r11830 r11837  
    3838    $dbname,                    # Database name
    3939    $workdir,                   # Working directory, for output files
    40     $no_update                  # Don't update the database?
     40    $no_update,                 # Don't update the database?
     41    $no_op,                     # Don't do any operations?
    4142    );
    4243GetOptions(
    … …  
    4748    'dbname|d=s'    => \$dbname, # Database name
    4849    'workdir|w=s'   => \$workdir,
    49     'no-update'     => \$no_update
     50    'no-update'     => \$no_update,
     51    'no-op'         => \$no_op,
    5052) or pod2usage( 2 );
    5153
    … …  
    8890$input = $ipprc->convert_filename_absolute( $input );
    8991
    90 # Run ppImage
    91 {
     92my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
     93unless ($no_op) {
     94    # Run ppImage
    9295    print "outputImage: $outputImage\n";
    9396    print "outputBin1: $outputBin1\n";
    9497    print "outputStats: $outputStats\n";
    95 
     98   
    9699    my $command = "$ppImage -file $input $outputRoot -recipe PPIMAGE " . RECIPE .
    97100        " -stat $outputStats"; # Command to run ppImage
    … …  
    106109    &my_die("Couldn't find expected output file: $outputBin2\n", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin2;
    107110    &my_die("Couldn't find expected output file: $outputStats\n", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStats;
    108 }
    109111
    110 # Get the statistics on the processed image
    111 my $stats;                      # Statistics from ppImage
    112 {
     112    # Get the statistics on the processed image
    113113    my $statsFile;              # File handle
    114114    open $statsFile, $outputStats or &my_die("Can't open statistics file $outputStats: $!", $exp_tag, $class_id, $PS_EXIT_SYS_ERROR);
    … …  
    118118    my $metadata = $mdcParser->parse(join "", @contents) or
    119119        &my_die("Unable to parse metadata config doc", $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
    120     $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
    121120    $stats->parse($metadata) or
    122121        &my_die("Unable to find all values in statistics output.\n", $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
    … …  
    127126$outputBin1  = $ipprc->convert_filename_relative( $outputBin1 );
    128127$outputBin2  = $ipprc->convert_filename_relative( $outputBin2 );
     128
     129my $bg = ($stats->bg_mean() or 'NAN');
     130my $bg_stdev = ($stats->bg_stdev() or 'NAN');
     131my $bg_mean_stdev = ($stats->bg_mean_stdev() or 'NAN');
     132
    129133unless ($no_update) {
    130134    # Command to run p2tool
    … …  
    135139    $command .= " -uri $outputImage";
    136140    $command .= " -b1_uri $outputRoot";
    137     $command .= " -bg " . $stats->bg_mean();
    138     $command .= " -bg_stdev " . $stats->bg_stdev();
    139     if ( defined $stats->bg_mean_stdev() ) {
    140         $command .= " -bg_mean_stdev " . $stats->bg_mean_stdev();
    141     } else {
    142         $command .= " -bg_mean_stdev 0.0";
    143     }
     141    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
    144142    $command .= " -dbname $dbname" if defined $dbname;
    145143

  • trunk/ippScripts/scripts/phase3.pl

    r11829 r11837  
    3333use Pod::Usage qw( pod2usage );
    3434
    35 my ($exp_tag, $camera, $dbname, $workdir, $no_update);
     35my ($exp_tag, $camera, $dbname, $workdir, $no_update, $no_op);
    3636GetOptions(
    3737    'exp_tag|e=s'       => \$exp_tag,
    … …  
    3939    'dbname|d=s'        => \$dbname, # Database name
    4040    'workdir|w=s'       => \$workdir,
    41     'no-update'         => \$no_update
     41    'no-update'         => \$no_update,
     42    'no-op'             => \$no_op,
    4243) or pod2usage( 2 );
    4344
    … …  
    111112        my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
    112113        $stats->add_data(@backgrounds);
    113         $bg = $stats->mean();
    114         $bg_mean_stdev = $stats->standard_deviation() || 0.0;
     114        $bg = ($stats->mean() or 'NAN');
     115        $bg_mean_stdev = ($stats->standard_deviation() or 'NAN');
    115116    }
    116117    {
    117118        my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
    118119        $stats->add_data(@variances);
    119         $bg_stdev = sqrt( $stats->mean() );
     120        $bg_stdev = (sqrt( $stats->mean() ) or 'NAN');
    120121    }
    121122}
    … …  
    169170my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $outputRoot); # MEF psastro output
    170171
    171 # run psastro +mosastro on the set of chips
    172 # XXX note that this is wrong if imfiles are cells
    173 if (@$files > 1) {
    174     my $command = "$psastro -list $list3Name $outputRoot +mosastro -chipastro";
    175     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    176         run(command => $command, verbose => 1);
    177     unless ($success) {
    178         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    179         &my_die("Unable to perform psastro: $error_code", $exp_tag, $error_code);
    180     }
    181     &my_die("Unable to find expected output file: $fpaObjects", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $fpaObjects;
    182 } else {
    183     $fpaObjects = $chipObjects;
    184 }
    185 
    186 # run addstar on either the single chip output or the single fpa output
    187 # XXX this construct requires the user to have a valid .ptolemyrc
    188 # XXX which in turn points at ippconfig/dvo.site
    189 {
    190     my $command = "addstar -D CAMERA $camera $fpaObjects";
    191     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    192         run(command => $command, verbose => 1);
    193     unless ($success) {
    194         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    195         &my_die("Unable to perform addstar: $error_code", $exp_tag, $error_code);
    196     }
    197 }
    198 
    199 # Make the jpeg for binning 1
    200 {
    201     my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE " . RECIPE1; # Command to run
    202     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    203         run(command => $command, verbose => 1);
    204     unless ($success) {
    205         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    206         &my_die("Unable to perform ppImage: $error_code", $exp_tag, $error_code);
    207     }
    208     &my_die("Unable to find expected output file: $jpeg1", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $jpeg1;
    209 }
    210 
    211 # Make the jpeg for binning 2
    212 {
    213     my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE " . RECIPE2; # Command to run
    214     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    215         run(command => $command, verbose => 1);
    216     unless ($success) {
    217         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    218         &my_die("Unable to perform ppImage: $error_code", $exp_tag, $error_code);
    219     }
    220     &my_die("Unable to find expected output file: $jpeg2", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $jpeg2;
     172unless ($no_op) {
     173
     174    # run psastro +mosastro on the set of chips
     175    # XXX note that this is wrong if imfiles are cells
     176    if (@$files > 1) {
     177        my $command = "$psastro -list $list3Name $outputRoot +mosastro -chipastro";
     178        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     179            run(command => $command, verbose => 1);
     180        unless ($success) {
     181            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     182            &my_die("Unable to perform psastro: $error_code", $exp_tag, $error_code);
     183        }
     184        &my_die("Unable to find expected output file: $fpaObjects", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $fpaObjects;
     185    } else {
     186        $fpaObjects = $chipObjects;
     187    }
     188
     189    # run addstar on either the single chip output or the single fpa output
     190    # XXX this construct requires the user to have a valid .ptolemyrc
     191    # XXX which in turn points at ippconfig/dvo.site
     192    {
     193        my $command = "addstar -D CAMERA $camera $fpaObjects";
     194        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     195            run(command => $command, verbose => 1);
     196        unless ($success) {
     197            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     198            &my_die("Unable to perform addstar: $error_code", $exp_tag, $error_code);
     199        }
     200    }
     201   
     202    # Make the jpeg for binning 1
     203    {
     204        my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE " . RECIPE1; # Command to run
     205        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     206            run(command => $command, verbose => 1);
     207        unless ($success) {
     208            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     209            &my_die("Unable to perform ppImage: $error_code", $exp_tag, $error_code);
     210        }
     211        &my_die("Unable to find expected output file: $jpeg1", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $jpeg1;
     212    }
     213
     214    # Make the jpeg for binning 2
     215    {
     216        my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE " . RECIPE2; # Command to run
     217        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     218            run(command => $command, verbose => 1);
     219        unless ($success) {
     220            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     221            &my_die("Unable to perform ppImage: $error_code", $exp_tag, $error_code);
     222        }
     223        &my_die("Unable to find expected output file: $jpeg2", $exp_tag, $PS_EXIT_PROG_ERROR) unless -f $jpeg2;
     224    }
    221225}
    222226
    … …  
    226230$jpeg1  = $ipprc->convert_filename_relative($jpeg1);
    227231$jpeg2  = $ipprc->convert_filename_relative($jpeg2);
     232
    228233unless ($no_update) {
    229234    my $command = "$p3tool -addprocessedexp -exp_tag $exp_tag -uri UNKNOWN " .

  • trunk/ippScripts/scripts/phase4_overlap.pl

    r11829 r11837  
    3232use Pod::Usage qw( pod2usage );
    3333
    34 my ($det_id, $dbname, $workdir, $no_update);
     34my ($p4_id, $dbname, $workdir, $no_update, $no_op);
    3535GetOptions(
    3636    'p4_id|i=s'         => \$p4_id, # Phase 4 identifier
    3737    'dbname|d=s'        => \$dbname, # Database name
    3838    'workdir|w=s'       => \$workdir, # Working directory, for output files
    39     'no-update'         => \$no_update
     39    'no-update'         => \$no_update, # Don't update the database?
     40    'no-op'             => \$no_op, # Don't do any operations
    4041) or pod2usage( 2 );
    4142
    … …  
    6768    }
    6869
     70    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
    6971    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    7072        &my_die("Unable to parse metadata config doc", $p4_id, $PS_EXIT_PROG_ERROR);
    … …  
    7375}
    7476
     77unless ($no_op) {
     78
    7579### Calculates, in some as-yet unknown way the overlaps between imfiles and skycells
    7680
    77 # Add in the skycell and tesselation for each imfile
    78 foreach my $imfile (@$imfiles) {
    79     $imfile->{skycell_id} = 'default';
    80     $imfile->{tess_id} = 'default';
     81} else {
     82    # Add in the skycell and tesselation for each imfile
     83    foreach my $imfile (@$imfiles) {
     84        $imfile->{skycell_id} = 'default';
     85        $imfile->{tess_id} = 'default';
     86    }
    8187}
    82 
     88   
    8389
    8490# Generate a MDC file with the overlaps
    8591$workdir = $ipprc->convert_filename_absolute( $workdir );
    86 my $overlapName = File::Spec->catfile( $workdir, $p4_id . ".overlap.mdc" );
    87 open my $overlapFile "> $overlapName" or
     92my $overlapName = File::Spec->catfile( $workdir, $p4_id . '.overlap.mdc' );
     93open my $overlapFile, "> $overlapName" or
    8894    &my_die("Unable to open mdc file $overlapName", $p4_id, $PS_EXIT_DATA_ERROR);
    8995print $overlapFile "p4SkyCellMap MULTI\n\n";

  • trunk/ippScripts/scripts/phase4_warp.pl

    r11830 r11837  
    3232use Pod::Usage qw( pod2usage );
    3333
    34 my ($p4_id, $skycell_id, $camera, $dbname, $workdir, $no_update);
     34my ($p4_id, $skycell_id, $camera, $dbname, $workdir, $no_update, $no_op);
    3535GetOptions(
    3636    'p4_id|i=s'         => \$p4_id, # Phase 4 identifier
    … …  
    3939    'dbname|d=s'        => \$dbname, # Database name
    4040    'workdir|w=s'       => \$workdir, # Working directory, for output files
    41     'no-update'         => \$no_update
     41    'no-update'         => \$no_update, # Don't update the database?
     42    'no-op'             => \$no_op, # Don't do any operations?
    4243) or pod2usage( 2 );
    4344
    … …  
    6263
    6364# Get list of component imfiles for exposure
     65my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    6466my $imfiles;
    6567{
    … …  
    9799open my $listFile, '>' . $listName;
    98100foreach my $imfile (@$imfiles) {
    99     my $uri = $ipprc->convert_filename_absolute( $file->{uri} );
     101    my $uri = $ipprc->convert_filename_absolute( $imfile->{uri} );
    100102
    101     print $listFile $uri . "\n";
     103    print $listFile "$uri\n";
    102104}
    103105close $listFile;
    104106
    105107# Run pswarp
    106 {
    107     my $command = "$pswarp -file $uri $outputRoot -stat $outputStats"; # Command to run pswarp
     108my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
     109unless ($no_op) {
     110    my $command = "$pswarp -list $listName $outputRoot -stat $outputStats"; # Command to run pswarp
    108111    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    109112        run(command => $command, verbose => 1);
    … …  
    116119    &my_die("Couldn't find expected output file: $outputBin1", $p4_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin1;
    117120    &my_die("Couldn't find expected output file: $outputBin2", $p4_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputBin2;
    118 }
    119121
    120 # Get the statistics on the warped image
    121 my $stats;                      # Statistics from pswarp
    122 {
     122    # Get the statistics on the warped image
    123123    my $statsFile;              # File handle
    124124    open $statsFile, "$outputStats" or die "Can't open statistics file $outputStats: $!\n";
    125125    my @contents = <$statsFile>; # Contents of file
    126126    close $statsFile;
    127     my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
    128127    my $metadata = $mdcParser->parse(join "", @contents)
    129128        or &my_die("Unable to parse metadata config", $p4_id, $skycell_id, $PS_EXIT_PROG_ERROR);
    130     $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
    131129    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $p4_id, $skycell_id, $PS_EXIT_PROG_ERROR);
    132130}
    … …  
    137135$outputBin2  = $ipprc->convert_filename_relative( $outputBin2  );
    138136
     137my $bg = ($stats->bg_mean() or 'NAN');
     138my $bg_stdev = ($stats->bg_stdev() or 'NAN');
     139
    139140# Add the processed file to the database
    140141unless ($no_update) {
    141142    my $command = "$p4tool -warped -p4_id $p4_id -skycell_id $skycell_id" .
    142143        " -uri $outputImage -b1_uri $outputRoot";  # Command to run dettool
    143     $command .= " -bg " . $stats->bg_mean();
    144     $command .= " -bg_stdev " . $stats->bg_stdev();
     144    $command .= " -bg $bg -bg_stdev $bg_stdev";
    145145    $command .= " -dbname $dbname" if defined $dbname;
    146146

  • trunk/ippScripts/scripts/phase5_subtract.pl

    r11830 r11837  
    3333use Pod::Usage qw( pod2usage );
    3434
    35 my ($p5_id, $camera, $dbname, $workdir, $no_update);
     35my ($p5_id, $camera, $dbname, $workdir, $no_update, $no_op);
    3636GetOptions(
    3737    'p5_id|d=s'         => \$p5_id, # Phase 5 identifier
    … …  
    3939    'dbname|d=s'        => \$dbname, # Database name
    4040    'workdir|w=s'       => \$workdir,   # Working directory, for output files
    41     'no-update'         => \$no_update,
     41    'no-update'         => \$no_update, # Don't update the database?
     42    'no-op'             => \$no_op, # Don't do any operations?
    4243) or pod2usage( 2 );
    4344
    … …  
    6162
    6263# Get list of components for subtraction
     64my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    6365my $files;
    6466{
    65     my $command = "$p5tool -inputscfile -p5_id $p4_id";
     67    my $command = "$p5tool -inputscfile -p5_id $p5_id";
    6668    $command .= " -dbname $dbname" if defined $dbname;
    6769    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    116118
    117119my $outputName = $outputRoot . ".fits";
    118 # my $bin1Name =  $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id);
    119 # my $bin2Name =  $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id);
    120 # my $outputStats = $outputRoot . '.stats';
     120my $bin1Name =  $ipprc->filename("PPIMAGE.BIN1", $outputRoot);
     121my $bin2Name =  $ipprc->filename("PPIMAGE.BIN2", $outputRoot);
     122my $outputStats = $outputRoot . '.stats';
    121123
    122124# Perform subtraction
    123 {
     125my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
     126unless ($no_op) {
    124127    my $command = "$pois $template $input $outputName"; # Command to run pois
    125128
    … …  
    134137#    &my_die("Couldn't find expected output file: $bin2Name",    $p5_id, $PS_EXIT_SYS_ERROR) unless -f $bin2Name;
    135138#    &my_die("Couldn't find expected output file: $outputStats", $p5_id, $PS_EXIT_SYS_ERROR) unless -f $outputStats;
    136 }
    137 
    138 # Get the statistics on the residual image
    139 if (0) { ### Disabled because pois doesn't output stats yet
    140     my $stats;                  # Statistics from ppImage
    141     {
     139
     140    # Get the statistics on the residual image
     141    if (0) { ### Disabled because pois doesn't output stats yet
    142142        my $statsFile;          # File handle
    143         open $statsFile, $outputStats or &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR);
     143        open $statsFile, $outputStats or &my_die("Can't open statistics file $outputStats: $!", $p5_id, $PS_EXIT_SYS_ERROR);
    144144        my @contents = <$statsFile>; # Contents of file
    145     close $statsFile;
    146         my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
     145        close $statsFile;
    147146        my $metadata = $mdcParser->parse(join "", @contents) or
    148             &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
    149         $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
    150         $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
     147            &my_die("Unable to parse metadata config doc", $p5_id, $PS_EXIT_PROG_ERROR);
     148        $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $p5_id, $PS_EXIT_PROG_ERROR);
    151149    }
    152150}
    … …  
    154152# Add the processed file to the database
    155153$outputName = $ipprc->convert_filename_relative( $outputName );
     154
     155my $bg = ($stats->bg_mean() or 'NAN');
     156my $bg_stdev = ($stats->bg_stdev() or 'NAN');
     157
    156158unless ($no_update) {
    157159
    158160    # Add the subtraction result
    159161    {
    160         my $command = "$p5tool -adddiffscfile -p5_id $p5_id -uri $outputName -b1_uri $outputRoot"; # Command to run p5tool
     162        my $command = "$p5tool -adddiffscfile -p5_id $p5_id -uri $outputName -b1_uri $outputRoot";
     163        $command .= " -bg $bg -bg_stdev $bg_stdev";
    161164        $command .= " -dbname $dbname" if defined $dbname;
    162        
    163         if (0) { ### Disabled because no statistics yet
    164             $command .= " -bg " . $stats->bg_mean();
    165             $command .= " -bg_stdev " . $stats->bg_stdev();
    166         }
    167165       
    168166        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    170168        unless ($success) {
    171169            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    172             &my_die("Unable to perform p5tool -adddiffscfile: $error_code", $p5_id, $iter, $exp_tag, $class_id, $error_code);
     170            &my_die("Unable to perform p5tool -adddiffscfile: $error_code", $p5_id, $error_code);
    173171        }
    174172       
    … …  
    199197
    200198    warn($msg);
    201     if ($det_id and $iter and $exp_tag and not $no_update) {
     199    if ($p5_id and not $no_update) {
    202200        my $command = "$p5tool -updaterun -p5_id $p5_id -state stop -code $exit_code";
    203201        $command .= " -dbname $dbname" if defined $dbname;
Note: See TracChangeset for help on using the changeset viewer.