IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 3, 2010, 8:50:52 AM (16 years ago)
Author:
eugene
Message:

updates from trunk

Location:
branches/simtest_nebulous_branches
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/simtest_nebulous_branches

  • branches/simtest_nebulous_branches/ippScripts/scripts/diff_skycell.pl

    r24179 r27840  
    77use Sys::Hostname;
    88my $host = hostname();
     9my $date = `date`;
    910print "\n\n";
    10 print "Starting script $0 on $host\n\n";
     11print "Starting script $0 on $host at $date\n\n";
    1112
    1213use DateTime;
    … …  
    3536}
    3637
    37 my ($diff_id, $dbname, $threads, $outroot, $reduction, $inverse, $verbose, $no_update, $no_op, $redirect);
     38my ($diff_id, $dbname, $threads, $outroot, $reduction, $inverse, $run_state, $verbose, $no_update, $no_op, $redirect);
    3839my ($skycell_id, $diff_skyfile_id);
     40my ($use_convolved);
    3941GetOptions(
    4042    'diff_id=s'         => \$diff_id, # Diff identifier
    … …  
    4345    'dbname|d=s'        => \$dbname, # Database name
    4446    'threads=s'         => \$threads,   # Number of threads to use
     47    'run-state=s'       => \$run_state,   # state for run: 'new' or 'update'
    4548    'outroot=s'         => \$outroot, # Output root name
    4649    'inverse'           => \$inverse, # Make inverse subtraction?
    4750    'reduction=s'       => \$reduction, # Reduction class
     51    'use_convolved'     => \$use_convolved, # Use convolved stacks instead of unconvolved.
    4852    'verbose'           => \$verbose,   # Print to stdout
    4953    'no-update'         => \$no_update, # Don't update the database?
    … …  
    5458pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    5559pod2usage(
    56     -msg => "Required options: --diff_id --skycell_id --outroot",
     60    -msg => "Required options: --diff_id --skycell_id --outroot --run-state --diff_skyfile_id",
    5761    -exitval => 3,
    5862          ) unless defined $diff_id
    5963    and defined $skycell_id
    6064    and defined $diff_skyfile_id
     65    and defined $run_state
    6166    and defined $outroot;
    6267
    … …  
    6671# my $logDest = $ipprc->filename("LOG.EXP", $outroot);
    6772my $logDest = "$outroot.log";
     73$logDest .= ".update" if $run_state eq "update";
    6874$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
    6975
    … …  
    97103my $camera;                     # Camera
    98104my ($inputMagic, $templateMagic); # Are the inputs been magicked?
     105my ($saveInConv, $saveRefConv);   # Save the input or reference convolved images?
     106# Prescan to decide if this is or is not a stack stack diff. The check above confirms we only have two entries.
     107if ((${ $files }[0]->{warp_id} == 0)&&
     108    (${ $files }[1]->{warp_id} == 0)) {
     109    # Both are zero, so stack stack diff;
     110    unless ($use_convolved) {
     111        $use_convolved = 0;
     112    }
     113}
     114else {
     115    # We're in some sort of warp stack or warp warp (don't care about the last one.)
     116    $use_convolved = 1;
     117}
     118
    99119foreach my $file (@$files) {
    100120    if (defined $file->{template} and $file->{template}) {
    101         $template = $file->{uri};
     121#        $template = $file->{uri};
    102122        $templatePath = $file->{path_base};
    103123        if ($file->{warp_id} == 0) {
    104             $templateMask = "PPSTACK.OUTPUT.MASK";
    105             $templateVariance = "PPSTACK.OUTPUT.VARIANCE";
     124            if ($use_convolved) {
     125                $template = "PPSTACK.OUTPUT";
     126                $templateMask = "PPSTACK.OUTPUT.MASK";
     127                $templateVariance = "PPSTACK.OUTPUT.VARIANCE";
     128            }
     129            else {
     130                $template     = "PPSTACK.UNCONV";
     131                $templateMask = "PPSTACK.UNCONV.MASK";
     132                $templateVariance = "PPSTACK.UNCONV.VARIANCE";
     133            }
    106134            $templateSources = "PSPHOT.OUT.CMF.MEF";  ## this must be consistent with the value in stack_skycell.pl:161
    107135            # template is a stack so it doesn't need to be magicked
    … …  
    109137            ## use an explicit stack name for psphot output objects
    110138        } else {
     139            $template     = "PSWARP.OUTPUT";
    111140            $templateMask = "PSWARP.OUTPUT.MASK";
    112141            $templateVariance = "PSWARP.OUTPUT.VARIANCE";
    113142            $templateSources = "PSWARP.OUTPUT.SOURCES";
    114143            $templateMagic = $file->{magicked};
     144            $saveRefConv = 1;
    115145        }
    116146    } else {
    117         $input = $file->{uri};
     147#        $input = $file->{uri};
    118148        $inputPath = $file->{path_base};
    119149        $inputMagic = $file->{magicked};    # if input is a stack the output can't be "magicked"
    120150        if ($file->{warp_id} == 0) {
    121             $inputMask = "PPSTACK.OUTPUT.MASK";
    122             $inputVariance = "PPSTACK.OUTPUT.VARIANCE";
     151            if ($use_convolved) {
     152                $input = "PPSTACK.OUTPUT";
     153                $inputMask = "PPSTACK.OUTPUT.MASK";
     154                $inputVariance = "PPSTACK.OUTPUT.VARIANCE";
     155            }
     156            else {
     157                $input     = "PPSTACK.UNCONV";
     158                $inputMask = "PPSTACK.UNCONV.MASK";
     159                $inputVariance = "PPSTACK.UNCONV.VARIANCE";
     160            }
    123161            $inputSources = "PSPHOT.OUT.CMF.MEF";  ## this must be consistent with the value in stack_skycell.pl:161
    124162        } else {
     163            $input     = "PSWARP.OUTPUT";
    125164            $inputMask = "PSWARP.OUTPUT.MASK";
    126165            $inputVariance = "PSWARP.OUTPUT.VARIANCE";
    127166            $inputSources = "PSWARP.OUTPUT.SOURCES";
     167            $saveInConv = 1;
    128168        }
    129169    }
    … …  
    134174        $tess_id = $file->{tess_id};
    135175    }
    136     if (defined $skycell_id) {
    137         &my_die("Skycell identifiers don't match", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless
    138             $file->{skycell_id} eq $skycell_id;
    139     } else {
    140         $skycell_id = $file->{skycell_id};
    141     }
     176    &my_die("Skycell identifiers don't match", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless
     177        $file->{skycell_id} eq $skycell_id;
    142178    if (defined $camera) {
    143179        &my_die("Cameras don't match", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $file->{camera} eq $camera;
    … …  
    155191# The output file will be considered magicked if the input has been magicked and the
    156192# template is either a stack or a warp that has been magicked.
    157 # note that difftool -inputskyfile outputs the magicked boolean as an int not T or F
    158 # because the output is constructed from a union of two selects
    159 my $magicked = $inputMagic && $templateMagic;
     193my $magicked = $inputMagic && $templateMagic ? $inputMagic : 0;
    160194
    161195# Recipes to use based on reduction class
    … …  
    166200    &my_die("Couldn't find selected reduction for DIFF_PPSUB and DIFF_PSPHOT: $reduction\n", $diff_id, $skycell_id, $PS_EXIT_CONFIG_ERROR);
    167201}
     202
     203print "reduction: $reduction\n";
     204print "recipe_ppSub: $recipe_ppSub\n";
    168205
    169206# print "templateMask: $templateMask\n";
    … …  
    176213# print "inputSources: $inputSources\n";
    177214
     215$input     = $ipprc->filename($input, $inputPath);
    178216$inputMask = $ipprc->filename($inputMask, $inputPath);
    179217$inputVariance = $ipprc->filename($inputVariance, $inputPath);
    180218$inputSources = $ipprc->filename($inputSources, $inputPath);
    181219
     220$template     = $ipprc->filename($template, $templatePath);
    182221$templateMask = $ipprc->filename($templateMask, $templatePath);
    183222$templateVariance = $ipprc->filename($templateVariance, $templatePath);
    184223$templateSources = $ipprc->filename($templateSources, $templatePath);
    185224
     225print "template:     $template\n";
    186226print "templateMask: $templateMask\n";
    187227print "templatePath: $templatePath\n";
     228print "input:     $input\n";
    188229print "inputMask: $inputMask\n";
    189230print "inputPath: $inputPath\n";
    … …  
    213254my $traceDest = $ipprc->filename("TRACE.EXP", $outroot);
    214255
     256if ($run_state eq 'update') {
     257    $traceDest .= '.update';
     258    $outputStats .= '.update';
     259}
     260
    215261my ($inverseName, $inverseMask, $inverseVariance, $inverseSources);
    216262if ($inverse) {
    … …  
    224270
    225271# Perform subtraction
    226 unless ($no_op) {
     272{
    227273    my $command = "$ppSub $outroot";
    228274    $command .= " -inimage $input";
    … …  
    236282    $command .= " -stats $outputStats";
    237283    $command .= " -threads $threads" if defined $threads;
     284    if ($run_state eq "new") {
     285        $command .= " -dumpconfig $configuration";
     286    } else {
     287        my $configurationReal = $ipprc->file_resolve($configuration) or &my_die("Couldn't resolve configuration file: $configuration", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR);
     288        $command .= " -ipprc $configurationReal";
     289    }
     290    $command .= " -save-inconv" if defined $saveInConv;
     291    $command .= " -save-refconv" if defined $saveRefConv;
    238292    $command .= " -recipe PPSUB $recipe_ppSub";
    239293    $command .= " -recipe PSPHOT $recipe_psphot";
    … …  
    242296    $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF";
    243297    $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF";
    244     $command .= " -photometry";
     298    if ($run_state eq "new") {
     299        $command .= " -photometry";
     300    }
    245301    $command .= " -inverse" if $inverse;
    246302    $command .= " -tracedest $traceDest -log $logDest";
    247     $command .= " -dumpconfig $configuration";
    248303    $command .= " -dbname $dbname" if defined $dbname;
    249304    $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id;
    250305    $command .= " -source_id $source_id" if defined $source_id;
    251306
    252     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    253         run(command => $command, verbose => $verbose);
    254     unless ($success) {
    255         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    256         &my_die("Unable to perform ppSub: $error_code", $diff_id, $skycell_id, $error_code);
    257     }
    258 
    259     my $outputStatsReal = $ipprc->file_resolve($outputStats);
    260     &my_die("Couldn't find expected output file: $outputStats", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal;
    261 
    262     # measure chip stats
    263     $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL";
    264     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    265         run(command => $command, verbose => $verbose);
    266     unless ($success) {
    267         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    268         &my_die("Unable to perform ppStatsFromMetadata: $error_code", $diff_id, $skycell_id, $error_code);
    269     }
    270     foreach my $line (@$stdout_buf) {
    271         $cmdflags .= " $line";
    272     }
    273     chomp $cmdflags;
    274 
    275     my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
    276 
    277     if (!$quality) {
    278         &my_die("Couldn't find expected output file: $outputName", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
    279         &my_die("Couldn't find expected output file: $outputMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
    280         &my_die("Couldn't find expected output file: $outputVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance);
    281         &my_die("Couldn't find expected output file: $outputSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
    282         &my_die("Couldn't find expected output file: $jpeg1Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg1Name);
    283         &my_die("Couldn't find expected output file: $jpeg2Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg2Name);
    284         if ($inverse) {
    285             &my_die("Couldn't find expected output file: $inverseName", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseName);
    286             &my_die("Couldn't find expected output file: $inverseMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseMask);
    287             &my_die("Couldn't find expected output file: $inverseVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseVariance);
    288             &my_die("Couldn't find expected output file: $inverseSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseSources);
    289         }
    290     }
    291 }
    292 
    293 unless ($no_update) {
    294 
    295     # Add the subtraction result
    296     {
    297         my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -path_base $outroot";
    298         $command .= " $cmdflags";
    299         $command .= " -magicked" if $magicked;
    300         $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    301         $command .= " -hostname $host" if defined $host;
    302         $command .= " -dbname $dbname" if defined $dbname;
    303 
     307    unless ($no_op) {
    304308        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    305309            run(command => $command, verbose => $verbose);
    306310        unless ($success) {
    307311            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    308             &my_die("Unable to perform difftool -adddiffskyfile: $error_code", $diff_id, $skycell_id, $error_code);
     312            &my_die("Unable to perform ppSub: $error_code", $diff_id, $skycell_id, $error_code);
     313        }
     314
     315        my $outputStatsReal = $ipprc->file_resolve($outputStats);
     316        &my_die("Couldn't find expected output file: $outputStats", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
     317
     318        # measure chip stats
     319        $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL";
     320        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     321            run(command => $command, verbose => $verbose);
     322        unless ($success) {
     323            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     324            &my_die("Unable to perform ppStatsFromMetadata: $error_code", $diff_id, $skycell_id, $error_code);
     325        }
     326        foreach my $line (@$stdout_buf) {
     327            $cmdflags .= " $line";
     328        }
     329        chomp $cmdflags;
     330
     331        my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
     332
     333        if (!$quality) {
     334            &my_die("Couldn't find expected output file: $outputName", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
     335            &my_die("Couldn't find expected output file: $outputMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
     336            &my_die("Couldn't find expected output file: $outputVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance);
     337            &my_die("Couldn't find expected output file: $outputSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
     338            &my_die("Couldn't find expected output file: $jpeg1Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg1Name);
     339            &my_die("Couldn't find expected output file: $jpeg2Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg2Name);
     340            if ($inverse) {
     341                &my_die("Couldn't find expected output file: $inverseName", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseName);
     342                &my_die("Couldn't find expected output file: $inverseMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseMask);
     343                &my_die("Couldn't find expected output file: $inverseVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseVariance);
     344                &my_die("Couldn't find expected output file: $inverseSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseSources);
     345                if ($run_state eq 'new') {
     346                    &my_die("Couldn't find expected output file: $configuration", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
     347                }
     348            }
     349        }
     350    } else {
     351        print "Not executing: $command\n";
     352    }
     353}
     354
     355unless ($no_update) {
     356
     357    # Add the subtraction result
     358    {
     359        my $command = "$difftool -diff_id $diff_id -skycell_id $skycell_id";
     360        $command .= " -magicked $magicked" if $magicked;
     361        if ($run_state eq 'new') {
     362            $command .= " -adddiffskyfile -path_base $outroot";
     363            $command .= " $cmdflags";
     364            $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
     365            $command .= " -hostname $host" if defined $host;
     366        } else {
     367            $command .= " -tofullskyfile";
     368        }
     369        $command .= " -dbname $dbname" if defined $dbname;
     370
     371        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     372            run(command => $command, verbose => $verbose);
     373        unless ($success) {
     374            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     375            my $err_message = $run_state eq "update" ?
     376                "Unable to perform difftool -adddiffskyfile" :
     377                "Unable to perform difftool -tofullskyfile";
     378            &my_die("$err_message: $error_code", $diff_id, $skycell_id, $error_code);
    309379        }
    310380    }
    … …  
    323393    warn($msg);
    324394    if (defined $diff_id and defined $skycell_id and not $no_update) {
    325         my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -fault $exit_code";
    326         $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    327         $command .= " -hostname $host" if defined $host;
    328         $command .= " -path_base $outroot" if defined $outroot;
    329         $command .= " -dbname $dbname" if defined $dbname;
     395        my $command = "$difftool -diff_id $diff_id -skycell_id $skycell_id -fault $exit_code";
     396        if ($run_state eq 'new') {
     397            $command .= " -adddiffskyfile";
     398            $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
     399            $command .= " -hostname $host" if defined $host;
     400            $command .= " -path_base $outroot" if defined $outroot;
     401            $command .= " -dbname $dbname" if defined $dbname;
     402        } else {
     403            $command .= " -updatediffskyfile";
     404        }
    330405        run(command => $command, verbose => $verbose);
    331406    }
Note: See TracChangeset for help on using the changeset viewer.