IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 19561


Ignore:
Timestamp:
Sep 15, 2008, 2:24:27 PM (18 years ago)
Author:
bills
Message:

update processing for camera stage

Location:
trunk/ippScripts/scripts
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/camera_exp.pl

    r19466 r19561  
    2727
    2828my ( $exp_tag, $cam_id, $camera, $outroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update,
    29      $no_op, $redirect, $save_temps );
     29     $no_op, $redirect, $save_temps, $run_state);
    3030GetOptions(
    3131    'exp_tag=s'          => \$exp_tag, # Exposure identifier
    … …  
    3737    'reduction=s'       => \$reduction, # Reduction class
    3838    'dvodb|w=s'         => \$dvodb,  # output DVO database
     39    'run-state=s'       => \$run_state, # 'new' or 'update'
    3940    'verbose'           => \$verbose,   # Print to stdout
    4041    'no-update'         => \$no_update, # Update the database?
    … …  
    5758
    5859my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
     60if ($run_state eq 'update') {
     61    $logDest .= '.update';
     62}
    5963
    6064$ipprc->redirect_output($logDest) if $redirect;
    … …  
    196200my $fpaStats   = $ipprc->filename("PSASTRO.STATS",      $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    197201my $traceDest  = $ipprc->filename("TRACE.EXP",          $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
     202my $configuration = $ipprc->filename("PSASTRO.CONFIG",  $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
     203
     204if ($run_state eq 'update') {
     205    $traceDest .= '.update';
     206    $fpaStats .= '.update';
     207}
    198208
    199209# convert supplied DVO database name to UNIX filename
    … …  
    210220
    211221    # Make the jpeg for binning 1
    212     {
     222    if ($run_state eq 'new') {
    213223        my $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run
    214224        $command .= " -dbname $dbname" if defined $dbname;
    … …  
    224234
    225235    # Make the jpeg for binning 2
    226     {
     236    if ($run_state eq 'new') {
    227237        my $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run
    228238        $command .= " -dbname $dbname" if defined $dbname;
    … …  
    244254        $command  = "$psastro -list $list3Name $outroot";
    245255        $command .= " -tracedest $traceDest -log $logDest";
    246         $command .= " -stats $fpaStats -recipe PPSTATS CHIPSTATS";
    247256        $command .= " -dbname $dbname" if defined $dbname;
     257
     258        my $do_stats;
     259        if ($run_state eq 'new') {
     260            $command .= " -stats $fpaStats -recipe PPSTATS CHIPSTATS";
     261            $command .= " -dumpconfig $configuration";
     262            $do_stats = 1;
     263        } elsif ($run_state eq 'update') {
     264            $command .= " -ipprc $configuration";
     265        } else {
     266            &my_die("invalid value for run-state: $run_state", $cam_id, $PS_EXIT_CONFIG_ERROR);
     267        }
    248268
    249269        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    255275        # XXX do we want to give an error if astrometry fails here?
    256276        &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects);
    257         &my_die("Unable to find expected output file: $fpaStats",   $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects);
    258 
    259         # Get the statistics on the processed image
    260         my $statsFile;              # File handle
    261         open $statsFile, $ipprc->file_resolve($fpaStats) or &my_die("Can't open statistics file $fpaStats: $!", $cam_id, $PS_EXIT_SYS_ERROR);
    262         my @contents = <$statsFile>; # Contents of file
    263         close $statsFile;
    264 
    265         # parse the statistics MDC file
    266         my $mdcParser = PS::IPP::Metadata::Config->new();   # Parser for metadata config files
    267         my $metadata = $mdcParser->parse(join "", @contents);
    268         unless ($metadata) {
    269             &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR);
    270         }
    271 
    272         # extract the stats from the metadata
    273         unless ($camStats->parse($metadata)) {
    274             &my_die("Failure extracting metadata from the statistics output file.\n", $cam_id, $PS_EXIT_PROG_ERROR);
     277
     278        if ($do_stats) {
     279            &my_die("Unable to find expected output file: $fpaStats",   $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects);
     280
     281            # Get the statistics on the processed image
     282            my $statsFile;              # File handle
     283            open $statsFile, $ipprc->file_resolve($fpaStats) or &my_die("Can't open statistics file $fpaStats: $!", $cam_id, $PS_EXIT_SYS_ERROR);
     284            my @contents = <$statsFile>; # Contents of file
     285            close $statsFile;
     286
     287            # parse the statistics MDC file
     288            my $mdcParser = PS::IPP::Metadata::Config->new();   # Parser for metadata config files
     289            my $metadata = $mdcParser->parse(join "", @contents);
     290            unless ($metadata) {
     291                &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR);
     292            }
     293
     294            # extract the stats from the metadata
     295            unless ($camStats->parse($metadata)) {
     296                &my_die("Failure extracting metadata from the statistics output file.\n", $cam_id, $PS_EXIT_PROG_ERROR);
     297            }
    275298        }
    276299
    277300        # run addstar on the output fpaObjects (if a DVO database is defined)
    278         if (defined $dvodbReal) {
     301        if (defined $dvodbReal and ($run_state eq 'new')) {
    279302
    280303            ## XXX the camera analysis can either save the full set of
    … …  
    319342}
    320343
    321 my $fpaCommand = "$camtool -addprocessedexp";
    322 $fpaCommand .= " -cam_id $cam_id";
    323 $fpaCommand .= " -uri UNKNOWN";
    324 $fpaCommand .= " -path_base $outroot";
    325 $fpaCommand .= " -hostname $host" if defined $host;
     344
     345my $fpaCommand = "$camtool -cam_id $cam_id";
     346if ($run_state eq 'new') {
     347    $fpaCommand .= " -addprocessedexp";
     348    $fpaCommand .= " -uri UNKNOWN";
     349    $fpaCommand .= " -path_base $outroot";
     350    $fpaCommand .= $chipStats->cmdflags();
     351    $fpaCommand .= $camStats->cmdflags();
     352    $fpaCommand .= " -hostname $host" if defined $host;
     353} else {
     354    $fpaCommand .= " -updaterun -state full";
     355}
    326356$fpaCommand .= " -dbname $dbname" if defined $dbname;
    327 $fpaCommand .= $chipStats->cmdflags();
    328 $fpaCommand .= $camStats->cmdflags();
    329357
    330358# Add the result into the database
    … …  
    350378    carp($msg);
    351379    if (defined $cam_id and not $no_update) {
    352         my $command = "$camtool -addprocessedexp";
    353         $command .= " -cam_id $cam_id";
    354         $command .= " -uri UNKNOWN";
    355         $command .= " -code $exit_code";
    356         $command .= " -path_base $outroot";
     380        my $command = "$camtool -cam_id $cam_id";
     381        if ($run_state eq 'new') {
     382            $command .= " -addprocessedexp";
     383            $command .= " -uri UNKNOWN";
     384            $command .= " -code $exit_code";
     385            $command .= " -path_base $outroot";
     386            $command .= " -path_base $outroot" if defined $outroot;
     387        } else {
     388            $command .= " -updateprocessedexp";
     389            $command .= " -code $exit_code";
     390        }
    357391        $command .= " -hostname $host" if defined $host;
    358392        $command .= " -dbname $dbname" if defined $dbname;
    359         $command .= " -path_base $outroot" if defined $outroot;
    360393        system ($command);
    361394    }

  • trunk/ippScripts/scripts/ipp_cleanup.pl

    r19397 r19561  
    5050}
    5151
     52my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff  => 1);
     53unless ($stages{$stage}) {
     54    die "unknown stage $stage for ipp_cleanup.pl\n";
     55}
     56
    5257$ipprc->define_camera($camera);
    5358
    5459my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    55 
    56 my $fault = 0;
    5760
    5861# choice of files to delete depends on the stage
    5962if ($stage eq "chip") {
    6063   
     64    die "--stage_id required for stage chip\n" if !$stage_id;
    6165    ### select the imfiles for this entry
    6266
    … …  
    8589        my $class_id = $imfile->{class_id};
    8690        my $path_base = $imfile->{path_base};
    87 
    88         # array of actual filenames to delete
    89         my @files = ();
    90 
    91         # delete the temporary image datafiles
    92         addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
    93         addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
    94         addFilename (\@files, "PPIMAGE.OUTPUT.WEIGHT", $path_base, $class_id);
    95         addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
    96         addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
    97         addFilename (\@files, "PPIMAGE.CHIP.WEIGHT", $path_base, $class_id);
    98         if ($mode eq "goto_purged") {
    99             # additional files to remove for 'purge' mode
    100             addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
    101             addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
    102             addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
    103             addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
    104             addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
    105             addFilename (\@files, "PPIMAGE.JPEG", $path_base, $class_id);
    106             addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
    107         }
     91        my $status = 1;
     92
     93        # don't clean up unless the data needed to update is available
     94        if ($mode eq "goto_cleaned") {
     95            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
     96
     97            if (!$config_file or ! -e $config_file) {
     98                print STDERR "skipping cleanup for chipRun $stage_id $class_id "
     99                    . " because config file is missing\n";
     100                $status = 0;
     101            }
     102        }
     103
     104        if ($status) {
     105            # array of actual filenames to delete
     106            my @files = ();
     107
     108            # delete the temporary image datafiles
     109            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
     110            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
     111            addFilename (\@files, "PPIMAGE.OUTPUT.WEIGHT", $path_base, $class_id);
     112            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
     113            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
     114            addFilename (\@files, "PPIMAGE.CHIP.WEIGHT", $path_base, $class_id);
     115            if ($mode eq "goto_purged") {
     116                # additional files to remove for 'purge' mode
     117                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
     118                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
     119                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     120                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
     121                addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
     122                addFilename (\@files, "PPIMAGE.JPEG", $path_base, $class_id);
     123                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
     124                addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
     125            }
    108126       
    109         # actual command to delete the files
    110         my $status = &delete_files (\@files);
     127            # actual command to delete the files
     128            $status = &delete_files (\@files);
     129        }
    111130
    112131        if ($status)  {
    … …  
    119138            $command .= " -dbname $dbname" if defined $dbname;
    120139
    121             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     140            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     141                    run(command => $command, verbose => $verbose);
    122142            unless ($success) {
    123143                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    125145            }
    126146        } else {
    127             my $command = "$chiptool -updateimfile -chip_id $stage_id -class_id $class_id -code 1";
     147            my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -code 1";
    128148            $command .= " -dbname $dbname" if defined $dbname;
    129149
    130             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     150            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     151                    run(command => $command, verbose => $verbose);
    131152            unless ($success) {
    132153                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    133154                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    134155            }
    135             $fault = 1;
    136156        }
    137157    }
    138158
    139 if (0) {
    140     # This block is not necessary
    141     # chiptool -tocleanedimfile and -topurgedimfile automatically update chipRun when the
    142     # last image gets set to purged or cleaned
    143 
    144     if (! $fault) {
    145         # set the state of the chipRun entry to "cleaned" or "purged" as needed
    146         my $command = "$chiptool -updaterun -chip_id $stage_id";
    147         if ($mode eq "goto_purged") {
    148             $command .= " -state purged";
    149         } else {
    150             $command .= " -state cleaned";
    151         }
    152         $command .= " -dbname $dbname" if defined $dbname;
    153 
    154         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    155         unless ($success) {
    156             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    157             &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    158         }
    159     }
    160 }
    161     exit 0;
    162 }
    163 
    164 # choice of files to delete depends on the stage
    165 if ($stage eq "camera") {
     159} elsif ($stage eq "camera") {
     160    die "--stage_id required for stage camera\n" if !$stage_id;
    166161    # this stage uses 'camtool'
    167162    my $camtool = can_run('camtool') or die "Can't find camtool";
    … …  
    169164    # Get list of component imfiles
    170165    # XXX may need a different my_die for each stage
    171     my $imfiles;                      # Array of component files
     166    my $exps;                      # Array of component files
    172167    my $command = "$camtool -pendingcleanupexp -cam_id $stage_id"; # Command to run
    173168    $command .= " -dbname $dbname" if defined $dbname;
    … …  
    175170    unless ($success) {
    176171        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    177         &my_die("Unable to perform camtool: $error_code", "cam", $stage_id, $error_code);
     172        &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code);
    178173    }
    179174    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    180         &my_die("Unable to parse metadata config doc", "cam", $stage_id, $PS_EXIT_PROG_ERROR);
    181 
     175        &my_die("Unable to parse metadata config doc", "camera", $stage_id, $PS_EXIT_PROG_ERROR);
     176
     177    $exps = parse_md_list($metadata) or
     178        &my_die("Unable to parse metadata list", "camera", $stage_id, $PS_EXIT_PROG_ERROR);
     179
     180    my $n_exps = @$exps;
     181    &my_die("unexpected number of exposures $n_exps", "camera", $stage_id, $PS_EXIT_PROG_ERROR)
     182        if $n_exps != 1;
     183
     184    my $exp = $exps->[0];
     185    my $path_base = $exp->{path_base};
     186
     187    my $status = 1;
     188    # don't clean up unless the data needed to update is available
     189    if ($mode eq "goto_cleaned") {
     190        my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base);
     191
     192        if (!$config_file or ! -e $config_file) {
     193            print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n";
     194            $status = 0;
     195        }
     196    }
     197    if ($status) {
     198        my @files = ();
     199        # delete the temporary image datafiles
     200        addFilename (\@files, "PSASTRO.OUTPUT", $path_base);
     201        if ($mode eq "goto_purged") {
     202            # additional files to remove for 'purge' mode
     203            addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
     204            addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
     205            addFilename (\@files, "PSASTRO.STATS", $path_base);
     206        }
     207        # actual command to delete the files
     208        $status = &delete_files (\@files);
     209    }
     210
     211    if ($status)  {
     212        my $command = "$camtool -cam_id $stage_id -updaterun";
     213        if ($mode eq "goto_cleaned") {
     214            $command .= " -state cleaned";
     215        } else {
     216            $command .= " -state purged";
     217        }
     218        $command .= " -dbname $dbname" if defined $dbname;
     219        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     220            run(command => $command, verbose => $verbose);
     221        unless ($success) {
     222            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     223            &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code);
     224        }
     225    } else {
     226        my $command = "$camtool -updateprocessedexp -cam_id $stage_id -code 1";
     227        $command .= " -dbname $dbname" if defined $dbname;
     228
     229        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     230            run(command => $command, verbose => $verbose);
     231        unless ($success) {
     232            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     233            &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code);
     234        }
     235        exit $PS_EXIT_UNKNOWN_ERROR;
     236    }
    182237    exit 0;
    183 }
    184 
     238} elsif ($stage eq "warp") {
     239    die "--stage_id required for stage warp\n" if !$stage_id;
     240    # this stage uses 'warptool'
     241    my $warptool = can_run('warptool') or die "Can't find warptool";
     242
     243    # Get list of component imfiles
     244    # XXX may need a different my_die for each stage
     245    my $skyfiles;                      # Array of component files
     246    my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run
     247    $command .= " -dbname $dbname" if defined $dbname;
     248    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     249            run(command => $command, verbose => $verbose);
     250    unless ($success) {
     251        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     252        &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
     253    }
     254    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     255        &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR);
     256
     257    $skyfiles = parse_md_list($metadata) or
     258        &my_die("Unable to parse metadata list", "warp", $stage_id, $PS_EXIT_PROG_ERROR);
     259
     260    my @files = ();
     261    foreach my $skyfile (@$skyfiles) {
     262        my $path_base = $skyfile->{path_base};
     263        my $skycell_id = $skyfile->{skycell_id};
     264
     265        my $status = 1;
     266        if ($mode eq "goto_cleaned") {
     267            my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);
     268
     269            if (!$config_file or ! -e $config_file) {
     270                print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
     271                    " because config file is missing\n";
     272                $status = 0;
     273            }
     274        }
     275        if ($status) {
     276            # delete the temporary image datafiles
     277            addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id );
     278            addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
     279            addFilename(\@files, "PSWARP.OUTPUT.WEIGHT", $path_base, $skycell_id);
     280            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
     281
     282            if ($mode eq "goto_purged") {
     283                # additional files to remove for 'purge' mode
     284                addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id );
     285                addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id );
     286                addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id );
     287                # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base);
     288
     289                # XXX: do we want to delete these?
     290                # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id);
     291                # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
     292            }
     293            # actual command to delete the files
     294            $status = &delete_files (\@files);
     295        }
     296
     297        if ($status)  {
     298            my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
     299            if ($mode eq "goto_purged") {
     300                $command .= " -topurgedskyfile";
     301            } else {
     302                $command .= " -tocleanedskyfile";
     303            }
     304            $command .= " -dbname $dbname" if defined $dbname;
     305
     306            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     307                    run(command => $command, verbose => $verbose);
     308            unless ($success) {
     309                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     310                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
     311            }
     312         } else {
     313            # XXX: -updateskyfile mode does not exist, need to add it
     314            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -code 1";
     315            $command .= " -dbname $dbname" if defined $dbname;
     316
     317            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     318                run(command => $command, verbose => $verbose);
     319            unless ($success) {
     320                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     321                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
     322            }
     323            exit $PS_EXIT_UNKNOWN_ERROR;
     324        }
     325    }
     326    exit 0;
     327}
     328
     329# left TODO
    185330# fake : faketool : -pendingcleanupimfile (loop over imfiles)
    186 # warp : warptool : -pendingcleanupskyfile (loop over skyfiles)
    187 
    188 die "unknown stage for ipp_cleanup.pl\n";
    189 exit 1;
     331# stack: stacktool : -pendingcleanupskyfile (loop over skyfiles)
     332# diff:  difftool : -pendingcleanupskyfile
     333
     334die "ipp_cleanup.pl -stage $stage not yet implemented\n";
    190335
    191336sub delete_files
    … …  
    196341    foreach my $file (@$files) {
    197342        print STDERR "unlinking $file\n";
     343        # XXX: need to handle files in nebulous
    198344        unlink $file;
    199345    }
    … …  
    217363{
    218364    my $msg = shift; # Warning message on die
    219     my $stage = shift; # Chiptool identifier
    220     my $stage_id = shift; # Chiptool identifier
    221     my $exit_code = shift; # Exit code to add
    222     # outputImage and outroot are globals
     365    my $stage = shift; # stage name
     366    my $stage_id = shift; # identifier
     367    my $exit_code = shift; # Exit code
     368    # outputImage and path_base are globals
    223369
    224370    carp($msg);

  • trunk/ippScripts/scripts/stack_skycell.pl

    r19545 r19561  
    2828use Pod::Usage qw( pod2usage );
    2929
    30 my ($stack_id, $dbname, $outroot, $debug, $threads, $verbose, $no_update, $no_op, $redirect, $save_temps);
     30my ($stack_id, $dbname, $outroot, $debug, $run_state, $threads, $verbose, $no_update, $no_op, $redirect, $save_temps);
    3131GetOptions(
    3232    'stack_id|d=s'      => \$stack_id, # Stack identifier
    3333    'dbname|d=s'        => \$dbname, # Database name
    3434    'outroot=s'         => \$outroot, # Output root name
     35    'run-state=s'       => \$run_state,
    3536    'debug'             => \$debug,   # Print to stdout
    3637    'threads=s'         => \$threads,   # Number of threads to use for ppStack
    … …  
    4445pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    4546pod2usage(
    46     -msg => "Required options: --stack_id --outroot",
     47    -msg => "Required options: --stack_id --outroot --run-state",
    4748    -exitval => 3,
    4849          ) unless defined $stack_id
    49     and defined $outroot;
     50    and defined $outroot
     51    and defined $run_state;
    5052
    5153# XXX camera is not known here; cannot use filerules...

  • trunk/ippScripts/scripts/warp_skycell.pl

    r19396 r19561  
    2525my $ipprc = PS::IPP::Config->new(); # IPP configuration
    2626
    27 my ($warp_id, $skycell_id, $tess_dir, $camera, $dbname, $outroot, $threads, $verbose, $no_update, $no_op, $redirect, $save_temps);
     27my ($warp_id, $skycell_id, $tess_dir, $camera, $dbname, $outroot, $threads, $run_state, $verbose, $no_update, $no_op, $redirect, $save_temps);
    2828GetOptions(
    2929    'warp_id|i=s'       => \$warp_id, # Warp identifier
    … …  
    3333    'dbname|d=s'        => \$dbname, # Database name
    3434    'outroot=s'         => \$outroot, # Output root name
    35     'threads=s'         => \$threads,   # Number of threads to use for ppImage
     35    'threads=s'         => \$threads,   # Number of threads to use for pswarp
     36    'run-state=s'       => \$run_state,  # 'new' or 'update'
    3637    'verbose'           => \$verbose,   # Print to stdout
    3738    'no-update'         => \$no_update, # Don't update the database?
    … …  
    4344pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    4445pod2usage(
    45     -msg => "Required options: --warp_id --skycell_id --tess_dir --camera --outroot",
     46    -msg => "Required options: --warp_id --skycell_id --tess_dir --camera --outroot --run-state",
    4647    -exitval => 3,
    4748) unless defined $warp_id
    … …  
    4950    and defined $tess_dir
    5051    and defined $camera
    51     and defined $outroot;
     52    and defined $outroot
     53    and defined $run_state;
    5254
    5355$ipprc->define_camera($camera);
    5456
    5557my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id);
     58$logDest .= ".update" if ($run_state eq 'update');
    5659
    5760$ipprc->redirect_output($logDest) if $redirect;
    … …  
    127130my $configuration =  $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id);
    128131
     132if ($run_state eq 'update') {
     133    $traceDest .= ".update";
     134    $outputStats .= ".update";
     135}
     136
    129137my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id );
    130138$ipprc->skycell_file( $tess_dir, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
    … …  
    170178# Run pswarp
    171179my $accept = 1;                 # Accept the skycell?
     180my $do_stats;
    172181unless ($no_op) {
    173182    my $command = "$pswarp";
    … …  
    177186    $command .= " -astromlist $astromName";
    178187    $command .= " $outroot $skyFile";
    179     $command .= " -stats $outputStats";
    180     $command .= " -recipe PPSTATS WARPSTATS";
    181188    $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE";
    182189    $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF";
    … …  
    185192    $command .= " -tracedest $traceDest -log $logDest";
    186193    $command .= " -threads $threads" if defined $threads;
    187     $command .= " -dumpconfig $configuration";
    188194    $command .= " -dbname $dbname" if defined $dbname;
     195    if ($run_state eq 'new') {
     196        $command .= " -dumpconfig $configuration";
     197        $do_stats = 1;
     198    } else {
     199        $command .= " -ipprc $configuration";
     200    }
     201    if ($do_stats) {
     202        $command .= " -recipe PPSTATS WARPSTATS";
     203        $command .= " -stats $outputStats";
     204    }
    189205
    190206    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    195211    }
    196212
    197     # Check first for the stats file, and if the ACCEPT flag is set.
    198     &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats);
    199     # Get the statistics on the warped image
    200     my $statsFile;              # File handle
    201     open $statsFile, $ipprc->file_resolve($outputStats) or die "Can't open statistics file $outputStats: $!\n";
    202     my @contents = <$statsFile>; # Contents of file
    203     close $statsFile;
    204     my $contents = join "", @contents;
    205 
    206     my $metadata = $mdcParser->parse($contents)
    207         or &my_die("Unable to parse metadata config", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR);
    208     $accept = metadataLookupBool($metadata, "ACCEPT");
    209 
    210     # $accept is set above based on the fraction of lit pixels
    211     # XXX for some files, there may not be enough stars to find a good psf.  these should be dropped as well
    212     if ($accept && !$ipprc->file_exists($outputPSF)) {
    213         $accept = 0;
    214     }
    215 
    216     if ($accept) {
    217         $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR);
    218 
    219         &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
    220         &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
    221         &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
    222         &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
    223 #    &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF);
    224 #    &my_die("Couldn't find expected output file: $outputBin1", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
    225 #    &my_die("Couldn't find expected output file: $outputBin2", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
    226     }
    227 }
    228 
    229 unless ($no_update) {
    230     # XXX change -tess_id to -tess_dir when db is updated
    231     my $command = "$warptool -addwarped";
    232     $command .= " -warp_id $warp_id";
    233     $command .= " -skycell_id $skycell_id";
    234     $command .= " -tess_id $tess_dir";
    235     $command .= " -path_base $outroot"; # needed for logfile lookups
    236     $command .= " -ignore"           if not $accept; # Completed succesfully, but can't produce product
    237     $command .= " -uri $outputImage" if $accept;
    238     $command .= $stats->cmdflags()   if $accept;
    239     $command .= " -hostname $host"   if defined $host;
    240     $command .= " -dbname $dbname"   if defined $dbname;
    241 
    242     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    243         run(command => $command, verbose => $verbose);
    244     unless ($success) {
    245         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    246         warn("Unable to perform warptool -addwarped: $error_code\n");
    247         exit($error_code);
     213    if ($do_stats) {
     214        # Check first for the stats file, and if the ACCEPT flag is set.
     215        &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats);
     216        # Get the statistics on the warped image
     217        my $statsFile;              # File handle
     218        open $statsFile, $ipprc->file_resolve($outputStats) or die "Can't open statistics file $outputStats: $!\n";
     219        my @contents = <$statsFile>; # Contents of file
     220        close $statsFile;
     221        my $contents = join "", @contents;
     222
     223        my $metadata = $mdcParser->parse($contents)
     224            or &my_die("Unable to parse metadata config", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR);
     225        $accept = metadataLookupBool($metadata, "ACCEPT");
     226
     227        # $accept is set above based on the fraction of lit pixels
     228        # XXX for some files, there may not be enough stars to find a good psf.  these should be dropped as well
     229        if ($accept && !$ipprc->file_exists($outputPSF)) {
     230            $accept = 0;
     231        }
     232
     233        if ($accept) {
     234            $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR);
     235
     236            &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
     237            &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
     238            &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
     239            &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
     240    #    &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF);
     241    #    &my_die("Couldn't find expected output file: $outputBin1", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
     242    #    &my_die("Couldn't find expected output file: $outputBin2", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
     243        }
     244
     245        unless ($no_update) {
     246            # XXX change -tess_id to -tess_dir when db is updated
     247            my $command = "$warptool -addwarped";
     248            $command .= " -warp_id $warp_id";
     249            $command .= " -skycell_id $skycell_id";
     250            $command .= " -tess_id $tess_dir";
     251            $command .= " -path_base $outroot"; # needed for logfile lookups
     252            $command .= " -ignore"           if not $accept; # Completed succesfully, but can't produce product
     253            $command .= " -uri $outputImage" if $accept;
     254            $command .= $stats->cmdflags()   if $accept;
     255            $command .= " -hostname $host"   if defined $host;
     256            $command .= " -dbname $dbname"   if defined $dbname;
     257
     258            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     259                run(command => $command, verbose => $verbose);
     260            unless ($success) {
     261                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     262                warn("Unable to perform warptool -addwarped: $error_code\n");
     263                exit($error_code);
     264            }
     265        }
     266    } else {
     267        # $run_state eq 'update'
     268        unless ($no_update) {
     269            my $command = "$warptool -tofullskyfile";
     270            $command .= " -warp_id $warp_id";
     271            $command .= " -skycell_id $skycell_id";
     272            $command .= " -dbname $dbname"   if defined $dbname;
     273
     274            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     275                run(command => $command, verbose => $verbose);
     276            unless ($success) {
     277                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     278                warn("Unable to perform warptool -addwarped: $error_code\n");
     279                exit($error_code);
     280            }
     281        }
    248282    }
    249283}
    … …  
    260294    if (defined $warp_id and defined $skycell_id and defined $tess_dir and not $no_update) {
    261295        # XXX change -tess_id to -tess_dir when db is updated
    262         my $command = "$warptool -addwarped";
     296        my $command = "$warptool";
     297        if ($run_state eq 'new') {
     298            $command .= " -addwarped";
     299            $command .= " -tess_id $tess_dir";
     300            $command .= " -path_base $outroot";
     301            $command .= " -hostname $host" if defined $host;
     302        } else {
     303            $command .= " -updateskyfile";
     304        }
    263305        $command .= " -warp_id $warp_id";
    264306        $command .= " -skycell_id $skycell_id";
    265         $command .= " -tess_id $tess_dir";
    266307        $command .= " -code $exit_code";
    267         $command .= " -path_base $outroot";
    268         $command .= " -hostname $host" if defined $host;
    269308        $command .= " -dbname $dbname" if defined $dbname;
    270309        run(command => $command, verbose => $verbose);
Note: See TracChangeset for help on using the changeset viewer.