IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Dec 1, 2010, 12:49:23 PM (15 years ago)
Author:
bills
Message:

Reorganize chip_imfile.pl and camera_exp.pl to replicate output files

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/chip_imfile.pl

    r28899 r29893  
    6666
    6767pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    68 pod2usage( -msg => "Required options: --exp_id --chip_id --chip_imfile_id --class_id --uri --camera --outroot --run-state",
     68pod2usage( -msg => "Required options: --exp_id --chip_id --chip_imfile_id --class_id --uri --camera --outroot --run-state --dbname",
    6969           -exitval => 3) unless
    7070    defined $exp_id and
    … …  
    7575    defined $camera and
    7676    defined $outroot and
     77    defined $dbname and
    7778    defined $run_state;
    7879
    7980my_die ("$run_state is an invalid value for run-state", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR) unless ($run_state eq 'new' or $run_state eq 'update');
    8081
    81 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    82 
    83 my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    84 $logDest .= ".update" if $run_state eq "update";
     82my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR );
     83
     84my $neb;
     85if (file_scheme($outroot) eq 'neb') {
     86    $neb = $ipprc->nebulous();
     87}
     88
     89my ($logDest, $traceDest);
     90if ($run_state eq 'new') {
     91    $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, 0);
     92    $traceDest = prepare_output("TRACE.IMFILE",  $outroot, $class_id, 1);
     93} else {
     94    $logDest = prepare_output("LOG.IMFILE.UPDATE", $outroot, $class_id, 1);
     95    $traceDest = prepare_output("TRACE.IMFILE.UPDATE",  $outroot, $class_id, 1);
     96}
    8597
    8698if ($redirect) {
    … …  
    90102    print STDOUT "FULL COMMAND: $0 @ARGS\n\n";
    91103}
     104
    92105
    93106# Recipes to use based on reduction class
    … …  
    114127
    115128## these names are used in ppImage, and thus may be URIs
    116 my $outputImage   = $ipprc->filename("PPIMAGE.CHIP",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    117 my $outputMask    = $ipprc->filename("PPIMAGE.CHIP.MASK",   $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    118 my $outputWeight  = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    119 my $outputBin1    = $ipprc->filename("PPIMAGE.BIN1",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    120 my $outputBin2    = $ipprc->filename("PPIMAGE.BIN2",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    121 my $outputStats   = $ipprc->filename("PPIMAGE.STATS",       $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    122 my $traceDest     = $ipprc->filename("TRACE.IMFILE",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    123 my $configuration = $ipprc->filename("PPIMAGE.CONFIG",      $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    124 
    125 if ($run_state eq 'update') {
    126     $outputStats .= '.update';
    127     $traceDest .= '.update';
     129my $outputImage   = prepare_output("PPIMAGE.CHIP",          $outroot, $class_id, 1 );
     130my $outputMask    = prepare_output("PPIMAGE.CHIP.MASK",     $outroot, $class_id, 1);
     131my $outputWeight  = prepare_output("PPIMAGE.CHIP.VARIANCE", $outroot, $class_id, 1);
     132my $pattern       = prepare_output("PPIMAGE.PATTERN",       $outroot, $class_id, 1);
     133my $backmdl       = prepare_output("PSPHOT.BACKMDL",        $outroot, $class_id, 1);
     134
     135my $configuration;
     136my $outputSources;
     137my $outputPsf;
     138my $outputStats;
     139my $outputBin1;
     140my $outputBin2;
     141my $dump_config = 1;
     142my $do_binned_images = 1;
     143if ($run_state eq 'new') {
     144    # prepare the files that are only created for a new run
     145    $configuration = prepare_output("PPIMAGE.CONFIG",        $outroot, $class_id, 1);
     146    $outputStats   = prepare_output("PPIMAGE.STATS",         $outroot, $class_id, 1);
     147} else {
     148    $configuration = $ipprc->filename('PPIMAGE.CONFIG', $outroot, $class_id)
     149            or &my_die("Missing entry from camera config: PPIMAGE.CONFIG", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     150    if ($ipprc->file_exists($configuration)) {
     151        $dump_config = 0;
     152    } else {
     153        print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n";
     154
     155        # XXX: should we create a new config dump file?
     156        # I vote yes but only if we can distingusing between temporarily unavailable and GONE.
     157        my $gone = 0;
     158        if (storage_object_exists($configuration, \$gone)) {
     159            if ($gone) {
     160                rename_gone_file($configuration);
     161                $configuration = prepare_output('PPIMAGE.CONFIG', $outroot, $class_id, 1);
     162                # if we dump the config we need to insure that the config dump represents
     163                # the full processing
     164            } else {
     165                # file is temporarily not available. Don't dump config.
     166                $dump_config = 0;
     167            }
     168        }
     169    }
     170
    128171    # make sure that any lingering destreak backup files are gone
    129172    $ipprc->delete_destreak_backup_file($outputImage)
    … …  
    134177        or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR);
    135178}
     179if ($do_binned_images) {
     180    $outputBin1    = prepare_output("PPIMAGE.BIN1",          $outroot, $class_id, 1);
     181    $outputBin2    = prepare_output("PPIMAGE.BIN2",          $outroot, $class_id, 1);
     182}
    136183
    137184my $cmdflags;
    … …  
    152199    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
    153200        &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     201
     202    my $do_photom = metadataLookupBool($recipeData, 'PHOTOM');
     203    if ($do_photom and ($run_state eq 'update')) {
     204        # If previous photometry outputs are ok skip running photometry
     205        if ($dump_config || rerun_photometry($outroot, $class_id)) {
     206            carp "Will rerun photometry\n";
     207        } else {
     208            $do_photom = 0;
     209        }
     210    }
     211    if ($do_photom) {
     212        $outputSources = prepare_output("PSPHOT.OUTPUT",   $outroot, $class_id, 1);
     213        $outputPsf     = prepare_output("PSPHOT.PSF.SAVE", $outroot, $class_id, 1);
     214    }
     215
    154216
    155217    ## XXX make the feature more general?
    … …  
    357419    }
    358420
     421    $command  = "$ppImage -file $uri $outroot";
     422    if ($dump_config) {
     423        $command .= " -recipe PPIMAGE $recipe_ppImage";
     424        $command .= " -dumpconfig $configuration";
     425    } else {
     426        $command .= " -ipprc $configuration";
     427    }
     428    if ($do_photom) {
     429        $command .= " -recipe PSPHOT $recipe_psphot";
     430    } else {
     431        $command .= " -Db PPIMAGE:PHOTOM FALSE";
     432    }
    359433    if ($run_state eq "new") {
    360         $command  = "$ppImage -file $uri $outroot";
    361         $command .= " -recipe PPIMAGE $recipe_ppImage";
    362         $command .= " -recipe PSPHOT $recipe_psphot";
    363         $command .= " -threads $threads" if defined $threads;
    364         $command .= " -dbname $dbname" if defined $dbname;
    365         $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
    366         $command .= " -source_id $source_id" if defined $source_id;
    367         $command .= " -dumpconfig $configuration";
    368         $command .= " -tracedest $traceDest -log $logDest";
    369         $do_stats = 1;
    370     } else {
    371         $command  = "$ppImage -file $uri $outroot";
    372         $command .= " -ipprc $configuration";
    373         $command .= " -threads $threads" if defined $threads;
    374         $command .= " -dbname $dbname" if defined $dbname;
    375         $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
    376         $command .= " -source_id $source_id" if defined $source_id;
    377         $command .= " -tracedest $traceDest -log $logDest";
    378         $command .= " -Db PPIMAGE:PHOTOM FALSE";
    379     }
    380     if ($do_stats) {
    381434        $command .= " -recipe PPSTATS CHIPSTATS";
    382435        $command .= " -stats $outputStats";
    383     }
     436        $do_stats = 1;
     437    }
     438    if (!$do_binned_images) {
     439        $command .= " -Db PPIMAGE:BIN1.FITS FALSE -Db PPIMAGE:BIN2.FITS FALSE";
     440    }
     441    $command .= " -threads $threads" if defined $threads;
     442    $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
     443    $command .= " -source_id $source_id" if defined $source_id;
     444    $command .= " -tracedest $traceDest -log $logDest";
     445    $command .= " -dbname $dbname" if defined $dbname;
    384446
    385447    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    394456    my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS');
    395457    my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.VARIANCE.FITS');
     458    my $outputBackmdlExpect = metadataLookupBool($recipeData, 'BACKGROUND');
     459    my $outputPatternExpect = (metadataLookupBool($recipeData, 'PATTERN.ROW') or metadataLookupBool($recipeData, 'PATTERN.CELL')) ;
    396460
    397461    my $quality;                # Quality flag
    … …  
    416480
    417481    if (!$quality) {
    418         &my_die("Couldn't find expected output file: $outputImage\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputImageExpect or $ipprc->file_exists($outputImage);
    419         &my_die("Couldn't find expected output file: $outputMask\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputMaskExpect or $ipprc->file_exists($outputMask);
    420         &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputWeightExpect or $ipprc->file_exists($outputWeight);
    421         &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
    422         &my_die("Couldn't find expected output file: $outputBin2\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
    423         if ($run_state eq 'new') {
    424             &my_die("Couldn't find expected output file: $configuration", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
    425         }
    426     }
    427 
     482        my $replicateImages = 0;
     483        check_output($outputImage, $replicateImages) if $outputImageExpect;
     484        check_output($outputMask, $replicateImages) if $outputMaskExpect;
     485        check_output($outputWeight, $replicateImages) if $outputWeightExpect;
     486        if ($do_binned_images) {
     487            check_output($outputBin1, 1);
     488            check_output($outputBin2, 1);
     489        }
     490        check_output($configuration, 1) if $dump_config;
     491        check_output($backmdl, 1) if $outputBackmdlExpect;
     492        check_output($pattern, 1) if $outputPatternExpect;
     493        if ($do_photom) {
     494            check_output($outputSources, 1);
     495            check_output($outputPsf, 1);
     496        }
     497        # XXX: Do we want to replicate the stats, logs, trace file?
     498    }
    428499}
    429500
    … …  
    462533}
    463534
     535exit 0;
     536
     537# check whether psphot outputs should be regenerated.
     538# Whether we need to or not is a somewhat complicated question.
     539sub rerun_photometry
     540{
     541    my $outroot = shift;
     542    my $class_id = shift;
     543    my $outputSources = $ipprc->filename('PSPHOT.OUTPUT', $outroot, $class_id)
     544                    or &my_die("Missing entry from camera config: PSPHOT.OUTPUT", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     545
     546    my $make_sources = 0;
     547    my $sources_available = 0;
     548    if ($ipprc->file_exists($outputSources)) {
     549        $sources_available = 1;
     550    } else {
     551        carp "WARNING: photometry sources file $outputSources is not available";
     552        my $gone;
     553        if (storage_object_exists($outputSources, \$gone)) {
     554            # check whether the file is permanantely or temporarily gone
     555            if ($gone) {
     556                carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
     557                rename_gone_file($outputSources);
     558                $make_sources = 1;
     559            }
     560        } else {
     561            # storage object must have been deleted
     562            $make_sources = 1;
     563        }
     564    }
     565
     566    my $make_psf = 0;
     567    my $psf_available = 0;
     568    my $outputPsf = $ipprc->filename("PSPHOT.PSF.SAVE",       $outroot, $class_id)
     569                    or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     570    if ($ipprc->file_exists($outputPsf)) {
     571        $psf_available = 1;
     572    } else {
     573        carp "PSF file $outputPsf is missing";
     574        my $gone = 0;
     575        if (storage_object_exists($outputPsf, \$gone)) {
     576            # object exists, but no instances are available. If they are permanently gone
     577            # rename the storage object
     578            if ($gone) {
     579                carp "WARNING: PSF storage object exists but all instances are permanently gone";
     580                rename_gone_file($outputPsf);
     581                $make_psf = 1;
     582            }
     583        } else {
     584            $make_psf = 1;
     585        }
     586    }
     587
     588    if ($sources_available && $psf_available) {
     589        return 0;
     590    }
     591
     592    # if either of the files are gone rerun photometry unless the other file is temporarily not available
     593
     594    if (!$sources_available && !$make_sources) {
     595        # destreak will die if the sources is not available
     596        &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     597    }
     598    if (!$psf_available && !$make_psf) {
     599        # warp updates need the psf file
     600        &my_die("PSPHOT.PSF.SAVE is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     601    }
     602
     603    return $make_psf || $make_sources;
     604}
     605
     606# subroutine to check the status of a nebulous file. Used to distinguish between a storage object that
     607# does not exist and one that all of the instances have been lost.
     608# XXXX This should be implemented properly in Nebulous
     609# For now uses Bill's script 'whichnode' which queries the nebulous database directly
     610
     611my $whichnode;
     612sub storage_object_exists
     613{
     614    my $file = shift;
     615    my $ref_all_gone = shift;
     616
     617    my $exists = $neb->storage_object_exists($file);
     618    if (!$exists) {
     619        return 0;
     620    }
     621
     622    if (!$whichnode) {
     623        $whichnode = can_run('whichnode') or
     624            &my_die("Can't find whichnode",  $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     625    }
     626
     627    my $command = "$whichnode $file";
     628
     629    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     630        run(command => $command, verbose => $verbose);
     631    unless ($success) {
     632        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     633        &my_die("Unable to perform whichnode: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     634    }
     635
     636    my @lines = split "\n", (join "", @$stdout_buf);
     637
     638    if (scalar @lines == 0) {
     639        # no output the file is really and truely gone
     640        # XXX: this is now caught above
     641        print STDERR "storage object for $file does not exist\n";
     642        return 0;
     643    }
     644
     645    my $numGone = 0;
     646    my $numNotGone = 0;
     647    foreach my $line (@lines) {
     648        chomp $line;
     649
     650        # output lines are either
     651        #   "volume available"
     652        # or
     653        #   "volume not available"
     654
     655        my ($volume, $answer, undef) = split " ", $line;
     656        # our hack is if the volume has an X in the name it's gone
     657        if ($volume =~ /X/) {
     658            print STDERR "$file is on $volume which is gone\n";
     659            $numGone++;
     660        } elsif ($answer eq 'available') {
     661            $numNotGone++;
     662        } elsif ($answer eq 'not') {
     663            print STDERR "$file is on $volume which is not available\n";
     664            $numNotGone++;
     665        } else {
     666            print STDERR "unexpected output from whichnode: $line\n";
     667        }
     668    }
     669    # if there are any instances that are not on a gone volume set all_gone to 0
     670    if ($numNotGone == 0 and $numGone > 0) {
     671        $$ref_all_gone = 1;
     672    } else {
     673        $$ref_all_gone = 0;
     674    }
     675
     676    # storage object exists so return true
     677    return 1;
     678}
     679sub rename_gone_file
     680{
     681    # check whether the only instance of file is on a lost volume
     682    # XXX: we don't have a proper interface for this.
     683    # For now try to use Bill's hack: the script 'whichnode'
     684
     685    my $file = shift;
     686    $neb->move($file, "$file.gone") or
     687                    &my_die("failed to rename: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     688}
     689
     690# Prepare to write to an output file
     691#   Lookup the filename in the rules.
     692#   Make sure that if file exists and is a nebulous file that there is only one instance
     693#   Deal with files that have been lost.
     694sub prepare_output
     695{
     696    my $filerule = shift;
     697    my $outroot  = shift;
     698    my $class_id = shift;
     699    my $delete = shift;
     700    $delete = 0 if !defined $delete;
     701
     702    my $error;
     703    my $output = $ipprc->prepare_output($filerule, $outroot, $class_id, $delete, \$error)
     704                    or &my_die("failed to prepare output file for: $filerule", $exp_id, $chip_id, $class_id, $error);
     705    return $output;
     706}
     707
     708sub check_output
     709{
     710    my $file = shift;
     711    my $replicate = shift;
     712
     713    if (!defined $file) {
     714        return;
     715    }
     716
     717    &my_die("Couldn't find expected output file: $file",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
     718
     719    if ($replicate and (file_scheme($file) eq 'neb')) {
     720        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     721    }
     722}
    464723
    465724sub my_die
Note: See TracChangeset for help on using the changeset viewer.