IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 35715


Ignore:
Timestamp:
Jun 26, 2013, 4:44:14 PM (13 years ago)
Author:
bills
Message:

Implement "Update" for camera stage.
This mode uses the existing smf file and updates the dynamic masks.
Will be used to create new masks from LAP which have correct
dynamic masking. Also builds (rebuilds) the background model files

Location:
trunk
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/camera_exp.pl

    r35367 r35715  
    5353    'skip-binned'       => \$skip_binned, # override recipe - don't create binned images
    5454    'skip-refmask'      => \$skip_masks, # override recipe - don't create refmask
    55     'bkg-only'          => \$bkg_only,  # override recipe - only do background continuity
     55#    'bkg-only'          => \$bkg_only,  # override recipe - only do background continuity
    5656    'verbose'           => \$verbose,   # Print to stdout
    5757    'no-update'         => \$no_update, # Update the database?
    … …  
    7979
    8080my $replicateOutputs = 1;
     81
     82$ipprc->outroot_prepare($outroot);
    8183
    8284my $logDest;
    … …  
    193195
    194196
    195 my $produceMasks;               # Produce masks?
     197my $do_masks;               # Produce masks?
    196198if (!$skip_masks) {
    197199    # Get the PSASTRO recipe
    … …  
    206208        &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR);
    207209
    208     $produceMasks = metadataLookupBool($recipeData, 'REFSTAR_MASK');
     210    $do_masks = metadataLookupBool($recipeData, 'REFSTAR_MASK');
    209211}
    210212
    … …  
    220222my ($list5File, $list5Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b5.list.XXXX", UNLINK => !$save_temps ); # For background models
    221223
    222 # XXX we perform astrometry iff photometry output exists
     224
     225my $do_stats;
     226my $do_bkg;
     227my $do_jpegs;
     228my $fpaStats;
     229my $psastroInputArg;
     230if ($run_state eq 'new') {
     231    $do_stats = 1;
     232    $do_bkg = 1;
     233    $do_jpegs = !$skip_binned;
     234    $fpaStats = prepare_output("PSASTRO.STATS",      $outroot, undef, 1);
     235    $psastroInputArg = " -list $list3Name";
     236} else {
     237    # for $run_state eq 'update' we onlly rebuild the masks using psastro
     238    $do_stats = 0;
     239    $do_jpegs = 0;
     240    $do_bkg = 1;        # we could skip this step if camProcessedExp.backgroun_model is non zero
     241
     242    if (!$do_masks) {
     243        &my_die("run_state is update but do_masks is F. I have nothing to do!!", $cam_id, $PS_EXIT_UNKNOWN_ERROR)
     244    }
     245
     246    # the input to psastro is the original PSASTRO.OUTPUT
     247   
     248    # Check for file with rule PSASTRO.OUTPUT.ORGINAL
     249    # This file will exist if we attempted to update this camRun before but faulted
     250    # XXX: make sure cleanup deals with these files
     251    my $inputObjects = $ipprc->filename("PSASTRO.OUTPUT.ORIGINAL", $outroot, undef);
     252    if (!$ipprc->file_exists($inputObjects)) {
     253        # not found so original file should still be in place.
     254        # Rename it.
     255        my $originalObjects = $ipprc->filename("PSASTRO.OUTPUT", $outroot, undef);
     256        if ($ipprc->file_exists($originalObjects)) {
     257
     258            unless ($ipprc->file_rename($originalObjects, $inputObjects)) {
     259                &my_die("failed to rename $originalObjects to $inputObjects", $cam_id, $PS_EXIT_UNKNOWN_ERROR);
     260            }
     261
     262            # ok ready to go
     263        } else {
     264            print STDERR "failed to find input objects to update\n";
     265            print STDERR "Original file name: $originalObjects\n";
     266            print STDERR "Saved file name:    $inputObjects\n";
     267
     268            &my_die("Cannot proceed.", $cam_id, $PS_EXIT_DATA_ERROR);
     269
     270            # XXX: actually we could use the chip stage cmfs as inputs and turn on astrometry
     271            # ... except that somebody (me) got the bright idea to save space and
     272            # changed chip stage cleanup to remove them. So we're kind of stuck.
     273        }
     274    }
     275    $psastroInputArg = " -file $inputObjects -skipastro";
     276}
     277   
     278
    223279my @outMasks;                   # Names of output masks
    224280my @bkg_models;                 # Names of output background models
    225281foreach my $file (@$files) {
     282    # we perform astrometry iff photometry output exists
    226283    next if $file->{quality} != 0;
     284
    227285    # use the path_base as OUTPUT root and convert the filenames with ipprc->filename:
    228286    my $class_id = $file->{class_id};
    … …  
    239297    print $list5File ($ipprc->filename("PSPHOT.BACKMDL", $file->{path_base}, $class_id) . "\n");
    240298
    241     push @outMasks, prepare_output("PSASTRO.OUTPUT.MASK", $outroot, $class_id, 1) if $produceMasks;
    242     push @bkg_models, prepare_output("PPIMAGE.BACKMDL", $outroot, $class_id, 1);
     299    push @outMasks, prepare_output("PSASTRO.OUTPUT.MASK", $outroot, $class_id, 1) if $do_masks;
     300    push @bkg_models, prepare_output("PPIMAGE.BACKMDL", $outroot, $class_id, 1) if $do_bkg;
    243301}
    244302close $list1File;
    … …  
    247305close $list4File;
    248306close $list5File;
    249 # Output products
    250 $ipprc->outroot_prepare($outroot);
     307
     308# Prepare the Output products
    251309
    252310# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs
    253 my $jpeg1      = prepare_output("PPIMAGE.JPEG1",      $outroot, undef, 1);
    254 my $jpeg2      = prepare_output("PPIMAGE.JPEG2",      $outroot, undef, 1);
    255 my $fpaObjects = prepare_output("PSASTRO.OUTPUT",     $outroot, undef, 1);
    256 my $configuration = prepare_output("PSASTRO.CONFIG",  $outroot, undef, 1);
    257 
    258 my $do_stats;
    259 my $fpaStats;
     311my $jpeg1      = prepare_output("PPIMAGE.JPEG1",      $outroot, undef, 1) if $do_jpegs;
     312my $jpeg2      = prepare_output("PPIMAGE.JPEG2",      $outroot, undef, 1) if $do_jpegs;
     313my $fpaObjects = prepare_output("PSASTRO.OUTPUT",     $outroot, undef, 1) if !$bkg_only;
     314
     315my $configuration;
    260316if ($run_state eq 'new') {
    261     $do_stats = 1;
    262     $fpaStats = prepare_output("PSASTRO.STATS",      $outroot, undef, 1);
     317    $configuration = prepare_output("PSASTRO.CONFIG",  $outroot, undef, 1);
    263318} else {
    264     $do_stats = 0;
    265 }
     319    # Do not use the original recipes because they might contain the
     320    # recipe values that caused the masks to get fouled up in the first place
     321    # $configuration = $ipprc->filename("PSASTRO.CONFIG",  $outroot, undef);
     322}
     323
    266324
    267325unless ($no_op) {
    … …  
    269327    ## build the output JPEG images first so we get them even if the astrometry fails
    270328
    271     # Make the jpeg for binning 1
    272     if (!$skip_binned and ($run_state eq 'new')) {
     329    if ($do_jpegs) {
     330        # Make the jpeg for binning 1
    273331        my $command = "$ppImage -list $list1Name $outroot"; # Command to run
    274332        $command .= " -recipe PPIMAGE $recipe1";
    … …  
    282340        }
    283341        check_output($jpeg1, $replicateOutputs);
    284     }
    285 
    286     # Make the jpeg for binning 2
    287     if (!$skip_binned and ($run_state eq 'new')) {
    288         my $command = "$ppImage -list $list2Name $outroot"; # Command to run
     342
     343        # Make the jpeg for binning 2
     344
     345        $command = "$ppImage -list $list2Name $outroot"; # Command to run
    289346        $command .= " -recipe PPIMAGE $recipe2";
    290347        $command .= " -dbname $dbname" if defined $dbname;
    291348
    292         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     349        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    293350            run(command => $command, verbose => $verbose);
    294351        unless ($success) {
    … …  
    302359        # run psastro on the chipObjects, producing fpaObjects
    303360        my $command;
    304         $command  = "$psastro";
    305         $command .= " -list $list3Name";
    306         $command .= " -masklist $list4Name" if $produceMasks;
     361        $command  = $psastro;
     362        $command .= " $psastroInputArg";
     363        $command .= " -masklist $list4Name" if $do_masks;
    307364        $command .= " $outroot";
    308365        $command .= " -recipe PSASTRO $recipe_psastro";
    309         $command .= " -Db PSASTRO:REFSTAR_MASK F" if $skip_masks;
     366        $command .= " -Db PSASTRO:REFSTAR_MASK F" if !$do_masks;
    310367        $command .= " -tracedest $traceDest -log $logDest";
    311368        $command .= " -dbname $dbname" if defined $dbname;
    … …  
    314371            $command .= " -dumpconfig $configuration";
    315372        } elsif ($run_state eq 'update') {
    316             $command .= " -ipprc $configuration";
     373            $command .= " -ipprc $configuration" if $configuration;
    317374        } else {
    318375            &my_die("invalid value for run-state: $run_state", $cam_id, $PS_EXIT_CONFIG_ERROR);
    … …  
    362419        }
    363420    }
     421
    364422    # Construct FPA continuity corrected background images
    365423    if ($camera =~ /ISP/) {
    366424        print "Skipping FPA continuity corrected background images for ISP\n";
    367     } else {
     425    } elsif ($do_bkg) {
    368426        my $command;
    369427        $command = "$ppImage";
    … …  
    381439            check_output($bkgModel, $replicateOutputs);
    382440        }
    383         $cmdflags = $cmdflags . " -background_model 1 ";
     441        if ($run_state eq 'new') {
     442            $cmdflags .= " -background_model 1 ";
     443        } else {
     444            my $command = "camtool -updateprocessedexp -set_background_model 1 -fault 0 -cam_id $cam_id";
     445            $command .= " -dbname $dbname" if $dbname;
     446            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     447                run(command => $command, verbose => $verbose);
     448            unless ($success) {
     449                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     450                &my_die("Unable to updateprocessedexp to set background_model: $error_code", $cam_id, $error_code);
     451            }
     452        }
    384453    }
    385454}

  • trunk/ippconfig/recipes/filerules-split.mdc

    r35678 r35715  
    1414
    1515PSASTRO.OUTPUT     STR PSASTRO.OUT.CMF.MEF
     16PSASTRO.OUTPUT.ORIGINAL     STR PSASTRO.OUT.ORIGINAL.CMF.MEF
    1617# PSASTRO.OUTPUT.SPL STR PSASTRO.OUT.CMF.SPL
    1718# PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF
    … …  
    275276PSASTRO.CONFIG               OUTPUT {OUTPUT}.psastro.mdc              TEXT            NONE       FPA        TRUE      NONE
    276277PSASTRO.OUTPUT.MASK          OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits      MASK            COMP_MASK  CHIP       TRUE      NONE
     278
     279PSASTRO.OUT.ORIGINAL.CMF.MEF OUTPUT {OUTPUT}.smf.original             CMF             NONE       FPA        TRUE      MEF
    277280                                                                                                             
    278281PSWARP.OUTPUT                OUTPUT {OUTPUT}.fits                     IMAGE           COMP_IMG   FPA        TRUE      NONE

  • trunk/psastro/src/psastro.h

    r28043 r35715  
    8383bool              psastroOneChipGrid (pmFPA *fpa, pmChip *chip, psArray *refset, psArray *rawset, psMetadata *recipe, psMetadata *updates);
    8484bool              psastroOneChipFit (pmFPA *fpa, pmChip *chip, pmReadout *readout, psArray *refset, psArray *rawset, psMetadata *recipe, psMetadata *updates);
    85 bool              psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source);
     85bool              psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source, bool saveExistingMatchedRefs);
    8686bool              psastroRefstarSubset (pmReadout *readout);
    8787pmLumFunc        *psastroLuminosityFunction (psArray *stars, pmLumFunc *rawFunc);

  • trunk/psastro/src/psastroAnalysis.c

    r28043 r35715  
    8686    }
    8787
    88     if (!psastroChooseRefstars (config, refs, "PSASTRO.INPUT")) {
     88    bool skipastro = psMetadataLookupBool (&status, config->arguments, "PSASTRO.SKIP.ASTRO");
     89
     90    if (!psastroChooseRefstars (config, refs, "PSASTRO.INPUT", skipastro)) {
    8991        psError (PSASTRO_ERR_UNKNOWN, false, "failed to select reference data for chips\n");
    9092        psFree(refs);
    … …  
    108110        mosastro  = psMetadataLookupBool (&status, recipe, "PSASTRO.MOSAIC.MODE");
    109111    }
    110     if (!chipastro && !mosastro) {
     112
     113    if (skipastro) {
     114        chipastro = false;
     115        mosastro = false;
     116        psLogMsg ("psastro", 3, "skip astrometry mode, accepting input astrometry\n");
     117    } else if (!chipastro && !mosastro) {
    111118        psLogMsg ("psastro", 3, "no astrometry mode selected, assuming chip astrometry\n");
    112119        chipastro = true;
    … …  
    144151    }
    145152
    146     if (!psastroZeroPoint (config)) {
    147         psError(psErrorCodeLast(), false, "Failed to calculate zero point.");
    148         return false;
    149     }
     153    if (!skipastro) {
     154        if (!psastroZeroPoint (config)) {
     155            psError(psErrorCodeLast(), false, "Failed to calculate zero point.");
     156            return false;
     157        }
    150158
    151     if (!psastroAstromGuessCheck (config)) {
    152         psError(psErrorCodeLast(), false, "Failed to check astrometry guess.");
    153         return false;
     159        if (!psastroAstromGuessCheck (config)) {
     160            psError(psErrorCodeLast(), false, "Failed to check astrometry guess.");
     161            return false;
     162        }
    154163    }
    155164

  • trunk/psastro/src/psastroArguments.c

    r26259 r35715  
    7474
    7575    // apply mosastro mode?
     76    bool mosastro = false;
    7677    if ((N = psArgumentGet (argc, argv, "-mosastro"))) {
    7778        psArgumentRemove (N, &argc, argv);
    … …  
    8182        psArgumentRemove (N, &argc, argv);
    8283        psMetadataAddBool (config->arguments, PS_LIST_TAIL, "PSASTRO.MOSAIC.MODE", PS_META_REPLACE, "", true);
     84        mosastro = true;
    8385    }
    8486
    8587    // apply chipastro mode?
     88    bool chipastro = false;
    8689    if ((N = psArgumentGet (argc, argv, "-chipastro"))) {
    8790        psArgumentRemove (N, &argc, argv);
    … …  
    9194        psArgumentRemove (N, &argc, argv);
    9295        psMetadataAddBool (config->arguments, PS_LIST_TAIL, "PSASTRO.CHIP.MODE", PS_META_REPLACE, "", true);
     96        chipastro = true;
     97    }
     98    if ((N = psArgumentGet (argc, argv, "-skipastro"))) {
     99        psArgumentRemove (N, &argc, argv);
     100        if (mosastro) {
     101            psError(PSASTRO_ERR_ARGUMENTS, true, "cannot specify +mosastro with -skipastro");
     102            psErrorStackPrint(stderr, "exit");
     103            return false;
     104        }
     105        if (chipastro) {
     106            psError(PSASTRO_ERR_ARGUMENTS, true, "cannot specify +chipastro with -skipastro");
     107            return false;
     108        }
     109        psMetadataAddBool (config->arguments, PS_LIST_TAIL, "PSASTRO.SKIP.ASTRO", PS_META_REPLACE, "", true);
    93110    }
    94111

  • trunk/psastro/src/psastroChooseRefstars.c

    r35419 r35715  
    2020}
    2121
    22 bool psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source) {
     22bool psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source, bool saveExistingMatchedRefs) {
    2323
    2424    bool status;
    … …  
    5555        // analysis.  however, we are re-doing the astrometry here, so remove
    5656        // that prior set of matched references
    57         psMetadataRemoveKey (fpa->analysis, "MATCHED_REFS");
     57        if (!saveExistingMatchedRefs) {
     58            psMetadataRemoveKey (fpa->analysis, "MATCHED_REFS");
     59        }
    5860    }
    5961

  • trunk/psastro/src/psastroExtractAnalysis.c

    r23592 r35715  
    4444    }
    4545
    46     if (!psastroChooseRefstars (config, refs, "PSASTRO.EXTRACT.ASTROM")) {
     46    if (!psastroChooseRefstars (config, refs, "PSASTRO.EXTRACT.ASTROM", false)) {
    4747        psError (PSASTRO_ERR_UNKNOWN, false, "failed to select reference data for chips\n");
    4848        psFree(refs);
Note: See TracChangeset for help on using the changeset viewer.