Changeset 29839

Timestamp:

Nov 26, 2010, 10:35:03 AM (15 years ago)

Author:

eugene

Message:

merge updates from trunk

Location:

branches/eam_branches/ipp-20101103/ippScripts

Files:

• . (modified) (1 prop)
• scripts/automate_stacks.pl (modified) (2 diffs)
• scripts/ipp_cleanup.pl (modified) (5 diffs)
• scripts/magic_destreak_cleanup.pl (modified) (1 diff)
• scripts/stack_skycell.pl (modified) (2 diffs)

branches/eam_branches/ipp-20101103/ippScripts/scripts/automate_stacks.pl

@@ -79 +79 @@
     'check_dqstats'        => \$check_dqstats,
     'queue_dqstats'        => \$queue_dqstats,
-    'check_diffs'          => \$check_diffs,
-    'queue_diffs'          => \$queue_diffs,
+    'check_diffs'          => \$check_stacks,
+    'queue_diffs'          => \$queue_stacks,
     'clean_old'            => \$clean_old,
     ) or pod2usage ( 2 );
@@ -1203 +1203 @@
         foreach my $filter (@filter_list) {
             if (exists($extra_processing{$target})) {
-                do_extra_processing($date,$target,$filter,$pretend);
+                my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter);
+                if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :(
+                    next;
+                }
+                if ($Nexposures == 0) {
+                    if ($debug == 1) {
+                        print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n";
+                    }
+                    next;
+                }                   
+                else {
+                    do_extra_processing($date,$target,$filter,$pretend);
+                }
+
             }
             if ($stackable_list{$target} == 1) {

branches/eam_branches/ipp-20101103/ippScripts/scripts/ipp_cleanup.pl

@@ -18 +18 @@
 
 # Parse the command-line arguments
-my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist);
+my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile);
 GetOptions('stage=s'        => \$stage,     # which analysis stage to clean?
            'camera|i=s'     => \$camera,    # user-supplied camera name
@@ -27 +27 @@
            'verbose'        => \$verbose,   # Print to stdout
            'no-op'          => \$no_op,     # pretend but don't actually inject
-           'helplist'       => \$helplist   # give help listing
+           'helplist'       => \$helplist,  # give help listing
+           'logfile=s'      => \$logfile    # destination for stdout and stderr
            ) or pod2usage( 2 );
 
@@ -53 +54 @@
 }
 
+$ipprc->redirect_output($logfile) or 
+        &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile;
+
 # set this to 1 to enable checking for files on dead nodes
 # it is off for now because the implementation is a hack
 # See comments below.
-my $check_for_gone = 0;
+my $check_for_gone = 1;
 
 my $error_state;
@@ -66 +70 @@
 
 my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff"  => 1,
+               "chip_bg" => 1, "warp_bg" => 1,
                "detrend.processed" => 1, "detrend.resid" => 1, "detrend.process.exp" => 0, "detrend.stack.imfile" => 0,
                "detrend.normstat.imfile" => 0, "detrend.norm.imfile" => 0, "detrend.norm.exp" => 0 );
@@ -1648 +1653 @@
 }
 
+if ($stage eq "chip_bg") {
+
+    die "--stage_id required for stage chip_bg\n" if !$stage_id;
+
+    &my_die("only mode goto_cleaned is supported for stage chip_bg", "chip_bg", $stage_id, $PS_EXIT_PROG_ERROR)
+        if $mode ne 'goto_cleaned';
+
+    ### select the imfiles for this entry
+
+    # this stage uses 'bgtool'
+    my $bgtool = can_run('bgtool') or die "Can't find bgtool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $imfiles;                      # Array of component files
+    my $command = "$bgtool -pendingcleanupchipimfile -chip_bg_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code);
+    }
+
+    # XXX: Is not having any files to process really a bug?
+    if (@$stdout_buf == 0)  {
+        my $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $error_state";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code);
+        }
+        exit 0;
+    }
+
+    # extract the metadata for the files into a hash list
+    $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "chip_bg", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # loop over all of the imfiles, determine the path_base and class_id for each
+    foreach my $imfile (@$imfiles) {
+        my $class_id = $imfile->{class_id};
+        my $path_base = $imfile->{path_base};
+        my $status = 1;
+        $status = 0 unless defined $path_base and $path_base ne "NULL";
+
+        # don't clean up unless the data needed to update is available
+        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
+        # goto_scrubbed now requires the config file to not exist.
+        if ($status) {
+            if ($mode eq "goto_cleaned") {
+                my $config_file = $ipprc->filename("PPBACKGROUND.CONFIG", $path_base, $class_id);
+
+                unless ($ipprc->file_exists($config_file)) {
+                    if (file_gone($config_file)) {
+                        print STDERR "forcing cleanup for chipRun $stage_id $class_id "
+                            . " because config file ($config_file) is gone\n";
+                    } else {
+                        print STDERR "skipping cleanup for chipRun $stage_id $class_id "
+                            . " because config file ($config_file) is missing\n";
+                        $status = 0;
+                    }
+                }
+            }
+            elsif ($mode eq "goto_scrubbed") {
+                my $config_file = $ipprc->filename("PPBACKGROUND.CONFIG", $path_base, $class_id);
+
+                if ($ipprc->file_exists($config_file)) {
+                    print STDERR "skipping scrubbed for chipBackgroundRun $stage_id $class_id "
+                        . " because config file ($config_file) is present\n";
+                    $status = 0;
+                }
+            }
+        }
+
+        if ($status) {
+            # array of actual filenames to delete
+            my @files = ();
+
+            # delete the image datafiles
+            addFilename (\@files, "PPBACKGROUND.OUTPUT", $path_base, $class_id);
+            addFilename (\@files, "PPBACKGROUND.OUTPUT.MASK", $path_base, $class_id);
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+                addFilename (\@files, "PPBACKGROUND.STATS", $path_base, $class_id);         #clean?
+                addFilename (\@files, "PPBACKGOROUND.CONFIG", $path_base, $class_id);
+            }
+
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+
+        if ($status)  {
+            my $command = "$bgtool -chip_bg_id $stage_id -class_id $class_id";
+            if ($mode eq "goto_cleaned") {
+                $command .= " -tocleanedchipimfile";
+            }
+            elsif ($mode eq "goto_purged") {
+                $command .= " -topurgedchipimfile";
+            }
+            elsif ($mode eq "goto_scrubbed") {
+                $command .= " -toscrubbedchipimfile";
+            }
+
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+            }
+
+        } else {
+
+            # if an error happens for one chip, the chipBackgroundRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
+            my $command = "$bgtool -updatechipimfile -chip_bg_id $stage_id -class_id $class_id -set_data_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code);
+            }
+
+            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
+            $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+            }
+
+        }
+    }
+    exit 0;
+}
+if ($stage eq "warp_bg") {
+
+    die "--stage_id required for stage warp_bg\n" if !$stage_id;
+
+    &my_die("only mode goto_cleaned is supported for stage warp_bg", "warp_bg", $stage_id, $PS_EXIT_PROG_ERROR)
+        if $mode ne 'goto_cleaned';
+
+    ### select the imfiles for this entry
+
+    # this stage uses 'bgtool'
+    my $bgtool = can_run('bgtool') or die "Can't find bgtool";
+
+    # Get list of component imfiles
+    # XXX may need a different my_die for each stage
+    my $imfiles;                      # Array of component files
+    my $command = "$bgtool -pendingcleanupwarpskyfile -warp_bg_id $stage_id"; # Command to run
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code);
+    }
+
+    # XXX: Is not having any files to process really a bug?
+    if (@$stdout_buf == 0)  {
+        my $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $error_state";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code);
+        }
+        exit 0;
+    }
+
+    # extract the metadata for the files into a hash list
+    $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", "warp_bg", $stage_id, $PS_EXIT_PROG_ERROR);
+
+    # loop over all of the imfiles, determine the path_base and skycell_id for each
+    foreach my $imfile (@$imfiles) {
+        my $skycell_id = $imfile->{skycell_id};
+        my $path_base = $imfile->{path_base};
+        my $status = 1;
+        $status = 0 unless defined $path_base and $path_base ne "NULL";
+
+        # don't clean up unless the data needed to update is available
+        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
+        # goto_scrubbed now requires the config file to not exist.
+        if ($status) {
+            if ($mode eq "goto_cleaned") {
+                my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);
+
+                unless ($ipprc->file_exists($config_file)) {
+                    if (file_gone($config_file)) {
+                        print STDERR "forcing cleanup for warpBackgroundSkyfile $stage_id $skycell_id "
+                            . " because config file ($config_file) is gone\n";
+                    } else {
+                        print STDERR "skipping cleanup for warpBackgroundSkyfile $stage_id $skycell_id "
+                            . " because config file ($config_file) is missing\n";
+                        $status = 0;
+                    }
+                }
+            }
+            elsif ($mode eq "goto_scrubbed") {
+                my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id);
+
+                if ($ipprc->file_exists($config_file)) {
+                    print STDERR "skipping scrubbed for chipBackgroundRun $stage_id $skycell_id "
+                        . " because config file ($config_file) is present\n";
+                    $status = 0;
+                }
+            }
+        }
+
+        if ($status) {
+            # array of actual filenames to delete
+            my @files = ();
+
+            # delete the image datafiles
+            addFilename (\@files, "PSWARP.OUTPUT", $path_base, $skycell_id);
+            addFilename (\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+                addFilename (\@files, "PSWARP.STATS", $path_base, $skycell_id);         #clean?
+                addFilename (\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
+            }
+
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+
+        if ($status)  {
+            my $command = "$bgtool -warp_bg_id $stage_id -skycell_id $skycell_id";
+            # recall that only goto_cleaned is supported (currently)
+            if ($mode eq "goto_cleaned") {
+                $command .= " -tocleanedwarpskyfile";
+            }
+            elsif ($mode eq "goto_purged") {
+                $command .= " -topurgedwarpskyfile";
+            }
+            elsif ($mode eq "goto_scrubbed") {
+                $command .= " -toscrubbedwarpskyfile";
+            }
+
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform bgtool: $error_code", "warp", $stage_id, $error_code);
+            }
+
+        } else {
+
+            # if an error happens for one skycell, the warpBackgroundRun will stay in goto_*, but the skycells will go to error_* (matching the goto_*)
+            my $command = "$bgtool -updatewarpskyfile -warp_bg_id $stage_id -skycell_id $skycell_id -set_data_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code);
+            }
+
+            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
+            $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform bgtool: $error_code", "warp", $stage_id, $error_code);
+            }
+        }
+    }
+    exit 0;
+}
 
 die "ipp_cleanup.pl -stage $stage not yet implemented\n";

branches/eam_branches/ipp-20101103/ippScripts/scripts/magic_destreak_cleanup.pl

@@ -130 +130 @@
         push @components, $comp;
         $num_components++;
+}
+if ($num_components == 0) {
+    # no components to clean up set run state to cleaned
+    my $command = "$magicdstool -updaterun -magic_ds_id $magic_ds_id -set_state cleaned";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            carp("failed to update database for $magic_ds_id");
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+    exit 0;
 }
 

branches/eam_branches/ipp-20101103/ippScripts/scripts/stack_skycell.pl

@@ -171 +171 @@
 my $convolve = metadataLookupBool($recipe, 'CONVOLVE'); # Convolve inputs?
 my $photometry = metadataLookupBool($recipe, 'PHOTOMETRY'); # perform photometry?
+my $output_nocomp = metadataLookupBool($recipe, 'OUTPUT.NOCOMP'); # change filerules to produced uncompressed output images
 
 
@@ -243 +244 @@
     $command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL";
     $command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID";
+    if ($output_nocomp) {
+        $command .= " -F PPSTACK.OUTPUT PPSTACK.OUTPUT.NOCOMP";
+        $command .= " -F PPSTACK.OUTPUT.VARIANCE PPSTACK.OUTPUT.VARIANCE.NOCOMP";
+        $command .= " -F PPSTACK.OUTPUT.EXPWT PPSTACK.OUTPUT.EXPWT.NOCOMP";
+        $command .= " -F PPSTACK.UNCONV PPSTACK.UNCONV.NOCOMP";
+        $command .= " -F PPSTACK.UNCONV.VARIANCE PPSTACK.UNCONV.VARIANCE.NOCOMP";
+        $command .= " -F PPSTACK.UNCONV.EXPWT PPSTACK.UNCONV.EXPWT.NOCOMP";
+    }
     $command .= " -threads $threads" if defined $threads;
     $command .= " -debug-stack" if defined $debug;