IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Dec 7, 2012, 11:33:47 AM (13 years ago)
Author:
bills
Message:

Enhancements to ipp cleanup for chip warp and diff stage.

Added --check-all which cleans all components regardless of data_state.
This may be used to add new files (such as cmfs) to the list of files
to clean up and to pick up files missed in previous cleanup attempts

Greatly reduced the log output primarily by changing the verbosity
of executed commands.

If error occurs for any component, set run state to error_cleaned so
that we can track errors. May remove this at some point because this
blocks updates.

bzip the log and log.update files

Never invoke magicdstool.

Do not require that config dump file to exist if either quality or fault is nonzero

For chip stage run censorObjects -checkinputonly on associated smf file(s)
if a good one is found clean chip cmfs. Otherwise leave them.
clean bin1 images

For warp stage clean sources and skycell template. They are regenerated
during updates.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/ipp_cleanup.pl

    r33664 r34773  
    1818
    1919# Parse the command-line arguments
    20 my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile);
     20my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile, $check_all);
     21my $very_verbose = 0;
     22
     23# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
     24#my $ds_done = 0;
     25# magic is dead
     26my $ds_done = 1;
     27
    2128GetOptions('stage=s'        => \$stage,     # which analysis stage to clean?
    2229           'camera|i=s'     => \$camera,    # user-supplied camera name
    … …  
    2431           'mode|m=s'       => \$mode,      # cleanup mode (clean / purge)
    2532           'path_base=s'    => \$path_base, # basename for files
     33           'check-all'      => \$check_all, # if set clean all chips regardless of data_state
    2634           'dbname|d=s'     => \$dbname,    # Database name
    2735           'verbose'        => \$verbose,   # Print to stdout
    … …  
    5664$ipprc->redirect_output($logfile) or
    5765        &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile;
     66
     67
     68my $bzip2 = can_run('bzip2') or die 'cannot find bzip2\n';
    5869
    5970# set this to 1 to enable checking for files on dead nodes
    … …  
    91102    # this stage uses 'chiptool'
    92103    my $chiptool = can_run('chiptool') or die "Can't find chiptool";
     104    my $censorObjects = can_run('censorObjects') or die "Can't find censorObjects";
    93105
    94106    # Get list of component imfiles
    … …  
    96108    my $imfiles;                      # Array of component files
    97109    my $command = "$chiptool -pendingcleanupimfile -chip_id $stage_id"; # Command to run
     110    $command .= ' -all' if ($check_all);
    98111    $command .= " -dbname $dbname" if defined $dbname;
    99     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     112
     113    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf )
     114        = run(command => $command, verbose => $very_verbose);
    100115    unless ($success) {
    101116        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    114129        unless ($success) {
    115130            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    116             &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
     131            &my_die("Unable to perform chiptool -processedimfile: $error_code", "chip", $stage_id, $error_code);
    117132        }
    118133        exit 0;
    … …  
    123138        &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
    124139
     140    my $numchips = scalar @$imfiles;
     141    print "Found $numchips to clean\n";
     142
     143    my $clean_sources = 0;
     144    if ((scalar @$imfiles > 0) and ($mode eq 'goto_cleaned')) {
     145        # go and find the smf file(s) for the associated camRun and check the status of the file
     146        # if a good one is found we have the sources for this chipRun and thus can clean the cmfs
     147        my $command = "$chiptool -listrun -chip_id $stage_id";
     148        $command .= " -dbname $dbname" if defined $dbname;
     149        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose);
     150        unless ($success) {
     151            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     152            &my_die("Unable to perform chiptool -listrun: $error_code", "chip", $stage_id, $error_code);
     153        }
     154        my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or
     155            &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
     156        my $good_smf = 0;
     157        foreach my $entry (@$entries) {
     158            my $camRun_state = $entry->{camRun_state};
     159            next if $camRun_state  ne 'full';
     160            my $cam_id = $entry->{cam_id};
     161            if (!$cam_id) {
     162                carp('no cam_id for listrun entry');
     163                next;
     164            }
     165            my $cam_path_base = $entry->{cam_path_base};
     166            if ( !defined $cam_path_base ) {
     167                carp("no path_base for $cam_id\n");
     168                next;
     169            }
     170
     171            # XXX: This assumes that the filerules are filerules-split
     172            my $smf =  $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
     173            if (!$ipprc->file_exists($smf)) {
     174                carp("smf for $cam_path_base not found");
     175                next;
     176            }
     177            # we run the program censorObjects in the check mode
     178            # If this program succeeds the smf is a valid fits file and each of the
     179            # extensions was succesfully read.
     180            # XXX: create a new program outside of magic that performs this check
     181
     182            my $command = "$censorObjects -checkinputonly -file $smf";
     183            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     184                                        run(command => $command, verbose => $very_verbose);
     185            unless ($success) {
     186                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     187                print STDERR "censorObjects failed:\n";
     188                print STDERR "\nSTDOUT:\n" . join "", @$stdout_buf;
     189                print STDERR "\nSTDERR:\n" . join "", @$stderr_buf;
     190                &my_die("Unable to perform censorObjects -checkinputonly: $error_code", "chip", $stage_id, $error_code);
     191            }
     192            $good_smf++;
     193        }
     194        if ($good_smf) {
     195            # we have a good one so we can clean the sources
     196            $clean_sources = 1;
     197            print "Found $good_smf good smf files will clean sources\n";
     198        } else {
     199            print "Unable to find good smf file will NOT clean sources\n";
     200        }
     201    }
     202
    125203    # loop over all of the imfiles, determine the path_base and class_id for each
     204    my $num_errors = 0;
    126205    foreach my $imfile (@$imfiles) {
    127206        my $class_id = $imfile->{class_id};
    128207        my $path_base = $imfile->{path_base};
     208        my $data_state = $imfile->{data_state};
    129209        my $status = 1;
    130210        $status = 0 unless defined $path_base and $path_base ne "NULL";
    131211
    132         my $poor_quality = $imfile->{quality} > 0;
     212        my $quality = $imfile->{quality};
     213        my $good_quality = ($quality == 0);
     214
     215        print "Starting cleanup for $class_id\n";
    133216
    134217        # don't clean up unless the data needed to update is available
    135218        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
    136219        # goto_scrubbed now requires the config file to not exist.
    137         if ($status and !$poor_quality) {
     220        if ($status and $good_quality) {
    138221            if ($mode eq "goto_cleaned") {
    139222                my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    140223
    141224                unless ($ipprc->file_exists($config_file)) {
     225                    my $fault = $imfile->{fault};
     226
    142227                    if (file_gone($config_file)) {
    143                         print STDERR "forcing cleanup for chipRun $stage_id $class_id "
     228                        # config file was lost. Clean up. If the chip is ever updated a new config
     229                        # file will be created
     230                        print STDERR "forcing cleanup chip $stage_id $class_id fault: $fault quality: $quality"
    144231                            . " because config file ($config_file) is gone\n";
     232                    } elsif ($fault == 0 and $quality == 0) {
     233                            print STDERR "skipping cleaning up chip $stage_id $class_id fault: $fault quality: $quality"
     234                                . " because config file ($config_file) is missing\n";
     235                            $status = 0;
    145236                    } else {
    146                         print STDERR "skipping cleanup for chipRun $stage_id $class_id "
    147                             . " because config file ($config_file) is missing\n";
    148                         $status = 0;
     237                            # config file is missing but this is a bad chip anyways so clean it
     238                            print STDERR "cleaning up chip $stage_id $class_id fault: $fault quality: $quality"
     239                                . " even though config file ($config_file) is missing\n";
    149240                    }
    150241                }
    … …  
    154245
    155246                if ($ipprc->file_exists($config_file)) {
    156                     print STDERR "skipping scrubbed for chipRun $stage_id $class_id "
     247                    print STDERR "skipping scrubbed for chip $stage_id $class_id "
    157248                        . " because config file ($config_file) is present\n";
    158249                    $status = 0;
    … …  
    165256            my @files = ();
    166257
    167             # delete the temporary image datafiles
    168 #            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
    169 #            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
    170 #            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
    171258            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1);
    172259            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1);
    173260            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1);
    174261            addFilename (\@files, "PPIMAGE.PATTERN", $path_base, $class_id, 0);
     262            if ($clean_sources) {
     263                addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id);
     264                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     265            }
    175266            if ($mode eq "goto_purged") {
    176267                # additional files to remove for 'purge' mode
     268                if (!$clean_sources) {
     269                    # these weren't added above but we do want to clean it
     270                    addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id);
     271                    addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     272                }
     273               
     274                # background model is needed to build stack background images so we do not remove it
     275                # addFilename (\@files, "PSPHOT.BACKMDL", $path_base, $class_id);
     276
     277                addFilename (\@files, "PSPHOT.PSF.SAVE", $path_base, $class_id);
    177278                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
    178279                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
    179                 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);          # clean?
    180                 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);          # clean?
     280                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
    181281                addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
    182282                addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id);
    183                 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);         #clean?
     283                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
    184284                addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
    185285            }
    … …  
    188288            $status = &delete_files (\@files);
    189289        }
     290        bzip2_file("LOG.IMFILE", $path_base, $class_id);
     291        bzip2_file("LOG.IMFILE.UPDATE", $path_base, $class_id);
    190292
    191293        if ($status)  {
     294            my $update_chip = 1;
    192295            my $command = "$chiptool -chip_id $stage_id -class_id $class_id";
    193296            if ($mode eq "goto_purged") {
    194297                $command .= " -topurgedimfile";
     298                if ($data_state eq 'purged') {
     299                    $update_chip = 0;
     300                }
    195301            }
    196302            elsif ($mode eq "goto_cleaned") {
    197303                $command .= " -tocleanedimfile";
     304                if ($data_state eq 'cleaned') {
     305                    $update_chip = 0;
     306                }
    198307            }
    199308            elsif ($mode eq "goto_scrubbed") {
    200309                $command .= " -toscrubbedimfile";
    201             }
    202 
    203             $command .= " -dbname $dbname" if defined $dbname;
    204 
    205             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    206                     run(command => $command, verbose => $verbose);
    207             unless ($success) {
    208                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    209                 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    210             }
    211 
    212             set_destreak_goto_cleaned();
    213 
     310                if ($data_state eq 'scrubbed') {
     311                    $update_chip = 0;
     312                }
     313            }
     314
     315            if ($update_chip) {
     316                $command .= " -dbname $dbname" if defined $dbname;
     317
     318                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     319                        run(command => $command, verbose => $verbose);
     320                unless ($success) {
     321                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     322                    &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
     323                }
     324                set_destreak_goto_cleaned();
     325            }
    214326        } else {
     327            $num_errors++;
    215328
    216329            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
    … …  
    218331            $command .= " -dbname $dbname" if defined $dbname;
    219332
    220 if (0) {
    221         # XXX Don't set components to error cleaned anymore
    222333            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    223334                    run(command => $command, verbose => $verbose);
    … …  
    226337                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    227338            }
    228 }
    229339
    230340            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
    … …  
    241351        }
    242352    }
     353    print "Cleanup completed for chip_id $stage_id.";
     354    print " num_errors: $num_errors" if $num_errors;
     355    print "\n";
    243356    exit 0;
    244357}
    … …  
    350463    my $skyfiles;                      # Array of component files
    351464    my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run
     465    $command .= ' -all' if $check_all;
    352466    $command .= " -dbname $dbname" if defined $dbname;
    353467    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    354             run(command => $command, verbose => $verbose);
     468            run(command => $command, verbose => $very_verbose);
    355469    unless ($success) {
    356470        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    378492        &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR);
    379493
     494    my $numskycells = scalar @$skyfiles;
     495    print "Found $numskycells to clean\n";
     496
    380497    my @files = ();
     498    my $num_errors = 0;
    381499    foreach my $skyfile (@$skyfiles) {
    382500        my $path_base = $skyfile->{path_base};
    383501        my $skycell_id = $skyfile->{skycell_id};
     502        my $data_state = $skyfile->{data_state};
    384503
    385504        my $status = 1;
    … …  
    391510
    392511                unless ($ipprc->file_exists($config_file)) {
     512                    my $fault = $skyfile->{fault};
     513                    my $quality = $skyfile->{quality};
    393514                    if (file_gone($config_file)) {
    394                         print STDERR "forcing cleanup for warpRun $stage_id $skycell_id" .
    395                             " because config file is gone\n";
     515                        print STDERR "forcing cleanup warp $stage_id $skycell_id fault: $fault quality: $quality"
     516                            . " because config file ($config_file) is gone\n";
     517                    } elsif ($fault == 0 and $quality == 0) {
     518                            print STDERR "skipping cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality"
     519                                . " because config file ($config_file) is missing\n";
     520                            $status = 0;
    396521                    } else {
    397                         print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
    398                             " because config file is missing\n";
    399                         $status = 0;
     522                            # config file is missing but this is a bad warp anyways so clean it
     523                            print STDERR "cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality"
     524                                . " even though config file ($config_file) is missing\n";
    400525                    }
    401526                }
    … …  
    413538
    414539        if ($status) {
    415             if ($skyfile->{quality} != 8007) {
     540            # XXX: what is special about quality == 8007?
     541            if ($skyfile->{quality} != 8007 || $check_all) {
    416542                my @files = ();
    417543
    … …  
    420546                addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1);
    421547                addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
    422 #            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
     548                # these are rebuilt during update so we can delete them here
     549                addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
     550                addFilename(\@files, "SKYCELL.TEMPLATE", $path_base, $skycell_id );
    423551                if ($mode eq "goto_purged") {
    424552                    # additional files to remove for 'purge' mode
    … …  
    426554                    addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id );
    427555                    addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id );
    428                     # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base);
    429 
    430                     # XXX: do we want to delete these?
     556                    addFilename(\@files, "SKYCELL.STATS.UPDATE", $path_base, $skycell_id );
     557                    addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
     558
     559                    # XXX: do we want to delete these? trace file is empty
    431560                    # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id);
    432                     # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
    433                 }
    434             # actual command to delete the files
     561                }
     562                # actual command to delete the files
    435563                $status = &delete_files (\@files);
    436564            }
    437565        }
     566        bzip2_file("LOG.EXP", $path_base, $skycell_id);
     567        bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id);
    438568
    439569        if ($status)  {
     570            my $update_skyfile = 1;
    440571            my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
    441572            if ($mode eq "goto_purged") {
    442573                $command .= " -topurgedskyfile";
     574                if ($data_state eq 'purged') {
     575                    $update_skyfile = 0;
     576                }
    443577            }
    444578            elsif ($mode eq "goto_cleaned") {
    445579                $command .= " -tocleanedskyfile";
     580                if ($data_state eq 'cleaned') {
     581                    $update_skyfile = 0;
     582                }
    446583            }
    447584            elsif ($mode eq "goto_scrubbed") {
    448585                $command .= " -toscrubbedskyfile";
    449             }
    450             $command .= " -dbname $dbname" if defined $dbname;
    451 
    452             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    453                     run(command => $command, verbose => $verbose);
     586                if ($data_state eq 'scrubbed') {
     587                    $update_skyfile = 0;
     588                }
     589            }
     590            $command .= " -dbname $dbname" if defined $dbname;
     591
     592            if ($update_skyfile) {
     593                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     594                        run(command => $command, verbose => $verbose);
     595                unless ($success) {
     596                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     597                    &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
     598                }
     599
     600                set_destreak_goto_cleaned();
     601            }
     602
     603         } else {
     604            $num_errors++;
     605            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
     606            $command .= " -dbname $dbname" if defined $dbname;
     607
     608            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     609                run(command => $command, verbose => $verbose);
    454610            unless ($success) {
    455611                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    457613            }
    458614
    459             set_destreak_goto_cleaned();
    460 
    461          } else {
    462             my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
    463             $command .= " -dbname $dbname" if defined $dbname;
    464 
    465             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     615            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
     616            $command = "$warptool -warp_id $stage_id -updaterun -set_state $error_state";
     617            $command .= " -dbname $dbname" if defined $dbname;
     618
     619            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    466620                run(command => $command, verbose => $verbose);
    467621            unless ($success) {
    … …  
    469623                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
    470624            }
    471 
    472             #            exit $PS_EXIT_UNKNOWN_ERROR;
    473         }
    474     }
     625        }
     626    }
     627    print "Cleanup completed for warp_id $stage_id.";
     628    print " num_errors: $num_errors" if $num_errors;
     629    print "\n";
    475630    exit 0;
    476631}
    … …  
    603758                &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
    604759            }
    605 #           exit $PS_EXIT_UNKNOWN_ERROR;
    606760        }
    607761    }
    … …  
    619773    my $skyfiles;                  # Array reference of component files
    620774    my $command = "difftool -pendingcleanupskyfile -diff_id $stage_id"; # Command to run
     775    $command .= ' -all' if $check_all;
    621776    $command .= " -dbname $dbname" if defined $dbname;
    622     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     777    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose);
    623778    unless ($success) {
    624779        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    646801        &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR);
    647802
     803    my $num_errors = 0;
    648804    my @files = ();
    649805    foreach my $skyfile (@{ $skyfiles }) {
    650806        my $path_base = $skyfile->{path_base};
    651807        my $skycell_id = $skyfile->{skycell_id};
     808        my $data_state = $skyfile->{data_state};
    652809
    653810        my $status = 1;
    … …  
    661818                my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
    662819
     820            if (0) {
    663821                unless ($ipprc->file_exists($config_file)) {
    664822                    if (file_gone($config_file)) {
    … …  
    672830                }
    673831            }
     832                unless ($ipprc->file_exists($config_file)) {
     833                    my $fault = $skyfile->{fault};
     834                    my $quality = $skyfile->{quality};
     835                    if (file_gone($config_file)) {
     836                        print STDERR "forcing cleanup diff $stage_id $skycell_id fault: $fault quality: $quality"
     837                            . " because config file ($config_file) is gone\n";
     838                    } elsif ($fault == 0 and $quality == 0) {
     839                            print STDERR "skipping cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality"
     840                                . " because config file ($config_file) is missing\n";
     841                            $status = 0;
     842                    } else {
     843                            # config file is missing but this is a bad diff anyways so clean it
     844                            print STDERR "cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality"
     845                                . " even though config file ($config_file) is missing\n";
     846                    }
     847                }
     848            }
    674849            elsif ($mode eq "goto_scrubbed") {
    675850                my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
    … …  
    712887
    713888            }
    714 #           print STDERR "MY FILES: @files\n";
    715889            $status = &delete_files(\@files);
    716890        }
    717 #       print STDERR "MY STATUS: $status\n";
     891
     892        bzip2_file("LOG.EXP", $path_base, $skycell_id);
     893        bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id);
     894
    718895        if ($status) {
    719896            my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id";
     897            my $update_skyfile = 1;
    720898
    721899            if ($mode eq "goto_purged") {
    722900                $command .= " -topurgedskyfile";
     901                if ($data_state eq 'purged') {
     902                    $update_skyfile = 0;
     903                }
    723904            }
    724905            elsif ($mode eq "goto_cleaned") {
    725906                $command .= " -tocleanedskyfile";
     907                if ($data_state eq 'cleaned') {
     908                    $update_skyfile = 0;
     909                }
    726910            }
    727911            elsif ($mode eq "goto_scrubbed") {
    728912                $command .= " -toscrubbedskyfile";
    729             }
    730 
     913                if ($data_state eq 'scrubbed') {
     914                    $update_skyfile = 0;
     915                }
     916            }
     917
     918            $command .= " -dbname $dbname" if defined $dbname;
     919
     920            if ($update_skyfile) {
     921                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     922                    run(command => $command, verbose => $verbose);
     923                unless ($success) {
     924                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     925                    &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
     926                }
     927                set_destreak_goto_cleaned();
     928            }
     929
     930
     931        } else {
     932            $num_errors++;
     933            my $command = "$difftool -updatediffskyfile -diff_id $stage_id -skycell_id $skycell_id -set_state $error_state";
    731934            $command .= " -dbname $dbname" if defined $dbname;
    732935
    … …  
    738941            }
    739942
    740             set_destreak_goto_cleaned();
    741 
    742         } else {
    743             my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
    744 
    745             $command .= " -dbname $dbname" if defined $dbname;
    746 
    747             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     943            $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
     944
     945            $command .= " -dbname $dbname" if defined $dbname;
     946
     947            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    748948                run(command => $command, verbose => $verbose);
    749949            unless ($success) {
    … …  
    751951                &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
    752952            }
    753 #           exit $PS_EXIT_UNKNOWN_ERROR;
    754         }
    755     }
     953        }
     954    }
     955    print "Cleanup completed for diff_id $stage_id.";
     956    print " num_errors: $num_errors" if $num_errors;
     957    print "\n";
    756958    exit 0;
    757959}
    … …  
    19172119
    19182120    foreach my $file (@$files) {
    1919         print STDERR "unlinking $stage $stage_id $file\n";
     2121        print STDERR "unlinking $stage $stage_id $file\n" if $very_verbose;
    19202122
    19212123        my $error_code = $ipprc->kill_file($file);
    … …  
    20082210}
    20092211
    2010 # this gets set to 1 the first time we set the corresponding destreak run to be cleaned
    2011 my $ds_done = 0;
    20122212sub set_destreak_goto_cleaned {
    20132213
    … …  
    20272227}
    20282228
     2229sub bzip2_file {
     2230    my $filerule = shift;
     2231    my $path_base = shift;
     2232    my $component = shift;
     2233
     2234    my $filename = $ipprc->filename($filerule, $path_base, $component);
     2235    if (!$ipprc->file_exists($filename)) {
     2236        return 1;
     2237    }
     2238    if (my $resolved = $ipprc->file_resolve($filename)) {
     2239        my $bzip2_filename = $filename . '.bz2';
     2240        if ($ipprc->file_exists($bzip2_filename)) {
     2241            $ipprc->kill_file($bzip2_filename);
     2242        }
     2243        my $bzip2_file = $ipprc->file_create($bzip2_filename);
     2244        my_die("Unable to create $bzip2_filename", $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file;
     2245
     2246        my $command = "$bzip2 < $resolved > $bzip2_file";
     2247        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     2248            run(command => $command, verbose => $very_verbose);
     2249        if ($success) {
     2250            # success delete the original file
     2251            my $error_code = $ipprc->kill_file($filename);
     2252        } else {
     2253            # if bzip2 failed. Carry on but don't delete the existing file
     2254            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     2255            print STDERR "Failed to bzip2 $filename: $error_code\n";
     2256            return 0;
     2257        }
     2258    }
     2259    return 1;
     2260}
     2261
    20292262# XXX we currently do not set the error state in the db on my_die
    20302263sub my_die
Note: See TracChangeset for help on using the changeset viewer.