IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jul 2, 2009, 1:24:16 PM (17 years ago)
Author:
watersc1
Message:

Added cleanup routines for diff and stack, and added a message for fake.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/ipp_cleanup.pl

    r24200 r24642  
    6060
    6161
    62 my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff  => 1);
     62my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff  => 1,
     63               detrend.process.imfile => 1, detrend.process.exp => 1, detrend.stack.imfile => 1,
     64               detrend.normstat.imfile => 1, detrend.norm.imfile => 1, detrend.norm.exp => 1,
     65               detrend.resid.imfile => 1, detrend.resid.exp => 1 );
    6366unless ($stages{$stage}) {
    6467    die "unknown stage $stage for ipp_cleanup.pl\n";
    … …  
    118121        if ($mode eq "goto_cleaned") {
    119122            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    120 
     123            print STDERR "CHIP: CONFIG_FILE : $config_file\n";
    121124            if (!$config_file or ! -e $config_file) {
    122125                print STDERR "skipping cleanup for chipRun $stage_id $class_id "
    … …  
    361364}
    362365
    363 # left TODO
     366### added for cleanup, based on warp stage entry
     367if ($stage eq 'stack') {
     368   
     369    die "--stage_id required for stage stack\n" if !$stage_id;
     370
     371    # this stage uses 'stacktool'
     372    my $stacktool = can_run('stacktool') or die "Can't find stacktool";
     373
     374    # Get list of component imfiles
     375    my $skyfiles;                  # Array reference of component files
     376    my $command = "stacktool -pendingcleanupskyfile -stack_id $stage_id"; # Command to run
     377    $command .= " -dbname $dbname" if defined $dbname;
     378    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     379    unless ($success) {
     380        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     381        &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
     382    }
     383    my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or
     384        &my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR);
     385
     386    $skyfiles = parse_md_list($metadata) or
     387        &my_die("Unable to parse metadata list", "stack", $stage_id, $PS_EXIT_PROG_ERROR);
     388   
     389    my @files = ();
     390    foreach my $skyfile (@{ $skyfiles }) {
     391        my $path_base = $skyfile->{path_base};
     392        my $skycell_id = $skyfile->{skycell_id};
     393
     394        my $status = 1;
     395        if ($mode eq "goto_cleaned") {
     396            my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id);
     397            print STDERR "MY CONFIG FILE = $config_file\n";
     398            printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file);
     399            $config_file =~ s%^file://%%;
     400            if (!$config_file or ! -e $config_file) {
     401                print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" .
     402                    " because config file is missing\n";
     403                $status = 0;
     404            }
     405            $config_file = 'file://' . $config_file;
     406        }
     407        if ($status) {
     408            # delete the temporary image datafiles
     409            addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id);
     410            addFilename(\@files, "PPSTACK.OUTPUT.MASK", $path_base, $skycell_id);
     411            addFilename(\@files, "PPSTACK.OUTPUT.VARIANCE", $path_base, $skycell_id);
     412
     413            if ($mode eq "goto_purged") {
     414                # additional files to remove for 'purge' mode
     415                addFilename(\@files, "PPSTACK.CONV.KERNEL", $path_base, $skycell_id);
     416                addFilename(\@files, "PPSTACK.OUTPUT.JPEG1", $path_base, $skycell_id);
     417                addFilename(\@files, "PPSTACK.OUTPUT.JPEG2", $path_base, $skycell_id);
     418                # Commented out to match warp files.
     419                #addFilename(\@files, "PPSTACK.TARGET.PSF", $path_base, $skycell_id);
     420                #addFilename(\@files, "PPSTACK.CONFIG", $path_base, $skycell_id);
     421            }
     422
     423            $status = &delete_files(\@files);
     424        }
     425
     426        if ($status) {
     427            my $command = "$stacktool -stack_id $stage_id";
     428            if ($mode eq "goto_purged") {
     429                $command .= " -updaterun -state purged";
     430            } else {
     431                $command .= " -updaterun -state cleaned";
     432            }
     433            $command .= " -dbname $dbname" if defined $dbname;
     434           
     435            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     436                run(command => $command, verbose => $verbose);
     437            unless ($success) {
     438                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     439                &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
     440            }
     441        } else {
     442            my $command = "$stacktool -updaterun  -stack_id $stage_id -state $error_state";
     443            $command .= " -dbname $dbname" if defined $dbname;
     444           
     445            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     446                run(command => $command, verbose => $verbose);
     447            unless ($success) {
     448                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     449                &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
     450            }
     451            exit $PS_EXIT_UNKNOWN_ERROR;
     452        }
     453    }
     454    exit 0;
     455}
     456
     457if ($stage eq 'diff') {
     458   
     459    die "--stage_id required for stage diff\n" if !$stage_id;
     460
     461    # this stage uses 'difftool'
     462    my $difftool = can_run('difftool') or die "Can't find difftool";
     463
     464    # Get list of component imfiles
     465    my $skyfiles;                  # Array reference of component files
     466    my $command = "difftool -pendingcleanupskyfile -diff_id $stage_id"; # Command to run
     467    $command .= " -dbname $dbname" if defined $dbname;
     468    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     469    unless ($success) {
     470        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     471        &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
     472    }
     473    my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or
     474        &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR);
     475
     476    $skyfiles = parse_md_list($metadata) or
     477        &my_die("Unable to parse metadata list", "diff", $stage_id, $PS_EXIT_PROG_ERROR);
     478   
     479    my @files = ();
     480    foreach my $skyfile (@{ $skyfiles }) {
     481        my $path_base = $skyfile->{path_base};
     482        my $skycell_id = $skyfile->{skycell_id};
     483
     484        my $status = 1;
     485        if ($mode eq "goto_cleaned") {
     486            my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
     487            print STDERR "MY CONFIG FILE = $config_file\n";
     488            printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file);
     489            $config_file =~ s%^file://%%;
     490            if (!$config_file or ! -e $config_file) {
     491                print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" .
     492                    " because config file ($config_file) is missing\n";
     493                $status = 0;
     494            }
     495            $config_file = 'file://' . $config_file;
     496        }
     497        if ($status) {
     498            # delete the temporary image datafiles
     499            addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id);
     500            addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id);
     501            addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id);
     502
     503            addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id);
     504            addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id);
     505            addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id);
     506            addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id);
     507            addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id);
     508            addFilename(\@files, "PPSUB.INPUT.CONV.VARIANCE", $path_base, $skycell_id);
     509           
     510            addFilename(\@files, "PPSUB.REF.CONV", $path_base, $skycell_id);
     511            addFilename(\@files, "PPSUB.REF.CONV.MASK", $path_base, $skycell_id);
     512            addFilename(\@files, "PPSUB.REF.CONV.VARIANCE", $path_base, $skycell_id);
     513           
     514            if ($mode eq "goto_purged") {
     515                # additional files to remove for 'purge' mode
     516                addFilename(\@files, "PPSUB.OUTPUT.KERNELS", $path_base, $skycell_id);
     517                addFilename(\@files, "PPSUB.OUTPUT.JPEG1", $path_base, $skycell_id);
     518                addFilename(\@files, "PPSUB.OUTPUT.JPEG2", $path_base, $skycell_id);
     519                # Commented out to match warp files.
     520                #addFilename(\@files, "PPSUB.CONFIG", $path_base, $skycell_id);
     521                addFilename(\@files, "PPSUB.OUTPUT.SOURCES", $path_base, $skycell_id);
     522                addFilename(\@files, "PPSUB.INVERSE.SOURCES", $path_base, $skycell_id);
     523               
     524            }
     525            print STDERR "MY FILES: @files\n";
     526            $status = &delete_files(\@files);
     527        }
     528        print STDERR "MY STATUS: $status\n";
     529        if ($status) {
     530            my $command = "$difftool -diff_id $stage_id";
     531#           my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id";
     532            if ($mode eq "goto_purged") {
     533                $command .= " -updaterun -state purged";
     534            } else {
     535                $command .= " -updaterun -state cleaned";
     536            }
     537            $command .= " -dbname $dbname" if defined $dbname;
     538           
     539            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     540                run(command => $command, verbose => $verbose);
     541            unless ($success) {
     542                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     543                &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
     544            }
     545        } else {
     546            my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state";
     547#           my $command = "$difftool -updaterun -diff_id $stage_id -skycell_id $skycell_id -state $error_state";
     548            $command .= " -dbname $dbname" if defined $dbname;
     549           
     550            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     551                run(command => $command, verbose => $verbose);
     552            unless ($success) {
     553                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     554                &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
     555            }
     556            exit $PS_EXIT_UNKNOWN_ERROR;
     557        }
     558    }
     559    exit 0;
     560}
     561if ($stage eq 'fake') {
     562    die "ipp_cleanup.pl -stage fake not yet implemented. Probably will just mark database cleaned.\n";
     563}
    364564# fake : faketool : -pendingcleanupimfile (loop over imfiles)
    365 # stack: stacktool : -pendingcleanupskyfile
    366 # diff:  difftool : -pendingcleanupskyfile (loop over skyfiles)
    367565
    368566die "ipp_cleanup.pl -stage $stage not yet implemented\n";
Note: See TracChangeset for help on using the changeset viewer.