IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 26721


Ignore:
Timestamp:
Jan 29, 2010, 10:51:08 AM (16 years ago)
Author:
eugene
Message:

various cleanup and automation operations

Location:
branches/eam_branches/20091201/ippScripts
Files:
2 added
6 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/20091201/ippScripts/Build.PL

    r26290 r26721  
    9898        scripts/rcserver_checkstatus.pl
    9999        scripts/publish_file.pl
     100        scripts/dqstats_bundle.pl
    100101        scripts/whichimage
     102        scripts/automate_stacks.pl
    101103    )],
    102104    dist_abstract => 'Scripts for running the Pan-STARRS IPP',

  • branches/eam_branches/20091201/ippScripts/MANIFEST

    r18564 r26721  
    3636scripts/isp_trans.pl
    3737scripts/ds9_cmf_regions.pl
     38scripts/automate_stacks.pl
    3839t/00_distribution.t

  • branches/eam_branches/20091201/ippScripts/scripts/ipp_cleanup.pl

    r25837 r26721  
    190190            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    191191            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    192             $command = "$magicdstool -dbname gpc1  -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     192            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     193            $command .= " -dbname $dbname" if defined $dbname;
    193194            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    194195                run(command => $command, verbose => $verbose);
    … …  
    309310        # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    310311        my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    311         $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     312        $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     313        $command .= " -dbname $dbname" if defined $dbname;
    312314        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    313315            run(command => $command, verbose => $verbose);
    … …  
    444446            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    445447            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    446             $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     448            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     449            $command .= " -dbname $dbname" if defined $dbname;
    447450            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    448451                run(command => $command, verbose => $verbose);
    … …  
    580583            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    581584            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    582             $command = "$magicdstool -dbname gpc1  -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     585            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     586            $command .= " -dbname $dbname" if defined $dbname;
    583587            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    584588                run(command => $command, verbose => $verbose);
    … …  
    732736            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    733737            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    734             $command = "$magicdstool -dbname gpc1 -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     738            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned";
     739            $command .= " -dbname $dbname" if defined $dbname;
    735740            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    736741                run(command => $command, verbose => $verbose);

  • branches/eam_branches/20091201/ippScripts/scripts/magic_destreak.pl

    r25029 r26721  
    1515
    1616use IPC::Cmd 0.36 qw( can_run run );
    17 use File::Temp qw( tempfile );
     17use File::Temp qw( tempfile tempdir );
    1818use File::Basename qw( basename dirname );
    1919use PS::IPP::Metadata::Config;
    … …  
    3232my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
    3333my $censorObjects = can_run('censorObjects') or (warn "Can't find censorObjects" and $missing_tools = 1);
     34my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
    3435if ($missing_tools) {
    3536    warn("Can't find required tools.");
    … …  
    3839
    3940# Parse the command-line arguments
    40 my ($magic_ds_id, $camera, $streaks, $inv_streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
     41my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
    4142my ($outroot, $recoveryroot);
    4243my ($replace, $release);
    … …  
    4849           'streaks=s'      => \$streaks,    # file containing the list of streaks
    4950           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
     51           'exp_id=s'       => \$exp_id,     # exp_id, chip_id, warp_id, or diff_id
    5052           'stage=s'        => \$stage,      # raw, chip, warp, or diff
    5153           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
    … …  
    7476    defined $stage and
    7577    defined $stage_id and
     78    defined $exp_id and
    7679    defined $component and
    7780    defined $uri and
    … …  
    145148    }
    146149}
     150
     151my $statsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.stats";
    147152
    148153my $backup_path_base;
    … …  
    166171        if (! -e $recoveryroot ) {
    167172            my $code = system "mkdir -p $recoveryroot";
    168             &my_die("cannot create output directory $outroot", $magic_ds_id, $component,
     173            &my_die("cannot create output directory $recoveryroot", $magic_ds_id, $component,
    169174                    $code >> 8) if $code;
    170175        }
    … …  
    173178    $recovery_path_base = "$recoveryroot/$basename/";
    174179}
     180
     181my $temp_dir;
    175182
    176183if ($stage ne "camera") {
    … …  
    195202                    &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
    196203
     204            $temp_dir = tempdir( CLEANUP => !$save_temps);
    197205            ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
    198206
    199207            foreach my $skycell (@$skycells) {
    200                 my $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
     208                my $skycell_uri;
     209                if ($skycell->{data_state} eq "full") {
     210                    $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
     211                } else {
     212                    # diff run must have been cleaned up, need to create this skycell file on the fly
     213                    my $skycell_id = $skycell->{skycell_id};
     214                    $skycell_uri = "$temp_dir/$skycell_id";
     215                    $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
     216                        &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
     217                }
    201218                print $sfh "$skycell_uri\n";
    202219            }
    203             close $sfh;
     220            close $sfh
    204221        }
    205222    }
    … …  
    260277        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
    261278
     279        $command .= " -stats $statsFile";
    262280        $command .= " -class_id $class_id" if defined $class_id;
    263281        $command .= " -recovery $recoveryroot" if defined $recoveryroot;
    … …  
    288306        $sources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $path_base);
    289307
     308        # Note: we create a stats file for the inverse procesing but we don't look at the results
     309        my $invStatsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.inv.stats";
     310
    290311        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
     312        $command .= " -stats $invStatsFile";
    291313
    292314        $command .= " -recovery $recoveryroot" if defined $recoveryroot;
    … …  
    371393}
    372394
    373 # if recovery and/or backup files were expected make sure they exist
     395my $statsFlags;
     396if ($stage ne "camera") {
     397    file_check($statsFile);
     398    {
     399        my $resolvedStatsFile = $ipprc->file_resolve($statsFile);
     400        my $command = "$ppStatsFromMetadata $resolvedStatsFile - STREAKSREMOVE";
     401
     402        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     403            run(command => $command, verbose => $verbose);
     404        unless ($success) {
     405            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     406            &my_die("Unable to perform ppStatsFromMetadata: $error_code", $magic_ds_id, $component, $error_code);
     407        }
     408        foreach my $line (@$stdout_buf) {
     409            $statsFlags .= " $line";
     410        }
     411        chomp $statsFlags;
     412    }
     413}
     414
     415# XXX: if recovery and/or backup files were expected make sure they exist
    374416
    375417# Input result into database
    … …  
    381423    $command   .= " -backup_path_base $backup_path_base" if $backup_path_base;
    382424    $command   .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
     425    $command   .= " $statsFlags" if $statsFlags;
    383426    $command   .= " -dbname $dbname" if defined $dbname;
    384427

  • branches/eam_branches/20091201/ippScripts/scripts/magic_destreak_cleanup.pl

    r25837 r26721  
    183183        }
    184184
    185         delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
     185        delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
    186186
    187187        if ($stage eq "diff" and $inverse) {
    … …  
    199199                $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base);
    200200            }
    201             delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources, undef);
     201            # undef is not necessary, it's just to keep the lists in the same order.
     202            delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources);
    202203        }
    203204}
    … …  
    211212if (!$no_update) {
    212213    my $result = $dbh->do("UPDATE magicDSRun SET state = 'cleaned' WHERE magic_ds_id = ?", undef, $magic_ds_id);
    213     my_die("attempt to update magicDSRun failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0";
     214    my_die("attempt to update magicDSRun.state failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0";
    214215} else {
    215216    print STDERR "skipping update of magicDSRun\n";
    … …  
    246247    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
    247248
    248     my $command = "$magicdstool -updaterun -fault $exit_code";
     249    my $command = "$magicdstool -updaterun -set_state failed_cleanup";
    249250    $command   .= " -magic_ds_id $magic_ds_id";
    250251    $command   .= " -dbname $dbname" if defined $dbname;

  • branches/eam_branches/20091201/ippScripts/scripts/magic_destreak_revert.pl

    r26290 r26721  
    294294        run(command => $command, verbose => $verbose);
    295295
     296    if (!defined $error_code) {
     297        print STDERR "run( $command ) returned undef error_code!!!!!\n";
     298        return $PS_EXIT_UNKNOWN_ERROR;
     299    }
    296300    return $error_code >> 8;
    297301}
    … …  
    392396    my $command = "$magicdstool -updaterun";
    393397    $command   .= " -magic_ds_id $magic_ds_id";
    394     $command   .= " -fault $exit_code";
     398    $command   .= " -set_state failed_revert";
    395399    $command   .= " -dbname $dbname" if defined $dbname;
    396400
Note: See TracChangeset for help on using the changeset viewer.