IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 30635


Ignore:
Timestamp:
Feb 14, 2011, 2:21:59 PM (15 years ago)
Author:
bills
Message:

Add new function to PS-IPP-Config kill_file() which moves nebulous files to the trash
before deleting, making the nebulous key available to be re-created

Location:
tags/ipp-20101215
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • tags/ipp-20101215/PS-IPP-Config

  • tags/ipp-20101215/PS-IPP-Config/lib/PS/IPP/Config.pm

    r30321 r30635  
    11481148    if ($neb->storage_object_exists($output)) {
    11491149        if ($delete_existing) {
    1150             # avoid dead instances by moving the file before deleting it.
    1151             # append current time to form new name
    1152             my $todelete;
    1153             eval {
    1154                 # parse the key so that we can compute the new name
    1155                 require Nebulous::Key;
    1156             };
    1157             if ($@) {
    1158                 carp "Can't find Nebulous::Key";
    1159                 $$r_error = $PS_EXIT_CONFIG_ERROR;
    1160                 return undef;
    1161             }
    1162             eval {
    1163                 # parse the key so that we can compute the new name
    1164                 my $neb_key = Nebulous::Key::parse_neb_key($output);
    1165                 die "parse_neb_key failed" if !$neb_key;
    1166                 my $path = $neb_key->path;
    1167                 die "neb_key has no path" if !$path;
    1168                 my $ticks = time();
    1169                 $todelete = "ipp_trash/$path.$ticks";
    1170                 $neb->move($output, $todelete);
    1171             };
    1172             if ($@) {
    1173                 carp "nebulous move failed for $output";
     1150            $$r_error = $self->_kill_nebulous_file($output);
     1151            if ($$r_error) {
    11741152                $output = undef;
    1175                 $$r_error = $PS_EXIT_SYS_ERROR;
    1176             }
    1177             if ($todelete) {
    1178                 eval {
    1179                     $neb->delete($todelete);
    1180                 };
    1181                 if ($@) {
    1182                     carp "nebulous delete for $todelete failed. Ignoring.\n";
    1183                     $$r_error = $PS_EXIT_SYS_ERROR;
    1184                 }
    11851153            }
    11861154        } else {
    11871155            # Make sure that there is only 1 instance.
    1188 
    11891156            eval {
    11901157                $neb->there_can_be_only_one($output);
    … …  
    11981165    }
    11991166    return $output;
     1167}
     1168
     1169# _kill_nebulous_file: reliably get a nebulous file out of way.
     1170# First move it to the trash then attempt to delete it. No failure if delete fails.
     1171# The file is in the trash.
     1172# Assumes that file is a nebulous file with a storage object that exists
     1173# Users should call kill_file()
     1174# Returns 0 on success otherwise and a PS_EXIT error code on failure
     1175sub _kill_nebulous_file {
     1176    my $self = shift;
     1177    my $filename = shift;
     1178    my $neb = $self->nebulous;
     1179
     1180    # avoid dead instances by moving the file before deleting it.
     1181    # append current time to form new name
     1182    my $todelete;
     1183    eval {
     1184        # parse the key so that we can compute the new name
     1185        require Nebulous::Key;
     1186    };
     1187    if ($@) {
     1188        carp "Can't find Nebulous::Key";
     1189        return $PS_EXIT_CONFIG_ERROR;
     1190    }
     1191    eval {
     1192        # parse the key so that we can compute the new name
     1193        my $neb_key = Nebulous::Key::parse_neb_key($filename);
     1194        die "parse_neb_key failed" if !$neb_key;
     1195        my $path = $neb_key->path;
     1196        die "neb_key has no path" if !$path;
     1197        my $ticks = time();
     1198        $todelete = "ipp_trash/$path.$ticks";
     1199        $neb->move($filename, $todelete);
     1200    };
     1201    if ($@) {
     1202        carp "nebulous move failed for $filename";
     1203        return $PS_EXIT_SYS_ERROR;
     1204    }
     1205    if ($todelete) {
     1206        eval {
     1207            $neb->delete($todelete);
     1208        };
     1209        if ($@) {
     1210            carp "nebulous delete for $todelete failed. Ignoring.\n";
     1211            return $PS_EXIT_SYS_ERROR;
     1212        }
     1213    }
     1214    return 0;
     1215}
     1216
     1217# user level interface to kill_file reliably get a file out of way.
     1218#
     1219# If a nebulous file, move it to the trash then attempt to delete it.
     1220# if non nebulous file just delete it.
     1221# Returns 0 on success otherwise and a PS_EXIT error code on failure
     1222
     1223sub kill_file {
     1224    my $self = shift;
     1225    my $filename = shift;
     1226    my $neb = $self->nebulous;
     1227
     1228    my $scheme = file_scheme($filename);
     1229    if (!$scheme or ($scheme ne 'neb')) {
     1230        if ($self->file_exists($filename)) {
     1231            if (!$self->file_delete($filename)) {
     1232                carp "failed to delete $filename";
     1233                return $PS_EXIT_SYS_ERROR;
     1234            }
     1235        }
     1236    } elsif ($neb->storage_object_exists($filename)) {
     1237        $self->_kill_nebulous_file($filename);
     1238    }
     1239    return 0;
    12001240}
    12011241

  • tags/ipp-20101215/ippScripts/scripts/ipp_cleanup.pl

    r30416 r30635  
    207207            }
    208208
    209             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    210             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    211             $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
    212             $command .= " -dbname $dbname" if defined $dbname;
    213             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    214                 run(command => $command, verbose => $verbose);
    215             unless ($success) {
    216                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    217                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    218             }
     209            set_destreak_goto_cleaned();
    219210
    220211        } else {
    … …  
    324315        }
    325316
    326         # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    327         my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    328         $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
    329         $command .= " -dbname $dbname" if defined $dbname;
    330         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    331             run(command => $command, verbose => $verbose);
    332         unless ($success) {
    333             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    334             &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    335         }
     317        set_destreak_goto_cleaned();
    336318
    337319    } else {
    … …  
    469451            }
    470452
    471             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    472             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    473             $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
    474             $command .= " -dbname $dbname" if defined $dbname;
    475             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    476                 run(command => $command, verbose => $verbose);
    477             unless ($success) {
    478                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    479                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    480             }
     453            set_destreak_goto_cleaned();
    481454
    482455         } else {
    … …  
    603576            }
    604577
    605             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    606             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    607             $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
    608             $command .= " -dbname $dbname" if defined $dbname;
    609             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    610                 run(command => $command, verbose => $verbose);
    611             unless ($success) {
    612                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    613                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    614             }
     578            set_destreak_goto_cleaned();
    615579
    616580        } else {
    … …  
    759723            }
    760724
    761             # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
    762             my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
    763             $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
    764             $command .= " -dbname $dbname" if defined $dbname;
    765             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    766                 run(command => $command, verbose => $verbose);
    767             unless ($success) {
    768                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    769                 &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
    770             }
     725            set_destreak_goto_cleaned();
    771726
    772727        } else {
    … …  
    19451900{
    19461901    my $files = shift; # reference to a list of files to unlink
    1947 #     my $test_verbose = 1;
    1948 
    1949 #     if ($test_verbose == 1) {
    1950 #       open(TMPLOG,">>/tmp/czw.cleanup.log");
    1951 #       flock(TMPLOG,2);
    1952 #     }
    1953 
    1954     # this script is, of course, very dangerous.
     1902
    19551903    foreach my $file (@$files) {
    19561904        print STDERR "unlinking $stage $stage_id $file";
    1957         unless ($ipprc->file_exists($file)) {
    1958             print STDERR "\t File not found\n";
    1959         }
    1960         else {
    1961             print STDERR "\n";
    1962         }
    1963 
    1964 #       if ($test_verbose == 1) {
    1965 #           print TMPLOG "$stage $stage_id $file";
    1966 
    1967 #           else {
    1968 #               print TMPLOG "\n";
    1969 #           }
    1970 #       }
    1971 
    1972         $ipprc->file_delete($file);
    1973     }
    1974 
    1975 #     if ($test_verbose == 1) {
    1976 #       flock(TMPLOG,8);
    1977 #       close(TMPLOG);
    1978 #     }
     1905
     1906        my $error_code = $ipprc->kill_file($file);
     1907
     1908        &my_die("failed to kill $file", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR) if $error_code;
     1909    }
    19791910
    19801911    return 1;
    … …  
    20621993}
    20631994
     1995# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
     1996my $ds_done = 0;
     1997sub set_destreak_goto_cleaned {
     1998
     1999    return if $ds_done;
     2000
     2001    # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
     2002    my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
     2003    my $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
     2004    $command .= " -dbname $dbname" if defined $dbname;
     2005    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     2006        run(command => $command, verbose => $verbose);
     2007    unless ($success) {
     2008        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     2009            &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
     2010    }
     2011    $ds_done = 1;
     2012}
     2013
    20642014# XXX we currently do not set the error state in the db on my_die
    20652015sub my_die

  • tags/ipp-20101215/ippScripts/scripts/magic_destreak_cleanup.pl

    r30534 r30635  
    4444           'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
    4545           'camera=s'       => \$camera,     # camera for evaluating file rules
    46            'stage=s'        => \$stage,     # camera for evaluating file rules
     46           'stage=s'        => \$stage,      # ipp stage for this magicDSRun
    4747           'save-temps'     => \$save_temps, # Save temporary files?
    4848           'dbname=s'       => \$dbname,     # Database name
    … …  
    160160        my $backup_path_base = $comp->{backup_path_base};
    161161        my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom);
    162         my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom);
     162#        my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom);
    163163
    164164        if ($stage eq "chip") {
    … …  
    219219        }
    220220
    221         delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
     221        delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask);
    222222
    223223        if ($stage eq "diff" and $warp_warp) {
    … …  
    252252    foreach my $file (@_) {
    253253        if ($file) {
    254             if ($ipprc->file_exists($file)) {
    255                 if (!$no_update) {
    256                     print STDERR "deleting $file\n" if $verbose;
    257                     $ipprc->file_delete($file, 1) or my_die("Failed to delete $file", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR);
    258                     } else {
    259                     print STDERR "skipping delete $file\n";
    260                 }
    261             } else {
    262                 print STDERR "$file not found\n" if $verbose;
    263             }
     254            my $error_code = $ipprc->kill_file($file);
     255            my_die("Failed to delete $file", $magic_ds_id, $error_code) if $error_code;
    264256        }
    265257    }
Note: See TracChangeset for help on using the changeset viewer.