IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 30583


Ignore:
Timestamp:
Feb 11, 2011, 3:30:25 PM (15 years ago)
Author:
watersc1
Message:

Added tested version of --continue option to convince it to continue
on. This should probably be watched a bit the first time it runs.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/ipp_apply_burntool_single.pl

    r30216 r30583  
    1515my $REALRUN = 1;
    1616
    17 my ( $class_id, $exp_id, $this_uri, $previous_uri, $imfile_state, $camera, $dbname, $logfile, $verbose, $save_temps, $rerun );
     17my ( $class_id, $exp_id, $this_uri, $previous_uri, $imfile_state, $camera, $dbname, $logfile, $verbose, $save_temps );
     18my ( $rerun, $fixit );
     19my $continue = 0;
    1820GetOptions(
    1921    'exp_id=s'         => \$exp_id,   # exposure identifier
    … …  
    2628    'logfile=s'         => \$logfile,
    2729    'rerun'             => \$rerun,
     30#    'fix|f'             => \$fixit,
     31    'continue=i'        => \$continue,
    2832    'verbose'           => \$verbose,   # Print to stdout
    2933    'save-temps'        => \$save_temps, # Save temporary files?
    … …  
    3337pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    3438pod2usage(
    35     -msg => "Required options: --class_id --this_uri --dbname",
     39    -msg => "Required options: --class_id --this_uri --exp_id --dbname",
    3640    -exitval => 3,
    3741    ) unless
    3842    defined $class_id and
    3943    defined $this_uri and
     44    defined $exp_id and
    4045    defined $dbname;
    4146
    … …  
    9398print ">>$burntoolStateGood<<\n";
    9499
     100# Determine what to do if we're fixing.
     101# if ($fixit) {
     102#     ($exp_id,$this_uri,$previous_uri,$continue) = fix_burntool($exp_id,$this_uri,$class_id);
     103# }
     104
     105
     106# Precalculate what we should be able to do if we're continuing
     107my @continue_uris;
     108my @continue_exp_ids;
     109my @continue_states;
     110if ($continue > 0) {
     111    my $regtool_get_date   = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -dbname $dbname";
     112    my $dateobs;
     113    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     114        run ( command => $regtool_get_date, verbose => $verbose );
     115    unless ($success) {
     116        $continue = 0;
     117        last;
     118    }
     119    my $imfile_data = $mdcParser->parse(join "", @$stdout_buf);
     120    my $imfile_data2 = parse_md_list($imfile_data);
     121    if ($#{ $imfile_data2 } > 0) {
     122        &my_die("Too many entries returned for date query", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
     123    }
     124    foreach my $entry (@$imfile_data2) {
     125        $dateobs = $entry->{dateobs};
     126    }
     127    my $date = $dateobs; $date =~ s/T.*//;
     128
     129    my $regtool_status_cmd = "$regtool -checkstatus -dateobs_begin $dateobs -dateobs_end ${date}T17:30:00 -class_id $class_id -dbname $dbname";
     130    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     131        run ( command => $regtool_status_cmd, verbose => $verbose );
     132    unless ($success) {
     133        $continue = 0;
     134    }
     135    if ($continue != 0) {
     136        my $night_data = $mdcParser->parse(join "", @$stdout_buf);
     137        my $night_data2 = parse_md_list($night_data);
     138        my $addable = 0;
     139       
     140        foreach my $entry (@$night_data2) {
     141            my ($this_exp_id,$this_uri,$this_state);
     142            if ($entry->{exp_id} == $exp_id) {
     143                $addable = 1;
     144                next;
     145            }
     146            if (($addable == 1)&&($#continue_exp_ids < $continue)) {
     147                $this_exp_id = $entry->{exp_id};
     148                $this_uri    = $entry->{uri};
     149                $this_state  = $entry->{imfile_state};
     150                print ">> $this_exp_id $this_uri $this_state\n";
     151                if ($this_state eq 'pending_burntool') {
     152                    push @continue_exp_ids, $this_exp_id;
     153                    push @continue_uris, $this_uri;
     154                    push @continue_states, $this_state;
     155                }
     156                else {
     157                    last;
     158                }
     159            }
     160            if ($#continue_exp_ids >= $continue) {
     161                last;
     162            }
     163        }
     164    }
     165}
     166$continue = $#continue_exp_ids + 1;
     167
     168# Look over at least one uri/imfile/exp_id and do burntool on it.
     169my $next_exp_id;
     170for (my $iteration = 0; $iteration <= $continue; $iteration ++) {
    95171# Set up files
    96 my $rawImfile = $this_uri;
    97 my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0);
    98 
    99 my $outTable  = $this_uri;
    100 $outTable  =~ s/fits$/burn.tbl/;
    101 my $outTableReal = $ipprc->file_resolve($outTable, 1);
    102 
    103 my $previousTable;
    104 my $previousTableReal;
    105 if ($previous_uri) {
    106     $previousTable = $previous_uri;
    107     $previousTable =~ s/fits$/burn.tbl/;
    108     $previousTableReal = $ipprc->file_resolve($previousTable, 0);
    109 }
    110 
     172    my $rawImfile = $this_uri;
     173    my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0);
     174   
     175    my $outTable  = $this_uri;
     176    $outTable  =~ s/fits$/burn.tbl/;
     177    my $outTableReal = $ipprc->file_resolve($outTable, 1);
     178   
     179    my $previousTable;
     180    my $previousTableReal;
     181    if ($previous_uri) {
     182        $previousTable = $previous_uri;
     183        $previousTable =~ s/fits$/burn.tbl/;
     184        $previousTableReal = $ipprc->file_resolve($previousTable, 0);
     185    }
     186   
    111187# Set state to processing:
    112 my $burntool_state = 0;
    113 
    114 my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN);
    115 if ($status) {
    116     &my_die("failed to update imfile");
    117 }
    118 $burntool_state = 1;
    119 
    120 
     188    my $burntool_state = 0;
     189   
     190    my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN);
     191    if ($status) {
     192        &my_die("failed to update imfile");
     193    }
     194    $burntool_state = 1;
     195   
     196   
    121197# funpack
    122 my $tempfile = new File::Temp ( TEMPLATE => "burntool.${exp_id}.${class_id}.XXXX",
    123                                 DIR => '/tmp/',
    124                                 UNLINK => !$save_temps,
    125                                 SUFFIX => '.fits');
    126 my $tempPixels = $tempfile->filename;
    127 
    128 $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN);
    129 if ($status) {
    130     &my_die("failed on funpack",$exp_id,$class_id);
    131 }
    132 
     198    my $tempfile = new File::Temp ( TEMPLATE => "burntool.${exp_id}.${class_id}.XXXX",
     199                                    DIR => '/tmp/',
     200                                    UNLINK => !$save_temps,
     201                                    SUFFIX => '.fits');
     202    my $tempPixels = $tempfile->filename;
     203   
     204    $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN);
     205    if ($status) {
     206        &my_die("failed on funpack",$exp_id,$class_id);
     207    }
     208   
    133209# burntool
    134 if (($previousTableReal)&&($previousTableReal ne '')) {
    135     $status = vsystem ("$burntool $tempPixels in=$previousTableReal out=$outTableReal tableonly=t persist=t", $REALRUN);
    136 }
    137 else {
    138     $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN);
    139 }
    140 if ($status) {
    141     &my_die("failed on burntool",$exp_id,$class_id);
    142 }
    143 &my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal);
    144 
     210    if (($previousTableReal)&&($previousTableReal ne '')) {
     211        $status = vsystem ("$burntool $tempPixels in=$previousTableReal out=$outTableReal tableonly=t persist=t", $REALRUN);
     212    }
     213    else {
     214        $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN);
     215    }
     216    if ($status) {
     217        &my_die("failed on burntool",$exp_id,$class_id);
     218    }
     219    &my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal);
     220   
    145221# Replicate files
    146 $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN);
    147 $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN);
    148 
     222    $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN);
     223    $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN);
     224
     225# Prep for next iteration if necessary.
     226    if ($iteration < $continue) {
     227        $next_exp_id = shift(@continue_exp_ids);
     228        my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $next_exp_id -class_id $class_id -set_state check_burntool";
     229        $status = vsystem ($command, $REALRUN);
     230        if ($status) {
     231            $continue = 0;
     232        }
     233    }
    149234# Set state to finished.
    150 my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState";
    151 if ($imfile_state ne 'full') {
    152     $command .= " -set_state full ";
    153 }
    154 $status = vsystem ($command, $REALRUN);
    155 if ($status) {
    156     &my_die("failed to update imfile");
    157 }
    158 
     235    my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState";
     236    if ($imfile_state ne 'full') {
     237        $command .= " -set_state full ";
     238    }
     239    $status = vsystem ($command, $REALRUN);
     240    if ($status) {
     241        &my_die("failed to update imfile");
     242    }
     243
     244# Set values for the next iteration if we are doing another iteration.
     245    $previous_uri = $this_uri;
     246    if ($iteration < $continue) {
     247        $this_uri = shift (@continue_uris);
     248        $exp_id   = $next_exp_id;
     249        $imfile_state = shift(@continue_states);
     250    }
     251
     252}
    159253exit 0;
    160254
    … …  
    183277    exit 1;
    184278}
    185 
    186 
    187    
     279       
     280# sub fix_burntool {
     281#     my ($exp_id,$this_uri,$class_id,$burntoolGoodState) = @_;
     282#     my ($dateobs,$exp_name);
     283#     # Calculate the date for this exposure, and calculate back to find the burntool period.
     284#     my $regtool_get_date   = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -dbname $dbname";
     285#     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     286#       run ( command => $regtool_get_date, verbose => $verbose );
     287#     unless ($success) {
     288#       $continue = 0;
     289#       return($exp_id,$this_uri,undef,0);
     290#     }
     291#     my $imfile_data = $mdcParser->parse(join "", @$stdout_buf);
     292#     foreach my $entry (@$imfile_data) {
     293#       if ($entry->{name} eq 'dateobs') {
     294#           $dateobs = $entry->{value};
     295#       }
     296#       if ($entry->{name} eq 'exp_name') {
     297#           $exp_name = $entry->{value};
     298#       }
     299#     }
     300#     my $date = $dateobs; $date =~ s/T.*//;
     301#     my $time = $dateobs; $time =~ s/.*T//;
     302#     my ($hour,$min,$sec) = split /\:/, $time; #/;
     303#     if ($min >= 30) {
     304#       $min -= 30;
     305#     }
     306#     else {
     307#       $min += 30;
     308#       $hour -= 1;
     309#     }
     310#     my $dateobs_begin = sprintf("%sT%02d:%02d:%02d",$date,$hour,$min,$sec);
     311
     312
     313#     # Get the night data for this date range.
     314#     my $regtool_status_cmd = "$regtool -checkstatus -dateobs_begin $dateobs_begin -dateobs_end $dateobs -class_id $class_id -dbname $dbname";
     315#     ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     316#       run ( command => $regtool_get_date, verbose => $verbose );
     317#     unless ($success) {
     318#       return($exp_id,$this_uri,undef,0);
     319#     }
     320
     321#     my $night_data = $mdcParser->parse(join "", @$stdout_buf);
     322
     323#     my $mode;
     324#     my ($row_exp_id,$row_uri,$row_data_state,$row_bt_state);
     325#     my $entry = @$night_data[-1];
     326#     foreach my $row (@$entry) {  # This is a single exposure, so let's get everything we'll need here.
     327#       if ($row->{name} eq 'exp_id') {
     328#           $row_exp_id = $row->{value};
     329#       }
     330#       elsif ($row->{name} eq 'uri') {
     331#           $row_uri = $row->{uri};
     332#       }
     333#       elsif ($row->{name} eq 'data_state') {
     334#           $row_data_state = $row->{data_state};
     335#       }               
     336#       elsif ($row->{name} eq 'burntool_state') {
     337#           $row_bt_state = $row->{burntool_state};
     338#       }               
     339#     }
     340#     # Determine what to do.
     341#     if ($row_exp_id == $exp_id) {
     342#       if (($row_bt_state != $burntoolStateGood)) {
     343#           # Never been run.
     344#           $mode = 1;
     345#       }
     346#       else {
     347#           $mode = 0;
     348#       }
     349#     }
     350#     if ($mode == 0) {
     351#       # If we're here, then the database thinks we have a table for this uri on disk. Let's see if that's true.
     352#       # Zero byte file/good alternate
     353#       # Inconsistent md5sums
     354#       # Missing this table only.
     355#       my $neb = $ipprc->nebulous();
     356#       my $table_key = $this_uri; $table_key =~ s/fits/burn.tbl/;
     357
     358#       my $neb_instances = $neb->find_instances($table_key, 'any');
     359
     360#       my $Ninstances = $#{ $neb_instances } + 1;
     361
     362#       if (($Ninstances == 0)||($Ninstances == 1)) {
     363#           #Missing this table only.  This should fall through to do a regular pass on this uri
     364#           $continue = 0;
     365#           $entry = @$night_data[-2];
     366#           foreach my $row (@$entry) {  # This is a single exposure, so let's get everything we'll need here.
     367#               if ($row->{name} eq 'exp_id') {
     368#                   $row_exp_id = $row->{value};
     369#               }
     370#               elsif ($row->{name} eq 'uri') {
     371#                   $row_uri = $row->{uri};
     372#               }
     373#               elsif ($row->{name} eq 'data_state') {
     374#                   $row_data_state = $row->{data_state};
     375#               }               
     376#               elsif ($row->{name} eq 'burntool_state') {
     377#                   $row_bt_state = $row->{burntool_state};
     378#               }               
     379#           }
     380#           $previous_uri = $row_uri;
     381#           # Go through to regular processing of this exposure.
     382#       }
     383#       elsif ($Ninstances == 2) {
     384#           $md5sum_A = md5sum(${ $neb_instances }[0]);
     385#           $md5sum_B = md5sum(${ $neb_instances }[1]);
     386#           if ($md5sum_A != $md5sum_B) {
     387#               # md5sums are different
     388#               if ($md5sum_A = 'd41d8cd98f00b204e9800998ecf8427e') {
     389#                   # copy B -> A
     390#               }
     391#               elsif ($md5sum_B = 'd41d8cd98f00b204e9800998ecf8427e') {
     392#                   # copy A -> B
     393#               }
     394#               else {
     395#                   # different but unknown why
     396#               }
     397#           }
     398#       }
     399#     }
     400
     401#     if ($mode == 1) {
     402#       foreach my $entry (reverse(@$night_data)) {  # reverse it so we can start with the exposure we have and go backwards
     403#           foreach my $row (@$entry) {  # This is a single exposure, so let's get everything we'll need here.
     404#               if ($row->{name} eq 'exp_id') {
     405#                   $row_exp_id = $row->{value};
     406#               }
     407#               elsif ($row->{name} eq 'uri') {
     408#                   $row_uri = $row->{uri};
     409#               }
     410#               elsif ($row->{name} eq 'data_state') {
     411#                   $row_data_state = $row->{data_state};
     412#               }               
     413#               elsif ($row->{name} eq 'burntool_state') {
     414#                   $row_bt_state = $row->{burntool_state};
     415#               }               
     416#           }
     417#       }
     418#     }
     419
     420#     return($exp_id,$this_uri,$previous_uri,$continue);
     421# }
Note: See TracChangeset for help on using the changeset viewer.