Index: /branches/czw_branch/20101203/Ohana/src/getstar/src/MatchCoords.c =================================================================== --- /branches/czw_branch/20101203/Ohana/src/getstar/src/MatchCoords.c (revision 30586) +++ /branches/czw_branch/20101203/Ohana/src/getstar/src/MatchCoords.c (revision 30587) @@ -53,7 +53,6 @@ } -#ifdef notdef - printf("%ld %8.1f %8.1f %s\n", i, x, y, dbImages[i].name); -#endif + + if (VERBOSE) fprintf(stderr, "%ld %8.1f %8.1f %s\n", i, x, y, dbImages[i].name); if ((x >= xmin && x <= xmax) && (y >= ymin && y <= ymax)) { Index: /branches/czw_branch/20101203/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- /branches/czw_branch/20101203/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 30586) +++ /branches/czw_branch/20101203/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 30587) @@ -34,4 +34,5 @@ $PSTAMP_NO_WAIT_FOR_UPDATE $PSTAMP_SUCCESS + $PSTAMP_FIRST_ERROR_CODE $PSTAMP_SYSTEM_ERROR $PSTAMP_NOT_IMPLEMENTED @@ -74,4 +75,5 @@ # NOTE: these must match the values in pstamp/src/pstamp.h our $PSTAMP_SUCCESS = 0; +our $PSTAMP_FIRST_ERROR_CODE = 10; our $PSTAMP_SYSTEM_ERROR = 10; our $PSTAMP_NOT_IMPLEMENTED = 11; Index: /branches/czw_branch/20101203/ippMonitor/raw/czartool_exposures.php =================================================================== --- /branches/czw_branch/20101203/ippMonitor/raw/czartool_exposures.php (revision 30586) +++ /branches/czw_branch/20101203/ippMonitor/raw/czartool_exposures.php (revision 30587) @@ -17,16 +17,34 @@ else {$myMenu = "ipp.imfiles.dat";} -menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']); - $pass = $ID['pass']; $proj = $ID['proj']; $menu = $ID['menu']; -$today = date("Y-m-d"); - +$date = $_POST['date']; +$survey = $_POST['survey']; + +// if no date chosen, use today +if ($date == "") $date = date("Y-m-d"); +if ($survey == "") $survey = ""; + +menu($myMenu, 'Exposure summary for '.$date , 'ipp.css', $ID['link'], $ID['proj']); + +// set up the form +echo "
\n"; + +echo "\n"; +echo "\n"; +echo "\n"; + + +echo ""; +echo "

Current status of all exposures taken on (UTC) "; +echo " for survey (leave blank for all surveys)"; +echo ""; echo "

"; -echo "

Current status of all exposures taken $today UTC

"; - -createTableForThisSurvey($gpc1db, $today, ""); + +echo "
\n"; + +createTableForThisSurvey($gpc1db, $date, $survey); menu_end(); @@ -58,5 +76,5 @@ $qry->fetchInto($count); - if ($surveyStr == "") $surveyText = "all surveys"; + if ($surveyStr == "") $surveyText = "all surveys"; else $surveyText = "$surveyStr survey"; @@ -83,34 +101,36 @@ $ssDiffProduced = array(); getSSDiffProduced($db, $date, $surveyStr, &$ssDiffProduced); - $class = "list"; - echo "\n"; + // create table + echo "
\n"; + + // table headers echo "\n"; - write_header_cell($class, ""); - write_header_cell($class, ""); - write_header_cell($class, ""); - write_header_cell($class, ""); - write_header_cell($class, "Raw"); - write_header_cell($class, "Chip"); - write_header_cell($class, "Camera"); - write_header_cell($class, "Warp"); - write_header_cell($class, "Stack"); - write_header_cell($class, "Warp-Stack Diff"); - write_header_cell($class, "Warp-Warp1 Diff"); - write_header_cell($class, "Warp-Warp2 Diff"); - write_header_cell($class, "Stack-Stack Diff"); + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; echo "\n"; - write_header_cell($class, "Exp ID"); - write_header_cell($class, "Exp Name"); - write_header_cell($class, "Filter"); - write_header_cell($class, "Comment"); - write_header_cell($class, "state"); - write_header_cell($class, "state/dist?"); - write_header_cell($class, "state/dist?"); - write_header_cell($class, "state/dist?"); - write_header_cell($class, "total/dist"); - write_header_cell($class, "total/dist"); - write_header_cell($class, "total/dist"); - write_header_cell($class, "total/dist"); - write_header_cell($class, "total/dist"); + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; $sql = @@ -147,49 +167,54 @@ echo "\n"; + echo ""; + echo ""; + echo ""; + echo ""; + echo ""; + // chip if (!$chipDist[$exp_id]) $chipDistStr = "N"; else $chipDistStr = $chipDist[$exp_id]; + echo ""; // cam if (!$camDist[$exp_id]) $camDistStr = "N"; else $camDistStr = $camDist[$exp_id]; + echo ""; // warp if (!$warpDist[$exp_id]) $warpDistStr = "N"; else $warpDistStr = $warpDist[$exp_id]; + echo ""; // stack - if (!$stack_hash[$exp_id] && !$stackDist[$exp_id]) $stackDistStr = ""; + if (!$stack_hash[$exp_id] && !$stackDist[$exp_id]) $stackDistStr = " "; else $stackDistStr = $stack_hash[$exp_id] . "/" . $stackDist[$exp_id]; + $color = getColour($stack_hash[$exp_id], $stackDist[$exp_id]); + echo ""; // WS diff - if (!$wsDiffProduced[$exp_id] && !$wsDiffDist[$exp_id]) $wsDistStr = ""; + if (!$wsDiffProduced[$exp_id] && !$wsDiffDist[$exp_id]) $wsDistStr = " "; else $wsDistStr = $wsDiffProduced[$exp_id] . "/" . $wsDiffDist[$exp_id]; + $color = getColour($wsDiffProduced[$exp_id], $wsDiffDist[$exp_id]); + echo ""; // WW1 diff - if (!$ww1DiffProduced[$exp_id] && !$ww1DiffDist[$exp_id]) $ww1DistStr = ""; + if (!$ww1DiffProduced[$exp_id] && !$ww1DiffDist[$exp_id]) $ww1DistStr = " "; else $ww1DistStr = $ww1DiffProduced[$exp_id] . "/" . $ww1DiffDist[$exp_id]; + $color = getColour($ww1DiffProduced[$exp_id], $ww1DiffDist[$exp_id]); + echo ""; // WW2 dist - if (!$ww2DiffProduced[$exp_id] && !$ww2DiffDist[$exp_id]) $ww2DistStr = ""; + if (!$ww2DiffProduced[$exp_id] && !$ww2DiffDist[$exp_id]) $ww2DistStr = " "; else $ww2DistStr = $ww2DiffProduced[$exp_id] . "/" . $ww2DiffDist[$exp_id]; + $color = getColour($ww2DiffProduced[$exp_id], $ww2DiffDist[$exp_id]); + echo ""; // SS dist - if (!$ssDiffProduced[$exp_id] && !$ssDiffDist[$exp_id]) $ssDistStr = ""; + if (!$ssDiffProduced[$exp_id] && !$ssDiffDist[$exp_id]) $ssDistStr = " "; else $ssDistStr = $ssDiffProduced[$exp_id] . "/" . $ssDiffDist[$exp_id]; - - write_table_cell($class, '%s', "", $exp_id); - write_table_cell($class, '%s', "", $exp_name); - write_table_cell($class, '%s', "", $filter); - write_table_cell($class, '%s', "", $comment); - write_table_cell($class, '%s', "", $rawExp); - write_table_cell($class, '%s', "", "$chipState/$chipDistStr"); - write_table_cell($class, '%s', "", "$camState/$camDistStr"); - write_table_cell($class, '%s', "", "$warpState/$warpDistStr"); - write_table_cell($class, '%s', "", $stackDistStr); - write_table_cell($class, '%s', "", $wsDistStr); - write_table_cell($class, '%s', "", $ww1DistStr); - write_table_cell($class, '%s', "", $ww2DistStr); - write_table_cell($class, '%s', "", $ssDistStr); + $color = getColour($ssDiffProduced[$exp_id], $ssDiffDist[$exp_id]); + echo ""; echo "\n"; @@ -199,4 +224,14 @@ } +########################################################################### +# +# Gets the right cell colour provided number created and distributed +# +########################################################################### +function getColour($created, $distributed) { + + if ($distributed > 0 && $created > 0 && $distributed % $created != 0) return "red"; + return ""; +} ########################################################################### Index: /branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl (revision 30586) +++ /branches/czw_branch/20101203/ippScripts/scripts/chip_imfile.pl (revision 30587) @@ -325,4 +325,5 @@ # get the UNIX version of the (possible) neb: or path: filename my $uriReal = $ipprc->file_resolve( $uri ); + &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal; # funpack into the temp file. Index: /branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl (revision 30586) +++ /branches/czw_branch/20101203/ippScripts/scripts/ipp_apply_burntool_single.pl (revision 30587) @@ -15,5 +15,7 @@ my $REALRUN = 1; -my ( $class_id, $exp_id, $this_uri, $previous_uri, $imfile_state, $camera, $dbname, $logfile, $verbose, $save_temps, $rerun ); +my ( $class_id, $exp_id, $this_uri, $previous_uri, $imfile_state, $camera, $dbname, $logfile, $verbose, $save_temps ); +my ( $rerun, $fixit ); +my $continue = 0; GetOptions( 'exp_id=s' => \$exp_id, # exposure identifier @@ -26,4 +28,6 @@ 'logfile=s' => \$logfile, 'rerun' => \$rerun, +# 'fix|f' => \$fixit, + 'continue=i' => \$continue, 'verbose' => \$verbose, # Print to stdout 'save-temps' => \$save_temps, # Save temporary files? @@ -33,9 +37,10 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --class_id --this_uri --dbname", + -msg => "Required options: --class_id --this_uri --exp_id --dbname", -exitval => 3, ) unless defined $class_id and defined $this_uri and + defined $exp_id and defined $dbname; @@ -93,68 +98,157 @@ print ">>$burntoolStateGood<<\n"; +# Determine what to do if we're fixing. +# if ($fixit) { +# ($exp_id,$this_uri,$previous_uri,$continue) = fix_burntool($exp_id,$this_uri,$class_id); +# } + + +# Precalculate what we should be able to do if we're continuing +my @continue_uris; +my @continue_exp_ids; +my @continue_states; +if ($continue > 0) { + my $regtool_get_date = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -dbname $dbname"; + my $dateobs; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $regtool_get_date, verbose => $verbose ); + unless ($success) { + $continue = 0; + last; + } + my $imfile_data = $mdcParser->parse(join "", @$stdout_buf); + my $imfile_data2 = parse_md_list($imfile_data); + if ($#{ $imfile_data2 } > 0) { + &my_die("Too many entries returned for date query", $exp_id, $class_id, $PS_EXIT_SYS_ERROR); + } + foreach my $entry (@$imfile_data2) { + $dateobs = $entry->{dateobs}; + } + my $date = $dateobs; $date =~ s/T.*//; + + my $regtool_status_cmd = "$regtool -checkstatus -dateobs_begin $dateobs -dateobs_end ${date}T17:30:00 -class_id $class_id -dbname $dbname"; + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $regtool_status_cmd, verbose => $verbose ); + unless ($success) { + $continue = 0; + } + if ($continue != 0) { + my $night_data = $mdcParser->parse(join "", @$stdout_buf); + my $night_data2 = parse_md_list($night_data); + my $addable = 0; + + foreach my $entry (@$night_data2) { + my ($this_exp_id,$this_uri,$this_state); + if ($entry->{exp_id} == $exp_id) { + $addable = 1; + next; + } + if (($addable == 1)&&($#continue_exp_ids < $continue)) { + $this_exp_id = $entry->{exp_id}; + $this_uri = $entry->{uri}; + $this_state = $entry->{imfile_state}; + print ">> $this_exp_id $this_uri $this_state\n"; + if ($this_state eq 'pending_burntool') { + push @continue_exp_ids, $this_exp_id; + push @continue_uris, $this_uri; + push @continue_states, $this_state; + } + else { + last; + } + } + if ($#continue_exp_ids >= $continue) { + last; + } + } + } +} +$continue = $#continue_exp_ids + 1; + +# Look over at least one uri/imfile/exp_id and do burntool on it. +my $next_exp_id; +for (my $iteration = 0; $iteration <= $continue; $iteration ++) { # Set up files -my $rawImfile = $this_uri; -my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0); - -my $outTable = $this_uri; -$outTable =~ s/fits$/burn.tbl/; -my $outTableReal = $ipprc->file_resolve($outTable, 1); - -my $previousTable; -my $previousTableReal; -if ($previous_uri) { - $previousTable = $previous_uri; - $previousTable =~ s/fits$/burn.tbl/; - $previousTableReal = $ipprc->file_resolve($previousTable, 0); -} - + my $rawImfile = $this_uri; + my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0); + + my $outTable = $this_uri; + $outTable =~ s/fits$/burn.tbl/; + my $outTableReal = $ipprc->file_resolve($outTable, 1); + + my $previousTable; + my $previousTableReal; + if ($previous_uri) { + $previousTable = $previous_uri; + $previousTable =~ s/fits$/burn.tbl/; + $previousTableReal = $ipprc->file_resolve($previousTable, 0); + } + # Set state to processing: -my $burntool_state = 0; - -my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN); -if ($status) { - &my_die("failed to update imfile"); -} -$burntool_state = 1; - - + my $burntool_state = 0; + + my $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -1", $REALRUN); + if ($status) { + &my_die("failed to update imfile"); + } + $burntool_state = 1; + + # funpack -my $tempfile = new File::Temp ( TEMPLATE => "burntool.${exp_id}.${class_id}.XXXX", - DIR => '/tmp/', - UNLINK => !$save_temps, - SUFFIX => '.fits'); -my $tempPixels = $tempfile->filename; - -$status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN); -if ($status) { - &my_die("failed on funpack",$exp_id,$class_id); -} - + my $tempfile = new File::Temp ( TEMPLATE => "burntool.${exp_id}.${class_id}.XXXX", + DIR => '/tmp/', + UNLINK => !$save_temps, + SUFFIX => '.fits'); + my $tempPixels = $tempfile->filename; + + $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN); + if ($status) { + &my_die("failed on funpack",$exp_id,$class_id); + } + # burntool -if (($previousTableReal)&&($previousTableReal ne '')) { - $status = vsystem ("$burntool $tempPixels in=$previousTableReal out=$outTableReal tableonly=t persist=t", $REALRUN); -} -else { - $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN); -} -if ($status) { - &my_die("failed on burntool",$exp_id,$class_id); -} -&my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal); - + if (($previousTableReal)&&($previousTableReal ne '')) { + $status = vsystem ("$burntool $tempPixels in=$previousTableReal out=$outTableReal tableonly=t persist=t", $REALRUN); + } + else { + $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN); + } + if ($status) { + &my_die("failed on burntool",$exp_id,$class_id); + } + &my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal); + # Replicate files -$status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN); -$status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN); - + $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN); + $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN); + +# Prep for next iteration if necessary. + if ($iteration < $continue) { + $next_exp_id = shift(@continue_exp_ids); + my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $next_exp_id -class_id $class_id -set_state check_burntool"; + $status = vsystem ($command, $REALRUN); + if ($status) { + $continue = 0; + } + } # Set state to finished. -my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState"; -if ($imfile_state ne 'full') { - $command .= " -set_state full "; -} -$status = vsystem ($command, $REALRUN); -if ($status) { - &my_die("failed to update imfile"); -} - + my $command = "$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState"; + if ($imfile_state ne 'full') { + $command .= " -set_state full "; + } + $status = vsystem ($command, $REALRUN); + if ($status) { + &my_die("failed to update imfile"); + } + +# Set values for the next iteration if we are doing another iteration. + $previous_uri = $this_uri; + if ($iteration < $continue) { + $this_uri = shift (@continue_uris); + $exp_id = $next_exp_id; + $imfile_state = shift(@continue_states); + } + +} exit 0; @@ -183,5 +277,145 @@ exit 1; } - - - + +# sub fix_burntool { +# my ($exp_id,$this_uri,$class_id,$burntoolGoodState) = @_; +# my ($dateobs,$exp_name); +# # Calculate the date for this exposure, and calculate back to find the burntool period. +# my $regtool_get_date = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -dbname $dbname"; +# my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $regtool_get_date, verbose => $verbose ); +# unless ($success) { +# $continue = 0; +# return($exp_id,$this_uri,undef,0); +# } +# my $imfile_data = $mdcParser->parse(join "", @$stdout_buf); +# foreach my $entry (@$imfile_data) { +# if ($entry->{name} eq 'dateobs') { +# $dateobs = $entry->{value}; +# } +# if ($entry->{name} eq 'exp_name') { +# $exp_name = $entry->{value}; +# } +# } +# my $date = $dateobs; $date =~ s/T.*//; +# my $time = $dateobs; $time =~ s/.*T//; +# my ($hour,$min,$sec) = split /\:/, $time; #/; +# if ($min >= 30) { +# $min -= 30; +# } +# else { +# $min += 30; +# $hour -= 1; +# } +# my $dateobs_begin = sprintf("%sT%02d:%02d:%02d",$date,$hour,$min,$sec); + + +# # Get the night data for this date range. +# my $regtool_status_cmd = "$regtool -checkstatus -dateobs_begin $dateobs_begin -dateobs_end $dateobs -class_id $class_id -dbname $dbname"; +# ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run ( command => $regtool_get_date, verbose => $verbose ); +# unless ($success) { +# return($exp_id,$this_uri,undef,0); +# } + +# my $night_data = $mdcParser->parse(join "", @$stdout_buf); + +# my $mode; +# my ($row_exp_id,$row_uri,$row_data_state,$row_bt_state); +# my $entry = @$night_data[-1]; +# foreach my $row (@$entry) { # This is a single exposure, so let's get everything we'll need here. +# if ($row->{name} eq 'exp_id') { +# $row_exp_id = $row->{value}; +# } +# elsif ($row->{name} eq 'uri') { +# $row_uri = $row->{uri}; +# } +# elsif ($row->{name} eq 'data_state') { +# $row_data_state = $row->{data_state}; +# } +# elsif ($row->{name} eq 'burntool_state') { +# $row_bt_state = $row->{burntool_state}; +# } +# } +# # Determine what to do. +# if ($row_exp_id == $exp_id) { +# if (($row_bt_state != $burntoolStateGood)) { +# # Never been run. +# $mode = 1; +# } +# else { +# $mode = 0; +# } +# } +# if ($mode == 0) { +# # If we're here, then the database thinks we have a table for this uri on disk. Let's see if that's true. +# # Zero byte file/good alternate +# # Inconsistent md5sums +# # Missing this table only. +# my $neb = $ipprc->nebulous(); +# my $table_key = $this_uri; $table_key =~ s/fits/burn.tbl/; + +# my $neb_instances = $neb->find_instances($table_key, 'any'); + +# my $Ninstances = $#{ $neb_instances } + 1; + +# if (($Ninstances == 0)||($Ninstances == 1)) { +# #Missing this table only. This should fall through to do a regular pass on this uri +# $continue = 0; +# $entry = @$night_data[-2]; +# foreach my $row (@$entry) { # This is a single exposure, so let's get everything we'll need here. +# if ($row->{name} eq 'exp_id') { +# $row_exp_id = $row->{value}; +# } +# elsif ($row->{name} eq 'uri') { +# $row_uri = $row->{uri}; +# } +# elsif ($row->{name} eq 'data_state') { +# $row_data_state = $row->{data_state}; +# } +# elsif ($row->{name} eq 'burntool_state') { +# $row_bt_state = $row->{burntool_state}; +# } +# } +# $previous_uri = $row_uri; +# # Go through to regular processing of this exposure. +# } +# elsif ($Ninstances == 2) { +# $md5sum_A = md5sum(${ $neb_instances }[0]); +# $md5sum_B = md5sum(${ $neb_instances }[1]); +# if ($md5sum_A != $md5sum_B) { +# # md5sums are different +# if ($md5sum_A = 'd41d8cd98f00b204e9800998ecf8427e') { +# # copy B -> A +# } +# elsif ($md5sum_B = 'd41d8cd98f00b204e9800998ecf8427e') { +# # copy A -> B +# } +# else { +# # different but unknown why +# } +# } +# } +# } + +# if ($mode == 1) { +# foreach my $entry (reverse(@$night_data)) { # reverse it so we can start with the exposure we have and go backwards +# foreach my $row (@$entry) { # This is a single exposure, so let's get everything we'll need here. +# if ($row->{name} eq 'exp_id') { +# $row_exp_id = $row->{value}; +# } +# elsif ($row->{name} eq 'uri') { +# $row_uri = $row->{uri}; +# } +# elsif ($row->{name} eq 'data_state') { +# $row_data_state = $row->{data_state}; +# } +# elsif ($row->{name} eq 'burntool_state') { +# $row_bt_state = $row->{burntool_state}; +# } +# } +# } +# } + +# return($exp_id,$this_uri,$previous_uri,$continue); +# } Index: /branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl (revision 30586) +++ /branches/czw_branch/20101203/ippScripts/scripts/ipp_cleanup.pl (revision 30587) @@ -207,14 +207,5 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + set_destreak_goto_cleaned(); } else { @@ -324,14 +315,5 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + set_destreak_goto_cleaned(); } else { @@ -469,14 +451,5 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + set_destreak_goto_cleaned(); } else { @@ -603,14 +576,5 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + set_destreak_goto_cleaned(); } else { @@ -759,14 +723,5 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + set_destreak_goto_cleaned(); } else { @@ -2062,4 +2017,23 @@ } +# this gets set to 1 the first time we set the corresponding destreak run to be cleaned +my $ds_done = 0; +sub set_destreak_goto_cleaned { + + return if $ds_done; + + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + my $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } + $ds_done = 1; +} + # XXX we currently do not set the error state in the db on my_die sub my_die Index: /branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl (revision 30586) +++ /branches/czw_branch/20101203/ippScripts/scripts/magic_destreak_cleanup.pl (revision 30587) @@ -159,5 +159,4 @@ my $component = $comp->{component}; my $backup_path_base = $comp->{backup_path_base}; - my $recovery_path_base = $comp->{recovery_path_base}; my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom); my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom); @@ -197,17 +196,4 @@ $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $component); } - if ($recovery_path_base) { - $rimage = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $component); - - if ($dynamicMasks) { - my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $component); - # This is kludgey but correct - $rmask = dirname($recovery_path_base) . "/" . basename($mask); - $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component); - } else { - $rmask = $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component); - } - $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $component); - } } elsif ($stage eq "camera") { if ($backup_path_base) { @@ -221,12 +207,20 @@ $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base); } - if ($recovery_path_base) { - $rimage = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base); - $rmask = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base); - $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base); - $rsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $recovery_path_base); - } } elsif ($stage eq "diff") { my $name = "PPSUB.OUTPUT"; + if ($backup_path_base) { + $bimage = $ipprc->filename($name, $backup_path_base); + $bmask = $ipprc->filename("$name.MASK", $backup_path_base); + $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); + # bills 2011-01-24 + # don't clean up the uncensored Diff sources file + # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); + } + } + + delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask); + + if ($stage eq "diff" and $warp_warp) { + my $name = "PPSUB.INVERSE"; if ($backup_path_base) { $bimage = $ipprc->filename($name, $backup_path_base); @@ -237,32 +231,5 @@ # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); } - if ($recovery_path_base) { - $rimage = $ipprc->filename($name, $recovery_path_base); - $rmask = $ipprc->filename("$name.MASK", $recovery_path_base); - $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base); - $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base); - } - } - - delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask); - - if ($stage eq "diff" and $warp_warp) { - my $name = "PPSUB.INVERSE"; - if ($backup_path_base) { - $bimage = $ipprc->filename($name, $backup_path_base); - $bmask = $ipprc->filename("$name.MASK", $backup_path_base); - $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); - # bills 2011-01-24 - # don't clean up the uncensored sources file - # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); - } - if ($recovery_path_base) { - $rimage = $ipprc->filename($name, $recovery_path_base); - $rmask = $ipprc->filename("$name.MASK", $recovery_path_base); - $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base); - $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base); - } - # undef is not necessary, it's just to keep the lists in the same order. - delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources); + delete_files($bimage, $bmask, $bweight, $bsources); } my $command = "$magicdstool -tocleanedfile -magic_ds_id $magic_ds_id -component $component"; Index: /branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl (revision 30586) +++ /branches/czw_branch/20101203/ippScripts/scripts/nightly_science.pl (revision 30587) @@ -1266,4 +1266,27 @@ } +sub multi_date_verify_uniqueness_diff { + my $warp_id_1 = shift; + my $warp_id_2 = shift; + my $date = shift; + my $target = shift; + + my $db = init_gpc_db(); + $date =~ s/-//g; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $count = 0; + + my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_1 AND warp2 = $warp_id_2"; + my $data_ref = $db->selectall_arrayref( $sth ); + $count += $#{ $data_ref } + 1; + + $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_2 AND warp2 = $warp_id_1"; + $data_ref = $db->selectall_arrayref( $sth ); + $count += $#{ $data_ref } + 1; + + return($count); +} + sub pre_diff_queue { my $date = shift; @@ -1375,5 +1398,13 @@ $metadata_out{nsDiffState} = 'DIFFING'; } - + if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') { + foreach my $target (sort (keys %science_config)) { + if ($science_config{$target}{DIFFABLE} == 1) { + foreach my $filter (@filter_list) { + multi_date_diff_queue($date,$target,$filter,$pretend); + } + } + } + } # if (($Npotential == $Nnoexp)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)) { # $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; @@ -1511,4 +1542,137 @@ } + +sub multi_date_diff_queue { + my $date = shift; + my $target = shift; + my $filter = shift; + my $pretend = shift; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + + my $db = init_gpc_db(); + + my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; + $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND rawExp.filter = '$filter' ORDER BY rawExp.object"; + + my $object_ref = $db->selectall_arrayref( $obj_sth ); + + my $Npotential = 0; + my $Nqueued = 0; + + foreach my $object_row (@{ $object_ref }) { + my $this_object = shift @{ $object_row }; +# my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) "; +# $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; +# $input_sth .= " ORDER BY dateobs "; + + my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM "; + $input_sth .= " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) "; + $input_sth .= " WHERE warpRun.label = '$label' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' "; + $input_sth .= " ORDER BY dateobs "; + + my $warps = $db->selectall_arrayref( $input_sth ); + + # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra. + my %comment_hash = (); + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + $comment_hash{$this_comment} = $this_exp_id; + } + + if (($#{ $warps } + 1) % 2 != 0) { + print STDERR "md_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } "; + if ($#{ $warps} + 1 == 1) { + print STDERR ": I can do no diffs with only one exposure.\n"; + next; + } + else { + print STDERR ": I should declare an exposure to be faulty.\n"; + my @keep_warps = (); +# print "@{ $warps }\n"; + foreach my $this_warp (@{ $warps }) { + my $this_comment = ${ $this_warp }[3]; + my $this_exp_id = ${ $this_warp }[0]; + if ($comment_hash{$this_comment} == $this_exp_id) { + push @keep_warps, $this_warp; + } + else { + print STDERR "md_diff_queue: excluding $this_exp_id for $this_object\n"; + } + } + @{ $warps } = @keep_warps; +# print "@{ $warps }\n"; + } + } + + while ($#{ $warps } > -1) { + my $input_warp = shift @{ $warps }; + my $input_exp_id = ${ $input_warp }[0]; + my $input_comment = ${ $input_warp }[3]; + + + my $template_warp = shift @{ $warps }; + + my $template_exp_id = ${ $template_warp }[0]; + + my $input_warp_id = ${ $input_warp }[1]; + my $template_warp_id = ${ $template_warp }[1]; + + my $input_warp_state = ${ $input_warp }[4]; + my $template_warp_state = ${ $template_warp }[4]; + + unless(defined($template_warp)&& defined($template_exp_id)) { + print STDERR "md_diff received an undef! $input_exp_id $input_comment $this_object T: $template_warp V: @$template_warp\n"; + next; + } + $Npotential++; + + unless (defined($input_warp_id) && defined($template_warp_id) && + ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) { + print STDERR "md_Diff for this $date $target $input_exp_id ($input_warp_id $input_warp_state) $template_exp_id ($template_warp_id $template_warp_state) not fully processed\n"; + next; + } + + if (multi_date_verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) { + $Nqueued++; + print STDERR "md_Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n"; + next; + } + + my $new_data_group = "${data_group}.multi"; + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label "; + $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting. + $cmd .= "-set_workdir $workdir -set_dist_group $dist_group -set_data_group $new_data_group "; + $cmd .= " -simple -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id "; +# $cmd .= " -pretend "; + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + print STDERR "md_Diffs would like to queue for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n"; + print STDERR "md_diff_queue: $cmd\n"; + print STDERR " $input_warp_id $template_warp_id\n"; + } + + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $Nqueued++; + } + } + } + $metadata_out{nsDiffPotential} += $Npotential; + $metadata_out{nsDiffQueued} += $Nqueued; +# if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) { +# $metadata_out{nsDiffState} = 'FINISHED_DIFFS'; +# } + +} Index: /branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl (revision 30586) +++ /branches/czw_branch/20101203/ippScripts/scripts/register_exp.pl (revision 30587) @@ -255,5 +255,5 @@ if (exists($nightlyscience_config{$target}{OBSMODE})) { $possible++; - if ($obsmode =~ /$nightlyscience_config{$target}{OBSMODE}/) { + if ($obsmode =~ /^$nightlyscience_config{$target}{OBSMODE}$/) { $match++; } @@ -261,5 +261,5 @@ if (exists($nightlyscience_config{$target}{OBJECT})) { $possible++; - if ($object =~ /$nightlyscience_config{$target}{OBJECT}/) { + if ($object =~ /^$nightlyscience_config{$target}{OBJECT}$/) { $match++; } @@ -267,5 +267,5 @@ if (exists($nightlyscience_config{$target}{COMMENT})) { $possible++; - if ($comment =~ /$nightlyscience_config{$target}{COMMENT}/) { + if ($comment =~ /^$nightlyscience_config{$target}{COMMENT}$/) { $match++; } Index: /branches/czw_branch/20101203/ippScripts/scripts/warp_skycell.pl =================================================================== --- /branches/czw_branch/20101203/ippScripts/scripts/warp_skycell.pl (revision 30586) +++ /branches/czw_branch/20101203/ippScripts/scripts/warp_skycell.pl (revision 30587) @@ -71,6 +71,20 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id) or my_die( "Unable to get log filename", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ); -$logDest .= ".update" if ($run_state eq 'update'); +my ($logDest, $traceDest); +if ($run_state eq 'new') { + $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, 0); + $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1); +} elsif ($run_state eq 'update') { + $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 0); + $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, $skycell_id, 1); +} else { + &my_die( "invalid run_state: $run_state", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR ); +} + +my $neb; +my $scheme = file_scheme($outroot); +if ($scheme and $scheme eq 'neb') { + $neb = $ipprc->nebulous(); +} $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect; @@ -117,8 +131,8 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $skycell_id, $tess_dir, $error_code); } my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); } @@ -141,21 +155,43 @@ -my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outroot, $skycell_id ); -my $outputMask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outroot, $skycell_id); -my $outputWeight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id); -my $outputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id); -my $outputPSF = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $outroot); -my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outroot, $skycell_id ); -my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outroot, $skycell_id ); -my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot, $skycell_id ); -my $traceDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id); -my $configuration = $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id); - -if ($run_state eq 'update') { - $traceDest .= ".update"; - $outputStats .= ".update"; -} - -my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id ); +my $outputImage = prepare_output ("PSWARP.OUTPUT", $outroot, $skycell_id, 1); +my $outputMask = prepare_output ("PSWARP.OUTPUT.MASK", $outroot, $skycell_id, 1); +my $outputWeight = prepare_output ("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id, 1); +my $outputSources = prepare_output ("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id, 1); +my $outputPSF = prepare_output ("PSPHOT.PSF.SKY.SAVE", $outroot, 1); +my $outputBin1 = prepare_output ("PSWARP.BIN1", $outroot, $skycell_id, 1); +my $outputBin2 = prepare_output ("PSWARP.BIN2", $outroot, $skycell_id, 1); +my $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1); +my $configuration; + +my $dump_config = 1; +if ($run_state eq 'new') { + $configuration = prepare_output ("PSWARP.CONFIG", $outroot, $skycell_id, 1); +} else { + $configuration = $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id) or + &my_die("Missing entry from camera config PSWARP.CONFIG", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR); + if ($ipprc->file_exists($configuration)) { + $dump_config = 0; + } else { + print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n"; + + # XXX: should we create a new config dump file? + # I vote yes but only if we can distingusing between temporarily unavailable and GONE. + my $gone = 0; + if (storage_object_exists($configuration, \$gone)) { + if ($gone) { + $configuration = prepare_output('PSWARP.CONFIG', $outroot, $skycell_id, 1); + # if we dump the config we need to insure that the config dump represents + # the full processing + } else { + # file is temporarily not available. Don't dump config. + $dump_config = 0; + } + } + } +} + + +my $skyFile = prepare_output ("SKYCELL.TEMPLATE", $outroot, $skycell_id, 1); $ipprc->skycell_file( $tess_dir, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); ## XXX this seems to have insufficient error checking: dvoImageExtract can fail to write and still return a valid exit status @@ -171,11 +207,11 @@ foreach my $imfile (@$imfiles) { my $image = $imfile->{uri}; # Image name - my $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name + my $weight = $ipprc->filename ("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name my $mask; # Mask name if ($dynamicMasks) { - $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); + $mask = $ipprc->filename ("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id}); } else { - $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id}); + $mask = $ipprc->filename ("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id}); } @@ -186,5 +222,5 @@ # Astrometry file: astrometry is done at the camera stage, and always results in a MEF file # XXX allow an option to use the image header astrometry? - my $astrom = $ipprc->filename($astromSource, $imfile->{cam_path_base}); + my $astrom = $ipprc->filename ($astromSource, $imfile->{cam_path_base}); &my_die("Couldn't find input file: $astrom", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless defined $astrom and $ipprc->file_exists($astrom); @@ -245,7 +281,5 @@ $do_stats = 1; } else { - #$command .= " -ipprc $configuration"; - my $resolved = $ipprc->file_resolve($configuration); - $command .= " -ipprc $resolved"; + $command .= " -ipprc $configuration"; } if ($do_stats) { @@ -263,9 +297,10 @@ if ($do_stats) { # Check first for the stats file + check_output($outputStats, 0); my $outputStatsReal = $ipprc->file_resolve($outputStats); - &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); +# &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); &my_die("Stats file has zero size: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless -s $outputStatsReal; - # measure chip stats + # measure skycell stats $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL"; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -283,11 +318,18 @@ if (!$quality) { - &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); - &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); - &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); - &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); - &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF); - if ($run_state eq 'new') { - &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration); + # &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); + # &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + # &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + # &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + # &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF); + + check_output($outputImage, 0); + check_output($outputMask, 0); + check_output($outputWeight, 0); + check_output($outputSources, 1); + check_output($outputPSF, 1) if metadataLookupBool($recipe, 'PSF') ; + if ($dump_config) { + check_output($configuration, 1); +# &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration); } } @@ -335,4 +377,117 @@ } } + +exit 0; + +# Prepare to write to an output file +# Lookup the filename in the rules. +# Make sure that if file exists and is a nebulous file that there is only one instance +# Deal with files that have been lost. +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $skycell_id = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, $skycell_id, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $warp_id, $skycell_id, $tess_dir, $error); + return $output; +} + +sub check_output +{ + my $file = shift; + my $replicate = shift; + + if (!defined $file) { + return; + } + + &my_die("Couldn't find expected output file: $file", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + if ($replicate and $neb) { + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); + } +} + +# subroutine to check the status of a nebulous file. Used to distinguish between a storage object that +# does not exist and one that all of the instances have been lost. +# XXXX This should be implemented properly in Nebulous +# For now uses Bill's script 'whichnode' which queries the nebulous database directly + +my $whichnode; +sub storage_object_exists +{ + return 0 if !$neb; + + my $file = shift; + my $ref_all_gone = shift; + + my $exists = $neb->storage_object_exists($file); + if (!$exists) { + return 0; + } + + if (!$whichnode) { + $whichnode = can_run('whichnode') or + &my_die("Can't find whichnode", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR); + } + + my $command = "$whichnode $file"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform whichnode: $error_code", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR); + } + + my @lines = split "\n", (join "", @$stdout_buf); + + if (scalar @lines == 0) { + # no output the file is really and truely gone + # XXX: this is now caught above + print STDERR "storage object for $file does not exist\n"; + return 0; + } + + my $numGone = 0; + my $numNotGone = 0; + foreach my $line (@lines) { + chomp $line; + + # output lines are either + # "volume available" + # or + # "volume not available" + + my ($volume, $answer, undef) = split " ", $line; + # our hack is if the volume has an X in the name it's gone + if ($volume =~ /X/) { + print STDERR "$file is on $volume which is gone\n"; + $numGone++; + } elsif ($answer eq 'available') { + $numNotGone++; + } elsif ($answer eq 'not') { + print STDERR "$file is on $volume which is not available\n"; + $numNotGone++; + } else { + print STDERR "unexpected output from whichnode: $line\n"; + } + } + # if there are any instances that are not on a gone volume set all_gone to 0 + if ($numNotGone == 0 and $numGone > 0) { + $$ref_all_gone = 1; + } else { + $$ref_all_gone = 0; + } + + # storage object exists so return true + return 1; +} + sub my_die Index: /branches/czw_branch/20101203/ippTasks/pstamp.pro =================================================================== --- /branches/czw_branch/20101203/ippTasks/pstamp.pro (revision 30586) +++ /branches/czw_branch/20101203/ippTasks/pstamp.pro (revision 30587) @@ -18,4 +18,21 @@ $pstampStopFaulted_DB = 0 +# give up on dependents with fault_count >= $PSTAMP_MAX_FAULT_COUNT +$PSTAMP_MAX_FAULT_COUNT = 5 +macro set.max.fault.count + $PSTAMP_MAX_FAULT_COUNT = $1 +end +macro get.max.fault.count + echo maximum fault count: $PSTAMP_MAX_FAULT_COUNT +end +$POLL_DEP = 500 + +macro set.dependent.poll + $POLL_DEP = $1 +end +macro get.dependent.poll + echo dependent poll limit: $POLL_DEP +end + # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config # warning: no quotes around the two words. That causes the variable to get passed to pstamptool as one word @@ -137,4 +154,5 @@ end end + macro pstamp.find.on task pstamp.request.find @@ -143,8 +161,26 @@ end macro pstamp.find.off - task pstamp.request.find.load - active false - end -end + task pstamp.request.find + active false + end +end + +macro pstamp.dependent.on + task pstamp.dependent.load + active true + end + task pstamp.dependent.run + active true + end +end +macro pstamp.dependent.off + task pstamp.dependent.load + active false + end + task pstamp.dependent.run + active false + end +end + macro pstamp.status.on @@ -158,4 +194,5 @@ end end + macro pstamp.status.set.exec task pstamp.save.status @@ -525,5 +562,5 @@ echo pstamp.job.run task.exit $JOB_ID status: $JOB_STATUS end - showcommand failure +# showcommand failure process_exit pstampJob $options:0 $JOB_STATUS end @@ -609,5 +646,5 @@ add_poll_args run add_poll_labels run - command $run + command $run -limit $POLL_DEP end @@ -664,4 +701,5 @@ book getword pstampDependent $pageName outdir -var OUTDIR book getword pstampDependent $pageName need_magic -var NEED_MAGIC + book getword pstampDependent $pageName fault_count -var FAULT_COUNT book getword pstampDependent $pageName dbname -var DBNAME @@ -682,5 +720,5 @@ stderr $MYLOGFILE - $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC + $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT add_standard_args run @@ -705,5 +743,5 @@ echo pstamp.job.run task.exit $DEP_ID status: $JOB_STATUS end - showcommand failure +# showcommand failure process_exit pstampDependent $options:0 $JOB_STATUS end Index: /branches/czw_branch/20101203/ippTasks/rcserver.pro =================================================================== --- /branches/czw_branch/20101203/ippTasks/rcserver.pro (revision 30586) +++ /branches/czw_branch/20101203/ippTasks/rcserver.pro (revision 30587) @@ -40,7 +40,4 @@ active false end - task rcserver.revert - active false - end end @@ -48,4 +45,10 @@ task rcserver.revert active true + end +end + +macro rcserver.revert.off + task rcserver.revert + active false end end Index: /branches/czw_branch/20101203/ippTasks/survey.pro =================================================================== --- /branches/czw_branch/20101203/ippTasks/survey.pro (revision 30586) +++ /branches/czw_branch/20101203/ippTasks/survey.pro (revision 30587) @@ -1135,5 +1135,5 @@ end - echo $run + # echo $run command $run Index: /branches/czw_branch/20101203/ippToPsps/perl/ippToPsps/Gpc1Db.pm =================================================================== --- /branches/czw_branch/20101203/ippToPsps/perl/ippToPsps/Gpc1Db.pm (revision 30586) +++ /branches/czw_branch/20101203/ippToPsps/perl/ippToPsps/Gpc1Db.pm (revision 30587) @@ -34,4 +34,22 @@ return 1; +} + +########################################################################### +# +# Returns a count of warps thyat contributed to the stack with this stack ID +# +########################################################################### +sub countContributingWarpsToStack { + my ($self, $stackId) = @_; + + my $query = $self->{_db}->prepare(<execute; + return scalar $query->fetchrow_array(); } Index: /branches/czw_branch/20101203/ippTools/share/chiptool_processedimfile.sql =================================================================== --- /branches/czw_branch/20101203/ippTools/share/chiptool_processedimfile.sql (revision 30586) +++ /branches/czw_branch/20101203/ippTools/share/chiptool_processedimfile.sql (revision 30587) @@ -29,6 +29,8 @@ rawImfile.magicked AS raw_magicked, rawImfile.burntool_state, - magicDSRun.state AS dsRun_state, - IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id + IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id, + IFNULL(magicDSRun.state,0) AS dsRun_state, + IFNULL(magicDSFile.fault,0) AS dsFile_fault, + IFNULL(magicDSFile.data_state,0) AS dsFile_data_state FROM chipRun JOIN chipImfile @@ -43,2 +45,3 @@ LEFT JOIN magicDSRun ON stage_id = chip_id AND stage = 'chip' AND magicDSRun.re_place AND magicDSRun.state != 'drop' +LEFT JOIN magicDSFile ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id AND chipProcessedImfile.class_id = magicDSFile.component Index: /branches/czw_branch/20101203/ippTools/share/difftool_pendingcleanupskyfile.sql =================================================================== --- /branches/czw_branch/20101203/ippTools/share/difftool_pendingcleanupskyfile.sql (revision 30586) +++ /branches/czw_branch/20101203/ippTools/share/difftool_pendingcleanupskyfile.sql (revision 30587) @@ -9,5 +9,5 @@ USING(diff_id) WHERE - ((diffRun.state = 'goto_cleaned' AND diffSkyfile.data_state = 'full') + ((diffRun.state = 'goto_cleaned' AND (diffSkyfile.data_state = 'full' OR diffSkyfile.data_state = 'update')) OR (diffRun.state = 'goto_scrubbed' AND diffSkyfile.data_state != 'scrubbed') Index: /branches/czw_branch/20101203/ippTools/share/difftool_setskyfiletoupdate.sql =================================================================== --- /branches/czw_branch/20101203/ippTools/share/difftool_setskyfiletoupdate.sql (revision 30586) +++ /branches/czw_branch/20101203/ippTools/share/difftool_setskyfiletoupdate.sql (revision 30587) @@ -1,11 +1,9 @@ UPDATE diffRun JOIN diffSkyfile USING(diff_id) - LEFT JOIN magicDSRun ON (stage_id = diff_id AND stage = 'diff') - LEFT JOIN magicDSFile ON (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND component = skycell_id) SET diffRun.state = 'update', - diffSkyfile.data_state = 'update' + diffSkyfile.data_state = 'update', + diffSkyfile.fault = 0 -- set hook %s WHERE diff_id = %lld AND (diffRun.state = 'cleaned' OR diffRun.state = 'update') - AND (diffSkyfile.data_state = 'cleaned') + AND (diffSkyfile.data_state = 'cleaned' OR diffSkyfile.data_state = 'update') Index: /branches/czw_branch/20101203/ippTools/share/magicdstool_setfiletoupdate.sql =================================================================== --- /branches/czw_branch/20101203/ippTools/share/magicdstool_setfiletoupdate.sql (revision 30586) +++ /branches/czw_branch/20101203/ippTools/share/magicdstool_setfiletoupdate.sql (revision 30587) @@ -7,3 +7,3 @@ WHERE magic_ds_id = %lld AND (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') - AND (magicDSFile.data_state = 'cleaned') + AND (magicDSFile.data_state = 'cleaned' OR magicDSFile.data_state ='update') Index: /branches/czw_branch/20101203/ippTools/share/warptool_change_skyfile_data_state.sql =================================================================== --- /branches/czw_branch/20101203/ippTools/share/warptool_change_skyfile_data_state.sql (revision 30586) +++ /branches/czw_branch/20101203/ippTools/share/warptool_change_skyfile_data_state.sql (revision 30587) @@ -1,5 +1,5 @@ -- handle changes in warpSkyfile.data_state. -- Used for the modes tofullskyfile, tocleanedskyfile and topurgedskyfile --- args are new data_state, string for magic hook, warp_id, skycell_id and current expected state for warpRun +-- arguments are are new data_state, string for magic hook, warp_id, skycell_id UPDATE warpSkyfile JOIN warpRun USING(warp_id) Index: /branches/czw_branch/20101203/ippTools/share/warptool_scmap.sql =================================================================== --- /branches/czw_branch/20101203/ippTools/share/warptool_scmap.sql (revision 30586) +++ /branches/czw_branch/20101203/ippTools/share/warptool_scmap.sql (revision 30587) @@ -12,4 +12,5 @@ IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id, IFNULL(magicDSRun.state, 0) AS dsRun_state, + IFNULL(magicDSFile.data_state, 0) AS dsFile_data_state, IFNULL(magicDSFile.fault, 0) as dsFile_fault FROM warpRun Index: /branches/czw_branch/20101203/ippTools/share/warptool_setskyfiletoupdate.sql =================================================================== --- /branches/czw_branch/20101203/ippTools/share/warptool_setskyfiletoupdate.sql (revision 30586) +++ /branches/czw_branch/20101203/ippTools/share/warptool_setskyfiletoupdate.sql (revision 30587) @@ -1,11 +1,9 @@ UPDATE warpRun JOIN warpSkyfile USING(warp_id) - LEFT JOIN magicDSRun ON (stage_id = warp_id AND stage = 'warp') - LEFT JOIN magicDSFile ON (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND component = skycell_id) SET warpRun.state = 'update', - warpSkyfile.data_state = 'update' + warpSkyfile.data_state = 'update', + warpSkyfile.fault = 0 -- set hook %s WHERE warp_id = %lld AND (warpRun.state = 'cleaned' OR warpRun.state = 'update') - AND (warpSkyfile.data_state = 'cleaned') + AND (warpSkyfile.data_state = 'cleaned' OR warpSkyfile.data_state = 'update') Index: /branches/czw_branch/20101203/ippTools/share/warptool_towarped.sql =================================================================== --- /branches/czw_branch/20101203/ippTools/share/warptool_towarped.sql (revision 30586) +++ /branches/czw_branch/20101203/ippTools/share/warptool_towarped.sql (revision 30587) @@ -11,4 +11,5 @@ exp_tag, workdir, + label, magicked, path_base, @@ -23,4 +24,5 @@ warpRun.state, warpRun.reduction, + warpRun.label, camRun.cam_id, rawExp.camera, @@ -76,4 +78,5 @@ warpRun.state, warpRun.reduction, + warpRun.label, camRun.cam_id, rawExp.camera, Index: /branches/czw_branch/20101203/ippTools/src/chiptool.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/chiptool.c (revision 30586) +++ /branches/czw_branch/20101203/ippTools/src/chiptool.c (revision 30587) @@ -838,4 +838,6 @@ PXOPT_COPY_S16(config->args, where, "-fault", "chipProcessedImfile.fault", "=="); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { @@ -874,4 +876,11 @@ } psFree(query); + + if (!fault) { + // If fault has not been supplied, don't revert update faults with the magic value + // We don't do this for new runs because then they would never complete + // quality should be used to drop bad components + psStringAppend(&query_update, " AND (chipProcessedImfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); + } if (!p_psDBRunQuery(config->dbh, query_update)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1919,4 +1928,7 @@ } + // we do not update components with the magic fault value. They are non-updateable + psStringAppend(&query, " AND (chipProcessedImfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); + if (!p_psDBRunQueryF(config->dbh, query, setHook, chip_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /branches/czw_branch/20101203/ippTools/src/difftool.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/difftool.c (revision 30586) +++ /branches/czw_branch/20101203/ippTools/src/difftool.c (revision 30587) @@ -917,4 +917,6 @@ PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); @@ -950,4 +952,10 @@ psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh)); + if (!fault) { + // If fault has not been supplied, don't revert update faults with the magic value + // We don't do this for new runs because then they would never complete + // quality should be used to drop bad components + psStringAppend(&query_updated, " AND (diffSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); + } if (!p_psDBRunQuery(config->dbh, query_updated)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -3198,4 +3206,7 @@ psStringAppend(&query, " AND (diffSkyfile.skycell_id = '%s')", skycell_id); } + // we do not update components with the magic fault value. They are non-updateable + // (But can be recovered with "difftool -revertwarped -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT) + psStringAppend(&query, " AND (diffSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); if (!p_psDBRunQueryF(config->dbh, query, setHook, diff_id)) { Index: /branches/czw_branch/20101203/ippTools/src/disttool.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/disttool.c (revision 30586) +++ /branches/czw_branch/20101203/ippTools/src/disttool.c (revision 30587) @@ -1185,5 +1185,5 @@ PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "rcDSFileset.fault", "=="); Index: /branches/czw_branch/20101203/ippTools/src/disttoolConfig.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/disttoolConfig.c (revision 30586) +++ /branches/czw_branch/20101203/ippTools/src/disttoolConfig.c (revision 30587) @@ -197,5 +197,5 @@ psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-stage_id",0, "define stage_id", 0); psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-state", 0, "define state", NULL); - psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); psMetadataAddS16(revertfilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); psMetadataAddBool(revertfilesetArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); Index: /branches/czw_branch/20101203/ippTools/src/magicdstool.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/magicdstool.c (revision 30586) +++ /branches/czw_branch/20101203/ippTools/src/magicdstool.c (revision 30587) @@ -1112,6 +1112,8 @@ PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); psString queryFile = NULL; + bool stateIsUpdate = false; if (state) { if (! strcmp(state, "new") || !strcmp(state, "goto_restored")) { @@ -1119,4 +1121,5 @@ } else if (!strcmp(state, "update")) { queryFile = "magicdstool_revertupdated.sql"; + stateIsUpdate = true; } else { psError(PXTOOLS_ERR_SYS, true, "%s is not a valid value for state", state); @@ -1141,4 +1144,12 @@ } psFree(where); + + if (stateIsUpdate && !fault) { + // If fault has not been supplied, don't revert update faults with + // the magic "do not update" value + // We don't do this for new runs because then they would never complete + // quality should be used to drop bad components + psStringAppend(&query, " AND magicDSFile.fault != %d", PXTOOL_DO_NOT_REVERT_FAULT); + } if (!p_psDBRunQuery(config->dbh, query)) { @@ -1744,4 +1755,8 @@ } + // we do not update components with the magic fault value. They are non-updateable + // (But can be recovered with "magicdstool -revertdestreakedfile -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT) + psStringAppend(&query, " AND (magicDSFile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); + if (!p_psDBRunQueryF(config->dbh, query, setHook, magic_ds_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /branches/czw_branch/20101203/ippTools/src/pstamptool.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/pstamptool.c (revision 30586) +++ /branches/czw_branch/20101203/ippTools/src/pstamptool.c (revision 30587) @@ -571,4 +571,6 @@ PXOPT_COPY_S32(config->args, where, "-fault", "fault", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-reqType", "reqType", "=="); + PXOPT_COPY_STR(config->args, where, "-name", "name", "LIKE"); PXOPT_COPY_TIME(config->args, where, "-timestamp_begin", "timestamp", ">="); PXOPT_COPY_TIME(config->args, where, "-timestamp_end", "timestamp", "<="); @@ -586,5 +588,5 @@ psStringAppend(&query, ", state = '%s'", state); if (!strcmp(state, "goto_cleaned")) { - psStringAppend(&stateCheck, " AND state != 'cleaned'"); + psStringAppend(&stateCheck, " AND (state != 'cleaned' AND state != 'goto_cleaned')"); } } Index: /branches/czw_branch/20101203/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/pstamptoolConfig.c (revision 30586) +++ /branches/czw_branch/20101203/ippTools/src/pstamptoolConfig.c (revision 30587) @@ -104,4 +104,6 @@ psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "search by reqType", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "search by reqType (LIKE comparsion)", NULL); psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); psMetadataAddTime(updatereqArgs, PS_LIST_TAIL, "-timestamp_begin", 0, "search by timestamp (>=)", NULL); Index: /branches/czw_branch/20101203/ippTools/src/pxtools.h =================================================================== --- /branches/czw_branch/20101203/ippTools/src/pxtools.h (revision 30586) +++ /branches/czw_branch/20101203/ippTools/src/pxtools.h (revision 30587) @@ -51,4 +51,7 @@ # define MAX_ROWS 10e9 # define PXTOOL_MODE_NONE 0x0 +// we do not revert or update components with this fault value +// This is the same value as PSTAMP_GONE +# define PXTOOL_DO_NOT_REVERT_FAULT 26 bool pxIsValidState(const char *state); Index: /branches/czw_branch/20101203/ippTools/src/regtool.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/regtool.c (revision 30586) +++ /branches/czw_branch/20101203/ippTools/src/regtool.c (revision 30587) @@ -211,5 +211,5 @@ // convert regular class_id format to summitImfile.class_id format rep = psStringCopy(class_id); - psFree(class_id); + class_id = rep; @@ -320,5 +320,5 @@ psStringSubstitute(&query,dateobs_end,"@DATEOBS_END@"); - // fprintf(stderr,"%s",query); + fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { @@ -345,5 +345,5 @@ } - // fprintf (stderr, "found %ld rows\n", output->n); + fprintf (stderr, "found %ld rows\n", output->n); if (!psArrayLength(output)) { psTrace("regtool", PS_LOG_INFO, "no rows found"); @@ -370,5 +370,5 @@ char *tmp_id = psMetadataLookupStr(&status,row,"summit_class_id"); if (!status) { - // fprintf (stderr, "incomplete on %s\n", psMetadataLookupStr(NULL,row,"exp_name")); + fprintf (stderr, "incomplete on %s\n", psMetadataLookupStr(NULL,row,"exp_name")); continue; } @@ -440,5 +440,5 @@ already_burned = false; - // fprintf (stderr, "missing uri: %s %s\n", psMetadataLookupStr(NULL,row,"exp_name"), this_class_id); + fprintf (stderr, "missing uri: %s %s\n", psMetadataLookupStr(NULL,row,"exp_name"), this_class_id); // Save this round for next round. @@ -1903,7 +1903,7 @@ psFree(query); query = rep; - psStringSubstitute(&class_id,"ota","XY"); psStringSubstitute(&query,class_id,"@CLASS_ID@"); - + psStringSubstitute(&query,"ota","XY"); + if (!date) { if (!dateobs_begin || !dateobs_end) { Index: /branches/czw_branch/20101203/ippTools/src/warptool.c =================================================================== --- /branches/czw_branch/20101203/ippTools/src/warptool.c (revision 30586) +++ /branches/czw_branch/20101203/ippTools/src/warptool.c (revision 30587) @@ -1485,4 +1485,6 @@ PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyfile.fault", "=="); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { @@ -1510,4 +1512,11 @@ } psFree(where); + + if (!fault) { + // If fault has not been supplied, don't revert update faults with the magic value + // We don't do this for new runs because then they would never complete + // quality should be used to drop bad components + psStringAppend(&query_updated, " AND warpSkyfile.fault != %d", PXTOOL_DO_NOT_REVERT_FAULT); + } if (!p_psDBRunQuery(config->dbh, query)) { @@ -2003,5 +2012,5 @@ // shared code for the modes -tocleanedskyfile -tofullskyfile -topurgedskyfile -static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state) +static bool change_skyfile_data_state(pxConfig *config, psString data_state) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -2065,17 +2074,17 @@ static bool tocleanedskyfileMode(pxConfig *config) { - return change_skyfile_data_state(config, "cleaned", "goto_cleaned"); + return change_skyfile_data_state(config, "cleaned"); } static bool tofullskyfileMode(pxConfig *config) { - return change_skyfile_data_state(config, "full", "update"); + return change_skyfile_data_state(config, "full"); } static bool topurgedskyfileMode(pxConfig *config) { - return change_skyfile_data_state(config, "purged", "goto_purged"); + return change_skyfile_data_state(config, "purged"); } static bool toscrubbedskyfileMode(pxConfig *config) { - return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); + return change_skyfile_data_state(config, "scrubbed"); } @@ -2101,11 +2110,11 @@ else { if (strcmp(state,"error_cleaned") == 0) { - change_skyfile_data_state(config,"error_cleaned","goto_cleaned"); + change_skyfile_data_state(config,"error_cleaned"); } else if (strcmp(state, "error_scrubbed") == 0) { - change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed"); + change_skyfile_data_state(config,"error_scrubbed"); } else if (strcmp(state, "error_purged") == 0) { - change_skyfile_data_state(config,"error_purged","goto_purged"); + change_skyfile_data_state(config,"error_purged"); } else { @@ -2526,4 +2535,7 @@ psStringAppend(&query, " AND (warpSkyfile.skycell_id = '%s')", skycell_id); } + // we do not update components with the magic fault value. They are non-updateable + // (But can be recovered with "warptool -revertwarped -fault 26" (PXTOOL_DO_NOT_REVERT_FAULT) + psStringAppend(&query, " AND (warpSkyfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); if (!p_psDBRunQueryF(config->dbh, query, setHook, warp_id)) { Index: /branches/czw_branch/20101203/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/czw_branch/20101203/ippconfig/recipes/filerules-split.mdc (revision 30586) +++ /branches/czw_branch/20101203/ippconfig/recipes/filerules-split.mdc (revision 30587) @@ -341,8 +341,8 @@ CENSOR.OUTPUT OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF -MAGIC.ORIGINAL.PNG OUTPUT {OUTPUT}_original.png JPEG NONE FPA TRUE NONE -MAGIC.RESIDUAL.PNG OUTPUT {OUTPUT}_residual.png JPEG NONE FPA TRUE NONE -MAGIC.MASK.PNG OUTPUT {OUTPUT}_mask.png JPEG NONE FPA TRUE NONE -MAGIC.DUPLICATE.PNG OUTPUT {OUTPUT}_duplicate.png JPEG NONE FPA TRUE NONE +MAGIC.ORIGINAL.PNG OUTPUT {OUTPUT}.original.png JPEG NONE FPA TRUE NONE +MAGIC.RESIDUAL.PNG OUTPUT {OUTPUT}.residual.png JPEG NONE FPA TRUE NONE +MAGIC.MASK.PNG OUTPUT {OUTPUT}.mask.png JPEG NONE FPA TRUE NONE +MAGIC.DUPLICATE.PNG OUTPUT {OUTPUT}.duplicate.png JPEG NONE FPA TRUE NONE # TYPE OUTPUT FILENAME.RULE FILE.TYPE FITS.TYPE DATA.LEVEL FILE.SAVE FILE.FORMAT Index: /branches/czw_branch/20101203/magic/remove/src/streaksreplace.c =================================================================== --- /branches/czw_branch/20101203/magic/remove/src/streaksreplace.c (revision 30586) +++ /branches/czw_branch/20101203/magic/remove/src/streaksreplace.c (revision 30587) @@ -75,5 +75,5 @@ if (sfiles->inWeight) { double recWeightValue = psImageGet(sfiles->recWeight->image, x, y); - if (recWeightValue != NAN) { + if (!isnan(recWeightValue)) { psImageSet(sfiles->inWeight->image, x, y, recWeightValue); } @@ -86,4 +86,8 @@ } while (streakFilesNextExtension(sfiles)); + + // check the weight and mask files for extra extensions that might be in files + // (covariance matrix for example) + copyExtraExtensions(sfiles); // close all files @@ -235,5 +239,5 @@ usage(); } - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.MASK", 0, + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY.MASK", 0, "name of input mask image", argv[argnum]); psArgumentRemove(argnum, &argc, argv); @@ -251,5 +255,5 @@ usage(); } - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.WEIGHT", 0, + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY.WEIGHT", 0, "name of input weight image", argv[argnum]); psArgumentRemove(argnum, &argc, argv); @@ -321,5 +325,4 @@ addDestreakKeyword(sf->outMask->header, false); - // XXX: TODO setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false); @@ -333,5 +336,5 @@ sf->outWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header); addDestreakKeyword(sf->outWeight->header, false); - setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false); + setupImageRefs(sf->outWeight, NULL, sf->inWeight, sf->extnum, false); copyFitsOptions(sf->outWeight, NULL, sf->inWeight, sf->tiles); Index: /branches/czw_branch/20101203/ppTranslate/src/ppMopsMerge.c =================================================================== --- /branches/czw_branch/20101203/ppTranslate/src/ppMopsMerge.c (revision 30586) +++ /branches/czw_branch/20101203/ppTranslate/src/ppMopsMerge.c (revision 30587) @@ -35,5 +35,5 @@ ppMopsDetections *merged = NULL; // Merged list - int num = 1; // Number of merged files + int num = 0; // Number of merged files for (int i = 0; i < detections->n; i++) { ppMopsDetections *det = detections->data[i]; // Detections of interest @@ -163,4 +163,9 @@ } + if (num == 0) { + //All detections were NULL?! + psTrace("ppMops.merge", 3, "All %d detections were NULL\n", detections->n); + return NULL; + } psTrace("ppMops.merge", 2, "%ld sources in merged detections list\n", merged->num); Index: /branches/czw_branch/20101203/psModules/src/detrend/pmFringeStats.c =================================================================== --- /branches/czw_branch/20101203/psModules/src/detrend/pmFringeStats.c (revision 30586) +++ /branches/czw_branch/20101203/psModules/src/detrend/pmFringeStats.c (revision 30587) @@ -1008,16 +1008,68 @@ } } + // Allocate array of vectors to hold data. + psArray *bins = psArrayAlloc(4000); + for (int j = 0; j < bins->n; j++) { + bins->data[j] = psVectorAllocEmpty(1,PS_TYPE_F32); + // psVector *v = psVectorAllocEmpty(1,PS_TYPE_F32); + // bins = psArrayAdd(bins,1,v); + } + + // Fill vectors + pmFringeStats *fringe = fringes->data[0]; + for (int j = 0; j < numRegions; j++) { + if (mask->data.PS_TYPE_VECTOR_MASK_DATA[j] == 0) { + int array_bin = (int) ((fringe->f->data.F32[j] - -0.1) / 5e-5); + psVector *bin = bins->data[array_bin]; + psVectorAppend(bin,science->f->data.F32[j]); + } + } + psVector *fringe_positions = psVectorAllocEmpty(4000,PS_TYPE_F32); + psVector *science_values = psVectorAllocEmpty(4000,PS_TYPE_F32); + psVector *science_errors = psVectorAllocEmpty(4000,PS_TYPE_F32); + + psStats *binStats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + for (int i = 0; i < 4000; i++) { + psVector *bin = bins->data[i]; + if (bin->n > 2) { + psStatsInit(binStats); + + psVectorStats(binStats,bin,NULL,NULL,1); + + if (isfinite(binStats->robustStdev) && + isfinite(binStats->robustMedian) && + binStats->robustStdev > 0) { + psVectorAppend(fringe_positions,-0.1 + i * 5e-5); + psVectorAppend(science_values, binStats->robustMedian); + psVectorAppend(science_errors, binStats->robustStdev); + } + } + psFree(bins->data[i]); + } + psFree(bins); + psFree(binStats); + + for (int i = 0; i < fringe_positions->n; i++) { + psTrace("psModules.detrend",7,"FITDATA: %f %f %f\n", + fringe_positions->data.F32[i], + science_values->data.F32[i], + science_errors->data.F32[i]); + } /* // Begin switch from old outlier removal and fitting code. */ psPolynomial1D *poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); - pmFringeStats *fringe = fringes->data[0]; - psVector *errors = psVectorAlloc(science->df->n,PS_TYPE_F32); - for (int j = 0; j < errors->n; j++) { - errors->data.F32[j] = 1 / science->df->data.F32[j]; - } - psVectorFitPolynomial1D(poly,mask,0xff,science->f,errors,fringe->f); - + // pmFringeStats *fringe = fringes->data[0]; +/* psVector *errors = psVectorAlloc(science->df->n,PS_TYPE_F32); */ +/* for (int j = 0; j < errors->n; j++) { */ +/* errors->data.F32[j] = 1 / science->df->data.F32[j]; */ +/* } */ +/* psVectorFitPolynomial1D(poly,mask,0xff,science->f,errors,fringe->f); */ + psVectorFitPolynomial1D(poly,NULL,0xff,science_values,science_errors,fringe_positions); + psFree(fringe_positions); + psFree(science_values); + psFree(science_errors); + for (int i = 0; i <= poly->nX; i++) { scale->coeff->data.F32[i] = poly->coeff[i]; @@ -1027,5 +1079,5 @@ psFree(poly); // psFree(fringe); - psFree(errors); + // psFree(errors); psFree(median); Index: /branches/czw_branch/20101203/psphot/src/psphotEfficiency.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotEfficiency.c (revision 30586) +++ /branches/czw_branch/20101203/psphot/src/psphotEfficiency.c (revision 30587) @@ -31,6 +31,15 @@ assert(covarFactor); + // apply the amplitude of kernel^2 to the covarFactor (why?) + // XXX this is simply undoing the scale scalculate in psImageCovarianceCalculateFactor + // if this is the right solution, make this extra calculation optional (and explain...) psKernel *kernel = psImageSmoothKernel(smoothSigma, smoothNsigma); // Kernel used for smoothing - *covarFactor = psImageCovarianceCalculateFactor(kernel, ro->covariance); + double sum2 = 0.0; // Sum of kernel squared + for (int y = kernel->yMin; y <= kernel->yMax; y++) { + for (int x = kernel->xMin; x <= kernel->xMax; x++) { + sum2 += PS_SQR(kernel->kernel[y][x]); + } + } + *covarFactor = sum2 * psImageCovarianceCalculateFactor(kernel, ro->covariance); psFree(kernel); @@ -70,4 +79,5 @@ float fluxLim = 2.0 * *norm * peakLim; // Limiting flux in original *magLim = -2.5 * log10f(fluxLim); + psTrace("psphot.fake", 1, "Covar Factor: %f\n", *covarFactor); psTrace("psphot.fake", 1, "Limiting peak: %f\n", peakLim); psTrace("psphot.fake", 1, "Limiting flux: %f\n", fluxLim); Index: /branches/czw_branch/20101203/psphot/src/psphotOutput.c =================================================================== --- /branches/czw_branch/20101203/psphot/src/psphotOutput.c (revision 30586) +++ /branches/czw_branch/20101203/psphot/src/psphotOutput.c (revision 30587) @@ -307,4 +307,7 @@ psMetadataItemSupplement (&status, header, analysis, "PSPHOT.CR.MAX.MAG"); + psMetadataItemSupplement (&status, header, analysis, "EFFECTIVE_AREA"); + psMetadataItemSupplement (&status, header, analysis, "SIGNIFICANCE_SCALE_FACTOR"); + // sky background model statistics psMetadataItemSupplement (&status, header, analysis, "MSKY_MN"); Index: /branches/czw_branch/20101203/pstamp/scripts/psstatus =================================================================== --- /branches/czw_branch/20101203/pstamp/scripts/psstatus (revision 30586) +++ /branches/czw_branch/20101203/pstamp/scripts/psstatus (revision 30587) @@ -13,6 +13,6 @@ my $running; my $finished; -my $req_faulted; -my $job_faulted; +my $finishing; +my $totals; my $dbname; my $dbserver; @@ -20,5 +20,4 @@ my $dbpassword; my $verbose; -my $finishing; GetOptions( @@ -26,6 +25,5 @@ 'finished|f', \$finished, 'finishing', \$finishing, - 'req_faulted', \$req_faulted, - 'job_faulted', \$job_faulted, + 'totals|t', \$totals, 'limit|l=i', \$limit, 'dbname=s', \$dbname, @@ -35,7 +33,7 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV ; -$running = 1 if (!$finished and !$finishing and !$running); - -my $no_args = ! (defined $running or defined $finished or defined $req_faulted or defined $job_faulted ); +$running = 1 if (!$finished and !$finishing and !$running and !$totals); + +my $no_args = ! (defined $running or defined $finished or defined $finishing or defined $totals ); die "cannot supply --running and --finished\n" if ($running and $finished); @@ -44,4 +42,11 @@ my $ipprc = PS::IPP::Config->new(); my $dbh = getDBHandle(); + +my $totals_query = " +SELECT label, count(job_id) AS 'Unfinished Jobs', priority +FROM pstampRequest join pstampJob USING(req_id) +JOIN Label USING(label) +WHERE pstampJob.state ='run' and pstampRequest.state ='run' +GROUP by label"; my $running_query = " @@ -51,5 +56,5 @@ label, reqType, - priority, + -- priority, state, fault, @@ -233,5 +238,7 @@ my $sql; -if ($finished) { +if ($totals) { + $sql = $totals_query; +} elsif ($finished) { $sql = $finished_query; } elsif ($running) { Index: /branches/czw_branch/20101203/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /branches/czw_branch/20101203/pstamp/scripts/pstamp_checkdependent.pl (revision 30586) +++ /branches/czw_branch/20101203/pstamp/scripts/pstamp_checkdependent.pl (revision 30587) @@ -4,6 +4,6 @@ # # Check the status of a pending pstampDependent insuring that -# the dependent processing has been queued and whether it is -# finished or not +# the any update processing that is needed has been queued and determine check +# whether the processing has finished or not use warnings; @@ -27,6 +27,6 @@ my $IPP_DIFF_MODE_STACK_STACK = 4; -my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic); -my ($dbname, $dbserver, $verbose, $save_temps, $no_update); +my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count); +my ($dbname, $ps_dbserver, $verbose, $save_temps, $no_update); GetOptions( @@ -35,9 +35,11 @@ 'stage_id=i' => \$stage_id, 'component=s' => \$component, - 'imagedb=s' => \$imagedb, # dbname for images. + 'imagedb=s' => \$imagedb, # dbname for images lookups. 'rlabel=s' => \$rlabel, 'need_magic' => \$need_magic, + 'fault_count=i' => \$fault_count, + 'max_fault_count=i' => \$max_fault_count, 'dbname=s' => \$dbname, # postage stamp server dbname - 'dbserver=s' => \$dbserver, # postage stamp server dbserver + 'dbserver=s' => \$ps_dbserver, # postage stamp server dbserver 'verbose' => \$verbose, 'save-temps' => \$save_temps, @@ -48,4 +50,7 @@ if !(defined $dep_id and defined $stage and defined $stage_id and defined $component and defined $imagedb); + +$max_fault_count = 5 if !$max_fault_count; +$fault_count = 0 if !defined $fault_count; my $missing_tools; @@ -61,5 +66,14 @@ } +my $ipprc = PS::IPP::Config->new(); + +if (!$ps_dbserver) { + $ps_dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); +} +$pstamptool .= " -dbname $dbname" if $dbname; +$pstamptool .= " -dbserver $ps_dbserver"; + # Append imagedb to the ippTools +# Note: configured DBSERVER is used for this server $chiptool .= " -dbname $imagedb"; $warptool .= " -dbname $imagedb"; @@ -68,42 +82,17 @@ $magicdstool .= " -dbname $imagedb"; -my $ipprc = PS::IPP::Config->new(); - -if (!$dbserver) { - $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); -} - my $tool; my $cmd; my $dsRun_state = ""; -my $whole_run = ($component eq 'all'); - -# XXX: whole_run was a concept that isn't practical. Having one dependent for -# a whole run makes finding errors too hard. We always have a dependent for each -# component. -# XXX: remove the unneeded code -my_die("component = 'all' not supported", $PS_EXIT_PROG_ERROR) if $whole_run;; - -if ($whole_run) { - if ($stage eq "chip") { - $cmd = "$chiptool -listrun -chip_id $stage_id"; - } elsif ($stage eq "warp") { - $cmd = "$warptool -listrun -warp_id $stage_id"; - } elsif ($stage eq "diff") { - $cmd = "$difftool -listrun -diff_id $stage_id"; - } else { - my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR); - } + +if ($stage eq "chip") { + $cmd = "$chiptool -processedimfile -allfiles -chip_id $stage_id -class_id $component"; +} elsif ($stage eq "warp") { + $cmd = "$warptool -warped -warp_id $stage_id -skycell_id $component"; +} elsif ($stage eq "diff") { + $cmd = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $component"; } else { - if ($stage eq "chip") { - $cmd = "$chiptool -processedimfile -allfiles -chip_id $stage_id -class_id $component"; - } elsif ($stage eq "warp") { - $cmd = "$warptool -warped -warp_id $stage_id -skycell_id $component"; - } elsif ($stage eq "diff") { - $cmd = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $component"; - } else { - my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR); - } + my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR); } @@ -124,11 +113,10 @@ } my $status = 0; -if (($it->{state} eq 'full') or ($it->{state} eq 'update') and ($whole_run or ($it->{data_state} eq 'full')) +if ((($it->{state} eq 'full') or ($it->{state} eq 'update')) and ($it->{data_state} eq 'full') and (!$need_magic or $magic_ok or $it->{magicked} > 0)) { - # This Dependency is satisfied. All done! + # This Dependency is satisfied. All done. Release the pstampJobs + # my $command = "$pstamptool -updatedependent -set_state full -dep_id $dep_id"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; if (!$no_update) { my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -143,23 +131,42 @@ } elsif (($it->{state} eq 'cleaned') or ($it->{state} eq 'update')) { # For warp and diff stages we need to call the 'queue_update' subroutines even if the - # data_state is update in order to check the earlier stages in the pipeline - # For example if warpSkyfile is in update state but the chip run that it depends on hasn't - # been updated we need to go and queue it. - - if (($it->{state} ne 'cleaned') and $it->{fault}) { - my_die("Component faulted on update dep_id: $dep_id", - $PS_EXIT_SYS_ERROR); + # data_state is update in order to check the state of inputs in earlier stages in the pipeline + # For example if warpSkyfile is in update state but the chipRun that it depends on hasn't + # been updated we need to go and queue the chips for processing. + + my $fault = $it->{fault}; + if (($fault eq $PSTAMP_GONE) or (($it->{state} eq 'update') and $fault)) { + $fault_count++; + print "$stage $stage_id $component has fault $fault\n"; + if ($it->{fault} eq $PSTAMP_GONE) { + faultJobs($PSTAMP_GONE); + exit 0; + } elsif ($fault_count >= $max_fault_count) { + print "$stage $stage_id $component has faulted $fault_count times. Giving up\n"; + + faultComponent($stage, $stage_id, $component, $PSTAMP_GONE); + + # fault the jobs + faultJobs($PSTAMP_GONE); + exit 0; + } + + # assume the fault is transient. + my_die("Component faulted on update dep_id: $dep_id", $PS_EXIT_SYS_ERROR); } if ($stage eq 'chip') { - # check_states_chip takes an array so that check_states_warp can pass it set of chips + # check_states_chip takes an array so that check_states_warp can pass it a set of chips my $chips = [$it]; - $status = check_states_chip($it->{chip_id}, $whole_run, $chips, $rlabel, $need_magic); + $status = check_states_chip($it->{chip_id}, $chips, $rlabel, $need_magic); } elsif ($stage eq 'warp') { - $status = check_states_warp($it, $whole_run, $rlabel, $need_magic); + $status = check_states_warp($it, $rlabel, $need_magic); } elsif ($stage eq 'diff') { - $status = check_states_diff($it, $whole_run, $rlabel, $need_magic); + $status = check_states_diff($it, $rlabel, $need_magic); } else { my_die("Unexpected stage found $stage", $PS_EXIT_PROG_ERROR); + } + if ($status >= $PSTAMP_FIRST_ERROR_CODE) { + faultJobs($status); } } else { @@ -173,20 +180,8 @@ my $job_fault = 0; -if (0) { - if ($stage eq 'chip') { - # XXX: There is no need to check this anymore. All magicked chipRuns have abs(burntool_state) >= 13 - # If something changes that causes an error, we will figure that out from the chip failure - my $burntool_state = $it->{burntool_state}; - # XXX: get the value of 13 from the ppImage recipe - if ($burntool_state and (abs($burntool_state) < 13)) { - print STDERR "chip $it->{chip_id} $it->{class_id} has burntool_state $burntool_state. Not avaiable.\n"; - $job_fault = $PSTAMP_NOT_AVAILABLE; - } - } -} if ($state eq 'error_cleaned') { $job_fault = $PSTAMP_NOT_AVAILABLE; - } elsif (($state =~ /scrub/) or ($state =~ /purge/)) { - # jobs must have changed state since depenency was made + } elsif (($state =~ /scrub/) or ($state =~ /purge/) or ($state eq 'drop')) { + # Component state must have been changed state since dependency was inserted. print STDERR "Dependency cannot be satisfied\n"; $job_fault = $PSTAMP_GONE; @@ -195,5 +190,10 @@ my_die ("Unexpected state for ${stage}Run $stage_id $state", $PS_EXIT_PROG_ERROR); } + if (!$job_fault and ($stage eq 'chip')) { + # should only get here with data_state 'full' and perhaps destreaking not done + my_die ("Unexpected state for ${stage}Run $stage_id $state", $PS_EXIT_PROG_ERROR) + if $it->{data_state} ne 'full'; + # chip processing is done, start destreaking. my @chips; @@ -201,6 +201,7 @@ $job_fault = check_states_magicDSRun($stage, $stage_id, \@chips, $rlabel, $need_magic, $it->{raw_magicked}, $it->{magic_ds_id}, $it->{dsRun_state}); } - if ($job_fault) { - faultJobs($state, $stage, $stage_id, $job_fault); + + if ($job_fault >= $PSTAMP_FIRST_ERROR_CODE) { + faultJobs($job_fault); } } @@ -211,9 +212,7 @@ sub check_states_chip { my $chip_id = shift; - my $whole_run = shift; # if true queue entire run for update - my $metadatas = shift; # an array of hashes, either from -processedimfile or -listrun + my $metadatas = shift; # an array of hashes, either from chiptool -processedimfile or warptool -scmap my $rlabel = shift; # if defined a new label for the chipRun my $need_magic = shift; - my $dsRun_state; @@ -221,138 +220,32 @@ my @chips; my $magic_ds_id; - if (!$whole_run) { - foreach my $chip (@$metadatas) { - $dsRun_state = $chip->{dsRun_state}; - $raw_all_magicked &= ($chip->{raw_magicked} > 0); - $magic_ds_id = $chip->{magic_ds_id}; - - push @chips, $chip->{class_id}; - - my $state = $chip->{state}; - my $data_state = $chip->{data_state}; - if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or - ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/)) { - - print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state cannot update\n"; - faultJobs('stop', undef, undef, $PSTAMP_GONE); - - return 0; - } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) { - - my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id -class_id $chip->{class_id}"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } elsif ($chip->{fault}) { - # fault the dependent - my_die("chip $chip->{chip_id} $chip->{class_id} faulted: $chip->{fault}", $chip->{fault}); - } elsif ($chip->{dsFile_fault}) { - # fault the dependent - my_die("magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} faulted: $chip->{dsFile_fault}", $chip->{dsFile_fault}); - } - } - } else { - my $run = $metadatas->[0]; - $dsRun_state = $run->{dsRun_state}; - $raw_all_magicked = ($run->{raw_magicked} > 0); - my $state = $run->{state}; - if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/)) { - print "chipRun state is $run->{chip_id} is in state $state cannot update\n"; - faultJobs('stop', undef, undef, $PSTAMP_GONE); - return 0; - } - - # providing no -class_id arguments changes all imfiles with data_state = 'cleaned' to 'update' - my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } - - my $status = check_states_magicDSRun('chip', $chip_id, \@chips, $rlabel, $need_magic, $raw_all_magicked, $magic_ds_id, $dsRun_state); - - return $status; -} - -sub check_states_warp { - # check status of input chips. If state is not updatable set error code for job - - # if chipProcessedImfile.state is cleaned call check_states_chip - - my $metadata = shift; - my $whole_run = shift; # if true queue entire run for update - my $rlabel = shift; # if defined a new label for the chipRun - my $need_magic = shift; - - my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked - - my $warp_id = $metadata->{warp_id}; - my $skycell_id = $metadata->{skycell_id}; - my $state = $metadata->{state}; - my $data_state = $metadata->{data_state}; - if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or - ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/)) { - print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs\n"; - faultJobs('stop', undef, undef, $PSTAMP_GONE); - exit 0; - } - if (($state eq 'update') and ($metadata->{fault})) { - # fault dependent. - my $fault = $metadata->{fault}; - my_die("warp $warp_id $skycell_id faulted: $fault", $fault); - } - - if (!$whole_run) { - my $skycell = $metadata; - my $skycell_id = $skycell->{skycell_id}; - - my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id"; - my $data = runToolAndParse($command, $verbose); - if (!$data or scalar @$data == 0) { - # This happens if the chipProcessedImfile disappears which happened when earlier - # versions of chiptool -revertprocessedimfile didn't check the chipRun.state before - # deleing the row. - # Fault the jobs so that the Request can finish ... - faultJobs('stop', undef, undef, $PSTAMP_GONE); - # ... and fault the dependent so that we have a record of the error - my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id", - $PSTAMP_GONE); - } - - - my $chips_ready = 1; - my @chipsToUpdate; - my $chip_id; - foreach my $chip (@$data) { - $chip_id = $chip->{chip_id}; - if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { - $chips_ready = 0; - $chip->{fault} = $chip->{chip_fault}; - push @chipsToUpdate, $chip; + + my $queued_update = 0; + foreach my $chip (@$metadatas) { + $dsRun_state = $chip->{dsRun_state}; + $raw_all_magicked &= ($chip->{raw_magicked} > 0); + $magic_ds_id = $chip->{magic_ds_id}; + + push @chips, $chip->{class_id}; + + my $state = $chip->{state}; + my $data_state = $chip->{data_state}; + if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or + ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) { + + print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state cannot update\n"; + my $error_code; + if (($state eq 'error_cleaned') or ($data_state ='error_cleaned')) { + $error_code = $PSTAMP_NOT_AVAILABLE; } else { - # this chip is good to go - } - } - - if ($chips_ready and $skycell->{data_state} ne 'update') { - # the reason we defer setting the warp to update is so that we can handle error conditions at previous - # stages more easily. - my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; + $error_code = $PSTAMP_GONE; + } + + # caller will fault the jobs + return $error_code; + } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) { + + # chiptool does more state checking to insure this isn't done prematurely. + my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id -class_id $chip->{class_id}"; $command .= " -set_label $rlabel" if $rlabel; @@ -366,52 +259,129 @@ print "skipping $command\n"; } - } elsif (scalar @chipsToUpdate > 0) { - return check_states_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic); - } - } else { - my $warpRun = $metadata; - my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}"; - my $data = runToolAndParse($command, $verbose); - my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR) - if !$data or scalar @$data != 1; - - my $chipRun = $data->[0]; - - my $chipRunState = $chipRun->{state}; - if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) { - print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n"; - faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE); - return 0; - } - my $warpRunState = $warpRun->{state}; - if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) { - # The inputs and outputs are ready. Queue the warpRun for update. - - # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update' - my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } elsif ($chipRunState eq 'cleaned' or - (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) { - my $data = [$chipRun]; - return check_states_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic); - } - } - - # return value may be the return status of script so zero is good - return 0; -} + $queued_update = 1; + } elsif ($chip->{fault}) { + $fault_count++; + my $fault = $chip->{fault}; + + if ($fault eq $PSTAMP_GONE) { + # caller will fault jobs + return $PSTAMP_GONE; + } elsif ($fault_count > $max_fault_count) { + print "$stage $stage_id has faulted $fault_count times. Giving up\n"; + $fault = $PSTAMP_GONE; + faultComponent('chip', $chip->{chip_id}, $chip->{class_id}, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + # fault the dependent + my_die("chip $chip->{chip_id} $chip->{class_id} faulted: $chip->{fault}", $chip->{fault}); + } elsif ($chip->{dsFile_fault} eq $PSTAMP_GONE) { + print STDERR "magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} is GONE"; + return $PSTAMP_GONE; + } elsif ($chip->{dsFile_fault} and ($chip->{dsFile_data_state} eq 'update')) { + # fault the dependent + my_die("magicDSFile $chip->{magic_ds_id} $chip->{chip_id} $chip->{class_id} faulted: $chip->{dsFile_fault}", $chip->{dsFile_fault}); + } + } + + my $status = 0; + if (!$queued_update) { + $status = check_states_magicDSRun('chip', $chip_id, \@chips, $rlabel, $need_magic, $raw_all_magicked, $magic_ds_id, $dsRun_state); + } + + return $status; +} + +sub check_states_warp { + my $metadata = shift; + my $rlabel = shift; # if defined a new label for the chipRun + my $need_magic = shift; + + my $exit_status = 0; + + my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked + + my $warp_id = $metadata->{warp_id}; + my $skycell_id = $metadata->{skycell_id}; + my $state = $metadata->{state}; + my $data_state = $metadata->{data_state}; + + if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or + ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) { + print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs\n"; + my $error_code; + if (($state eq 'error_cleaned') or ($data_state eq 'error_cleaned')) { + $error_code = $PSTAMP_NOT_AVAILABLE; + } else { + $error_code = $PSTAMP_GONE; + } + return $error_code + } + if (($state eq 'update') and ($metadata->{fault})) { + # fault dependent. + my $fault = $metadata->{fault}; + print STDERR "warp $warp_id $skycell_id faulted: $fault"; + return $fault; + } + + my $skycell = $metadata; + + # get the list of input chips for this skycell + my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id"; + my $data = runToolAndParse($command, $verbose); + if (!$data or scalar @$data == 0) { + # This happens if the chipProcessedImfile disappears which happened when earlier + # versions of chiptool -revertprocessedimfile didn't check the chipRun.state before + # deleting the row. + print STDERR "failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id"; + return $PSTAMP_GONE; + } + + my $chips_ready = 1; + my @chipsToUpdate; + my $chip_id; + foreach my $chip (@$data) { + $chip_id = $chip->{chip_id}; + if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { + $chips_ready = 0; + $chip->{fault} = $chip->{chip_fault}; + push @chipsToUpdate, $chip; + } else { + # this chip is done + } + } + + if ($chips_ready and $skycell->{data_state} ne 'update') { + # the reason we defer setting the warp to update is so that we can handle error conditions at previous + # stages more easily. + my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } + } elsif (scalar @chipsToUpdate > 0) { + my $fault = check_states_chip($chip_id, \@chipsToUpdate, $rlabel, $need_magic); + if ($fault) { + if ($fault eq $PSTAMP_GONE) { + # chip or dsfile that this skycell depends on has faulted in a way that is not recoverable + # fault the skycell + faultComponent('warp', $warp_id, $skycell->{skycell_id}, $PSTAMP_GONE); + } + $exit_status = $fault; + } + } + + # return value may be used as the return status of script so we use zero as success + return $exit_status; +} + sub check_states_diff { my $metadata = shift; - my $whole_run = shift; # if true queue entire run for update my $rlabel = shift; # if defined a new label for the chipRun my $need_magic = shift; @@ -419,126 +389,132 @@ my $diff_id = $metadata->{diff_id}; my $diff_mode = $metadata->{diff_mode}; - if (!$whole_run) { - my $skycell = $metadata; - my $skycell_id = $skycell->{skycell_id}; - - if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) { - # check the state of the template stack - my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; - my $stack = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; - - if ($stack->{state} ne 'full') { - print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; - # this faults all jobs depending on this dep_id - faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE); - return $PSTAMP_GONE; - } - - # now check the warp - $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; - my $warp = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp; - - if ($warp->{data_state} ne 'full') { - return check_states_warp($warp, 0, $rlabel, $need_magic); - } - # warps are ready fall through and queue the diff update - } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) { - my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; - my $warp1 = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1; - - my $warps_ready = 1; - my $status = 0; - if ($warp1->{data_state} ne 'full') { - $warps_ready = 0; - $status = check_states_warp($warp1, 0, $rlabel, $need_magic); - if ($status) { - return $status; + my $skycell = $metadata; + my $skycell_id = $skycell->{skycell_id}; + + if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) { + # check the state of the template stack + my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; + my $stack = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; + + if ($stack->{state} ne 'full') { + print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + + # now check the warp + $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; + my $warp = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp; + + if ($warp->{data_state} ne 'full') { + my $warp_status = check_states_warp($warp, 0, $rlabel, $need_magic); + if ($warp_status eq $PSTAMP_GONE) { + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + } + return $warp_status; + } + # warps are ready fall through and queue the diff update + } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) { + my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id"; + my $warp1 = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1; + + my $warps_ready = 1; + my $warp_status = 0; + if ($warp1->{data_state} ne 'full') { + $warps_ready = 0; + $warp_status = check_states_warp($warp1, 0, $rlabel, $need_magic); + if ($warp_status) { + if ($warp_status eq $PSTAMP_GONE) { + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); } - } - $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id"; - my $warp2 = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2; - - if ($warp2->{data_state} ne 'full') { - $warps_ready = 0; - $status = check_states_warp($warp2, 0, $rlabel, $need_magic); - } - - if (!$warps_ready) { - # don't queue the diff update yet - return $status; - } - - } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_STACK ) { - # check the state of the input stack - my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; - my $stack1 = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1; - - if ($stack1->{state} ne 'full') { - print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; - # this faults all jobs depending on this dep_id - faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE); - return $PSTAMP_GONE; - } - # check the state of the template stack - $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; - my $stack2 = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack2; - - if ($stack2->{state} ne 'full') { - print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; - # this faults all jobs depending on this dep_id - faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE); - return $PSTAMP_GONE; - } - - # inputs are ready fall through and queue the diff update - } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_WARP ) { - # check the state of the input stack - my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}"; - my $stack = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; - - if ($stack->{state} ne 'full') { - print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; - # this faults all jobs depending on this dep_id - faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE); - return $PSTAMP_GONE; - } - - # now check the template warp - $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id"; - my $warp = runToolAndParseExpectOne($command, $verbose); - my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp; - - if ($warp->{data_state} ne 'full') { - return check_states_warp($warp, 0, $rlabel, $need_magic); - } - # warps are ready fall through and queue the diff update + return $warp_status; + } + } + $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id"; + my $warp2 = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2; + + if ($warp2->{data_state} ne 'full') { + $warps_ready = 0; + $warp_status = check_states_warp($warp2, 0, $rlabel, $need_magic); + if ($warp_status eq $PSTAMP_GONE) { + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + } + } + + if (!$warps_ready) { + # don't queue the diff update yet + return $warp_status; + } + # inputs are ready fall through and queue the diff update + + } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_STACK ) { + # check the state of the input stack + my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; + my $stack1 = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1; + + if ($stack1->{state} ne 'full') { + print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + # check the state of the template stack + $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}"; + my $stack2 = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack2; + + if ($stack2->{state} ne 'full') { + print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + + # inputs are ready fall through and queue the diff update + } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_WARP ) { + # check the state of the input stack + my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}"; + my $stack = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack; + + if ($stack->{state} ne 'full') { + print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n"; + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + + # now check the template warp + $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id"; + my $warp = runToolAndParseExpectOne($command, $verbose); + my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp; + + if ($warp->{data_state} ne 'full') { + my $warp_status = check_states_warp($warp, 0, $rlabel, $need_magic); + if ($warp_status eq $PSTAMP_GONE) { + faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE); + } + return $warp_status; + } + # warps are ready fall through and queue the diff update + } else { + my_die("unexpected diff_mode found: $diff_mode", $PS_EXIT_PROG_ERROR); + } + + if ($skycell->{data_state} ne 'update') { + my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id"; + $command .= " -set_label $rlabel" if $rlabel; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); + } } else { - my_die("unexpected diff_mode found: $diff_mode", $PS_EXIT_PROG_ERROR); - } - - if ($skycell->{data_state} ne 'update') { - my $command = "$difftool -setskyfiletoupdate -diff_id $diff_id -skycell_id $skycell_id"; - $command .= " -set_label $rlabel" if $rlabel; - - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } - } - } else { - # XXX todo whole_run - return $PS_EXIT_PROG_ERROR; + print "skipping $command\n"; + } } @@ -633,22 +609,8 @@ } -sub faultJobs { - my ($state, $stage, $stage_id, $job_fault) = @_; - - my $command = "$pstamptool -updatejob -set_state stop -set_fault $job_fault -dep_id $dep_id"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to set pstampJob.fault for dep_id: $dep_id", - $PS_EXIT_UNKNOWN_ERROR); - } - } else { - print "skipping $command\n"; - } -} - +# Check the data_state of the magicDSFile associated with this component (currently only chip stage is supported) +# Returns zero on success. +# returns the PSTAMP fault code to use for the jobs if an unrecoverable error is detected +# And faults the dependent if transient errors occur sub check_states_magicDSRun { my $stage = shift; @@ -661,5 +623,5 @@ my $dsRun_state = shift; - # XXX: this code assumes that destreaking is handled at the chip stage + # XXX: this code assumes that for update destreaking is only performed for chip stage my_die ("check_states_magicDSRun only implemented for chip stage", $PS_EXIT_PROG_ERROR) if $stage ne 'chip'; @@ -675,6 +637,7 @@ if ($need_magic and !$input_magicked) { if (!defined($dsRun_state) or ($dsRun_state eq 'NULL')) { + # it is arguably a programming error if we get here print "No magicDSRun for chipRun $stage_id and magic is required\n"; - faultJobs('stop', undef, undef, $PSTAMP_NOT_DESTREAKED); + return $PSTAMP_NOT_DESTREAKED; } elsif (($dsRun_state eq 'cleaned') or ($dsRun_state eq 'update')) { foreach my $c (@$components) { @@ -684,5 +647,17 @@ my_die("failed to find magicDSFile for ${stage}Run $stage_id $c", $PS_EXIT_UNKNOWN_ERROR); } - if ($dsfile->{fault} > 0) { + if ($dsfile->{fault} eq $PSTAMP_GONE) { + print "magicDSFile has fault $PSTAMP_GONE\n"; + return $PSTAMP_GONE; + } + # destreak faults get cleared when the component is set to be updated + if (($dsfile->{data_state} eq 'update') and ($dsfile->{fault} > 0)) { + $fault_count++; + if ($fault_count > $max_fault_count) { + print "Destreak file $magic_ds_id $component for $stage $stage_id has faulted $fault_count times. Giving up\n"; + faultComponent('destreak', $magic_ds_id, $component, $PSTAMP_GONE); + return $PSTAMP_GONE; + } + # Assume fault is transient my_die("faulted magicDSFile for ${stage}Run $stage_id $c fault: $dsfile->{fault}", $PS_EXIT_UNKNOWN_ERROR); @@ -690,5 +665,6 @@ if ($dsfile->{data_state} eq 'cleaned') { $command = "$magicdstool -setfiletoupdate -magic_ds_id $magic_ds_id -component $c"; - # XXX: get the recoveryroot from a config file (it isn't actually used except to check whether it is in nebulous) + # XXX: get the recoveryroot from a config file + # (It isn't actually used except to check whether it is a nebulous path) $command .= " -set_recoveryroot neb://any/gpc1/destreak/recover"; $command .= " -set_label $rlabel" if $rlabel; @@ -708,6 +684,7 @@ } } elsif ($dsRun_state eq 'failed_revert') { + # XXX: revert failures are rarely fixed. give up but say it's just not available not GONE print "magicDSRun.state = $dsRun_state for chipRun $stage_id is in state failed_revert cannot update\n"; - faultJobs('stop', undef, undef, $PSTAMP_NOT_AVAILABLE); + return $PSTAMP_NOT_AVAILABLE; } else { print "magicDSRun.state = $dsRun_state for chipRun $stage_id"; @@ -716,5 +693,51 @@ } } + return 0; +} + +sub faultJobs { + my ($job_fault) = @_; + + my $command = "$pstamptool -updatejob -set_state stop -set_fault $job_fault -dep_id $dep_id"; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to set pstampJob.fault for dep_id: $dep_id", + $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; + } +} + +sub faultComponent { + my ($stage, $stage_id, $component, $fault) = @_; + + my $command; + if ($stage eq 'chip') { + $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $component"; + } elsif ($stage eq 'warp') { + $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $component"; + } elsif ($stage eq 'diff') { + $command = "$difftool -updatediffskyfile -diff_id $stage_id -skycell_id $component"; + } elsif ($stage eq 'destreak') { + $command = "$magicdstool -updatedestreakedfile -magic_ds_id $stage_id -component $component"; + } else { + my_die("unexpected stage $stage found", $PS_EXIT_PROG_ERROR); + } + + $command .= " -fault $fault"; + + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + carp "$cmd failed"; + } + } else { + print "skipping $command\n"; + } } @@ -726,6 +749,4 @@ my $command = "$pstamptool -updatedependent -set_fault $fault -dep_id $dep_id"; - $command .= " -dbname $dbname" if $dbname; - $command .= " -dbserver $dbserver" if $dbserver; if (!$no_update) { my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = Index: /branches/czw_branch/20101203/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /branches/czw_branch/20101203/pstamp/scripts/pstamp_job_run.pl (revision 30586) +++ /branches/czw_branch/20101203/pstamp/scripts/pstamp_job_run.pl (revision 30587) @@ -14,5 +14,5 @@ use File::Basename; use File::Copy; -use File::Temp qw(tempfile); +use File::Temp qw(tempfile tempdir); use Digest::MD5::File qw( file_md5_hex ); use PS::IPP::PStamp::RequestFile qw( :standard ); @@ -144,6 +144,10 @@ } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) { # user required uncensored but since stage isn't chip we can't rebuild them - my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE); - } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($stage ne 'chip')) { + my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE, 'stop'); + } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} eq 'update') and ($stage ne 'chip')) { + # we can only restore pixels for chip stage images if the data has been updated. + # XXX: this test is not quite good enough. If all components have been updated then the + # state will be 'full' But this will get us going. + print "Run state is update: will make stamps from destreaked $stage images.\n"; # make stamps from uncensored images $muggle = 0; @@ -151,8 +155,6 @@ # Try and replace the streaks from the recovery images - @file_list = (); - $tmproot = "$outdir/$job_id"; - mkdir $tmproot or - my_die( "failed to create temporary directory $tmproot", $job_id, $PS_EXIT_UNKNOWN_ERROR); + my $temp_dir = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR") or &my_die("Unable to find temporary directory in site configuration", $job_id, $PS_EXIT_CONFIG_ERROR); + $tmproot = tempdir("$temp_dir/psjob.$job_id.XXXX", CLEANUP => !$save_temps); my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot"; # find the "directory" of the input path_base @@ -160,4 +162,6 @@ my $base = basename($image); $tmpImage = "$tmproot/$base"; + + @file_list = (); # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and @@ -180,5 +184,5 @@ $base = basename($variance); $tmpVariance = "$tmproot/$base"; - $newFileArgs .= " -weight $tmpVariance"; + $newFileArgs .= " -variance $tmpVariance"; my $recVariance = "$inputdir/REC_$base"; $muggle_command .= " -weight $variance -recweight $recVariance"; @@ -225,11 +229,4 @@ } - if (!$save_temps) { - unlink $tmpImage if $tmpImage; - unlink $tmpMask if $tmpMask; - unlink $tmpVariance if $tmpVariance; - rmdir $tmproot if $tmproot; - } - if ($exitStatus == 0) { my $reglist = "$outdir/reglist$job_id"; Index: /branches/czw_branch/20101203/pstamp/scripts/pstamp_server_status =================================================================== --- /branches/czw_branch/20101203/pstamp/scripts/pstamp_server_status (revision 30586) +++ /branches/czw_branch/20101203/pstamp/scripts/pstamp_server_status (revision 30587) @@ -46,9 +46,4 @@ -my $down = 0; -if ($down) { - print "Postage Stamp Server will be down for maintenance it will back at approximately 00:00 2010-12-11 UTC\n"; - exit 0; -} my $now = `date -u`; @@ -56,97 +51,119 @@ print "Status updated:     $now

\n"; -my ($pts, $pantasks_script) = tempfile ('/tmp/pts.XXXX', UNLINK => !$save_temps); +my $down = 0; +if ($down) { + print "Postage Stamp Server will be down for maintenance it will back at approximately 00:00 2010-12-11 UTC\n"; + # exit 0; +} else { + # talk to pantasks_server and get the status -print $pts "status\n"; -print $pts "quit\n"; -close $pts; + my ($pts, $pantasks_script) = tempfile ('/tmp/pts.XXXX', UNLINK => !$save_temps); -my $command = "$pantasks_client < $pantasks_script"; -my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); -unless ($success) { - $error_code = (($error_code >> 8) or 1); - if ($error_code == 12) { - #print "Postage Stamp Server is not running\n"; - print "Postage Stamp Server will be down for maintenance it will back shortly.\n"; - exit 0; + print $pts "status\n"; + print $pts "quit\n"; + close $pts; + + + my $command = "$pantasks_client < $pantasks_script"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or 1); + if ($error_code == 12) { + #print "Postage Stamp Server is not running\n"; + print "Postage Stamp Server will be down for maintenance it will back shortly.\n"; + exit 0; + } else { + warn("$command failed. exit status: $error_code"); + exit $error_code; + } } - warn("$command failed. exit status: $error_code"); - exit $error_code; -} -my $output = join "", @$stdout_buf; + my $output = join "", @$stdout_buf; -print "$output" if $verbose; + print "$output" if $verbose; -my @lines = split "\n", $output; + my @lines = split "\n", $output; -my ($scheduler_state, $controller_state); -my $got_state = 0; -my %tasks; -foreach my $line (@lines) { - chomp $line; - next if !$line; - my ($first, $second, $third); + my ($scheduler_state, $controller_state); + my $got_state = 0; + my %tasks; + foreach my $line (@lines) { + chomp $line; + next if !$line; + my ($first, $second, $third); - if ($line =~ /Scheduler is/) { - ($first, $second, $scheduler_state) = split " ", $line; - $got_state = 1; -# print "$first $second $third $scheduler_state\n"; - } elsif ($line =~ /Controller is/) { - ($first, $second, $controller_state) = split " ", $line; -# print "$first $second $third $controller_state\n"; - } elsif ($got_state == 1 and $line =~ /\+|\-/) { -# print "$line\n"; - my ($task_state, $task, $nrun, $njobs, $ngood, $nfail, $ntime, $cmd) = split " ", $line; -# print "$task $task_state\n"; - my %this_task = ( name => $task, nrun=> $nrun, njobs => $njobs, ngood => $ngood, nfail => $nfail, ntime => $ntime, cmd => $cmd ); + if ($line =~ /Scheduler is/) { + ($first, $second, $scheduler_state) = split " ", $line; + $got_state = 1; + # print "$first $second $third $scheduler_state\n"; + } elsif ($line =~ /Controller is/) { + ($first, $second, $controller_state) = split " ", $line; + # print "$first $second $third $controller_state\n"; + } elsif ($got_state == 1 and $line =~ /\+|\-/) { + # print "$line\n"; + my ($task_state, $task, $nrun, $njobs, $ngood, $nfail, $ntime, $cmd) = split " ", $line; + # print "$task $task_state\n"; + my %this_task = ( name => $task, nrun=> $nrun, njobs => $njobs, ngood => $ngood, nfail => $nfail, ntime => $ntime, cmd => $cmd ); - # change '.' in task name to '_' so we have a valid hash key - $task =~ s/\./\_/g; - $tasks{$task} = \%this_task; + # change '.' in task name to '_' so we have a valid hash key + $task =~ s/\./\_/g; + $tasks{$task} = \%this_task; + } + } + + if ($scheduler_state) { + print "Pantasks Scheduler $scheduler_state.\n"; + if ($controller_state) { + print $br . "Pantasks Controller $controller_state.\n"; + } else { + print STDERR "Controller state not found"; + exit 1; + } + print "$br$br\n"; + } else { + print STDERR "Scheduler state not found"; + exit 1; + } + + my $request_run = $tasks{'pstamp_request_run'}; + my $request_fin = $tasks{'request_finish_run'}; + my $job_run = $tasks{'pstamp_job_run'}; + + + if ($request_run) { + # print "
\n"; + print "Parser Status: $request_run->{nrun} running. $request_run->{ngood} parsed successfully.  $request_run->{nfail} failed to parse."; + if ($request_fin) { + print " $request_fin->{ngood} Requests finished."; + } else { + print "Task request.finish.run not found.\n"; + } + print "\n"; + print "\n"; + if ($job_run) { + print "
Job Status:    $job_run->{nrun} running.   $job_run->{ngood} completed successfully.   $job_run->{nfail} failed"; + print "
\n"; + } else { + print "Task pstamp.job.run not found.
\n"; + } + } else { + print "Task pstamp.request.run not found.\n"; } } -if ($scheduler_state) { - print "Pantasks Scheduler $scheduler_state.\n"; - if ($controller_state) { - print $br . "Pantasks Controller $controller_state.\n"; - } else { - print STDERR "Controller state not found"; - exit 1; - } - print "$br$br\n"; -} else { - print STDERR "Scheduler state not found"; - exit 1; -} +# now run the script psstatus to check the status of running requests and finished requests +print "
Unfinished Requests
\n"; +print "
\n";
+system "$status_cmd --totals";
+print "
\n"; -my $request_run = $tasks{'pstamp_request_run'}; -my $request_fin = $tasks{'request_finish_run'}; -my $job_run = $tasks{'pstamp_job_run'}; +#print "
\n"; -if ($request_run) { - print "
Parser Status: $request_run->{nrun} running. $request_run->{ngood} parsed successfully.  $request_run->{nfail} failed to parse."; - if ($request_fin) { - print " $request_fin->{ngood} Requests finished."; - } else { - print "Task request.finish.run not found.\n"; - } - print "\n"; - print "\n"; - if ($job_run) { - print "
Job Status:    $job_run->{nrun} running.   $job_run->{ngood} completed successfully.   $job_run->{nfail} failed"; - print "
\n"; - } else { - print "Task pstamp.job.run not found.
\n"; - } - - # now run the script psstatus to check the status of running requests and finished requests - print "
Unfinished Requests
\n"; - print "
\n";
-    system "$status_cmd --running";
-    print "
\n"; +# print "
Unfinished Requests
\n"; +print "
\n";
+system "$status_cmd --running";
+print "
\n"; if (0) { @@ -158,13 +175,8 @@ } - print "
Requests completed in last 24 hours (most recently completed first)
\n"; - print "
\n";
-    system "$status_cmd --finished";
-    print "
\n"; - -} else { - print "Task pstamp.request.run not found.\n"; -} - +print "
Requests completed in last 24 hours (most recently completed first)
\n"; +print "
\n";
+system "$status_cmd --finished";
+print "
\n"; Index: /branches/czw_branch/20101203/pstamp/scripts/pstampparse.pl =================================================================== --- /branches/czw_branch/20101203/pstamp/scripts/pstampparse.pl (revision 30586) +++ /branches/czw_branch/20101203/pstamp/scripts/pstampparse.pl (revision 30587) @@ -932,10 +932,21 @@ my $data_state = $image->{data_state}; $data_state = $run_state if $stage eq 'stack'; - if (($run_state =~ /purged/) or ($run_state =~ /scrubbed/) or ($run_state eq 'drop')) { + my $component_fault = $image->{fault}; + my $stage_id = $image->{stage_id}; + my $component = $image->{component}; + + if ($component_fault eq $PSTAMP_GONE) { # image is gone and it's not coming back + print STDERR "$stage $stage_id $component is gone\n"; + $$r_newState = 'stop'; + $$r_fault = $PSTAMP_GONE; + } elsif (($run_state =~ /purged/) or ($run_state =~ /scrubbed/) or ($run_state eq 'drop')) { + # image is gone and it's not coming back + print STDERR "$stage $stage_id $component has state: $run_state data_state: $data_state\n"; $$r_newState = 'stop'; $$r_fault = $PSTAMP_GONE; } elsif (($run_state eq "error_cleaned") or ($data_state eq 'error_cleaned')) { # if cleanup had an error don't get the user's hopes up set fault to gone + print STDERR "$stage $stage_id $component has state: $run_state data_state: $data_state\n"; $$r_newState = 'stop'; $$r_fault = $PSTAMP_GONE; Index: /branches/czw_branch/20101203/tools/log_parsing/README =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/README (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/README (revision 30587) @@ -0,0 +1,16 @@ +Nebulous apache log analysis. + +Both scripts (parse_apache_log.pl and parse_apache_log.py) do the same thing: parse an apache log file +and give some stat about the requests. The Python script is about 5 times faster + +Usually, you will use it like (bash): + cat access-log-20100204 | ./parse_apache_log.py > status.20100204 2> status.20100204.errors +or: + zcat access-log-20100204.gz | ./parse_apache_log.py > status.20100204 2> status.20100204.errors + +You can 'tail -f status.20100204.errors' to see the progression. + +Unit tests can be run using runUnitTests.sh + +Note: parsing can take a LONG time. For instance, on ippc01, parsing 100 million lines requires about 1 hour. + Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.create.pl =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.create.pl (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.create.pl (revision 30587) @@ -0,0 +1,2 @@ +#epoch create +1282187400 1 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.create.py =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.create.py (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.create.py (revision 30587) @@ -0,0 +1,2 @@ +# Epoch | Create | FindIn | Delete | FailSt | Stat | Unkn. | NonNeb +2010-08-19T03:10:00 | 1 | 0 | 0 | 0 | 0 | 0 | 0 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.delete.pl =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.delete.pl (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.delete.pl (revision 30587) @@ -0,0 +1,2 @@ +#epoch delete +1282187400 1 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.delete.py =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.delete.py (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.delete.py (revision 30587) @@ -0,0 +1,2 @@ +# Epoch | Create | FindIn | Delete | FailSt | Stat | Unkn. | NonNeb +2010-08-19T03:10:00 | 0 | 0 | 1 | 0 | 0 | 0 | 0 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.failure_stat.pl =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.failure_stat.pl (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.failure_stat.pl (revision 30587) @@ -0,0 +1,2 @@ +#epoch failure_stat +1282187400 1 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.failure_stat.py =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.failure_stat.py (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.failure_stat.py (revision 30587) @@ -0,0 +1,2 @@ +# Epoch | Create | FindIn | Delete | FailSt | Stat | Unkn. | NonNeb +2010-08-19T03:10:00 | 0 | 0 | 0 | 1 | 0 | 0 | 0 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.find_instance.pl =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.find_instance.pl (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.find_instance.pl (revision 30587) @@ -0,0 +1,2 @@ +#epoch find_instance +1282187400 1 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.find_instance.py =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.find_instance.py (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.find_instance.py (revision 30587) @@ -0,0 +1,2 @@ +# Epoch | Create | FindIn | Delete | FailSt | Stat | Unkn. | NonNeb +2010-08-19T03:10:00 | 0 | 1 | 0 | 0 | 0 | 0 | 0 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.non_nebulous.pl =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.non_nebulous.pl (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.non_nebulous.pl (revision 30587) @@ -0,0 +1,1 @@ +#epoch Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.non_nebulous.py =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.non_nebulous.py (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.non_nebulous.py (revision 30587) @@ -0,0 +1,2 @@ +# Epoch | Create | FindIn | Delete | FailSt | Stat | Unkn. | NonNeb +2010-08-19T03:10:00 | 0 | 0 | 0 | 0 | 0 | 0 | 1 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.stat.pl =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.stat.pl (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.stat.pl (revision 30587) @@ -0,0 +1,2 @@ +#epoch stat +1282187400 1 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.stat.py =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.stat.py (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.stat.py (revision 30587) @@ -0,0 +1,2 @@ +# Epoch | Create | FindIn | Delete | FailSt | Stat | Unkn. | NonNeb +2010-08-19T03:10:00 | 0 | 0 | 0 | 0 | 1 | 0 | 0 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.unknown.pl =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.unknown.pl (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.unknown.pl (revision 30587) @@ -0,0 +1,2 @@ +#epoch unknown +1282187400 1 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.unknown.py =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.unknown.py (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/expectations/log.unknown.py (revision 30587) @@ -0,0 +1,2 @@ +# Epoch | Create | FindIn | Delete | FailSt | Stat | Unkn. | NonNeb +2010-08-19T03:10:00 | 0 | 0 | 0 | 0 | 0 | 1 | 0 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.create =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.create (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.create (revision 30587) @@ -0,0 +1,1 @@ +166.122.172.32 - - [19/Aug/2010:03:12:45 -1000] "POST /nebulous HTTP/1.1" 200 631 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.delete =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.delete (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.delete (revision 30587) @@ -0,0 +1,1 @@ +166.122.172.68 - - [19/Aug/2010:03:12:27 -1000] "POST /nebulous HTTP/1.1" 200 488 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.failure_stat =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.failure_stat (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.failure_stat (revision 30587) @@ -0,0 +1,1 @@ +166.122.172.26 - - [19/Aug/2010:03:12:27 -1000] "POST /nebulous HTTP/1.1" 200 2092 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.find_instance =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.find_instance (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.find_instance (revision 30587) @@ -0,0 +1,1 @@ +166.122.172.67 - - [19/Aug/2010:03:12:45 -1000] "POST /nebulous HTTP/1.1" 200 655 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.non_nebulous =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.non_nebulous (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.non_nebulous (revision 30587) @@ -0,0 +1,1 @@ +::1 - - [19/Aug/2010:03:12:27 -1000] "OPTIONS * HTTP/1.0" 200 - Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.stat =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.stat (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.stat (revision 30587) @@ -0,0 +1,1 @@ +166.122.172.72 - - [19/Aug/2010:03:12:49 -1000] "POST /nebulous HTTP/1.1" 200 859 Index: /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.unknown =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.unknown (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/data/unittests/input/log.unknown (revision 30587) @@ -0,0 +1,1 @@ +166.122.172.47 - - [19/Aug/2010:03:13:47 -1000] "POST /nebulous HTTP/1.1" 200 420 Index: /branches/czw_branch/20101203/tools/log_parsing/parse_apache_log.pl =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/parse_apache_log.pl (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/parse_apache_log.pl (revision 30587) @@ -0,0 +1,81 @@ +#!/usr/bin/env perl + +use warnings; +use DateTime; + +%months = ('Jan' => 1, 'Feb' => 2, 'Mar' => 3, 'Apr' => 4, + 'May' => 5, 'Jun' => 6, 'Jul' => 7, 'Aug' => 8, + 'Sep' => 9, 'Oct' => 10, 'Nov' => 11, 'Dec' => 12); + +my ($ip,$day,$month,$year,$hour,$min,$sec,$bytes); + +while (<>) { + chomp; +# print STDERR "$i\n"; + if ($_ =~ /POST \/nebulous HTTP\/1.1/) { # This is a nebulous query + s/(\d+?\.\d+?\.\d+?\.\d+?) - - \[(\d\d)\/(\w\w\w)\/(\d\d\d\d)\:(\d\d)\:(\d\d)\:(\d\d).*\] "POST \/nebulous HTTP\/1.1" 200 (\d+?)/$1 $2 $3 $4 $5 $6 $7 $8/; + ($ip,$day,$month,$year,$hour,$min,$sec,$bytes) = split / /; + $month = $months{$month}; + $type = size_to_type($bytes); + count_it($type,$year,$month,$day,$hour,$min,$sec); + } +} + +print "#epoch\t"; +foreach $tag (keys %tags) { + print "$tag\t"; +} +print "\n"; +foreach $time (sort {$a <=> $b} (keys %data)) { + print "$time\t"; + foreach $tag (sort (keys %tags)) { + unless (exists($data{$time}{$tag})) { + $data{$time}{$tag} = 0; + } + print "$data{$time}{$tag}\t"; + } + print "\n"; +} + +sub size_to_type { + my $bytes = shift; + if ($bytes == 488) { # == 488 + return("delete"); + } + elsif ($bytes > 1800) { # 1577 + 3*ext_key for stat, 1597 + 3*ext_key for delete + return("failure_stat"); + } + elsif ($bytes > 850) { # 819 + ext_key for stat + return("stat"); + } + elsif ($bytes > 650) { # 600 + ext_key + return("find_instance"); + } + elsif ($bytes > 575) { # 526 + ext_key + return("create"); + } +# else { +# print "$bytes\n"; +# } + return("unknown"); +} + +sub count_it { + my $tag = shift; + my $year = shift; + my $month = shift; + my $day = shift; + my $hour = shift; + my $min = shift; + my $sec = shift; + + # Truncate to hours; + $sec = 0; + $min = int($min / 10) * 10; +# $min = 0; + my $dt = DateTime->new( year => $year, month => $month, day => $day, hour => $hour, minute => $min, second => $sec); + my $time = $dt->epoch; + + $data{$time}{$tag} += 1; + $tags{$tag} = 1; +} Index: /branches/czw_branch/20101203/tools/log_parsing/parse_apache_log.py =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/parse_apache_log.py (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/parse_apache_log.py (revision 30587) @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +import sys +import re +import datetime + +nebulousPattern = re.compile('POST /nebulous HTTP/1.1') +# Resolution: 10 minutes of 60 seconds of 1000000 microseconds +datetime.resolution = 10*60*1000000 + +def message(size): + if size == 488: + # delete == 488 + return 2 + elif size > 1800: + # failure_stat + # 1577 + 3*ext_key for stat, 1597 + 3*ext_key for delete + return 3 + elif size > 850: + # stat + # 819 + ext_key for stat + return 4 + elif size > 650: + # find_instance + # 600 + ext_key + return 1 + elif size > 575: + # create + # 526 + ext_key + return 0 + else: + # unknown + return 5 + +def show_results(counts, output=sys.stderr): + output.write('# Epoch | Create | FindIn | Delete | FailSt | Stat | Unkn. | NonNeb\n') + for count in sorted(counts.iterkeys()): + output.write('%19s | %6d | %6d | %6d | %6d | %6d | %6d | %6d\n' % (count, + counts[count][0], + counts[count][1], + counts[count][2], + counts[count][3], + counts[count][4], + counts[count][5], + counts[count][6])) + +if __name__ == '__main__': + counts = dict() + + linesCount = 1 + for line in sys.stdin: + if linesCount % 10000 == 0: + sys.stderr.write('Analyzed lines: %d\n' % linesCount) + if linesCount % 100000 == 0: + show_results(counts) + linesCount += 1 + if nebulousPattern.search(line): + elements = line.split(' ') + date = elements[3] + date = date[1:] + size = elements[-1] + # print '[%s] %s' % (date, size) + dt = datetime.datetime.strptime(date, + '%d/%b/%Y:%H:%M:%S') + dt = dt.replace(minute = int(dt.minute/10)*10, + second = 0) + try: + counts[dt.isoformat()][message(int(size))] += 1 + except KeyError: + counts[dt.isoformat()] = [0, 0, 0, 0, 0, 0, 0] + counts[dt.isoformat()][message(int(size))] += 1 + else: + elements = line.split(' ') + date = elements[3] + date = date[1:] + size = elements[-1] + # print '[%s] %s' % (date, size) + dt = datetime.datetime.strptime(date, + '%d/%b/%Y:%H:%M:%S') + dt = dt.replace(minute = int(dt.minute/10)*10, + second = 0) + try: + counts[dt.isoformat()][6] += 1 + except KeyError: + counts[dt.isoformat()] = [0, 0, 0, 0, 0, 0, 0] + counts[dt.isoformat()][6] += 1 + + show_results(counts, sys.stdout) Index: /branches/czw_branch/20101203/tools/log_parsing/runUnitTests.sh =================================================================== --- /branches/czw_branch/20101203/tools/log_parsing/runUnitTests.sh (revision 30587) +++ /branches/czw_branch/20101203/tools/log_parsing/runUnitTests.sh (revision 30587) @@ -0,0 +1,50 @@ +#!/bin/bash + +# 1) Input is processed from INPUT_DIRECTORY +# 2) Output is saved to ACTUAL_DIRECTORY +# 3) Output is compared to expectation from EXPECTATION_DIRECTORY +INPUT_DIRECTORY=data/unittests/input +ACTUAL_DIRECTORY=data/unittests/actual +EXPECTATION_DIRECTORY=data/unittests/expectations + +/bin/mkdir -p $ACTUAL_DIRECTORY +/bin/rm -f $ACTUAL_DIRECTORY/* +DIFF="/usr/bin/diff --brief" + +function performUnitTest() { + LOGFILE=$1 + TESTNAME=$2 + TESTRESULT="PASS" + cat $INPUT_DIRECTORY/$LOGFILE | ./parse_apache_log.pl > $ACTUAL_DIRECTORY/$LOGFILE.pl + $DIFF $EXPECTATION_DIRECTORY/$LOGFILE.pl $ACTUAL_DIRECTORY/$LOGFILE.pl + if [ "$?" != "0" ]; then + TESTRESULT="FAIL" + fi + cat $INPUT_DIRECTORY/$LOGFILE | ./parse_apache_log.py > $ACTUAL_DIRECTORY/$LOGFILE.py + $DIFF $EXPECTATION_DIRECTORY/$LOGFILE.py $ACTUAL_DIRECTORY/$LOGFILE.py + if [ "$?" != "0" ]; then + TESTRESULT="FAIL" + fi + echo "Test [$TESTNAME] Result: $TESTRESULT" +} + +LOGFILE=log.create +performUnitTest $LOGFILE "CREATE" + +LOGFILE=log.delete +performUnitTest $LOGFILE "DELETE" + +LOGFILE=log.failure_stat +performUnitTest $LOGFILE "FAILURE_STAT" + +LOGFILE=log.find_instance +performUnitTest $LOGFILE "FIND_INSTANCE" + +LOGFILE=log.stat +performUnitTest $LOGFILE "STAT" + +LOGFILE=log.unknown +performUnitTest $LOGFILE "Nebulous (unknown)" + +LOGFILE=log.non_nebulous +performUnitTest $LOGFILE "Non Nebulous"
RawChipCameraWarpStackWarp-Stack DiffWarp-Warp1 DiffWarp-Warp2 DiffStack-Stack Diff
Exp IDExp NameFilterCommentstatestate/dist?state/dist?state/dist?total/disttotal/disttotal/disttotal/disttotal/dist
$exp_id$exp_name$filter$comment$rawExp$chipState/$chipDistStr$camState/$camDistStr$warpState/$warpDistStr$stackDistStr$wsDistStr$ww1DistStr$ww2DistStr$ssDistStr