Index: branches/eam_branches/ipp-20110710/ippScripts/Build.PL =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/Build.PL (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/Build.PL (revision 32337) @@ -118,4 +118,6 @@ scripts/diffphot.pl scripts/lap_science.pl + scripts/videophot_process.pl + scripts/listvideocells.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/automate_stacks.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/automate_stacks.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/automate_stacks.pl (revision 32337) @@ -545,5 +545,5 @@ my $mdcParser = PS::IPP::Metadata::Config->new; - my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL.STATE.GOOD | uniq"; + my $config_cmd = "$ppConfigDump -camera $camera -get-key BURNTOOL.STATE.GOOD"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run ( command => $config_cmd, verbose => $verbose); Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/camera_exp.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/camera_exp.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/camera_exp.pl (revision 32337) @@ -155,5 +155,6 @@ { # Determine if FWHM is too large to bother continuing. - my $command = "$ppConfigDump -camera $camera -dump-camera -"; + my $maxFWHM = 0; + my $command = "$ppConfigDump -camera $camera -get-key MAX_ALLOWED_FWHM"; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -162,8 +163,9 @@ &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_CONFIG_ERROR); } - my $cameraConfig = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR); - - my $maxFWHM = metadataLookupStr($cameraConfig, 'MAX_ALLOWED_FWHM'); + if (@$stdout_buf) { + my $cameraConfig = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_CONFIG_ERROR); + $maxFWHM = metadataLookupStr($cameraConfig, 'MAX_ALLOWED_FWHM'); + } if ($maxFWHM) { my $expFWHM; Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/chip_imfile.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/chip_imfile.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/chip_imfile.pl (revision 32337) @@ -44,4 +44,7 @@ my ( $exp_id, $chip_id, $class_id, $chip_imfile_id, $uri, $camera, $outroot, $dbname, $run_state, $reduction, $threads, $verbose, $no_update, $save_temps, $no_op, $redirect, $magicked, $deburned ); + +my $zaplog = 0; + GetOptions( 'exp_id=s' => \$exp_id, # Exposure identifier @@ -62,4 +65,5 @@ 'no-op' => \$no_op, # Don't do any operations? 'redirect-output' => \$redirect, + 'zaplog' => \$zaplog, 'save-temps' => \$save_temps, # Save temporary files? ) or pod2usage( 2 ); @@ -90,5 +94,5 @@ my ($logDest, $traceDest); if ($run_state eq 'new') { - $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, 0); + $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, $zaplog); $traceDest = prepare_output("TRACE.IMFILE", $outroot, $class_id, 1); } else { @@ -115,5 +119,5 @@ my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -&my_die("Couldn't find input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); +# &my_die("Couldn't find input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); # outroot examples (HOST components must be set) @@ -262,7 +266,7 @@ } - ## Read camera config to get the current good burntool state : - ## XXX This is extremely slow. Any better way to do this? - my $camera_config_cmd = "$ppConfigDump -camera $camera -dump-camera -"; + ## Read camera config to get the current good burntool state + ## this returns just the values of interest in a tiny mdc file + my $camera_config_cmd = "$ppConfigDump -camera $camera -get-key BURNTOOL.STATE.GOOD -get-key BURNTOOL.STATE.GOOD.UPDATE"; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $camera_config_cmd, verbose => 0); Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/diff_skycell.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/diff_skycell.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/diff_skycell.pl (revision 32337) @@ -40,4 +40,6 @@ my ($skycell_id, $diff_skyfile_id); my ($use_convolved); +my $zaplog = 0; + GetOptions( 'diff_id=s' => \$diff_id, # Diff identifier @@ -55,4 +57,5 @@ 'no-op' => \$no_op, # Don't do any operations? 'redirect-output' => \$redirect, + 'zaplog' => \$zaplog, # delete any previous log file ) or pod2usage( 2 ); @@ -82,4 +85,6 @@ $logDest .= '.update'; # ... well we can just do the dirty work here + $ipprc->kill_file($logDest); +} elsif ($zaplog) { $ipprc->kill_file($logDest); } Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/ipp_apply_burntool.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/ipp_apply_burntool.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/ipp_apply_burntool.pl (revision 32337) @@ -70,5 +70,5 @@ # Determine the value of a "good" burntool run. -my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL | uniq"; +my $config_cmd = "$ppConfigDump -camera $camera -get-key BURNTOOL.STATE.GOOD"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run ( command => $config_cmd, verbose => $verbose); Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/ipp_apply_burntool_single.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/ipp_apply_burntool_single.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/ipp_apply_burntool_single.pl (revision 32337) @@ -74,5 +74,5 @@ # Determine the value of a "good" burntool run. -my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL | uniq"; +my $config_cmd = "$ppConfigDump -camera $camera -get-key BURNTOOL.STATE.GOOD"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run ( command => $config_cmd, verbose => $verbose); Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/ipp_cleanup.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/ipp_cleanup.pl (revision 32337) @@ -172,4 +172,5 @@ addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1); addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1); + addFilename (\@files, "PPIMAGE.PATTERN", $path_base, $class_id, 0); if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/lap_science.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/lap_science.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/lap_science.pl (revision 32337) @@ -1,4 +1,3 @@ #!/usr/bin/env perl - use warnings; @@ -29,4 +28,9 @@ my ( $lap_id ); my ( $chip_mode, $monitor_mode, $cleanup_mode); + +# +# Global configuration constants that probably should be read from elsewhere. +my $qstack_threshold = 0.05; # Only make a quickstack if more than 5% of the exposures require it. +my $minimum_stack_inputs = 6; # We can avoid magicking stack inputs if we have more than this number. GetOptions( @@ -255,9 +259,9 @@ $exposure->{pair_id} = 9223372036854775807; update_this_exposure($exposure); + $exposure = remake_this_exposure($exposure); $counter++; next; } } - if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id @@ -280,5 +284,7 @@ $counter++; } - print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; + if ($verbose) { + print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; + } } @@ -319,5 +325,7 @@ # Scan all exposures, and ensure that pairwise and private are set correctly foreach my $exposure (@$exposures) { - print "YY: $exposure\n"; + if ($verbose) { + print "YY: $exposure\n"; + } if ($exposure->{pairwise} && !($exposure->{pair_id})) { $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. @@ -448,13 +456,17 @@ # Things I want to know before I'm through - my $needs_qstack = 0; +# my $needs_qstack = 0; my $needs_something_remade = 0; my $needs_something_private = 0; - my $can_qstack = 0; - my $have_diff = 0; - my $can_diff = 0; - my $can_fstack = 0; +# my $can_qstack = 0; + my $have_diff = 0; + my $can_diff = 0; +# my $can_fstack = 0; my $total_exposures = 0; - my $lonely_exposures = 0; + my @lonely_exposures = (); + + my $are_warped = 0; + my $are_magicked = 0; + foreach my $exposure (@$exposures) { $total_exposures++; @@ -472,7 +484,18 @@ } } + + if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. + $are_warped++; +# $can_qstack ++; + $can_diff ++; + $have_diff ++; + $are_magicked ++; +# $can_fstack ++; + next; + } if ($exposure->{private}) { # I've declared this exposure private to this lapRun. - $needs_qstack = 1; +# $needs_qstack++; + push @lonely_exposures, $exposure; # $lonely_exposures++; } @@ -512,5 +535,6 @@ unless ((defined($exposure->{diffRun_state}))&& ($exposure->{diffRun_state} eq 'full')) { - $needs_qstack = 1; + $are_warped++; +# $needs_qstack++; $needs_something_private = 1; if ($companion) { @@ -518,4 +542,5 @@ $companion->{pairwise} = 0; &update_this_exposure($companion); + push @lonely_exposures, $companion; # $lonely_exposures++; } @@ -525,23 +550,17 @@ $exposure->{data_state} = 'drop'; &update_this_exposure($exposure); - } - if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. - $can_qstack ++; - $can_diff ++; - $have_diff ++; - $can_fstack ++; - next; - } if (($exposure->{warpRun_state})&& # This exposure has a warp ($exposure->{warpRun_state} eq 'full')) { # This exposure's warp is done. - $can_qstack ++; + $are_warped++; +# $can_qstack ++; $can_diff ++; } if (($exposure->{magicked}&& &S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. - $can_fstack ++; + $are_magicked++; +# $can_fstack ++; } if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) { @@ -556,11 +575,13 @@ print "STATUS: HAVE_FSTACK: $have_fstack\n"; - print "STATUS: NEEDS_QSTACK: $needs_qstack\n"; +# print "STATUS: NEEDS_QSTACK: $needs_qstack\n"; print "STATUS: NEEDS_REMADE: $needs_something_remade\n"; print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n"; - print "STATUS: CAN_QSTACK: $can_qstack\n"; + print "STATUS: ARE WARPED: $are_warped\n"; + print "STATUS: ARE MAGICKED: $are_magicked\n"; +# print "STATUS: CAN_QSTACK: $can_qstack\n"; print "STATUS: CAN_DIFF: $can_diff\n"; print "STATUS: HAVE_DIFF: $have_diff\n"; - print "STATUS: CAN_FSTACK: $can_fstack\n"; +# print "STATUS: CAN_FSTACK: $can_fstack\n"; print "STATUS: TOTAL_EXPOSURES: $total_exposures\n"; @@ -569,42 +590,73 @@ exit(0); } - if (($needs_qstack == 1)&& # Do we need the quick stack? - ($defined_qstack == 0)&& # Have we not made it already? - ($can_qstack == $total_exposures)&& # Are all warps done? - ($needs_something_remade == 0) # Do we need to wait for a stolen exposure? - ) { - print "STATUS: Will now queue quickstacks\n"; - &queue_quickstack($exposures); - } - - if (($can_diff == $total_exposures)&& # Are all warps done? - ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? - ($have_diff < $can_diff)&& # We haven't already done them all. - (($needs_qstack == 0)|| # Are we just doing pairwise diffs? - ($needs_qstack == 1)&&($have_qstack == 1)) ) { # If we need a quickstack, do we have it? - print "STATUS: Will now queue diffs\n"; - &queue_diffs($exposures); - } - - if (($can_fstack == $total_exposures)&& # Are all warps done? - ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? - ($defined_fstack == 0)) { # Have we not made it already? - print "STATUS: Will now queue final stacks\n"; - &queue_finalstack($exposures); - } - - if (($have_fstack == 1)) { # Are we all done making the final product? - print "STATUS: Will now deactivate exposures for this complete lapRun.\n"; - &deactivate_exposures($exposures); - $command = "$laptool -updaterun -lap_id $lap_id"; - $command .= " -dbname $dbname " if defined $dbname; - $command .= " -set_state full "; - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); - } - } + + # Do the decision making based on what we've learned + if ($are_warped == $total_exposures) { # Are all the warps done? + if ($defined_fstack == 0) { # We have not yet made any stacks. + print "STATUS: We can attempt to queue deep final stacks.\n"; + my $Nstacks_made = &queue_muggle_finalstack($exposures); + print "STATUS: Made $Nstacks_made final stacks.\n"; + exit($Nstacks_made); + # If this doesn't do anything, we may end up stuck here. Therefore, we need a more robust test. + } + else { # We have made a stack before for this lapRun. + if ($have_fstack == $defined_fstack) { # We have made all the stacks we have asked for (so far) + if ($are_magicked < $total_exposures) { # But we have not yet made diffs + if (($can_diff == $total_exposures)&& # And we have exposures that should be diffed. + ($have_diff < $can_diff)) { # and we haven't made them all yet, either. + print "STATUS: Dropping unpaired exposures and making pairwise diffs.\n"; +# foreach my $exposure (@lonely_exposures) { +# $exposure->{data_state} = 'drop'; +# &update_this_exposure($exposure); +# } + &queue_diffs($exposures); + exit(0); + } + else { + # This is apparently the case where we've queued diffs to run, but they have not yet + # completed though the destreak phase. + print "STATUS: Waiting for diffs/magic/destreak to complete.\n"; + exit(0); + } + } + elsif ($are_magicked == $total_exposures) { # We have made all of the diffs, and propagated through magic + print "STATUS: We can create all remaining stacks possible.\n"; + my $Nstacks_made = &queue_magic_finalstack($exposures); + if ($Nstacks_made == 0) { # We have made all the stacks that we will ever make. + print "STATUS: No more stacks generated. Deactivating exposures for this complete lapRun.\n"; + &deactivate_exposures($exposures); + $command = "$laptool -updaterun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_state full "; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + exit(0); + } + else { + print "STATUS: $Nstacks_made stacks made.\n"; + exit(0); + } + } + else { # What? + print "STATUS: More exposures magicked than exist. This is a mistake.\n"; + exit(0); + } + } + else { # We have not yet made all the stacks we've asked for. + print "STATUS: Defined $defined_fstack, completed $have_fstack. Waiting for stacking to finish.\n"; + exit(0); + } + } + } + else { # Not all warps finished. + # do nothing + print "STATUS: Not all warps are finished, waiting for that to finish.\n"; + exit(0); + } + print "\n"; return(0); @@ -626,5 +678,5 @@ my $warps = ''; foreach $exposure (@$exposures) { - if (($exposure->{data_state} != 'drop')&& + if (($exposure->{data_state} ne 'drop')&& (S64_IS_NOT_NULL($exposure->{warp_id}))) { $warps .= " -warp_id $exposure->{warp_id} "; @@ -638,10 +690,11 @@ my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; - my $data_group = "${label}.${proj_cell}.quick.${date}"; + my $data_group = "${label}.${proj_cell}.quick.${lap_id}"; my $command = "$stacktool "; $command .= " -pretend " if defined $debug; $command .= " -dbname $dbname " if defined $dbname; - $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; + $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; +# $command .= " -select_label $label "; # Removed to allow exposure sharing to work $command .= " -set_label ${label} -set_data_group $data_group "; $command .= " -set_workdir $workdir -set_dist_group NODIST "; @@ -655,4 +708,6 @@ &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); } + my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); $command = "$stacktool "; @@ -688,7 +743,7 @@ } - -} -sub queue_finalstack { + return($#{ $stacks_made } + 1); +} +sub queue_muggle_finalstack { my $exposures = shift; # reference to exposure array; my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information @@ -705,6 +760,6 @@ my $warps = ''; foreach $exposure (@$exposures) { - if (($exposure->{data_state} != 'drop')&& - (S64_IS_NOT_NULL($exposure->{magicked}))&& + if (($exposure->{data_state} ne 'drop')&& +# (S64_IS_NOT_NULL($exposure->{magicked}))&& (S64_IS_NOT_NULL($exposure->{warp_id}))) { $warps .= " -warp_id $exposure->{warp_id} "; @@ -718,12 +773,13 @@ my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; - my $data_group = "${label}.${proj_cell}.final.${date}"; + my $data_group = "${label}.${proj_cell}.final.${lap_id}"; my $command = "$stacktool "; $command .= " -pretend " if defined $debug; $command .= " -dbname $dbname " if defined $dbname; - $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; + $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; +# $command .= " -select_label $label "; # Removed to allow exposure sharing to work $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group "; - $command .= " -min_num 2 -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} "; + $command .= " -min_num ${minimum_stack_inputs} -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} "; $command .= " $warps "; @@ -734,5 +790,6 @@ &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); } - + my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); $command = "$stacktool "; @@ -767,4 +824,85 @@ } + return($#{ $stacks_made } + 1); +} +sub queue_magic_finalstack { + my $exposures = shift; # reference to exposure array; + my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information + + my $lap_id = $exposure->{lap_id}; + my $label = $exposure->{label}; + my $filter = $exposure->{filter}; + my $proj_cell = $exposure->{projection_cell}; + + unless (defined($label) && defined($filter) && defined($proj_cell)) { + &my_die("Unable to perform stacktool. Insufficient information.", $lap_id); + } + + my $warps = ''; + foreach $exposure (@$exposures) { + if (($exposure->{data_state} ne 'drop')&& + (S64_IS_NOT_NULL($exposure->{magicked}))&& + (S64_IS_NOT_NULL($exposure->{warp_id}))) { + $warps .= " -warp_id $exposure->{warp_id} "; + } + } + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${proj_cell}.final.${lap_id}"; + + my $command = "$stacktool "; + $command .= " -pretend " if defined $debug; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; +# $command .= " -select_label $label "; # Removed to allow exposure sharing to work + $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group "; + $command .= " -min_num 2 -min_new 2 -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} "; + $command .= " $warps "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); + } + my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); + + $command = "$stacktool "; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -sassskyfile -data_group $data_group "; + $command .= " -filter $filter -projection_cell ${proj_cell} "; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -sassskyfile: $error_code", $lap_id); + } + + my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stacktool -sassskyfile", $lap_id, ""); + + my $stack = ${ $stacks }[0]; + my $sass_id = $stack->{sass_id}; + unless (defined($sass_id)) { + &my_die("Unable to parse metadata from stacktool for sass_id", $lap_id, ""); + } + print "FINAL_SASS_ID: $sass_id\n"; + $command = "$laptool -updaterun -lap_id $lap_id -set_final_sass_id $sass_id"; + $command .= " -dbname $dbname " if defined $dbname; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + return($#{ $stacks_made } + 1); } sub queue_diffs { @@ -817,5 +955,5 @@ my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; $command .= " -definewarpwarp "; - $command .= "-input_label $label -template_label $label "; +# $command .= "-input_label $label -template_label $label "; $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} "; $retry_command = $command; @@ -826,7 +964,9 @@ } else { # warp-qstack - $command .= " -definewarpstack -set_reduction NOCONVDIFF -available -good_frac 0.2 "; - $command .= " -warp_id $exposure->{warp_id} -stack_label ${label}.quick "; - $already_queued{$exposure->{warp_id}} = 1; +# next; +# $command .= '-pretend'; + $command .= " -definewarpstack -available -good_frac 0.2 "; + $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} "; + $already_queued{$exposure->{warp_id}} = 1; } @@ -1060,5 +1200,6 @@ $command .= " -nopairwise "; } - if ($exposure->{active}) { +# if ($exposure->{active}) { + unless ($exposure->{active} == 0) { $command .= " -active "; } Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/listvideocells.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/listvideocells.pl (revision 32337) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/listvideocells.pl (revision 32337) @@ -0,0 +1,71 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $fhead = can_run('fhead') or (warn "Can't find fhead" and $missing_tools = 1); + +my ($uri, $verbose); + +# Parse the command-line arguments +GetOptions( + 'file=s' => \$uri, # uri to examine + 'verbose' => \$verbose, # Print stuff? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --file", + -exitval => 3) unless + defined $uri; + +my $num_ext = 0; +my $num_video_ext = 0; +for (my $i = 0; ; $i++) { + my $command = "$fhead -x $i $uri | grep EXTNAME"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + if ($error_code != 1) { + carp("Unable to perform $command: $error_code"); + exit $error_code; + } + last; + } + $num_ext++; + + my $output = join "", @$stdout_buf; + chomp $output; + last if ! $output; + + my ($exttag, undef, $extname) = split " ", $output; + + last if !$exttag; + + die ("unexpected output from fhead: $output") unless $exttag eq 'EXTNAME'; + + if ($extname =~ /video_table/) { + $num_video_ext++; + # get rid of the quotes in the ftable output + $extname =~ s/\'//g; + my (undef, undef, $cell) = split "_", $extname; + print "$cell\n"; + } +} + +die "no extensions found in $uri" unless $num_ext > 0; +print "no video extensions found in $uri" unless $num_video_ext > 0; +exit 0; Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/magic_destreak_revert.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/magic_destreak_revert.pl (revision 32337) @@ -419,5 +419,5 @@ my $o_basename = basename($o_path); my $b_basename = basename($b_path); - if (($o_basename =~ /SR_/) and !($b_basename =~ /SR_/)) { + if (!($o_basename =~ /SR_/) and ($b_basename =~ /SR_/)) { print " basenames are as expected it is safe to revert\n"; } else { Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/nightly_science.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/nightly_science.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/nightly_science.pl (revision 32337) @@ -521,5 +521,5 @@ my $mdcParser = PS::IPP::Metadata::Config->new; - my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL.STATE.GOOD | uniq"; + my $config_cmd = "$ppConfigDump -camera $camera -get-key BURNTOOL.STATE.GOOD"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run ( command => $config_cmd, verbose => $verbose); Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/register_imfile.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/register_imfile.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/register_imfile.pl (revision 32337) @@ -147,5 +147,5 @@ elsif (($isGPC1 == 1) && ($burntoolStateCurrent == 1)) { # print STDERR "In the good region: >>$burntoolStateCurrent<<\n"; - my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -"; + my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -get-key BURNTOOL.STATE.GOOD"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = IPC::Cmd::run(command => $ppConfigDump_cmd, verbose => $verbose); @@ -522,4 +522,5 @@ $command .= " -class_id $class_id"; $command .= " -fault $exit_code"; + $command .= " -set_state pending_burntool "; $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/videophot_process.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/videophot_process.pl (revision 32337) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/videophot_process.pl (revision 32337) @@ -0,0 +1,177 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $vptool = can_run('vptool') or (warn "Can't find vptool" and $missing_tools = 1); +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $psvideophot = can_run('psvideophot') or (warn "Can't find psvideophot" and $missing_tools = 1); +my $listvideocells = can_run('listvideocells.pl') or (warn "Can't find listvideocells.pl" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} +my ($vp_id, $exp_id, $camera, $outroot, $dbname, $verbose, $no_update, $no_op, $redirect); + +GetOptions( + 'vp_id=s' => \$vp_id, + 'exp_id=s' => \$exp_id, + 'camera=s' => \$camera, + 'outroot=s' => \$outroot, + 'dbname|d=s' => \$dbname, + 'verbose' => \$verbose, + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, # redirect output streams to logfile +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --vp_id --exp_id --outroot --camera", + -exitval => 3, + ) unless defined $vp_id + and defined $exp_id + and defined $camera + and defined $outroot; + + +$no_update = 1 if $no_op; + +my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $vp_id, 'nochip', 'nocell', $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot); +$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $vp_id, 'nochip', 'nocell', $PS_EXIT_SYS_ERROR ) if $redirect; + +my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF); + +# Get list of components for subtraction +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ +# my $command = "$vptool -pendingimfile -vp_id $vp_id"; +# $command .= " -dbname $dbname" if defined $dbname; + my $command = "$regtool -processedimfile -video_cells -exp_id $exp_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, 'nochip', 'nocell', $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $vp_id, 'nochip', 'nocell', $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $vp_id, 'nochip', 'nocell', $PS_EXIT_PROG_ERROR); +} + +foreach my $file (@$files) { + my $class_id = $file->{class_id}; + my $uri = $file->{uri}; + + my $resolved = $ipprc->file_resolve($uri); + &my_die("Unable to resolve $uri", $vp_id, $class_id, 'nocell', $PS_EXIT_UNKNOWN_ERROR) unless $resolved; + + my $command = "$listvideocells --file $resolved"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $vp_id, $class_id, + 'nocell', $error_code); + } + + my @video_cells = split "\n", (join "", @$stdout_buf); + &my_die("No video cells found in $uri", $vp_id, $class_id, + 'nocell', $PS_EXIT_UNKNOWN_ERROR) unless scalar @video_cells; + + foreach my $cell_id (@video_cells) { + my $command = "$psvideophot $outroot"; + $command .= " -file $uri"; + $command .= " -cell_id $cell_id"; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppSub: $error_code", $vp_id, $class_id, $cell_id, $error_code); + } + + # check_output($outputStats, 1); + } else { + print "Not executing: $command\n"; + } + unless ($no_update) { + my $command = "$vptool -addvpcell -vp_id $vp_id -class_id $class_id -cell_id $cell_id"; + $command .= " -path_base $outroot"; + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = "Unable to perform vptool -addvpcell"; + &my_die("$err_message: $error_code", $vp_id, $class_id, $cell_id, $error_code); + } + } + } +} + +exit 0; + +sub my_die +{ + my $msg = shift; # Warning message on die + my $vp_id = shift; + my $class_id = shift; # chip identifier + my $cell_id = shift; # chip identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $vp_id and defined $class_id and not $no_update) { + my $command = "$vptool -vp_id $vp_id -class_id $class_id -cell_id -fault $exit_code"; + $command .= " -addvpcell"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: branches/eam_branches/ipp-20110710/ippScripts/scripts/warp_skycell.pl =================================================================== --- branches/eam_branches/ipp-20110710/ippScripts/scripts/warp_skycell.pl (revision 31852) +++ branches/eam_branches/ipp-20110710/ippScripts/scripts/warp_skycell.pl (revision 32337) @@ -38,4 +38,6 @@ my ($warp_id, $skycell_id, $warp_skyfile_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $run_state, $magicked, $verbose, $no_update, $no_op, $redirect, $save_temps); +my $zaplog = 0; + GetOptions( 'warp_id|i=s' => \$warp_id, # Warp identifier @@ -53,4 +55,5 @@ 'no-update' => \$no_update, # Don't update the database? 'no-op' => \$no_op, # Don't do any operations? + 'zaplog' => \$zaplog, # delete previous logfile 'redirect-output' => \$redirect, 'save-temps' => \$save_temps, # Save temporary files? @@ -74,5 +77,5 @@ my $do_stats; if ($run_state eq 'new') { - $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, 0); + $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, $zaplog); $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1); $do_stats = 1;