Index: /branches/haf_add201112/ippScripts/scripts/addstar_run.pl =================================================================== --- /branches/haf_add201112/ippScripts/scripts/addstar_run.pl (revision 33020) +++ /branches/haf_add201112/ippScripts/scripts/addstar_run.pl (revision 33021) @@ -108,6 +108,54 @@ # it was PSASTRO.OUTPUT -my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - + + +if ($stage =~ /cam/) { + # if it is cam stage we need to be careful when grabbing the filename. + # This breaks down into a few steps: + + #get info about the cam_id + my $magicked; + { + + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$camtool -processedexp -cam_id " . $stage_id; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get info on camRun", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + &my_die( "empty camRun", $add_id, $PS_EXIT_SYS_ERROR + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $mparsed = $comp->{magicked}; + + if (!defined($mparsed)) { + &my_die("Unable to parse magicked", $add_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found a value for magicked:$mparsed\n"; + $magicked = $mparsed; + } + + + + #is this cam_id magicked or not? + if ($magicked) { + my $fpaObjects = $ipprc->destreak_filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + } else { + + my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + } + + +} else { + + my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); +} if (($stage =~/staticsky/) || ($stage =~/stack/)) { $fpaObjects =~ s/smf$/cmf/; Index: /branches/haf_add201112/ippScripts/scripts/mergedvodb_merge.pl =================================================================== --- /branches/haf_add201112/ippScripts/scripts/mergedvodb_merge.pl (revision 33020) +++ /branches/haf_add201112/ippScripts/scripts/mergedvodb_merge.pl (revision 33021) @@ -41,8 +41,9 @@ } -my ( $mergedvodb_path, $merge_id, $minidvodb_path, $dbname,$camera,$verbose, $logfile, $no_op, $redirect, $save_temps); +my ( $mergedvodb_path, $merge_id, $mergedvodb, $minidvodb_path, $dbname,$camera,$verbose, $logfile, $no_op, $redirect, $save_temps); GetOptions( 'mergedvodb_path|w=s' => \$mergedvodb_path, # output DVO database 'minidvodb_path|w=s' => \$minidvodb_path, #minidvodb database + 'mergedvodb|w=s' => \$mergedvodb, #mergedvodb 'merge_id|w=s' => \$merge_id, #minidvodb_id 'dbname|d=s' => \$dbname, # Database name @@ -56,9 +57,10 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --merge_id -- minidvodb_path --mergedvodb --camera", + -msg => "Required options: --merge_id -- minidvodb_path --mergedvodb_path --mergedvodb --camera", -exitval => 3, ) unless + defined $mergedvodb_path and + defined $minidvodb_path and defined $mergedvodb and - defined $minidvodb_path and defined $merge_id and defined $camera; @@ -78,138 +80,122 @@ unless ($no_op) { - #this is chopped into several parts: firstcheck, addstar, relphot, dvoverify, merge - - #first check that there were no faulted merges before. If there are, fault this merge with PS_EXIT_UNKNOWN_ERROR - { - my $nothing_faulted = 0; - my $mdcParser = PS::IPP::Metadata::Config->new; - - my $command = "$mergetool -listmerged -faulted -mergedvodb" . $mergedvodb; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - &my_die( "Unable to get list of faulted mergedvodbs", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; - if (scalar @$stdout_buf == 0 ) { #it lists nothing if nothign has faulted - $nothing_faulted =1; - print "previous merges are okay.\n"; - } - &my_die( "Previous merges faulted, do not proceed until they are investigated:", $minidvodb_group, $PS_EXIT_UNKNOWN_ERROR) unless $nothing_faulted; - - - - } - - { - my $command = "$dvoverify -s $minidvodb_path"; - print "$command\n"; - my $mjd_dvoverify_start = DateTime->now->mjd; # MJD of starting script - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dvoverify: $error_code", $merge_id, $error_code); - } - $dtime_verify = 86400.0*(DateTime->now->mjd - $mjd_dvoverify_start); # MJD of starting script - print "dvoverify time $dtime_verify\n"; - } - - - #Merge - - my $this_is_the_first; - { - - my $mdcParser = PS::IPP::Metadata::Config->new; - - my $command = "$mergetool -listmerged -state full -mergedvodb " . $mergedvodb; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - &my_die( "Unable to get list of mergedvodbruns", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; - if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first - $this_is_the_first =1; - print "listing nothing\n"; - } else { - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config", $merge_id, $PS_EXIT_PROG_ERROR); - #this fails if there is nothing listed. I checked. - my $components = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $merge_id, $PS_EXIT_PROG_ERROR); - my $comp = $$components[0]; - my $mergedvodb_path_tmp = $comp->{mergedvodb_path}; - - if (!defined($mergedvodb_path_tmp)) { - &my_die("Unable to parse mergedvodb_path", $merge_id, $PS_EXIT_PROG_ERROR); - } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. - print "found at least 1 mergedvodbrun in merged state\n"; - $this_is_the_first = 0; - } - } - print "$this_is_the_first $mergedvodb_path/Image.dat\n"; + #this is chopped into several parts: firstcheck, addstar, relphot, dvoverify, merge + + #first check that there were no faulted merges before. If there are, fault this merge with PS_EXIT_UNKNOWN_ERROR + { + my $nothing_faulted = 0; + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$mergetool -listmerged -faulted -mergedvodb " . $mergedvodb; + $command .= " -dbname $dbname" if defined $dbname; + print "looking for faulted:\n\n$command\n\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of faulted mergedvodbs", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if nothign has faulted + $nothing_faulted =1; + print "previous merges are okay.\n"; + } + &my_die( "Previous merges faulted, do not proceed until they are investigated:", $merge_id, $PS_EXIT_UNKNOWN_ERROR) unless $nothing_faulted; + } + { + my $command = "$dvoverify -s $minidvodb_path"; + print "$command\n"; + my $mjd_dvoverify_start = DateTime->now->mjd; # MJD of starting script + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dvoverify: $error_code", $merge_id, $error_code); + } + $dtime_verify = 86400.0*(DateTime->now->mjd - $mjd_dvoverify_start); # MJD of starting script + print "dvoverify time $dtime_verify\n"; + } + #Merge + + my $this_is_the_first; + { - - if (-e "$mergedvodb_path/Image.dat") { - if ($this_is_the_first == 1) { - &my_die("refusing to merge, this is the first, but files already exist in dir", $merge_id, 4); - } - $this_is_the_first =0; - } + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$mergetool -listmerged -mergedvodb " . $mergedvodb; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of mergedvodbruns", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + $this_is_the_first =1; + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $merge_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $merge_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $mergedvodb_path_tmp = $comp->{mergedvodb_path}; + + if (!defined($mergedvodb_path_tmp)) { + &my_die("Unable to parse mergedvodb_path", $merge_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found at least 1 mergedvodbrun in merged state\n"; + $this_is_the_first = 0; + } + } + print "$this_is_the_first $mergedvodb_path/Image.dat\n"; + if (-e "$mergedvodb_path/Image.dat") { + if ($this_is_the_first == 1) { + &my_die("refusing to merge, this is the first, but files already exist in dir", $merge_id, 4); + } + $this_is_the_first =0; + } - print "$this_is_the_first $mergedvodb_path/Image.dat\n"; - { - my $merge_command; - my $mjd_merge_start = DateTime->now->mjd; # MJD of starting script - if ($this_is_the_first) { - - $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path"; - } else { - $merge_command = "$dvomerge $minidvodb_path into $mergedvodb_path"; - } - - print "\n$merge_command\n"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $merge_command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to merge: $error_code", $merge_id, $error_code); - } - - $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start); # MJD of starting script - print "merge time $dtime_merge\n"; + print "$this_is_the_first $mergedvodb_path/Image.dat\n"; + { + my $merge_command; + my $mjd_merge_start = DateTime->now->mjd; # MJD of starting script + if ($this_is_the_first) { + $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path"; + } else { + $merge_command = "$dvomerge $minidvodb_path into $mergedvodb_path"; + } + print "\n$merge_command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $merge_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to merge: $error_code", $merge_id, $error_code); + } + $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start); # MJD of starting script + print "merge time $dtime_merge\n"; # } - } - - { - - $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start); - - - my $command = "mergetool -merge_id $merge_id"; - $command .= " -addmerged"; - $command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; - $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; - $command .= " -dtime_script $dtime_script" if defined $dtime_script; - $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify; - $command .= " -dbname $dbname" if defined $dbname; - #print $command; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to add to mergedvodbprocessed: $error_code", $merge_id, $error_code); - } - } - - } else { - &my_die("dvodb: $mergedvodb_path not found", $merge_id, $PS_EXIT_UNKNOWN_ERROR); - } -} else { + } + + { + + $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start); + my $command = "mergetool -merge_id $merge_id"; + $command .= " -addmerged"; + $command .= " -mergedvodb $mergedvodb" if defined $mergedvodb; +# $command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to mergedvodbprocessed: $error_code", $merge_id, $error_code); + } + } + +#} else { +# &my_die("dvodb: $mergedvodb_path not found", $merge_id, $PS_EXIT_UNKNOWN_ERROR); +} +else { print "skipping processing for $mergedvodb_path\n"; } - + exit 0; @@ -218,15 +204,16 @@ { my $msg = shift; # Warning message on die - my $minidbvodb_id = shift; + my $merge_id = shift; my $exit_code = shift; # Exit code to add print STDERR "$msg $mergedvodb_path\n"; -if (defined $minidvodb_id ) { +if (defined $merge_id ) { my $command = "mergetool -merge_id $merge_id"; $command .= " -addmerged"; + $command .= " -mergedvodb $mergedvodb" if defined $mergedvodb; $command .= " -fault $exit_code"; - $command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; + #$command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; $command .= " -dtime_script $dtime_script" if defined $dtime_script; @@ -236,5 +223,5 @@ - #print $command; + print $command; system ($command); } Index: /branches/haf_add201112/ippScripts/scripts/mergedvodb_queue.pl =================================================================== --- /branches/haf_add201112/ippScripts/scripts/mergedvodb_queue.pl (revision 33020) +++ /branches/haf_add201112/ippScripts/scripts/mergedvodb_queue.pl (revision 33021) @@ -118,5 +118,5 @@ #create the mergedvodb entry (well, the command for it) - my $fpaCommand = "$mergetool -addmergedvodbrun"; + my $fpaCommand = "$mergetool -definebyquery"; $fpaCommand .= " -mergedvodb $mergedvodb"; $fpaCommand .= " -minidvodb_group $minidvodb_group"; @@ -125,5 +125,5 @@ print "running the following: \n\n"; - print "fpaCommand\n\n"; + print "$fpaCommand\n\n"; unless ($no_update) { Index: /branches/haf_add201112/ippTasks/mergedvodb.pro =================================================================== --- /branches/haf_add201112/ippTasks/mergedvodb.pro (revision 33020) +++ /branches/haf_add201112/ippTasks/mergedvodb.pro (revision 33021) @@ -10,12 +10,9 @@ if (not($?havemergedvodbBooks)) - book create MERGEDVODB - book create MERGEDVODB_MERGE + book create MERGEDVODB_MERGE $havemergedvodbBooks = TRUE end $MERGEDVODB_DB = 0 - - $MERGEDVODB_MERGE_DB = 0 @@ -49,16 +46,12 @@ ## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. macro add.mergedvodb - if ($0 != 5) - echo "USAGE: add.mergedvodb (mergedvodb) (minidvodb_group) (camera)" + if ($0 != 2) + echo "USAGE: add.mergedvodb (mergedvodb)" break end - - #merge book newpage MERGEDVODB_MERGE $1 book setword MERGEDVODB_MERGE $1 MERGEDVODB $1 - book setword MERGEDVODB_MERGE $1 MINIDVODB_GROUP $2 book setword MERGEDVODB_MERGE $1 STATE PENDING - end @@ -78,4 +71,7 @@ echo "mergedvodb merge" book listbook MERGEDVODB_MERGE + echo "n pages" + book npages MERGEDVODB_MERGE -var N + echo $N end @@ -95,12 +91,10 @@ book npages MERGEDVODB_MERGE -var N if ($N == 0) -# echo "No labels for processing" + echo "No labels for processing" break endif - book getpage MERGEDVODB_MERGE 0 -var mergedvodb -key STATE NEW if ("$mergedvodb" == "NULL") # All labels have been done --- reset -# echo "Resetting labels" for i 0 $N book getpage MERGEDVODB_MERGE $i -var mergedvodb @@ -113,6 +107,6 @@ if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 end - #finds the minidvodbs in a state of 'to_be_merged' - book setword MERGEVODB_MERGE $mergedvodb STATE DONE + book setword MERGEDVODB_MERGE $mergedvodb STATE DONE + $run = mergetool -pendingmerge -limit 1 $run = $run -mergedvodb $mergedvodb @@ -126,6 +120,6 @@ if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 end - add_poll_args run - + #add_poll_args run + #echo $run command $run end @@ -133,4 +127,5 @@ # success task.exit 0 + #echo "success" # convert 'stdout' to book format ipptool2book stdout mergedvodbMergelist -key merge_id -uniq -setword dbname $options:0 -setword pantaskState INIT @@ -156,7 +151,4 @@ end end - - - task mergedvodb.merge.run @@ -173,15 +165,12 @@ # if we are unable to run the 'exec', use a long retry time periods -exec $RUNEXEC - book npages mergedvodbMergelist -var N if ($N == 0) break - # look for new images in minidvodbWaitlist (pantaskState == INIT) book getpage mergedvodbMergelist 0 -var pageName -key pantaskState INIT if ("$pageName" == "NULL") break - book setword mergedvodbMergelist $pageName pantaskState RUN book getword mergedvodbMergelist $pageName merge_id -var MERGE_ID - book getword mergedvodbMergelist $pageName mergedvodb -var MERGEDVODB_GROUP + book getword mergedvodbMergelist $pageName mergedvodb -var MERGEDVODB book getword mergedvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH book getword mergedvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH @@ -189,7 +178,9 @@ stdout $LOGDIR/mergedvodb.merge.run.log stderr $LOGDIR/mergedvodb.merge.run.log - - $run = mergedvodb_merge.pl --camera GPC1 --mergedvodb_path $MERGEDVODB_PATH --minidvodb_path $MINIDVODB_PATH --merged_id $MERGE_ID - + $run = mergedvodb_merge.pl --camera GPC1 + $run = $run --mergedvodb_path $MERGEDVODB_PATH + $run = $run --minidvodb_path $MINIDVODB_PATH + $run = $run --merge_id $MERGE_ID --mergedvodb $MERGEDVODB + echo $run if ($DB:n == 0) option DEFAULT Index: /branches/haf_add201112/ippTools/share/addtool_checkminidvodbaddrun.sql =================================================================== --- /branches/haf_add201112/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 33020) +++ /branches/haf_add201112/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 33021) @@ -1,4 +1,4 @@ -- this selects the minidvodbs that have complete addRuns -SELECT minidvodb_id, minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg +SELECT minidvodb_id, minidvodb_name, minidvodb_path, state, minidbg AS minidvodb_group, (cnt - cnt2) as not_full, cnt as addRun_count FROM minidvodbRun Index: /branches/haf_add201112/ippTools/share/mergetool_find_pending_merge.sql =================================================================== --- /branches/haf_add201112/ippTools/share/mergetool_find_pending_merge.sql (revision 33020) +++ /branches/haf_add201112/ippTools/share/mergetool_find_pending_merge.sql (revision 33021) @@ -1,4 +1,4 @@ SELECT - mergedvodbRun.* + mergedvodbRun.*, minidvodbRun.minidvodb_path FROM minidvodbRun