Changeset 33030 for trunk

Timestamp:

Dec 27, 2011, 3:09:17 PM (14 years ago)

Author:

heather

Message:

crap. I don't know why it thinks some of these are modified, but diffing them
against a fresh checkout of the trunk shows no differences.

These are the merge of addstar changes into the trunk. This adds tables,
breaking apart minidvodbRun into 2 parts (building/relphot/addstar) and
(verify/merging), which will make it easier to rebuild dvodbs in the
future. There's some debugging of various tasks here (addstar.revert, for
example), and a lot of cleanup of code.

Location:

trunk

Files:

• . (modified) (1 prop)
• Ohana/src/libdvo/src/dvo_util.c (modified) (1 diff)
• dbconfig/mergedvodb.md (copied) (copied from branches/haf_add201112/dbconfig/mergedvodb.md )
• dbconfig/minidvodb.md (modified) (2 diffs)
• extperl (modified) (1 prop)
• ippMonitor (modified) (1 prop)
• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/scripts/addstar_run.pl (modified) (9 diffs)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (1 prop)
• ippScripts/scripts/mergedvodb_copy.pl (copied) (copied from branches/haf_add201112/ippScripts/scripts/mergedvodb_copy.pl )
• ippScripts/scripts/mergedvodb_merge.pl (copied) (copied from branches/haf_add201112/ippScripts/scripts/mergedvodb_merge.pl )
• ippScripts/scripts/mergedvodb_queue.pl (copied) (copied from branches/haf_add201112/ippScripts/scripts/mergedvodb_queue.pl )
• ippScripts/scripts/minidvodb_createdb.pl (modified) (1 diff)
• ippScripts/scripts/minidvodb_premerge.pl (copied) (copied from branches/haf_add201112/ippScripts/scripts/minidvodb_premerge.pl )
• ippTasks/Makefile.am (modified) (1 diff)
• ippTasks/addstar.pro (modified) (7 diffs)
• ippTasks/mergedvodb.pro (copied) (copied from branches/haf_add201112/ippTasks/mergedvodb.pro )
• ippTasks/minidvodb.pro (modified) (17 diffs)
• ippTasks/survey.pro (modified) (8 diffs)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/addtool_checkminidvodbaddrun.sql (modified) (1 diff)
• ippTools/share/addtool_find_minidvodbrun2.sql (modified) (1 diff)
• ippTools/share/addtool_queue_minidvodbrun.sql (modified) (1 diff)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/chiptool_setimfiletoupdate.sql (modified) (1 prop)
• ippTools/share/mergetool_find_merge_id_dvo.sql (copied) (copied from branches/haf_add201112/ippTools/share/mergetool_find_merge_id_dvo.sql )
• ippTools/share/mergetool_find_mergedvodbcopy.sql (copied) (copied from branches/haf_add201112/ippTools/share/mergetool_find_mergedvodbcopy.sql )
• ippTools/share/mergetool_find_minidvodb_id.sql (copied) (copied from branches/haf_add201112/ippTools/share/mergetool_find_minidvodb_id.sql )
• ippTools/share/mergetool_find_pending_merge.sql (copied) (copied from branches/haf_add201112/ippTools/share/mergetool_find_pending_merge.sql )
• ippTools/share/mergetool_find_pendingmergeprocess.sql (copied) (copied from branches/haf_add201112/ippTools/share/mergetool_find_pendingmergeprocess.sql )
• ippTools/share/mergetool_find_processed.sql (copied) (copied from branches/haf_add201112/ippTools/share/mergetool_find_processed.sql )
• ippTools/share/mergetool_queue_merge_id.sql (copied) (copied from branches/haf_add201112/ippTools/share/mergetool_queue_merge_id.sql )
• ippTools/share/mergetool_queue_mergecopy_id.sql (copied) (copied from branches/haf_add201112/ippTools/share/mergetool_queue_mergecopy_id.sql )
• ippTools/share/mergetool_queue_minidvodb_id.sql (copied) (copied from branches/haf_add201112/ippTools/share/mergetool_queue_minidvodb_id.sql )
• ippTools/share/mergetool_revertmergedvodbcopy.sql (copied) (copied from branches/haf_add201112/ippTools/share/mergetool_revertmergedvodbcopy.sql )
• ippTools/share/mergetool_revertmergedvodbprocessed.sql (copied) (copied from branches/haf_add201112/ippTools/share/mergetool_revertmergedvodbprocessed.sql )
• ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (3 diffs, 1 prop)
• ippTools/share/warptool_towarped.sql (modified) (1 prop)
• ippTools/src (modified) (2 props)
• ippTools/src/addtool.c (modified) (21 diffs)
• ippTools/src/addtoolConfig.c (modified) (5 diffs)
• ippTools/src/magictool.c (modified) (1 prop)
• ippTools/src/mergetool.c (copied) (copied from branches/haf_add201112/ippTools/src/mergetool.c )
• ippTools/src/mergetool.h (copied) (copied from branches/haf_add201112/ippTools/src/mergetool.h )
• ippTools/src/mergetoolConfig.c (copied) (copied from branches/haf_add201112/ippTools/src/mergetoolConfig.c )
• ippTools/src/minidvodbtool.c (modified) (1 diff)
• ippTools/src/pxmerge.c (copied) (copied from branches/haf_add201112/ippTools/src/pxmerge.c )
• ippTools/src/pxmerge.h (copied) (copied from branches/haf_add201112/ippTools/src/pxmerge.h )
• ippconfig (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 prop)
• ppImage/src (modified) (1 prop)
• psLib/src/fits/psFitsTableNew.c (modified) (1 diff)
• psModules (modified) (1 prop)
• psphot (modified) (1 prop)
• psphot/src (modified) (1 prop)
• psphot/src/psphotStackImageLoop.c (modified) (1 prop)
• pstamp/scripts (modified) (1 prop)
• psvideophot (modified) (1 prop)

trunk/Ohana/src/libdvo/src/dvo_util.c

@@ -276 +276 @@
   FREE(detections);
 
-  if (NInvalidDetID) fprintf (stderr, "ERROR: Encountered %ld invalid detection IDs\n", NInvalidDetID);
+  if (NInvalidDetID) fprintf (stderr, "ERROR: Encountered %zx invalid detection IDs\n", NInvalidDetID);
 
   return (Ndetect);

trunk/dbconfig/minidvodb.md

@@ -4 +4 @@
     minidvodb_group STR    64
     minidvodb_path  STR    255
-    mergedvodb_path STR    255
     state           STR    64
     creation_date   UTC    0001-01-01T00:00:00Z
@@ -11 +10 @@
 minidvodbProcessed METADATA
     minidvodb_id    S64    0
-    merge_order     S64    0
     dtime_resort    F32    0.0
     dtime_relphot   F32    0.0
-    dtime_verify    F32    0.0
-    dtime_merge     F32    0.0
     dtime_script    F32    0.0
     epoch           UTC    0001-01-01T00:00:00Z
-    mergedvodb_path STR    255
     fault           S16    0
 END

trunk/extperl

@@ -31 +31 @@
 Digest-SHA1-2.11
 ExtUtils-MakeMaker-6.31
+ExtUtils-MakeMaker-6.54
 ExtUtils-Manifest-1.51
 File-ExtAttr-1.04
@@ -66 +67 @@
 Params-Validate-0.92
 Parse-RecDescent-1.94
+Pod-Simple-3.14
+Pod-Escapes-1.04
 SOAP-Lite-0.69
 SQL-Interp-1.06

@@ -31 +31 @@
 Digest-SHA1-2.11
 ExtUtils-MakeMaker-6.31
+ExtUtils-MakeMaker-6.54
 ExtUtils-Manifest-1.51
 File-ExtAttr-1.04
@@ -66 +67 @@
 Params-Validate-0.92
 Parse-RecDescent-1.94
+Pod-Simple-3.14
+Pod-Escapes-1.04
 SOAP-Lite-0.69
 SQL-Interp-1.06

trunk/ippScripts/Build.PL

@@ -65 +65 @@
         scripts/minidvodb_createdb.pl
         scripts/minidvodb_merge.pl
+        scripts/minidvodb_premerge.pl
         scripts/minidvodb_copy.pl
+        scripts/mergedvodb_copy.pl
+        scripts/mergedvodb_merge.pl
+        scripts/mergedvodb_queue.pl
         scripts/ippdb.pl
         scripts/ipp_cleanup.pl

trunk/ippScripts/scripts/addstar_run.pl

@@ -30 +30 @@
 my $missing_tools;
 my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1);
+my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1);
@@ -37 +38 @@
 }
 my $minidvodb_path;
-my ( $add_id, $camera, $stage, $multi_num, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update,
+my $stage_extra1;
+my ( $add_id, $camera, $stage, $stage_id, $multi_num, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update,
      $no_op, $redirect, $save_temps);
 GetOptions(
@@ -43 +45 @@
     'camera|c=s'        => \$camera, # Camera
     'stage|s=s'        => \$stage, # Camera
+    'stage_id|w=s'   => \$stage_id,
     'multi_num|w=s'    => \$multi_num, # the number for a staticskymulti (for finding cmf)
     'dbname|d=s'        => \$dbname, # Database name
@@ -71 +74 @@
     defined $dvodb and
     defined $camera;
+if ($stage =~ /cam/ && !defined $stage_id) {
+    my_die("cam stage requires -stage_id", $add_id, 3);
+
+}
 if ($minidvodb && !defined($minidvodb_group)) {
                 my_die( "missing minidvodb_group", $add_id, 3 );
@@ -108 +115 @@
 
 # it was PSASTRO.OUTPUT
-my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",     $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR);
-
+
+my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR);
+
+if ($stage =~ /cam/) {
+    # if it is cam stage we need to be careful when grabbing the filename. 
+    # This breaks down into a few steps: 
+    
+    #get info about the cam_id 
+    my $magicked;
+    {
+        my $mdcParser = PS::IPP::Metadata::Config->new;
+        my $command = "$camtool -processedexp -cam_id " . $stage_id;
+        $command .= " -dbname $dbname" if defined $dbname;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        &my_die( "Unable to get info on camRun", $add_id, $PS_EXIT_SYS_ERROR) unless $success;
+        if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first
+            &my_die( "empty camRun", $add_id, $PS_EXIT_SYS_ERROR);
+            print "listing nothing\n";
+        } else {
+            my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+                &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR);
+            #this fails if there is nothing listed. I checked.
+            my $components = parse_md_list($metadata) or
+                &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
+            my $comp = $$components[0];
+            my  $mparsed = $comp->{magicked};
+            
+            if (!defined($mparsed)) {
+                &my_die("Unable to parse magicked", $add_id, $PS_EXIT_PROG_ERROR);
+            } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk.
+            print "found a value for magicked:$mparsed\n";
+            $magicked = $mparsed;
+        }
+        
+        #is this cam_id magicked or not?
+        if ($magicked) {
+            $stage_extra1 = $magicked;
+            $fpaObjects = $ipprc->destreaked_filename("$fpaObjects") or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR);
+            print "cam_id is magicked, using $fpaObjects for the cam smf\n";
+        } else {
+            print "cam_id is NOT magicked, using $fpaObjects for the cam smf\n";
+        }
+    }
+}
 if (($stage =~/staticsky/) || ($stage =~/stack/)) {
     $fpaObjects =~ s/smf$/cmf/;
-    
-}
-
+}
 if ($stage =~/staticsky_multi/) {
     $fpaObjects =~ s/smf$/cmf/; #this should do nothing because it was already modded above
@@ -120 +168 @@
     my $nice_num = sprintf ("%03d", $multi_num);
     $fpaObjects =~ s/cmf$/$nice_num.cmf/;  #this make it look for .001.cmf, etc
- 
 }
 
 my $traceDest  = $ipprc->filename("TRACE.EXP",          $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR);
-
+        
 # convert supplied DVO database name to UNIX filename
 my $dvodbReal;
@@ -131 +178 @@
     $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $add_id, $PS_EXIT_CONFIG_ERROR);
 }
-
-
-
-
 my $dtime_addstar = 0;
-
 if (defined $dvodbReal) {
-        if ($minidvodb) {
-
-            #if (!defined($minidvodb_name)) {
-                #take the active one, if it's not defined on the command line
-                #reverts would have this already set, for example.
-                my $command = "addtool -listminidvodbrun ";
-                $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group;
-                $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
-                $command .= " -state 'active' -limit 1";
-                $command .= " -dbname $dbname" if defined $dbname;
-                print $command;
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-                &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success;
-                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-                    &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR);
-
-                my $components = parse_md_list($metadata) or
-                    &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
-                my $comp = $$components[0];
-                $minidvodb_path = $comp->{minidvodb_path};
-                $minidvodb_name = $comp->{minidvodb_name};
-
-                if (!defined($minidvodb_path)) {
-                    &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR);
-                }
-                if (!defined($minidvodb_name)) {
-                    &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR);
-                }
-            #}
-            # tack on the minidvodb part to the db.
-#           $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/';
-#we don't need this now that I fixed the paths
-
-        }
-        else {
-
-            $minidvodb_path = $dvodbReal;
-        }
-
-unless ($no_op) {
+    if ($minidvodb) {
+        my $command = "addtool -listminidvodbrun ";
+        $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group;
+        $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
+        $command .= " -state 'active' -limit 1";
+        $command .= " -dbname $dbname" if defined $dbname;
+        print $command;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success;
+        my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR);
+
+        my $components = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
+        my $comp = $$components[0];
+        $minidvodb_path = $comp->{minidvodb_path};
+        $minidvodb_name = $comp->{minidvodb_name};
+        
+        if (!defined($minidvodb_path)) {
+            &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR);
+        }
+        if (!defined($minidvodb_name)) {
+            &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR);
+        }
+    }
+    else {
+        $minidvodb_path = $dvodbReal;
+    }
+    
+    unless ($no_op) {
             print $dvodbReal;
-
-        ## addstar can either save the full set of detections, or just
-        ## the image metadata, in the dvodb.  this is set in the
-        ## database table addRun
-
-        # addstar requires the user to have a valid .ptolemyrc which
-        # in turn points at ippconfig/dvo.site
-
-        # get the names for the camera and the real input file
-        my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar
-        my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR);
-
-        # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default)
-        my $command  = "$addstar -update"; # XXX optionally set -update?
-        $command .= " -D CAMERA $camdir";
-        $command .= " -D CATDIR $minidvodb_path";
-        $command .= " $realFile";
-        $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference
+            
+            ## addstar can either save the full set of detections, or just
+            ## the image metadata, in the dvodb.  this is set in the
+            ## database table addRun
+
+            # addstar requires the user to have a valid .ptolemyrc which
+            # in turn points at ippconfig/dvo.site
+
+            # get the names for the camera and the real input file
+            my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar
+            my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR);
+            
+            # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default)
+            my $command  = "$addstar -update"; # XXX optionally set -update?
+            $command .= " -D CAMERA $camdir";
+            $command .= " -D CATDIR $minidvodb_path";
+            $command .= " $realFile";
+            $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference
             $command .= " -image" if $image_only;
             if ($stage =~ /staticsky/) {
                 $command .= " -accept-astrom ";
             }  #careful here - this matches staticsky and staticsky_multi
-        my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
-
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform addstar: $error_code", $add_id, $error_code);
-        }
-        $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);   # MJD of starting script
+            my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform addstar: $error_code", $add_id, $error_code);
+            }
+            $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);   # MJD of starting script
     }
 }
@@ -221 +253 @@
 $fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path;
 $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
+$fpaCommand .= " -stage_extra1" if defined $stage_extra1;
 $fpaCommand .= " -dbname $dbname" if defined $dbname;
 print $fpaCommand;
@@ -253 +286 @@
         $command .= " -path_base $outroot" if defined $outroot;
         $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400));
-        $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
+   #     $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; don't think we want it recorded (not sure)
+        $command .= " -stage_extra1 $stage_extra1" if defined $stage_extra1;
         $command .= " -dbname $dbname" if defined $dbname;
+        print $command;
         system ($command);
     }

trunk/ippScripts/scripts/minidvodb_createdb.pl

@@ -226 +226 @@
     $fpaCommand .= " -set_minidvodb_group $minidvodb_group";
     $fpaCommand .= " -set_minidvodb_path  $minidvodbReal" if defined $minidvodbReal;
-    $fpaCommand .= " -set_mergedvodb_path $dvodbReal";
     $fpaCommand .= " -dbname $dbname" if defined $dbname;
 

trunk/ippTasks/Makefile.am

@@ -38 +38 @@
         minidvodb.pro \
         minidvodbcopy.pro \
+        mergedvodb.pro \
         nightly_science.pro \
         burntool.pro \

trunk/ippTasks/addstar.pro

@@ -53 +53 @@
 
 macro addstar.revert.off
-  task addstar.revert
+  task addstar.revert.cam
+    active false
+  end
+ task addstar.revert.stack
+    active false
+  end
+ task addstar.revert.staticsky
+    active false
+  end
+ task addstar.revert.staticsky_multi
     active false
   end
 end
 macro addstar.revert.on
-  task addstar.revert
+  task addstar.revert.cam
+    active true
+  end
+  task addstar.revert.stack
+    active true
+  end
+  task addstar.revert.staticsky_multi
+    active true
+  end
+  task addstar.revert.staticsky
     active true
   end
@@ -65 +83 @@
 
 # this variable will cycle through the known database names
-$addstar_DB = 0
-$addstar_revert_DB = 0
+$addstar_DB = 0 
+#this may not work for more databases (addstar) will do that later)
+$addstar_revert_DB_C = 0
+$addstar_revert_DB_S = 0
+$addstar_revert_DB_SS = 0
+$addstar_revert_DB_SSM = 0
 
 
@@ -318 +340 @@
     book getword addPendingExp $pageName stage_extra1 -var STAGE_EXTRA1
     book getword addPendingExp $pageName stageroot -var STAGEROOT
-    book getword addPendingExp $pageName stage -var STAGE  
+    book getword addPendingExp $pageName stage -var STAGE
+    book getword addPendingExp $pageName stage_id -var STAGE_ID
     book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE
     book getword addPendingExp $pageName reduction -var REDUCTION
@@ -367 +390 @@
     if ("$STAGE" == "staticsky_multi")
       $run = $run --multi_num $STAGE_EXTRA1
+    end
+    if ("$STAGE" == "cam") 
+      $run = $run --stage_id $STAGE_ID
     end
     if ("$IMAGE_ONLY" == "T")
@@ -411 +437 @@
 end
 
-task addstar.revert
+task addstar.revert.cam
   host         local
 
@@ -425 +451 @@
   task.exec
     if ($LABEL:n == 0) break
-    $run = addtool -revertprocessedexp
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$addstar_revert_DB
-      $run = $run -dbname $DB:$addstar_revert_DB
-      $addstar_revert_DB ++
-      if ($addstar_revert_DB >= $DB:n) set addstar_revert_DB = 0
+    $run = addtool -revertprocessedexp -stage cam
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$addstar_revert_DB_C
+      $run = $run -dbname $DB:$addstar_revert_DB_C
+      $addstar_revert_DB_C ++
+      if ($addstar_revert_DB_C >= $DB:n) set addstar_revert_DB_C = 0
     end
     add_poll_labels run
@@ -457 +483 @@
   end
 end
+task addstar.revert.stack
+  host         local
+
+  periods      -poll 5.0
+  periods      -exec 60.0
+  periods      -timeout 1200
+  npending     1
+  active        false
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = addtool -revertprocessedexp -stage stack
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$addstar_revert_DB_S
+      $run = $run -dbname $DB:$addstar_revert_DB_S
+      $addstar_revert_DB_S ++
+      if ($addstar_revert_DB_S >= $DB:n) set addstar_revert_DB_S = 0
+    end
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+task addstar.revert.staticsky
+  host         local
+
+  periods      -poll 5.0
+  periods      -exec 60.0
+  periods      -timeout 1200
+  npending     1
+  active        false
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = addtool -revertprocessedexp -stage staticsky
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$addstar_revert_DB_SS
+      $run = $run -dbname $DB:$addstar_revert_DB_SS
+      $addstar_revert_DB_SS ++
+      if ($addstar_revert_DB_SS >= $DB:n) set addstar_revert_DB_SS = 0
+    end
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+task addstar.revert.staticsky_multi
+  host         local
+
+  periods      -poll 5.0
+  periods      -exec 60.0
+  periods      -timeout 1200
+  npending     1
+  active        false
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = addtool -revertprocessedexp -stage staticsky_multi
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$addstar_revert_DB_SSM
+      $run = $run -dbname $DB:$addstar_revert_DB_SSM
+      $addstar_revert_DB_SSM ++
+      if ($addstar_revert_DB_SSM >= $DB:n) set addstar_revert_DB_SSM = 0
+    end
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end

trunk/ippTasks/minidvodb.pro

@@ -11 +11 @@
 if (not($?haveminidvodbBooks))
  book create MINIDVODB
- book create MINIDVODB_MERGE
+ book create MINIDVODB_PREMERGE
  book create MINIDVODB_CREATE
  book create MINIDVODB_ACTIVE   
@@ -20 +20 @@
 
 $MINIDVODB_WAIT_DB = 0
-$MINIDVODB_MERGE_DB = 0
+$MINIDVODB_PREMERGE_DB = 0
 $MINIDVODB_CREATE_DB = 0
 $MINIDVODB_ACTIVE_DB = 0
 
 book init minidvodbWaitlist
-book init minidvodbMergelist
+book init minidvodbPreMergelist
 book init minidvodbCreatelist
 book init minidvodbActivelist
@@ -54 +54 @@
 end
 
-macro minidvodb.merge.status
-  book listbook minidvodbMergelist
-end
-
-macro minidvodb.merge.reset
-  book init minidvodbMergelist
+macro minidvodb.premerge.status
+  book listbook minidvodbPreMergelist
+end
+
+macro minidvodb.premerge.reset
+  book init minidvodbPreMergelist
 end
 
@@ -112 +112 @@
 ##these merge the dbs
 
-macro minidvodb.merge.on
-  task minidvodb.merge.load
+macro minidvodb.premerge.on
+  task minidvodb.premerge.load
     active true
   end
-  task minidvodb.merge.run
+  task minidvodb.premerge.run
     active true
   end  
 end
-macro minidvodb.merge.off
-  task minidvodb.merge.load
+macro minidvodb.premerge.off
+  task minidvodb.premerge.load
     active false
   end
-  task minidvodb.merge.run
+  task minidvodb.premerge.run
     active false
   end  
@@ -142 +142 @@
   book setword MINIDVODB $1 STATE PENDING
   #merge  
-  book newpage MINIDVODB_MERGE $1
-  book setword MINIDVODB_MERGE $1 MINIDVODB_GROUP $1
-  book setword MINIDVODB_MERGE $1 DVODB $3
-  book setword MINIDVODB_MERGE $1 STATE PENDING
+  book newpage MINIDVODB_PREMERGE $1
+  book setword MINIDVODB_PREMERGE $1 MINIDVODB_GROUP $1
+  book setword MINIDVODB_PREMERGE $1 DVODB $3
+  book setword MINIDVODB_PREMERGE $1 STATE PENDING
   #active  
   book newpage MINIDVODB_ACTIVE $1
@@ -168 +168 @@
   end
   book delpage MINIDVODB $1
-  book delpage MINIDVODB_MERGE $1
+  book delpage MINIDVODB_PREMERGE $1
   book delpage MINIDVODB_CREATE $1  
   book delpage MINIDVODB_ACTIVE $1
@@ -180 +180 @@
   echo "minidvodb wait"
   book listbook MINIDVODB
-  echo "minidvodb merge"
-  book listbook MINIDVODB_MERGE
+  echo "minidvodb premerge"
+  book listbook MINIDVODB_PREMERGE
   echo "minidvodb create"
   book listbook MINIDVODB_CREATE 
@@ -339 +339 @@
 
 
-task           minidvodb.merge.load
+task           minidvodb.premerge.load
   host         local
 
@@ -348 +348 @@
 
   stdout NULL
-  stderr $LOGDIR/minidvodb.merge.load.log
+  stderr $LOGDIR/minidvodb.premerge.load.log
 
   task.exec
-    book npages MINIDVODB_MERGE -var N
+    book npages MINIDVODB_PREMERGE -var N
     if ($N == 0)
 #      echo "No labels for processing"
@@ -357 +357 @@
     endif
 
-    book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW
+    book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW
     if ("$minidvodb_group" == "NULL")
       # All labels have been done --- reset
 #      echo "Resetting labels"
       for i 0 $N
-        book getpage MINIDVODB_MERGE $i -var minidvodb_group
-        book setword MINIDVODB_MERGE $minidvodb_group STATE NEW
+        book getpage MINIDVODB_PREMERGE $i -var minidvodb_group
+        book setword MINIDVODB_PREMERGE $minidvodb_group STATE NEW
       end
-      book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW
+      book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW
 
       # Select different database
@@ -372 +372 @@
     end
     #finds the minidvodbs in a state of 'to_be_merged' 
-    book setword MINIDVODB_MERGE $minidvodb_group STATE DONE
+    book setword MINIDVODB_PREMERGE $minidvodb_group STATE DONE
     $run = addtool -listminidvodbrun -state to_be_merged -limit 1
     $run = $run -minidvodb_group $minidvodb_group
@@ -392 +392 @@
   task.exit    0
     # convert 'stdout' to book format
-    ipptool2book stdout minidvodbMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    ipptool2book stdout minidvodbPreMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     if ($VERBOSE > 2)
-      book listbook minidvodbMergelist
+      book listbook minidvodbPreMergelist
     end
     # delete existing entries in the appropriate pantaskStates
-    process_cleanup minidvodbMergelist
+    process_cleanup minidvodbPreMergelist
   end
 
@@ -418 +418 @@
 
 
-task           minidvodb.merge.run
+task           minidvodb.premerge.run
   periods      -poll $RUNPOLL
   periods      -exec $RUNEXEC
@@ -432 +432 @@
     periods -exec $RUNEXEC
 
-    book npages minidvodbMergelist -var N
+    book npages minidvodbPreMergelist -var N
     if ($N == 0) break
     
     # look for new images in minidvodbWaitlist (pantaskState == INIT)
-    book getpage minidvodbMergelist 0 -var pageName -key pantaskState INIT
+    book getpage minidvodbPreMergelist 0 -var pageName -key pantaskState INIT
     if ("$pageName" == "NULL") break
 
-    book setword minidvodbMergelist $pageName pantaskState RUN
-    book getword minidvodbMergelist $pageName minidvodb_id -var MINIDVODB_ID
-    book getword minidvodbMergelist $pageName minidvodb_group -var MINIDVODB_GROUP
-    book getword minidvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH
-    book getword minidvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH
-    book getword minidvodbMergelist $pageName state -var STATE
-    stdout $LOGDIR/minidvodb.merge.run.log
-    stderr $LOGDIR/minidvodb.merge.run.log
-
-    $run = minidvodb_merge.pl --camera GPC1 --mergedvodb $MERGEDVODB_PATH --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID
+    book setword minidvodbPreMergelist $pageName pantaskState RUN
+    book getword minidvodbPreMergelist $pageName minidvodb_id -var MINIDVODB_ID
+    book getword minidvodbPreMergelist $pageName minidvodb_group -var MINIDVODB_GROUP
+    book getword minidvodbPreMergelist $pageName minidvodb_path -var MINIDVODB_PATH
+    book getword minidvodbPreMergelist $pageName state -var STATE
+    stdout $LOGDIR/minidvodb.premerge.run.log
+    stderr $LOGDIR/minidvodb.premerge.run.log
+
+    $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID
     
   if ($DB:n == 0)
@@ -472 +471 @@
   # default exit status
     task.exit    default
-    process_exit minidvodbMergelist $options:0 $JOB_STATUS
+    process_exit minidvodbPreMergelist $options:0 $JOB_STATUS
   end
 
@@ -482 +481 @@
     # Set a fault code in the database
     
-    process_exit minidvodbMergelist $options:0 $EXIT_CRASH_ERR
+    process_exit minidvodbPreMergelist $options:0 $EXIT_CRASH_ERR
   end
 
@@ -488 +487 @@
     task.exit    timeout
         showcommand timeout
-    book setword minidvodbMergelist $options:0 pantaskState TIMEOUT
+    book setword minidvodbPreMergelist $options:0 pantaskState TIMEOUT
   end
 end

trunk/ippTasks/survey.pro

@@ -3 +3 @@
 # test for required global variables
 check.globals
-
-# there is a bug in 'getbook', use a different method below
-# book getbook SURVEY_DIFF -var isBook
-# if ("$isBook" != "SURVEY_DIFF")
-#   book create SURVEY_DIFF
-# end
-# book getbook SURVEY_MAGIC -var isBook
-# if ("$isBook" == "NULL")
-#   book create SURVEY_MAGIC
-# end
-# book getbook SURVEY_DESTREAK -var isBook
-# if ("$isBook" == "NULL")
-#   book create SURVEY_DESTREAK
-# end
-# book getbook SURVEY_DIST -var isBook
-# if ("$isBook" == "NULL")
-#   book create SURVEY_DIST
-# end
 
 if (not($?haveSurveyBooks))
@@ -30 +12 @@
  book create SURVEY_DIST
  book create SURVEY_ADDSTAR   
+ book create SURVEY_MERGEDVODB
  book create SURVEY_CHIP_BG
  book create SURVEY_WARP_BG
@@ -44 +27 @@
 $SURVEY_DIST_DB = 0
 $SURVEY_ADDSTAR_DB = 0
+$SURVEY_MERGEDVODB_DB = 0
 $SURVEY_CHIP_BG_DB = 0
 $SURVEY_WARP_BG_DB = 0
@@ -68 +52 @@
     active true
   end
+  task survey.mergedvodb
+    active true
+  end
   task survey.destreak
     active true
@@ -104 +91 @@
     active false
   end
+  task survey.mergedvodb
+    active false
+  end  
   task survey.destreak
     active false
@@ -270 +260 @@
   book listbook SURVEY_ADDSTAR
 end
+
+macro survey.add.mergedvodb
+  if ($0 != 4)
+    echo "USAGE: survey.add.mergedvodb (tag) (mergedvodb) (minidvodb_group)"
+    break
+  end
+  book newpage SURVEY_MERGEDVODB $1
+  book setword SURVEY_MERGEDVODB $1 MERGEDVODB $2
+  book setword SURVEY_MERGEDVODB $1 MINIDVODB_GROUP $3
+end
+
+macro survey.del.mergedvodb
+  if ($0 != 2)
+    echo "USAGE: survey.del.mergedvodb (tag)"
+    break
+  end
+  book delpage SURVEY_MERGEDVODB $1
+end
+
+macro survey.show.mergedvodb
+  if ($0 != 1)
+    echo "USAGE: survey.show.mergedvodb"
+    break
+  end
+  book listbook SURVEY_MERGEDVODB
+end
+
 
 # user functions to manipulate destreak labels
@@ -826 +843 @@
     if ("$stage" == "cam") 
         #only queue destreaked cams. stacks and staticsky don't need this
-        if ("$DB:$SURVEY_ADDSTAR_DB" == "isp")
+    #    if ("$DB:$SURVEY_ADDSTAR_DB" == "isp")
             #this is the only way I can think of how to handle this (but it is messy). If it is a database that we KNOW doesn't use magicked (ie, isp), then do not queue destreaked
+           # now with no magic, all are uncensored
            $run = $run -uncensored
-        else
+#       else
            #if not isp, then run destreaked version only
-           $run = $run -destreaked
-        end
+#          $run = $run -destreaked
+#       end
     end
     if ("$stage" == "staticsky") 
@@ -875 +893 @@
   end
 end
+
+
+task survey.mergedvodb
+  host local
+ 
+  periods      -poll $SURVEY_POLL
+  periods      -exec $SURVEY_EXEC
+  periods      -timeout $SURVEY_TIMEOUT
+  npending     1
+
+  stdout $LOGDIR/survey.mergedvodb.log
+  stderr $LOGDIR/survey.mergedvodb.log
+
+  task.exec
+    book npages SURVEY_MERGEDVODB -var N
+    if ($N == 0)
+#      echo "No labels for processing"
+      break
+    endif
+
+    book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW
+    if ("$tag" == "NULL")
+      # All labels have been done --- reset
+    # echo "Resetting labels"
+      for i 0 $N
+        book getpage SURVEY_MERGEDVODB $i -var tag
+        book setword SURVEY_MERGEDVODB $tag STATE NEW
+      end
+      book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW
+
+      # Select different database
+      $SURVEY_MERGEDVODB_DB ++
+      if ($SURVEY_MERGEDVODB_DB >= $DB:n) set SURVEY_MERGEDVODB_DB = 0
+    end
+   book setword SURVEY_MERGEDVODB $tag STATE DONE
+   book getword SURVEY_MERGEDVODB $tag MERGEDVODB -var mergedvodb
+   book getword SURVEY_MERGEDVODB $tag MINIDVODB_GROUP -var minidvodb_group
+     
+ #   $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb
+    #$run = mergetool 
+    $run = mergedvodb_queue.pl --outroot $mergedvodb --mergedvodb $mergedvodb --minidvodb_group $minidvodb_group
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      $run = $run --dbname $DB:$SURVEY_MERGEDVODB_DB
+      option $DB:$SURVEY_MERGEDVODB_DB
+    end
+    #this run doesn't work because there's no workdir 
+    #$run = $run -definebyquery -mergedvodb $mergedvodb -minidvodb_group $minidvodb_group 
+    
+    
+    # echo $run
+
+  echo $run
+    command $run
+  end
+
+  # success
+   
+  task.exit    0
+#    echo "Success"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
 
 

trunk/ippTools/share/Makefile.am

@@ -310 +310 @@
         minidvodbtool_queue_minidvodb_id.sql \
         minidvodbtool_revertminidvodbcopy.sql \
+        mergetool_find_minidvodb_id.sql \
+        mergetool_find_mergedvodbcopy.sql \
+        mergetool_find_merge_id_dvo.sql \
+        mergetool_find_pendingmergeprocess.sql \
+        mergetool_find_pending_merge.sql \
+        mergetool_find_processed.sql \
+        mergetool_queue_mergecopy_id.sql \
+        mergetool_queue_merge_id.sql \
+        mergetool_revertmergedvodbcopy.sql \
+        mergetool_revertmergedvodbprocessed.sql \
         pstamptool_completedreq.sql \
         pstamptool_datastore.sql \

trunk/ippTools/share/addtool_checkminidvodbaddrun.sql

@@ -1 +1 @@
 -- this selects the minidvodbs that have complete addRuns
-SELECT minidvodb_id, minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg 
+SELECT minidvodb_id, minidvodb_name, minidvodb_path, state, minidbg 
 AS minidvodb_group, (cnt - cnt2) as not_full, cnt as addRun_count
 FROM minidvodbRun 

trunk/ippTools/share/addtool_find_minidvodbrun2.sql

@@ -1 +1 @@
 -- this selects the minidvodbs that have complete addRuns
-SELECT minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg 
+SELECT minidvodb_name, minidvodb_path, state, minidbg 
 AS minidvodb_group 
 FROM minidvodbRun 

trunk/ippTools/share/addtool_queue_minidvodbrun.sql

@@ -4 +4 @@
     minidvodb_name,
     minidvodb_path,
-    mergedvodb_path,
     'new'
 

trunk/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql

@@ -1 +1 @@
 SELECT minidvodbRun.* FROM minidvodbRun
 JOIN minidvodbProcessed USING(minidvodb_id)
-WHERE minidvodbRun.state = 'merged'
+WHERE minidvodbRun.state = 'merged'     
+      AND minidvodbProcessed.fault = 0
     AND minidvodb_id NOT IN (SELECT minidvodb_id
        FROM minidvodbCopy

trunk/ippTools/share/pxadmin_create_tables.sql

@@ -1964 +1964 @@
        minidvodb_group VARCHAR(64) NOT NULL,
        minidvodb_path VARCHAR(255) NOT NULL,
-       mergedvodb_path VARCHAR(255) NOT NULL,
        state VARCHAR(64) NOT NULL,
        creation_date TIMESTAMP,
@@ -1972 +1971 @@
 CREATE TABLE minidvodbProcessed (
       minidvodb_id BIGINT,
-      merge_order BIGINT,
       dtime_resort FLOAT,
       dtime_relphot FLOAT,
-      dtime_verify FLOAT,
-      dtime_merge FLOAT,
       dtime_script FLOAT,
       epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-      mergedvodb_path VARCHAR(255) NOT NULL,
       fault SMALLINT NOT NULL,
       UNIQUE KEY (minidvodb_id),
@@ -2000 +1995 @@
     FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE mergedvodbRun (
+       merge_id BIGINT AUTO_INCREMENT,
+       minidvodb_id BIGINT,
+       mergedvodb VARCHAR(64) NOT NULL,
+       mergedvodb_path VARCHAR(255) NOT NULL,
+       state VARCHAR(64) NOT NULL,
+       creation_date TIMESTAMP,
+       PRIMARY KEY(merge_id), KEY(minidvodb_id), KEY(mergedvodb), INDEX(merge_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE mergedvodbProcessed (
+      merge_id BIGINT,
+      dtime_verify FLOAT,
+      dtime_merge FLOAT,
+      dtime_script FLOAT,
+      epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+      fault SMALLINT NOT NULL,
+      UNIQUE KEY (merge_id),
+      KEY(merge_id),
+      FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE mergedvodbCopy (
+    mergedvodbcopy_id     BIGINT AUTO_INCREMENT,
+    merge_id     BIGINT,
+    mergedvodb_rsync_path VARCHAR(255) NOT NULL,
+    destination_host VARCHAR(255) NOT NULL,
+    fault  SMALLINT NOT NULL,
+    state   VARCHAR(64) NOT NULL,
+    epoch TIMESTAMP,
+    dtime FLOAT,
+    PRIMARY KEY(mergedvodbcopy_id),
+    KEY(merge_id),
+    FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
 CREATE TABLE vpRun (
     vp_id BIGINT AUTO_INCREMENT,

trunk/ippTools/src

@@ -7 +7 @@
 Makefile.in
 addtool
+mergetool
 caltool
 camtool

@@ -7 +7 @@
 Makefile.in
 addtool
+mergetool
 caltool
 camtool

trunk/ippTools/src/addtool.c

@@ -292 +292 @@
     } else {
       //This picks only the unmagicked/uncensored ones
-      if (strcmp(stage,"cam") == 0) {
-        psStringAppend(&query, " AND (camRun.magicked = 0)");
-      }
+      //if (strcmp(stage,"cam") == 0) {
+        //we can now properly handle the magicked case in ippScripts, so we queue camRuns in any magic state now if -uncensored.
+        //psStringAppend(&query, " AND (camRun.magicked = 0)");
+        //}
       if (strcmp(stage,"stack") == 0) {
         psStringAppend(&query, " AND (stackRun.magicked = 0)");
@@ -824 +825 @@
     PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-minidvodb_name", false, false);
     PXOPT_LOOKUP_S16(fault,         config->args, "-fault", false, false);
-
+    PXOPT_LOOKUP_S64(stage_extra1, config->args, "-stage_extra1", false, false);
     // generate restrictions
     psMetadata *where = psMetadataAlloc();
@@ -902 +903 @@
       }
     }
-
+    //if there is a stage_extra1, set it in addRun (it's not known until it is processed)
+    if (stage_extra1) {
+      psString setExtra = NULL;
+      psStringAppend (&setExtra, "UPDATE addRun set stage_extra1 = %" PRId64, stage_extra1);
+      psStringAppend (&setExtra, " where add_id = %" PRId64, row->add_id);
+
+      if (!p_psDBRunQuery(config->dbh, setExtra)) {
+        if (!psDBRollback(config->dbh)) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+
+        return false;
+      }
+    }
 
 
@@ -1049 +1064 @@
     PXOPT_COPY_S64(config->args, where, "-add_id",    "addRun.add_id",         "==");
     PXOPT_COPY_S64(config->args, where, "-stage_id",    "addRun.stage_id",         "==");
-    PXOPT_LOOKUP_STR(stage,       config->args, "-stage", false, false);
+    PXOPT_LOOKUP_STR(stage,       config->args, "-stage", true, false);
     pxcamGetSearchArgs (config, where);
     pxAddLabelSearchArgs (config, where, "-label",    "addRun.label",     "==");
@@ -1221 +1236 @@
 
   PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false);
-  PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", true, false);
-
-  //optional
+    //optional
   PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false);
   PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false);
@@ -1238 +1251 @@
   if (minidvodb_path) {
     minidvodbpath = minidvodb_path;
+  } else {
+    psError(PS_ERR_UNKNOWN, false, "require minidvodb_path");
+    return false;
   }
 
@@ -1245 +1261 @@
             minidvodb_group,
             minidvodbpath,
-            mergedvodb_path,
-            "new",
+            "new",
             0
             )) {
@@ -1268 +1283 @@
     psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id);
   }
-
-  if (!minidvodb_path) {
-    psStringAppend(&minidvodb_path,"%s/%s",mergedvodb_path,minidvodb_name);
-  }
-
   if (minidvodb_path) {
     psStringAppend(&minidvodb_path,"/%s",minidvodb_name);
@@ -1304 +1314 @@
     PXOPT_COPY_STR(config->args, where, "-state",     "state", "==");
     PXOPT_COPY_STR(config->args, where, "-minidvodb_path", "minidvodb_path", "==");
-    PXOPT_COPY_STR(config->args, where, "-mergedvodb_path", "mergedvodb_path", "==");
     PXOPT_COPY_STR(config->args, where, "-minidvodb_group",     "minidvodb_group", "==");
 
@@ -1310 +1319 @@
     PXOPT_LOOKUP_STR(minidvodb_path,  config->args, "-set_minidvodb_path", false, false);
     PXOPT_LOOKUP_STR(state,  config->args, "-set_state", false, false);
-    PXOPT_LOOKUP_STR(mergedvodb_path,  config->args, "-set_mergedvodb_path", false, false);
     PXOPT_LOOKUP_STR(minidvodb_group,  config->args, "-set_minidvodb_group", false, false);
 
@@ -1345 +1353 @@
     }
 
-    if (mergedvodb_path) {
-      if (cnt) {
-        psStringAppend(&query, "%s", comma);
-      }
-      psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path);
-      cnt++;
-    }
-
     if (minidvodb_group) {
       if (cnt) {
@@ -1396 +1396 @@
   }
 
-  psString firstquery = psStringCopy("SELECT * from  minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL");
+  psString firstquery = psStringCopy("SELECT * from  minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL");
 
   psString firstwhereClause = psDBGenerateWhereConditionSQL(where, NULL);
@@ -1444 +1444 @@
 
   //now flip new -> active
-  psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL ");
+  psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL ");
   psStringAppend(&query2, " AND minidvodb_group = '%s' limit 1;", minidvodb_group);
   if (!p_psDBRunQuery(config->dbh, query2)) {
@@ -1621 +1621 @@
     // required
   PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false);
-  PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-mergedvodb_path", true, false);
   PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", true, false);
 
     // optional
-  PXOPT_LOOKUP_U64(merge_order,     config->args, "-merge_order", false, false);
   PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-dtime_relphot", false, false);
   PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false);
-  PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false);
-  PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false);
   PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
   
@@ -1680 +1676 @@
   minidvodbProcessedRow *row = minidvodbProcessedRowAlloc(
                pendingRow->minidvodb_id,
-               merge_order,
                dtime_resort,
                dtime_relphot,
-               dtime_verify,
-               dtime_merge,
                dtime_script,
                epoch,
-               mergedvodb_path,
                fault
                );
@@ -1702 +1694 @@
   }
 
-
-
-
-  //this finds the # of merged things (for the merge order)
-  psString query3 = NULL;
-  psStringAppend(&query3, "select count(*) from minidvodbRun join minidvodbProcessed using (minidvodb_id) where state = 'merged' and minidvodb_group = '%s';", minidvodb_group);
-
-  if (!p_psDBRunQuery(config->dbh, query3)) {
-    // rollback
-    if (!psDBRollback(config->dbh)) {
-      psError(PS_ERR_UNKNOWN, false, "database error");
-    }
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    psFree(query3);
-    return false;
-  }
-  psArray *output2 = p_psDBFetchResult(config->dbh);
-  if (!output2) {
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    return false;
-  }
-  if (!psArrayLength(output2)) {
-    psTrace("addtool", PS_LOG_INFO, "no rows found");
-    psFree(output);
-    return true;
-  }
-  bool status;
-  psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)");
-  if (!status) {
-
-    psAbort("failed to lookup value for count column");
-    return false;
-  }
-  psString final = NULL;
-  psStringAppend(&final, "%" PRIu64, m_order);
-    //return false;
-  psFree(query3);
-  psFree(output2);
-
-
-
-  //update the merge_order
-
-  psString query4 = NULL;
-  psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order);
-  psStringAppend(&query4," where minidvodb_id = %" PRIu64,  minidvodb_id);
-  //printf("%s", query4);
-  if (!p_psDBRunQuery(config->dbh, query4)) {
-    // rollback
-    if (!psDBRollback(config->dbh)) {
-      psError(PS_ERR_UNKNOWN, false, "database error");
-    }
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    psFree(query4);
-    return false;
-  }
-  //
-psFree(query4);
-
-
-
-// since there is only one exp per 'new' set mindvodbRun.state = 'merged'
-
   psString query2 = NULL ;
   psStringAppend(&query2, "UPDATE minidvodbRun SET state = 'merged' WHERE minidvodb_id = %'" PRIu64, row->minidvodb_id);
@@ -1777 +1706 @@
     return false;
   }
-
-
-
-
-
-
-
-
   psFree(row);
   psFree(pendingRow);
@@ -1797 +1718 @@
     return false;
   }
-
-
-  //print the merge_order (why not!)
-  printf("%s", final);
-  psFree(final);
-
   return true;
 }
@@ -1961 +1876 @@
 
   PXOPT_LOOKUP_U64(minidvodb_id,  config->args, "-minidvodb_id", true, false);
-  PXOPT_LOOKUP_U64(merge_order,  config->args, "-set_merge_order", false, false);
   PXOPT_LOOKUP_S16(fault,  config->args, "-set_fault", false, false);
   PXOPT_LOOKUP_F32(dtime_relphot,  config->args, "-set_dtime_relphot", false, false);
   PXOPT_LOOKUP_F32(dtime_resort,  config->args, "-set_dtime_resort", false, false);
-  PXOPT_LOOKUP_F32(dtime_merge,  config->args, "-set_dtime_merge", false, false);
-PXOPT_LOOKUP_F32(dtime_verify,  config->args, "-set_dtime_verify", false, false);
 PXOPT_LOOKUP_F32(dtime_script,  config->args, "-set_dtime_script", false, false);
   PXOPT_COPY_S64(config->args, where, "-minidvodb_id",     "minidvodbProcessed.minidvodb_id", "==");
@@ -1986 +1898 @@
   }
 
-  if (merge_order) {
-    if (cnt) {
-      psStringAppend(&query, "%s", comma);
-    }
-
-    psStringAppend(&query, " merge_order = %" PRId64, merge_order);
-    cnt++;
-  }
-
   if (dtime_relphot) {
     if (cnt) {
@@ -2011 +1914 @@
   }
 
-  if (dtime_merge) {
-    if (cnt) {
-      psStringAppend(&query, "%s", comma);
-    }
-    psStringAppend(&query, " dtime_merge = %f", dtime_merge);
-    cnt++;
- }
-if (dtime_verify) {
-    if (cnt) {
-      psStringAppend(&query, "%s", comma);
-    }
-    psStringAppend(&query, " dtime_verify = %f", dtime_verify);
-    cnt++;
- }
-if (dtime_script) {
+  if (dtime_script) {
     if (cnt) {
       psStringAppend(&query, "%s", comma);

trunk/ippTools/src/addtoolConfig.c

@@ -115 +115 @@
     psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN);
     psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
+psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-stage_extra1",  0,            "set stage_extra1", 0);
+
 
     // -processedexp
@@ -165 +167 @@
     psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group",        0, "define minidvodb_group (required)", NULL);
     psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path",        0, "define path for minidvodb", NULL);
-    psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path",        0, "define path for the merged dvodb (required)", NULL);
     psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_state",        0, "define state", NULL);
 
@@ -174 +175 @@
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name",        0, "search by minidvodb_name (LIKE)", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_path",        0, "search by path for minidvodb", NULL);
-    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-mergedvodb_path",        0, "search by path for the merged dvodb", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-state",        0, "search by state", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name",        0, "define minidvodb_name", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path",        0, "define path for minidvodb", NULL);
-    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path",        0, "define path for minidvodb", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group",        0, "define path for the merged dvodb", NULL);
     psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_state",        0, "define state", NULL);
@@ -205 +204 @@
     psMetadata *addminidvodbprocessedArgs = psMetadataAlloc();
     psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0,    "define minidvodb_id (required)", 0);
-    psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-merge_order",    0,    "define merge order", 0);
     psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_relphot",  0,    "define elapsed time for relphot (seconds)", NAN);
     psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_resort", 0,    "define elapsed time for resort (seconds)", NAN);
-    psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_verify",    0,    "define elapsed time for DVO verify (seconds)", NAN);
-    psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_merge",    0,    "define elapsed time for DVO merge (seconds)", NAN);
     psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_script",    0,    "define elapsed time for script (seconds)", NAN);
     psMetadataAddTime(addminidvodbprocessedArgs, PS_LIST_TAIL, "-epoch",         0,    "time merge is finished", NULL);
-    psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-mergedvodb_path",0,    "path of merged dvodb", NULL);
     psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",0,    "minidvodb_group", NULL);
     psMetadataAddS16(addminidvodbprocessedArgs, PS_LIST_TAIL, "-fault",          0,    "set fault code", 0);
@@ -232 +227 @@
     psMetadataAddStr(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name",        0, "search by minidvodb_name", NULL);
     psMetadataAddS16(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_fault",  0,            "set fault code", 0);
-    psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_merge_order",    0,    "define merge order", 0);
     psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_relphot",  0,    "define elapsed time for relphot (seconds)", 0);
     psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_resort", 0,    "define elapsed time for resort (seconds)", 0);
-    psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_verify",    0,    "define elapsed time for DVO verify (seconds)", 0);
-    psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_merge",    0,    "define elapsed time for DVO merge (seconds)", 0);
     psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_script",    0,    "define elapsed time for script (seconds)", 0);
     psMetadata *argSets = psMetadataAlloc();

trunk/ippTools/src/minidvodbtool.c

@@ -89 +89 @@
     psMetadata *where = psMetadataAlloc();
     pxcamGetSearchArgs (config, where);
-    PXOPT_COPY_S64(config->args, where,  "-minidvodb_id",    "minidvodbRun.minidvodbRun_id", "==");
+    PXOPT_COPY_S64(config->args, where,  "-minidvodb_id",    "minidvodbRun.minidvodb_id", "==");
     pxAddLabelSearchArgs (config, where, "-minidvodb_group",     "minidvodbRun.minidvodb_group", "=="); // define using camRun label
    

trunk/psLib/src/fits/psFitsTableNew.c

@@ -130 +130 @@
     if (numCols > 0) {
         table->columns = psAlloc(sizeof(psFitsTableColumn) * numCols);
-        memset(table->columns, sizeof(psFitsTableColumn) * numCols, 0);
+        memset(table->columns, 0, sizeof(psFitsTableColumn) * numCols);
     } else {
         table->columns = NULL;

@@ -31 +31 @@
 Digest-SHA1-2.11
 ExtUtils-MakeMaker-6.31
+ExtUtils-MakeMaker-6.54
 ExtUtils-Manifest-1.51
 File-ExtAttr-1.04
@@ -66 +67 @@
 Params-Validate-0.92
 Parse-RecDescent-1.94
+Pod-Simple-3.14
+Pod-Escapes-1.04
 SOAP-Lite-0.69
 SQL-Interp-1.06

@@ -7 +7 @@
 Makefile.in
 addtool
+mergetool
 caltool
 camtool