Changeset 28181 for trunk

Timestamp:

Jun 1, 2010, 3:09:47 PM (16 years ago)

Author:

Paul Price

Message:

Merging branches/haf_branches/ipp.20100512/ into trunk: addtool development to ingest sources into separate dvo catalogues and dvodb merging. (Committed by PAP for HAF)

Location:

trunk

Files:

• . (modified) (1 prop)
• dbconfig/add.md (modified) (3 diffs)
• dbconfig/ipp.m4 (modified) (1 diff)
• dbconfig/minidvodb.md (copied) (copied from branches/haf_branches/ipp.20100512/dbconfig/minidvodb.md )
• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/scripts/addstar_run.pl (modified) (7 diffs)
• ippScripts/scripts/minidvodb_createdb.pl (copied) (copied from branches/haf_branches/ipp.20100512/ippScripts/scripts/minidvodb_createdb.pl )
• ippScripts/scripts/minidvodb_merge.pl (copied) (copied from branches/haf_branches/ipp.20100512/ippScripts/scripts/minidvodb_merge.pl ) (5 diffs)
• ippTasks/Makefile.am (modified) (1 diff)
• ippTasks/addstar.pro (modified) (4 diffs)
• ippTasks/minidvodb.pro (copied) (copied from branches/haf_branches/ipp.20100512/ippTasks/minidvodb.pro )
• ippTasks/survey.pro (modified) (3 diffs)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/addtool_checkminidvodbaddrun.sql (copied) (copied from branches/haf_branches/ipp.20100512/ippTools/share/addtool_checkminidvodbaddrun.sql )
• ippTools/share/addtool_find_minidvodbprocessed.sql (copied) (copied from branches/haf_branches/ipp.20100512/ippTools/share/addtool_find_minidvodbprocessed.sql )
• ippTools/share/addtool_find_minidvodbrun.sql (copied) (copied from branches/haf_branches/ipp.20100512/ippTools/share/addtool_find_minidvodbrun.sql )
• ippTools/share/addtool_find_minidvodbrun2.sql (copied) (copied from branches/haf_branches/ipp.20100512/ippTools/share/addtool_find_minidvodbrun2.sql )
• ippTools/share/addtool_find_pendingmergeprocess.sql (copied) (copied from branches/haf_branches/ipp.20100512/ippTools/share/addtool_find_pendingmergeprocess.sql )
• ippTools/share/addtool_find_processedexp.sql (modified) (1 diff)
• ippTools/share/addtool_queue_cam_id.sql (modified) (1 diff)
• ippTools/share/addtool_queue_minidvodbrun.sql (copied) (copied from branches/haf_branches/ipp.20100512/ippTools/share/addtool_queue_minidvodbrun.sql )
• ippTools/share/addtool_reset_faulted_minidvodbruns.sql (copied) (copied from branches/haf_branches/ipp.20100512/ippTools/share/addtool_reset_faulted_minidvodbruns.sql )
• ippTools/share/addtool_revertminidvodbprocessed.sql (copied) (copied from branches/haf_branches/ipp.20100512/ippTools/share/addtool_revertminidvodbprocessed.sql )
• ippTools/src/addtool.c (modified) (11 diffs)
• ippTools/src/addtool.h (modified) (1 diff)
• ippTools/src/addtoolConfig.c (modified) (5 diffs)
• ippTools/src/flatcorr.c (modified) (1 diff)
• ippTools/src/pstamptool.h (modified) (1 diff)
• ippTools/src/pxadd.c (modified) (2 diffs)
• ippTools/src/pxadd.h (modified) (1 diff)

trunk/dbconfig/add.md

@@ -12 +12 @@
     note            STR     255
     image_only      BOOL    f
+    minidvodb       BOOL    f
+    minidvodb_group STR     64
+    minidvodb_name  STR     64
 END
 
@@ -18 +21 @@
     dtime_addstar   F32     0.0
     path_base       STR     255
+    dvodb_path      STR     255
     fault           S16     0       # Key NOT NULL
 END
@@ -24 +28 @@
     label           STR    64       # Primary Key
 END
-    
+
+
+

trunk/dbconfig/ipp.m4

@@ -24 +24 @@
 include(label.md)
 include(magic.md)
+include(minidvodb.md)
 include(calibration.md)
 include(flatcorr.md)

trunk/ippScripts/Build.PL

@@ -62 +62 @@
         scripts/magic_destreak_cleanup.pl
         scripts/magic_destreak_defineruns.pl
+        scripts/minidvodb_createdb.pl
+        scripts/minidvodb_load.pl
         scripts/ippdb.pl
         scripts/ipp_cleanup.pl

trunk/ippScripts/scripts/addstar_run.pl

@@ -37 +37 @@
 }
 
-my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $image_only, $verbose, $no_update,
+my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update,
      $no_op, $redirect, $save_temps);
 GetOptions(
@@ -48 +48 @@
     'reduction=s'       => \$reduction, # Reduction class
     'dvodb|w=s'         => \$dvodb,  # output DVO database
+    'minidvodb'         => \$minidvodb,  # use minidvodb?
+    'minidvodb_name|w=s'=> \$minidvodb_name,  # miniDVO database name
+    'minidvodb_group|w=s' => \$minidvodb_group, # miniDVO database group
     'image-only'        => \$image_only,   # Print to stdout
     'verbose'           => \$verbose,   # Print to stdout
@@ -58 +61 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
-          -msg => "Required options: --exp_tag --add_id --camera --outroot",
+          -msg => "Required options: --exp_tag --add_id --camera --outroot --dvodb --camroot",
           -exitval => 3,
           ) unless
@@ -65 +68 @@
     defined $outroot and
     defined $camroot and
+    defined $dvodb and
     defined $camera;
-
+if ($minidvodb && !defined($minidvodb_group)) {
+                my_die( "missing minidvodb_group", $add_id, 3 );
+            }
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 
@@ -103 +109 @@
 my $dtime_addstar = 0;
 
+if (defined $dvodbReal) {
+        if ($minidvodb) {
+            
+            if (!defined($minidvodb_name)) {
+                #take the active one, if it's not defined on the command line
+                #reverts would have this already set, for example.
+                my $command = "addtool -listminidvodbrun ";
+                $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group;
+                $command .= " -state 'active' -limit 1";
+                $command .= " -dbname $dbname" if defined $dbname;
+                print $command;
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success;
+                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+                    &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR);
+
+                my $components = parse_md_list($metadata) or
+                    &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
+                my $comp = $$components[0];
+                $minidvodb_name = $comp->{minidvodb_name};
+        
+                if (!defined($minidvodb_name)) {
+                    &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR);
+                } 
+                
+            }
+            # tack on the minidvodb part to the db.
+            $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/';
+            
+        }
 unless ($no_op) {
-    if (defined $dvodbReal) {
+            print $dvodbReal;
 
         ## addstar can either save the full set of detections, or just
@@ -142 +179 @@
 $fpaCommand .= " -path_base $outroot";
 $fpaCommand .= " -dtime_addstar $dtime_addstar";
+$fpaCommand .= " -dvodb_path $dvodbReal";
 $fpaCommand .= " -dbname $dbname" if defined $dbname;
-
+print $fpaCommand;
 # Add the result into the database
 unless ($no_update) {
@@ -171 +209 @@
         $command .= " -addprocessedexp";
         $command .= " -fault $exit_code";
+        $command .= " -dvodb_path $dvodbReal" if defined $dvodbReal;
         $command .= " -path_base $outroot" if defined $outroot;
         $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400));

trunk/ippScripts/scripts/minidvodb_merge.pl

@@ -87 +87 @@
 unless ($no_op) {
     if (defined $mergedvodbReal) {
-        {
+        {
             my $command  = "$addstar -resort";
             $command .= " -D CAMERA $camera";
@@ -101 +101 @@
             }
             $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);  $dtime_resort = $dtime_addstar;
-            # MJD of starting script
+            # MJD of starting script
             print "addstar -resort time $dtime_addstar\n";
         }
@@ -123 +123 @@
             print "relphot time $dtime_relphot\n";
         }
-        
-        {
-
-            my $mdcParser = PS::IPP::Metadata::Config->new;
-            
-
-            my $command = "$addtool -listminidvodbrun -state mergedsdfsad -minidvodb_group " . $minidvodb_group;
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+
+        my $this_is_the_first;
+        {
+
+            my $mdcParser = PS::IPP::Metadata::Config->new;
+
+            my $command = "$addtool -listminidvodbrun -state mergedsdfsad -minidvodb_group " . $minidvodb_group;
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                     run(command => $command, verbose => $verbose);
-            &my_die( "Unable to get list of minidvodbruns", $minidvodb_id, $PS_EXIT_SYS_ERROR) unless $success;
-            my $this_is_the_first;
-            if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first
-                $this_is_the_first =1;
-            } else {
-                 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die( "Unable to get list of minidvodbruns", $minidvodb_id, $PS_EXIT_SYS_ERROR) unless $success;
+            if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first
+                $this_is_the_first =1;
+            } else {
+                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
                     &my_die("Unable to parse metadata config", $minidvodb_id, $PS_EXIT_PROG_ERROR);
-                 #this fails if there is nothing listed. I checked.
+                #this fails if there is nothing listed. I checked.
                 my $components = parse_md_list($metadata) or
                     &my_die("Unable to parse metadata list", $minidvodb_id, $PS_EXIT_PROG_ERROR);
                 my $comp = $$components[0];
-           my  $minidvodb_name = $comp->{minidvodb_name};
-        
+                my  $minidvodb_name = $comp->{minidvodb_name};
+
                 if (!defined($minidvodb_name)) {
                     &my_die("Unable to parse minidvodb_name", $minidvodb_id, $PS_EXIT_PROG_ERROR);
                 } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk.
-                 $this_is_the_first = 0;
-            }
-
-            my $merge_command;
-            if ($this_is_the_first) {
-                
-                $merge_command = "cp -rp $minidvodb $mergedvodb"
-
-            } else {
-
-                $merge_command = "$dvomerge $minidvodb into $mergedvodb"
-            }
-
-
-
-            system ($merge_command);
-
-            my ( $success2, $error_code2, $full_buf2, $stdout_buf2, $stderr_buf2 ) =
-                run(command => $merge_command, verbose => $verbose);
-        unless ($success2) {
-            $error_code2 = (($error_code2 >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to merge: $error_code", $mergedvodb, $error_code2);
-        }
-
-                   
-            $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_relphot_start);   # MJD of starting script
+                $this_is_the_first = 0;
+            }
+        }
+
+        {
+            my $merge_command;
+            if ($this_is_the_first) {
+                $merge_command = "cp -rp $minidvodb $mergedvodb"
+            } else {
+                $merge_command = "$dvomerge $minidvodb into $mergedvodb"
+            }
+
+            my $mjd_merge_start = DateTime->now->mjd;   # MJD of starting script
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $merge_command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to merge: $error_code", $mergedvodb, $error_code);
+            }
+
+            $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start);   # MJD of starting script
             print "merge time $dtime_merge\n";
-            
-        }
-        
-
-        {
-        my $command = "addtool -minidvodb_id $minidvodb_id";
-        $command .= " -addminidvodbprocessed";
-        $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal;
-        $command .= " -minidvodb_group $minidvodb_group";
-        $command .= " -dtime_relphot $dtime_relphot"  if defined $dtime_relphot;
-        $command .= " -dtime_relphot $dtime_resort" if defined $dtime_resort;
-        $command .= " -dtime_relphot $dtime_merge" if defined $dtime_merge;
-        $command .= " -dbname $dbname" if defined $dbname;
-    #print $command;
-        system ($command);
-
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to add to minidvodbprocessed: $error_code", $mergedvodb, $error_code);
-        }
-
-        }
+        }
+
+
+        {
+            my $command = "addtool -minidvodb_id $minidvodb_id";
+            $command .= " -addminidvodbprocessed";
+            $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal;
+            $command .= " -minidvodb_group $minidvodb_group";
+            $command .= " -dtime_relphot $dtime_relphot"  if defined $dtime_relphot;
+            $command .= " -dtime_relphot $dtime_resort" if defined $dtime_resort;
+            $command .= " -dtime_relphot $dtime_merge" if defined $dtime_merge;
+            $command .= " -dbname $dbname" if defined $dbname;
+            #print $command;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to add to minidvodbprocessed: $error_code", $mergedvodb, $error_code);
+            }
+        }
 
     } else {
@@ -217 +207 @@
 {
     my $msg = shift; # Warning message on die
-    my $minidbvodb_id = shift; 
+    my $minidbvodb_id = shift;
     my $exit_code = shift; # Exit code to add
 
@@ -223 +213 @@
 
 if (defined $minidvodb_id ) {
-    
+
     my $command = "addtool -minidvodb_id $minidvodb_id";
     $command .= " -addminidvodbprocessed";
         $command .= " -fault $exit_code";
-        $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal;
-        $command .= " -minidvodb_group $minidvodb_group";
+        $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal;
+        $command .= " -minidvodb_group $minidvodb_group";
         $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot;
         $command .= " -dtime_relphot $dtime_resort" if defined $dtime_resort;
         $command .= " -dtime_relphot $dtime_merge" if defined $dtime_merge;
         $command .= " -dbname $dbname" if defined $dbname;
-    
-    
+
+
 
     #print $command;

trunk/ippTasks/Makefile.am

@@ -32 +32 @@
         dqstats.pro \
         science.cleanup.pro \
+        minidvodb.pro \
         nightly_stacks.pro \
         lossy_compress.pro

trunk/ippTasks/addstar.pro

@@ -45 +45 @@
 end
 
+
 # this variable will cycle through the known database names
 $addstar_DB = 0
 $addstar_revert_DB = 0
+
 
 # select images ready for addstar analysis
@@ -64 +66 @@
 
   task.exec
-    if ($LABEL:n == 0) break
+   # if ($LABEL:n == 0) break
     $run = addtool -pendingexp
     if ($DB:n == 0)
@@ -134 +136 @@
     book getword addPendingExp $pageName reduction -var REDUCTION
     book getword addPendingExp $pageName dvodb  -var DVODB
+    book getword addPendingExp $pageName minidvodb  -var MINIDVODB
+    book getword addPendingExp $pageName minidvodb_name  -var MINIDVODB_NAME
+    book getword addPendingExp $pageName minidvodb_group  -var MINIDVODB_GROUP
     book getword addPendingExp $pageName image_only -var IMAGE_ONLY
     book getword addPendingExp $pageName dbname -var DBNAME
@@ -164 +169 @@
       $run = $run --image-only
     end
+    if ("$MINIDVODB" == "T")
+    $run = $run --minidvodb
+    $run = $run --minidvodb_group
+        if (("$MINIDVODB_NAME" != "NULL") && ("$MINIDVODB_NAME" != "(null)"))
+           $run = $run --minidvodb_name  #have addstar_run.pl grab the 'active' name if it is NULL
+        end
+    end 
     
     add_standard_args run

trunk/ippTasks/survey.pro

@@ -207 +207 @@
 
 macro survey.add.addstar
-  if ($0 != 3)
-    echo "USAGE: survey.add.addstar (label) (dvodb)"
+  if ($0 != 4)
+    echo "USAGE: survey.add.addstar (label) (dvodb) (minidvodb_group)"
     break
   end
   book newpage SURVEY_ADDSTAR $1
   book setword SURVEY_ADDSTAR $1 DVODB $2
+  book setword SURVEY_ADDSTAR $1 MINIDVODB_GROUP $3
   book setword SURVEY_ADDSTAR $1 STATE PENDING
 end
@@ -638 +639 @@
     book setword SURVEY_ADDSTAR $label STATE DONE
     book getword SURVEY_ADDSTAR $label DVODB -var dvodb
+    book getword SURVEY_ADDSTAR $label MINIDVODB_GROUP -var minidvodb_group
   
  #   $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb
@@ -648 +650 @@
     end
     
-    $run = $run -definebyquery -destreaked -label $label -set_dvodb $dvodb
+    $run = $run -definebyquery -destreaked -label $label -set_dvodb $dvodb -set_minidvodb_group $minidvodb_group
     echo $run
     command $run

trunk/ippTools/share/Makefile.am

@@ -4 +4 @@
 
 dist_pkgdata_DATA = \
+     addtool_checkminidvodbaddrun.sql \
      addtool_donecleanup.sql \
      addtool_find_cam_id.sql \
      addtool_find_pendingexp.sql \
+     addtool_find_pendingmergeprocess.sql \
      addtool_find_processedexp.sql \
+     addtool_find_minidvodbprocessed.sql \
+     addtool_find_minidvodbrun.sql \
      addtool_pendingcleanupexp.sql \
      addtool_pendingcleanuprun.sql \
      addtool_queue_cam_id.sql \
+     addtool_queue_minidvodbrun.sql \
+     addtool_reset_faulted_minidvodbruns.sql \
      addtool_reset_faulted_runs.sql \
+     addtool_revertminidvodbprocessed.sql \
      addtool_revertprocessedexp.sql \
      camtool_donecleanup.sql \

trunk/ippTools/share/addtool_find_processedexp.sql

@@ -2 +2 @@
     addProcessedExp.*,
     addRun.workdir
-FROM addRun
-JOIN camProcessedExp
+FROM addProcessedExp
+JOIN addRun
+    USING(add_id)
+JOIN camRun
     USING(cam_id)
 JOIN chipRun

trunk/ippTools/share/addtool_queue_cam_id.sql

@@ -11 +11 @@
         '%s',           -- dvodb 
         '%s',           -- note
-        %d              -- image_only
+        %d,             -- image_only
+        %d,             -- minidvodb
+        '%s',           -- minidvodb_group 
+        '%s'            -- minidvodb_name
     FROM camRun
     WHERE

trunk/ippTools/src/addtool.c

@@ -40 +40 @@
 static bool maskedMode(pxConfig *config);
 static bool unblockMode(pxConfig *config);
+static bool addminidvodbrunMode(pxConfig *config);
+static bool updateminidvodbrunMode(pxConfig *config);
+static bool listminidvodbrunMode(pxConfig *config);
+static bool flipminidvodbrunMode(pxConfig *config);
+static bool checkminidvodbrunaddrunMode(pxConfig *config);
+static bool addminidvodbprocessedMode(pxConfig *config);
+static bool listminidvodbprocessedMode(pxConfig *config);
+static bool revertminidvodbprocessedMode(pxConfig *config);
+static bool updateminidvodbprocessedMode(pxConfig *config);
+
+
 
 # define MODECASE(caseName, func) \
@@ -69 +80 @@
         MODECASE(ADDTOOL_MODE_MASKED,               maskedMode);
         MODECASE(ADDTOOL_MODE_UNBLOCK,              unblockMode);
+        MODECASE(ADDTOOL_MODE_ADDMINIDVODBRUN,      addminidvodbrunMode);
+        MODECASE(ADDTOOL_MODE_UPDATEMINIDVODBRUN,   updateminidvodbrunMode);
+        MODECASE(ADDTOOL_MODE_LISTMINIDVODBRUN,     listminidvodbrunMode);
+        MODECASE(ADDTOOL_MODE_FLIPMINIDVODBRUN,     flipminidvodbrunMode);
+        MODECASE(ADDTOOL_MODE_CHECKMINIDVODBRUNADDRUN, checkminidvodbrunaddrunMode);
+        MODECASE(ADDTOOL_MODE_ADDMINIDVODBPROCESSED,addminidvodbprocessedMode);
+        MODECASE(ADDTOOL_MODE_LISTMINIDVODBPROCESSED,listminidvodbprocessedMode);
+        MODECASE(ADDTOOL_MODE_REVERTMINIDVODBPROCESSED,revertminidvodbprocessedMode);
+        MODECASE(ADDTOOL_MODE_UPDATEMINIDVODBPROCESSED,updateminidvodbprocessedMode);
+
         default:
             psAbort("invalid option (this should not happen)");
@@ -102 +123 @@
     PXOPT_COPY_STR(config->args, where,  "-reduction", "camRun.reduction", "==");
 
+
     if (!psListLength(where->list)) {
         psFree(where);
@@ -114 +136 @@
     PXOPT_LOOKUP_STR(reduction,   config->args, "-set_reduction", false, false);
     PXOPT_LOOKUP_STR(note,        config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_STR(minidvodb_name,        config->args, "-set_minidvodb_name", false, false);
+    PXOPT_LOOKUP_STR(minidvodb_group,        config->args, "-set_minidvodb_group", false, false);
     PXOPT_LOOKUP_BOOL(image_only, config->args, "-image_only", false);
+    PXOPT_LOOKUP_BOOL(minidvodb,  config->args, "-set_minidvodb", false);
     PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false);
     PXOPT_LOOKUP_BOOL(pretend,    config->args, "-pretend", false);
     PXOPT_LOOKUP_BOOL(simple,     config->args, "-simple", false);
+
+
 
     // find the cam_id of all the exposures that we want to queue up.
@@ -245 +272 @@
                                dvodb       ? dvodb     : row->dvodb,
                                note        ? note      : NULL,
-                               image_only
+                               image_only,
+                               minidvodb,
+                               minidvodb_group,
+                               minidvodb_name
         )) {
             if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
+                psError(PS_ERR_UNKNOWN, false, "database error sfg");
             }
             psError(PS_ERR_UNKNOWN, false,
@@ -291 +321 @@
 
     // pxUpdateRun gets parameters from config->args and runs the update query
-    bool result = pxUpdateRun(config, where, &query, "addRun", "add_id", 
+    bool result = pxUpdateRun(config, where, &query, "addRun", "add_id",
         "addProcessedExp", false);
 
@@ -374 +404 @@
     PXOPT_LOOKUP_STR(path_base,     config->args, "-path_base", false, false);
     PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false);
+    PXOPT_LOOKUP_STR(dvodb_path, config->args, "-dvodb_path", false, false);
     PXOPT_LOOKUP_S16(fault,         config->args, "-fault", false, false);
 
@@ -399 +430 @@
         return false;
     }
+    //psError(PS_ERR_UNKNOWN, false, "database error %s", query);
     psFree(query);
 
@@ -424 +456 @@
         dtime_addstar,
         path_base,
+        dvodb_path,
         fault
         );
@@ -522 +555 @@
 
     if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+      psError(PS_ERR_UNKNOWN, false, "database error %s ", query);
         psFree(query);
         return false;
@@ -733 +766 @@
     return true;
 }
+
+static bool addminidvodbrunMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+  // required
+
+  PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false);
+  PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", true, false);
+
+  //optional
+  PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false);
+  PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false);
+  PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
+
+  if (!psDBTransaction(config->dbh)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+
+  psString minidvodbpath = "NULL";
+
+  // I don't know how to get around the complaints of minidvodb_path can't be null. this 'fixes' it, but someone smarter can fix it properly.
+  if (minidvodb_path) {
+    minidvodbpath = minidvodb_path;
+  }
+
+  if (!minidvodbRunInsert(config->dbh,
+            0, // job_id
+            minidvodb_name,
+            minidvodb_group,
+            minidvodbpath,
+            mergedvodb_path,
+            "new",
+            0
+            )) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+
+  psU64 affected = psDBAffectedRows(config->dbh);
+  if (affected != 1) {
+    psError(PS_ERR_UNKNOWN, false,
+            "should have affected one row but %" PRIu64 " rows were modified",
+            affected);
+    return false;
+  }
+
+  psS64 minidvodb_id = psDBLastInsertID(config->dbh);
+  printf("%" PRId64 "\n", minidvodb_id);
+
+
+  if (!minidvodb_name) {
+    psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id);
+  }
+
+  if (!minidvodb_path) {
+    psStringAppend(&minidvodb_path,"%s/%s",mergedvodb_path,minidvodb_name);
+  }
+
+  psString query = NULL;
+
+  psStringAppend(&query, "UPDATE minidvodbRun SET minidvodb_path = '%s', minidvodb_name = '%s' where minidvodb_id = %" PRIu64";", minidvodb_path, minidvodb_name, minidvodb_id);
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+
+  if (!psDBCommit(config->dbh)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+
+  return true;
+}
+
+static bool updateminidvodbrunMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-minidvodb_id",     "minidvodb_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-minidvodb_name",     "minidvodb_name", "==");
+    PXOPT_COPY_STR(config->args, where, "-state",     "state", "==");
+    PXOPT_COPY_STR(config->args, where, "-minidvodb_path", "minidvodb_path", "==");
+    PXOPT_COPY_STR(config->args, where, "-mergedvodb_path", "mergedvodb_path", "==");
+    PXOPT_COPY_STR(config->args, where, "-minidvodb_group",     "minidvodb_group", "==");
+
+    PXOPT_LOOKUP_STR(minidvodb_name,  config->args, "-set_minidvodb_name", false, false);
+    PXOPT_LOOKUP_STR(minidvodb_path,  config->args, "-set_minidvodb_path", false, false);
+    PXOPT_LOOKUP_STR(state,  config->args, "-set_state", false, false);
+    PXOPT_LOOKUP_STR(mergedvodb_path,  config->args, "-set_mergedvodb_path", false, false);
+    PXOPT_LOOKUP_STR(minidvodb_group,  config->args, "-set_minidvodb_group", false, false);
+
+
+    if (!psListLength(where->list)) {
+      psFree(where);
+      psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+      return false;
+    }
+
+    psString query = psStringCopy("UPDATE minidvodbRun SET ");
+    int cnt = 0;
+    psString comma = ",";
+    if (minidvodb_name) {
+      psStringAppend(&query, " minidvodb_name = '%s'", minidvodb_name);
+      cnt++;
+    }
+
+    if (minidvodb_path) {
+      if (cnt) {
+          psStringAppend(&query, "%s", comma);
+      }
+
+      psStringAppend(&query, " minidvodb_path = '%s'", minidvodb_path);
+      cnt++;
+    }
+
+    if (state) {
+      if (cnt) {
+        psStringAppend(&query, "%s", comma);
+      }
+      psStringAppend(&query, " state = '%s'", state);
+      cnt++;
+    }
+
+    if (mergedvodb_path) {
+      if (cnt) {
+        psStringAppend(&query, "%s", comma);
+      }
+      psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path);
+      cnt++;
+    }
+
+    if (minidvodb_group) {
+      if (cnt) {
+        psStringAppend(&query, "%s", comma);
+      }
+      psStringAppend(&query, " minidvodb_group = '%s'", minidvodb_group);
+      cnt++;
+    }
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " WHERE %s", whereClause);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+
+    psFree(query);
+    psFree(where);
+
+    return true;
+
+}
+
+static bool flipminidvodbrunMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_LOOKUP_STR(minidvodb_group,  config->args, "-minidvodb_group",true, false);
+  PXOPT_COPY_STR(config->args, where, "-minidvodb_group",     "minidvodb_group", "==");
+
+//this flips the new - > active
+// and the active - > waiting in one action
+
+
+// the first query looks to find things that are new and where all the fields are filled (ie, ready to be flipped to active)
+
+// start a transaction eraly so it will contain any row level locks
+  if (!psDBTransaction(config->dbh)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+
+  psString firstquery = psStringCopy("SELECT * from  minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL");
+
+  psString firstwhereClause = psDBGenerateWhereConditionSQL(where, NULL);
+  psStringAppend(&firstquery, " AND %s", firstwhereClause);
+
+
+
+  if (!p_psDBRunQuery(config->dbh, firstquery)) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    if (!psDBRollback(config->dbh)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+    }
+    return false;
+  }
+  psFree(firstquery);
+
+ //we don't care what the stuff is that is found, just that there is stuff. This is a check to see that there is something in the 'new' state, before flipping (so that if there is nothing in new, it won't flip the active to waiting.  the flipminidvo is just to make it easy to flip from new -> active.
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("addtool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+
+  //ok, there's something, so flip active -> waiting
+
+  psString query = psStringCopy("UPDATE minidvodbRun SET state = 'waiting' WHERE state = 'active' ");
+
+  psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+  psStringAppend(&query, " AND %s", whereClause);
+
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    if (!psDBRollback(config->dbh)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+    }
+    return false;
+  }
+
+
+  //now flip new -> active
+  psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL ");
+  psStringAppend(&query2, " AND minidvodb_group = '%s' limit 1;", minidvodb_group);
+  if (!p_psDBRunQuery(config->dbh, query2)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    if (!psDBRollback(config->dbh)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+    }
+    psFree(query);
+    return false;
+  }
+
+
+  if (!psDBCommit(config->dbh)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    if (!psDBRollback(config->dbh)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+    }
+    psFree(query);
+  return false;
+  }
+ psFree(query2);
+ psFree(where);
+
+ return true;
+
+}
+
+
+static bool checkminidvodbrunaddrunMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, false);
+  psMetadata *where = psMetadataAlloc();
+
+  //this checks to see if a minidvod_group/name is has completed addRun processing
+  PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "==");
+  PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "==");
+  PXOPT_COPY_STR(config->args, where, "-state", "minidvodbRun.state", "==");
+
+
+  PXOPT_LOOKUP_U64(limit,      config->args, "-limit", false, false);
+  PXOPT_LOOKUP_BOOL(all_addrun_states, config->args, "-all_addrun_states", false);
+  //this doesn't care what state the addRun is in (useful for counting addRuns)
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+  if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+  }
+
+  psString query = pxDataGet("addtool_checkminidvodbaddrun.sql");
+
+  if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+  }
+
+
+  if (psListLength(where->list)) {
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+  }
+
+  if (!all_addrun_states) {
+    psStringAppend(&query, " AND (cnt2 = cnt) ");
+  }
+
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, " %s", limitString);
+    psFree(limitString);
+  }
+
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("addtool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  if (!ippdbPrintMetadatas(stdout, output, "minidvodbRun", !simple)) {
+    psError(PS_ERR_UNKNOWN, false, "failed to print array");
+    psFree(output);
+    return false;
+  }
+
+  psFree(output);
+
+  return true;
+}
+
+
+
+static bool listminidvodbrunMode(pxConfig *config) {
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "==");
+    PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "==");
+
+    PXOPT_COPY_STR(config->args, where, "-state", "minidvodbRun.state", "==");
+    PXOPT_LOOKUP_U64(limit,      config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+
+    if (!psListLength(where->list)) {
+      psFree(where);
+      psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+      return false;
+    }
+
+    psString query = pxDataGet("addtool_find_minidvodbrun.sql");
+
+    if (!query) {
+      psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+      return false;
+    }
+
+    if (psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+      psStringAppend(&query, " WHERE %s", whereClause);
+      psFree(whereClause);
+    }
+
+    if (limit) {
+      psString limitString = psDBGenerateLimitSQL(limit);
+      psStringAppend(&query, " %s", limitString);
+      psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+
+    psFree(query);
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
+    if (!psArrayLength(output)) {
+      psTrace("addtool", PS_LOG_INFO, "no rows found");
+      psFree(output);
+      return true;
+    }
+
+    if (!ippdbPrintMetadatas(stdout, output, "minidvodbRun",  !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+
+    psFree(output);
+
+return true;
+}
+
+static bool addminidvodbprocessedMode(pxConfig *config) {
+
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+  PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false);
+  PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-mergedvodb_path", true, false);
+  PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", true, false);
+
+    // optional
+  PXOPT_LOOKUP_U64(merge_order,     config->args, "-merge_order", false, false);
+  PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-dtime_relphot", false, false);
+  PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false);
+  PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false);
+  PXOPT_LOOKUP_TIME(epoch, config->args, "-epoch", false, false);
+  PXOPT_LOOKUP_S16(fault,         config->args, "-fault", false, false);
+    //generate restrictions
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-minidvodb_id",   "minidvodbRun.minidvodb_id",   "==");
+
+
+  if (!psDBTransaction(config->dbh)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+
+
+  psString query = pxDataGet("addtool_find_pendingmergeprocess.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    return false;
+  }
+  // use psDBGenerateWhereSQL because the SQL yields an intermediate table
+  if (psListLength(where->list)) {
+    psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClaus);
+    psFree(whereClaus);
+  }
+  psFree(where);
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("addtool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  minidvodbRunRow *pendingRow = minidvodbRunObjectFromMetadata(output->data[0]);
+  psFree(output);
+  minidvodbProcessedRow *row = minidvodbProcessedRowAlloc(
+               pendingRow->minidvodb_id,
+               merge_order,
+               dtime_resort,
+               dtime_relphot,
+               dtime_merge,
+               epoch,
+               mergedvodb_path,
+               fault
+               );
+
+  if (!minidvodbProcessedInsertObject(config->dbh, row)) {
+    // rollback
+    if (!psDBRollback(config->dbh)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+    }
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(row);
+    psFree(pendingRow);
+    return false;
+  }
+
+
+
+
+  //this finds the # of merged things (for the merge order)
+  psString query3 = NULL;
+  psStringAppend(&query3, "select count(*) from minidvodbRun join minidvodbProcessed using (minidvodb_id) where state = 'merged' and minidvodb_group = '%s';", minidvodb_group);
+
+  if (!p_psDBRunQuery(config->dbh, query3)) {
+    // rollback
+    if (!psDBRollback(config->dbh)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+    }
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query3);
+    return false;
+  }
+  psArray *output2 = p_psDBFetchResult(config->dbh);
+  if (!output2) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+  if (!psArrayLength(output2)) {
+    psTrace("addtool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+  bool status;
+  psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)");
+  if (!status) {
+
+    psAbort("failed to lookup value for count column");
+    return false;
+  }
+  psString final = NULL;
+  psStringAppend(&final, "%" PRIu64, m_order);
+    //return false;
+  psFree(query3);
+  psFree(output2);
+
+
+
+  //update the merge_order
+
+  psString query4 = NULL;
+  psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order);
+  psStringAppend(&query4," where minidvodb_id = %" PRIu64,  minidvodb_id);
+  printf("%s", query4);
+  if (!p_psDBRunQuery(config->dbh, query4)) {
+    // rollback
+    if (!psDBRollback(config->dbh)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+    }
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query4);
+    return false;
+  }
+  //
+psFree(query4);
+
+
+
+// since there is only one exp per 'new' set mindvodbRun.state = 'merged'
+
+  psString query2 = NULL ;
+  psStringAppend(&query2, "UPDATE minidvodbRun SET state = 'merged' WHERE minidvodb_id = %'" PRIu64, row->minidvodb_id);
+
+  if (!p_psDBRunQuery(config->dbh, query2)) {
+    // rollback
+    if (!psDBRollback(config->dbh)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+    }
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query2);
+    return false;
+  }
+
+
+
+
+
+
+
+
+  psFree(row);
+  psFree(pendingRow);
+
+
+
+
+
+  //commit the changes
+  if (!psDBCommit(config->dbh)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+
+
+  //print the merge_order (why not!)
+  printf("%s", final);
+  psFree(final);
+
+  return true;
+}
+
+
+
+static bool listminidvodbprocessedMode(pxConfig *config) {
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbProcessed.minidvodb_name", "==");
+  PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbProcessed.minidvodb_group", "==");
+  PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
+  if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+  psString query = pxDataGet("addtool_find_minidvodbprocessed.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    }
+
+// we either add AND (condition) or WHERE (condition):
+    if (where->list && faulted) {
+        // list only faulted rows
+        psStringAppend(&query, " %s", " AND minidvodbProcessed.fault != 0");
+    }
+    if (where->list && !faulted) {
+        // don't list faulted rows
+        psStringAppend(&query, " %s", " AND minidvodbProcessed.fault = 0");
+    }
+    if (!where->list && faulted) {
+        // list only faulted rows
+        psStringAppend(&query, " %s", " WHERE minidvodbProcessed.fault != 0");
+    }
+    if (!where->list && !faulted) {
+        // don't list faulted rows
+        psStringAppend(&query, " %s", " WHERE minidvodbProcessed.fault = 0");
+    }
+    psFree(where);
+
+    // order by epoch
+    psStringAppend(&query, " ORDER BY minidvodb_id");
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error ");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("addtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negate simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "minidvodbProcessed", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+return true;
+}
+
+
+
+static bool revertminidvodbprocessedMode(pxConfig *config) {
+  psMetadata *where = psMetadataAlloc();
+  PS_ASSERT_PTR_NON_NULL(config, false);
+  PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "addRun.minidvodb_group", "==");
+
+  if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+    psFree(where);
+    psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+    return false;
+  }
+
+  if (!psDBTransaction(config->dbh)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(where);
+      return false;
+  }
+
+  {
+        psString query = pxDataGet("addtool_reset_faulted_minidvodbruns.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+
+        // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE
+
+        if (where && psListLength(where->list)) {
+          psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+          psStringAppend(&query, " AND %s", whereClause);
+          psFree(whereClause);
+        }
+
+        if (!p_psDBRunQuery(config->dbh, query)) {
+          // rollback
+          if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+          }
+          psError(PS_ERR_UNKNOWN, false, "database error");
+          psFree(query);
+          psFree(where);
+          return false;
+        }
+        psFree(query);
+  }
+  {
+    psString query = pxDataGet("addtool_revertminidvodbprocessed.sql");
+    if (!query) {
+      // rollback
+      if (!psDBRollback(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+      }
+      psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+      psFree(where);
+      return false;
+    }
+
+    // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE
+    if (where && psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+      psStringAppend(&query, " AND %s", whereClause);
+      psFree(whereClause);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      // rollback
+      if (!psDBRollback(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+      }
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      psFree(where);
+            return false;
+    }
+    psFree(query);
+  }
+  psFree(where);
+
+  if (!psDBCommit(config->dbh)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return false;
+  }
+
+  return true;
+}
+
+
+
+static bool updateminidvodbprocessedMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, false);
+  psMetadata *where = psMetadataAlloc();
+
+  PXOPT_LOOKUP_U64(minidvodb_id,  config->args, "-minidvodb_id", true, false);
+  PXOPT_LOOKUP_U64(merge_order,  config->args, "-set_merge_order", false, false);
+  PXOPT_LOOKUP_S16(fault,  config->args, "-set_fault", false, false);
+  PXOPT_LOOKUP_F32(dtime_relphot,  config->args, "-set_dtime_relphot", false, false);
+  PXOPT_LOOKUP_F32(dtime_resort,  config->args, "-set_dtime_resort", false, false);
+  PXOPT_LOOKUP_F32(dtime_merge,  config->args, "-set_dtime_merge", false, false);
+
+  PXOPT_COPY_S64(config->args, where, "-minidvodb_id",     "minidvodbProcessed.minidvodb_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-minidvodb_name",     "minidvodbRun.minidvodb_name", "==");
+
+
+  if (!psListLength(where->list)) {
+    psFree(where);
+    psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+    return false;
+  }
+
+  psString query = psStringCopy("UPDATE minidvodbProcessed JOIN minidvodbRun USING (minidvodb_id) SET ");
+  int cnt = 0;
+  psString comma = ",";
+  if (fault) {
+    psStringAppend(&query, " fault = %d", fault);
+  cnt++;
+  }
+
+  if (merge_order) {
+    if (cnt) {
+      psStringAppend(&query, "%s", comma);
+    }
+
+    psStringAppend(&query, " merge_order = %" PRId64, merge_order);
+    cnt++;
+  }
+
+  if (dtime_relphot) {
+    if (cnt) {
+      psStringAppend(&query, "%s", comma);
+    }
+    psStringAppend(&query, " dtime_relphot = %f", dtime_relphot);
+    cnt++;
+  }
+
+  if (dtime_resort) {
+    if (cnt) {
+      psStringAppend(&query, "%s", comma);
+    }
+    psStringAppend(&query, " dtime_resort = %f", dtime_resort);
+    cnt++;
+  }
+
+  if (dtime_merge) {
+    if (cnt) {
+      psStringAppend(&query, "%s", comma);
+    }
+    psStringAppend(&query, " dtime_merge = %f", dtime_merge);
+    cnt++;
+ }
+
+  psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+  psStringAppend(&query, " WHERE %s", whereClause);
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+   psError(PS_ERR_UNKNOWN, false, "database error %s", query);
+   psFree(query);
+   return false;
+  }
+
+  psFree(query);
+  psFree(where);
+
+  return true;
+}
+
+

trunk/ippTools/src/addtool.h

@@ -39 +39 @@
     ADDTOOL_MODE_DONECLEANUP,
     ADDTOOL_MODE_EXPORTRUN,
-    ADDTOOL_MODE_IMPORTRUN
+    ADDTOOL_MODE_IMPORTRUN,
+    ADDTOOL_MODE_ADDMINIDVODBRUN,
+    ADDTOOL_MODE_UPDATEMINIDVODBRUN,
+    ADDTOOL_MODE_LISTMINIDVODBRUN,
+    ADDTOOL_MODE_FLIPMINIDVODBRUN,
+    ADDTOOL_MODE_CHECKMINIDVODBRUNADDRUN,
+    ADDTOOL_MODE_ADDMINIDVODBPROCESSED,
+    ADDTOOL_MODE_LISTMINIDVODBPROCESSED,
+    ADDTOOL_MODE_REVERTMINIDVODBPROCESSED,
+    ADDTOOL_MODE_UPDATEMINIDVODBPROCESSED
 } addtoolMode;
 

trunk/ippTools/src/addtoolConfig.c

@@ -63 +63 @@
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend",           0, "do not actually modify the database", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple",            0, "use the simple output format", false);
-
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-set_minidvodb",            0, "use minidvodb", false);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0,   "define minidvodb_group", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0,   "define minidvodb_bname", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0,   "define new data_group", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0,   "define new dist_group", NULL);
@@ -84 +86 @@
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0,   "define new dist_group", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0,         "define new note", NULL);
+
     // -pendingexp
     psMetadata *pendingexpArgs = psMetadataAlloc();
@@ -98 +101 @@
     psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0,            "define addtool ID (required)", 0);
     psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0,            "define base output location", NULL);
+     psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-dvodb_path", 0,            "define base output location", NULL);
     psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN);
     psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
@@ -142 +146 @@
     psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label",  0,            "name of a label to unmask (required)", NULL);
 
+    // -addminidvodbruns
+    psMetadata *addminidvodbrunArgs = psMetadataAlloc();
+    psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name",        0, "define minidvodb_name", NULL);
+    psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group",        0, "define minidvodb_group (required)", NULL);
+    psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path",        0, "define path for minidvodb", NULL);
+    psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path",        0, "define path for the merged dvodb (required)", NULL);
+    psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_state",        0, "define state", NULL);
+
+    // -updateminidvodbruns
+    psMetadata *updateminidvodbrunArgs = psMetadataAlloc();
+    psMetadataAddU64(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_id",        0, "search by minidvodb_id ", 0);
+    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_group",        0, "search by minidvodb_name (LIKE)", NULL);
+    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name",        0, "search by minidvodb_name (LIKE)", NULL);
+    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_path",        0, "search by path for minidvodb", NULL);
+    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-mergedvodb_path",        0, "search by path for the merged dvodb", NULL);
+    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-state",        0, "search by state", NULL);
+    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name",        0, "define minidvodb_name", NULL);
+    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path",        0, "define path for minidvodb", NULL);
+    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path",        0, "define path for minidvodb", NULL);
+    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group",        0, "define path for the merged dvodb", NULL);
+    psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_state",        0, "define state", NULL);
+    
+    // -listminidvodbrunArgs
+    psMetadata *listminidvodbrunArgs = psMetadataAlloc();
+    psMetadataAddU64(listminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_id",        0, "search by minidvodb_id", 0);
+    psMetadataAddStr(listminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name",        0, "search by minidvodb_name (LIKE)", NULL);
+    psMetadataAddStr(listminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_group",     0,    "search by minidvodb.minidvodb_group", NULL);
+    psMetadataAddStr(listminidvodbrunArgs, PS_LIST_TAIL, "-state",        0, "search by state", NULL);
+    psMetadataAddU64(listminidvodbrunArgs, PS_LIST_TAIL, "-limit",        0, "limit to N items", 0);
+    psMetadataAddBool(listminidvodbrunArgs, PS_LIST_TAIL, "-simple",        0, "simple output", false);
+    
+    //psMetadataAddBool(listminidvodbrunArgs, PS_LIST_TAIL, "-finished_addrun",        0, "limit to minidvodbs with completed addRuns (none in new state)", false);
+    psMetadata *flipminidvodbrunArgs = psMetadataAlloc();
+    psMetadataAddStr(flipminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_group",     0,    "for the supplied minidvodb_group (required): flip the current 'new' to 'active', the current 'active' to 'waiting'", NULL);
+    
+    psMetadata *checkminidvodbrunaddrunArgs = psMetadataAlloc();
+    psMetadataAddStr(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-minidvodb_group",     0,    "for the supplied minidvodb_group (required): check if the addRun stage is complete (all in addRun.state = full) ", NULL);
+    psMetadataAddStr(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-minidvodb_name",     0,    "for the supplied minidvodb_name: check if the addRun stage is complete (all in addRun.state = full) ", NULL);
+    psMetadataAddStr(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-state",     0,    "limit by minidvodbRun state ", NULL);
+    psMetadataAddU64(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-limit",        0, "limit to N items", 0);
+    psMetadataAddBool(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-simple",        0, "simple output", false);
+    psMetadataAddBool(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-all_addrun_states",        0, "list all minidvodbRun.minidvodb_names, not just ones that have complete addRuns", false);
+
+    psMetadata *addminidvodbprocessedArgs = psMetadataAlloc();
+    psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0,    "define minidvodb_id (required)", 0);
+    psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-merge_order",    0,    "define merge order", 0);
+    psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_relphot",  0,    "define elapsed time for relphot (seconds)", NAN);
+    psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_resort", 0,    "define elapsed time for resort (seconds)", NAN);
+    psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_merge",    0,    "define elapsed time for DVO merge (seconds)", NAN);
+    psMetadataAddTime(addminidvodbprocessedArgs, PS_LIST_TAIL, "-epoch",         0,    "time merge is finished", NULL);
+    psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-mergedvodb_path",0,    "path of merged dvodb", NULL);
+    psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",0,    "minidvodb_group", NULL);
+    psMetadataAddS16(addminidvodbprocessedArgs, PS_LIST_TAIL, "-fault",          0,    "set fault code", 0);
+
+    psMetadata *listminidvodbprocessedArgs = psMetadataAlloc();
+    psMetadataAddU64(listminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id",        0, "search by minidvodb_id", 0);
+    psMetadataAddStr(listminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name",        0, "search by minidvodb_name", NULL);
+    psMetadataAddStr(listminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",        0, "search by minidvodb.minidvodb_group", NULL);
+    psMetadataAddU64(listminidvodbprocessedArgs, PS_LIST_TAIL, "-limit",        0, "limit to N items", 0);
+    psMetadataAddBool(listminidvodbprocessedArgs, PS_LIST_TAIL, "-simple",        0, "simple output", false);
+    psMetadataAddBool(listminidvodbprocessedArgs, PS_LIST_TAIL, "-faulted",        0, "limit to faulted state", false);
+
+    psMetadata *revertminidvodbprocessedArgs = psMetadataAlloc();
+    psMetadataAddU64(revertminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id",        0, "search by minidvodb_id", 0);
+    psMetadataAddStr(revertminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",        0, "search by addRun.minidvodb_group", NULL);
+
+    psMetadata *updateminidvodbprocessedArgs = psMetadataAlloc();
+    psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id",        0, "search by minidvodb_id", 0);
+    psMetadataAddStr(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name",        0, "search by minidvodb_name", NULL);
+    psMetadataAddS16(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_fault",  0,            "set fault code", 0);
+    psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_merge_order",    0,    "define merge order", 0);
+    psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_relphot",  0,    "define elapsed time for relphot (seconds)", 0);
+    psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_resort", 0,    "define elapsed time for resort (seconds)", 0);
+    psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_merge",    0,    "define elapsed time for DVO merge (seconds)", 0);
+    
     psMetadata *argSets = psMetadataAlloc();
     psMetadata *modes = psMetadataAlloc();
@@ -154 +233 @@
     PXOPT_ADD_MODE("-block",                "set a label block",                    ADDTOOL_MODE_BLOCK,         blockArgs);
     PXOPT_ADD_MODE("-masked",               "show blocked labels",                  ADDTOOL_MODE_MASKED,        maskedArgs);
-    PXOPT_ADD_MODE("-unblock",              "remove a label block",                 ADDTOOL_MODE_UNBLOCK,       unblockArgs);
+    PXOPT_ADD_MODE("-addminidvodbrun",      "create minidvodbs ",                   ADDTOOL_MODE_ADDMINIDVODBRUN, addminidvodbrunArgs);
+    PXOPT_ADD_MODE("-updateminidvodbrun",   "change minidvodb properties",          ADDTOOL_MODE_UPDATEMINIDVODBRUN,     updateminidvodbrunArgs);
+    PXOPT_ADD_MODE("-listminidvodbrun",     "list minidvodbs",                      ADDTOOL_MODE_LISTMINIDVODBRUN,       listminidvodbrunArgs);
+    PXOPT_ADD_MODE("-flipminidvodbrun",     "flip minidvodbs",                      ADDTOOL_MODE_FLIPMINIDVODBRUN,       flipminidvodbrunArgs);
+    PXOPT_ADD_MODE("-checkminidvodbrunaddrun", "check minidvodbs to see if addRuns are completed", ADDTOOL_MODE_CHECKMINIDVODBRUNADDRUN,       checkminidvodbrunaddrunArgs);
+    PXOPT_ADD_MODE("-addminidvodbprocessed","add a processed minidvodb",            ADDTOOL_MODE_ADDMINIDVODBPROCESSED,  addminidvodbprocessedArgs);
+    PXOPT_ADD_MODE("-listminidvodbprocessed","list processed minidvodbs",           ADDTOOL_MODE_LISTMINIDVODBPROCESSED, listminidvodbprocessedArgs);
+    PXOPT_ADD_MODE("-revertminidvodbprocessed","revert processed minidvobs",        ADDTOOL_MODE_REVERTMINIDVODBPROCESSED,     revertminidvodbprocessedArgs);
+    PXOPT_ADD_MODE("-updateminidvodbprocessed","change processed minidvodb properties",ADDTOOL_MODE_UPDATEMINIDVODBPROCESSED,  updateminidvodbprocessedArgs);
+ 
+ 
+
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

trunk/ippTools/src/flatcorr.c

@@ -671 +671 @@
                 row->dvodb,
                 NULL,       // note is not propagated
-                0)) {
+                0,
+                0,  //The minidvodb stuff is off
+                NULL,
+                NULL)) {
             if (!psDBRollback(config->dbh)) {
                 psError(PS_ERR_UNKNOWN, false, "database error");

trunk/ippTools/src/pstamptool.h

@@ -22 +22 @@
 
 #include "pxtools.h"
+
+#define PSTAMP_FIRST_ERROR_CODE 0       // First error code value to consider
 
 typedef enum {

trunk/ippTools/src/pxadd.c

@@ -130 +130 @@
                        char *dvodb,
                        char *note,
-                       bool image_only)
+                       bool image_only,
+                       bool minidvodb,
+                       char *minidvodb_group,
+                       char *minidvodb_name)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -158 +161 @@
                          note     ? note     : "NULL",
                          image_only,
+                         minidvodb,
+                         minidvodb_group,
+                         minidvodb_name,
                          (long long) cam_id
     )) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+      psError(PS_ERR_UNKNOWN, false, "database error %s", query);
         return false;
     }

trunk/ippTools/src/pxadd.h

@@ -38 +38 @@
                        char *dvodb,
                        char *note,
-                       bool image_only);
+                       bool image_only,
+                       bool minidvodb,
+                       char *minidvodb_group,
+                       char *minidvodb_name);
 
 #endif // PXADD_H