Changeset 28375 for trunk

Timestamp:

Jun 16, 2010, 5:30:27 PM (16 years ago)

Author:

watersc1

Message:

merge of stack association and warp/stack/diff jpegs

Location:

trunk

Files:

• . (modified) (1 prop)
• Ohana (modified) (1 prop)
• dbconfig/changes.txt (modified) (4 diffs)
• dbconfig/diff.md (modified) (1 diff)
• dbconfig/stack.md (modified) (1 diff)
• dbconfig/warp.md (modified) (1 diff)
• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/MANIFEST (modified) (1 diff)
• ippScripts/scripts/skycell_jpeg.pl (copied) (copied from branches/czw_branch/20100519/ippScripts/scripts/skycell_jpeg.pl )
• ippTasks/diff.pro (modified) (7 diffs)
• ippTasks/stack.pro (modified) (7 diffs)
• ippTasks/warp.pro (modified) (6 diffs)
• ippTools/share/Makefile.am (modified) (4 diffs)
• ippTools/share/difftool_addsummary.sql (copied) (copied from branches/czw_branch/20100519/ippTools/share/difftool_addsummary.sql )
• ippTools/share/difftool_tosummary.sql (copied) (copied from branches/czw_branch/20100519/ippTools/share/difftool_tosummary.sql )
• ippTools/share/pxadmin_create_tables.sql (modified) (3 diffs)
• ippTools/share/pxadmin_drop_tables.sql (modified) (1 diff)
• ippTools/share/stacktool_addsummary.sql (copied) (copied from branches/czw_branch/20100519/ippTools/share/stacktool_addsummary.sql )
• ippTools/share/stacktool_associationdefine_select.sql (copied) (copied from branches/czw_branch/20100519/ippTools/share/stacktool_associationdefine_select.sql ) (1 diff)
• ippTools/share/stacktool_sassskyfile.sql (copied) (copied from branches/czw_branch/20100519/ippTools/share/stacktool_sassskyfile.sql )
• ippTools/share/stacktool_tosummary.sql (copied) (copied from branches/czw_branch/20100519/ippTools/share/stacktool_tosummary.sql )
• ippTools/share/warptool_addsummary.sql (copied) (copied from branches/czw_branch/20100519/ippTools/share/warptool_addsummary.sql )
• ippTools/share/warptool_tosummary.sql (copied) (copied from branches/czw_branch/20100519/ippTools/share/warptool_tosummary.sql )
• ippTools/src/difftool.c (modified) (3 diffs)
• ippTools/src/difftool.h (modified) (1 diff)
• ippTools/src/difftoolConfig.c (modified) (3 diffs)
• ippTools/src/stacktool.c (modified) (15 diffs)
• ippTools/src/stacktool.h (modified) (1 diff)
• ippTools/src/stacktoolConfig.c (modified) (4 diffs)
• ippTools/src/warptool.c (modified) (3 diffs)
• ippTools/src/warptool.h (modified) (1 diff)
• ippTools/src/warptoolConfig.c (modified) (2 diffs)
• ippconfig/recipes/filerules-mef.mdc (modified) (1 diff)
• ippconfig/recipes/filerules-simple.mdc (modified) (1 diff)
• ippconfig/recipes/filerules-split.mdc (modified) (1 diff)
• ippconfig/recipes/ppSkycell.config (modified) (1 diff)
• ppImage/src/ppImageLoop.c (modified) (2 diffs)
• ppImage/src/ppImagePhotom.c (modified) (3 diffs)
• ppSkycell/src/ppSkycell.h (modified) (1 diff)
• ppSkycell/src/ppSkycellCamera.c (modified) (2 diffs)
• ppSkycell/src/ppSkycellLoop.c (modified) (2 diffs)
• psModules (modified) (1 prop)
• psconfig/tagsets/ipp-2.9.dist (modified) (1 diff)
• psphot (modified) (1 prop)
• psphot/src/psphotReadout.c (modified) (18 diffs)

trunk/dbconfig/changes.txt

@@ -1634 +1634 @@
 ALTER TABLE diffSkyfile ADD COLUMN maskfrac_advisory FLOAT after maskfrac_magic;
 
+
 ALTER TABLE chipRun CHANGE maskfrac_npix  maskfrac_npix FLOAT;
 ALTER TABLE camRun CHANGE maskfrac_npix  maskfrac_npix FLOAT;
@@ -1730 +1731 @@
 -- this adds the new tables 
 
-
 CREATE TABLE minidvodbRun (
 minidvodb_id BIGINT AUTO_INCREMENT,
@@ -1741 +1741 @@
        PRIMARY KEY(minidvodb_id), KEY(minidvodb_name), INDEX(minidvodb_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
-
-
 
 CREATE TABLE minidvodbProcessed (
@@ -1791 +1789 @@
     FOREIGN KEY(diff_phot_id) REFERENCES diffPhotRun(diff_phot_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+
+-- Version 1.1.69 warp/stack/diff summary and stack associations.
+
+CREATE TABLE warpSummary (
+    warp_id     BIGINT,
+    projection_cell VARCHAR(64) NOT NULL,
+    path_base   VARCHAR(255) NOT NULL,
+    PRIMARY KEY(warp_id),
+    KEY(projection_cell),
+    FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE diffSummary (
+    diff_id     BIGINT,
+    projection_cell VARCHAR(64) NOT NULL,
+    path_base   VARCHAR(255) NOT NULL,
+    PRIMARY KEY(diff_id),
+    KEY(projection_cell),
+    FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE stackAssociation (
+    sass_id      BIGINT AUTO_INCREMENT,
+    data_group   VARCHAR(64) NOT NULL,
+    projection_cell VARCHAR(64) NOT NULL,
+    tess_id       VARCHAR(64) NOT NULL,
+    filter        VARCHAR(64) NOT NULL,
+    PRIMARY KEY(sass_id),
+    KEY(data_group),
+    KEY(projection_cell),
+    KEY(tess_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE stackAssociationMap (
+    sass_id      BIGINT,
+    stack_id     BIGINT,
+    PRIMARY KEY(sass_id, stack_id),
+    FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id),
+    FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE stackSummary (
+    sass_id     BIGINT,
+    projection_cell VARCHAR(64) NOT NULL,
+    path_base   VARCHAR(255) NOT NULL,
+    PRIMARY KEY(sass_id),
+    KEY(projection_cell),
+    FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;

trunk/dbconfig/diff.md

@@ -75 +75 @@
     maskfrac_advisory F32   0.0
 END
+
+diffSummary METADATA
+    diff_id         S64      0       # Primary Key fkey(diff_id) ref diffRun(diff_id)
+    projection_cell STR     32      # Primary Key
+    path_base       STR     255
+END

trunk/dbconfig/stack.md

@@ -53 +53 @@
     quality            S16    0
 END
+
+stackSummary METADATA
+    sass_id        S64      0       # Primary Key fkey(sass_id) ref stackAssociation(sass_id)
+    projection_cell  STR            32      # Primary Key
+    path_base      STR      255
+END
+
+stackAssociation METADATA
+    sass_id        S64      0       # Primary Key AUTO_INCREMENT
+    data_group     STR      64      # Key
+    projection_cell  STR      64      # Key
+    tess_id        STR      64      # Key
+    filter         STR      64
+END
+
+stackAssociationMap METADATA
+    sass_id        S64      0       # Primary Key fkey(sass_id) ref stackAssociation(sass_id)
+    stack_id       S64      0       # Primary Key fkey(stack_id) ref stackRun(stack_id)
+END

trunk/dbconfig/warp.md

@@ -79 +79 @@
     label       STR         64      # Primary Key
 END
+
+warpSummary METADATA
+    warp_id        S64      0       # Primary Key fkey(warp_id) ref warpRun(warp_id)
+    projection_cell  STR            32      # Primary Key
+    path_base      STR      255
+END

trunk/ippScripts/Build.PL

@@ -109 +109 @@
         scripts/bundle_detrends.pl
         scripts/ipp_cluster_load_monitor.pl
+        scripts/skycell_jpeg.pl
         scripts/diffphot.pl
     )],

trunk/ippScripts/MANIFEST

@@ -42 +42 @@
 scripts/bundle_detrends.pl
 scripts/ipp_cluster_load_monitor.pl
+scripts/skycell_jpeg.pl
 t/00_distribution.t

trunk/ippTasks/diff.pro

@@ -11 +11 @@
 book init diffSkyfile
 #book init diffCleanup
+book init diffPendingSummary
 
 ### Database lists
@@ -16 +17 @@
 $diffAdvance_DB = 0
 $diff_revert_DB = 0
+$diffSummary_DB = 0
 #$diffCleanup_DB = 0
 
@@ -27 +29 @@
 macro diff.reset
   book init diffSkyfile
+  book init diffPendingSummary
 #  book init diffCleanup
 end
@@ -44 +47 @@
     active false
   end
+  task diff.summary.load
+    active true
+  end
+  task diff.summary.run
+    active true
+  end
 end
 
@@ -58 +67 @@
   end
   task diff.revert
+    active false
+  end
+  task diff.summary.load
+    active false
+  end
+  task diff.summary.run
     active false
   end
@@ -193 +208 @@
     # host anyhost
     # $WORKDIR = $WORKDIR_TEMPLATE
-
     if (($DIFF_MODE == 1)||("$DIFF_MODE" == "NULL")) 
         $DIFF_TAG = ""
@@ -477 +491 @@
   end
 end
+
+### Load tasks for doing the diffs
+### Tasks are loaded into diffPendingSkyCell.
+task           diff.summary.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 1200
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/diff.summary.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = difftool -tosummary
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$diffSummary_DB
+      $run = $run -dbname $DB:$diffSummary_DB
+      $diffSummary_DB ++
+      if ($diffSummary_DB >= $DB:n) set diffSummary_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    # XXX change tess_id to tess_dir when db is updated
+    ipptool2book stdout diffPendingSummary -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook diffPendingSummary
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup diffPendingSummary
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+### Run tasks for calculating the diff overlaps
+### Tasks are taken from diffPendingSkyCell.
+task           diff.summary.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $RUNEXEC
+
+    book npages diffPendingSummary -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images in diffPendingSkyCell (pantaskState == INIT)
+    book getpage diffPendingSummary 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword diffPendingSummary $pageName pantaskState RUN
+    book getword diffPendingSummary $pageName diff_id -var DIFF_ID
+    book getword diffPendingSummary $pageName camera -var CAMERA
+    book getword diffPendingSummary $pageName workdir -var WORKDIR_TEMPLATE
+    book getword diffPendingSummary $pageName dbname -var DBNAME
+    book getword diffPendingSummary $pageName tess_id -var TESS_DIR
+    book getword diffPendingSummary $pageName diff_mode -var DIFF_MODE
+    book getword diffPendingSummary $pageName state -var RUN_STATE
+
+    # set the host and workdir based on the skycell hash
+    host anyhost
+    strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR
+#    set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+
+#    if ("$PATH_BASE" == "NULL") 
+
+    if (($DIFF_MODE == 1)||("$DIFF_MODE" == "NULL")) 
+        $DIFF_TAG = ""
+    end
+    if ($DIFF_MODE == 2)
+        $DIFF_TAG = "WS."
+    end 
+    if ($DIFF_MODE == 3)
+        $DIFF_TAG = "SW."
+    end
+    if ($DIFF_MODE == 4)
+        $DIFF_TAG = "SS."
+    end
+
+    basename $TESS_DIR -var TESS_ID
+
+    ## generate outroot specific to this exposure
+    sprintf outroot "%s/%s/%s.%sdif.%s.summary" $WORKDIR $TESS_ID $TESS_ID $DIFF_TAG $DIFF_ID
+
+
+    stdout $LOGDIR/diff.summary.log
+    stderr $LOGDIR/diff.summary.log
+
+    $run = skycell_jpeg.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --outroot $outroot
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit diffPendingSummary $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword diffPendingSummary $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword diffPendingSummary $options:0 pantaskState TIMEOUT
+  end
+end

trunk/ippTasks/stack.pro

@@ -10 +10 @@
 ### Initialise the books containing the tasks to do
 book init stackSumSkyfile
+book init stackPendingSummary
 #book init stackCleanup
 
@@ -16 +17 @@
 $stack_revert_DB = 0
 #$stackCleanup_DB = 0
+$stackSummary_DB = 0
 
 ### Check status of stacking tasks
@@ -26 +28 @@
 macro stack.reset
   book init stackSumSkyfile
+  book init stackPendingSummary
 #  book init stackCleanup
 end
@@ -40 +43 @@
     active false
   end
+  task stack.summary.load
+    active true
+  end
+  task stack.summary.run
+    active true
+  end
 end
 
@@ -51 +60 @@
   end
   task stack.revert
+    active false
+  end
+  task stack.summary.load
+    active false
+  end
+  task stack.summary.run
     active false
   end
@@ -349 +364 @@
 # end
 
+
+
+
 task stack.revert
   host         local
@@ -400 +418 @@
 end
 
+### Load tasks for doing the stack summary
+### Tasks are loaded into stackPendingSummary
+task           stack.summary.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 1200
+  trange       07:00:00 08:00:00 -nmax 1
+  npending     1
+
+  stdout       NULL
+  stderr       $LOGDIR/stack.summary.load
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = stacktool -tosummary
+    if ($DB:n == 0) 
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$stackSummary_DB
+      $run = $run -dbname $DB:$stackSummary_DB
+      $stackSummary_DB ++
+      if ($stackSummary_DB >= $DB:n) set stackSummary_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit     0
+    # convert 'stdout' to book format
+    ipptool2book stdout stackPendingSummary -key sass_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook stackPendingSummary
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup stackPendingSummary
+  end
+
+  # locked list
+  task.exit     default
+    showcommand failure
+  end
+
+  task.exit     crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit     timeout
+    showcommand timeout
+  end
+end
+
+### Run tasks for doing the stack summary
+task           stack.summary.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $RUNEXEC
+
+    book npages stackPendingSummary -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+
+    # look for new images in stackPendingSummary (pantaskState == INIT)
+    book getpage stackPendingSummary 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword stackPendingSummary $pageName pantaskState RUN
+    book getword stackPendingSummary $pageName sass_id -var SASS_ID
+    book getword stackPendingSummary $pageName camera -var CAMERA
+    book getword stackPendingSummary $pageName workdir -var WORKDIR_TEMPLATE
+    book getword stackPendingSummary $pageName dbname -var DBNAME
+    book getword stackPendingSummary $pageName tess_id -var TESS_DIR
+    book getword stackPendingSummary $pageName state -var RUN_STATE
+    book getword stackPendingSummary $pageName projection_cell -var projection_cell
+
+    # set the host and workdir
+    host anyhost
+    strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR
+
+    basename $TESS_DIR -var TESS_ID
+
+    ## generate outroot specific to this association
+    sprintf outroot "%s/%s/%s.stk.%s.summary" $WORKDIR $TESS_ID $TESS_ID $SASS_ID
+
+    stdout $LOGDIR/stack.summary.log
+    stderr $LOGDIR/stack.summary.log
+
+    $run = skycell_jpeg.pl --stage stack --stage_id $SASS_ID --camera $CAMERA --outroot $outroot
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+        echo command $run
+    end
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit       default
+    process_exit stackPendingSummary $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit       crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword stackPendingSummary $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit       timeout
+    showcommand timeout
+    book setword stackPendingSummary $options:0 pantaskState TIMEOUT
+  end
+end
+

trunk/ippTasks/warp.pro

@@ -13 +13 @@
 book init warpPendingSkyCell
 book init warpPendingCleanup
+book init warpPendingSummary
 
 ### Database lists
@@ -19 +20 @@
 $warp_revert_overlap_DB = 0
 $warp_revert_warped_DB = 0
+$warpSummary_DB = 0
 
 ### Check status of warping tasks
@@ -30 +32 @@
   book init warpInputExp
   book init warpPendingSkyCell
+  book init warpPendingSummary
 end
 
@@ -55 +58 @@
     active true
   end
+  task warp.summary.load
+    active true
+  end
+  task warp.summary.run
+    active true
+  end
 end
 
@@ -78 +87 @@
   end
   task warp.revert.warped
+    active false
+  end
+  task warp.summary.load
+    active false
+  end
+  task warp.summary.run
     active false
   end
@@ -525 +540 @@
   end
 end
+
+### Load tasks for doing the warps
+### Tasks are loaded into warpPendingSkyCell.
+task           warp.summary.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 1200
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/warp.summary.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = warptool -tosummary
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$warpSummary_DB
+      $run = $run -dbname $DB:$warpSummary_DB
+      $warpSummary_DB ++
+      if ($warpSummary_DB >= $DB:n) set warpSummary_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    # XXX change tess_id to tess_dir when db is updated
+    ipptool2book stdout warpPendingSummary -key warp_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook warpPendingSummary
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup warpPendingSummary
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+### Run tasks for calculating the warp overlaps
+### Tasks are taken from warpPendingSkyCell.
+task           warp.summary.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec $RUNEXEC
+
+    book npages warpPendingSummary -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new images in warpPendingSkyCell (pantaskState == INIT)
+    book getpage warpPendingSummary 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword warpPendingSummary $pageName pantaskState RUN
+    book getword warpPendingSummary $pageName warp_id -var WARP_ID
+    book getword warpPendingSummary $pageName camera -var CAMERA
+    book getword warpPendingSummary $pageName workdir -var WORKDIR_TEMPLATE
+    book getword warpPendingSummary $pageName dbname -var DBNAME
+    book getword warpPendingSummary $pageName tess_id -var TESS_DIR
+    book getword warpPendingSummary $pageName exp_tag -var EXP_TAG
+    book getword warpPendingSummary $pageName state -var RUN_STATE
+
+    # set the host and workdir based on the skycell hash
+    host anyhost
+    strsub $WORKDIR_TEMPLATE @HOST@.0 $default_host -var WORKDIR
+#    set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+
+#    if ("$PATH_BASE" == "NULL") 
+        ## generate outroot specific to this exposure
+    basename $TESS_DIR -var TESS_ID
+
+    sprintf outroot "%s/%s/%s.wrp.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID $TESS_ID
+
+
+    stdout $LOGDIR/warp.summary.log
+    stderr $LOGDIR/warp.summary.log
+
+    $run = skycell_jpeg.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --outroot $outroot
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit warpPendingSummary $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword warpPendingSummary $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword warpPendingSummary $options:0 pantaskState TIMEOUT
+  end
+end

trunk/ippTools/share/Makefile.am

@@ -126 +126 @@
      difftool_skyfile.sql \
      difftool_todiffskyfile.sql \
+     difftool_tosummary.sql \
+     difftool_addsummary.sql \
      disttool_definebyquery_camera.sql \
      disttool_definebyquery_chip.sql \
@@ -278 +280 @@
      regtool_updatebyquery.sql \
      regtool_updatebyqueryimfile.sql \
+     stacktool_associationdefine_select.sql \
      stacktool_definebyquery_insert.sql \
      stacktool_definebyquery_insert_random_part1.sql \
@@ -293 +296 @@
      stacktool_revertsumskyfile_delete.sql \
      stacktool_sumskyfile.sql \
+     stacktool_sassskyfile.sql \
      stacktool_tosum.sql \
+     stacktool_tosummary.sql \
+     stacktool_addsummary.sql \
      staticskytool_definebyquery_select.sql \
      staticskytool_definebyquery_inputs.sql \
@@ -326 +332 @@
      warptool_updateskyfile.sql \
      warptool_warped.sql \
+     warptool_tosummary.sql \
+     warptool_addsummary.sql \
         diffphottool_definerun.sql \
         diffphottool_input.sql \

trunk/ippTools/share/pxadmin_create_tables.sql

@@ -981 +981 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
+CREATE TABLE warpSummary (
+       warp_id BIGINT,
+       projection_cell VARCHAR(64) NOT NULL,
+       path_base     VARCHAR(255) NOT NULL,
+       PRIMARY KEY(warp_id),
+       KEY(projection_cell),
+       FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
 CREATE TABLE stackRun (
         stack_id BIGINT AUTO_INCREMENT,
@@ -1049 +1058 @@
         KEY(quality),
         FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE stackAssociation (
+    sass_id      BIGINT AUTO_INCREMENT,
+    data_group   VARCHAR(64) NOT NULL,
+    projection_cell VARCHAR(64) NOT NULL,
+    tess_id       VARCHAR(64) NOT NULL,
+    filter        VARCHAR(64) NOT NULL,
+    PRIMARY KEY(sass_id),
+    KEY(data_group),
+    KEY(projection_cell),
+    KEY(tess_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE stackAssociationMap (
+    sass_id      BIGINT,
+    stack_id     BIGINT,
+    PRIMARY KEY(sass_id, stack_id),
+    FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id),
+    FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE stackSummary (
+    sass_id     BIGINT,
+    projection_cell VARCHAR(64) NOT NULL,
+    path_base   VARCHAR(255) NOT NULL,
+    PRIMARY KEY(sass_id),
+    KEY(projection_cell),
+    FOREIGN KEY(sass_id) REFERENCES stackAssociation(sass_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1147 +1185 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
+CREATE TABLE diffSummary (
+    diff_id     BIGINT,
+    projection_cell VARCHAR(64) NOT NULL,
+    path_base   VARCHAR(255) NOT NULL,
+    PRIMARY KEY(diff_id),
+    KEY(projection_cell),
+    FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
 CREATE TABLE magicRun (
         magic_id BIGINT AUTO_INCREMENT,

trunk/ippTools/share/pxadmin_drop_tables.sql

@@ -41 +41 @@
 DROP TABLE IF EXISTS warpSkyfile;
 DROP TABLE IF EXISTS warpMask;
+DROP TABLE IF EXISTS warpSummary;
 DROP TABLE IF EXISTS diffRun;
 DROP TABLE IF EXISTS diffInputSkyfile;
 DROP TABLE IF EXISTS diffSkyfile;
+DROP TABLE IF EXISTS diffSummary;
 DROP TABLE IF EXISTS stackRun;
 DROP TABLE IF EXISTS stackInputSkyfile;
 DROP TABLE IF EXISTS stackSumSkyfile;
+DROP TABLE IF EXISTS stackSummary;
+DROP TABLE IF EXISTS stackAssociation;
+DROP TABLE IF EXISTS stackAssociationMap;
 DROP TABLE IF EXISTS magicRun;
 DROP TABLE IF EXISTS magicInputSkyfile;

trunk/ippTools/share/stacktool_associationdefine_select.sql

@@ -2 +2 @@
        sass_id,
        data_group,
+       projection_cell,
        tess_id,
-       filter,
-       CASE WHEN LOCATE('.',skycell_id,9) > 0 THEN
-            SUBSTRING_INDEX(skycell_id,'.',2) ELSE
-            SUBSTRING_INDEX(skycell_id,'.',1) END
-       AS projection_cell
+       filter
+       FROM stackAssociation
+RIGHT JOIN (
+      SELECT 
+             data_group,
+             tess_id,
+             filter,
+             CASE WHEN LOCATE('.',skycell_id,9) > 0 THEN
+                  SUBSTRING_INDEX(skycell_id,'.',2) ELSE
+                  SUBSTRING_INDEX(skycell_id,'.',1) END
+             AS projection_cell
        FROM stackRun 
-       LEFT JOIN stackAssociationMap USING (stack_id)
-       WHERE stackRun.stack_id = @STACK_ID@
+       WHERE stackRun.stack_id = @STACK_ID@ 
+             ) AS RUN USING (data_group,tess_id,filter,projection_cell) LIMIT 1

trunk/ippTools/src/difftool.c

@@ -44 +44 @@
 static bool definewarpwarpMode(pxConfig *config);
 static bool definestackstackMode(pxConfig *config);
+static bool tosummaryMode(pxConfig *config);
+static bool addsummaryMode(pxConfig *config);
 static bool pendingcleanuprunMode(pxConfig *config);
 static bool pendingcleanupskyfileMode(pxConfig *config);
@@ -93 +95 @@
         MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
         MODECASE(DIFFTOOL_MODE_DEFINESTACKSTACK,      definestackstackMode);
+        MODECASE(DIFFTOOL_MODE_TOSUMMARY,             tosummaryMode);
+        MODECASE(DIFFTOOL_MODE_ADDSUMMARY,            addsummaryMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode);
@@ -2250 +2254 @@
 }
 
+static bool tosummaryMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-warp_id",    "diffSkyfile.warp_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-tess_id",    "diffSkyfile.tess_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-state",      "diffRun.state", "==");
+  PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs",  ">=");
+  PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawExp.dateobs",  "<=");
+  PXOPT_COPY_STR(config->args, where, "-filter",    "rawExp.filter", "LIKE");
+  PXOPT_COPY_S64(config->args, where, "-magicked", "diffSkyfile.magicked", "==");
+  pxAddLabelSearchArgs (config, where, "-label",   "diffRun.label", "LIKE");
+  pxAddLabelSearchArgs (config, where, "-data_group",   "diffRun.data_group", "LIKE");
+
+  PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+
+  PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  
+  // find all rawImfiles matching the default query
+  psString query = pxDataGet("difftool_tosummary.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    return false;
+  }
+
+  // generate where strings for arguments that require extra processing
+  // beyond PXOPT_COPY*
+  psString where2 = NULL;
+  if (!pxmagicAddWhere(config, &where2, "diffSkyfile")) {
+    psError(psErrorCodeLast(), false, "pxMagicAddWhere failed");
+    return false;
+  }
+  
+  if (psListLength(where->list)) {
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
+    psFree(whereClause);
+  } else if (!all && !where2) {
+    psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
+    return false;
+  }
+  
+  if (where2) {
+    if (psListLength(where->list)) {
+      psStringAppend(&query, " %s", where2);
+    } else {
+      psStringAppend(&query, " AND 1 %s", where2);
+    }
+  }
+  psFree(where);
+
+  // treat limit == 0 as "no limit"
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, " %s", limitString);
+    psFree(limitString);
+  }
+  
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+    
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("difftool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+  
+  if (psArrayLength(output)) {
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+  }
+  
+  psFree(output);
+  return(true);
+}
+static bool addsummaryMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+  PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
+  PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false);
+  PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
+
+  psString query = pxDataGet("difftool_addsummary.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    return(false);
+  }
+  if (!p_psDBRunQueryF(config->dbh, query, diff_id, projection_cell, path_base)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return(false);
+  }
+  psS64 numUpdated = psDBAffectedRows(config->dbh);
+  
+  if (numUpdated != 1) {
+    psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
+    psFree(query);
+    return false;
+  }
+  
+  psFree(query);
+
+  // Print anything here?
+  
+  return(true);
+}
+
+
 static bool pendingcleanuprunMode(pxConfig *config)
 {

trunk/ippTools/src/difftool.h

@@ -32 +32 @@
     DIFFTOOL_MODE_ADDDIFFSKYFILE,
     DIFFTOOL_MODE_ADVANCE,
+    DIFFTOOL_MODE_TOSUMMARY,
+    DIFFTOOL_MODE_ADDSUMMARY,
     DIFFTOOL_MODE_DIFFSKYFILE,
     DIFFTOOL_MODE_REVERTDIFFSKYFILE,

trunk/ippTools/src/difftoolConfig.c

@@ -323 +323 @@
     psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
 
+    // -tosummary
+    psMetadata *tosummaryArgs = psMetadataAlloc();
+    psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL,  "-diff_id", 0,           "search by diff ID", 0);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL,  "-tess_id",  0,          "search by tessellation ID", NULL);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-state",  0,            "search by state", NULL);
+    psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL,  "-exp_id",  0,           "search by exposure ID", 0);
+    psMetadataAddStr(tosummaryArgs , PS_LIST_TAIL, "-exp_name",  0,        "search by exposure name", NULL);
+    psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-warp_id", 0,         "search by warp ID", 0);
+    
+    psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_begin", 0,    "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_end", 0,      "search for exposures by time (<=)", NULL);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL,  "-filter", 0,           "search for filter", NULL);
+    psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL,  "-magicked", 0,         "search by magic id", 0);
+    psMetadataAddStr(tosummaryArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL);
+    psMetadataAddStr(tosummaryArgs,  PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(tosummaryArgs,  PS_LIST_TAIL, "-dist_group",  PS_META_DUPLICATE_OK, "search by diffRun dist_group (LIKE comparison)", NULL);
+    psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL,  "-destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL,  "-not_destreaked", 0, "search for runs that have not been destreaked", false);
+    
+    psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-all",  0,             "search without arguments", false);
+    psMetadataAddU64(tosummaryArgs, PS_LIST_TAIL,  "-limit",  0,            "limit result set to N items", 0);
+    psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
+    
+    // -addsummary
+    psMetadata *addsummaryArgs = psMetadataAlloc();
+    psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL,  "-diff_id", 0,           "search by diff ID", 0);
+    psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-projection_cell", 0, "set projection cell", NULL);
+    psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL);
+
     // -pendingcleanupskyfile
     psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc();
@@ -380 +409 @@
     psMetadata *importrunArgs = psMetadataAlloc();
     psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
-
 
 
@@ -415 +443 @@
     PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs);
     PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full", DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs);
+    PXOPT_ADD_MODE("-tosummary",            "show runs that can be summarized", DIFFTOOL_MODE_TOSUMMARY, tosummaryArgs);
+    PXOPT_ADD_MODE("-addsummary",           "add entry to the summary table", DIFFTOOL_MODE_ADDSUMMARY, addsummaryArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

trunk/ippTools/src/stacktool.c

@@ -39 +39 @@
 static bool addsumskyfileMode(pxConfig *config);
 static bool sumskyfileMode(pxConfig *config);
+static bool sassskyfileMode(pxConfig *config);
 static bool revertsumskyfileMode(pxConfig *config);
+static bool tosummaryMode(pxConfig *config);
+static bool addsummaryMode(pxConfig *config);
 static bool pendingcleanuprunMode(pxConfig *config);
 static bool pendingcleanupskyfileMode(pxConfig *config);
@@ -75 +78 @@
         MODECASE(STACKTOOL_MODE_ADDSUMSKYFILE,         addsumskyfileMode);
         MODECASE(STACKTOOL_MODE_SUMSKYFILE,            sumskyfileMode);
+        MODECASE(STACKTOOL_MODE_SASSSKYFILE,            sassskyfileMode);
         MODECASE(STACKTOOL_MODE_REVERTSUMSKYFILE,      revertsumskyfileMode);
+        MODECASE(STACKTOOL_MODE_TOSUMMARY,             tosummaryMode);
+        MODECASE(STACKTOOL_MODE_ADDSUMMARY,            addsummaryMode);
         MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
         MODECASE(STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode);
@@ -102 +108 @@
     exit(exit_status);
 }
+//stackAssociationRow *association = pxStackAssociationDefine(data_group,tess_id,filter,skycell_id);
+stackAssociationRow *pxStackAssociationDefine(pxConfig *config, psS64 stack_id) {
+  psString select = pxDataGet("stacktool_associationdefine_select.sql");
+  if (!select) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    return false;
+  }
+
+  psString idString = NULL;
+  psStringAppend(&idString, "%" PRId64, stack_id);
+  // Copy string to get around the issue with psStringSubstitute not believing
+  // that select is a psString.
+  psString rep = psStringCopy(select);
+  psFree(select);
+  select = rep;
+  psStringSubstitute(&select, idString, "@STACK_ID@");
+  psFree(idString);
+
+  if (!p_psDBRunQuery(config->dbh, select)) {
+    psError(PS_ERR_UNKNOWN,false, "database error");
+    psFree(select);
+    return(NULL);
+  }
+  psFree(select);
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+    return(NULL);
+  }
+  if (psArrayLength(output) != 1) {
+    psWarning("stacktool: incorrect number of rows found");
+    psFree(output);
+    return(NULL);
+  }
+  psMetadata *outrow = psMetadataAlloc();
+  for (long i = 0; i < output->n; i++) {
+    psMetadata *row = output->data[i];
+
+    printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,row,"sass_id"),
+           psMetadataLookupStr(NULL,row,"data_group"),
+           psMetadataLookupStr(NULL,row,"tess_id"),
+           psMetadataLookupStr(NULL,row,"filter"),
+           psMetadataLookupStr(NULL,row,"projection_cell"));
+
+    if (psMetadataLookupS64(NULL,row,"sass_id") == PS_MAX_S64) {
+      psMetadataAddS64(outrow,PS_LIST_TAIL,"sass_id",PS_META_REPLACE,"",0);
+    }
+    else {
+      psMetadataAddS64(outrow,PS_LIST_TAIL,"sass_id",PS_META_REPLACE,"",psMetadataLookupS64(NULL,row,"sass_id"));
+    }
+    psMetadataAddStr(outrow,PS_LIST_TAIL,"data_group",PS_META_REPLACE,"",psMetadataLookupStr(NULL,row,"data_group"));
+    psMetadataAddStr(outrow,PS_LIST_TAIL,"tess_id",PS_META_REPLACE,"",psMetadataLookupStr(NULL,row,"tess_id"));
+    psMetadataAddStr(outrow,PS_LIST_TAIL,"filter",PS_META_REPLACE,"",psMetadataLookupStr(NULL,row,"filter"));
+    psMetadataAddStr(outrow,PS_LIST_TAIL,"projection_cell",PS_META_REPLACE,"",
+                     psMetadataLookupStr(NULL,row,"projection_cell"));
+  }
+  printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,outrow,"sass_id"),
+         psMetadataLookupStr(NULL,outrow,"data_group"),
+         psMetadataLookupStr(NULL,outrow,"tess_id"),
+         psMetadataLookupStr(NULL,outrow,"filter"),
+         psMetadataLookupStr(NULL,outrow,"projection_cell"));
+  
+
+  psFree(output);
+  stackAssociationRow *sassRow = stackAssociationObjectFromMetadata(outrow);
+  psFree(outrow);
+  return(sassRow);
+}
+      
+      
+  
+                                              
 
 
@@ -395 +481 @@
         psFree(run);
 
+        //CZW Add an association entry here.
+        // Define the requested association, and insert it if it doesn't already exist
+        stackAssociationRow *association = pxStackAssociationDefine(config,stack_id);
+        psS64 sass_id;
+        if (!association->sass_id) {
+          psTrace("stacktool.association",2,"No required Association found. Adding.");
+
+          if (!stackAssociationInsertObject(config->dbh,association)) {
+            if (!psDBRollback(config->dbh)) {
+              psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(output);
+            psFree(run);
+            psFree(insert);
+            psFree(list);
+            psFree(association);
+            if (!psDBRollback(config->dbh)) {
+              psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return(false);
+          }
+          sass_id = psDBLastInsertID(config->dbh);
+          association->sass_id = sass_id;
+        }
+        // Insert the map entry for this row.
+        stackAssociationMapRow *maprow = stackAssociationMapRowAlloc(sass_id,stack_id);
+        if (!stackAssociationMapInsertObject(config->dbh,maprow)) {
+          if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+          }
+          psError(PS_ERR_UNKNOWN, false, "database error");
+          psFree(output);
+          psFree(run);
+          psFree(insert);
+          psFree(list);
+          psFree(association);
+          if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+          }
+          return(false);
+        }
+        
+        
         // Create a suitable insertion query for this run
         psString thisInsert = psStringCopy(insert);
@@ -552 +682 @@
     run->stack_id = psDBLastInsertID(config->dbh);
 
+    //CZW Add an association entry here.
+    
     // insert the stackInputSkyfile rows
     psListIterator *iter = psListIteratorAlloc(warp_ids->data.list, 0, false);
@@ -603 +735 @@
 #endif
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-stack_id",  "stack_id",   "==");
-    PXOPT_COPY_STR(config->args, where, "-label",     "label",     "==");
-    PXOPT_COPY_STR(config->args, where, "-state",     "state",     "==");
+    PXOPT_COPY_S64(config->args, where, "-stack_id",  "stackRun.stack_id",   "==");
+    PXOPT_COPY_STR(config->args, where, "-label",     "stackRun.label",     "==");
+    PXOPT_COPY_STR(config->args, where, "-state",     "stackRun.state",     "==");
+    PXOPT_COPY_STR(config->args, where, "-sass_id",   "stackAssociationMap.sass_id",  "==");
     if (!psListLength(where->list)) {
         psFree(where);
@@ -612 +745 @@
     }
 
+    //CZW join against stackAssociationMap
     psString query = psStringCopy("UPDATE stackRun");
 
@@ -1038 +1172 @@
 }
 
-
-static bool revertsumskyfileMode(pxConfig *config)
+static bool sassskyfileMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
-    pxAddLabelSearchArgs(config, where, "-label", "stackRun.label", "==");
-    PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
-
-    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
-        psFree(where);
-        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
-        return false;
-    }
-
-    // Delete product
-    psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql");
-    if (!delete) {
+    PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociation.sass_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-tess_id", "stackAssociation.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-projection_cell", "stackAssociation.projection_cell", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter", "stackAssociation.filter", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "stackAssociation.data_group", "LIKE");
+
+//  The following three selectors are incompatible with the sql so omit them
+//    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "==");
+//     PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
+//    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("stacktool_sassskyfile.sql");
+    if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
@@ -1063 +1201 @@
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&delete, " AND %s", whereClause);
+        psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, delete)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(delete);
-        psFree(where);
-        return false;
-    }
-    psFree(delete);
-
-    int numRows = psDBAffectedRows(config->dbh); // Number of row affected
-    psLogMsg("stacktool", PS_LOG_INFO, "Deleted %d rows", numRows);
-
-    psFree(where);
-
-    return true;
-}
-
-
-static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state)
-{
-    PS_ASSERT_PTR_NON_NULL(state, false);
-
-    // check that state is a valid string value
-    if (!pxIsValidState(state)) {
-        psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state);
-        return false;
-    }
-
-    char *query = "UPDATE stackRun SET state = '%s' WHERE stack_id = %"PRId64;
-    if (!p_psDBRunQueryF(config->dbh, query, state, stack_id)) {
-        psError(PS_ERR_UNKNOWN, false,
-                "failed to change state for stack_id %"PRId64, stack_id);
-        return false;
-    }
-
-    return true;
-}
-
-#ifdef notdef
-static bool setstackRunStateByLabel(pxConfig *config, const char *label, const char *state)
-{
-    PS_ASSERT_PTR_NON_NULL(state, false);
-
-    // check that state is a valid string value
-    if (!pxIsValidState(state)) {
-        psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state);
-        return false;
-    }
-
-    char *query = "UPDATE stackRun SET state = '%s' WHERE label = '%s'";
-    if (!p_psDBRunQueryF(config->dbh, query, state, label)) {
-        psError(PS_ERR_UNKNOWN, false,
-                "failed to change state for label %s", label);
-        return false;
-    }
-
-    return true;
-}
-#endif
-
-static bool pendingcleanuprunMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-
-    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
-    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-    psMetadata *where = psMetadataAlloc();
-    pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "==");
-
-    psString query = pxDataGet("stacktool_pendingcleanuprun.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (where && psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
-        psFree(whereClause);
-    }
+    } else if (!all) {
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
+        return false;
+    }
+
     psFree(where);
 
@@ -1165 +1226 @@
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
         return false;
     }
@@ -1174 +1244 @@
     }
 
+    if (psArrayLength(output)) {
+        if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool revertsumskyfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "stackRun.label", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
+
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    // Delete product
+    psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql");
+    if (!delete) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&delete, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, delete)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(delete);
+        psFree(where);
+        return false;
+    }
+    psFree(delete);
+
+    int numRows = psDBAffectedRows(config->dbh); // Number of row affected
+    psLogMsg("stacktool", PS_LOG_INFO, "Deleted %d rows", numRows);
+
+    psFree(where);
+
+    return true;
+}
+
+
+static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state)
+{
+    PS_ASSERT_PTR_NON_NULL(state, false);
+
+    // check that state is a valid string value
+    if (!pxIsValidState(state)) {
+        psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state);
+        return false;
+    }
+
+    char *query = "UPDATE stackRun SET state = '%s' WHERE stack_id = %"PRId64;
+    if (!p_psDBRunQueryF(config->dbh, query, state, stack_id)) {
+        psError(PS_ERR_UNKNOWN, false,
+                "failed to change state for stack_id %"PRId64, stack_id);
+        return false;
+    }
+
+    return true;
+}
+
+#ifdef notdef
+static bool setstackRunStateByLabel(pxConfig *config, const char *label, const char *state)
+{
+    PS_ASSERT_PTR_NON_NULL(state, false);
+
+    // check that state is a valid string value
+    if (!pxIsValidState(state)) {
+        psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state);
+        return false;
+    }
+
+    char *query = "UPDATE stackRun SET state = '%s' WHERE label = '%s'";
+    if (!p_psDBRunQueryF(config->dbh, query, state, label)) {
+        psError(PS_ERR_UNKNOWN, false,
+                "failed to change state for label %s", label);
+        return false;
+    }
+
+    return true;
+}
+#endif
+static bool tosummaryMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-stack_id",    "stackSumSkyfile.warp_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-sass_id",     "stackAssociationMap.sass_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-tess_id",    "stackSumSkyfile.tess_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-state",      "stackRun.state", "==");
+  PXOPT_COPY_STR(config->args, where, "-filter",    "stackRun.filter", "LIKE");
+  pxAddLabelSearchArgs (config, where, "-label",   "stackRun.label", "LIKE");
+  pxAddLabelSearchArgs (config, where, "-data_group",   "stackRun.data_group", "LIKE");
+  
+  PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+
+  PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  
+  // find all rawImfiles matching the default query
+  psString query = pxDataGet("stacktool_tosummary.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    return false;
+  }
+
+  // generate where strings for arguments that require extra processing
+  // beyond PXOPT_COPY*
+  if (psListLength(where->list)) {
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
+    psFree(whereClause);
+  } else if (!all) {
+    psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
+    return false;
+  }
+  
+  psFree(where);
+
+  // treat limit == 0 as "no limit"
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, " %s", limitString);
+    psFree(limitString);
+  }
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+    
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("stacktool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+  
+  if (psArrayLength(output)) {
     // negative simple so the default is true
-    if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupRun", !simple)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print array");
-        psFree(output);
-        return false;
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-static bool pendingcleanupskyfileMode(pxConfig *config)
+    if (!ippdbPrintMetadatas(stdout, output, "stackRun", !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+  }
+  
+  psFree(output);
+  return(true);
+}
+static bool addsummaryMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+  PXOPT_LOOKUP_S64(sass_id, config->args, "-sass_id", true, false);
+  PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false);
+  PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
+
+  psString query = pxDataGet("stacktool_addsummary.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    return(false);
+  }
+  if (!p_psDBRunQueryF(config->dbh, query, sass_id, projection_cell, path_base)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return(false);
+  }
+  psS64 numUpdated = psDBAffectedRows(config->dbh);
+  
+  if (numUpdated != 1) {
+    psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
+    psFree(query);
+    return false;
+  }
+  
+  psFree(query);
+
+  // Print anything here?
+  
+  return(true);
+}
+
+static bool pendingcleanuprunMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, NULL);
 
-    PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false);
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
     psMetadata *where = psMetadataAlloc();
-    if (stack_id) {
-        PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
-    }
+    PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociationMap.sass_id", "==");
+    
     pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "==");
 
-    psString query = pxDataGet("stacktool_pendingcleanupskyfile.sql");
+    psString query = pxDataGet("stacktool_pendingcleanuprun.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -1239 +1511 @@
 
     // negative simple so the default is true
-    if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupSkyfile", !simple)) {
+    if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupRun", !simple)) {
         psError(PS_ERR_UNKNOWN, false, "failed to print array");
         psFree(output);
@@ -1250 +1522 @@
 }
 
-
-static bool donecleanupMode(pxConfig *config)
+static bool pendingcleanupskyfileMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, NULL);
 
+    PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false);
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
-
-    psString query = pxDataGet("stacktool_donecleanup.sql");
+    if (stack_id) {
+        PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
+    }
+    pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "==");
+
+    psString query = pxDataGet("stacktool_pendingcleanupskyfile.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -1300 +1575 @@
 
     // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "stackPendingCleanupSkyfile", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool donecleanupMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    PXOPT_COPY_STR(config->args, where, "-sass_id", "sass_id", "==");
+
+    psString query = pxDataGet("stacktool_donecleanup.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (where && psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("stacktool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negative simple so the default is true
     if (!ippdbPrintMetadatas(stdout, output, "stackDoneCleanup", !simple)) {
         psError(PS_ERR_UNKNOWN, false, "failed to print array");
@@ -1329 +1666 @@
 }
 
+//CZW I have not added sass information to the export/import run modes yet.
 bool exportrunMode(pxConfig *config)
 {

trunk/ippTools/src/stacktool.h

@@ -33 +33 @@
     STACKTOOL_MODE_ADDSUMSKYFILE,
     STACKTOOL_MODE_SUMSKYFILE,
+    STACKTOOL_MODE_SASSSKYFILE,
     STACKTOOL_MODE_REVERTSUMSKYFILE,
+    STACKTOOL_MODE_TOSUMMARY,
+    STACKTOOL_MODE_ADDSUMMARY,
     STACKTOOL_MODE_PENDINGCLEANUPRUN,
     STACKTOOL_MODE_PENDINGCLEANUPSKYFILE,

trunk/ippTools/src/stacktoolConfig.c

@@ -197 +197 @@
     psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-all",  0,            "enable search without arguments", false);
 
+    // -sassskyfile
+    psMetadata *sassskyfileArgs = psMetadataAlloc();
+    psMetadataAddS64(sassskyfileArgs, PS_LIST_TAIL, "-sass_id", 0,           "search by stack association ID", 0);
+    psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID", 0);
+    psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-projection_cell", 0,         "search by projection cell", 0);
+
+    psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-data_group", 0,        "search by stackAssociation.data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(sassskyfileArgs, PS_LIST_TAIL, "-filter", 0,            "search by filter (LIKE comparison)", NULL);
+    psMetadataAddU64(sassskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
+    psMetadataAddBool(sassskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
+    psMetadataAddBool(sassskyfileArgs, PS_LIST_TAIL, "-all",  0,            "enable search without arguments", false);
+
     // -revertsumskyfile
     psMetadata *revertsumskyfileArgs= psMetadataAlloc();
@@ -204 +216 @@
     psMetadataAddBool(revertsumskyfileArgs, PS_LIST_TAIL, "-all",  0, "allow no search terms", 0);
 
+    // -tosummary
+    psMetadata *tosummaryArgs = psMetadataAlloc();
+    psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-stack_id", 0,  "search by stack ID", 0);
+    psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-sass_id", 0,  "search by stack association ID", 0);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-tess_id", 0,   "search by tessellation ID", NULL);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-state", 0,     "search by state", NULL);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-filter", 0,    "search by filter", NULL);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison)", NULL);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by stackRun dist_group (LIKE comparison)", NULL);
+
+    psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-all",  0,             "search without arguments", false);
+    psMetadataAddU64(tosummaryArgs, PS_LIST_TAIL,  "-limit",  0,            "limit result set to N items", 0);
+    psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
+
+    // -addsummary
+    psMetadata *addsummaryArgs = psMetadataAlloc();
+    psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-sass_id", 0,      "set stack Association ID", 0);
+    psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-projection_cell", 0, "set projection cell", NULL);
+    psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0,     "set summary path base", NULL);              
+    
     // -pendingcleanuprun
     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
@@ -252 +285 @@
     PXOPT_ADD_MODE("-addsumskyfile",   "", STACKTOOL_MODE_ADDSUMSKYFILE,   addsumskyfileArgs);
     PXOPT_ADD_MODE("-sumskyfile",      "list results of stackRun", STACKTOOL_MODE_SUMSKYFILE,      sumskyfileArgs);
+    PXOPT_ADD_MODE("-sassskyfile",      "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE,      sassskyfileArgs);
     PXOPT_ADD_MODE("-revertsumskyfile","", STACKTOOL_MODE_REVERTSUMSKYFILE,      revertsumskyfileArgs);
     PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);
@@ -257 +291 @@
     PXOPT_ADD_MODE("-donecleanup",           "show runs that have been cleaned",     STACKTOOL_MODE_DONECLEANUP,          donecleanupArgs);
     PXOPT_ADD_MODE("-updatesumskyfile",      "update fault code for sumskyfile",     STACKTOOL_MODE_UPDATESUMSKYFILE,          updatesumskyfileArgs);
+    PXOPT_ADD_MODE("-tosummary",            "show runs that can be summarized", STACKTOOL_MODE_TOSUMMARY, tosummaryArgs);
+    PXOPT_ADD_MODE("-addsummary",           "add entry to the summary table", STACKTOOL_MODE_ADDSUMMARY, addsummaryArgs);
     PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", STACKTOOL_MODE_EXPORTRUN, exportrunArgs);
     PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           STACKTOOL_MODE_IMPORTRUN, importrunArgs);

trunk/ippTools/src/warptool.c

@@ -48 +48 @@
 static bool maskedMode(pxConfig *config);
 static bool unblockMode(pxConfig *config);
+static bool tosummaryMode(pxConfig *config);
+static bool addsummaryMode(pxConfig *config);
 static bool pendingcleanuprunMode(pxConfig *config);
 static bool pendingcleanupwarpMode(pxConfig *config);
@@ -101 +103 @@
         MODECASE(WARPTOOL_MODE_MASKED,             maskedMode);
         MODECASE(WARPTOOL_MODE_UNBLOCK,            unblockMode);
+        MODECASE(WARPTOOL_MODE_TOSUMMARY,          tosummaryMode);
+        MODECASE(WARPTOOL_MODE_ADDSUMMARY,         addsummaryMode);
         MODECASE(WARPTOOL_MODE_PENDINGCLEANUPRUN,  pendingcleanuprunMode);
         MODECASE(WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupwarpMode);
@@ -1548 +1552 @@
 }
 
+static bool tosummaryMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpSkyfile.warp_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-tess_id",    "warpSkyfile.tess_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-state",      "warpRun.state", "==");
+  PXOPT_COPY_S64(config->args, where, "-exp_id",     "rawExp.exp_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-exp_name",   "rawExp.exp_name", "==");
+  PXOPT_COPY_S64(config->args, where, "-fake_id",    "fakeRun.fake_id", "==");
+  PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs",  ">=");
+  PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawExp.dateobs",  "<=");
+  PXOPT_COPY_STR(config->args, where, "-filter",    "rawExp.filter", "LIKE");
+  PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "==");
+  pxAddLabelSearchArgs (config, where, "-label",   "warpRun.label", "LIKE");
+  pxAddLabelSearchArgs (config, where, "-data_group",   "warpRun.data_group", "LIKE");
+
+  PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+
+  PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  
+  // find all rawImfiles matching the default query
+  psString query = pxDataGet("warptool_tosummary.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    return false;
+  }
+
+  // generate where strings for arguments that require extra processing
+  // beyond PXOPT_COPY*
+  psString where2 = NULL;
+  if (!pxmagicAddWhere(config, &where2, "warpSkyfile")) {
+    psError(psErrorCodeLast(), false, "pxMagicAddWhere failed");
+    return false;
+  }
+  
+  if (psListLength(where->list)) {
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
+    psFree(whereClause);
+  } else if (!all && !where2) {
+    psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
+    return false;
+  }
+  
+  if (where2) {
+    psStringAppend(&query, " %s", where2);
+  }
+  psFree(where);
+
+  // treat limit == 0 as "no limit"
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, " %s", limitString);
+    psFree(limitString);
+  }
+  
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+    
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("warptool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+  
+  if (psArrayLength(output)) {
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+  }
+  
+  psFree(output);
+  return(true);
+}
+static bool addsummaryMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+  PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
+  PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false);
+  PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
+
+  psString query = pxDataGet("warptool_addsummary.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    return(false);
+  }
+  if (!p_psDBRunQueryF(config->dbh, query, warp_id, projection_cell, path_base)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return(false);
+  }
+  psS64 numUpdated = psDBAffectedRows(config->dbh);
+  
+  if (numUpdated != 1) {
+    psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
+    psFree(query);
+    return false;
+  }
+  
+  psFree(query);
+
+  // Print anything here?
+  
+  return(true);
+}
+
 static bool pendingcleanuprunMode(pxConfig *config)
 {

trunk/ippTools/src/warptool.h

@@ -44 +44 @@
     WARPTOOL_MODE_MASKED,
     WARPTOOL_MODE_UNBLOCK,
+    WARPTOOL_MODE_TOSUMMARY,
+    WARPTOOL_MODE_ADDSUMMARY,
     WARPTOOL_MODE_PENDINGCLEANUPRUN,
     WARPTOOL_MODE_PENDINGCLEANUPSKYFILE,

trunk/ippTools/src/warptoolConfig.c

@@ -321 +321 @@
     psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label",  0,            "name of a label to unmask (required)", NULL);
 
+    // -tosummary
+    psMetadata *tosummaryArgs = psMetadataAlloc();
+    psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-warp_id", 0,         "search by warp ID", 0);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL,  "-tess_id",  0,          "search by tessellation ID", NULL);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-state",  0,            "search by state", NULL);
+    psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-exp_id", 0,            "search by exposure tag", 0);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL, "-exp_name", 0,          "search by exposure tag", 0);
+    psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL, "-fake_id", 0,           "search by phase 3 version of exposure tag", 0);
+    psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_begin", 0,    "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(tosummaryArgs, PS_LIST_TAIL, "-dateobs_end", 0,      "search for exposures by time (<=)", NULL);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL,  "-filter", 0,           "search for exposures by filter", NULL);
+    psMetadataAddS64(tosummaryArgs, PS_LIST_TAIL,  "-magicked", 0,         "search by magic id", 0);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL,  "-label",  PS_META_DUPLICATE_OK, "search by warpRun label", NULL);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL,  "-data_group",  PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL);
+    psMetadataAddStr(tosummaryArgs, PS_LIST_TAIL,  "-dist_group",  PS_META_DUPLICATE_OK, "search by warpRun dist_group", NULL);
+    psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for runs that have not been destreaked", false);
+    
+    psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-all",  0,             "search without arguments", false);
+    psMetadataAddU64(tosummaryArgs, PS_LIST_TAIL,  "-limit",  0,            "limit result set to N items", 0);
+    psMetadataAddBool(tosummaryArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
+
+    // -addsummary
+    psMetadata *addsummaryArgs = psMetadataAlloc();
+    psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-warp_id", 0,         "set warp ID", 0);
+    psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-projection_cell", 0,   "set projection cell", NULL);
+    psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0,       "set summary path base", NULL);
+    
     // -pendingcleanuprun
     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
@@ -430 +458 @@
     PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs);
     PXOPT_ADD_MODE("-setskyfiletoupdate", "set cleaned skyfile to be updated", WARPTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateArgs);
+    PXOPT_ADD_MODE("-tosummary",            "show runs that can be summarized", WARPTOOL_MODE_TOSUMMARY, tosummaryArgs);
+    PXOPT_ADD_MODE("-addsummary",           "add entry to the summary table", WARPTOOL_MODE_ADDSUMMARY, addsummaryArgs);
+
     PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", WARPTOOL_MODE_EXPORTRUN, exportrunArgs);
     PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           WARPTOOL_MODE_IMPORTRUN, importrunArgs);

trunk/ippconfig/recipes/filerules-mef.mdc

@@ -312 +312 @@
 PPSKYCELL.JPEG1         OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg     JPEG      NONE       CHIP       TRUE      NONE
 PPSKYCELL.JPEG2         OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg     JPEG      NONE       CHIP       TRUE      NONE
+PPSKYCELL.BIN1          OUTPUT {OUTPUT}.{FILE.INDEX}.b1.fits     IMAGE     NONE       FPA        TRUE      NONE
+PPSKYCELL.BIN2          OUTPUT {OUTPUT}.{FILE.INDEX}.b2.fits     IMAGE     NONE       FPA        TRUE      NONE
 
 LOG.IMFILE              OUTPUT {OUTPUT}.{CHIP.NAME}.log          TEXT      NONE       CHIP       TRUE      NONE

trunk/ippconfig/recipes/filerules-simple.mdc

@@ -277 +277 @@
 PPSKYCELL.JPEG1              OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg JPEG            NONE       CHIP       TRUE      NONE
 PPSKYCELL.JPEG2              OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg JPEG            NONE       CHIP       TRUE      NONE
+PPSKYCELL.BIN1               OUTPUT {OUTPUT}.{FILE.INDEX}.b1.fits     IMAGE     NONE       FPA        TRUE      NONE
+PPSKYCELL.BIN2               OUTPUT {OUTPUT}.{FILE.INDEX}.b2.fits     IMAGE     NONE       FPA        TRUE      NONE
 
 LOG.IMFILE                   OUTPUT {OUTPUT}.imfile.log           TEXT            NONE       FPA        TRUE      NONE

trunk/ippconfig/recipes/filerules-split.mdc

@@ -297 +297 @@
 PPSKYCELL.JPEG1              OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg     JPEG            NONE       CHIP       TRUE      NONE
 PPSKYCELL.JPEG2              OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg     JPEG            NONE       CHIP       TRUE      NONE
+PPSKYCELL.BIN1          OUTPUT {OUTPUT}.{FILE.INDEX}.b1.fits     IMAGE     NONE       FPA        TRUE      NONE
+PPSKYCELL.BIN2          OUTPUT {OUTPUT}.{FILE.INDEX}.b2.fits     IMAGE     NONE       FPA       TRUE      NONE
 
 LOG.IMFILE                   OUTPUT {OUTPUT}.{CHIP.NAME}.log          TEXT            NONE       CHIP       TRUE      NONE

trunk/ippconfig/recipes/ppSkycell.config

@@ -4 +4 @@
 BIN1            S32     4               # Binning factor for output 1
 BIN2            S32     4               # Binning factor for output 2
-
+MAKEFITS        BOOL    T               # Make fits image as well?

trunk/ppImage/src/ppImageLoop.c

@@ -175 +175 @@
             ESCAPE("Unable to measures pixel stats for image");
         }
+
         if (!ppImageMosaicChip(config, options, view, "PPIMAGE.CHIP", "PPIMAGE.OUTPUT")) {
             ESCAPE("Unable to mosaic chip");
@@ -181 +182 @@
 
         // we perform photometry on the readouts of this chip in the output
+
         psTimerStart(TIMER_PHOT);
         if (options->doPhotom) {

trunk/ppImage/src/ppImagePhotom.c

@@ -11 +11 @@
     pmCell *cell;
     pmReadout *readout;
-
+    printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     psphotInit();
-
+    printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // find or define a pmFPAfile PSPHOT.INPUT
     pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.INPUT");
@@ -20 +20 @@
         return false;
     }
-
+    printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // we make a new copy of the output chip to keep psphot from modifying the output image
     pmChip *oldChip = pmFPAviewThisChip (view, input->src);
     pmChip *newChip = pmFPAviewThisChip (view, input->fpa);
     pmChipCopy (newChip, oldChip);
-
+    printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // iterate over the cells and readout for this chip
     while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) {
@@ -34 +34 @@
         while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) {
             if (! readout->data_exists) { continue; }
-
+            printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
             // run the actual photometry analysis
             if (!psphotReadout (config, view)) {

trunk/ppSkycell/src/ppSkycell.h

@@ -16 +16 @@
     int bin1, bin2;                     // Binning factors
     pmConfig *config;                   // Configuration
+  bool doFits;                          // hold whether to do fits as well.
 } ppSkycellData;
 

trunk/ppSkycell/src/ppSkycellCamera.c

@@ -103 +103 @@
         return false;
     }
+    if (!pmFPAfileDefineOutput(data->config, NULL, "PPSKYCELL.BIN1")) {
+        psError(psErrorCodeLast(), false, "Unable to define output.");
+        return false;
+    }
+    if (!pmFPAfileDefineOutput(data->config, NULL, "PPSKYCELL.BIN2")) {
+        psError(psErrorCodeLast(), false, "Unable to define output.");
+        return false;
+    }
 
     // Now the camera has been determined, we can read the recipe
@@ -116 +124 @@
     data->bin2 = psMetadataLookupS32(NULL, recipe, "BIN2");
 
+    data->doFits = psMetadataLookupBool(NULL, recipe, "MAKEFITS");
+    
     if (data->bin1 <= 0 || data->bin2 <= 0) {
         psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find legitimate values for BIN1 and BIN2");

trunk/ppSkycell/src/ppSkycellLoop.c

@@ -284 +284 @@
         pmFPAfileActivate(data->config->files, true, "PPSKYCELL.JPEG1");
         pmFPAfileActivate(data->config->files, true, "PPSKYCELL.JPEG2");
+        if (data->doFits) {
+          pmFPAfileActivate(data->config->files, true, "PPSKYCELL.BIN1");
+          pmFPAfileActivate(data->config->files, true, "PPSKYCELL.BIN2");
+        }
+        
         pmFPAview *view = filesIterateDown(data->config); // View to readout
 
+        
         pmCell *cell1 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.JPEG1"); // Rebinned cell 1
         pmCell *cell2 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.JPEG2"); // Rebinned cell 2
-        psFree(view);
+
         pmReadout *ro1 = pmReadoutAlloc(cell1), *ro2 = pmReadoutAlloc(cell2); // Binned readouts
 
@@ -298 +304 @@
         ro1->data_exists = cell1->data_exists = cell1->parent->data_exists = true;
         ro2->data_exists = cell2->data_exists = cell2->parent->data_exists = true;
-
+        
         pmFPAfile *file1 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.JPEG1", 0);
         file1->save = true;
-        file1->index = i;
+        file1->fileIndex = i;
         pmFPAfile *file2 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.JPEG2", 0);
         file2->save = true;
-        file2->index = i;
+        file2->fileIndex = i;
+
+        if (data->doFits) {
+          pmCell *Fcell1 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.BIN1"); // Rebinned cell 1
+          pmCell *Fcell2 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.BIN2"); // Rebinned cell 2
+
+          // This is a hack to get a functioning header created so the fits images can be written out.
+          psMetadataAddS32(Fcell1->concepts,PS_LIST_TAIL,"CELL.XPARITY", PS_META_REPLACE,"",1);
+          psMetadataAddS32(Fcell1->concepts,PS_LIST_TAIL,"CELL.YPARITY", PS_META_REPLACE,"",1);
+          psMetadataAddS32(Fcell1->concepts,PS_LIST_TAIL,"CELL.READDIR", PS_META_REPLACE,"",1);
+          psMetadataAddS32(Fcell2->concepts,PS_LIST_TAIL,"CELL.XPARITY", PS_META_REPLACE,"",1);
+          psMetadataAddS32(Fcell2->concepts,PS_LIST_TAIL,"CELL.YPARITY", PS_META_REPLACE,"",1);
+          psMetadataAddS32(Fcell2->concepts,PS_LIST_TAIL,"CELL.READDIR", PS_META_REPLACE,"",1);
+
+          psMetadataAddS32(Fcell1->parent->concepts,PS_LIST_TAIL,"CHIP.XPARITY", PS_META_REPLACE,"",1);
+          psMetadataAddS32(Fcell1->parent->concepts,PS_LIST_TAIL,"CHIP.YPARITY", PS_META_REPLACE,"",1);
+          psMetadataAddS32(Fcell2->parent->concepts,PS_LIST_TAIL,"CHIP.XPARITY", PS_META_REPLACE,"",1);
+          psMetadataAddS32(Fcell2->parent->concepts,PS_LIST_TAIL,"CHIP.YPARITY", PS_META_REPLACE,"",1);
+
+          pmReadout *Fro1 = pmReadoutAlloc(Fcell1), *Fro2 = pmReadoutAlloc(Fcell2); // Binned readouts
+          
+          Fro1->image = image1;
+          Fro2->image = image2;
+          Fro1->mask = mask1;
+          Fro2->mask = mask2;
+          
+          Fro1->data_exists = Fcell1->data_exists = Fcell1->parent->data_exists = true;
+          Fro2->data_exists = Fcell2->data_exists = Fcell2->parent->data_exists = true;
+          
+          pmFPAfile *fits1 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.BIN1", 0);
+          fits1->save = true;
+          fits1->fileIndex = i;
+          pmFPAfile *fits2 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.BIN2", 0);
+          fits2->save = true;
+          fits2->fileIndex = i;
+        }
+
+        psFree(view);
 #if 0
         {

trunk/psconfig/tagsets/ipp-2.9.dist

@@ -73 +73 @@
   YYYYY  ppTranslate            ipp-2-9          -0
   YYYYY  ppViz                  ipp-2-9          -0
+  YYYYY  ppSkycell              ipp-2-9          -0
 
   YYYYY  extsrc/gpcsw           ipp-2-9          -0

trunk/psphot/src/psphotReadout.c

@@ -14 +14 @@
     // by the multiple threads, not the total time used by all threads.
     psTimerStart ("psphotReadout");
-
+    printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     pmModelClassSetLimits(PM_MODEL_LIMITS_LAX);
 
@@ -26 +26 @@
     char *breakPt = psMetadataLookupStr (NULL, recipe, "BREAK_POINT");
     psAssert (breakPt, "configuration error: set BREAK_POINT");
-
+    printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // set the photcode for this image
     if (!psphotAddPhotcode (config, view, "PSPHOT.INPUT")) {
@@ -32 +32 @@
         return false;
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // Generate the mask and weight images, including the user-defined analysis region of interest
     if (!psphotSetMaskAndVariance (config, view, "PSPHOT.INPUT")) {
@@ -40 +40 @@
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // generate a background model (median, smoothed image)
     if (!psphotModelBackground (config, view, "PSPHOT.INPUT")) {
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
+            printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     if (!psphotSubtractBackground (config, view, "PSPHOT.INPUT")) {
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
@@ -51 +52 @@
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // load the psf model, if suppled.  FWHM_X,FWHM_Y,etc are determined and saved on
     // readout->analysis XXX this function currently only works with a single PSPHOT.INPUT
@@ -58 +59 @@
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // find the detections (by peak and/or footprint) in the image.
     if (!psphotFindDetections (config, view, "PSPHOT.INPUT", true)) { // pass 1
@@ -65 +66 @@
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // construct sources and measure basic stats (saved on detections->newSources)
     if (!psphotSourceStats (config, view, "PSPHOT.INPUT", true)) { // pass 1
@@ -74 +75 @@
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // find blended neighbors of very saturated stars (detections->newSources)
     if (!psphotDeblendSatstars (config, view, "PSPHOT.INPUT")) {
@@ -80 +81 @@
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // mark blended peaks PS_SOURCE_BLEND (detections->newSources)
     if (!psphotBasicDeblend (config, view, "PSPHOT.INPUT")) {
@@ -86 +87 @@
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // classify sources based on moments, brightness.  if a PSF model has been loaded, the PSF
     // clump defined for it is used not measured (detections->newSources)
@@ -93 +94 @@
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
+            printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // if we were not supplied a PSF model, determine the IQ stats here (detections->newSources)
     if (!psphotImageQuality (config, view, "PSPHOT.INPUT")) { // pass 1
@@ -101 +103 @@
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // use bright stellar objects to measure PSF if we were supplied a PSF for any input file,
     // this step is skipped
@@ -115 +117 @@
     // XXX fix this in the new multi-input context
     // psphotLoadExtSources (config, view, "PSPHOT.INPUT"); // pass 1
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // construct an initial model for each object, set the radius to fitRadius, set circular
     // fit mask (detections->newSources)
     psphotGuessModels (config, view, "PSPHOT.INPUT"); // pass 1
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // merge the newly selected sources into the existing list
     // NOTE: merge OLD and NEW
     psphotMergeSources (config, view, "PSPHOT.INPUT");
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // linear PSF fit to source peaks, subtract the models from the image (in PSF mask)
     psphotFitSourcesLinear (config, view, "PSPHOT.INPUT", false); // pass 1 (detections->allSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // identify CRs and extended sources (only unmeasured sources are measured)
     psphotSourceSize (config, view, "PSPHOT.INPUT", true); // pass 1 (detections->allSources)
@@ -132 +134 @@
         goto finish;
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // non-linear PSF and EXT fit to brighter sources
     // replace model flux, adjust mask as needed, fit, subtract the models (full stamp)
     psphotBlendFit (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // replace all sources
     psphotReplaceAllSources (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // linear fit to include all sources (subtract again)
     // NOTE : apply to ALL sources (extended + psf)
     psphotFitSourcesLinear (config, view, "PSPHOT.INPUT", true); // pass 2 (detections->allSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // if we only do one pass, skip to extended source analysis
     if (!strcasecmp (breakPt, "PASS1")) goto pass1finish;
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // NOTE: possibly re-measure background model here with objects subtracted / or masked
 
     // add noise for subtracted objects
     psphotAddNoise (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // find fainter sources
     // NOTE: finds new peaks and new footprints, OLD and FULL set are saved on detections
     psphotFindDetections (config, view, "PSPHOT.INPUT", false); // pass 2 (detections->peaks, detections->footprints)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // remove noise for subtracted objects (ie, return to normal noise level)
     // NOTE: this needs to operate only on the OLD sources
     psphotSubNoise (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // define new sources based on only the new peaks
     // NOTE: new sources are saved on detections->newSources
     psphotSourceStats (config, view, "PSPHOT.INPUT", false); // pass 2 (detections->newSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // set source type
     // NOTE: apply only to detections->newSources
@@ -170 +172 @@
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // create full input models, set the radius to fitRadius, set circular fit mask
     // NOTE: apply only to detections->newSources
     psphotGuessModels (config, view, "PSPHOT.INPUT"); // pass 2 (detections->newSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // replace all sources so fit below applies to all at once
     // NOTE: apply only to OLD sources (which have been subtracted)
     psphotReplaceAllSources (config, view, "PSPHOT.INPUT"); // pass 2
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // merge the newly selected sources into the existing list
     // NOTE: merge OLD and NEW
     // XXX check on free of sources...
     psphotMergeSources (config, view, "PSPHOT.INPUT"); // (detections->newSources + detections->allSources -> detections->allSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // NOTE: apply to ALL sources
     psphotFitSourcesLinear (config, view, "PSPHOT.INPUT", true); // pass 3 (detections->allSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
 pass1finish:
 
@@ -192 +194 @@
     // NOTE: applies only to NEW (unmeasured) sources
     psphotSourceSize (config, view, "PSPHOT.INPUT", false); // pass 2 (detections->allSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     psphotExtendedSourceAnalysis (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources)
     psphotExtendedSourceFits (config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources)
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
 finish:
 
     // plot positive sources
     // psphotSourcePlots (readout, sources, recipe);
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // measure aperture photometry corrections
     if (!psphotApResid (config, view, "PSPHOT.INPUT")) { // pass 1 (detections->allSources)
@@ -206 +208 @@
         return psphotReadoutCleanup (config, view, "PSPHOT.INPUT");
     }
-
+        printf("%x %s\n",psErrorCodeLast(),psErrorCodeString(psErrorCodeLast()));
     // calculate source magnitudes
-    psphotMagnitudes(config, view, "PSPHOT.INPUT"); // pass 1 (detections->allSources)
-
+    if (!psphotMagnitudes(config, view, "PSPHOT.INPUT")) { // pass 1 (detections->allSources)
+      psErrorStackPrint(stderr, "Unable to do magnitudes.");
+        psErrorClear();
+    }      
     if (!psphotEfficiency(config, view, "PSPHOT.INPUT")) { // pass 1
         psErrorStackPrint(stderr, "Unable to determine detection efficiencies from fake sources");
@@ -219 +223 @@
 
     // replace background in residual image
-    psphotSkyReplace (config, view, "PSPHOT.INPUT"); // pass 1
-
+    if (!psphotSkyReplace (config, view, "PSPHOT.INPUT")) { // pass 1
+      psErrorStackPrint(stderr, "Unable to replace sky");
+      psErrorClear();
+
+/*       psLogMsg("psphot", 3, "failed on psphotSkyReplace"); */
+/*       return(psphotReadoutCleanup(config, view, "PSPHOT.INPUT")); */
+    }
     // drop the references to the image pixels held by each source
-    psphotSourceFreePixels (config, view, "PSPHOT.INPUT"); // pass 1
-
+    if (!psphotSourceFreePixels (config, view, "PSPHOT.INPUT")) { // pass 1
+      psErrorStackPrint(stderr, "Unable to free source pixels");
+      psErrorClear();
+
+/*       psLogMsg ("psphot", 3, "failed on psphotSourceFreePixels"); */
+/*       return(psphotReadoutCleanup(config, view, "PSPHOT.INPUT")); */
+    }
     // create the exported-metadata and free local data
     return psphotReadoutCleanup(config, view, "PSPHOT.INPUT");