IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 25816


Ignore:
Timestamp:
Oct 8, 2009, 5:09:53 PM (17 years ago)
Author:
eugene
Message:

updates to the addstar stage; modify flatcorr to interact correctly with the addstar stage

Location:
trunk
Files:
1 added
22 edited

Legend:

Unmodified
Added
Removed

  • trunk/dbconfig/add.md

    r25299 r25816  
     1# XXX rename these 'addstarRun', etc... (add_id -> addstar_id)
    12addRun METADATA
    23    add_id          S64     0       # Primary Key AUTO_INCREMENT
    … …  
    56    workdir         STR     255
    67    workdir_state   STR     64
     8    reduction       STR     64
    79    label           STR     64
    810    dvodb           STR     255
    9     magicked        S64     0
     11    image_only      BOOL    f
    1012END
    1113
    … …  
    1315    add_id          S64     0       # Primary Key AUTO_INCREMENT
    1416    dtime_addstar   F32     0.0
    15 
    16     n_stars         S32     0
    17    
    1817    path_base       STR     255
    1918    fault           S16     0       # Key NOT NULL

  • trunk/dbconfig/changes.txt

    r25795 r25816  
    13001300
    13011301ALTER TABLE pstampJob ADD COLUMN options BIGINT;
     1302
     1303ALTER TABLE addRun ADD COLUMN reduction VARCHAR(64) AFTER workdir_state;
     1304ALTER TABLE addRun DROP COLUMN magicked;
     1305ALTER TABLE addRun ADD COLUMN image_only tinyint after dvodb;
     1306ALTER TABLE addProcessedExp DROP COLUMN n_stars;
     1307
     1308CREATE TABLE flatcorrAddstarLink (
     1309        corr_id BIGINT,
     1310        cam_id BIGINT,
     1311        add_id BIGINT,
     1312        include TINYINT,
     1313        PRIMARY KEY(corr_id, cam_id, add_id),
     1314        FOREIGN KEY (corr_id)  REFERENCES  flatcorrRun(corr_id),
     1315        FOREIGN KEY (cam_id)  REFERENCES  camRun(cam_id),
     1316        FOREIGN KEY (add_id)  REFERENCES  addRun(add_id)
     1317) ENGINE=innodb DEFAULT CHARSET=latin1;
     1318
    13021319ALTER TABLE pstampJob ADD COLUMN dep_id BIGINT;
    13031320CREATE TABLE pstampDependent (
    … …  
    13111328        PRIMARY KEY(dep_id)
    13121329) ENGINE=innodb DEFAULT CHARSET=latin1;
    1313 

  • trunk/dbconfig/flatcorr.md

    r20397 r25816  
    3333END
    3434
     35# table of exposures sent to addstar
     36flatcorrAddstarLink METADATA
     37    corr_id     S64         0       # Primary Key fkey(corr_id) ref flatcorrRun(corr_id)
     38    cam_id      S64         0       # Primary Key fkey(chip_id) ref chipRun(chip_id)
     39    add_id      S64         0       # Primary Key fkey(chip_id) ref chipRun(chip_id)
     40    include     BOOL        t
     41END
     42

  • trunk/ippScripts/scripts/addstar_run.pl

    r25299 r25816  
    2828# Look for programs we need
    2929my $missing_tools;
    30 #my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
    3130my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1);
    32 #my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
    3331my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
    34 #my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
    35 #my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1);
    3632my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1);
    3733if ($missing_tools) {
    … …  
    4036}
    4137
    42 my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update,
    43      $no_op, $redirect, $save_temps, $run_state);
     38my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $image_only, $verbose, $no_update,
     39     $no_op, $redirect, $save_temps);
    4440GetOptions(
    4541    'exp_tag=s'          => \$exp_tag, # Exposure identifier
    4642    'add_id=s'          => \$add_id, # Camtool identifier
    47     'recipe=s'          => \$recipe, # Recipe to use
    4843    'camera|c=s'        => \$camera, # Camera
    4944    'dbname|d=s'        => \$dbname, # Database name
    … …  
    5247    'reduction=s'       => \$reduction, # Reduction class
    5348    'dvodb|w=s'         => \$dvodb,  # output DVO database
    54     'run-state=s'       => \$run_state, # 'new' or 'update'
     49    'image-only'        => \$image_only,   # Print to stdout
    5550    'verbose'           => \$verbose,   # Print to stdout
    5651    'no-update'         => \$no_update, # Update the database?
    … …  
    7469my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR);
    7570
    76 if (not defined $run_state) { $run_state = 'new'; }
    77 if ($run_state eq 'update') {
    78     $logDest .= '.update';
    79 }
    80 
    8171if ($redirect) {
    8272    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $add_id, $PS_EXIT_SYS_ERROR );
    … …  
    8979$reduction = 'DEFAULT' unless defined $reduction;
    9080
     81# XXX This is now not used: do we still need it?
    9182my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use
    9283&my_die("Unrecognised ADDSTAR recipe", $add_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar;
    9384
    94 #my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use
    95 #&my_die("Unrecognised PSASTRO recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro;
    96 
    9785my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    98 
    99 my $cmdflags;
    100 
    101 # Get list of component files
    102 my $files;                      # Array of component files
    103 {
    104     my $command = "$addtool -pendingexp -add_id $add_id"; # Command to run
    105     $command .= " -dbname $dbname" if defined $dbname;
    106     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    107         run(command => $command, verbose => $verbose);
    108     unless ($success) {
    109         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    110         &my_die("Unable to perform addtool: $error_code", $add_id, $error_code);
    111     }
    112     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    113         &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_PROG_ERROR);
    114    
    115     # extract the metadata for the files into a hash list
    116     $files = parse_md_list($metadata) or
    117         &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);   
    118 }
    119 
    120 my $chipObjectsExist = 0;
    12186
    12287# Output products
    … …  
    12691my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",     $camroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR);
    12792my $traceDest  = $ipprc->filename("TRACE.EXP",          $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR);
    128 
    129 if ($run_state eq 'update') {
    130     $traceDest .= '.update';
    131 }
    13293
    13394# convert supplied DVO database name to UNIX filename
    … …  
    141102
    142103unless ($no_op) {
    143     if (defined $dvodbReal and ($run_state eq 'new')) {
    144         ## XXX the camera analysis can either save the full set of
    145         ## detections, or just the image metadata, in the dvodb
     104    if (defined $dvodbReal) {
     105
     106        ## addstar can either save the full set of detections, or just
     107        ## the image metadata, in the dvodb.  this is set in the
     108        ## database table addRun
    146109       
    147         ## get the addstar recipe for this camera and CAMERA reduction
    148         my $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -";
    149         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    150             run(command => $command, verbose => $verbose);
    151         unless ($success) {
    152             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    153             &my_die("Unable to perform ppConfigDump: $error_code", $add_id, $PS_EXIT_SYS_ERROR);
    154         }
    155         my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
    156             &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_SYS_ERROR);
    157        
    158         ## allow the dvodb to save only images, or the full detection set
    159         my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY');
    160        
    161         # XXX this construct requires the user to have a valid .ptolemyrc
    162         # XXX which in turn points at ippconfig/dvo.site
     110        # addstar requires the user to have a valid .ptolemyrc which
     111        # in turn points at ippconfig/dvo.site
     112
     113        # get the names for the camera and the real input file
     114        my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar
     115        my $realFile = $ipprc->file_resolve($fpaObjects);
     116
    163117        # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default)
    164         # XXX this needs to be converted to addstar_client...
    165 
    166         my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar
    167         $command  = "$addstar -D CAMERA $camdir -update";
    168         $command .= " -image" if $imagesOnly;
     118        my $command  = "$addstar -update"; # XXX optionally set -update?
     119        $command .= " -D CAMERA $camdir";
    169120        $command .= " -D CATDIR $dvodbReal";
    170        
    171         my $realFile = $ipprc->file_resolve($fpaObjects);
    172121        $command .= " $realFile";
     122        $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference
     123        $command .= " -image" if $image_only;
    173124       
    174125        my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
    175126       
    176         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     127        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    177128            run(command => $command, verbose => $verbose);
    178129        unless ($success) {
    … …  
    184135}
    185136
    186 
    187137# This needs to be updated when addtool is written. BROKEN
    188138my $fpaCommand = "$addtool -add_id $add_id";
    189 if ($run_state eq 'new') {
    190     $fpaCommand .= " -addprocessedexp";
    191     $fpaCommand .= " -path_base $outroot";
    192     $fpaCommand .= " $cmdflags";
    193     $fpaCommand .= " -dtime_addstar $dtime_addstar";
    194 } else {
    195     $fpaCommand .= " -updaterun -state full";
    196 }
     139$fpaCommand .= " -addprocessedexp";
     140$fpaCommand .= " -path_base $outroot";
     141$fpaCommand .= " -dtime_addstar $dtime_addstar";
    197142$fpaCommand .= " -dbname $dbname" if defined $dbname;
    198143
    … …  
    223168# This needs to be updated when addtool is written. BROKEN
    224169        my $command = "$addtool -add_id $add_id";
    225         if ($run_state eq 'new') {
    226             $command .= " -addprocessedexp";
    227             $command .= " -uri UNKNOWN";
    228             $command .= " -fault $exit_code";
    229             $command .= " -path_base $outroot";
    230             $command .= " -path_base $outroot" if defined $outroot;
    231             $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    232         } else {
    233             $command .= " -updateprocessedexp";
    234             $command .= " -fault $exit_code";
    235         }
     170        $command .= " -addprocessedexp";
     171        $command .= " -fault $exit_code";
     172        $command .= " -path_base $outroot" if defined $outroot;
     173        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    236174        $command .= " -dbname $dbname" if defined $dbname;
    237175        system ($command);
    … …  
    239177    exit $exit_code;
    240178}
    241 
    242179
    243180END {

  • trunk/ippScripts/scripts/camera_exp.pl

    r25299 r25816  
    3939}
    4040
    41 my ( $exp_tag, $cam_id, $camera, $outroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update,
     41my ( $exp_tag, $cam_id, $camera, $outroot, $dbname, $reduction, $dvodb, $verbose, $no_update,
    4242     $no_op, $redirect, $save_temps, $run_state);
    4343GetOptions(
    4444    'exp_tag=s'          => \$exp_tag, # Exposure identifier
    4545    'cam_id=s'          => \$cam_id, # Camtool identifier
    46     'recipe=s'          => \$recipe, # Recipe to use
    4746    'camera|c=s'        => \$camera, # Camera
    4847    'dbname|d=s'        => \$dbname, # Database name

  • trunk/ippScripts/scripts/flatcorr_proc.pl

    r23688 r25816  
    104104my ($DECs, $DECe) = split (",", $coords[1]);
    105105
    106 # Run addstar -resort to ensure the db is indexed
    107 {
    108     my $command = "$addstar -resort";
     106# Run addstar -resort to ensure the db is indexed
     107# XXX addstar should be able to recognize and skip indexed tables
     108{
     109    my $command = "$addstar -resort";
    109110    $command .= " -D CATDIR $dvodb";
    110111    $command .= " -region $RAs $RAe $DECs $DECe";

  • trunk/ippTasks/addstar.pro

    r25693 r25816  
    121121    book getword addPendingExp $pageName exp_tag -var EXP_TAG
    122122    book getword addPendingExp $pageName add_id -var ADD_ID
    123     book getword addPendingExp $pageName cam_id -var CAM_ID
     123    book getword addPendingExp $pageName camroot -var CAMROOT
    124124    book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE
     125    book getword addPendingExp $pageName reduction -var REDUCTION
    125126    book getword addPendingExp $pageName dvodb  -var DVODB
     127    book getword addPendingExp $pageName image_only -var IMAGE_ONLY
    126128    book getword addPendingExp $pageName dbname -var DBNAME
    127     book getword addPendingExp $pageName reduction -var REDUCTION
    128     book getword addPendingExp $pageName state -var RUN_STATE
    129129
    130130    # specify choice of remote host based on camera and chip (class_id)
    … …  
    144144    ## generate outroot specific to this exposure (& chip)
    145145    sprintf outroot "%s/%s/%s.add.%s" $WORKDIR $EXP_TAG $EXP_TAG $ADD_ID
    146     sprintf camroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID
    147146
    148147    stdout $LOGDIR/addstar.exp.log
    149148    stderr $LOGDIR/addstar.exp.log
    150149
    151     $run = addstar_run.pl --exp_tag $EXP_TAG --add_id $ADD_ID --camera $CAMERA --outroot $outroot --camroot $camroot --redirect-output --run-state $RUN_STATE
     150    $run = addstar_run.pl --exp_tag $EXP_TAG --add_id $ADD_ID --camera $CAMERA --dvodb $DVODB --camroot $CAMROOT --outroot $outroot --redirect-output
    152151    if ("$REDUCTION" != "NULL")
    153152      $run = $run --reduction $REDUCTION
    154153    end
    155     if ("$DVODB" != "NULL")
    156       $run = $run --dvodb $DVODB
    157     end
     154    if ("$IMAGE_ONLY" == "T")
     155      $run = $run --image-only
     156    end
     157   
    158158    add_standard_args run
    159159

  • trunk/ippTasks/flatcorr.pro

    r25785 r25816  
    22## this file contains the tasks for running the flatcorr stage
    33## these tasks use the book 'flatcorrBook'
    4 
    5 # test for required global variables
    6 check.globals
    7 
    8 book init flatcorrBook
    9 
    10 macro flatcorr.reset
    11   book init flatcorrBook
    12 end
    13 
    14 macro flatcorr.status
    15   book listbook flatcorrBook
    16 end
    17 
    18 macro flatcorr.on
    19   task flatcorr.advancecamera
    20     active true
    21   end
    22   task flatcorr.load
    23     active true
    24   end
    25   task flatcorr.run
    26     active true
    27   end
    28 end
    29 
    30 macro flatcorr.off
    31   task flatcorr.advancecamera
    32     active false
    33   end
    34   task flatcorr.load
    35     active false
    36   end
    37   task flatcorr.run
    38     active false
    39   end
    40 end
    41 
    42 # these variables will cycle through the known ippdb database names
    43 $flatcorr_advancecamera_DB = 0
    44 $flatcorr_pendingprocess_DB = 0
    454
    465# a flat-field correction run is initiated (manually) with a command like:
    … …  
    5716# flatcorr -advancecamera
    5817
    59 # we wait for the completed chip and camera analysis and search for
     18# as the camera analysis progresses, we need to occasionally migrate the
     19# completed exposures to the addstar stage by calling:
     20# flatcorr -advanceaddstar
     21
     22# we wait for the completed addstar jobs and search for
    6023# completed processing analysis with:
    6124# flatcorr -pendingprocess
    … …  
    6629# and failures with:
    6730# flatcorr -addprocess -fault N
     31
     32# test for required global variables
     33check.globals
     34
     35book init flatcorrBook
     36
     37macro flatcorr.reset
     38  book init flatcorrBook
     39end
     40
     41macro flatcorr.status
     42  book listbook flatcorrBook
     43end
     44
     45macro flatcorr.on
     46  task flatcorr.advancecamera
     47    active true
     48  end
     49  task flatcorr.advanceaddstar
     50    active true
     51  end
     52  task flatcorr.load
     53    active true
     54  end
     55  task flatcorr.run
     56    active true
     57  end
     58end
     59
     60macro flatcorr.off
     61  task flatcorr.advancecamera
     62    active false
     63  end
     64  task flatcorr.advanceaddstar
     65    active false
     66  end
     67  task flatcorr.load
     68    active false
     69  end
     70  task flatcorr.run
     71    active false
     72  end
     73end
     74
     75# these variables will cycle through the known ippdb database names
     76$flatcorr_advancecamera_DB = 0
     77$flatcorr_advanceaddstar_DB = 0
     78$flatcorr_pendingprocess_DB = 0
    6879
    6980# migrate complete flatcorr chips to the camera stage analysis
    … …  
    114125end
    115126
     127# migrate complete flatcorr exposures to the addstar process
     128task           flatcorr.advanceaddstar
     129  host         local
     130
     131  # check the list of pending flatcorr runs
     132  periods      -poll 5
     133  periods      -exec 30
     134  periods      -timeout 60
     135  npending 1
     136
     137  task.exec
     138    # define the command (does not depend on previous queries)
     139    if ($DB:n == 0)
     140      command flatcorr -advanceaddstar
     141    else
     142      # save the DB name for the exit tasks
     143      # note that this DB name refers to the ippdb, not the dvodb
     144      option $DB:$flatcorr_advanceaddstar_DB
     145      command flatcorr -advanceaddstar -dbname $DB:$flatcorr_advanceaddstar_DB
     146      $flatcorr_advanceaddstar_DB ++
     147      if ($flatcorr_advanceaddstar_DB >= $DB:n) set flatcorr_advanceaddstar_DB = 0
     148    end
     149  end
     150
     151  # silently drop stdout
     152  stdout NULL
     153  stderr $LOGDIR/flatcorr.log
     154
     155  # success (no action required)
     156  task.exit $EXIT_SUCCESS
     157  end
     158
     159  # locked list
     160  task.exit    default
     161    showcommand failure
     162  end
     163
     164  task.exit    crash
     165    showcommand crash
     166  end
     167
     168  # operation times out?
     169  task.exit    timeout
     170    showcommand timeout
     171  end
     172end
     173
    116174# create new flatcorr entries for the currently known DVO databases
    117175# run this multiple times once an hour - on pass for each db
    … …  
    123181  periods      -exec $LOADEXEC
    124182  periods      -timeout 60
    125   active false
    126183  npending 1
    127184
    … …  
    180237  periods      -exec $LOADEXEC
    181238  periods      -timeout 60
    182   active false
    183239
    184240  # silently drop stdout

  • trunk/ippTools/share/Makefile.am

    r25800 r25816  
    152152     faketool_unmasked.sql \
    153153     flatcorr_chiprundone.sql \
     154     flatcorr_camerarundone.sql \
    154155     flatcorr_pendingprocess.sql \
    155156     flatcorr_inputimfile.sql \

  • trunk/ippTools/share/addtool_find_pendingexp.sql

    r25789 r25816  
    11SELECT
    22    addRun.*,
     3    camProcessedExp.path_base as camroot,
    34    rawExp.exp_tag,
    45    rawExp.exp_id,
    … …  
    910FROM addRun
    1011JOIN camRun
     12    USING(cam_id)
     13JOIN camProcessedExp
    1114    USING(cam_id)
    1215JOIN chipRun
    … …  
    2124    camRun.state = 'full'
    2225    AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update')
     26    AND addRun.dvodb IS NOT NULL
     27    AND addRun.workdir IS NOT NULL
    2328    AND addMask.label IS NULL
    2429

  • trunk/ippTools/share/addtool_queue_cam_id.sql

    r25299 r25816  
    66        '%s',           -- workdir
    77        '%s',           -- workdir_state
     8        '%s',           -- reduction
    89        '%s',           -- label
    910        '%s',           -- dvodb
    10         0               -- magicked
     11        %d              -- image_only
    1112    FROM camRun
    1213    WHERE

  • trunk/ippTools/share/flatcorr_chiprundone.sql

    r20397 r25816  
    11SELECT
    22    corr_id,
    3     cam_id,
    43    chipRun.*
    54FROM flatcorrRun
    … …  
    1413    AND flatcorrChipLink.include = 1
    1514    AND chipRun.state = 'full'
    16     AND camRun.cam_id is NULL;
     15    AND camRun.cam_id is NULL

  • trunk/ippTools/share/flatcorr_pendingprocess.sql

    r20397 r25816  
    1010  reduction,
    1111  region,
    12   chip_count,
    13   cam_count
     12  cam_count,
     13  add_count
    1414FROM
    1515  (SELECT
    1616     flatcorrRun.*,
    17      flatcorrChipLink.corr_id as chip_corr_id,
    18      count(flatcorrChipLink.chip_id) as chip_count
     17     flatcorrCamLink.corr_id as cam_corr_id,
     18     count(flatcorrCamLink.cam_id) as cam_count
    1919   FROM flatcorrRun
    20    JOIN flatcorrChipLink
    21      ON flatcorrChipLink.corr_id = flatcorrRun.corr_id
    22   WHERE flatcorrChipLink.include = 1
     20   JOIN flatcorrCamLink
     21  USING (corr_id)
     22  WHERE flatcorrCamLink.include = 1
    2323   GROUP BY
    24      flatcorrChipLink.corr_id) AS t1
     24     flatcorrCamLink.corr_id) AS t1
    2525LEFT JOIN
    2626  (SELECT
    27      flatcorrCamLink.corr_id as cam_corr_id,
    28      count(flatcorrCamLink.cam_id) as cam_count
    29    FROM flatcorrCamLink
    30    JOIN camRun
    31        ON flatcorrCamLink.cam_id = camRun.cam_id
    32    WHERE camRun.state = 'full'
    33      AND flatcorrCamLink.include = 1
     27     flatcorrAddstarLink.corr_id as add_corr_id,
     28     count(flatcorrAddstarLink.add_id) as add_count
     29   FROM flatcorrAddstarLink
     30   JOIN addRun
     31   USING (add_id)
     32   WHERE addRun.state = 'full'
     33     AND flatcorrAddstarLink.include = 1
    3434   GROUP BY
    35        flatcorrCamLink.corr_id) AS t2
    36 ON t1.chip_corr_id = t2.cam_corr_id
    37 WHERE chip_count = cam_count
     35       flatcorrAddstarLink.corr_id) AS t2
     36ON t1.cam_corr_id = t2.add_corr_id
     37WHERE cam_count = add_count
    3838AND state = 'new'

  • trunk/ippTools/share/pxadmin_create_tables.sql

    r25795 r25816  
    464464    workdir VARCHAR(255),
    465465    workdir_state VARCHAR(64),
     466    reduction VARCHAR(64),
    466467    label VARCHAR(64),
    467468    dvodb VARCHAR(255),
    468     magicked BIGINT,
     469    image_only TINYINT,
    469470    PRIMARY KEY(add_id),
    470471    KEY(add_id),
    … …  
    480481    add_id BIGINT AUTO_INCREMENT,
    481482    dtime_addstar FLOAT,
    482     n_stars INT,
    483483    path_base VARCHAR(255),
    484484    fault SMALLINT NOT NULL,
    … …  
    11821182) ENGINE=innodb DEFAULT CHARSET=latin1;
    11831183
    1184 -- these two tables link the flatcorrRun to the associated chip and camera runs
     1184-- these three tables link the flatcorrRun to the associated chip, camera, and addstar runs
    11851185CREATE TABLE flatcorrChipLink (
    11861186        corr_id BIGINT,
    … …  
    12011201        FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id),
    12021202        FOREIGN KEY(cam_id) REFERENCES camRun(cam_id)
     1203) ENGINE=innodb DEFAULT CHARSET=latin1;
     1204
     1205CREATE TABLE flatcorrAddstarLink (
     1206        corr_id BIGINT,
     1207        cam_id BIGINT,
     1208        add_id BIGINT,
     1209        include TINYINT,
     1210        PRIMARY KEY(corr_id, cam_id, add_id),
     1211        FOREIGN KEY (corr_id)  REFERENCES  flatcorrRun(corr_id),
     1212        FOREIGN KEY (cam_id)  REFERENCES  camRun(cam_id),
     1213        FOREIGN KEY (add_id)  REFERENCES  addRun(add_id)
    12031214) ENGINE=innodb DEFAULT CHARSET=latin1;
    12041215

  • trunk/ippTools/share/pxadmin_drop_tables.sql

    r25795 r25816  
    5959DROP TABLE IF EXISTS flatcorrChipLink;
    6060DROP TABLE IF EXISTS flatcorrCamLink;
     61DROP TABLE IF EXISTS flatcorrAddstarLink;
    6162DROP TABLE IF EXISTS pstampDataStore;
    6263DROP TABLE IF EXISTS pstampProject;

  • trunk/ippTools/src/addtool.c

    r25789 r25816  
    4040static bool maskedMode(pxConfig *config);
    4141static bool unblockMode(pxConfig *config);
    42 static bool pendingcleanuprunMode(pxConfig *config);
    43 static bool pendingcleanupexpMode(pxConfig *config);
    44 static bool donecleanupMode(pxConfig *config);
    45 static bool exportrunMode(pxConfig *config);
    46 static bool importrunMode(pxConfig *config);
    4742
    4843# define MODECASE(caseName, func) \
    … …  
    7469        MODECASE(ADDTOOL_MODE_MASKED,               maskedMode);
    7570        MODECASE(ADDTOOL_MODE_UNBLOCK,              unblockMode);
    76         MODECASE(ADDTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunMode);
    77         MODECASE(ADDTOOL_MODE_PENDINGCLEANUPEXP,    pendingcleanupexpMode);
    78         MODECASE(ADDTOOL_MODE_DONECLEANUP,          donecleanupMode);
    79         MODECASE(ADDTOOL_MODE_EXPORTRUN,            exportrunMode);
    80         MODECASE(ADDTOOL_MODE_IMPORTRUN,            importrunMode);
    8171        default:
    8272            psAbort("invalid option (this should not happen)");
    … …  
    118108    }
    119109
    120     PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false);
    121     PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
    122     PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false);
    123     PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false);
    124     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
    125     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     110    PXOPT_LOOKUP_STR(workdir,     config->args, "-set_workdir", false, false);
     111    PXOPT_LOOKUP_STR(dvodb,       config->args, "-set_dvodb", false, false);
     112    PXOPT_LOOKUP_STR(label,       config->args, "-set_label", false, false);
     113    PXOPT_LOOKUP_STR(reduction,   config->args, "-set_reduction", false, false);
     114    PXOPT_LOOKUP_BOOL(image_only, config->args, "-image_only", false);
     115    PXOPT_LOOKUP_BOOL(pretend,    config->args, "-pretend", false);
     116    PXOPT_LOOKUP_BOOL(simple,     config->args, "-simple", false);
    126117
    127118    // find the cam_id of all the exposures that we want to queue up.
    … …  
    170161    }
    171162
     163    // loop over our list of camRun rows to check the supplied and selected dvodb and workdir values:
     164    for (long i = 0; i < psArrayLength(output); i++) {
     165        psMetadata *md = output->data[i];
     166
     167        camRunRow *row = camRunObjectFromMetadata(md);
     168        if (!row) {
     169            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun");
     170            psFree(output);
     171            return false;
     172        }
     173
     174        if (!dvodb && !row->dvodb) {
     175            psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, cam_id %" PRId64, row->label, row->cam_id);
     176            psFree(output);
     177            return false;
     178        }
     179        if (!workdir && !row->workdir) {
     180            psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, cam_id %" PRId64, row->label, row->cam_id);
     181            psFree(output);
     182            return false;
     183        }
     184
     185        psFree(row);
     186    }
     187
    172188    // start a transaction so we don't end up with an exp without any associted
    173189    // imfiles
    … …  
    183199    // old values in place (i.e., passing the values through).
    184200
    185 
    186     // loop over our list of addRun rows
     201    // loop over our list of camRun rows
    187202    for (long i = 0; i < psArrayLength(output); i++) {
    188203        psMetadata *md = output->data[i];
    … …  
    190205        camRunRow *row = camRunObjectFromMetadata(md);
    191206        if (!row) {
    192             psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into addRun");
     207            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun");
    193208            psFree(output);
    194209            return false;
    … …  
    199214                               row->cam_id,
    200215                               workdir     ? workdir   : row->workdir,
     216                               reduction   ? reduction : row->reduction,
    201217                               label       ? label     : row->label,
    202                                "RECIPE",
    203                                dvodb       ? dvodb     : row->dvodb
    204                                
     218                               dvodb       ? dvodb     : row->dvodb,
     219                               image_only
    205220        )) {
    206221            if (!psDBRollback(config->dbh)) {
    … …  
    234249    PXOPT_COPY_S64(config->args, where, "-cam_id",    "camRun.cam_id", "==");
    235250    pxcamGetSearchArgs (config, where); // most search arguments based on camera
    236 
    237251    PXOPT_COPY_STR(config->args, where, "-label",     "addRun.label", "==");
    238252    PXOPT_COPY_STR(config->args, where, "-state",     "addRun.state", "==");
    … …  
    287301    pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "==");
    288302    PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "==");
    289 
    290303    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    291304    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    … …  
    349362
    350363    // optional
    351     PXOPT_LOOKUP_F32(dtime_addstar, config->args,  "-dtime_addstar", false, false);
    352     PXOPT_LOOKUP_S32(n_stars, config->args,        "-n_stars", false, false);
    353     PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false);
    354     PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
     364    PXOPT_LOOKUP_STR(path_base,     config->args, "-path_base", false, false);
     365    PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false);
     366    PXOPT_LOOKUP_S16(fault,         config->args, "-fault", false, false);
    355367
    356368    // generate restrictions
    … …  
    401413        pendingRow->add_id,
    402414        dtime_addstar,
    403         n_stars,
    404415        path_base,
    405         0
     416        fault
    406417        );
    407418
    … …  
    418429
    419430    // since there is only one exp per 'new' set addRun.state = 'full'
    420     if (!pxaddRunSetState(config, row->add_id, "full", magicked)) {
     431    if (!pxaddRunSetState(config, row->add_id, "full")) {
    421432        psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id);
    422433        psFree(row);
    … …  
    439450{
    440451    PS_ASSERT_PTR_NON_NULL(config, false);
    441 
    442     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    443     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    444     PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
    445452
    446453    // generate restrictions
    … …  
    451458    pxAddLabelSearchArgs (config, where, "-label",    "addRun.label",     "==");
    452459    PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "==");
     460
     461    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     462    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     463    PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
    453464
    454465    if (!psListLength(where->list) &&
    … …  
    628639    PS_ASSERT_PTR_NON_NULL(config, false);
    629640
    630     PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
    631 
    632641    psMetadata *where = psMetadataAlloc();
    633642    PXOPT_COPY_S64(config->args, where, "-add_id",   "add_id",   "==");
    634643    PXOPT_COPY_S64(config->args, where, "-cam_id",  "cam_id",  "==");
     644    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
    635645
    636646    if (!pxSetFaultCode(config->dbh, "addProcessedExp", where, fault)) {
    … …  
    713723    return true;
    714724}
    715 
    716 static bool pendingcleanuprunMode(pxConfig *config)
    717 {
    718     PS_ASSERT_PTR_NON_NULL(config, NULL);
    719 
    720     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    721     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    722 
    723     psMetadata *where = psMetadataAlloc();
    724     pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "==");
    725 
    726     psString query = pxDataGet("addtool_pendingcleanuprun.sql");
    727     if (!query) {
    728         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    729         return false;
    730     }
    731 
    732     if (where && psListLength(where->list)) {
    733         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    734         psStringAppend(&query, " AND %s", whereClause);
    735         psFree(whereClause);
    736     }
    737     psFree(where);
    738 
    739     // treat limit == 0 as "no limit"
    740     if (limit) {
    741         psString limitString = psDBGenerateLimitSQL(limit);
    742         psStringAppend(&query, " %s", limitString);
    743         psFree(limitString);
    744     }
    745 
    746     if (!p_psDBRunQuery(config->dbh, query)) {
    747         psError(PS_ERR_UNKNOWN, false, "database error");
    748         psFree(query);
    749         return false;
    750     }
    751     psFree(query);
    752 
    753     psArray *output = p_psDBFetchResult(config->dbh);
    754     if (!output) {
    755         psError(PS_ERR_UNKNOWN, false, "database error");
    756         return false;
    757     }
    758     if (!psArrayLength(output)) {
    759         psTrace("addtool", PS_LOG_INFO, "no rows found");
    760         psFree(output);
    761         return true;
    762     }
    763 
    764     // negative simple so the default is true
    765     if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupRun", !simple)) {
    766         psError(PS_ERR_UNKNOWN, false, "failed to print array");
    767         psFree(output);
    768         return false;
    769     }
    770 
    771     psFree(output);
    772 
    773     return true;
    774 }
    775 
    776 
    777 static bool pendingcleanupexpMode(pxConfig *config)
    778 {
    779     PS_ASSERT_PTR_NON_NULL(config, NULL);
    780 
    781     PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", false, false);
    782     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    783     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    784 
    785     psMetadata *where = psMetadataAlloc();
    786     if (add_id) {
    787         PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "==");
    788     }
    789     pxAddLabelSearchArgs (config, where, "-label", "label", "==");
    790 
    791     psString query = pxDataGet("addtool_pendingcleanupexp.sql");
    792     if (!query) {
    793         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    794         return false;
    795     }
    796 
    797     if (where && psListLength(where->list)) {
    798         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    799         psStringAppend(&query, " AND %s", whereClause);
    800         psFree(whereClause);
    801     }
    802     psFree(where);
    803 
    804     // treat limit == 0 as "no limit"
    805     if (limit) {
    806         psString limitString = psDBGenerateLimitSQL(limit);
    807         psStringAppend(&query, " %s", limitString);
    808         psFree(limitString);
    809     }
    810 
    811     if (!p_psDBRunQuery(config->dbh, query)) {
    812         psError(PS_ERR_UNKNOWN, false, "database error");
    813         psFree(query);
    814         return false;
    815     }
    816     psFree(query);
    817 
    818     psArray *output = p_psDBFetchResult(config->dbh);
    819     if (!output) {
    820         psError(PS_ERR_UNKNOWN, false, "database error");
    821         return false;
    822     }
    823     if (!psArrayLength(output)) {
    824         psTrace("chiptool", PS_LOG_INFO, "no rows found");
    825         psFree(output);
    826         return true;
    827     }
    828 
    829     // negative simple so the default is true
    830     if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupExp", !simple)) {
    831         psError(PS_ERR_UNKNOWN, false, "failed to print array");
    832         psFree(output);
    833         return false;
    834     }
    835 
    836     psFree(output);
    837 
    838     return true;
    839 }
    840 
    841 
    842 static bool donecleanupMode(pxConfig *config)
    843 {
    844     PS_ASSERT_PTR_NON_NULL(config, NULL);
    845 
    846     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    847     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    848 
    849     psMetadata *where = psMetadataAlloc();
    850     PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
    851 
    852     psString query = pxDataGet("addtool_donecleanup.sql");
    853     if (!query) {
    854         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    855         return false;
    856     }
    857 
    858     if (where && psListLength(where->list)) {
    859         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    860         psStringAppend(&query, " AND %s", whereClause);
    861         psFree(whereClause);
    862     }
    863     psFree(where);
    864 
    865     // treat limit == 0 as "no limit"
    866     if (limit) {
    867         psString limitString = psDBGenerateLimitSQL(limit);
    868         psStringAppend(&query, " %s", limitString);
    869         psFree(limitString);
    870     }
    871 
    872     if (!p_psDBRunQuery(config->dbh, query)) {
    873         psError(PS_ERR_UNKNOWN, false, "database error");
    874         psFree(query);
    875         return false;
    876     }
    877     psFree(query);
    878 
    879     psArray *output = p_psDBFetchResult(config->dbh);
    880     if (!output) {
    881         psError(PS_ERR_UNKNOWN, false, "database error");
    882         return false;
    883     }
    884     if (!psArrayLength(output)) {
    885         psTrace("addtool", PS_LOG_INFO, "no rows found");
    886         psFree(output);
    887         return true;
    888     }
    889 
    890     // negative simple so the default is true
    891     if (!ippdbPrintMetadatas(stdout, output, "addDoneCleanup", !simple)) {
    892         psError(PS_ERR_UNKNOWN, false, "failed to print array");
    893         psFree(output);
    894         return false;
    895     }
    896 
    897     psFree(output);
    898 
    899     return true;
    900 }
    901 
    902 bool exportrunMode(pxConfig *config)
    903 {
    904   typedef struct ExportTable {
    905     char tableName[80];
    906     char sqlFilename[80];
    907   } ExportTable;
    908 
    909   int numExportTables = 2;
    910 
    911   PS_ASSERT_PTR_NON_NULL(config, NULL);
    912 
    913   PXOPT_LOOKUP_S64(det_id, config->args, "-add_id", true,  false);
    914   PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
    915   PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
    916   PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
    917 
    918   FILE *f = fopen (outfile, "w");
    919   if (f == NULL) {
    920     psError(PS_ERR_UNKNOWN, false, "failed to open output file");
    921     return false;
    922   }
    923 
    924   psMetadata *where = psMetadataAlloc();
    925   PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "==");
    926 
    927   ExportTable tables [] = {
    928     {"addRun", "addtool_export_run.sql"},
    929     {"addProcessedExp", "addtool_export_processed_exp.sql"},
    930   };
    931 
    932   for (int i=0; i < numExportTables; i++) {
    933     psString query = pxDataGet(tables[i].sqlFilename);
    934     if (!query) {
    935       psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    936       return false;
    937     }
    938 
    939     if (where && psListLength(where->list)) {
    940       psString whereClause = psDBGenerateWhereSQL(where, NULL);
    941       psStringAppend(&query, " %s", whereClause);
    942       psFree(whereClause);
    943     }
    944 
    945     // treat limit == 0 as "no limit"
    946     if (limit) {
    947       psString limitString = psDBGenerateLimitSQL(limit);
    948       psStringAppend(&query, " %s", limitString);
    949       psFree(limitString);
    950     }
    951 
    952     if (!p_psDBRunQuery(config->dbh, query)) {
    953       psError(PS_ERR_UNKNOWN, false, "database error");
    954       psFree(query);
    955       return false;
    956     }
    957     psFree(query);
    958 
    959     psArray *output = p_psDBFetchResult(config->dbh);
    960     if (!output) {
    961       psError(PS_ERR_UNKNOWN, false, "database error");
    962       return false;
    963     }
    964     if (!psArrayLength(output)) {
    965       psError(PS_ERR_UNKNOWN, true, "no rows found");
    966       psFree(output);
    967       return false;
    968     }
    969 
    970     if (clean) {
    971         if (!strcmp(tables[i].tableName, "addRun")) {
    972             if (!pxSetStateCleaned("addRun", "state", output)) {
    973                 psFree(output);
    974                 psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
    975                 return false;
    976             }
    977         }
    978     }
    979 
    980     // we must write the export table in non-simple (true) format
    981     if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
    982       psError(PS_ERR_UNKNOWN, false, "failed to print array");
    983       psFree(output);
    984       return false;
    985     }
    986     psFree(output);
    987   }
    988 
    989   fclose (f);
    990 
    991   return true;
    992 }
    993 
    994 bool importrunMode(pxConfig *config)
    995 {
    996   unsigned int nFail;
    997 
    998   PS_ASSERT_PTR_NON_NULL(config, NULL);
    999 
    1000   PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
    1001 
    1002   psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
    1003 
    1004   fprintf (stdout, "---- input ----\n");
    1005   psMetadataPrint (stderr, input, 1);
    1006 
    1007   psMetadataItem *item = psMetadataLookup (input, "addRun");
    1008   psAssert (item, "entry not in input?");
    1009   psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
    1010 
    1011   psMetadataItem *entry = psListGet (item->data.list, 0);
    1012   assert (entry);
    1013   assert (entry->type == PS_DATA_METADATA);
    1014   addRunRow *addRun = addRunObjectFromMetadata (entry->data.md);
    1015   addRunInsertObject (config->dbh, addRun);
    1016 
    1017   // fprintf (stdout, "---- add run ----\n");
    1018   // psMetadataPrint (stderr, entry->data.md, 1);
    1019 
    1020   item = psMetadataLookup (input, "addProcessedExp");
    1021   psAssert (item, "entry not in input?");
    1022   psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
    1023 
    1024   for (int i = 0; i < item->data.list->n; i++) {
    1025     psMetadataItem *entry = psListGet (item->data.list, i);
    1026     assert (entry);
    1027     assert (entry->type == PS_DATA_METADATA);
    1028     addProcessedExpRow *addProcessedExp = addProcessedExpObjectFromMetadata (entry->data.md);
    1029     addProcessedExpInsertObject (config->dbh, addProcessedExp);
    1030 
    1031     // fprintf (stdout, "---- row %d ----\n", i);
    1032     // psMetadataPrint (stderr, entry->data.md, 1);
    1033   }
    1034 
    1035   return true;
    1036 }

  • trunk/ippTools/src/addtoolConfig.c

    r25789 r25816  
    5858    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction",      0, "define reduction class", NULL);
    5959    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb",          0, "define DVO db", NULL);
     60    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-image_only",        0, "addstar image metadata but not detections", false);
    6061    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend",           0, "do not actually modify the database", false);
     62    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple",            0, "use the simple output format", false);
    6163    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all",               0, "allow everything to be queued without search terms", false);
    62     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple",            0, "use the simple output format", false);
    6364
    6465    // -updaterun
    … …  
    8788    psMetadata *addprocessedexpArgs = psMetadataAlloc();
    8889    psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0,            "define addtool ID (required)", 0);
     90    psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0,            "define base output location", NULL);
    8991    psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN);
    90     psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0,            "define number of stars", 0);
    91     psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0,            "define base output location", NULL);
    92     psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-magicked", 0,             "set magicked", 0);
     92    psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
    9393
    9494    // -processedexp
    … …  
    101101
    102102    psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit",    0,            "limit result set to N items", 0);
     103    psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     104    psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0,            "only return imfiles with a fault status set", false);
    103105    psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-all",     0,            "list everything without restriction", false);
    104     psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    105106
    106107    // -revertprocessedexp
    … …  
    118119    psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0,            "search by addtool ID", 0);
    119120    psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id",  0,            "search by camtool ID", 0);
     121    psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
    120122
    121123    // -block
    … …  
    130132    psMetadata *unblockArgs = psMetadataAlloc();
    131133    psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label",  0,            "name of a label to unmask (required)", NULL);
    132 
    133     // -pendingcleanuprun
    134     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
    135     psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
    136     psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    137     psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    138 
    139     // -pendingcleanupexp
    140     psMetadata *pendingcleanupexpArgs = psMetadataAlloc();
    141     psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
    142     psMetadataAddS64(pendingcleanupexpArgs, PS_LIST_TAIL, "-add_id", 0,            "search by addstar ID", 0);
    143     psMetadataAddBool(pendingcleanupexpArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    144     psMetadataAddU64(pendingcleanupexpArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    145 
    146     // -donecleanup
    147     psMetadata *donecleanupArgs = psMetadataAlloc();
    148     psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
    149     psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    150     psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    151 
    152     // -exportrun
    153     psMetadata *exportrunArgs = psMetadataAlloc();
    154     psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-add_id", 0,          "export this addstar ID (required)", 0);
    155     psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
    156     psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
    157     psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean",  0,          "export tables as cleaned", false);
    158 
    159     // -importrun
    160     psMetadata *importrunArgs = psMetadataAlloc();
    161     psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
    162 
    163134
    164135    psMetadata *argSets = psMetadataAlloc();
    … …  
    175146    PXOPT_ADD_MODE("-masked",               "show blocked labels",                  ADDTOOL_MODE_MASKED,        maskedArgs);
    176147    PXOPT_ADD_MODE("-unblock",              "remove a label block",                 ADDTOOL_MODE_UNBLOCK,       unblockArgs);
    177     PXOPT_ADD_MODE("-pendingcleanuprun",    "show runs that need to be cleaned up", ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs);
    178     PXOPT_ADD_MODE("-pendingcleanupexp",    "show exps for cleanup runs",           ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpArgs);
    179     PXOPT_ADD_MODE("-donecleanup",          "show runs that have been cleaned",     ADDTOOL_MODE_DONECLEANUP,       donecleanupArgs);
    180     PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", ADDTOOL_MODE_EXPORTRUN, exportrunArgs);
    181     PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           ADDTOOL_MODE_IMPORTRUN, importrunArgs);
    182148
    183149    if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

  • trunk/ippTools/src/flatcorr.c

    r25784 r25816  
    3333#include "pxtools.h"
    3434#include "pxchip.h"
     35#include "pxcam.h"
     36#include "pxadd.h"
    3537#include "flatcorr.h"
    3638
    … …  
    3840static bool definerunMode(pxConfig *config);
    3941static bool addchipMode(pxConfig *config);
     42static bool dropchipMode(pxConfig *config);
    4043static bool addcameraMode(pxConfig *config);
     44static bool dropcameraMode(pxConfig *config);
    4145static bool advancecameraMode(pxConfig *config);
    42 static bool dropchipMode(pxConfig *config);
    43 static bool dropcameraMode(pxConfig *config);
     46static bool advanceaddstarMode(pxConfig *config);
    4447static bool pendingprocessMode(pxConfig *config);
    4548static bool addprocessMode(pxConfig *config);
    … …  
    4750static bool inputexpMode(pxConfig *config);
    4851static bool inputimfileMode(pxConfig *config);
    49 static bool exportrunMode(pxConfig *config);
    50 static bool importrunMode(pxConfig *config);
    5152
    5253static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state);
    … …  
    7374        MODECASE(FLATCORR_MODE_DEFINERUN,      definerunMode);
    7475        MODECASE(FLATCORR_MODE_ADDCHIP,        addchipMode);
     76        MODECASE(FLATCORR_MODE_DROPCHIP,       dropchipMode);
    7577        MODECASE(FLATCORR_MODE_ADDCAMERA,      addcameraMode);
     78        MODECASE(FLATCORR_MODE_DROPCAMERA,     dropcameraMode);
    7679        MODECASE(FLATCORR_MODE_ADVANCECAMERA,  advancecameraMode);
    77         MODECASE(FLATCORR_MODE_DROPCHIP,       dropchipMode);
    78         MODECASE(FLATCORR_MODE_DROPCAMERA,     dropcameraMode);
     80        MODECASE(FLATCORR_MODE_ADVANCEADDSTAR, advanceaddstarMode);
    7981        MODECASE(FLATCORR_MODE_PENDINGPROCESS, pendingprocessMode);
    8082        MODECASE(FLATCORR_MODE_ADDPROCESS,     addprocessMode);
    … …  
    8284        MODECASE(FLATCORR_MODE_INPUTEXP,       inputexpMode);
    8385        MODECASE(FLATCORR_MODE_INPUTIMFILE,    inputimfileMode);
    84         MODECASE(FLATCORR_MODE_EXPORTRUN,      exportrunMode);
    85         MODECASE(FLATCORR_MODE_IMPORTRUN,      importrunMode);
    8686        default:
    8787            psAbort("invalid option (this should not happen)");
    … …  
    404404}
    405405
     406static bool dropcameraMode(pxConfig *config)
     407{
     408    PS_ASSERT_PTR_NON_NULL(config, false);
     409
     410    // required
     411    PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false);
     412    PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false);
     413
     414    // UPDATE flatcorrCamLink set include = 0 where corr_id = %lld AND cam_id = %lld
     415    psString query = pxDataGet("flatcorr_dropcamera.sql");
     416    if (!query) {
     417        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
     418        return false;
     419    }
     420
     421    if (!p_psDBRunQueryF(config->dbh, query, (long long) corr_id, (long long) cam_id)) {
     422        psError(PS_ERR_UNKNOWN, false, "database error");
     423        psFree(query);
     424        return false;
     425    }
     426
     427    return true;
     428}
     429
    406430// select the flatcorr chip runs that have completed and for which there is no camera entry
    407431// queue a new camera run for them
    … …  
    413437    PXOPT_LOOKUP_BOOL(limit,   config->args, "-limit",   false);
    414438    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
     439
     440    psMetadata *where = psMetadataAlloc();
     441    pxAddLabelSearchArgs (config, where, "-label", "flatcorrRun.label", "==");
    415442
    416443    psString query = pxDataGet("flatcorr_chiprundone.sql");
    … …  
    419446        return false;
    420447    }
     448
     449    if (where && psListLength(where->list)) {
     450        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     451        psStringAppend(&query, " AND %s", whereClause);
     452        psFree(whereClause);
     453    }
     454    psFree(where);
    421455
    422456    // treat limit == 0 as "no limit"
    … …  
    528562}
    529563
    530 static bool dropcameraMode(pxConfig *config)
    531 {
    532     PS_ASSERT_PTR_NON_NULL(config, false);
    533 
    534     // required
    535     PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false);
    536     PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false);
    537 
    538     // UPDATE flatcorrCamLink set include = 0 where corr_id = %lld AND cam_id = %lld
    539     psString query = pxDataGet("flatcorr_dropcamera.sql");
    540     if (!query) {
    541         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    542         return false;
    543     }
    544 
    545     if (!p_psDBRunQueryF(config->dbh, query, (long long) corr_id, (long long) cam_id)) {
    546         psError(PS_ERR_UNKNOWN, false, "database error");
    547         psFree(query);
    548         return false;
    549     }
    550 
    551     return true;
    552 }
    553 
    554 // select the flatcorr chip runs that have completed and for which there is no camera entry
    555 // queue a new camera run for them
    556 static bool pendingprocessMode(pxConfig *config)
     564// Select the flatcorr camera runs that have completed and for which there is no addstar
     565// entry.  Queue a new addstar run for them
     566static bool advanceaddstarMode(pxConfig *config)
    557567{
    558568    PS_ASSERT_PTR_NON_NULL(config, false);
    … …  
    560570    PXOPT_LOOKUP_BOOL(simple,  config->args, "-simple",  false);
    561571    PXOPT_LOOKUP_BOOL(limit,   config->args, "-limit",   false);
     572    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
    562573
    563574    psMetadata *where = psMetadataAlloc();
    564     pxAddLabelSearchArgs (config, where, "-label", "label", "==");
    565 
    566     psString query = pxDataGet("flatcorr_pendingprocess.sql");
     575    pxAddLabelSearchArgs (config, where, "-label", "flatcorrRun.label", "==");
     576
     577    psString query = pxDataGet("flatcorr_camerarundone.sql");
    567578    if (!query) {
    568579        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    … …  
    601612    }
    602613
     614    if (pretend) {
     615        // negative simple so the default is true
     616        if (!ippdbPrintMetadatas(stdout, output, "flatcorr_addcamera", !simple)) {
     617            psError(PS_ERR_UNKNOWN, false, "failed to print array");
     618            psFree(output);
     619            return false;
     620        }
     621    }
     622
     623    // start a transaction so we don't end up with an exp without any associted
     624    // imfiles
     625    if (!psDBTransaction(config->dbh)) {
     626        psError(PS_ERR_UNKNOWN, false, "database error");
     627        psFree(output);
     628        return false;
     629    }
     630
     631    // loop over our list of chipRun rows
     632    for (long i = 0; i < psArrayLength(output); i++) {
     633        psMetadata *md = output->data[i];
     634
     635        bool status;
     636        psS64 corr_id = psMetadataLookupS64(&status, md, "corr_id");
     637        if (!status) {
     638            if (!psDBRollback(config->dbh)) {
     639                psError(PS_ERR_UNKNOWN, false, "database error");
     640            }
     641            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for corr_id");
     642            psFree(output);
     643            return false;
     644        }
     645
     646        camRunRow *row = camRunObjectFromMetadata(md);
     647        if (!row) {
     648            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun");
     649            psFree(output);
     650            return false;
     651        }
     652
     653        // queue the exp : force image_only to be false (flatcorr is meaningless with just image info)
     654        if (!pxaddQueueByCamID(
     655                config,
     656                row->cam_id,
     657                row->workdir,
     658                row->reduction,
     659                row->label,
     660                row->dvodb,
     661                0)) {
     662            if (!psDBRollback(config->dbh)) {
     663                psError(PS_ERR_UNKNOWN, false, "database error");
     664            }
     665            psError(PS_ERR_UNKNOWN, false,
     666                    "failed to trying to queue cam_id: %" PRId64, row->cam_id);
     667            psFree(row);
     668            psFree(output);
     669            return false;
     670        }
     671
     672        // figure out the ID of the flatcorrRun we just created
     673        psS64 add_id = psDBLastInsertID(config->dbh);
     674
     675        // add the addRun entry to the flatcorrAddstarLink table (include is TRUE)
     676        if (!flatcorrAddstarLinkInsert(config->dbh, corr_id, row->cam_id, add_id, 1)) {
     677            if (!psDBRollback(config->dbh)) {
     678                psError(PS_ERR_UNKNOWN, false, "database error");
     679            }
     680            psError(PS_ERR_UNKNOWN, false, "database error");
     681            return false;
     682        }
     683        psFree(row);
     684    }
     685    psFree(output);
     686
     687    if (!psDBCommit(config->dbh)) {
     688        psError(PS_ERR_UNKNOWN, false, "database error");
     689        return false;
     690    }
     691
     692    return false;
     693}
     694
     695// select the flatcorr chip runs that have completed and for which there is no camera entry
     696// queue a new camera run for them
     697static bool pendingprocessMode(pxConfig *config)
     698{
     699    PS_ASSERT_PTR_NON_NULL(config, false);
     700
     701    PXOPT_LOOKUP_BOOL(simple,  config->args, "-simple",  false);
     702    PXOPT_LOOKUP_BOOL(limit,   config->args, "-limit",   false);
     703
     704    psMetadata *where = psMetadataAlloc();
     705    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
     706
     707    psString query = pxDataGet("flatcorr_pendingprocess.sql");
     708    if (!query) {
     709        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
     710        return false;
     711    }
     712
     713    if (where && psListLength(where->list)) {
     714        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     715        psStringAppend(&query, " AND %s", whereClause);
     716        psFree(whereClause);
     717    }
     718    psFree(where);
     719
     720    // treat limit == 0 as "no limit"
     721    if (limit) {
     722        psString limitString = psDBGenerateLimitSQL(limit);
     723        psStringAppend(&query, " %s", limitString);
     724        psFree(limitString);
     725    }
     726
     727    if (!p_psDBRunQuery(config->dbh, query)) {
     728        psError(PS_ERR_UNKNOWN, false, "database error");
     729        psFree(query);
     730        return false;
     731    }
     732
     733    psArray *output = p_psDBFetchResult(config->dbh);
     734    if (!output) {
     735        psError(PS_ERR_UNKNOWN, false, "database error");
     736        return false;
     737    }
     738    if (!psArrayLength(output)) {
     739        psTrace("flatcorr", PS_LOG_INFO, "no rows found");
     740        psFree(output);
     741        return true;
     742    }
     743
    603744    if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) {
    604745        psError(PS_ERR_UNKNOWN, false, "failed to print array");
    605746        psFree(output);
    606747        return false;
     748    }
     749
     750    return true;
     751}
     752
     753// XXX need a fault state
     754static bool addprocessMode(pxConfig *config)
     755{
     756    PS_ASSERT_PTR_NON_NULL(config, false);
     757
     758    PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false);
     759    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
     760    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
     761
     762    char *query = "UPDATE flatcorrRun SET state = 'full', hostname = '%s', fault = '%hd' WHERE corr_id = %" PRId64;
     763
     764    if (!p_psDBRunQueryF(config->dbh, query, hostname, code, corr_id)) {
     765        psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id);
     766        return false;
     767    }
     768
     769    return true;
     770}
     771
     772static bool updaterunMode(pxConfig *config)
     773{
     774    PS_ASSERT_PTR_NON_NULL(config, false);
     775
     776    PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false);
     777    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
     778
     779    if (!setflatcorrRunState(config, corr_id, state)) {
     780        if (!psDBRollback(config->dbh)) {
     781            psError(PS_ERR_UNKNOWN, false, "database error");
     782        }
     783        psError(PS_ERR_UNKNOWN, false, "failed to set run state");
     784        return false;
    607785    }
    608786
    … …  
    714892}
    715893
    716 // XXX need a fault state
    717 static bool addprocessMode(pxConfig *config)
    718 {
    719     PS_ASSERT_PTR_NON_NULL(config, false);
    720 
    721     PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false);
    722     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
    723     PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
    724 
    725     char *query = "UPDATE flatcorrRun SET state = 'full', hostname = '%s', fault = '%hd' WHERE corr_id = %" PRId64;
    726 
    727     if (!p_psDBRunQueryF(config->dbh, query, hostname, code, corr_id)) {
    728         psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id);
    729         return false;
    730     }
    731 
    732     return true;
    733 }
    734 
    735 static bool updaterunMode(pxConfig *config)
    736 {
    737     PS_ASSERT_PTR_NON_NULL(config, false);
    738 
    739     PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false);
    740     PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
    741 
    742     if (!setflatcorrRunState(config, corr_id, state)) {
    743         if (!psDBRollback(config->dbh)) {
    744             psError(PS_ERR_UNKNOWN, false, "database error");
    745         }
    746         psError(PS_ERR_UNKNOWN, false, "failed to set run state");
    747         return false;
    748     }
    749 
    750     return true;
    751 }
    752 
    753894static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state)
    754895{
    … …  
    774915    return true;
    775916}
    776 
    777 bool exportrunMode(pxConfig *config)
    778 {
    779   typedef struct ExportTable {
    780     char tableName[80];
    781     char sqlFilename[80];
    782   } ExportTable;
    783  
    784   int numExportTables = 3;
    785 
    786   PS_ASSERT_PTR_NON_NULL(config, NULL);
    787 
    788   PXOPT_LOOKUP_S64(det_id, config->args, "-corr_id", true,  false);
    789   PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
    790   PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
    791 
    792   FILE *f = fopen (outfile, "w");
    793   if (f == NULL) {
    794     psError(PS_ERR_UNKNOWN, false, "failed to open output file");
    795     return false;
    796   }
    797 
    798   psMetadata *where = psMetadataAlloc();
    799   PXOPT_COPY_S64(config->args, where, "-corr_id", "corr_id", "==");
    800 
    801   ExportTable tables [] = {
    802     {"flatcorrRun", "flatcorr_export_run.sql"},
    803     {"flatcorrCamLink", "flatcorr_export_cam_link.sql"},
    804     {"flatcorrChipLink", "flatcorr_export_chip_link.sql"},
    805   };
    806 
    807   for (int i=0; i < numExportTables; i++) {
    808     psString query = pxDataGet(tables[i].sqlFilename);
    809     if (!query) {
    810       psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    811       return false;
    812     }
    813 
    814     if (where && psListLength(where->list)) {
    815       psString whereClause = psDBGenerateWhereSQL(where, NULL);
    816       psStringAppend(&query, " %s", whereClause);
    817       psFree(whereClause);
    818     }
    819 
    820     // treat limit == 0 as "no limit"
    821     if (limit) {
    822       psString limitString = psDBGenerateLimitSQL(limit);
    823       psStringAppend(&query, " %s", limitString);
    824       psFree(limitString);
    825     }
    826 
    827     if (!p_psDBRunQuery(config->dbh, query)) {
    828       psError(PS_ERR_UNKNOWN, false, "database error");
    829       psFree(query);
    830       return false;
    831     }
    832     psFree(query);
    833 
    834     psArray *output = p_psDBFetchResult(config->dbh);
    835     if (!output) {
    836       psError(PS_ERR_UNKNOWN, false, "database error");
    837       return false;
    838     }
    839     if (!psArrayLength(output)) {
    840       psTrace("regtool", PS_LOG_INFO, "no rows found");
    841       psFree(output);
    842       return true;
    843     }
    844 
    845     // we must write the export table in non-simple (true) format
    846     if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
    847       psError(PS_ERR_UNKNOWN, false, "failed to print array");
    848       psFree(output);
    849       return false;
    850     }
    851     psFree(output);
    852   }
    853 
    854     fclose (f);
    855 
    856     return true;
    857 }
    858 
    859 bool importrunMode(pxConfig *config)
    860 {
    861   unsigned int nFail;
    862   psMetadataItem *item, *entry;
    863  
    864   int numImportTables = 3;
    865  
    866   char tables[3] [80] = {"flatcorrRun", "flatcorrCamLink", "flatcorrChipLink"};
    867 
    868   PS_ASSERT_PTR_NON_NULL(config, NULL);
    869  
    870   PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
    871 
    872   psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
    873 
    874   fprintf (stdout, "---- input ----\n");
    875   psMetadataPrint (stderr, input, 1);
    876 
    877   for (int i = 0; i < numImportTables; i++) {
    878     item = psMetadataLookup (input, tables[i]);
    879     psAssert (item, "entry not in input?");
    880     psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
    881    
    882     entry = psListGet (item->data.list, 0);
    883     assert (entry);
    884     assert (entry->type == PS_DATA_METADATA);
    885  
    886     switch (i) {
    887       case 0:
    888       {
    889         flatcorrRunRow *flatcorrRun = flatcorrRunObjectFromMetadata (entry->data.md);
    890         flatcorrRunInsertObject (config->dbh, flatcorrRun);
    891 
    892         // fprintf (stdout, "---- flatcorr run ----\n");
    893         // psMetadataPrint (stderr, entry->data.md, 1);
    894         break;
    895       }
    896       case 1:
    897       {
    898         flatcorrCamLinkRow *flatcorrCamLink = flatcorrCamLinkObjectFromMetadata (entry->data.md);
    899         flatcorrCamLinkInsertObject (config->dbh, flatcorrCamLink);
    900 
    901         // fprintf (stdout, "---- flatcorr cam link ----\n");
    902         // psMetadataPrint (stderr, entry->data.md, 1);
    903         break;
    904       }
    905       case 2:
    906       {
    907         flatcorrChipLinkRow *flatcorrChipLink = flatcorrChipLinkObjectFromMetadata (entry->data.md);
    908         flatcorrChipLinkInsertObject (config->dbh, flatcorrChipLink);
    909 
    910         // fprintf (stdout, "---- flatcorr chip link ----\n");
    911         // psMetadataPrint (stderr, entry->data.md, 1);
    912         break;
    913       }
    914     }
    915   }
    916   return true;
    917 }

  • trunk/ippTools/src/flatcorr.h

    r25784 r25816  
    2828    FLATCORR_MODE_DEFINERUN,
    2929    FLATCORR_MODE_ADDCHIP,
     30    FLATCORR_MODE_DROPCHIP,
    3031    FLATCORR_MODE_ADDCAMERA,
     32    FLATCORR_MODE_DROPCAMERA,
    3133    FLATCORR_MODE_ADVANCECAMERA,
    32     FLATCORR_MODE_DROPCHIP,
    33     FLATCORR_MODE_DROPCAMERA,
     34    FLATCORR_MODE_ADVANCEADDSTAR,
    3435    FLATCORR_MODE_PENDINGPROCESS,
    3536    FLATCORR_MODE_ADDPROCESS,
    … …  
    3738    FLATCORR_MODE_INPUTEXP,
    3839    FLATCORR_MODE_INPUTIMFILE,
    39     FLATCORR_MODE_EXPORTRUN,
    40     FLATCORR_MODE_IMPORTRUN
    4140} flatcorrMode;
    4241

  • trunk/ippTools/src/flatcorrConfig.c

    r25784 r25816  
    8383    psMetadataAddS64(addchipArgs, PS_LIST_TAIL, "-chip_id", 0,            "define Chip ID (required)", 0);
    8484
     85    // -dropchip
     86    psMetadata *dropchipArgs = psMetadataAlloc();
     87    psMetadataAddS64(dropchipArgs, PS_LIST_TAIL, "-corr_id", 0,            "define Flat Correction ID (required)", 0);
     88    psMetadataAddS64(dropchipArgs, PS_LIST_TAIL, "-chip_id", 0,            "define Chip ID (required)", 0);
     89
    8590    // -addcamera
    8691    psMetadata *addcameraArgs = psMetadataAlloc();
    … …  
    8994    psMetadataAddS64(addcameraArgs, PS_LIST_TAIL, "-cam_id", 0,             "define Camera ID (required)", 0);
    9095
    91     // -dropchip
    92     psMetadata *dropchipArgs = psMetadataAlloc();
    93     psMetadataAddS64(dropchipArgs, PS_LIST_TAIL, "-corr_id", 0,            "define Flat Correction ID (required)", 0);
    94     psMetadataAddS64(dropchipArgs, PS_LIST_TAIL, "-chip_id", 0,            "define Chip ID (required)", 0);
     96    // -dropcamera
     97    psMetadata *dropcameraArgs = psMetadataAlloc();
     98    psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-corr_id", 0,      "define Flat Correction ID (required)", 0);
     99    psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-cam_id", 0,       "define Camera ID (required)", 0);
    95100
    96101    // -advancecamera
    … …  
    99104    psMetadataAddBool(advancecameraArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
    100105    psMetadataAddBool(advancecameraArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false);
     106    psMetadataAddStr(advancecameraArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by label", NULL);
    101107
    102     // -dropcamera
    103     psMetadata *dropcameraArgs = psMetadataAlloc();
    104     psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-corr_id", 0,      "define Flat Correction ID (required)", 0);
    105     psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-cam_id", 0,       "define Camera ID (required)", 0);
     108    // -advanceaddstar
     109    psMetadata *advanceaddstarArgs = psMetadataAlloc();
     110    psMetadataAddU64 (advanceaddstarArgs, PS_LIST_TAIL, "-limit",   0, "limit result set to N items", 0);
     111    psMetadataAddBool(advanceaddstarArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
     112    psMetadataAddBool(advanceaddstarArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false);
     113    psMetadataAddStr(advanceaddstarArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by label", NULL);
    106114
    107115    // -pendingprocess
    … …  
    155163    PXOPT_ADD_MODE("-addcamera",      "add an existing camera run to a flat correction run", FLATCORR_MODE_ADDCAMERA,      addcameraArgs);
    156164    PXOPT_ADD_MODE("-advancecamera",  "migrate completed chips to camera stage analysis",    FLATCORR_MODE_ADVANCECAMERA,  advancecameraArgs);
     165    PXOPT_ADD_MODE("-advanceaddstar", "migrate completed exposures to addstar processing",   FLATCORR_MODE_ADVANCEADDSTAR,  advanceaddstarArgs);
    157166    PXOPT_ADD_MODE("-dropchip",       "drop a chip from a flat correction run",              FLATCORR_MODE_DROPCHIP,       dropchipArgs);
    158167    PXOPT_ADD_MODE("-dropcamera",     "drop an exposure (camera stage analysis)",            FLATCORR_MODE_DROPCAMERA,     dropcameraArgs);
    … …  
    162171    PXOPT_ADD_MODE("-inputexp",       "list exposures for a correction run",                 FLATCORR_MODE_INPUTEXP,       inputexpArgs);
    163172    PXOPT_ADD_MODE("-inputimfile",    "list imfiles for a chip run",                         FLATCORR_MODE_INPUTIMFILE,    inputimfileArgs);
    164     PXOPT_ADD_MODE("-exportrun",            "export run for import on other database",       FLATCORR_MODE_EXPORTRUN,      exportrunArgs);
    165     PXOPT_ADD_MODE("-importrun",            "import run from metadata file",                 FLATCORR_MODE_IMPORTRUN,      importrunArgs);
    166173
    167174    if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

  • trunk/ippTools/src/pxadd.c

    r25789 r25816  
    2929#include "pxadd.h"
    3030
    31 bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked)
     31bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state)
    3232{
    3333    PS_ASSERT_PTR_NON_NULL(config, false);
    … …  
    4141    }
    4242
    43     char *query = "UPDATE addRun SET state = '%s', magicked = %" PRId64 " WHERE add_id = %" PRId64;
    44     if (!p_psDBRunQueryF(config->dbh, query, state, magicked, add_id)) {
    45         psError(PS_ERR_UNKNOWN, false,
    46                 "failed to change state for add_id %" PRId64, add_id);
     43    char *query = "UPDATE addRun SET state = '%s' WHERE add_id = %" PRId64;
     44    if (!p_psDBRunQueryF(config->dbh, query, state, add_id)) {
     45        psError(PS_ERR_UNKNOWN, false, "failed to change state for add_id %" PRId64, add_id);
    4746        return false;
    4847    }
    … …  
    123122}
    124123
    125 // Need to think more about this to see what we want it to do. BROKEN
    126124bool pxaddQueueByCamID(pxConfig *config,
    127125                       psS64 cam_id,
    128126                       char *workdir,
     127                       char *reduction,
    129128                       char *label,
    130                        char *recipe,
    131                        char *dvodb)
     129                       char *dvodb,
     130                       bool image_only)
    132131{
    133132    PS_ASSERT_PTR_NON_NULL(config, false);
    … …  
    149148    if (!p_psDBRunQueryF(config->dbh, query,
    150149                         "new", // state
    151                          workdir  ? workdir  : "NULL",
     150                         workdir  ? workdir   : "NULL",
    152151                         "dirty", //workdir_state
    153                          label    ? label    : "NULL",
    154                          dvodb    ? dvodb    : "NULL",
    155                          (long long)cam_id
     152                         reduction? reduction : "NULL",
     153                         label    ? label     : "NULL",
     154                         dvodb    ? dvodb     : "NULL",
     155                         image_only,
     156                         (long long) cam_id
    156157    )) {
    157158        psError(PS_ERR_UNKNOWN, false, "database error");

  • trunk/ippTools/src/pxadd.h

    r25789 r25816  
    2525#include "pxtools.h"
    2626
    27 bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked);
     27bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state);
    2828bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state);
    2929bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label);
    3030bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label);
    3131
    32 // Likely BROKEN
    3332bool pxaddQueueByCamID(pxConfig *config,
    34                         psS64 cam_id,
    35                         char *workdir,
    36                         char *label,
    37                         char *recipe,
    38                         char *dvodb);
    39 
     33                       psS64 cam_id,
     34                       char *workdir,
     35                       char *reduction,
     36                       char *label,
     37                       char *dvodb,
     38                       bool image_only);
    4039
    4140#endif // PXADD_H
Note: See TracChangeset for help on using the changeset viewer.