IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 26567


Ignore:
Timestamp:
Jan 12, 2010, 2:07:51 PM (16 years ago)
Author:
bills
Message:

Make update processing compatible with magic. Fix some bugs in update for the warp stage
Make script changes to implement update for diff stage.
Change value of 'magicked' in the various runs and files to contain the value
the magic_id of the magicRun that corresponds to the streaks file used for desreaking.

Location:
trunk
Files:
38 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/chip_imfile.pl

    r25555 r26567  
    5353    'reduction=s'       => \$reduction, # Reduction class
    5454    'run-state=s'       => \$run_state, # current state of the run (new, update)
    55     'magicked'          => \$magicked,  # magicked state of input file
     55    'magicked=s'        => \$magicked,  # magicked state of input file
    5656    'deburned=s'        => \$deburned,  # does deburned image exist?
    5757    'threads=s'         => \$threads,   # Number of threads to use for ppImage
    … …  
    411411    $command .= " -uri $outputImage";
    412412    $command .= " -path_base $outroot";
    413     $command .= " -magicked" if $magicked;
     413    $command .= " -magicked $magicked" if $magicked;
    414414    $command .= " -hostname $host" if defined $host;
    415415    $command .= " -dbname $dbname" if defined $dbname;

  • trunk/ippScripts/scripts/diff_skycell.pl

    r24179 r26567  
    3535}
    3636
    37 my ($diff_id, $dbname, $threads, $outroot, $reduction, $inverse, $verbose, $no_update, $no_op, $redirect);
     37my ($diff_id, $dbname, $threads, $outroot, $reduction, $inverse, $run_state, $verbose, $no_update, $no_op, $redirect);
    3838my ($skycell_id, $diff_skyfile_id);
    3939GetOptions(
    … …  
    4343    'dbname|d=s'        => \$dbname, # Database name
    4444    'threads=s'         => \$threads,   # Number of threads to use
     45    'run-state=s'       => \$run_state,   # state for run: 'new' or 'update'
    4546    'outroot=s'         => \$outroot, # Output root name
    4647    'inverse'           => \$inverse, # Make inverse subtraction?
    … …  
    5455pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    5556pod2usage(
    56     -msg => "Required options: --diff_id --skycell_id --outroot",
     57    -msg => "Required options: --diff_id --skycell_id --outroot --run-state",
    5758    -exitval => 3,
    5859          ) unless defined $diff_id
    5960    and defined $skycell_id
    6061    and defined $diff_skyfile_id
     62    and defined $run_state
    6163    and defined $outroot;
    6264
    … …  
    6668# my $logDest = $ipprc->filename("LOG.EXP", $outroot);
    6769my $logDest = "$outroot.log";
     70$logDest .= ".update" if $run_state eq "update";
    6871$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
    6972
    … …  
    155158# The output file will be considered magicked if the input has been magicked and the
    156159# template is either a stack or a warp that has been magicked.
    157 # note that difftool -inputskyfile outputs the magicked boolean as an int not T or F
    158 # because the output is constructed from a union of two selects
    159 my $magicked = $inputMagic && $templateMagic;
     160my $magicked = $inputMagic && $templateMagic ? $inputMagic : 0;
    160161
    161162# Recipes to use based on reduction class
    … …  
    212213my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot);
    213214my $traceDest = $ipprc->filename("TRACE.EXP", $outroot);
     215
     216if ($run_state eq 'update') {
     217    $traceDest .= '.update';
     218    $outputStats .= '.update';
     219}
    214220
    215221my ($inverseName, $inverseMask, $inverseVariance, $inverseSources);
    … …  
    236242    $command .= " -stats $outputStats";
    237243    $command .= " -threads $threads" if defined $threads;
     244    my $configurationReal = $ipprc->file_resolve($configuration);
     245    if ($run_state eq "new") {
     246        $command .= " -dumpconfig $configurationReal";
     247    } else {
     248        $command .= " -ipprc $configurationReal";
     249    }
    238250    $command .= " -recipe PPSUB $recipe_ppSub";
    239251    $command .= " -recipe PSPHOT $recipe_psphot";
    … …  
    245257    $command .= " -inverse" if $inverse;
    246258    $command .= " -tracedest $traceDest -log $logDest";
    247     $command .= " -dumpconfig $configuration";
    248259    $command .= " -dbname $dbname" if defined $dbname;
    249260    $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id;
    … …  
    295306    # Add the subtraction result
    296307    {
    297         my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -path_base $outroot";
    298         $command .= " $cmdflags";
    299         $command .= " -magicked" if $magicked;
    300         $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    301         $command .= " -hostname $host" if defined $host;
     308        my $command = "$difftool -diff_id $diff_id -skycell_id $skycell_id";
     309        $command .= " -magicked $magicked" if $magicked;
     310        if ($run_state eq 'new') {
     311            $command .= " -adddiffskyfile -path_base $outroot";
     312            $command .= " $cmdflags";
     313            $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
     314            $command .= " -hostname $host" if defined $host;
     315        } else {
     316            $command .= " -tofullskyfile";
     317        }
    302318        $command .= " -dbname $dbname" if defined $dbname;
    303319
    … …  
    306322        unless ($success) {
    307323            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    308             &my_die("Unable to perform difftool -adddiffskyfile: $error_code", $diff_id, $skycell_id, $error_code);
     324            my $err_message = $run_state eq "update" ?
     325                "Unable to perform difftool -adddiffskyfile" :
     326                "Unable to perform difftool -tofullskyfile";
     327            &my_die("$err_message: $error_code", $diff_id, $skycell_id, $error_code);
    309328        }
    310329    }
    … …  
    323342    warn($msg);
    324343    if (defined $diff_id and defined $skycell_id and not $no_update) {
    325         my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -fault $exit_code";
    326         $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
    327         $command .= " -hostname $host" if defined $host;
    328         $command .= " -path_base $outroot" if defined $outroot;
    329         $command .= " -dbname $dbname" if defined $dbname;
     344        my $command = "$difftool -diff_id $diff_id -skycell_id $skycell_id -fault $exit_code";
     345        if ($run_state eq 'new') {
     346            $command .= " -adddiffskyfile";
     347            $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
     348            $command .= " -hostname $host" if defined $host;
     349            $command .= " -path_base $outroot" if defined $outroot;
     350            $command .= " -dbname $dbname" if defined $dbname;
     351        } else {
     352            $command .= " -updatediffskyfile";
     353        }
    330354        run(command => $command, verbose => $verbose);
    331355    }

  • trunk/ippScripts/scripts/warp_skycell.pl

    r25518 r26567  
    4848    'threads=s'           => \$threads,   # Number of threads to use for pswarp
    4949    'run-state=s'         => \$run_state,  # 'new' or 'update'
    50     'magicked'            => \$magicked,  # input run has been magicked already?
     50    'magicked=s'          => \$magicked,  # input run has been magicked already?
    5151    'verbose'             => \$verbose,   # Print to stdout
    5252    'no-update'           => \$no_update, # Don't update the database?
    … …  
    276276            $command .= " -tess_id $tess_dir";
    277277            $command .= " -path_base $outroot"; # needed for logfile lookups
    278             $command .= " -magicked" if $magicked;
     278            $command .= " -magicked $magicked" if $magicked;
    279279
    280280            $command .= " -uri $outputImage" if !$quality;

  • trunk/ippTasks/chip.pro

    r25693 r26567  
    138138    book getword chipPendingImfile $pageName state -var RUN_STATE
    139139
    140     if ("$RAW_MAGICKED" == "T")
    141         $MAGICKED_ARG = "--magicked"
     140    if ($RAW_MAGICKED > 0)
     141        $MAGICKED_ARG = "--magicked $RAW_MAGICKED"
    142142    else
    143143        $MAGICKED_ARG = ""

  • trunk/ippTasks/diff.pro

    r25570 r26567  
    155155    book getword diffSkyfile $pageName bothways -var BOTHWAYS
    156156    book getword diffSkyfile $pageName workdir -var WORKDIR_TEMPLATE
     157    book getword diffSkyfile $pageName state -var RUN_STATE
    157158    book getword diffSkyfile $pageName dbname -var DBNAME
    158159
    … …  
    171172    stderr $LOGDIR/diff.skycell.log
    172173
    173     $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --skycell_id $SKYCELL_ID --diff_skyfile_id $DIFF_SKYFILE_ID --outroot $outroot --redirect-output
     174    $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --skycell_id $SKYCELL_ID --diff_skyfile_id $DIFF_SKYFILE_ID --outroot $outroot --redirect-output --run-state $RUN_STATE
    174175    if ("$BOTHWAYS" == "T")
    175176       $run = $run --inverse

  • trunk/ippTasks/dist.cleanup.pro

    r26087 r26567  
    4848
    4949  stdout NULL
    50   stderr $LOGDIR/dist.cleanup.log
     50  stderr $LOGDIR/dist.cleanup.load.log
    5151
    5252  task.exec
    5353    if ($LABEL:n == 0) break
    54     $run = disttool -pendingcleanup
     54    $run = disttool -pendingcleanup -all
    5555    if ($DB:n == 0)
    5656      option DEFAULT
    … …  
    9898  periods      -poll $RUNPOLL
    9999  periods      -exec $RUNEXEC
    100   periods      -timeout 60
     100  periods      -timeout 600
    101101  active       false
    102102

  • trunk/ippTasks/warp.pro

    r25693 r26567  
    295295    book getword warpPendingSkyCell $pageName state -var RUN_STATE
    296296    book getword warpPendingSkyCell $pageName magicked -var CHIP_MAGICKED
    297     if ("$CHIP_MAGICKED" == "T")
    298         $MAGICKED_ARG = "--magicked"
     297    if ($CHIP_MAGICKED > 0)
     298        $MAGICKED_ARG = "--magicked $CHIP_MAGICKED"
    299299    else
    300300        $MAGICKED_ARG = ""

  • trunk/ippTools/share/camtool_find_pendingexp.sql

    r23617 r26567  
    99    rawExp.camera,
    1010    rawExp.telescope,
    11     rawExp.filelevel
     11    rawExp.filelevel,
     12    chipRun.magicked AS chip_magicked
    1213FROM camRun
    1314JOIN chipRun

  • trunk/ippTools/share/camtool_queue_chip_id.sql

    r25835 r26567  
    1616        '%s',           -- tess_id
    1717        '%s',           -- end_stage
    18         0,              -- magicked
    19         '%s'           -- note
     18        %lld,           -- magicked
     19        '%s'            -- note
    2020    FROM chipRun
    2121    WHERE

  • trunk/ippTools/share/chiptool_change_exp_state.sql

    r19702 r26567  
    44-- the chipProcessedImfile sub query
    55UPDATE chipRun
    6     SET state = '%s'
     6JOIN rawExp using(exp_id)
     7    SET chipRun.state = '%s'
     8    -- set magicked hook %s
    79    WHERE
    810    chipRun.chip_id = %lld

  • trunk/ippTools/share/chiptool_completely_processed_exp.sql

    r25835 r26567  
    1414    tess_id,
    1515    end_stage,
    16     all_files_magicked as magicked,
     16    imfile_magicked as magicked,
    1717    note
    1818FROM
    … …  
    2121        rawImfile.class_id as rawimfile_class_id,
    2222        chipProcessedImfile.class_id,
    23         SUM(!chipProcessedImfile.magicked) = 0 as all_files_magicked
     23        -- XXX using chipProcessedImfile assumes that all imfile's have
     24        -- the same magicked value if that isn't right then more than one
     25        -- row will be returned. In practice this is the case but bugs could
     26        -- cause it to not be true.
     27        -- We could use rawExp.magicked but that would make it possible for
     28        -- the chipRun to have a different magicked value than the imfiles
     29        chipProcessedImfile.magicked AS imfile_magicked
    2430    FROM chipRun
    2531    JOIN rawImfile

  • trunk/ippTools/share/difftool_change_run_state.sql

    r25800 r26567  
    55UPDATE diffRun
    66    SET state = '%s'
     7    -- set magicked hook %s
    78    WHERE
    89    diffRun.diff_id = %lld

  • trunk/ippTools/share/difftool_change_skyfile_data_state.sql

    r25800 r26567  
    55    SET
    66    data_state = '%s'
     7    -- magicked hook %s
    78WHERE
    89    diff_id = %lld

  • trunk/ippTools/share/difftool_todiffskyfile.sql

    r24174 r26567  
    6565    OR (diffRun.state = 'update'
    6666    AND diffSkyfile.fault = 0
    67     AND diffSkyfile.quality = 0)
     67    AND diffSkyfile.data_state = 'cleaned')
    6868    )
    6969-- Ensure input warps are available

  • trunk/ippTools/share/disttool_pending_camera.sql

    r25838 r26567  
    3030    AND distRun.stage = 'camera'
    3131    AND distComponent.dist_id IS NULL
    32     AND (distRun.clean OR (chipRun.magicked AND camRun.magicked) OR distRun.no_magic)
     32    AND (((chipRun.magicked > 0) AND (camRun.magicked > 0)) OR distRun.no_magic)
    3333    AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned'))

  • trunk/ippTools/share/disttool_pending_chip.sql

    r25838 r26567  
    2828    AND distRun.stage = 'chip'
    2929    AND distComponent.dist_id IS NULL
    30     AND (distRun.clean OR chipRun.magicked OR distRun.no_magic)
     30    AND ((chipRun.magicked > 0) OR distRun.no_magic)
    3131    AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned'))

  • trunk/ippTools/share/disttool_pending_diff.sql

    r25838 r26567  
    3636    AND distRun.stage = 'diff'
    3737    AND distComponent.dist_id IS NULL
    38     AND (distRun.clean OR diffRun.magicked OR distRun.no_magic)
     38    AND ((diffRun.magicked > 0) OR distRun.no_magic)
    3939    AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned'))

  • trunk/ippTools/share/disttool_pending_warp.sql

    r25838 r26567  
    3131    AND distRun.stage = 'warp'
    3232    AND distComponent.dist_id IS NULL
    33     AND (distRun.clean OR warpRun.magicked OR distRun.no_magic)
     33    AND ((warpRun.magicked > 0) OR distRun.no_magic)
    3434    AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned'))

  • trunk/ippTools/share/warptool_change_run_state.sql

    r19521 r26567  
    44-- the chipProcessedImfile sub query
    55UPDATE warpRun
    6     SET state = '%s'
     6JOIN fakeRun USING(fake_id)
     7JOIN camRun USING(cam_id)
     8JOIN chipRun USING(chip_id)
     9    SET warpRun.state = '%s'
     10    -- set magicked  hook %s
    711    WHERE
    812    warpRun.warp_id = %lld
    … …  
    1216        WHERE
    1317            warpSkyfile.warp_id = warpRun.warp_id
    14             AND data_state != '%s'
     18            AND warpSkyfile.data_state != '%s'
    1519        ) = 0

  • trunk/ippTools/share/warptool_change_skyfile_data_state.sql

    r23418 r26567  
    11-- handle changes in warpSkyfile.data_state.
    2 -- Used for the modes tocleanedskyfile and topurgedskyfile
    3 -- args are new data_state, warp_id, skycell_id and current expected state for warpRun
     2-- Used for the modes tofullskyfile, tocleanedskyfile and topurgedskyfile
     3-- args are new data_state, string for magic hook, warp_id, skycell_id and current expected state for warpRun
    44UPDATE warpSkyfile
     5JOIN warpRun USING(warp_id)
     6JOIN fakeRun USING(fake_id)
     7JOIN camRun USING(cam_id)
     8JOIN chipRun USING(chip_id)
    59    SET
    610    data_state = '%s'
     11    -- set magicked hook %s
    712WHERE
    813    warp_id = %lld

  • trunk/ippTools/share/warptool_finish_run.sql

    r23430 r26567  
    11UPDATE warpRun
    22    SET state = 'full',
    3     magicked = %d
     3    magicked = %lld
    44WHERE warp_id = %lld
    55

  • trunk/ippTools/share/warptool_finished_run_select.sql

    r24043 r26567  
    88       warpSkyCellMap.warp_id as foo,
    99       warpSkyfile.warp_id as bar,
    10        SUM(!warpSkyfile.magicked) = 0 as magicked
     10       warpSkyfile.magicked as magicked
    1111   FROM warpRun
    1212   JOIN warpSkyCellMap

  • trunk/ippTools/share/warptool_towarped.sql

    r24487 r26567  
    4040        AND warpSkyfile.tess_id IS NULL)
    4141    OR (warpRun.state = 'update'
     42        AND chipRun.magicked >= 0
     43        AND warpSkyfile.fault = 0
    4244        AND warpSkyfile.data_state = 'cleaned')
    4345    )

  • trunk/ippTools/src/camtool.c

    r26336 r26567  
    222222                    tess_id     ? tess_id   : row->tess_id,
    223223                    end_stage   ? end_stage : row->end_stage,
     224                    row->magicked,
    224225                    note
    225226        )) {
    … …  
    480481    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
    481482
    482     PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
     483//    Get this from the chipRun
     484//    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
    483485
    484486    // generate restrictions
    … …  
    523525        return false;
    524526    }
     527
     528    bool status;
     529    psS64 chip_magicked = psMetadataLookupS64(&status, output->data[0], "chip_magicked");
    525530
    526531    camRunRow *pendingRow = camRunObjectFromMetadata(output->data[0]);
    … …  
    615620
    616621    // since there is only one exp per 'new' set camRun.state = 'full'
    617     if (!pxcamRunSetState(config, row->cam_id, "full", magicked)) {
     622    // propagate magicked state from chipRun
     623    if (!pxcamRunSetState(config, row->cam_id, "full", chip_magicked)) {
    618624        psError(PS_ERR_UNKNOWN, false, "failed to change camRun.state for cam_id: %" PRId64, row->cam_id);
    619625        psFree(row);

  • trunk/ippTools/src/chiptool.c

    r26465 r26567  
    720720    PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
    721721    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL);
     722    if (psListLength(where->list) == 0) {
     723        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
     724        return false;
     725    }
    722726       
    723727    if (!state) {
    … …  
    12541258                    chipRun->tess_id,
    12551259                    chipRun->end_stage,
     1260                    chipRun->magicked,
    12561261                    NULL    // note does not propagate
    12571262        )) {
    … …  
    12951300    }
    12961301
    1297     psString set_magic = "";
     1302    char *set_magicked_imfile = "";
     1303    char *set_magicked_run = "";
    12981304    if (!strcmp(data_state, "full")) {
    1299         // copy the magicked state from the input to the output when transitioning to full state
    1300         set_magic = "\n , chipProcessedImfile.magicked = rawImfile.magicked";
    1301     }
    1302 
    1303     // note only updates if chipRun.state = run_state
    1304     // XXX note that we have removed this constraint for now
    1305     if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magic, chip_id, class_id)) {
     1305        // if (chipProcessedImfile.magicked < 0 and rawImfile.magicked = 0) leave magicked unchanged. This will
     1306        // block warp processing until destreaking has been done
     1307        // otherwise copy magicked from the rawImfile
     1308        // Same thing for chipRun/rawExp
     1309        set_magicked_imfile = "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0 AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)";
     1310        set_magicked_run = "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), chipRun.magicked, rawExp.magicked)";
     1311    } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
     1312        // if magicked is non-zero set it to -1
     1313        set_magicked_imfile = "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1)";
     1314        set_magicked_run = "\n, chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)";
     1315    }
     1316
     1317    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_imfile, chip_id, class_id)) {
    13061318        psFree(query);
    13071319        psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    13201332
    13211333    query = pxDataGet("chiptool_change_exp_state.sql");
    1322     if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, data_state)) {
     1334    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, chip_id, data_state)) {
    13231335        psFree(query);
    13241336        // rollback

  • trunk/ippTools/src/chiptoolConfig.c

    r25835 r26567  
    154154    psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
    155155    psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
    156     psMetadataAddBool(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", false);
     156    psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", 0);
    157157
    158158    // -processedimfile

  • trunk/ippTools/src/difftool.c

    r26381 r26567  
    5555static bool topurgedskyfileMode(pxConfig *config);
    5656static bool toscrubbedskyfileMode(pxConfig *config);
     57static bool tofullskyfileMode(pxConfig *config);
    5758
    5859
    … …  
    9495        MODECASE(DIFFTOOL_MODE_EXPORTRUN,             exportrunMode);
    9596        MODECASE(DIFFTOOL_MODE_IMPORTRUN,             importrunMode);
    96         MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE,   tocleanedskyfileMode);
    97         MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE,    topurgedskyfileMode);
    98         MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,  toscrubbedskyfileMode);
     97        MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE,      tocleanedskyfileMode);
     98        MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE,       topurgedskyfileMode);
     99        MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,     toscrubbedskyfileMode);
     100        MODECASE(DIFFTOOL_MODE_TOFULLSKYFILE,         tofullskyfileMode);
    99101
    100102        default:
    … …  
    20362038  PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
    20372039
     2040
     2041  psS64 magicked = 0;
     2042  if (!strcmp(data_state, "full")) {
     2043      PXOPT_LOOKUP_S64(set_magicked, config->args, "-magicked", 0, false);
     2044      magicked = set_magicked;
     2045  }
     2046
    20382047  psString query = pxDataGet("difftool_change_skyfile_data_state.sql");
    20392048
    … …  
    20432052  }
    20442053
     2054  psString set_magicked_skyfile = psStringCopy("");
     2055  psString set_magicked_run = psStringCopy("");
     2056  if (magicked != 0 && !strcmp(data_state, "full")) {
     2057    psStringAppend(&set_magicked_skyfile,  "\n , diffSkyfile.magicked = %" PRId64, magicked);
     2058    psStringAppend(&set_magicked_run,      "\n , diffRun.magicked = %" PRId64, magicked);
     2059
     2060  } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
     2061    psStringAppend(&set_magicked_skyfile, "\n, diffSkyfile.magicked = IF(diffSkyfile.magicked = 0, 0, -1)");
     2062    psStringAppend(&set_magicked_run, "\n, diffRun.magicked = IF(diffRun.magicked = 0, 0, -1)");
     2063  }
     2064
    20452065  // Uses the unconstrained (diffRun.state [NEED NOT EQUAL] run_state) version from warptool.c
    20462066
    2047   if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, skycell_id)) {
     2067  if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, diff_id, skycell_id)) {
    20482068    psError(PS_ERR_UNKNOWN, false, "database error");
    20492069    // rollback
    … …  
    20522072    }
    20532073    psError(PS_ERR_UNKNOWN, false, "database error");
     2074    psFree(set_magicked_skyfile);
    20542075    return(false);
    20552076  }
     2077  psFree(set_magicked_skyfile);
    20562078  psFree(query);
    20572079
    20582080  query = pxDataGet("difftool_change_run_state.sql");
    2059   if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, data_state)) {
     2081  if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, diff_id, data_state)) {
    20602082    // rollback
    20612083    if (!psDBRollback(config->dbh)) {
    … …  
    20632085    }
    20642086    psError(PS_ERR_UNKNOWN, false, "database error");
     2087    psFree(set_magicked_run);
    20652088    return(false);
    20662089  }
    … …  
    20682091  if (!psDBCommit(config->dbh)) {
    20692092    psError(PS_ERR_UNKNOWN, false, "database error");
     2093    psFree(set_magicked_run);
    20702094    return(false);
    20712095  }
     2096  psFree(set_magicked_run);
    20722097
    20732098  return(true);
    … …  
    20822107static bool toscrubbedskyfileMode(pxConfig *config) {
    20832108  return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed");
     2109}
     2110static bool tofullskyfileMode(pxConfig *config) {
     2111  return change_skyfile_data_state(config, "full", "update");
    20842112}
    20852113

  • trunk/ippTools/src/difftool.h

    r25800 r26567  
    4545    DIFFTOOL_MODE_TOCLEANEDSKYFILE,
    4646    DIFFTOOL_MODE_TOPURGEDSKYFILE,
    47     DIFFTOOL_MODE_TOSCRUBBEDSKYFILE
     47    DIFFTOOL_MODE_TOSCRUBBEDSKYFILE,
     48    DIFFTOOL_MODE_TOFULLSKYFILE
    4849   
    4950} difftoolMode;

  • trunk/ippTools/src/difftoolConfig.c

    r26388 r26567  
    122122    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0,   "define hostname", 0);
    123123    psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac",  0, "define %% of good pixels", NAN);
    124     psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", false);
     124    psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", 0);
    125125
    126126    // -advance
    … …  
    289289    psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
    290290
     291    // -tofullskyfile
     292    psMetadata *tofullskyfileArgs = psMetadataAlloc();
     293    psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0);
     294    psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
     295    psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", 0);
     296
    291297    // -exportrun
    292298    psMetadata *exportrunArgs = psMetadataAlloc();
    … …  
    329335    PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs);
    330336    PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs);
     337    PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full", DIFFTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs);
    331338
    332339    if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

  • trunk/ippTools/src/flatcorr.c

    r25913 r26567  
    535535                row->tess_id,
    536536                "camera",
     537                row->magicked,
    537538                NULL // note does not propragate
    538539                )) {

  • trunk/ippTools/src/magicdstool.c

    r26495 r26567  
    4444static bool tocleanupMode(pxConfig *config);
    4545
    46 static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psMetadata *where, const char *state);
     46static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state);
    4747
    4848# define MODECASE(caseName, func) \
    … …  
    326326    PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false);
    327327
     328    // optional
     329    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
     330    psString setString = NULL;
     331    if (set_label) {
     332        psStringAppend(&setString, ", label = '%s'", set_label);
     333    }
     334
    328335    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false);
    329336    if (magic_ds_id) {
    330337
    331         return setmagicDSRunState(config, magic_ds_id, NULL, state);
     338        return setmagicDSRunState(config, magic_ds_id, setString, NULL, state);
    332339
    333340    } else if (!strcmp(state, "full")) {
    … …  
    336343    }
    337344    // we can transition by query as well
     345
    338346
    339347    psMetadata *where = psMetadataAlloc();
    … …  
    342350    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
    343351
    344 
    345352    if (psListLength(where->list) < 2) {
    346353        psError(PS_ERR_UNKNOWN, true, "at least 2 search arguments are required");
    … …  
    353360        psMetadataAddS32(where, PS_LIST_TAIL, "re_place", 0, ">", 0);
    354361    }
    355     bool result = setmagicDSRunState(config, magic_ds_id, where, state);
     362    bool result = setmagicDSRunState(config, magic_ds_id, setString, where, state);
    356363    psFree(where);
    357364
    … …  
    445452{
    446453    // first query the magicDSRun to find the stage and the stage_id
    447     psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64;
     454    psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64;
    448455
    449456    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
    … …  
    472479    psString stage= psMetadataLookupStr(NULL, row, "stage");
    473480    psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
     481    psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id");
    474482
    475483    ippStage stageNum = ippStringToStage(stage);
    … …  
    499507    }
    500508
    501     if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) {
     509    if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id, component)) {
    502510        psError(PS_ERR_UNKNOWN, false, "database error");
    503511        return false;
    … …  
    518526{
    519527    // first query the magicDSRun to find the stage and the stage_id
    520     psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64;
     528    psString query = "SELECT stage, stage_id, magic_id from magicDSRun where magic_ds_id = %" PRId64;
    521529
    522530    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
    … …  
    545553    psString stage = psMetadataLookupStr(NULL, row, "stage");
    546554    psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
     555    psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id");
    547556
    548557    ippStage stageNum = ippStringToStage(stage);
    … …  
    570579        return false;
    571580    }
    572     if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id)) {
     581    if (!p_psDBRunQueryF(config->dbh, query, magic_id, stage_id)) {
    573582        psError(PS_ERR_UNKNOWN, false, "database error");
    574583        return false;
    … …  
    710719
    711720        // set magicDSRun.state to 'full'
    712         if (!setmagicDSRunState(config, magic_ds_id, NULL, "full")) {
     721        if (!setmagicDSRunState(config, magic_ds_id, NULL, NULL, "full")) {
    713722            psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64,
    714723                magic_ds_id);
    … …  
    927936}
    928937
    929 static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psMetadata *where, const char *state)
     938static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psString extraSetStr, psMetadata *where, const char *state)
    930939{
    931940    PS_ASSERT_PTR_NON_NULL(state, false);
    … …  
    937946    }
    938947
    939     psString query = psStringCopy("UPDATE magicDSRun SET state = '%s'");
     948    psString query = NULL;
     949    psStringAppend(&query, "UPDATE magicDSRun SET state = '%s' %s\n", state, extraSetStr ? extraSetStr : "");
    940950    if (magic_ds_id) {
    941951        psStringAppend(&query, " WHERE magic_ds_id = %" PRId64, magic_ds_id);
    … …  
    949959    }
    950960
    951     if (!p_psDBRunQueryF(config->dbh, query, state)) {
     961    if (!p_psDBRunQuery(config->dbh, query)) {
    952962        psError(PS_ERR_UNKNOWN, false,
    953963                "failed to change state for magic_id %" PRId64, magic_ds_id);

  • trunk/ippTools/src/magicdstoolConfig.c

    r26449 r26567  
    7474    psMetadata *updaterunArgs = psMetadataAlloc();
    7575    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL);
     76    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL);
    7677    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID", 0);
    7778    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage",     0, "define stage", NULL);

  • trunk/ippTools/src/pxcam.c

    r25835 r26567  
    152152                    char *tess_id,
    153153                    char *end_stage,
     154                    psS64 magicked,
    154155                    char *note)
    155156{
    … …  
    168169
    169170    // queue the exp
    170     // XXX chip_id is being cast here work around psS64 have a different type
     171    // XXX chip_id and magicked are being cast here work around psS64 have a different type
    171172    // different on 32/64
    172173    if (!p_psDBRunQueryF(config->dbh, query,
    … …  
    182183                tess_id  ? tess_id  : "NULL",
    183184                end_stage ? end_stage : "NULL",
     185                (long long) magicked,
    184186                note     ? note     : "NULL",
    185187                (long long)chip_id

  • trunk/ippTools/src/pxcam.h

    r25835 r26567  
    4040                        char *tess_id,
    4141                        char *end_stage,
     42                        psS64 magicked,
    4243                        char *note);
    4344

  • trunk/ippTools/src/pxchip.c

    r25835 r26567  
    129129}
    130130
    131 bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked)
     131bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked)
    132132{
    133133    PS_ASSERT_PTR_NON_NULL(config, false);
    … …  
    140140    }
    141141
    142     char *query = "UPDATE chipRun SET state = '%s', magicked = %d WHERE chip_id = %" PRId64;
     142    char *query = "UPDATE chipRun SET state = '%s', magicked = %" PRId64 " WHERE chip_id = %" PRId64;
    143143    if (!p_psDBRunQueryF(config->dbh, query, state, magicked, chip_id)) {
    144144        psError(PS_ERR_UNKNOWN, false,
    … …  
    174174
    175175    psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET state = '%s'");
     176
     177    if (!strcmp(state, "cleaned") || !strcmp(state, "purged")) {
     178        // if magicked is non-zero set it to -1
     179        psStringAppend(&query, ", chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)");
     180    }
    176181
    177182    if (where && psListLength(where->list) > 0) {
    … …  
    219224        psStringAppend(&query, " %s", whereClause);
    220225        psFree(whereClause);
     226    } else {
     227        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
     228        return false;
    221229    }
    222230

  • trunk/ippTools/src/pxchip.h

    r25835 r26567  
    2727
    2828
    29 bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked);
     29bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const psS64 magicked);
    3030bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state);
    3131

  • trunk/ippTools/src/warptool.c

    r25935 r26567  
    16461646    }
    16471647
    1648     // note only updates if warpRun.state = run_state
    1649     // XXX note that we have removed this constraint for now
    1650     if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id)) {
     1648    char *set_magicked_skyfile = "";
     1649    char *set_magicked_run = "";
     1650    if (!strcmp(data_state, "full")) {
     1651        set_magicked_skyfile = "\n , warpSkyfile.magicked = IF((warpSkyfile.magicked < 0 AND chipRun.magicked = 0), warpSkyfile.magicked, chipRun.magicked)";
     1652        set_magicked_run = "\n,  warpRun.magicked = IF((warpRun.magicked < 0 AND chipRun.magicked = 0), warpRun.magicked, chipRun.magicked)";
     1653
     1654    } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
     1655        set_magicked_skyfile = "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1)";
     1656        set_magicked_run = "\n, warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)";
     1657    }
     1658
     1659    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, warp_id, skycell_id)) {
    16511660        psError(PS_ERR_UNKNOWN, false, "database error");
    16521661        // rollback
    … …  
    16601669
    16611670    query = pxDataGet("warptool_change_run_state.sql");
    1662     if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, data_state)) {
     1671    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, warp_id, data_state)) {
    16631672        // rollback
    16641673        if (!psDBRollback(config->dbh)) {

  • trunk/ippTools/src/warptoolConfig.c

    r25835 r26567  
    212212    psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0,            "define hostname", 0);
    213213    psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac",  0,            "define %% of good pixels", NAN);
    214     psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-magicked",  0, "define if this skycell has been magicked", false);
     214    psMetadataAddS64(addwarpedArgs, PS_LIST_TAIL, "-magicked",  0, "define if this skycell has been magicked", 0);
    215215    psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
    216216    psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
    … …  
    310310    psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
    311311
    312     // -toupdateskyfile
     312    // -updateskyfile
    313313    psMetadata *updateskyfileArgs = psMetadataAlloc();
    314314    psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,    "warptool ID to update", 0);
Note: See TracChangeset for help on using the changeset viewer.