IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 25419


Ignore:
Timestamp:
Sep 16, 2009, 9:47:16 AM (17 years ago)
Author:
bills
Message:

Change magicdstool -todestreak to query the individual stages separately.
This greatly decreases the query time.
Add support for magicDSRun.state = 'goto_restore' and 'goto_censored' which
will restore destreaked files back to their original state.
Split up ippTasks/magic.pro into magic.pro and destreak.pro so that processing
can be easily magnaged separately

Location:
trunk
Files:
7 added
1 deleted
14 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/Makefile.am

    r25324 r25419  
    1717        warp.pro \
    1818        magic.pro \
     19        destreak.pro \
    1920        diff.pro \
    2021        stack.pro \

  • trunk/ippTasks/magic.pro

    r25021 r25419  
    1010book init magicToTree
    1111book init magicToProcess
    12 book init magicToDS
    13 book init magicDSToRevert
    1412
    1513### Database lists
    1614$magicToTree_DB = 0
    1715$magicToProcess_DB = 0
    18 $magicToDS_DB = 0
    19 $magicDSToRevert_DB = 0
    20 
    21 #list of stages
    22 $STAGE:n = 0
    23 list STAGE -add "raw"
    24 list STAGE -add "chip"
    25 list STAGE -add "camera"
    26 list STAGE -add "warp"
    27 list STAGE -add "diff"
    28 $magicDSRevertStage = 0
    2916
    3017### Check status of tasks
    … …  
    3219  book listbook magicToTree
    3320  book listbook magicToProcess
    34   book listbook magicToDS
    35   book listbook magicDSToRevert
    3621end
    3722
    … …  
    4025  book init magicToTree
    4126  book init magicToProcess
    42   book init magicToDS
    43   book init magicDSToRevert
    4427end
    4528
    … …  
    5841    active true
    5942  end
    60   task magic.destreak.load
    61     active true
    62   end
    63   task magic.destreak.run
    64     active true
    65   end
    66 end
    67 
    68 macro magic.ds.revert.on
    69     task magic.ds.revert.load
    70         active true
    71     end
    72     task magic.ds.revert.run
    73         active true
    74     end
    75 end
     43end
     44
    7645### Turn tasks off
    7746macro magic.off
    … …  
    8857    active false
    8958  end
    90   task magic.destreak.load
    91     active false
    92   end
    93   task magic.destreak.run
    94     active false
    95   end
    96 end
    97 
    98 macro magic.ds.revert.off
    99     task magic.ds.revert.load
    100         active false
    101     end
    102     task magic.ds.revert.run
    103         active false
    104     end
    105 end
    106 macro magic.ds.off
    107     task magic.destreak.load
    108         active false
    109     end
    110     task magic.destreak.run
    111         active false
    112     end
    113 end
    114 macro magic.ds.on
    115     task magic.destreak.load
    116         active true
    117     end
    118     task magic.destreak.run
    119         active true
    120     end
    121 end
    122 
    123 macro magic.ds.status
    124     echo magicToDS
    125     book listbook magicToDS
    126     echo ""
    127     echo magicDSToRevert
    128     book listbook magicDSToRevert
    129 end
    130 
    131 
     59end
    13260
    13361task           magic.tree.load
    … …  
    400328  end
    401329end
    402 
    403 task           magic.destreak.load
    404   host         local
    405 
    406   periods      -poll $LOADPOLL
    407   # this query can take a long time
    408   periods      -exec 10
    409   periods      -timeout 120
    410   npending     1
    411 
    412   stdout NULL
    413   stderr $LOGDIR/magic.destreak.log
    414 
    415   task.exec
    416     $run = magicdstool -todestreak -limit 120
    417     if ($DB:n == 0)
    418       option DEFAULT
    419     else
    420       # save the DB name for the exit tasks
    421       option $DB:$magicToDS_DB
    422       $run = $run -dbname $DB:$magicToDS_DB
    423       $magicToDS_DB ++
    424       if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0
    425     end
    426     add_poll_args run
    427     add_poll_labels run
    428     command $run
    429   end
    430 
    431   # success
    432   task.exit    0
    433     # convert 'stdout' to book format
    434     ipptool2book stdout magicToDS -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT
    435     if ($VERBOSE > 2)
    436       book listbook magicToDS
    437     end
    438 
    439     # delete existing entries in the appropriate pantaskStates
    440     process_cleanup magicToDS
    441   end
    442 
    443   # locked list
    444   task.exit    default
    445     showcommand failure
    446   end
    447 
    448   task.exit    crash
    449     showcommand crash
    450   end
    451 
    452   # operation times out?
    453   task.exit    timeout
    454     showcommand timeout
    455   end
    456 end
    457 
    458 task           magic.destreak.run
    459   periods      -poll $RUNPOLL
    460   periods      -exec $RUNEXEC
    461   periods      -timeout 60
    462 
    463   task.exec
    464     book npages magicToDS -var N
    465     if ($N == 0) break
    466     if ($NETWORK == 0) break
    467    
    468     # look for new images (pantaskState == INIT)
    469     book getpage magicToDS 0 -var pageName -key pantaskState INIT
    470     if ("$pageName" == "NULL") break
    471 
    472     book setword magicToDS $pageName pantaskState RUN
    473     book getword magicToDS $pageName exp_id -var EXP_ID
    474     book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID
    475     book getword magicToDS $pageName camera -var CAMERA
    476     book getword magicToDS $pageName streaks_uri -var STREAKS
    477     book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS
    478     book getword magicToDS $pageName stage -var STAGE
    479     book getword magicToDS $pageName stage_id -var STAGE_ID
    480     book getword magicToDS $pageName component -var COMPONENT
    481     book getword magicToDS $pageName uri -var URI
    482     book getword magicToDS $pageName path_base -var PATH_BASE
    483     book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE
    484     book getword magicToDS $pageName outroot -var OUTROOT
    485     book getword magicToDS $pageName recoveryroot -var RECROOT
    486     book getword magicToDS $pageName re_place -var REPLACE
    487     book getword magicToDS $pageName dbname -var DBNAME
    488 
    489     sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT
    490 
    491     substr $COMPONENT 0 3 COMP_HEAD
    492     if ("$COMP_HEAD" == "sky")
    493         set.host.for.skycell $COMPONENT
    494     else
    495         # assume component is a class_id, if not we will default to anyhost
    496         set.host.for.camera $CAMERA $COMPONENT
    497     end
    498 
    499     # TODO: do not add recoveryroot or replace if they are null or zero
    500 
    501     $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE
    502 
    503     add_standard_args run
    504 
    505     # save the pageName for future reference below
    506     options $pageName
    507 
    508     # create the command line
    509     if ($VERBOSE > 1)
    510       echo command $run
    511     end
    512     command $run
    513   end
    514 
    515   # default exit status
    516   task.exit    0
    517     process_exit magicToDS $options:0 $JOB_STATUS
    518    end
    519 
    520   # locked list
    521   task.exit    default
    522     showcommand failure
    523     process_exit magicToDS $options:0 $JOB_STATUS
    524   end
    525 
    526   task.exit    crash
    527     showcommand crash
    528     book setword magicToDS $options:0 pantaskState CRASH
    529   end
    530 
    531   # operation timed out?
    532   task.exit    timeout
    533     showcommand timeout
    534     book setword magicToDS $options:0 pantaskState TIMEOUT
    535   end
    536 end
    537 
    538 task           magic.ds.revert.load
    539   host         local
    540 
    541   periods      -poll $LOADPOLL
    542   periods      -exec $LOADEXEC
    543   periods      -timeout 20
    544   npending     1
    545   active       false
    546 
    547   stdout NULL
    548   stderr $LOGDIR/magic.ds.revert.log
    549 
    550   task.exec
    551     $run = magicdstool -torevert -stage $STAGE:$magicDSRevertStage
    552     $magicDSRevertStage ++
    553     if ($magicDSRevertStage >= $STAGE:n) set magicDSRevertStage = 0
    554 
    555     if ($DB:n == 0)
    556       option DEFAULT
    557     else
    558 
    559       # save the DB name for the exit tasks
    560       option $DB:$magicDSToRevert_DB
    561       $run = $run -dbname $DB:$magicDSToRevert_DB
    562 
    563       # only bump database number after we have gone through all of the stages
    564       if ($magicDSRevertStage == 0)
    565              $magicDSToRevert_DB ++
    566       end
    567       if ($magicDSToRevert_DB >= $DB:n) set magicDSToRevert_DB = 0
    568     end
    569     add_poll_args run
    570     add_poll_labels run
    571     command $run
    572   end
    573 
    574   # success
    575   task.exit    0
    576     # convert 'stdout' to book format
    577     ipptool2book stdout magicDSToRevert -key magic_ds_id:component -uniq -setword dbname $options:0 -setword pantaskState INIT
    578     if ($VERBOSE > 2)
    579       book listbook magicDSToRevert
    580     end
    581 
    582     # delete existing entries in the appropriate pantaskStates
    583     process_cleanup magicDSToRevert
    584   end
    585 
    586   # locked list
    587   task.exit    default
    588     showcommand failure
    589   end
    590 
    591   task.exit    crash
    592     showcommand crash
    593   end
    594 
    595   # operation times out?
    596   task.exit    timeout
    597     showcommand timeout
    598   end
    599 end
    600 
    601 task           magic.ds.revert.run
    602   periods      -poll $RUNPOLL
    603   periods      -exec $RUNEXEC
    604   periods      -timeout 60
    605   active       false
    606 
    607   task.exec
    608     book npages magicDSToRevert -var N
    609     if ($N == 0) break
    610     if ($NETWORK == 0) break
    611    
    612     # look for new images (pantaskState == INIT)
    613     book getpage magicDSToRevert 0 -var pageName -key pantaskState INIT
    614     if ("$pageName" == "NULL") break
    615 
    616     book setword magicDSToRevert $pageName pantaskState RUN
    617     book getword magicDSToRevert $pageName exp_id -var EXP_ID
    618     book getword magicDSToRevert $pageName magic_ds_id -var MAGIC_DS_ID
    619     book getword magicDSToRevert $pageName camera -var CAMERA
    620     book getword magicDSToRevert $pageName stage -var STAGE
    621     book getword magicDSToRevert $pageName stage_id -var STAGE_ID
    622     book getword magicDSToRevert $pageName component -var COMPONENT
    623     book getword magicDSToRevert $pageName path_base -var PATH_BASE
    624     book getword magicDSToRevert $pageName cam_path_base -var CAM_PATH_BASE
    625     book getword magicDSToRevert $pageName outroot -var OUTROOT
    626     book getword magciDSToRevert $pageName bytes -var BYTES
    627     book getword magciDSToRevert $pageName md5sum -var md5sum
    628 #    book getword magicDSToRevert $pageName recoveryroot -var RECROOT
    629     book getword magicDSToRevert $pageName re_place -var REPLACE
    630     book getword magicDSToRevert $pageName dbname -var DBNAME
    631 
    632     sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT
    633 
    634     substr $COMPONENT 0 3 COMP_HEAD
    635     if ("$COMP_HEAD" == "sky")
    636         set.host.for.skycell $COMPONENT
    637     else
    638         # assume component is a class_id, if not we will default to anyhost
    639         set.host.for.camera $CAMERA $COMPONENT
    640     end
    641 
    642     $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --replace $REPLACE
    643 
    644     add_standard_args run
    645 
    646     # save the pageName for future reference below
    647     options $pageName
    648 
    649     # create the command line
    650     if ($VERBOSE > 1)
    651       echo command $run
    652     end
    653     command $run
    654   end
    655 
    656   # default exit status
    657   task.exit    0
    658     process_exit magicDSToRevert $options:0 $JOB_STATUS
    659    end
    660 
    661   # locked list
    662   task.exit    default
    663     showcommand failure
    664     process_exit magicDSToRevert $options:0 $JOB_STATUS
    665   end
    666 
    667   task.exit    crash
    668     showcommand crash
    669     book setword magicDSToRevert $options:0 pantaskState CRASH
    670   end
    671 
    672   # operation timed out?
    673   task.exit    timeout
    674     showcommand timeout
    675     book setword magicDSToRevert $options:0 pantaskState TIMEOUT
    676   end
    677 end
    678 

  • trunk/ippTools/share/Makefile.am

    r25324 r25419  
    165165     magictool_revertnode.sql \
    166166     magicdstool_completed_runs.sql \
     167     magicdstool_completedrevert.sql \
    167168     magicdstool_definebyquery_raw.sql \
    168169     magicdstool_definebyquery_chip.sql \
    … …  
    172173     magicdstool_getrunids.sql \
    173174     magicdstool_getskycells.sql \
    174      magicdstool_todestreak.sql \
     175     magicdstool_todestreak_camera.sql \
     176     magicdstool_todestreak_chip.sql \
     177     magicdstool_todestreak_diff.sql \
     178     magicdstool_todestreak_raw.sql \
     179     magicdstool_todestreak_warp.sql \
    175180     magicdstool_toremove.sql \
    176181     magicdstool_torestore.sql \

  • trunk/ippTools/share/magicdstool_torevert_camera.sql

    r24883 r25419  
    1919    JOIN rawExp using(exp_id)
    2020WHERE magicDSRun.stage = 'camera'
    21     AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored')
    22     AND magicDSFile.fault > 0
     21    AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0)
     22       OR magicDSRun.state = 'goto_censored'
     23       OR magicDSRun.state = 'goto_restored')

  • trunk/ippTools/share/magicdstool_torevert_chip.sql

    r24883 r25419  
    1818    JOIN rawExp using(exp_id)
    1919WHERE magicDSRun.stage = 'chip'
    20     AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored')
    21     AND magicDSFile.fault > 0
     20    AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0)
     21       OR magicDSRun.state = 'goto_censored'
     22       OR magicDSRun.state = 'goto_restored')

  • trunk/ippTools/share/magicdstool_torevert_diff.sql

    r24883 r25419  
    1818    JOIN rawExp USING(exp_id)
    1919WHERE magicDSRun.stage = 'diff'
    20     AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored')
    21     AND magicDSFile.fault > 0
     20    AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0)
     21       OR magicDSRun.state = 'goto_censored'
     22       OR magicDSRun.state = 'goto_restored')

  • trunk/ippTools/share/magicdstool_torevert_raw.sql

    r24883 r25419  
    1818                      AND rawImfile.tmp_class_id = summitImfile.class_id)
    1919WHERE magicDSRun.stage = 'raw'
    20     AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored')
    21     AND magicDSFile.fault > 0
     20    AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0)
     21       OR magicDSRun.state = 'goto_censored'
     22       OR magicDSRun.state = 'goto_restored')

  • trunk/ippTools/share/magicdstool_torevert_warp.sql

    r24883 r25419  
    2121    JOIN rawExp USING(exp_id)
    2222WHERE magicDSRun.stage = 'warp'
    23     AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored')
    24     AND magicDSFile.fault > 0
     23    AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0)
     24       OR magicDSRun.state = 'goto_censored'
     25       OR magicDSRun.state = 'goto_restored')

  • trunk/ippTools/share/magictool_censor_chip.sql

    r24883 r25419  
    11UPDATE chipRun
    22    JOIN chipProcessedImfile USING(chip_id, exp_id)
    3     JOIN magicDSRun ON chipRun.magicked = magicDSRun.magic_ds_id
     3    JOIN magicDSRun ON stage_id = chip_id AND stage = 'chip' AND re_place
    44    JOIN magicDSFile USING(magic_ds_id)
    5     JOIN magicRun USING(magic_id, exp_id)
     5    JOIN magicRun using(magic_id)
    66SET chipProcessedImfile.magicked = 0,
    77    chipRun.magicked = 0,

  • trunk/ippTools/share/magictool_censor_diff.sql

    r24883 r25419  
    1 UPDATE magicRun
    2     JOIN magicDSRun USING(magic_id)
     1UPDATE diffRun
     2    JOIN diffSkyfile USING(diff_id)
     3    JOIN magicRun USING(diff_id)
     4    JOIN magicDSRun ON stage = 'diff' AND stage_id = diff_id AND re_place
    35    JOIN magicDSFile USING(magic_ds_id)
    4     JOIN diffRun ON magic_ds_id = magicked
    5     JOIN diffSkyfile ON diffRun.diff_id = diffSkyfile.diff_id
    66SET diffRun.magicked = 0,
    77    diffSkyfile.magicked = 0,

  • trunk/ippTools/share/magictool_censor_raw.sql

    r24885 r25419  
    22    JOIN magicDSRun USING(magic_id)
    33    JOIN magicDSFile USING(magic_ds_id)
    4     JOIN rawExp ON rawExp.magicked = magic_ds_id
    5     JOIN rawImfile ON rawExp.exp_id = rawImfile.exp_id
     4    JOIN rawImfile ON rawImfile.magicked = magic_ds_id
     5--    JOIN rawExp ON rawExp.magicked = magic_ds_id
    66SET rawImfile.magicked = 0,
    7     rawExp.magicked = 0,
    8     magicDSRun.state = 'censored',
     7--    rawExp.magicked = 0,
     8--    magicDSRun.state = 'censored',
    99    magicDSFile.fault = 42
    1010

  • trunk/ippTools/src/magicdstool.c

    r25015 r25419  
    4242static bool torestoreMode(pxConfig *config);
    4343static bool torevertMode(pxConfig *config);
     44static bool completedrevertMode(pxConfig *config);
    4445
    4546static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state);
    … …  
    7576        MODECASE(MAGICDSTOOL_MODE_TORESTORE,           torestoreMode);
    7677        MODECASE(MAGICDSTOOL_MODE_TOREVERT,            torevertMode);
     78        MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT,     completedrevertMode);
    7779        default:
    7880            psAbort("invalid option (this should not happen)");
    … …  
    396398    PS_ASSERT_PTR_NON_NULL(config, false);
    397399
     400    PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
     401
    398402    psMetadata *where = psMetadataAlloc();
    399403    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
    400404    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
    401     pxAddLabelSearchArgs (config, where, "-label", "label", "==");
    402     PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
     405    pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "==");
    403406
    404407    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    405408    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    406409
    407     // look for "inputs" that need to processed
    408     psString query = pxDataGet("magicdstool_todestreak.sql");
     410    psString sql_file = NULL;
     411    psStringAppend(&sql_file, "magicdstool_todestreak_%s.sql", stage);
     412
     413    psString query = pxDataGet(sql_file);
    409414    if (!query) {
    410         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    411         return false;
    412     }
     415        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file);
     416        psFree(sql_file);
     417        return false;
     418    }
     419    psFree(sql_file);
    413420
    414421    if (psListLength(where->list)) {
    415422        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    416         psStringAppend(&query, " WHERE %s", whereClause);
     423        psStringAppend(&query, " AND %s", whereClause);
    417424        psFree(whereClause);
    418425    }
    … …  
    817824    return true;
    818825}
    819 
    820 static bool getskycellsMode(pxConfig *config)
    821 {
    822     // required
    823     PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
     826static bool completedrevertMode(pxConfig *config)
     827{
     828    PS_ASSERT_PTR_NON_NULL(config, false);
    824829
    825830    psMetadata *where = psMetadataAlloc();
    826     PXOPT_COPY_STR(config->args, where, "-class_id",    "warpSkyCellMap.class_id", "==");
    827     PXOPT_COPY_STR(config->args, where, "-skycell_id",  "warpSkyCellMap.skycell_id", "==");
    828 
    829     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    830 
    831     psString query = pxDataGet("magicdstool_getskycells.sql");
    832     if (!query) {
    833         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    834         return false;
    835     }
    836 
     831    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "==");
     832    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
     833
     834    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     835
     836    psString query = pxDataGet("magicdstool_completedrevert.sql");
     837    // treat limit == 0 as "no limit"
     838    if (limit) {
     839        psString limitString = psDBGenerateLimitSQL(limit);
     840        psStringAppend(&query, " %s", limitString);
     841        psFree(limitString);
     842    }
     843
     844    psString whereString = NULL;
    837845    if (psListLength(where->list)) {
    838846        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    839         psStringAppend(&query, " AND %s", whereClause);
     847        psStringAppend(&whereString, "\nAND %s", whereClause);
    840848        psFree(whereClause);
    841849    }
    842850    psFree(where);
    843851
    844     if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
    845         psError(PS_ERR_UNKNOWN, false, "database error");
    846         psFree(query);
    847         return false;
    848     }
     852    if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString : "")) {
     853        psFree(whereString);
     854        psError(PS_ERR_UNKNOWN, false, "failed to revert");
     855        return false;
     856    }
     857    psFree(whereString);
    849858    psFree(query);
    850 
    851859    psArray *output = p_psDBFetchResult(config->dbh);
    852860    if (!output) {
    … …  
    868876        return true;
    869877    }
    870 
    871     if (psArrayLength(output)) {
    872         // negative simple so the default is true
    873         if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) {
    874             psError(PS_ERR_UNKNOWN, false, "failed to print array");
     878    for (int i=0; i<psArrayLength(output); i++) {
     879        psMetadata *row = output->data[i];
     880        psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id");
     881        psString old_state = psMetadataLookupStr(NULL, row, "state");
     882        psString new_state;
     883        if (!strcmp(old_state, "goto_censored")) {
     884            new_state = "censored";
     885        } else if (!strcmp(old_state, "goto_restored")) {
     886            new_state = "restored";
     887        } else {
     888            psError(PXTOOLS_ERR_PROG, true, "unexpected state found: %s", old_state);
    875889            psFree(output);
    876890            return false;
    877891        }
    878     }
    879 
     892        char *query2 = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64;
     893        if (!p_psDBRunQueryF(config->dbh, query2, magic_ds_id, new_state)) {
     894            psError(PS_ERR_UNKNOWN, false, "failed to set run magicDSRun.state to %s", new_state);
     895            return false;
     896        }
     897    }
    880898    psFree(output);
    881899
    … …  
    883901}
    884902
    885 static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state)
    886 {
    887     PS_ASSERT_PTR_NON_NULL(state, false);
    888 
    889     // check that state is a valid string value
    890     if (!(
    891             (strncmp(state, "new", 4) == 0)
    892             || (strncmp(state, "full", 5) == 0)
    893         )
    894     ) {
    895         psError(PS_ERR_UNKNOWN, false,
    896                 "invalid magicDSRun state: %s", state);
    897         return false;
    898     }
    899 
    900     char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64;
    901     if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) {
    902         psError(PS_ERR_UNKNOWN, false,
    903                 "failed to change state for magic_id %" PRId64, magic_ds_id);
    904         return false;
    905     }
    906 
    907     return true;
    908 }
    909 
    910 static bool toremoveMode(pxConfig *config)
    911 {
    912     PS_ASSERT_PTR_NON_NULL(config, false);
     903static bool getskycellsMode(pxConfig *config)
     904{
     905    // required
     906    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
    913907
    914908    psMetadata *where = psMetadataAlloc();
    915     PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
    916     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
    917     pxAddLabelSearchArgs (config, where, "-label", "label", "==");
    918 
    919     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     909    PXOPT_COPY_STR(config->args, where, "-class_id",    "warpSkyCellMap.class_id", "==");
     910    PXOPT_COPY_STR(config->args, where, "-skycell_id",  "warpSkyCellMap.skycell_id", "==");
     911
    920912    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    921913
    922     // look for "inputs" that need to processed
    923     psString query = pxDataGet("magicdstool_toremove.sql");
     914    psString query = pxDataGet("magicdstool_getskycells.sql");
    924915    if (!query) {
    925916        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    … …  
    929920    if (psListLength(where->list)) {
    930921        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    931         psStringAppend(&query, " WHERE %s", whereClause);
     922        psStringAppend(&query, " AND %s", whereClause);
    932923        psFree(whereClause);
    933924    }
    934925    psFree(where);
    935926
    936     // treat limit == 0 as "no limit"
    937     if (limit) {
    938         psString limitString = psDBGenerateLimitSQL(limit);
    939         psStringAppend(&query, " %s", limitString);
    940         psFree(limitString);
    941     }
    942 
    943     if (!p_psDBRunQuery(config->dbh, query)) {
     927    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
    944928        psError(PS_ERR_UNKNOWN, false, "database error");
    945929        psFree(query);
    … …  
    970954    if (psArrayLength(output)) {
    971955        // negative simple so the default is true
    972         if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) {
     956        if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) {
    973957            psError(PS_ERR_UNKNOWN, false, "failed to print array");
    974958            psFree(output);
    … …  
    981965    return true;
    982966}
    983 static bool torestoreMode(pxConfig *config)
     967
     968static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state)
     969{
     970    PS_ASSERT_PTR_NON_NULL(state, false);
     971
     972    // check that state is a valid string value
     973    if (!(
     974            (strncmp(state, "new", 4) == 0)
     975            || (strncmp(state, "full", 5) == 0)
     976        )
     977    ) {
     978        psError(PS_ERR_UNKNOWN, false,
     979                "invalid magicDSRun state: %s", state);
     980        return false;
     981    }
     982
     983    char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64;
     984    if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) {
     985        psError(PS_ERR_UNKNOWN, false,
     986                "failed to change state for magic_id %" PRId64, magic_ds_id);
     987        return false;
     988    }
     989
     990    return true;
     991}
     992
     993static bool toremoveMode(pxConfig *config)
    984994{
    985995    PS_ASSERT_PTR_NON_NULL(config, false);
    … …  
    9941004
    9951005    // look for "inputs" that need to processed
    996     psString query = pxDataGet("magicdstool_torestore.sql");
     1006    psString query = pxDataGet("magicdstool_toremove.sql");
    9971007    if (!query) {
    9981008        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
    … …  
    10431053    if (psArrayLength(output)) {
    10441054        // negative simple so the default is true
    1045         if (!ippdbPrintMetadatas(stdout, output, "torestore", !simple)) {
     1055        if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) {
    10461056            psError(PS_ERR_UNKNOWN, false, "failed to print array");
    10471057            psFree(output);
    … …  
    10541064    return true;
    10551065}
    1056 
    1057 
    1058 static bool torevertMode(pxConfig *config)
     1066static bool torestoreMode(pxConfig *config)
    10591067{
    10601068    PS_ASSERT_PTR_NON_NULL(config, false);
    10611069
    10621070    psMetadata *where = psMetadataAlloc();
    1063     PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
    1064 
    10651071    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
    10661072    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
    1067     pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "==");
     1073    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
    10681074
    10691075    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    10701076    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    10711077
    1072     psString sql_file = NULL;
    1073     psStringAppend(&sql_file, "magicdstool_torevert_%s.sql", stage);
    1074 
    1075     psString query = pxDataGet(sql_file);
     1078    // look for "inputs" that need to processed
     1079    psString query = pxDataGet("magicdstool_torestore.sql");
    10761080    if (!query) {
    1077         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file);
    1078         psFree(sql_file);
    1079         return false;
    1080     }
    1081     psFree(sql_file);
     1081        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
     1082        return false;
     1083    }
    10821084
    10831085    if (psListLength(where->list)) {
    10841086        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    1085         psStringAppend(&query, " AND %s", whereClause);
     1087        psStringAppend(&query, " WHERE %s", whereClause);
    10861088        psFree(whereClause);
    10871089    }
    … …  
    11241126    if (psArrayLength(output)) {
    11251127        // negative simple so the default is true
     1128        if (!ippdbPrintMetadatas(stdout, output, "torestore", !simple)) {
     1129            psError(PS_ERR_UNKNOWN, false, "failed to print array");
     1130            psFree(output);
     1131            return false;
     1132        }
     1133    }
     1134
     1135    psFree(output);
     1136
     1137    return true;
     1138}
     1139
     1140
     1141static bool torevertMode(pxConfig *config)
     1142{
     1143    PS_ASSERT_PTR_NON_NULL(config, false);
     1144
     1145    psMetadata *where = psMetadataAlloc();
     1146    PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
     1147
     1148    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
     1149    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
     1150    pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "==");
     1151
     1152    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     1153    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     1154
     1155    psString sql_file = NULL;
     1156    psStringAppend(&sql_file, "magicdstool_torevert_%s.sql", stage);
     1157
     1158    psString query = pxDataGet(sql_file);
     1159    if (!query) {
     1160        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file);
     1161        psFree(sql_file);
     1162        return false;
     1163    }
     1164    psFree(sql_file);
     1165
     1166    if (psListLength(where->list)) {
     1167        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     1168        psStringAppend(&query, " AND %s", whereClause);
     1169        psFree(whereClause);
     1170    }
     1171    psFree(where);
     1172
     1173    // treat limit == 0 as "no limit"
     1174    if (limit) {
     1175        psString limitString = psDBGenerateLimitSQL(limit);
     1176        psStringAppend(&query, " %s", limitString);
     1177        psFree(limitString);
     1178    }
     1179
     1180    if (!p_psDBRunQuery(config->dbh, query)) {
     1181        psError(PS_ERR_UNKNOWN, false, "database error");
     1182        psFree(query);
     1183        return false;
     1184    }
     1185    psFree(query);
     1186
     1187    psArray *output = p_psDBFetchResult(config->dbh);
     1188    if (!output) {
     1189        psErrorCode err = psErrorCodeLast();
     1190        switch (err) {
     1191            case PS_ERR_DB_CLIENT:
     1192                psError(PXTOOLS_ERR_SYS, false, "database error");
     1193            case PS_ERR_DB_SERVER:
     1194                psError(PXTOOLS_ERR_PROG, false, "database error");
     1195            default:
     1196                psError(PXTOOLS_ERR_PROG, false, "unknown error");
     1197        }
     1198
     1199        return false;
     1200    }
     1201    if (!psArrayLength(output)) {
     1202        psTrace("magicdstool", PS_LOG_INFO, "no rows found");
     1203        psFree(output);
     1204        return true;
     1205    }
     1206
     1207    if (psArrayLength(output)) {
     1208        // negative simple so the default is true
    11261209        if (!ippdbPrintMetadatas(stdout, output, "torevert", !simple)) {
    11271210            psError(PS_ERR_UNKNOWN, false, "failed to print array");

  • trunk/ippTools/src/magicdstool.h

    r24683 r25419  
    3535    MAGICDSTOOL_MODE_TORESTORE,
    3636    MAGICDSTOOL_MODE_TOREVERT,
     37    MAGICDSTOOL_MODE_COMPLETEDREVERT,
    3738} MAGICDStoolMode;
    3839

  • trunk/ippTools/src/magicdstoolConfig.c

    r24937 r25419  
    9393    // -todestreak
    9494    psMetadata *todestreakArgs = psMetadataAlloc();
     95    psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage (required)", NULL);
    9596    psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0);
    9697    psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
    9798    psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL);
    98     psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage", NULL);
    9999    psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
    100100    psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
    … …  
    147147    psMetadataAddU64(torevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
    148148    psMetadataAddBool(torevertArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     149
     150    // -completedrevert
     151    psMetadata *completedrevertArgs = psMetadataAlloc();
     152    psMetadataAddS64(completedrevertArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0);
     153    psMetadataAddStr(completedrevertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL);
     154    psMetadataAddU64(completedrevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
    149155
    150156    psFree(now);
    … …  
    174180                    MAGICDSTOOL_MODE_TOREVERT, torevertArgs);
    175181
     182    PXOPT_ADD_MODE("-completedrevert", "change state for runs that have finished reverting",
     183                    MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertArgs);
     184
    176185    if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {
    177186        psError(PS_ERR_UNKNOWN, true, "option parsing failed");
Note: See TracChangeset for help on using the changeset viewer.