Index: /trunk/ippScripts/scripts/magic_destreak.pl =================================================================== --- /trunk/ippScripts/scripts/magic_destreak.pl (revision 29560) +++ /trunk/ippScripts/scripts/magic_destreak.pl (revision 29561) @@ -42,5 +42,5 @@ # Parse the command-line arguments my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base, $cam_reduction); -my ($streaks_path_base, $inv_streaks_path_base); +my ($streaks_path_base, $inv_streaks_path_base, $run_state); my ($outroot, $recoveryroot, $magicked); my ($replace, $release); @@ -50,4 +50,5 @@ 'magic_ds_id=s' => \$magic_ds_id,# Magic destreak run identifier 'camera=s' => \$camera, # camera for evaluating file rules + 'run-state=s' => \$run_state, # state of run (new or update) 'streaks_path_base=s' => \$streaks_path_base, # path_base for streaks data 'inv_streaks_path_base=s' => \$inv_streaks_path_base, #path_base for streaks from inverse diff @@ -77,8 +78,9 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --uri --path_base --outroot --magicked", +pod2usage( -msg => "Required options: --magic_ds_id --camera --run_state --streaks --stage --stage_id --component --uri --path_base --outroot --magicked", -exitval => 3) unless defined $magic_ds_id and defined $camera and + defined $run_state and defined $streaks and defined $streaks_path_base and @@ -113,4 +115,8 @@ &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); } + +&my_die("Invalid value for run-state: $run_state", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) + unless ($run_state eq 'new') or ($run_state eq 'update'); + $inv_streaks_path_base = undef if defined($inv_streaks_path_base) and ($inv_streaks_path_base eq "NULL"); $inv_streaks = undef if defined($inv_streaks) and ($inv_streaks eq "NULL"); @@ -479,11 +485,16 @@ # Input result into database { - my $command = "$magicdstool -adddestreakedfile"; + my $command = "$magicdstool"; $command .= " -magic_ds_id $magic_ds_id"; $command .= " -component $component"; $command .= " -setmagicked" if $replace; - $command .= " -backup_path_base $backup_path_base" if $backup_path_base; - $command .= " -recovery_path_base $recovery_path_base" if $recovery_path_base; - $command .= " $statsFlags" if $statsFlags; + if ($run_state eq 'new') { + $command .= " -adddestreakedfile"; + $command .= " -backup_path_base $backup_path_base" if $backup_path_base; + $command .= " -recovery_path_base $recovery_path_base" if $recovery_path_base; + $command .= " $statsFlags" if $statsFlags; + } else { + $command .= " -tofullfile"; + } $command .= " -dbname $dbname" if defined $dbname; @@ -581,5 +592,11 @@ $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; - my $command = "$magicdstool -adddestreakedfile"; + my $command = "$magicdstool"; + + if ($run_state eq 'new') { + $command .= " -adddestreakedfile"; + } else { + $command .= " -updatedestreakedfile"; + } $command .= " -magic_ds_id $magic_ds_id"; $command .= " -component $component"; Index: /trunk/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- /trunk/ippScripts/scripts/magic_destreak_cleanup.pl (revision 29560) +++ /trunk/ippScripts/scripts/magic_destreak_cleanup.pl (revision 29561) @@ -89,5 +89,5 @@ $q1 .= " WHERE magic_ds_id = $magic_ds_id"; -my $q2 = "SELECT * from magicDSFile where magic_ds_id = $magic_ds_id"; +my $q2 = "SELECT * from magicDSFile WHERE (data_state = 'full' OR data_state = 'update') AND magic_ds_id = $magic_ds_id"; my $stmt1 = $dbh->prepare($q1); @@ -245,21 +245,17 @@ delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources); } -} - -$dbh = DBI->connect($dsn, $dbuser, $dbpassword) or die "Cannot connect to mysql server\n"; - -if (!$no_update and ($num_components > 0)) { - my $result = $dbh->do("DELETE FROM magicDSFile WHERE magic_ds_id = ?", undef, $magic_ds_id); - # my $result = $stmt3->do($magic_ds_id); - my_die("attempt to delete magicDSFiles failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0"; -} - -if (!$no_update) { - my $result = $dbh->do("UPDATE magicDSRun SET state = 'cleaned' WHERE magic_ds_id = ?", undef, $magic_ds_id); - my_die("attempt to update magicDSRun.state failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0"; -} else { - print STDERR "skipping update of magicDSRun\n"; -} - + my $command = "$magicdstool -tocleanedfile -magic_ds_id $magic_ds_id -component $component"; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + carp("failed to update database for $magic_ds_id"); + } + } else { + print "Skipping command: $command\n"; + } +} ### Pau. @@ -292,5 +288,5 @@ $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; - my $command = "$magicdstool -updaterun -set_state failed_cleanup"; + my $command = "$magicdstool -updaterun -set_state error_cleaned"; $command .= " -magic_ds_id $magic_ds_id"; $command .= " -dbname $dbname" if defined $dbname; Index: /trunk/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- /trunk/ippScripts/scripts/magic_destreak_revert.pl (revision 29560) +++ /trunk/ippScripts/scripts/magic_destreak_revert.pl (revision 29561) @@ -38,5 +38,5 @@ # Parse the command-line arguments -my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction, $magicked); +my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction, $magicked, $run_state); my ($outroot, $recoveryroot, $replace, $release, $bytes, $md5sum); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); @@ -47,4 +47,5 @@ 'stage=s' => \$stage, # raw, chip, warp, or diff 'stage_id=s' => \$stage_id, # exp_id, chip_id, warp_id, or diff_id + 'run-state=s' => \$run_state, # current state of run 'component=s' => \$component, # the class_id or skycell_id 'path_base=s' => \$path_base, # path_base of the input @@ -74,4 +75,5 @@ defined $path_base and defined $magicked and + defined $run_state and defined $outroot; @@ -248,4 +250,5 @@ { my $command = "$magicdstool -revertdestreakedfile -i_am_sure"; + $command .= " -state $run_state"; $command .= " -magic_ds_id $magic_ds_id"; $command .= " -component $component"; Index: /trunk/ippTasks/destreak.cleanup.pro =================================================================== --- /trunk/ippTasks/destreak.cleanup.pro (revision 29560) +++ /trunk/ippTasks/destreak.cleanup.pro (revision 29561) @@ -50,5 +50,4 @@ periods -timeout 20 npending 1 - active false stdout NULL @@ -105,5 +104,4 @@ periods -exec $RUNEXEC periods -timeout 60 - active false task.exec Index: /trunk/ippTasks/destreak.pro =================================================================== --- /trunk/ippTasks/destreak.pro (revision 29560) +++ /trunk/ippTasks/destreak.pro (revision 29561) @@ -185,4 +185,5 @@ book getword magicToDS $pageName exp_id -var EXP_ID book getword magicToDS $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicToDS $pageName state -var RUN_STATE book getword magicToDS $pageName camera -var CAMERA book getword magicToDS $pageName streaks_uri -var STREAKS @@ -215,5 +216,5 @@ # TODO: do not add recoveryroot or replace if they are null or zero - $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED + $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED --run-state $RUN_STATE add_standard_args run @@ -390,4 +391,5 @@ book getword magicDSToRevert $pageName camera -var CAMERA book getword magicDSToRevert $pageName stage -var STAGE + book getword magicDSToRevert $pageName state -var RUN_STATE book getword magicDSToRevert $pageName stage_id -var STAGE_ID book getword magicDSToRevert $pageName component -var COMPONENT @@ -414,5 +416,5 @@ end - $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --replace $REPLACE --bothways $BOTHWAYS --magicked $MAGICKED + $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --replace $REPLACE --bothways $BOTHWAYS --magicked $MAGICKED --run-state $RUN_STATE add_standard_args run Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 29560) +++ /trunk/ippTools/share/Makefile.am (revision 29561) @@ -238,4 +238,6 @@ magictool_exposure.sql \ magicdstool_clearstatefaults.sql \ + magicdstool_change_file_data_state.sql \ + magicdstool_change_run_state.sql \ magicdstool_completed_runs.sql \ magicdstool_completedrevert.sql \ @@ -252,4 +254,6 @@ magicdstool_getskycells.sql \ magicdstool_revertdestreakedfile.sql \ + magicdstool_revertupdated.sql \ + magicdstool_setfiletoupdate.sql \ magicdstool_tocleanup.sql \ magicdstool_todestreak_camera.sql \ Index: /trunk/ippTools/share/chiptool_setimfiletoupdate.sql =================================================================== --- /trunk/ippTools/share/chiptool_setimfiletoupdate.sql (revision 29560) +++ /trunk/ippTools/share/chiptool_setimfiletoupdate.sql (revision 29561) @@ -11,6 +11,7 @@ AND (chipRun.state = 'cleaned' OR chipRun.state = 'update') AND (chipProcessedImfile.data_state = 'cleaned') - -- don't queue update if the associated magicDSFile exists + -- don't queue update if the associated magicDSFile exists and isn't cleaned AND (chipRun.magicked = 0 - OR (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'new') - AND magicDSFile.component IS NULL) + OR ((magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') + AND (magicDSFile.data_state = 'cleaned`' OR magicDSFile.data_state = 'update')) + ) Index: /trunk/ippTools/share/magicdstool_change_file_data_state.sql =================================================================== --- /trunk/ippTools/share/magicdstool_change_file_data_state.sql (revision 29561) +++ /trunk/ippTools/share/magicdstool_change_file_data_state.sql (revision 29561) @@ -0,0 +1,8 @@ +-- handle changes in data_state. +UPDATE magicDSFile + JOIN magicDSRun USING(magic_ds_id) +SET + magicDSFile.data_state = '%s' +WHERE + magic_ds_id = %lld + AND component = '%s' Index: /trunk/ippTools/share/magicdstool_change_run_state.sql =================================================================== --- /trunk/ippTools/share/magicdstool_change_run_state.sql (revision 29561) +++ /trunk/ippTools/share/magicdstool_change_run_state.sql (revision 29561) @@ -0,0 +1,15 @@ +-- change state of magicDSRun from goto_cleaned to cleaned or goto_purged to purged +-- when all of the constituant imfiles are in the right state +-- arguments are new state (cleaned or purged) chip_id and new state again for +-- the chipProcessedImfile sub query +UPDATE magicDSRun + SET magicDSRun.state = '%s' +WHERE + magicDSRun.magic_ds_id = %lld + AND (SELECT + COUNT(magic_ds_id) + FROM magicDSFile + WHERE + magicDSFile.magic_ds_id = magicDSRun.magic_ds_id + AND data_state != '%s' + ) = 0 Index: /trunk/ippTools/share/magicdstool_revertupdated.sql =================================================================== --- /trunk/ippTools/share/magicdstool_revertupdated.sql (revision 29561) +++ /trunk/ippTools/share/magicdstool_revertupdated.sql (revision 29561) @@ -0,0 +1,7 @@ +UPDATE magicDSRun + JOIN magicDSFile using(magic_ds_id) +SET magicDSFile.fault = 0 +WHERE + magicDSRun.state = 'update' + AND magicDSFile.data_state = 'update' + AND magicDSFile.fault != 0 Index: /trunk/ippTools/share/magicdstool_setfiletoupdate.sql =================================================================== --- /trunk/ippTools/share/magicdstool_setfiletoupdate.sql (revision 29561) +++ /trunk/ippTools/share/magicdstool_setfiletoupdate.sql (revision 29561) @@ -0,0 +1,9 @@ +UPDATE magicDSRun + JOIN magicDSFile USING(magic_ds_id) +SET magicDSRun.state = 'update', + magicDSFile.data_state = 'update', + magicDSFile.fault = 0 + -- set hook %s +WHERE magic_ds_id = %lld + AND (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') + AND (magicDSFile.data_state = 'cleaned') Index: /trunk/ippTools/share/magicdstool_todestreak_camera.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak_camera.sql (revision 29560) +++ /trunk/ippTools/share/magicdstool_todestreak_camera.sql (revision 29561) @@ -2,4 +2,5 @@ magicDSRun.magic_ds_id, magicDSRun.magic_id, + magicDSRun.state, chipRun.exp_id, camRun.magicked, Index: /trunk/ippTools/share/magicdstool_todestreak_chip.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak_chip.sql (revision 29560) +++ /trunk/ippTools/share/magicdstool_todestreak_chip.sql (revision 29561) @@ -2,4 +2,5 @@ magicDSRun.magic_ds_id, magicDSRun.magic_id, + magicDSRun.state, chipRun.exp_id, chipRun.magicked, @@ -37,9 +38,10 @@ ON magicDSRun.label = Label.label WHERE - magicDSRun.state = 'new' + ((magicDSRun.state = 'new' AND magicDSFile.component IS NULL) + OR (magicDSRun.state = 'update' AND magicDSFile.data_state = 'update' + AND magicDSFile.fault = 0)) AND magicDSRun.stage = 'chip' AND (chipRun.state = 'full' OR (chipRun.state = 'update' and chipProcessedImfile.data_state = 'full')) AND chipProcessedImfile.fault = 0 AND chipProcessedImfile.quality = 0 - AND magicDSFile.component IS NULL AND (Label.active OR Label.active IS NULL) Index: /trunk/ippTools/share/magicdstool_todestreak_diff.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak_diff.sql (revision 29560) +++ /trunk/ippTools/share/magicdstool_todestreak_diff.sql (revision 29561) @@ -3,4 +3,5 @@ magicDSRun.magic_ds_id, magicRun.magic_id, + magicDSRun.state, magicRun.exp_id, magicDSRun.label, @@ -50,4 +51,5 @@ magicDSRun.magic_ds_id, magicRun.magic_id, + magicDSRun.state, magicRun.exp_id, magicDSRun.label, Index: /trunk/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak_raw.sql (revision 29560) +++ /trunk/ippTools/share/magicdstool_todestreak_raw.sql (revision 29561) @@ -2,4 +2,5 @@ magicDSRun.magic_ds_id, magicRun.magic_id, + magicDSRun.state, magicRun.exp_id, magicDSRun.label, Index: /trunk/ippTools/share/magicdstool_todestreak_warp.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak_warp.sql (revision 29560) +++ /trunk/ippTools/share/magicdstool_todestreak_warp.sql (revision 29561) @@ -2,4 +2,5 @@ magicDSRun.magic_ds_id, magicRun.magic_id, + magicDSRun.state, magicRun.exp_id, magicDSRun.label, Index: /trunk/ippTools/share/magicdstool_torevert_chip.sql =================================================================== --- /trunk/ippTools/share/magicdstool_torevert_chip.sql (revision 29560) +++ /trunk/ippTools/share/magicdstool_torevert_chip.sql (revision 29561) @@ -22,5 +22,7 @@ JOIN rawExp ON chipRun.exp_id = rawExp.exp_id WHERE magicDSRun.stage = 'chip' - AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0) + AND (((magicDSRun.state = 'new' OR (magicDSRun.state = 'update')) + AND magicDSFile.fault > 0) + -- why don't we require a fault for these states? OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored') AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL)) Index: /trunk/ippTools/src/magicdstool.c =================================================================== --- /trunk/ippTools/src/magicdstool.c (revision 29560) +++ /trunk/ippTools/src/magicdstool.c (revision 29561) @@ -39,4 +39,5 @@ static bool advancerunMode(pxConfig *config); static bool revertdestreakedfileMode(pxConfig *config); +static bool updatedestreakedfileMode(pxConfig *config); static bool clearstatefaultsMode(pxConfig *config); static bool getskycellsMode(pxConfig *config); @@ -45,4 +46,7 @@ static bool completedrevertMode(pxConfig *config); static bool tocleanupMode(pxConfig *config); +static bool tofullfileMode(pxConfig *config); +static bool tocleanedfileMode(pxConfig *config); +static bool setfiletoupdateMode(pxConfig *config); static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state); @@ -73,4 +77,5 @@ MODECASE(MAGICDSTOOL_MODE_ADVANCERUN, advancerunMode); MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode); + MODECASE(MAGICDSTOOL_MODE_UPDATEDESTREAKEDFILE,updatedestreakedfileMode); MODECASE(MAGICDSTOOL_MODE_CLEARSTATEFAULTS, clearstatefaultsMode); MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsMode); @@ -79,4 +84,7 @@ MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertMode); MODECASE(MAGICDSTOOL_MODE_TOCLEANUP, tocleanupMode); + MODECASE(MAGICDSTOOL_MODE_TOFULLFILE, tofullfileMode); + MODECASE(MAGICDSTOOL_MODE_TOCLEANEDFILE, tocleanedfileMode); + MODECASE(MAGICDSTOOL_MODE_SETFILETOUPDATE, setfiletoupdateMode); default: psAbort("invalid option (this should not happen)"); @@ -558,4 +566,9 @@ PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + if (!strcmp(state, "update")) { + fprintf(stderr, "'-updaterun -set_state update' is not supported. Use -setfiletoupdate"); + return false; + } + // optional PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); @@ -907,4 +920,71 @@ return false; } + + return true; +} +static bool updatedestreakedfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required values + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(backup_path_base, config->args, "-backup_path_base", false, false); + PXOPT_LOOKUP_STR(recovery_path_base, config->args, "-recovery_path_base", false, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", false, false); + PXOPT_LOOKUP_F32(streak_frac, config->args, "-streak_frac", false, false); + PXOPT_LOOKUP_F32(nondiff_frac, config->args, "-nondiff_frac", false, false); + PXOPT_LOOKUP_F32(run_time, config->args, "-run_time", false, false); + + psString query = psStringCopy("UPDATE magicDSFile"); + + + char *initial_separator = " SET"; + char *sep = initial_separator; + if (fault) { + psStringAppend(&query, "%s fault = %d", sep, fault); + sep = ", "; + } + if (backup_path_base) { + psStringAppend(&query, "%s backup_path_base = '%s'", sep, backup_path_base); + sep = ", "; + } + if (recovery_path_base) { + psStringAppend(&query, "%s recovery_path_base = '%s'", sep, recovery_path_base); + sep = ", "; + } + if (data_state) { + psStringAppend(&query, "%s data_state = '%s'", sep, data_state); + sep = ", "; + } + if (streak_frac) { + psStringAppend(&query, "%s streak_frac = '%f'", sep, streak_frac); + sep = ", "; + } + if (nondiff_frac) { + psStringAppend(&query, "%s nondiff_frac = '%f'", sep, nondiff_frac); + sep = ", "; + } + if (run_time) { + psStringAppend(&query, "%s run_time = '%f'", sep, run_time); + sep = ", "; + } + if (sep == initial_separator) { + psFree(query); + psError(PS_ERR_UNKNOWN, true, "must set at least one value"); + return false; + } + + psStringAppend(&query, "\nWHERE magic_ds_id = %" PRId64 " AND component = '%s'\n", magic_ds_id, component); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); return true; @@ -1019,8 +1099,28 @@ PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - psString query = pxDataGet("magicdstool_revertdestreakedfile.sql"); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + psString queryFile = NULL; + if (state) { + if (! strcmp(state, "new") ) { + queryFile = "magicdstool_revertdestreakedfile.sql"; + } else if (!strcmp(state, "update")) { + queryFile = "magicdstool_revertupdated.sql"; + } else { + psError(PXTOOLS_ERR_SYS, true, "%s is not a valid value for state", state); + return false; + } + } else { + queryFile = "magicdstool_revertdestreakedfile.sql"; + } + psString query = pxDataGet(queryFile); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } if (psListLength(where->list)) { @@ -1241,4 +1341,5 @@ (strcmp(state, "censored") == 0) || (strcmp(state, "cleaned") == 0) || + (strcmp(state, "update") == 0) || (strcmp(state, "goto_restored") == 0) || (strcmp(state, "goto_censored") == 0) || @@ -1520,2 +1621,125 @@ return true; } + +// update magicDSFile.data_state to given value. +// afterwards, if all files in the magicDSRun have the new state, +// update the state for it as well +// shared code for the modes -tocleanedfile -tofullfile + +static bool change_file_data_state(pxConfig *config, psString data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // magic_id, component + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); + + psString query = pxDataGet("magicdstool_change_file_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!strcmp(data_state, "full")) { + // if -tofullfile optoinally set the magicked value for the component + PXOPT_LOOKUP_BOOL(setmagicked, config->args, "-setmagicked", false); + // set the image file's magicked flag + if (!setMagicked(config, magic_ds_id, component)) { + psError(PS_ERR_UNKNOWN, false, "setMagicked failed"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, magic_ds_id, component)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + query = pxDataGet("magicdstool_change_run_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, magic_ds_id, data_state)) { + psFree(query); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool tocleanedfileMode(pxConfig *config) +{ + return change_file_data_state(config, "cleaned"); +} +static bool tofullfileMode(pxConfig *config) +{ + return change_file_data_state(config, "full"); +} +/* +static bool topurgedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "purged", "goto_purged"); +} +static bool toscrubbedfileMode(pxConfig *config) +{ + return change_file_data_state(config, "scrubbed", "goto_scrubbed"); +} +*/ + +// a very specfic function to queue a cleaned magicDSFile to be updated +static bool setfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("magicdstool_setfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , magicDSRun.label = '%s'", label); + } + + if (component) { + psStringAppend(&query, " AND (magicDSFile.component = '%s')", component); + } + + if (!p_psDBRunQueryF(config->dbh, query, setHook, magic_ds_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} + + Index: /trunk/ippTools/src/magicdstool.h =================================================================== --- /trunk/ippTools/src/magicdstool.h (revision 29560) +++ /trunk/ippTools/src/magicdstool.h (revision 29561) @@ -38,4 +38,8 @@ MAGICDSTOOL_MODE_COMPLETEDREVERT, MAGICDSTOOL_MODE_TOCLEANUP, + MAGICDSTOOL_MODE_TOCLEANEDFILE, + MAGICDSTOOL_MODE_TOFULLFILE, + MAGICDSTOOL_MODE_SETFILETOUPDATE, + MAGICDSTOOL_MODE_UPDATEDESTREAKEDFILE, } MAGICDStoolMode; Index: /trunk/ippTools/src/magicdstoolConfig.c =================================================================== --- /trunk/ippTools/src/magicdstoolConfig.c (revision 29560) +++ /trunk/ippTools/src/magicdstoolConfig.c (revision 29561) @@ -137,7 +137,22 @@ psMetadataAddS16(adddestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + // -updatedestreakedfile + psMetadata *updatedestreakedfileArgs = psMetadataAlloc(); + psMetadataAddS64(updatedestreakedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID (required)", 0); + psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "define component name (required)", NULL); + psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-state", 0, "current value for magicDSRun.state", NULL); + psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-backup_path_base", 0, "define backup URI", NULL); + psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-recovery_path_base", 0, "define recovery pixels URI", NULL); + psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-data_state", 0, "change data_state", NULL); + psMetadataAddBool(updatedestreakedfileArgs, PS_LIST_TAIL, "-setmagicked", 0, "update the magicked state of the file", false); + psMetadataAddF32(updatedestreakedfileArgs, PS_LIST_TAIL, "-streak_frac", 0, "set fraction of pixels masked by streaks", 0); + psMetadataAddF32(updatedestreakedfileArgs, PS_LIST_TAIL, "-nondiff_frac", 0, "set fraction of pixels masked because nondiffed", 0); + psMetadataAddF32(updatedestreakedfileArgs, PS_LIST_TAIL, "-run_time", 0, "set the streaksremove run time for component ", 0); + psMetadataAddS16(updatedestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + // -revertdestreakedfile psMetadata *revertdestreakedfileArgs = psMetadataAlloc(); psMetadataAddS64(revertdestreakedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0); + psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-state", 0, "current value for magicDSRun.state", NULL); psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "search by component", NULL); psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); @@ -197,4 +212,22 @@ psMetadataAddU64(tocleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(tocleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -tocleanedfile + psMetadata *tocleanedfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic_ds_id", 0); + psMetadataAddStr(tocleanedfileArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); + + // -tofullfile + psMetadata *tofullfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic_ds_id", 0); + psMetadataAddStr(tofullfileArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); + psMetadataAddBool(tofullfileArgs, PS_LIST_TAIL, "-setmagicked", 0, "update the magicked state of the file", false); + + // -setfiletoupdate + psMetadata *setfiletoupdateArgs = psMetadataAlloc(); + psMetadataAddS64(setfiletoupdateArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic_ds_id", 0); + psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); + psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "set new label", NULL); + psFree(now); @@ -230,4 +263,13 @@ MAGICDSTOOL_MODE_TOCLEANUP, tocleanupArgs); + PXOPT_ADD_MODE("-tofullfile", "set component's data_state to full", + MAGICDSTOOL_MODE_TOFULLFILE, tofullfileArgs); + PXOPT_ADD_MODE("-tocleanedfile", "set component's data_state to cleaned", + MAGICDSTOOL_MODE_TOCLEANEDFILE, tocleanedfileArgs); + PXOPT_ADD_MODE("-setfiletoupdate", "set component's data_state to update", + MAGICDSTOOL_MODE_SETFILETOUPDATE, setfiletoupdateArgs); + PXOPT_ADD_MODE("-updatedestreakedfile", "update parameters of destreaked file", + MAGICDSTOOL_MODE_UPDATEDESTREAKEDFILE, updatedestreakedfileArgs); + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /trunk/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 29560) +++ /trunk/pstamp/scripts/pstamp_checkdependent.pl (revision 29561) @@ -197,5 +197,7 @@ if (!$job_fault and ($stage eq 'chip')) { # chip processing is done, start destreaking. - $job_fault = check_states_magicDSRun($stage, $stage_id, $rlabel, $need_magic, $it->{raw_magicked}, $it->{dsRun_state}); + my @chips; + push @chips, $it->{class_id}; + $job_fault = check_states_magicDSRun($stage, $stage_id, \@chips, $rlabel, $need_magic, $it->{raw_magicked}, $it->{magic_ds_id}, $it->{dsRun_state}); } if ($job_fault) { @@ -217,8 +219,13 @@ my $dsRun_state; my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked + my @chips; + my $magic_ds_id; if (!$whole_run) { foreach my $chip (@$metadatas) { $dsRun_state = $chip->{dsRun_state}; $raw_all_magicked &= ($chip->{raw_magicked} > 0); + $magic_ds_id = $chip->{magic_ds_id}; + + push @chips, $chip->{class_id}; if (($chip->{state} =~ /error/) or ($chip->{state} =~ /purged/) or ($chip->{state} =~ /scrubbed/)) { @@ -273,5 +280,5 @@ } - my $status = check_states_magicDSRun('chip', $chip_id, $rlabel, $need_magic, $raw_all_magicked, $dsRun_state); + my $status = check_states_magicDSRun('chip', $chip_id, \@chips, $rlabel, $need_magic, $raw_all_magicked, $magic_ds_id, $dsRun_state); return $status; @@ -633,11 +640,13 @@ my $stage = shift; my $stage_id = shift; + my $components = shift; my $rlabel = shift; my $need_magic = shift; my $input_magicked = shift; + my $magic_ds_id = shift; my $dsRun_state = shift; # XXX: this code assumes that destreaking is handled at the chip stage - my_die ("check_states_magicDSRun only works for stage chip", $PS_EXIT_PROG_ERROR) if $stage ne 'chip'; + my_die ("check_states_magicDSRun only implemented for chip stage", $PS_EXIT_PROG_ERROR) if $stage ne 'chip'; # if called from check_states_warp dsRun_state is unknown. Go find it. @@ -654,16 +663,18 @@ print "No magicDSRun for chipRun $stage_id and magic is required\n"; faultJobs('stop', undef, undef, $PSTAMP_NOT_DESTREAKED); - } elsif ($dsRun_state eq 'cleaned') { - my $command = "$magicdstool -updaterun -set_state new -stage $stage -stage_id $stage_id"; - $command .= " -set_label $rlabel" if $rlabel; - if (!$no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - my_die("failed to set destreak run to 'new' for ${stage}Run $stage_id", - $PS_EXIT_UNKNOWN_ERROR); + } elsif (($dsRun_state eq 'cleaned') or ($dsRun_state eq 'update')) { + foreach my $c (@$components) { + my $command = "$magicdstool -setfiletoupdate -magic_ds_id $magic_ds_id -component $c"; + $command .= " -set_label $rlabel" if $rlabel; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + my_die("failed to set destreaked component to 'update' for ${stage}Run $stage_id $c", + $PS_EXIT_UNKNOWN_ERROR); + } + } else { + print "skipping $command\n"; } - } else { - print "skipping $command\n"; } } elsif ($dsRun_state eq 'failed_revert') { Index: /trunk/pstamp/scripts/pstamp_get_image_job.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_get_image_job.pl (revision 29560) +++ /trunk/pstamp/scripts/pstamp_get_image_job.pl (revision 29561) @@ -105,4 +105,7 @@ $command .= " --prefix $prefix"; $command .= " --magicked" if $magicked; + # DANGER DANGER do not commit next line +# $command .= " --no_magic"; + # DANGER DANGER do not commit last line $command .= " --dbname $dbname" if $dbname; $command .= " --verbose" if $verbose; Index: /trunk/pstamp/scripts/pstamp_server_status =================================================================== --- /trunk/pstamp/scripts/pstamp_server_status (revision 29560) +++ /trunk/pstamp/scripts/pstamp_server_status (revision 29561) @@ -57,5 +57,6 @@ $error_code = (($error_code >> 8) or 1); if ($error_code == 12) { - print "Postage Stamp Server is not running\n"; + #print "Postage Stamp Server is not running\n"; + print "Postage Stamp Server will be down for maintenance until approximately 0400 UTC October 26\n"; exit 0; }