Index: /branches/eam_branches/ippTools-20240208/share/addtool_find_xcff_id_dvo.sql =================================================================== --- /branches/eam_branches/ippTools-20240208/share/addtool_find_xcff_id_dvo.sql (revision 42618) +++ /branches/eam_branches/ippTools-20240208/share/addtool_find_xcff_id_dvo.sql (revision 42618) @@ -0,0 +1,14 @@ +SELECT xcForceRun.*, xcForceResult.warp_id +FROM xcForceRun +JOIN xcForceResult using (xcff_id) +JOIN skycell using (skycell_id, tess_id) + +WHERE xcForceRun.state = 'full' + AND xcForceResult.quality = 0 + AND xcForceRun.ff_id NOT IN + (SELECT xcForceRun.ff_id + FROM addRun + JOIN xcForceRun ON (xcForceRun.xcff_id = addRun.stage_id AND addRun.stage = 'xcff') + JOIN xcForceResult ON (xcForceRun.xcff_id = xcForceResult.xcff_id) + WHERE addRun.stage = 'xcff' AND %s + ) Index: /branches/eam_branches/ippTools-20240208/share/addtool_find_xcffsummary_id_dvo.sql =================================================================== --- /branches/eam_branches/ippTools-20240208/share/addtool_find_xcffsummary_id_dvo.sql (revision 42618) +++ /branches/eam_branches/ippTools-20240208/share/addtool_find_xcffsummary_id_dvo.sql (revision 42618) @@ -0,0 +1,14 @@ +SELECT xcForceRun.* FROM xcForceRun +JOIN xcForceSummary USING (xcff_id) +JOIN skycell USING (skycell_id, tess_id) + +WHERE xcForceRun.state = 'full' +AND xcForceSummary.quality = 0 +AND xcForceSummary.fault = 0 +AND xcForceRun.xcff_id NOT IN ( + SELECT xcForceRun.xcff_id + FROM addRun + JOIN xcForceRun ON (xcForceRun.xcff_id = addRun.stage_id AND addRun.stage = 'xcff_summary') + JOIN xcForceSummary ON (xcForceRun.xcff_id = xcForceSummary.xcff_id) + WHERE addRun.stage = 'xcff_summary' AND %s + ) Index: /branches/eam_branches/ippTools-20240208/src/addtool.c =================================================================== --- /branches/eam_branches/ippTools-20240208/src/addtool.c (revision 42617) +++ /branches/eam_branches/ippTools-20240208/src/addtool.c (revision 42618) @@ -119,77 +119,103 @@ // pxcamGetSearchArgs (config, where); PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); - PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); - PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); - PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); - PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); - PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); // XXX drop + PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); // XXX drop + PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-xcff_id", "xcForceRun.ff_id", "=="); - // PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); - PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", false, false); PXOPT_LOOKUP_STR(minidvodb_host, config->args, "-set_minidvodb_host", false, false); - PXOPT_LOOKUP_BOOL(image_only, config->args, "-image_only", false); - PXOPT_LOOKUP_BOOL(minidvodb, config->args, "-set_minidvodb", false); - PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); - PXOPT_LOOKUP_BOOL(uncensored, config->args, "-uncensored", false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - - PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(image_only, config->args, "-image_only", false); + PXOPT_LOOKUP_BOOL(minidvodb, config->args, "-set_minidvodb", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_F32(minra, config->args, "-set_minra", false, false); - PXOPT_LOOKUP_F32(maxra, config->args, "-set_maxra", false, false); - PXOPT_LOOKUP_F32(mindec, config->args, "-set_mindec", false, false); - PXOPT_LOOKUP_F32(maxdec, config->args, "-set_maxdec", false, false); + PXOPT_LOOKUP_F32(minra, config->args, "-set_minra", false, false); + PXOPT_LOOKUP_F32(maxra, config->args, "-set_maxra", false, false); + PXOPT_LOOKUP_F32(mindec, config->args, "-set_mindec", false, false); + PXOPT_LOOKUP_F32(maxdec, config->args, "-set_maxdec", false, false); // Handle stages - PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + + bool validStage = false; if (strcmp(stage, "cam") == 0) { - pxcamGetSearchArgs (config, where); - pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); // define using camRun label - pxAddLabelSearchArgs (config, where, "-data_group","camRun.data_group", "=="); // define using camRun label - pxAddLabelSearchArgs (config, where, "-filter", "rawExp.filter", "LIKE"); // define using camRun label - PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); - } - else if (strcmp(stage, "stack") == 0) { - pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); // define using camRun label - pxAddLabelSearchArgs (config, where, "-data_group","stackRun.data_group", "=="); // define using camRun label - PXOPT_COPY_STR(config->args, where, "-reduction", "stackRun.reduction", "=="); - pxAddLabelSearchArgs (config, where, "-filter", "stackRun.filter", "=="); // define using camRun label - } - else if (strcmp(stage, "staticsky") == 0) { - pxAddLabelSearchArgs (config, where, "-label", "staticskyRun.label", "=="); // define using camRun label - pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label - PXOPT_COPY_STR(config->args, where, "-reduction", "staticskyRun.reduction", "=="); - //no filter here - } - else if (strcmp(stage, "skycal") == 0) { - pxAddLabelSearchArgs (config, where, "-label", "skycalRun.label", "=="); //define using skycalRun label - pxAddLabelSearchArgs (config, where, "-data_group","skycalRun.data_group", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "skycalRun.reduction", "=="); - } - else if (strcmp(stage, "diff") == 0) { - pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); //define using diffRun label - pxAddLabelSearchArgs (config, where, "-data_group","diffRun.data_group", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "diffRun.reduction", "=="); - } - else if (strcmp(stage, "fullforce") == 0) { - pxAddLabelSearchArgs (config, where, "-label", "fullForceRun.label", "=="); //define using fullForceRun label - pxAddLabelSearchArgs (config, where, "-data_group","fullForceRun.data_group", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "fullForceRun.reduction", "=="); - } - else if (strcmp(stage, "fullforce_summary")==0) { - //should be nearly identical to fullforce (uses the same tables) - pxAddLabelSearchArgs (config, where, "-label", "fullForceRun.label", "=="); - pxAddLabelSearchArgs (config, where, "-data_group","fullForceRun.data_group", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "fullForceRun.reduction", "=="); - } - + pxcamGetSearchArgs (config, where); + pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); // define using camRun label + pxAddLabelSearchArgs (config, where, "-data_group","camRun.data_group", "=="); // define using camRun label + pxAddLabelSearchArgs (config, where, "-filter", "rawExp.filter", "LIKE"); // define using camRun label + PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); + validStage = true; + } + if (strcmp(stage, "skycal") == 0) { + pxAddLabelSearchArgs (config, where, "-label", "skycalRun.label", "=="); //define using skycalRun label + pxAddLabelSearchArgs (config, where, "-data_group","skycalRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "skycalRun.reduction", "=="); + validStage = true; + } + if (strcmp(stage, "diff") == 0) { + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); //define using diffRun label + pxAddLabelSearchArgs (config, where, "-data_group","diffRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "diffRun.reduction", "=="); + validStage = true; + } + if (strcmp(stage, "fullforce") == 0) { + pxAddLabelSearchArgs (config, where, "-label", "fullForceRun.label", "=="); //define using fullForceRun label + pxAddLabelSearchArgs (config, where, "-data_group","fullForceRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "fullForceRun.reduction", "=="); + validStage = true; + } + if (strcmp(stage, "fullforce_summary")==0) { + //should be nearly identical to fullforce (uses the same tables) + pxAddLabelSearchArgs (config, where, "-label", "fullForceRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-data_group","fullForceRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "fullForceRun.reduction", "=="); + validStage = true; + } + if (strcmp(stage, "xcff") == 0) { + pxAddLabelSearchArgs (config, where, "-label", "xcForceRun.label", "=="); //define using fullForceRun label + pxAddLabelSearchArgs (config, where, "-data_group","xcForceRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "xcForceRun.reduction", "=="); + validStage = true; + } + if (strcmp(stage, "xcff_summary")==0) { + //should be nearly identical to fullforce (uses the same tables) + pxAddLabelSearchArgs (config, where, "-label", "xcForceRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-data_group","xcForceRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "xcForceRun.reduction", "=="); + validStage = true; + } + if (!validStage) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "invalid stage specified"); + return false; + } + +# if (0) + if (strcmp(stage, "stack") == 0) { + pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); // define using camRun label + pxAddLabelSearchArgs (config, where, "-data_group","stackRun.data_group", "=="); // define using camRun label + PXOPT_COPY_STR(config->args, where, "-reduction", "stackRun.reduction", "=="); + pxAddLabelSearchArgs (config, where, "-filter", "stackRun.filter", "=="); // define using camRun label + } + if (strcmp(stage, "staticsky") == 0) { + pxAddLabelSearchArgs (config, where, "-label", "staticskyRun.label", "=="); // define using camRun label + pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label + PXOPT_COPY_STR(config->args, where, "-reduction", "staticskyRun.reduction", "=="); + //no filter here + } +# endif + if (!psListLength(where->list)) { psFree(where); @@ -198,31 +224,9 @@ } - // Select either destreaked or uncensored data. CZW: Should this be defaulted to uncensored? - if (!(destreaked && uncensored) && (destreaked || uncensored)) { - //if one of destreaked and uncensored is true, but not both - if (destreaked) { - psTrace("addtool.c", PS_LOG_INFO, "using -destreaked\n"); - } else { - psTrace("addtool.c", PS_LOG_INFO, "using -unscensored\n"); - } - } else { - if (destreaked && uncensored) { - psError(PS_ERR_UNKNOWN, false, "Both -destreaked and -uncensored are selected. Please use one or the other, not both, \n"); - return false; - } - if (!destreaked && !uncensored) { - psError(PS_ERR_UNKNOWN, false, "Neither -destreaked or -uncensored are selected. Please use one.\n"); - return false; - } - } - if ((strcmp(stage,"diff") == 0) || - (strcmp(stage,"fullforce") == 0) || - (strcmp(stage,"fullforce_summary") == 0)) { - if (! (dvodb)) { - psError(PS_ERR_UNKNOWN, false, "The SQL is not correctly written to handle this case (-set_dvodb not specified), so stopping here."); + // only cam stage can inherit dvodb from the camRun, for all others it must be supplied + if (strcmp(stage, "cam") && !dvodb) { + psError(PS_ERR_UNKNOWN, false, "the dvodb must be supplied for stage %s", stage); return(false); - } - } - + } // prevent queueing an addRun if a given exposure has already been added to @@ -231,71 +235,29 @@ psString bare_query = NULL; + // camRun can be supplied with a dvodb, in which case we can default to that if (strcmp(stage,"cam") == 0 ) { - if (dvodb) { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_cam_id_dvo.sql\n%s\n", dvodb,stage); - // find the cam_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_cam_id_dvo.sql"); + if (dvodb) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_cam_id_dvo.sql\n%s\n", dvodb,stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_cam_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_cam_id.sql\n%s\n",stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_cam_id.sql"); + // inherit dvodb from camRun, avoid matching NULL + psStringAppend(&dvodb_string, "(camRun.dvodb IS NOT NULL AND previous_dvodb = camRun.dvodb)"); + } + } + if (strcmp(stage,"skycal") == 0) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_skycal_id_dvo.sql\n%s\n", dvodb,stage); + // find the skycal_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_skycal_id_dvo.sql"); // user supplied dvodb psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); - } else { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_cam_id.sql\n%s\n",stage); - // find the cam_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_cam_id.sql"); - // inherit dvodb from camRun, avoid matching NULL - psStringAppend(&dvodb_string, "(camRun.dvodb IS NOT NULL AND previous_dvodb = camRun.dvodb)"); - } - } - - else if (strcmp(stage,"stack") == 0) { - if (dvodb) { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_stack_id_dvo.sql\n%s\n", dvodb,stage); - // find the cam_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_stack_id_dvo.sql"); - // user supplied dvodb - psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); - } else { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_stack_id.sql\n%s\n",stage); - // find the cam_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_stack_id.sql"); - // inherit dvodb from camRun, avoid matching NULL - psStringAppend(&dvodb_string, "(stackRun.dvodb IS NOT NULL AND previous_dvodb = stackRun.dvodb)"); - } - } - - else if (strcmp(stage,"staticsky") == 0) { - if (dvodb ) { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_multi_dvo.sql\n%s\n", dvodb,stage); - // find the cam_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_sky_id_multi_dvo.sql"); - // user supplied dvodb - psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); - } else { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_sky_id_multi.sql\n%s\n",stage); - // find the cam_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_sky_id_multi.sql"); - // inherit dvodb from camRun, avoid matching NULL - psStringAppend(&dvodb_string, "(staticskyRun.dvodb IS NOT NULL AND previous_dvodb = staticskyRun.dvodb)"); - } - } - - else if (strcmp(stage,"skycal") == 0) { - if (dvodb) { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_skycal_id_dvo.sql\n%s\n", dvodb,stage); - // find the skycal_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_skycal_id_dvo.sql"); - // user supplied dvodb - psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); - } else { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_skycal_id.sql\n%s\n",stage); - // find the skycal_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_skycal_id.sql"); - // inherit dvodb from skycalRun, avoid matching NULL - psStringAppend(&dvodb_string, "(skycalRun.dvodb IS NOT NULL AND previous_dvodb = skycalRun.dvodb)"); - // this is silly, there is no dvodb in skycalRun...? - } - } - - else if (strcmp(stage,"diff") == 0) { - if (dvodb) { + } + + if (strcmp(stage,"diff") == 0) { psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_diff_id_dvo.sql\n%s\n", dvodb,stage); // find the skycal_id of all the exposures that we want to queue up. @@ -303,16 +265,6 @@ // user supplied dvodb psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); - } else { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_diff_id.sql\n%s\n",stage); - // find the skycal_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_diff_id.sql"); - // inherit dvodb from skycalRun, avoid matching NULL - psStringAppend(&dvodb_string, "(diffRun.dvodb IS NOT NULL AND previous_dvodb = diffRun.dvodb)"); - // this is silly, there is no dvodb in skycalRun...? - } - } - - else if (strcmp(stage,"fullforce") == 0) { - if (dvodb) { + } + if (strcmp(stage,"fullforce") == 0) { psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_ff_id_dvo.sql\n%s\n", dvodb,stage); // find the skycal_id of all the exposures that we want to queue up. @@ -320,15 +272,6 @@ // user supplied dvodb psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); - } else { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_ff_id.sql\n%s\n",stage); - // find the skycal_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_ff_id.sql"); - // inherit dvodb from skycalRun, avoid matching NULL - psStringAppend(&dvodb_string, "(fullForceRun.dvodb IS NOT NULL AND previous_dvodb = fullForceRun.dvodb)"); - // this is silly, there is no dvodb in skycalRun...? - } - } - else if (strcmp(stage,"fullforce_summary") == 0) { - if (dvodb) { + } + if (strcmp(stage,"fullforce_summary") == 0) { psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_ffsummary_id_dvo.sql\n%s\n", dvodb,stage); // find the skycal_id of all the exposures that we want to queue up. @@ -336,14 +279,54 @@ // user supplied dvodb psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); - } else { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_ffsummary_id.sql\n%s\n",stage); + } + if (strcmp(stage,"xcff") == 0) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_xcff_id_dvo.sql\n%s\n", dvodb,stage); // find the skycal_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_ffsummary_id.sql"); - // inherit dvodb from skycalRun, avoid matching NULL - psStringAppend(&dvodb_string, "(fullForceRun.dvodb IS NOT NULL AND previous_dvodb = fullForceRun.dvodb)"); - // this is silly, there is no dvodb in skycalRun...? - } - } - + bare_query = pxDataGet("addtool_find_xcff_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } + if (strcmp(stage,"xcff_summary") == 0) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_xcffsummary_id_dvo.sql\n%s\n", dvodb,stage); + // find the skycal_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_xcffsummary_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } + +// XXX EAM 20240204 : stack and staticsky are deprecated +# if (0) + if (strcmp(stage,"stack") == 0) { + if (dvodb) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_stack_id_dvo.sql\n%s\n", dvodb,stage); + // find the stack_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_stack_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_stack_id.sql\n%s\n",stage); + // find the stack_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_stack_id.sql"); + // inherit dvodb from camRun, avoid matching NULL + psStringAppend(&dvodb_string, "(stackRun.dvodb IS NOT NULL AND previous_dvodb = stackRun.dvodb)"); + } + } + if (strcmp(stage,"staticsky") == 0) { + if (dvodb ) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_multi_dvo.sql\n%s\n", dvodb,stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_sky_id_multi_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_sky_id_multi.sql\n%s\n",stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_sky_id_multi.sql"); + // inherit dvodb from camRun, avoid matching NULL + psStringAppend(&dvodb_string, "(staticskyRun.dvodb IS NOT NULL AND previous_dvodb = staticskyRun.dvodb)"); + } + } +# endif + if (!bare_query) { psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); @@ -363,65 +346,62 @@ psStringAppend(&query, " AND %s", whereClause); - if ((strcmp(stage,"diff") == 0)|| - (strcmp(stage,"fullforce") == 0)|| - (strcmp(stage,"fullforce_summary") == 0)) { - //diff needs the ra /deg stuff - //ff needs the ra /deg stuff - //ff summary needs the ra /deg stuff - psStringAppend(&query, " AND radeg >= %f", minra); - psStringAppend(&query, " AND radeg <= %f", maxra); - psStringAppend(&query, " AND decdeg >= %f", mindec); - psStringAppend(&query, " AND decdeg <= %f", maxdec); - } - - psFree(whereClause); + // diff, ff, ff_summary, xcff, xcff_summary all need the RA,DEC range restricted + // (to avoid excessively long query times) + + if (!strcmp(stage,"diff") || + !strcmp(stage,"fullforce") || + !strcmp(stage,"fullforce_summary") || + !strcmp(stage,"xcff") || + !strcmp(stage,"xcff_summary")) { + if (!isfinite(minra) || !isfinite(maxra) || !isfinite(mindec) || !isfinite(maxra)) { + psError(PS_ERR_UNKNOWN, true, "invalid or missing RA,DEC search parameters"); + return false; + } + psStringAppend(&query, " AND radeg >= %f", minra); + psStringAppend(&query, " AND radeg <= %f", maxra); + psStringAppend(&query, " AND decdeg >= %f", mindec); + psStringAppend(&query, " AND decdeg <= %f", maxdec); + } + psFree(whereClause); } else { - psError(PS_ERR_UNKNOWN, true, "search parameters are required"); - return false; + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; } psFree(where); - if (destreaked) { - //This picks the magicked/censored ones - if ((strcmp(stage,"cam") == 0)|| - (strcmp(stage,"stack") == 0)) { - psStringAppend(&query, " AND (camRun.magicked > 0)"); - } - // staticSky/skycal have no magicked column. - } else { - //This picks only the unmagicked/uncensored ones - //if (strcmp(stage,"cam") == 0) { - //we can now properly handle the magicked case in ippScripts, so we queue camRuns in any magic state now if -uncensored. - //psStringAppend(&query, " AND (camRun.magicked = 0)"); - //} - if (strcmp(stage,"stack") == 0) { - psStringAppend(&query, " AND (stackRun.magicked = 0)"); - } - } - // if we grab a group of camRun/stackRuns that have multiple camRun/stackRun pointing to the same exp_id/stack_id, and that exp_id/stack_id has never before been added to addRun, we need to group by exp_id/stack_id to ensure that only 1 of the camRun/stackRun for that exp_id is added to the addRun stage. + + // XXX EAM 20240204 : this code is bad because it allows the definebyquery to select a set of exposures that have been + // processed multiple times. the GROUP BY selects an entry without any guarantee this selection is the one + // which is desired. if (strcmp(stage,"cam") == 0) { - psStringAppend(&query, " GROUP BY exp_id"); + psStringAppend(&query, " GROUP BY exp_id"); } else if (strcmp(stage,"stack") == 0) { - psStringAppend(&query, " GROUP BY stack_id"); + psStringAppend(&query, " GROUP BY stack_id"); } else if (strcmp(stage,"staticsky") == 0) { - psStringAppend(&query, " GROUP BY sky_id, stack_id"); //some reason it needs this + psStringAppend(&query, " GROUP BY sky_id, stack_id"); //some reason it needs this } else if (strcmp(stage,"skycal") == 0) { - psStringAppend(&query, " GROUP BY skycal_id, sky_id, stack_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there + psStringAppend(&query, " GROUP BY skycal_id, sky_id, stack_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there } //needs to be checked HAF xxx else if (strcmp(stage,"diff") == 0) { - psStringAppend(&query, " GROUP BY diff_id, diff_skyfile_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there + psStringAppend(&query, " GROUP BY diff_id, diff_skyfile_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there } //needs to be checked HAF xxx else if (strcmp(stage,"fullforce") == 0) { - psStringAppend(&query, " GROUP BY ff_id, warp_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there + psStringAppend(&query, " GROUP BY ff_id, warp_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there } else if (strcmp(stage,"fullforce_summary") == 0) { - psStringAppend(&query, " GROUP BY ff_id "); //needs to be checked, but should be fine: want 1 ff summary cmf. + psStringAppend(&query, " GROUP BY ff_id "); //needs to be checked, but should be fine: want 1 ff summary cmf. + } + else if (strcmp(stage,"xcff") == 0) { + psStringAppend(&query, " GROUP BY xcff_id, warp_id "); //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there + } + else if (strcmp(stage,"xcff_summary") == 0) { + psStringAppend(&query, " GROUP BY xcff_id "); //needs to be checked, but should be fine: want 1 ff summary cmf. } @@ -438,7 +418,7 @@ if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error, \n%s\n", query); - psFree(query); - return false; + psError(PS_ERR_UNKNOWN, false, "database error, \n%s\n", query); + psFree(query); + return false; } psFree(query); @@ -446,216 +426,268 @@ psArray *output = p_psDBFetchResult(config->dbh); if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; } if (!psArrayLength(output)) { - psTrace("addtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; } if (pretend) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "addRun", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } psFree(output); - return false; - } - psFree(output); - return true; + return true; } // loop over our list of camRun rows to check the supplied and selected dvodb and workdir values: if (strcmp(stage,"cam") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - camRunRow *row = camRunObjectFromMetadata(md); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); - psFree(output); - return false; - } - - if (!dvodb && !row->dvodb) { - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, cam_id %" PRId64, row->label, row->cam_id); - psFree(output); - return false; - } - if (!workdir && !row->workdir) { - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, cam_id %" PRId64, row->label, row->cam_id); - psFree(output); - return false; - } - - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + camRunRow *row = camRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + if (!dvodb && !row->dvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, cam_id %" PRId64, row->label, row->cam_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, cam_id %" PRId64, row->label, row->cam_id); + psFree(output); + return false; + } + + psFree(row); + } } else if (strcmp(stage,"stack") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - stackRunRow *row = stackRunObjectFromMetadata(md); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); - psFree(output); - return false; - } - - if (!dvodb && !row->dvodb) { - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, stack_id %" PRId64, row->label, row->stack_id); - psFree(output); - return false; - } - if (!workdir && !row->workdir) { - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, stack_id %" PRId64, row->label, row->stack_id); - psFree(output); - return false; - } - - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + stackRunRow *row = stackRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + if (!dvodb && !row->dvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, stack_id %" PRId64, row->label, row->stack_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, stack_id %" PRId64, row->label, row->stack_id); + psFree(output); + return false; + } + + psFree(row); + } } else if (strcmp(stage,"staticsky") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - bool status = false; - psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs"); - if (!status) { - psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs"); - return false; - } - if (num_inputs < 0) { - psError(PS_ERR_UNKNOWN, true, "invalid value for num_inputs"); - return false; - } - - staticskyRunRow *row = staticskyRunObjectFromMetadata(md); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); - psFree(output); - return false; - } - - if (!dvodb) { //there's no staticsky.dvodb - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, sky_id %" PRId64, row->label, row->sky_id); - psFree(output); - return false; - } - if (!workdir && !row->workdir) { - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, sky_id %" PRId64, row->label, row->sky_id); - psFree(output); - return false; - } - - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + bool status = false; + psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs"); + return false; + } + if (num_inputs < 0) { + psError(PS_ERR_UNKNOWN, true, "invalid value for num_inputs"); + return false; + } + + staticskyRunRow *row = staticskyRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + if (!dvodb) { //there's no staticsky.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, sky_id %" PRId64, row->label, row->sky_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, sky_id %" PRId64, row->label, row->sky_id); + psFree(output); + return false; + } + + psFree(row); + } } else if (strcmp(stage,"skycal") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - skycalRunRow *row = skycalRunObjectFromMetadata(md); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); - psFree(output); - return false; - } - - if (!dvodb) { // there's no skycalRun.dvodb - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, skycal_id %" PRId64, row->label, row->skycal_id); - psFree(output); - return false; - } - if (!workdir && !row->workdir) { - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, skycal_id %" PRId64, row->label, row->skycal_id); - psFree(output); - return false; - } - - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + skycalRunRow *row = skycalRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); + psFree(output); + return false; + } + + if (!dvodb) { // there's no skycalRun.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, skycal_id %" PRId64, row->label, row->skycal_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, skycal_id %" PRId64, row->label, row->skycal_id); + psFree(output); + return false; + } + + psFree(row); + } } else if (strcmp(stage,"diff") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - diffRunRow *row = diffRunObjectFromMetadata(md); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into diffRun"); - psFree(output); - return false; - } - - if (!dvodb) { // there's no skycalRun.dvodb - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, diff_id %" PRId64, row->label, row->diff_id); - psFree(output); - return false; - } - if (!workdir && !row->workdir) { - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, diff_id %" PRId64, row->label, row->diff_id); - psFree(output); - return false; - } - - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + diffRunRow *row = diffRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into diffRun"); + psFree(output); + return false; + } + + if (!dvodb) { // there's no skycalRun.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, diff_id %" PRId64, row->label, row->diff_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, diff_id %" PRId64, row->label, row->diff_id); + psFree(output); + return false; + } + + psFree(row); + } } else if (strcmp(stage,"fullforce") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - fullForceRunRow *row = fullForceRunObjectFromMetadata(md); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fullforceRun"); - psFree(output); - return false; - } - - if (!dvodb) { // there's no skycalRun.dvodb - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, ff_id %" PRId64, row->label, row->ff_id); - psFree(output); - return false; - } - if (!workdir && !row->workdir) { - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, ff_id %" PRId64, row->label, row->ff_id); - psFree(output); - return false; - } - - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + fullForceRunRow *row = fullForceRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fullforceRun"); + psFree(output); + return false; + } + + if (!dvodb) { // there's no skycalRun.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, ff_id %" PRId64, row->label, row->ff_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, ff_id %" PRId64, row->label, row->ff_id); + psFree(output); + return false; + } + + psFree(row); + } } else if (strcmp(stage,"fullforce_summary") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - //i believe this is correct for ff_summary as well: - fullForceRunRow *row = fullForceRunObjectFromMetadata(md); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fullforceRun"); - psFree(output); - return false; - } - - if (!dvodb) { // there's no skycalRun.dvodb - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, ff_id %" PRId64, row->label, row->ff_id); - psFree(output); - return false; - } - if (!workdir && !row->workdir) { - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, ff_id %" PRId64, row->label, row->ff_id); - psFree(output); - return false; - } - - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + //i believe this is correct for ff_summary as well: + fullForceRunRow *row = fullForceRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fullforceRun"); + psFree(output); + return false; + } + + if (!dvodb) { // there's no skycalRun.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, ff_id %" PRId64, row->label, row->ff_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, ff_id %" PRId64, row->label, row->ff_id); + psFree(output); + return false; + } + + psFree(row); + } + } + else if (strcmp(stage,"xcff") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + xcForceRunRow *row = xcForceRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into xcforceRun"); + psFree(output); + return false; + } + + if (!dvodb) { // there's no skycalRun.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, xcff_id %" PRId64, row->label, row->xcff_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, xcff_id %" PRId64, row->label, row->xcff_id); + psFree(output); + return false; + } + + psFree(row); + } + } + else if (strcmp(stage,"xcff_summary") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + //i believe this is correct for ff_summary as well: + xcForceRunRow *row = xcForceRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fullforceRun"); + psFree(output); + return false; + } + + if (!dvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, ff_id %" PRId64, row->label, row->xcff_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, ff_id %" PRId64, row->label, row->xcff_id); + psFree(output); + return false; + } + + psFree(row); + } } @@ -676,324 +708,416 @@ if (strcmp(stage,"cam") == 0) { - // loop over our list of camRun rows - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - psS64 stage_id =0; - - camRunRow *row = camRunObjectFromMetadata(md); - stage_id = row->cam_id; - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); - psFree(output); - return false; - } - - // queue the exp - if (!pxaddQueueByCamID(config, - stage, - stage_id, - 0, - workdir ? workdir : row->workdir, - reduction ? reduction : row->reduction, - label ? label : row->label, - data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), - dvodb ? dvodb : row->dvodb, - note ? note : NULL, - image_only, - minidvodb, - minidvodb_group, - minidvodb_name, - minidvodb_host - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error sfg"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue stage %s %" PRId64,stage, stage_id); - psFree(row); - psFree(output); - return false; - } - psFree(row); - } + // loop over our list of camRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + camRunRow *row = camRunObjectFromMetadata(md); + stage_id = row->cam_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + 0, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : row->dvodb, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name, + minidvodb_host + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } } else if (strcmp(stage,"stack") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - psS64 stage_id =0; - - stackRunRow *row = stackRunObjectFromMetadata(md); - stage_id = row->stack_id; - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); - psFree(output); - return false; - } - - // queue the exp - if (!pxaddQueueByCamID(config, - stage, - stage_id, - 0, - workdir ? workdir : row->workdir, - reduction ? reduction : row->reduction, - label ? label : row->label, - data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), - dvodb ? dvodb : row->dvodb, - note ? note : NULL, - image_only, - minidvodb, - minidvodb_group, - minidvodb_name, - minidvodb_host - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error sfg"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue stage %s %" PRId64,stage, stage_id); - psFree(row); - psFree(output); - return false; - } - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + stackRunRow *row = stackRunObjectFromMetadata(md); + stage_id = row->stack_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + 0, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : row->dvodb, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name, + minidvodb_host + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } } else if (strcmp(stage,"staticsky") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - psS64 stage_id =0; - bool status = false; - //psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs"); - psS32 stack_id = psMetadataLookupS32(&status, md, "stack_id"); - - if (!status) { - psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item stack_id"); - return false; - } - staticskyRunRow *row = staticskyRunObjectFromMetadata(md); - stage_id = row->sky_id; - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); - psFree(output); - return false; - } - // queue the exp - //for (int filter_id = 0; filter_id < num_inputs; filter_id++) { - if (!pxaddQueueByCamID(config, - stage, - stage_id, - stack_id, - workdir ? workdir : row->workdir, - reduction ? reduction : row->reduction, - label ? label : row->label, - data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), - dvodb ? dvodb : NULL, - note ? note : NULL, - image_only, - minidvodb, - minidvodb_group, - minidvodb_name, - minidvodb_host - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error sfg"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue stage %s %" PRId64,stage, stage_id); - psFree(row); - psFree(output); - return false; - } - //} - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + bool status = false; + //psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs"); + psS32 stack_id = psMetadataLookupS32(&status, md, "stack_id"); + + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item stack_id"); + return false; + } + staticskyRunRow *row = staticskyRunObjectFromMetadata(md); + stage_id = row->sky_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + // queue the exp + //for (int filter_id = 0; filter_id < num_inputs; filter_id++) { + if (!pxaddQueueByCamID(config, + stage, + stage_id, + stack_id, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name, + minidvodb_host + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + //} + psFree(row); + } } else if (strcmp(stage,"skycal") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - psS64 stage_id =0; - - skycalRunRow *row = skycalRunObjectFromMetadata(md); - stage_id = row->skycal_id; - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); - psFree(output); - return false; - } - - // queue the exp - if (!pxaddQueueByCamID(config, - stage, - stage_id, - 0, - workdir ? workdir : row->workdir, - reduction ? reduction : row->reduction, - label ? label : row->label, - data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), - dvodb ? dvodb : NULL, - note ? note : NULL, - image_only, - minidvodb, - minidvodb_group, - minidvodb_name, - minidvodb_host - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error sfg"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue stage %s %" PRId64,stage, stage_id); - psFree(row); - psFree(output); - return false; - } - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + skycalRunRow *row = skycalRunObjectFromMetadata(md); + stage_id = row->skycal_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + 0, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name, + minidvodb_host + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } } else if (strcmp(stage,"diff") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - psS64 stage_id =0; - // psS64 stage_extra1 = 0; - diffRunRow *row = diffRunObjectFromMetadata(md); - stage_id = row->diff_id; - bool status = false; - psS32 stage_extra1 = psMetadataLookupS32(&status, md, "diff_skyfile_id"); - if (!status) { - psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item diff_skyfile_id"); - return false; - } - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); - psFree(output); - return false; - } - - // queue the exp - if (!pxaddQueueByCamID(config, - stage, - stage_id, - stage_extra1, - workdir ? workdir : row->workdir, - reduction ? reduction : row->reduction, - label ? label : row->label, - data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), - dvodb ? dvodb : NULL, - note ? note : NULL, - image_only, - minidvodb, - minidvodb_group, - minidvodb_name, - minidvodb_host - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error sfg"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue stage %s %" PRId64,stage, stage_id); - psFree(row); - psFree(output); - return false; - } - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + // psS64 stage_extra1 = 0; + diffRunRow *row = diffRunObjectFromMetadata(md); + stage_id = row->diff_id; + bool status = false; + psS32 stage_extra1 = psMetadataLookupS32(&status, md, "diff_skyfile_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item diff_skyfile_id"); + return false; + } + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + stage_extra1, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name, + minidvodb_host + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } } else if (strcmp(stage,"fullforce") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - psS64 stage_id =0; - - fullForceRunRow *row = fullForceRunObjectFromMetadata(md); - stage_id = row->ff_id; - bool status = false; - psS32 stage_extra1 = psMetadataLookupS32(&status, md, "warp_id"); - if (!status) { - psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item warp_id"); - return false; - } - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); - psFree(output); - return false; - } - - // queue the exp - if (!pxaddQueueByCamID(config, - stage, - stage_id, - stage_extra1, - workdir ? workdir : row->workdir, - reduction ? reduction : row->reduction, - label ? label : row->label, - data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), - dvodb ? dvodb : NULL, - note ? note : NULL, - image_only, - minidvodb, - minidvodb_group, - minidvodb_name, - minidvodb_host - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error sfg"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue stage %s %" PRId64,stage, stage_id); - psFree(row); - psFree(output); - return false; - } - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + fullForceRunRow *row = fullForceRunObjectFromMetadata(md); + stage_id = row->ff_id; + bool status = false; + psS32 stage_extra1 = psMetadataLookupS32(&status, md, "warp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item warp_id"); + return false; + } + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + stage_extra1, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name, + minidvodb_host + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } } else if (strcmp(stage,"fullforce_summary") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - psS64 stage_id =0; - - fullForceRunRow *row = fullForceRunObjectFromMetadata(md); - stage_id = row->ff_id; - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fullforceRun"); - psFree(output); - return false; - } - - // queue the exp - if (!pxaddQueueByCamID(config, - stage, - stage_id, - 0, - workdir ? workdir : row->workdir, - reduction ? reduction : row->reduction, - label ? label : row->label, - data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), - dvodb ? dvodb : NULL, - note ? note : NULL, - image_only, - minidvodb, - minidvodb_group, - minidvodb_name, - minidvodb_host - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error sfg"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue stage %s %" PRId64,stage, stage_id); - psFree(row); - psFree(output); - return false; - } - psFree(row); - } + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + fullForceRunRow *row = fullForceRunObjectFromMetadata(md); + stage_id = row->ff_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fullforceRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + 0, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name, + minidvodb_host + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + } + else if (strcmp(stage,"xcff") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + xcForceRunRow *row = xcForceRunObjectFromMetadata(md); + stage_id = row->xcff_id; + bool status = false; + psS32 stage_extra1 = psMetadataLookupS32(&status, md, "warp_id"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item warp_id"); + return false; + } + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + stage_extra1, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name, + minidvodb_host + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + } + else if (strcmp(stage,"xcff_summary") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + xcForceRunRow *row = xcForceRunObjectFromMetadata(md); + stage_id = row->xcff_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fullforceRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + 0, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name, + minidvodb_host + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } } psFree(output); Index: /branches/eam_branches/ippTools-20240208/src/addtoolConfig.c =================================================================== --- /branches/eam_branches/ippTools-20240208/src/addtoolConfig.c (revision 42617) +++ /branches/eam_branches/ippTools-20240208/src/addtoolConfig.c (revision 42618) @@ -280,7 +280,4 @@ PXOPT_ADD_MODE("-revertminidvodbprocessed","revert processed minidvobs", ADDTOOL_MODE_REVERTMINIDVODBPROCESSED, revertminidvodbprocessedArgs); PXOPT_ADD_MODE("-updateminidvodbprocessed","change processed minidvodb properties",ADDTOOL_MODE_UPDATEMINIDVODBPROCESSED, updateminidvodbprocessedArgs); - - - if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {