Index: /tags/ipp-20150329/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /tags/ipp-20150329/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 38118) +++ /tags/ipp-20150329/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 38119) @@ -83,7 +83,9 @@ ($req_type ne "byskycell"); +if (0) { if (!$data_group and !$default_data_groups) { $default_data_groups = load_data_groups($verbose); } +} my $dateobs_begin; @@ -211,7 +213,11 @@ my $mjd_max = $row->{MJD_MAX}; my $data_group = $row->{DATA_GROUP}; + + # default_data_groups is a concept far past it's sell by date +if (0) { if (isnull($data_group) and !$default_data_groups) { $default_data_groups = load_data_groups($verbose); } +} my $rownum = $row->{ROWNUM}; @@ -273,5 +279,17 @@ my ($release_name, $survey, $default_tess_id) = get_release_info($row); if (!$tess_id and $default_tess_id) { - $tess_id = $default_tess_id; + # Don't want to set default tess_id in this context + # byid: no the tess_id is defined by the run + # byexp warp: This is the one case where it might make sense to use a default to avoid double lookup. + # this could be avoided if we did a two level lookup in that case. Find the warpRun first then use the + # run's tess id to find the specific skycell(s). + # bydiff: no tess_id defined by the diff + # byskycell requires tess_id so this does not apply + # note: currently get_release_info only sets default_tess_id for byskycell and bycoord requests + if (ref($default_tess_id) eq 'ARRAY') { + $tess_id = $default_tess_id->[0]; + } else { + $tess_id = $default_tess_id; + } } @@ -393,5 +411,12 @@ my $release_args; if (($req_type eq 'byexp' and $stage ne 'raw') or ($req_type eq 'byskycell' and $stage ne 'diff')) { - if ($release_name or $survey) { + # XXX: Think hard before committing the change to use releasetool for byexp requests regardless. + # XXX: using releasetool by default for byexp requests will break for nightly data when + # the release table survey task breaks. + # (MOPS one off requests for last night's data) + # we could fall back and do the old style lookup in that case. + # Need to fix the duplicate camRun problem. + my $force_releasetool_for_byexp = 0; + if ($release_name or $survey or ($force_releasetool_for_byexp and ($req_type eq 'byexp'))) { $use_releasetool = 1; if ($release_name) { @@ -599,5 +624,6 @@ $command .= " -data_group $data_group"; } elsif (!$use_releasetool and $req_type eq 'byskycell') { - # XXX: Why am I using default data_groups only for byskycell requests? + # Default data_groups are used by bycoord in the camera run lookup and + # here for byskycell. $command .= $default_data_groups; } @@ -773,5 +799,11 @@ my ($release_name, $survey, $default_tess_id) = get_release_info($row); if (!$tess_id and $default_tess_id) { - $tess_id = $default_tess_id; + if (ref($default_tess_id) eq 'ARRAY') { + # XXX: handle the case where default_tess_id is an array + # For now just take the first entry (RINGS.V3) + $tess_id = $default_tess_id->[0]; + } else { + $tess_id = $default_tess_id; + } } @@ -1061,12 +1093,19 @@ + my @tess_ids; my ($release_name, $survey, $default_tess_id) = get_release_info($row); - if (!$tess_id and $default_tess_id) { - $tess_id = $default_tess_id; + if ($tess_id) { + push @tess_ids, $tess_id; + } elsif ($default_tess_id) { + if (ref($default_tess_id) eq 'ARRAY') { + @tess_ids = @$default_tess_id; + } else { + push @tess_ids, $default_tess_id; + } } my $rowList = [$row]; - my $results = (); + my $results = []; if (($stage eq "raw") or ($stage eq "chip")) { @@ -1081,5 +1120,8 @@ next if $component and ($chip->{component} ne $component); my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $stage, $chip->{id}, - $tess_id, $chip->{component}, $need_magic, + # lookup doesn't need tess_id in this context + # $tess_id, + undef, + $chip->{component}, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $fwhm_min, $fwhm_max, @@ -1097,21 +1139,24 @@ and ($stage ne "stack") and ($stage ne "diff"); - my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose); - - if (!$skycells or scalar @$skycells == 0) { + foreach my $tess_id (@tess_ids) { + my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose); + + if ($skycells and scalar @$skycells != 0) { + # XXX: We are not applying the fwhm cuts unless we use releasetool + foreach my $skycell (@$skycells) { + my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef, + $skycell->{tess_id}, $skycell->{component}, $need_magic, + $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, + $fwhm_min, $fwhm_max, + undef, 0, $release_name, $survey, + $verbose); + + next if !$these_results; + push @$results, @$these_results; + } + } + } + if (scalar @$results == 0) { setErrorCodes($rowList, $PSTAMP_NO_IMAGE_MATCH); - } else { - # XXX: We are not applying the fwhm cuts unless we use releasetool - foreach my $skycell (@$skycells) { - my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef, - $skycell->{tess_id}, $skycell->{component}, $need_magic, - $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, - $fwhm_min, $fwhm_max, - undef, 0, $release_name, $survey, - $verbose); - - next if !$these_results; - push @$results, @$these_results; - } } } @@ -1976,4 +2021,6 @@ } +# get user supplied release paramters and the asssociated default_tess_id. +# If release parameters are not supplied set defaults. sub get_release_info { my $row = shift; @@ -1985,19 +2032,85 @@ # no release check for survey $survey = isnull($row->{SURVEY_NAME}) ? "" : $row->{SURVEY_NAME}; - } else { - # if release is given leave survey empty - $survey = ""; - } - - my $tess_id; - if ($release_name) { - # hack to optimize bycoord and byskycell lookups for well known releases - # XXX: todo query database and return a list for the survey or release - if ($release_name eq '3PI.PV1') { - $tess_id = 'RINGS.V3'; - } - } - - return ($release_name, $survey, $tess_id); + } + + # XXX: From here down this function is currently very PS1 specific and depends on + # conventions being used for release names. + # The comments following the lines marked 'XXX to improve' give suggestions on + # how these hacks can be cleaned up. + + my $tess_id = $row->{TESS_ID}; + if (!$release_name and !$survey) { + # no release or survey. If user does not supply + # a data group use a default survey + my $req_type = $row->{REQ_TYPE}; + if ($req_type eq "bycoord" or $req_type eq 'byskycell') { + # USING 3PI for bycoord and byskycell requests + # XXX to improve + # get default survey from project + my $data_group = $row->{DATA_GROUP}; + if (isnull($data_group) and isnull($tess_id)) { + print "No release information supplied for this row. Defaulting to 3PI survey\n"; + $survey = '3PI'; + } + } + } + + # if survey was not supplied, guess it from the release_name. This is used below + # to choose the default_tess_id + if (!$survey) { + if ($release_name) { + # XXX: to improve + # get survey from ippRelease(release_name) in database. Would need to cache + # such results + if ($release_name =~ "^3PI") { + $survey = '3PI'; + } elsif ($release_name =~ "^MD") { + $survey = substr($release_name, 0, 4); + } elsif ($release_name =~ "^SSS") { + $survey = 'SSS'; + } elsif ($release_name =~ "^SAS") { + $survey = 'SAS'; + } else { + # we don't know + print "Can't guess survey for $release_name\n"; + } + } elsif (!isnull($tess_id)) { + # finally if it is a MD tessellation use the MD field as the survey + # (This will also work for M31 and STS when that data is released. + if ($tess_id =~ "^MD") { + $survey = substr($tess_id, 0, 4); + } + } + } + + # choose default tess_id based on the survey + + # Note: Setting a default_tess_id effectively disables lookup bycoord requests + # for previous tessellations unless the user supplies the tess_id (which is + # applied by the caller to this function). + # bills has decided that this is desirable behaviour since most everything has + # been processed on a V3 tess. + my $default_tess_id; + if ($survey) { + # XXX: to improve + # Put default tess_id in survey table. + # note users will need to supply tessellation to get at 3pi skycells + # in the cnp tessellation. Hmm. Maybe we need to be clever and look + # at declination or perhaps allow default_tess_id to be an array. + if ($survey eq '3PI') { + $default_tess_id = ['RINGS.V3', 'CNP.V3']; + } elsif ($survey eq 'SSS' or $survey eq 'SAS') { + $default_tess_id = 'RINGS.V3'; + } elsif ($survey =~ 'MD') { + $default_tess_id = $survey . '.V3'; + } + } + + # leave the returned survey blank if release_name was supplied + # XXX: Why? Answer: this is how this function always behaved and I'm not sure if the + # callers depend on that behavior. + $survey = "" if $release_name; + + return ($release_name, $survey, $default_tess_id); } @@ -2007,4 +2120,5 @@ if (!isnull($run_type)) { $run_type = lc($run_type); + # XXX: these diff_mode values should be defined somewhere else. if ($run_type eq 'warp_warp') { $diff_mode = 1; Index: /tags/ipp-20150329/dbconfig/changes.txt =================================================================== --- /tags/ipp-20150329/dbconfig/changes.txt (revision 38118) +++ /tags/ipp-20150329/dbconfig/changes.txt (revision 38119) @@ -2526,5 +2526,5 @@ -- -UPDATE dbversion set schema_version = '1.1.80, updated= CURRENT_TIMESTAMP(); +UPDATE dbversion set schema_version = '1.1.80', updated= CURRENT_TIMESTAMP(); ALTER TABLE remoteComponent ADD COLUMN jobs INT after stage_id; ALTER TABLE remoteComponent ADD COLUMN state VARCHAR(64) NOT NULL after jobs; Index: /tags/ipp-20150329/ppSkycell/src/ppSkycellLoop.c =================================================================== --- /tags/ipp-20150329/ppSkycell/src/ppSkycellLoop.c (revision 38118) +++ /tags/ipp-20150329/ppSkycell/src/ppSkycellLoop.c (revision 38119) @@ -240,5 +240,5 @@ maxCRPIX1 = wcs->crpix1; } - if (wcs->crpix2 < maxCRPIX2) { + if (wcs->crpix2 > maxCRPIX2) { maxCRPIX2 = wcs->crpix2; } Index: /tags/ipp-20150329/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /tags/ipp-20150329/pstamp/scripts/pstamp_job_run.pl (revision 38118) +++ /tags/ipp-20150329/pstamp/scripts/pstamp_job_run.pl (revision 38119) @@ -92,5 +92,5 @@ my $stack_bkg_mk_mdc = can_run('stack_bkg_mk_mdc.pl') or (warn "Can't find stack_bkg_mk_mdc.pl" and $missing_tools = 1); my $fpack = can_run('fpack') or (warn "Can't find fpack" and $missing_tools = 1); - +my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); if ($missing_tools) { @@ -177,4 +177,19 @@ # be copied from the cmf file. $argString .= " -recipe PSPHOT STACKPHOT" + } + } elsif ($stage eq 'stack') { + # no astrom file for stack (skycal cmf) Go find a staticsky cmf and use that + # otherwise silently ignore the request for sources since the stack smf is not useful. + my $staticsky_cmf = findStaticskyCMF($params); + if ($staticsky_cmf) { + print "Using staticsky cmf file $staticsky_cmf for sources\n"; + $argString .= " -write_cmf"; + $fileArgs .= " -sources $staticsky_cmf"; + # Set psphot recipe to STACKPHOT so that the extended source paramters will + # be copied from the cmf file. + $argString .= " -recipe PSPHOT STACKPHOT"; + push @file_list, $staticsky_cmf; + } else { + print "No skycal or staticsky cmf for $params->{stack_id} ignoring request for sources\n"; } } elsif ($params->{cmf}) { @@ -864,4 +879,34 @@ return 0; } +sub findStaticskyCMF { + my $params = shift; + my $stack_id = $params->{stack_id}; + + &my_die( "NO stack_id found in findStaticskyCMF\n", + $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$stack_id; + + my $command = "$staticskytool -dbname $params->{imagedb} -result -stack_id $stack_id"; + my $results = runToolAndParse($command, $verbose); + + if (!$results) { + print "no staticsky results found for stack $stack_id\n"; + return undef; + } + + my $latest; + my $latest_sky_id = 0; + foreach my $result (@$results) { + if ($result->{sky_id} > $latest_sky_id) { + $latest = $result; + $latest_sky_id = $result->{sky_id}; + } + } + if ($latest) { + # XXXX Use proper file rule + return $latest->{path_base} . ".stk.$stack_id.cmf"; + } + # Can't happen can it? + return undef; +} sub my_die Index: /tags/ipp-20150329/tools/checkexp =================================================================== --- /tags/ipp-20150329/tools/checkexp (revision 38118) +++ /tags/ipp-20150329/tools/checkexp (revision 38119) @@ -53,5 +53,6 @@ summitExp.fault AS summit_fault, pzDownloadExp.state AS download_state, count(pzDownloadImfile.class_id) AS download_count, - newExp.state AS newExp_state, newExp.exp_id + newExp.state AS newExp_state, newExp.exp_id, + rawExp.state as rawExp_state FROM summitExp LEFT JOIN pzDownloadExp USING(exp_name) LEFT JOIN pzDownloadImfile USING(exp_name) @@ -88,4 +89,6 @@ my $first_exp_id; my $first_exp_name; +my $last_registered_exp_id = ""; +my $last_registered_exp_name = ""; while (my $se_ref = $se_stmt->fetchrow_hashref()) { push @summit_exps, $se_ref; @@ -98,4 +101,5 @@ my $imfiles = $se_ref->{imfiles}; my $dateobs = $se_ref->{dateobs}; + my $raw_state = $se_ref->{rawExp_state}; if ($summit_fault) { push @summit_faults, $se_ref; @@ -115,4 +119,8 @@ $last_new_exp_id = $exp_id; $last_new_exp_name = $exp_name; + } + if ($raw_state and $raw_state eq 'full') { + $last_registered_exp_id = $exp_id; + $last_registered_exp_name = $exp_name; } } @@ -180,4 +188,6 @@ print "first exposure: $first_exp_name $first_exp_id\n"; +print "last registered: $last_registered_exp_name $last_registered_exp_id\n" + if $last_registered_exp_id ne $last_exp_id; print "last copied: $last_new_exp_name $last_new_exp_id\n" if $last_new_exp_id ne $last_exp_id; Index: /tags/ipp-20150329/tools/mysql-dump/delete_log_spacing.sh =================================================================== --- /tags/ipp-20150329/tools/mysql-dump/delete_log_spacing.sh (revision 38119) +++ /tags/ipp-20150329/tools/mysql-dump/delete_log_spacing.sh (revision 38119) @@ -0,0 +1,91 @@ +#!/usr/bin/env mana + +if ($argv:n != 2) + echo $argv:n + echo "USAGE: delete_log_spacing (namebase) [test/commit]" + break +end + +# times scales: +# all for 2 days +# 1/day for 1 week +# 1/week for 1 month +# 1/month for 1 year +# 1/quarter forever + +$TIMEFORMAT = mjd +$TIMEREF = 2000/01/01,00:00:00 + +macro do_delete + if ($0 != 3) + echo "USAGE: do_delete (start) (stop)" + break + end + + local i + + list files -x "find . -maxdepth 1 -name '$argv:0\*' -cmin -$1 -cmin +$2 | sort" + if ($files:n == 0) + echo "no files in this date range, skipping" + return + end + echo $files:0 $1 - $2 + list keep -add $files:0 + for i 1 $files:n + echo rm $files:$i + if ($commit) + exec rm $files:$i + end + end +end + +macro delete_log_spacing + + $keep:n = 0 + + # keep 1/day for 1 week (stating 2 days ago) + for dt 2 9 + echo "$dt days old (daily)" + $dts = ($dt + 1)*1440 + $dte = ($dt + 0)*1440 + do_delete $dts $dte + end + + # keep 1/week for 1 month (stating 9 days ago) + for dt 9 37 7 + echo "$dt days old (weekly)" + $dts = ($dt + 7)*1440 + $dte = ($dt + 0)*1440 + do_delete $dts $dte + end + + # keep 1/month for 1 year (stating 39 days ago) + for dt 37 397 30 + echo "$dt days old (monthly)" + $dts = ($dt + 30)*1440 + $dte = ($dt + 0)*1440 + do_delete $dts $dte + end + + echo "---- keeping these database copies ----" + for i 0 $keep:n + echo $keep:$i + end +end + +if ($SCRIPT) + if ("$argv:1" == "commit") + $commit = 1 + delete_log_spacing + exit 0 + end + + if ("$argv:1" == "test") + $commit = 0 + delete_log_spacing + exit 0 + end + + echo "invalid option: use either 'test' or 'commit'" + exit 0 +end Index: /tags/ipp-20150329/tools/mysql-dump/gpc1_install.sh =================================================================== --- /tags/ipp-20150329/tools/mysql-dump/gpc1_install.sh (revision 38118) +++ /tags/ipp-20150329/tools/mysql-dump/gpc1_install.sh (revision 38119) @@ -13,4 +13,5 @@ # - Moves the MD5 checksum to the distribution directory # 4) Import gpc1 into gpc1_0 or gpc1_1 (ensured by gpc1Import.py utility) +# NOTE step 4 is no longer done . /home/panstarrs/ipp/mysql-dump/functions.sh @@ -60,8 +61,9 @@ fi -log INFO "Symlinking backup file [$TARGET/$BACKUP_FILE] to distribution [$DISTRIBUTION_TARGET]" +log INFO "Hard linking backup file [$TARGET/$BACKUP_FILE] to distribution [$DISTRIBUTION_TARGET]" /usr/bin/ln -f $TARGET/$BACKUP_FILE $DISTRIBUTION_TARGET /usr/bin/cp -f $TARGET/$MD5FILE $DISTRIBUTION_MD5 +# We no longer ingest the backup into the database. # if it is between 0 and 4, ingest into gpc1_0 or gpc1_1 #HOUR=`date +"%H"`