Index: /branches/eam_branches/ipp-20130904/ippScripts/scripts/rawcheck.pl =================================================================== --- /branches/eam_branches/ipp-20130904/ippScripts/scripts/rawcheck.pl (revision 36369) +++ /branches/eam_branches/ipp-20130904/ippScripts/scripts/rawcheck.pl (revision 36370) @@ -81,5 +81,5 @@ my $regtool_cmd = "$regtool -processedimfile -exp_id $exp_id -dbname $dbname"; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $regtool_cmd, verbose => 1); + run(command => $regtool_cmd, verbose => 0); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -133,9 +133,11 @@ $existance[$i] = 1; $existing_copies++; - my $fh; - open($fh,$files[$i]); - $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest; + $md5sums[$i] = local_md5sum($files[$i]); +# my $fh; +# open($fh,$files[$i]); +# $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest; +# close($fh); $md5sum_uniq{$md5sums[$i]} = 1; - close($fh); + } else { @@ -223,8 +225,9 @@ # Begin my best validation thought { - my $fh; - open($fh,$files[0]); - my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest; - close($fh); + my $tmpmd5 = local_md5sum($files[0]); +# my $fh; +# open($fh,$files[0]); +# my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest; +# close($fh); if ($tmpmd5 ne $md5sum) { die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; @@ -257,8 +260,9 @@ my $uris = $neb->find_instances($key,$rep_vol); @$uris = map {URI->new($_)->file if $_} @$uris; - my $fh; - open($fh,${ $uris }[0]); - my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest; - close($fh); + my $tmpmd5 = local_md5sum(${ $uris }[0]); +# my $fh; +# open($fh,${ $uris }[0]); +# my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest; +# close($fh); if ($tmpmd5 ne $md5sum) { die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; @@ -282,9 +286,9 @@ # print $uris . " " . ${ $uris }[0] . " \n"; - my $fh; - open($fh,${ $uris }[0]); - - my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest; - close($fh); + my $tmpmd5 = local_md5sum(${ $uris }[0]); +# my $fh; +# open($fh,${ $uris }[0]); +# my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest; +# close($fh); if ($tmpmd5 ne $md5sum) { die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; @@ -318,4 +322,20 @@ } +sub local_md5sum { + my $filename = shift; + my $volume = (split /\//, $filename)[2]; + my $host = $volume; + $host =~ s/\.\d//; +# print "$filename $host $volume\n"; + my $response = `ssh $host md5sum $filename`; + chomp($response); + my ($sum, undef) = split /\s+/, $response; + unless(defined($sum)) { + my_die("Failed to calculate md5sum locally. $filename $host $volume"); + } + return($sum); +} + + sub parse_volume { my $filename = shift(@_); @@ -341,7 +361,4 @@ } - - - sub my_die { my $msg = shift(@_); Index: /branches/eam_branches/ipp-20130904/ippScripts/scripts/skycell_jpeg.pl =================================================================== --- /branches/eam_branches/ipp-20130904/ippScripts/scripts/skycell_jpeg.pl (revision 36369) +++ /branches/eam_branches/ipp-20130904/ippScripts/scripts/skycell_jpeg.pl (revision 36370) @@ -271,5 +271,5 @@ my %products = ('image' => "PPSTACK.UNCONV", - 'mask' => "PPSTACK.UNCONV.MASK", +# 'mask' => "PPSTACK.UNCONV.MASK", 'variance' => "PPSTACK.UNCONV.VARIANCE", 'exp' => "PPSTACK.UNCONV.EXP", @@ -327,14 +327,13 @@ $command .= " -Di BIN1 1 -Di BIN2 1 "; } -# elsif ($key eq 'image') { -# $command .= " -masks $tangents{$projection_cell}{mask}{NAME} "; -# } -# elsif ($key eq 'mask') { -# next; # This should be made with the images. -# } - else { - # Append the image list to other objects, in case the WCS information is unpopulated - $command .= " -wcsref $tangents{$projection_cell}{image}{NAME} "; - } + elsif ($key eq 'image') { + $command .= " -exptimeOrder 1 "; + } + elsif ($key eq 'variance') { + $command .= " -exptimeOrder 2 "; + } + # Append the image list to other objects, in case the WCS information is unpopulated + $command .= " -wcsref $tangents{$projection_cell}{image}{NAME} "; + if ($tangents{$projection_cell}{$key}{N} > 0) { print "$command\n"; Index: /branches/eam_branches/ipp-20130904/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20130904/ippTools/share/Makefile.am (revision 36369) +++ /branches/eam_branches/ipp-20130904/ippTools/share/Makefile.am (revision 36370) @@ -400,4 +400,5 @@ stacktool_tosummary.sql \ stacktool_addsummary.sql \ + stacktool_revertsummary.sql \ stacktool_summary.sql \ staticskytool_definebyquery_select.sql \ Index: /branches/eam_branches/ipp-20130904/ippTools/share/stacktool_addsummary.sql =================================================================== --- /branches/eam_branches/ipp-20130904/ippTools/share/stacktool_addsummary.sql (revision 36369) +++ /branches/eam_branches/ipp-20130904/ippTools/share/stacktool_addsummary.sql (revision 36370) @@ -1,4 +1,4 @@ INSERT INTO stackSummary (sass_id,projection_cell,path_base) VALUES ( - %lld, -- warp_id + %lld, -- sass_id '%s', -- projection_cell '%s' -- path_base Index: /branches/eam_branches/ipp-20130904/ippTools/share/stacktool_revertsummary.sql =================================================================== --- /branches/eam_branches/ipp-20130904/ippTools/share/stacktool_revertsummary.sql (revision 36370) +++ /branches/eam_branches/ipp-20130904/ippTools/share/stacktool_revertsummary.sql (revision 36370) @@ -0,0 +1,3 @@ +DELETE FROM stackSummary + WHERE + sass_id = %lld Index: /branches/eam_branches/ipp-20130904/ippTools/src/regtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20130904/ippTools/src/regtoolConfig.c (revision 36369) +++ /branches/eam_branches/ipp-20130904/ippTools/src/regtoolConfig.c (revision 36370) @@ -292,5 +292,5 @@ psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-state", 0, "search by exposure state", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); - psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); Index: /branches/eam_branches/ipp-20130904/ippTools/src/stacktool.c =================================================================== --- /branches/eam_branches/ipp-20130904/ippTools/src/stacktool.c (revision 36369) +++ /branches/eam_branches/ipp-20130904/ippTools/src/stacktool.c (revision 36370) @@ -44,4 +44,5 @@ static bool tosummaryMode(pxConfig *config); static bool addsummaryMode(pxConfig *config); +static bool revertsummaryMode(pxConfig *config); static bool summaryMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); @@ -85,4 +86,5 @@ MODECASE(STACKTOOL_MODE_TOSUMMARY, tosummaryMode); MODECASE(STACKTOOL_MODE_ADDSUMMARY, addsummaryMode); + MODECASE(STACKTOOL_MODE_REVERTSUMMARY, revertsummaryMode); MODECASE(STACKTOOL_MODE_SUMMARY, summaryMode); MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); @@ -1570,4 +1572,32 @@ } +static bool revertsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, NULL); + PXOPT_LOOKUP_S64(sass_id, config->args, "-sass_id", true, false); + + psString query = pxDataGet("stacktool_revertsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, sass_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psS64 numUpdated = psDBAffectedRows(config->dbh); + + if (numUpdated != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); + psFree(query); + return(false); + } + psLogMsg("stacktool", PS_LOG_INFO, "Deleted %ld rows", numUpdated); + + psFree(query); + + return(true); +} + static bool summaryMode(pxConfig *config) { Index: /branches/eam_branches/ipp-20130904/ippTools/src/stacktool.h =================================================================== --- /branches/eam_branches/ipp-20130904/ippTools/src/stacktool.h (revision 36369) +++ /branches/eam_branches/ipp-20130904/ippTools/src/stacktool.h (revision 36370) @@ -38,4 +38,5 @@ STACKTOOL_MODE_TOSUMMARY, STACKTOOL_MODE_ADDSUMMARY, + STACKTOOL_MODE_REVERTSUMMARY, STACKTOOL_MODE_SUMMARY, STACKTOOL_MODE_PENDINGCLEANUPRUN, Index: /branches/eam_branches/ipp-20130904/ippTools/src/stacktoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20130904/ippTools/src/stacktoolConfig.c (revision 36369) +++ /branches/eam_branches/ipp-20130904/ippTools/src/stacktoolConfig.c (revision 36370) @@ -265,4 +265,8 @@ psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0, "set summary path base", NULL); + // -revertsummary + psMetadata *revertsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-sass_id", 0, "search by stack association ID", 0); + // -summary psMetadata *summaryArgs = psMetadataAlloc(); @@ -335,13 +339,18 @@ PXOPT_ADD_MODE("-addsumskyfile", "", STACKTOOL_MODE_ADDSUMSKYFILE, addsumskyfileArgs); PXOPT_ADD_MODE("-sumskyfile", "list results of stackRun", STACKTOOL_MODE_SUMSKYFILE, sumskyfileArgs); - PXOPT_ADD_MODE("-sassskyfile", "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE, sassskyfileArgs); + PXOPT_ADD_MODE("-revertsumskyfile","", STACKTOOL_MODE_REVERTSUMSKYFILE, revertsumskyfileArgs); + PXOPT_ADD_MODE("-updatesumskyfile", "update fault code for sumskyfile", STACKTOOL_MODE_UPDATESUMSKYFILE, updatesumskyfileArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs); PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", STACKTOOL_MODE_DONECLEANUP, donecleanupArgs); - PXOPT_ADD_MODE("-updatesumskyfile", "update fault code for sumskyfile", STACKTOOL_MODE_UPDATESUMSKYFILE, updatesumskyfileArgs); + + PXOPT_ADD_MODE("-sassskyfile", "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE, sassskyfileArgs); PXOPT_ADD_MODE("-tosummary", "show runs that can be summarized", STACKTOOL_MODE_TOSUMMARY, tosummaryArgs); PXOPT_ADD_MODE("-summary", "show runs that have been summarized", STACKTOOL_MODE_SUMMARY, summaryArgs); PXOPT_ADD_MODE("-addsummary", "add entry to the summary table", STACKTOOL_MODE_ADDSUMMARY, addsummaryArgs); + PXOPT_ADD_MODE("-revertsummary", "revert entry in the summary table", STACKTOOL_MODE_REVERTSUMMARY, revertsummaryArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", STACKTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", STACKTOOL_MODE_IMPORTRUN, importrunArgs); Index: /branches/eam_branches/ipp-20130904/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/eam_branches/ipp-20130904/ippconfig/recipes/filerules-split.mdc (revision 36369) +++ /branches/eam_branches/ipp-20130904/ippconfig/recipes/filerules-split.mdc (revision 36370) @@ -382,6 +382,6 @@ PPSKYCELL.JPEG1 OUTPUT {OUTPUT}.0.b1.jpeg JPEG NONE CHIP TRUE NONE PPSKYCELL.JPEG2 OUTPUT {OUTPUT}.0.b2.jpeg JPEG NONE CHIP TRUE NONE -PPSKYCELL.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE COMP_IMG FPA TRUE NONE -PPSKYCELL.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE COMP_IMG FPA TRUE NONE +PPSKYCELL.BIN1 OUTPUT {OUTPUT}.b1.fits IMAGE NONE FPA TRUE NONE +PPSKYCELL.BIN2 OUTPUT {OUTPUT}.b2.fits IMAGE NONE FPA TRUE NONE PPSKYCELL.BIN1.MASK OUTPUT {OUTPUT}.b1.mk.fits MASK COMP_MASK FPA TRUE NONE PPSKYCELL.BIN2.MASK OUTPUT {OUTPUT}.b2.mk.fits MASK COMP_MASK FPA TRUE NONE