Index: /branches/eam_branches/ipp-20111110/Nebulous-Server/bin/neb-admin =================================================================== --- /branches/eam_branches/ipp-20111110/Nebulous-Server/bin/neb-admin (revision 32693) +++ /branches/eam_branches/ipp-20111110/Nebulous-Server/bin/neb-admin (revision 32694) @@ -534,4 +534,13 @@ { my %destination; + my %vol_id_list = ('ipp004.0' => 1,'ipp005.0' => 2, 'ipp006.0' => 3, 'ipp007.0' => 4,'ipp008.0' => 50,'ipp009.0' => 6,'ipp010.0' => 7, + 'ipp011.0' => 8,'ipp012.0' => 15,'ipp013.0' => 16, 'ipp014.0' => 18,'ipp015.0' => 17,'ipp016.0' => 9,'ipp017.0' => 10,'ipp018.0' => 11,'ipp019.0' => 12, + 'ipp020.0' => 52,'ipp021.0' => 14,'ipp023.0' => 19,'ipp024.0' => 20,'ipp025.0' => 32,'ipp026.0' => 21, 'ipp027.0' => 49, + 'ipp028.0' => 23,'ipp029.0' => 24,'ipp030.0' => 25,'ipp031.0' => 26,'ipp032.0' => 27,'ipp033.0' => 28,'ipp034.0' => 29, + 'ipp035.0' => 30,'ipp036.0' => 31,'ipp037.0' => 51,'ipp038.0' => 33,'ipp039.0' => 34,'ipp040.0' => 35,'ipp041.0' => 36, + 'ipp042.0' => 37,'ipp043.0' => 38,'ipp044.0' => 39,'ipp045.0' => 40,'ipp046.0' => 41,'ipp047.0' => 42,'ipp048.0' => 43, + 'ipp049.0' => 44,'ipp050.0' => 45,'ipp051.0' => 46,'ipp052.0' => 47,'ipp053.0' => 48,'ipp054.0' => 62,'ipp055.0' => 63, + 'ipp056.0' => 64,'ipp057.0' => 65,'ipp058.0' => 66,'ipp059.0' => 67,'ipp060.0' => 68,'ipp061.0' => 69,'ipp062.0' => 70, + 'ipp063.0' => 71,'ipp064.0' => 72,'ipp065.0' => 73,'ipp066.0' => 74); %{ $destination{VOL} } = ('ota01' => 7, 'ota02' => 8, 'ota03' => 3, 'ota04' => 3, 'ota05' => 4, 'ota06' => 4, 'ota10' => 50, 'ota11' => 50, 'ota12' => 6, 'ota13' => 6, 'ota14' => 7, 'ota15' => 8, 'ota16' => 15, 'ota17' => 15, @@ -542,20 +551,23 @@ 'ota60' => 34, 'ota61' => 35, 'ota62' => 36, 'ota63' => 37, 'ota64' => 38, 'ota65' => 39, 'ota66' => 41, 'ota67' => 42, 'ota71' => 43, 'ota72' => 44, 'ota73' => 45, 'ota74' => 46, 'ota75' => 47, 'ota76' => 48); - %{ $destination{HOST} } = ('ota01' => 'ipp010','ota02' => 'ipp011','ota03' => 'ipp006','ota04' => 'ipp006','ota05' => 'ipp007','ota06' => 'ipp007', - 'ota10' => 'ipp008','ota11' => 'ipp008','ota12' => 'ipp009','ota13' => 'ipp009','ota14' => 'ipp010','ota15' => 'ipp011','ota16' => 'ipp012','ota17' => 'ipp012', - 'ota20' => 'ipp013','ota21' => 'ipp014','ota22' => 'ipp014','ota23' => 'ipp015','ota24' => 'ipp016','ota25' => 'ipp016','ota26' => 'ipp017','ota27' => 'ipp018', - 'ota30' => 'ipp019','ota31' => 'ipp020','ota32' => 'ipp021','ota33' => 'ipp021','ota34' => 'ipp023','ota35' => 'ipp013','ota36' => 'ipp024','ota37' => 'ipp019', + %{ $destination{HOST} } = ('ota01' => 'ipp010','ota02' => 'ipp011','ota03' => 'ipp006','ota04' => 'ipp054','ota05' => 'ipp007','ota06' => 'ipp055', + 'ota10' => 'ipp008','ota11' => 'ipp056','ota12' => 'ipp009','ota13' => 'ipp057','ota14' => 'ipp058','ota15' => 'ipp059','ota16' => 'ipp012','ota17' => 'ipp060', + 'ota20' => 'ipp013','ota21' => 'ipp014','ota22' => 'ipp061','ota23' => 'ipp015','ota24' => 'ipp016','ota25' => 'ipp062','ota26' => 'ipp064','ota27' => 'ipp065', + 'ota30' => 'ipp066','ota31' => 'ipp020','ota32' => 'ipp063','ota33' => 'ipp021','ota34' => 'ipp023','ota35' => 'ipp013','ota36' => 'ipp024','ota37' => 'ipp019', 'ota40' => 'ipp025','ota41' => 'ipp018','ota42' => 'ipp017','ota43' => 'ipp015','ota44' => 'ipp027','ota45' => 'ipp028','ota46' => 'ipp029','ota47' => 'ipp030', 'ota50' => 'ipp031','ota51' => 'ipp032','ota52' => 'ipp033','ota53' => 'ipp034','ota54' => 'ipp035','ota55' => 'ipp036','ota56' => 'ipp037','ota57' => 'ipp038', 'ota60' => 'ipp039','ota61' => 'ipp040','ota62' => 'ipp041','ota63' => 'ipp042','ota64' => 'ipp043','ota65' => 'ipp044','ota66' => 'ipp046','ota67' => 'ipp047', 'ota71' => 'ipp048','ota72' => 'ipp049','ota73' => 'ipp050','ota74' => 'ipp051','ota75' => 'ipp052','ota76' => 'ipp053'); - %{ $destination{NAME} } = ('ota01' => 'ipp010.0','ota02' => 'ipp011.0','ota03' => 'ipp006.0','ota04' => 'ipp006.0','ota05' => 'ipp007.0','ota06' => 'ipp007.0', - 'ota10' => 'ipp008.0','ota11' => 'ipp008.0','ota12' => 'ipp009.0','ota13' => 'ipp009.0','ota14' => 'ipp010.0','ota15' => 'ipp011.0','ota16' => 'ipp012.0','ota17' => 'ipp012.0', - 'ota20' => 'ipp013.0','ota21' => 'ipp014.0','ota22' => 'ipp014.0','ota23' => 'ipp015.0','ota24' => 'ipp016.0','ota25' => 'ipp016.0','ota26' => 'ipp017.0','ota27' => 'ipp018.0', - 'ota30' => 'ipp019.0','ota31' => 'ipp020.0','ota32' => 'ipp021.0','ota33' => 'ipp021.0','ota34' => 'ipp023.0','ota35' => 'ipp013.0','ota36' => 'ipp024.0','ota37' => 'ipp019.0', + %{ $destination{NAME} } = ('ota01' => 'ipp010.0','ota02' => 'ipp011.0','ota03' => 'ipp006.0','ota04' => 'ipp054.0','ota05' => 'ipp007.0','ota06' => 'ipp055.0', + 'ota10' => 'ipp008.0','ota11' => 'ipp056.0','ota12' => 'ipp009.0','ota13' => 'ipp057.0','ota14' => 'ipp058.0','ota15' => 'ipp059.0','ota16' => 'ipp012.0','ota17' => 'ipp060.0', + 'ota20' => 'ipp013.0','ota21' => 'ipp014.0','ota22' => 'ipp061.0','ota23' => 'ipp015.0','ota24' => 'ipp016.0','ota25' => 'ipp062.0','ota26' => 'ipp064.0','ota27' => 'ipp065.0', + 'ota30' => 'ipp066.0','ota31' => 'ipp020.0','ota32' => 'ipp063.0','ota33' => 'ipp021.0','ota34' => 'ipp023.0','ota35' => 'ipp013.0','ota36' => 'ipp024.0','ota37' => 'ipp019.0', 'ota40' => 'ipp025.0','ota41' => 'ipp018.0','ota42' => 'ipp017.0','ota43' => 'ipp015.0','ota44' => 'ipp027.0','ota45' => 'ipp028.0','ota46' => 'ipp029.0','ota47' => 'ipp030.0', 'ota50' => 'ipp031.0','ota51' => 'ipp032.0','ota52' => 'ipp033.0','ota53' => 'ipp034.0','ota54' => 'ipp035.0','ota55' => 'ipp036.0','ota56' => 'ipp037.0','ota57' => 'ipp038.0', 'ota60' => 'ipp039.0','ota61' => 'ipp040.0','ota62' => 'ipp041.0','ota63' => 'ipp042.0','ota64' => 'ipp043.0','ota65' => 'ipp044.0','ota66' => 'ipp046.0','ota67' => 'ipp047.0', 'ota71' => 'ipp048.0','ota72' => 'ipp049.0','ota73' => 'ipp050.0','ota74' => 'ipp051.0','ota75' => 'ipp052.0','ota76' => 'ipp053.0'); + foreach my $k (keys %{ $destination{NAME} }) { + $destination{VOL}{$k} = $vol_id_list{$destination{NAME}{$k}}; + } my %blocked_volume = ('ipp027' => 1, 'ipp037' => 1, Index: /branches/eam_branches/ipp-20111110/PS-IPP-Config/lib/PS/IPP/Config.pm =================================================================== --- /branches/eam_branches/ipp-20111110/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 32693) +++ /branches/eam_branches/ipp-20111110/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 32694) @@ -398,5 +398,8 @@ return $self->{nebulous} if defined $self->{nebulous}; # Already started - my $server = metadataLookupStr( $self->{_siteConfig}, 'NEB_SERVER' ); # Nebulous server + # For consistency with the rest of the IPP, don't use site.config + # to set NEB_SERVER use environment variable. + # my $server = metadataLookupStr( $self->{_siteConfig}, 'NEB_SERVER' ); # Nebulous server + my $server = $ENV{NEB_SERVER}; unless (defined $server) { carp "Unable to find NEB_SERVER in camera configuration file."; Index: /branches/eam_branches/ipp-20111110/ippScripts/scripts/diff_skycell.pl =================================================================== --- /branches/eam_branches/ipp-20111110/ippScripts/scripts/diff_skycell.pl (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippScripts/scripts/diff_skycell.pl (revision 32694) @@ -289,8 +289,4 @@ } - if (!$ipprc->file_exists($configuration)) { - print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n"; - $configuration = undef; - } # use WARPSTATS if we're updating, as we don't care about the new stuff from the STACK and DIFF STATS recipes $recipe_ppstats = 'WARPSTATS'; Index: /branches/eam_branches/ipp-20111110/ippScripts/scripts/dist_defineruns.pl =================================================================== --- /branches/eam_branches/ipp-20111110/ippScripts/scripts/dist_defineruns.pl (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippScripts/scripts/dist_defineruns.pl (revision 32694) @@ -88,5 +88,5 @@ push @stages, $stage; } else { - @stages = qw( raw chip chip_bg camera fake warp warp_bg diff stack SSdiff sky); + @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky); } Index: /branches/eam_branches/ipp-20111110/ippScripts/scripts/magic_destreak.pl =================================================================== --- /branches/eam_branches/ipp-20111110/ippScripts/scripts/magic_destreak.pl (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippScripts/scripts/magic_destreak.pl (revision 32694) @@ -221,4 +221,5 @@ my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells"; $command .= " $skycell_args"; + $command .= " -dbname $dbname" if $dbname; my @diff_components; if ($mismatched_tess) { Index: /branches/eam_branches/ipp-20111110/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- /branches/eam_branches/ipp-20111110/ippScripts/scripts/magic_destreak_revert.pl (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippScripts/scripts/magic_destreak_revert.pl (revision 32694) @@ -511,5 +511,9 @@ my $command = "$magicdstool -updaterun"; $command .= " -magic_ds_id $magic_ds_id"; - $command .= " -set_state failed_revert"; + if ($run_state ne 'update') { + $command .= " -set_state failed_revert"; + } else { + $command .= " -set_state failed_revert_ud"; + } $command .= " -dbname $dbname" if defined $dbname; Index: /branches/eam_branches/ipp-20111110/ippTasks/diskbalance.pro =================================================================== --- /branches/eam_branches/ipp-20111110/ippTasks/diskbalance.pro (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippTasks/diskbalance.pro (revision 32694) @@ -318,5 +318,5 @@ $TARGET_LIMIT = 500 $TARGET_OFFSET = 0 -$TARGET_TARGET = "SOURCE" +$TARGET_TARGET = SOURCE # Select Nebulous objects which should be shuffled task target.load @@ -423,7 +423,7 @@ # Fix this with multihost restriction when possible. if ("$TARGET_TARGET" == "DESTINATION") - set.host.for.target $DEST_HOST + set.host.for.balance $DEST_HOST else - set.host.for.target $SOURCE_HOST + set.host.for.balance $SOURCE_HOST end Index: /branches/eam_branches/ipp-20111110/ippTasks/ipphosts.mhpcc.config =================================================================== --- /branches/eam_branches/ipp-20111110/ippTasks/ipphosts.mhpcc.config (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippTasks/ipphosts.mhpcc.config (revision 32694) @@ -6,90 +6,106 @@ # randomly. the only exception is ipp004, which is the DVO db host -# NOTE: wave 1 machines have 10TB; wave 2 & 3 machines have 20TB -# for balance, use 1 host per 10 TB in the partition +# I've allocated hosts based on disk size. +# 40TB => 2 units of skycells +# 20TB => 1 unit of skycells +# Each host gets one OTA/chip of data. +# These hosts are skipped: +# ipp004 : dvo +# ipp005 : dvo +# ipp026 : crashes +# ipp029 : crashes +# ipp045 : reference cat ipphosts METADATA camera STR skycell - count S32 76 - sky00 STR ipp005 + count S32 86 + sky00 STR ipp006 sky01 STR ipp006 sky02 STR ipp007 - sky03 STR ipp008 - sky04 STR ipp009 - sky05 STR ipp010 - sky06 STR ipp011 - sky07 STR ipp012 - sky08 STR ipp012 - sky09 STR ipp013 - sky10 STR ipp014 - sky11 STR ipp014 - sky12 STR ipp015 - sky13 STR ipp016 - sky14 STR ipp016 - sky15 STR ipp017 - sky16 STR ipp018 - sky17 STR ipp019 - sky18 STR ipp020 - sky19 STR ipp021 - sky20 STR ipp023 - sky21 STR ipp024 - sky22 STR ipp025 - sky23 STR ipp026 - sky24 STR ipp027 - sky25 STR ipp027 - sky26 STR ipp028 - sky27 STR ipp028 - sky28 STR ipp029 - sky29 STR ipp029 - sky30 STR ipp030 - sky31 STR ipp030 - sky32 STR ipp031 - sky33 STR ipp031 - sky34 STR ipp032 - sky35 STR ipp032 - sky36 STR ipp033 - sky37 STR ipp033 - sky38 STR ipp034 - sky39 STR ipp034 - sky40 STR ipp035 - sky41 STR ipp035 - sky42 STR ipp036 + sky03 STR ipp007 + sky04 STR ipp008 + sky05 STR ipp008 + sky06 STR ipp009 + sky07 STR ipp009 + sky08 STR ipp010 + sky09 STR ipp010 + sky10 STR ipp011 + sky11 STR ipp011 + sky12 STR ipp012 + sky13 STR ipp012 + sky14 STR ipp013 + sky15 STR ipp013 + sky16 STR ipp014 + sky17 STR ipp014 + sky18 STR ipp015 + sky19 STR ipp015 + sky20 STR ipp016 + sky21 STR ipp016 + sky22 STR ipp017 + sky23 STR ipp017 + sky24 STR ipp018 + sky25 STR ipp018 + sky26 STR ipp019 + sky27 STR ipp019 + sky28 STR ipp020 + sky29 STR ipp020 + sky30 STR ipp021 + sky31 STR ipp021 + sky32 STR ipp023 + sky33 STR ipp024 + sky34 STR ipp025 + sky35 STR ipp027 + sky36 STR ipp028 + sky37 STR ipp030 + sky38 STR ipp031 + sky39 STR ipp032 + sky40 STR ipp033 + sky41 STR ipp034 + sky42 STR ipp035 sky43 STR ipp036 sky44 STR ipp037 - sky45 STR ipp037 - sky46 STR ipp038 - sky47 STR ipp038 - sky48 STR ipp039 - sky49 STR ipp039 - sky50 STR ipp040 - sky51 STR ipp040 - sky52 STR ipp041 - sky53 STR ipp041 - sky54 STR ipp042 - sky55 STR ipp042 - sky56 STR ipp043 - sky57 STR ipp043 - sky58 STR ipp044 - sky59 STR ipp044 - sky60 STR ipp046 - sky61 STR ipp046 - sky62 STR ipp047 - sky63 STR ipp047 - sky64 STR ipp048 - sky65 STR ipp048 - sky66 STR ipp049 - sky67 STR ipp049 - sky68 STR ipp050 - sky69 STR ipp050 - sky70 STR ipp051 - sky71 STR ipp051 - sky72 STR ipp052 - sky73 STR ipp052 - sky74 STR ipp053 - sky75 STR ipp053 -END - -# we have 18 10TB nodes and 30 20TB nodes (ipp037 is used for dvo) -# 1 x W1 + 1 x W2 + 1 x W3 + 12 more nodes... - + sky45 STR ipp038 + sky46 STR ipp039 + sky47 STR ipp040 + sky48 STR ipp041 + sky49 STR ipp042 + sky50 STR ipp043 + sky51 STR ipp044 + sky52 STR ipp046 + sky53 STR ipp047 + sky54 STR ipp048 + sky55 STR ipp049 + sky56 STR ipp050 + sky57 STR ipp051 + sky58 STR ipp052 + sky59 STR ipp053 + sky60 STR ipp054 + sky61 STR ipp054 + sky62 STR ipp055 + sky63 STR ipp055 + sky64 STR ipp056 + sky65 STR ipp056 + sky66 STR ipp057 + sky67 STR ipp057 + sky68 STR ipp058 + sky69 STR ipp058 + sky70 STR ipp059 + sky71 STR ipp059 + sky72 STR ipp060 + sky73 STR ipp060 + sky74 STR ipp061 + sky75 STR ipp061 + sky76 STR ipp062 + sky77 STR ipp062 + sky78 STR ipp063 + sky79 STR ipp063 + sky80 STR ipp064 + sky81 STR ipp064 + sky82 STR ipp065 + sky83 STR ipp065 + sky84 STR ipp066 + sky85 STR ipp066 +END + +# These are ordered oddly to prevent excessive targetting shuffles. ipphosts METADATA camera STR GPC1 @@ -98,32 +114,32 @@ XY02 STR ipp011 XY03 STR ipp006 - XY04 STR ipp006 + XY04 STR ipp054 XY05 STR ipp007 - XY06 STR ipp007 + XY06 STR ipp055 XY10 STR ipp008 - XY11 STR ipp008 + XY11 STR ipp056 XY12 STR ipp009 - XY13 STR ipp009 - XY14 STR ipp010 - XY15 STR ipp011 + XY13 STR ipp057 + XY14 STR ipp058 + XY15 STR ipp059 XY16 STR ipp012 - XY17 STR ipp012 + XY17 STR ipp060 XY20 STR ipp013 XY21 STR ipp014 - XY22 STR ipp014 + XY22 STR ipp061 XY23 STR ipp015 XY24 STR ipp016 - XY25 STR ipp016 - - XY26 STR ipp017 - XY27 STR ipp018 - XY30 STR ipp019 + XY25 STR ipp062 + + XY26 STR ipp064 + XY27 STR ipp065 + XY30 STR ipp066 XY31 STR ipp020 - XY32 STR ipp021 + XY32 STR ipp063 XY33 STR ipp021 XY34 STR ipp023 @@ -177,71 +193,71 @@ ota02 STR ipp011 ota03 STR ipp006 - ota04 STR ipp006 - + ota04 STR ipp054 + ota05 STR ipp007 - ota06 STR ipp007 + ota06 STR ipp055 ota10 STR ipp008 - ota11 STR ipp008 - + ota11 STR ipp056 + ota12 STR ipp009 - ota13 STR ipp009 - ota14 STR ipp010 - ota15 STR ipp011 - + ota13 STR ipp057 + ota14 STR ipp058 + ota15 STR ipp059 + ota16 STR ipp012 - ota17 STR ipp012 + ota17 STR ipp060 ota20 STR ipp013 ota21 STR ipp014 - - ota22 STR ipp014 + + ota22 STR ipp061 ota23 STR ipp015 ota24 STR ipp016 - ota25 STR ipp016 - - ota26 STR ipp017 - ota27 STR ipp018 - ota30 STR ipp019 + ota25 STR ipp062 + + ota26 STR ipp064 + ota27 STR ipp065 + ota30 STR ipp066 ota31 STR ipp020 - - ota32 STR ipp021 + + ota32 STR ipp063 ota33 STR ipp021 ota34 STR ipp023 ota35 STR ipp013 - + ota36 STR ipp024 ota37 STR ipp019 ota40 STR ipp025 ota41 STR ipp018 - + ota42 STR ipp017 ota43 STR ipp015 ota44 STR ipp027 ota45 STR ipp028 - + ota46 STR ipp029 ota47 STR ipp030 ota50 STR ipp031 ota51 STR ipp032 - + ota52 STR ipp033 ota53 STR ipp034 ota54 STR ipp035 ota55 STR ipp036 - + ota56 STR ipp037 ota57 STR ipp038 ota60 STR ipp039 ota61 STR ipp040 - + ota62 STR ipp041 ota63 STR ipp042 ota64 STR ipp043 ota65 STR ipp044 - + ota66 STR ipp046 ota67 STR ipp047 ota71 STR ipp048 ota72 STR ipp049 - + ota73 STR ipp050 ota74 STR ipp051 @@ -288,71 +304,71 @@ XY02 STR ipp011 XY03 STR ipp006 - XY04 STR ipp006 - + XY04 STR ipp054 + XY05 STR ipp007 - XY06 STR ipp007 + XY06 STR ipp055 XY10 STR ipp008 - XY11 STR ipp008 - + XY11 STR ipp056 + XY12 STR ipp009 - XY13 STR ipp009 - XY14 STR ipp010 - XY15 STR ipp011 - + XY13 STR ipp057 + XY14 STR ipp058 + XY15 STR ipp059 + XY16 STR ipp012 - XY17 STR ipp012 + XY17 STR ipp060 XY20 STR ipp013 XY21 STR ipp014 - - XY22 STR ipp014 + + XY22 STR ipp061 XY23 STR ipp015 XY24 STR ipp016 - XY25 STR ipp016 - - XY26 STR ipp017 - XY27 STR ipp018 - XY30 STR ipp019 + XY25 STR ipp062 + + XY26 STR ipp064 + XY27 STR ipp065 + XY30 STR ipp066 XY31 STR ipp020 - - XY32 STR ipp021 + + XY32 STR ipp063 XY33 STR ipp021 XY34 STR ipp023 XY35 STR ipp013 - + XY36 STR ipp024 XY37 STR ipp019 XY40 STR ipp025 XY41 STR ipp018 - + XY42 STR ipp017 XY43 STR ipp015 XY44 STR ipp027 XY45 STR ipp028 - + XY46 STR ipp029 XY47 STR ipp030 XY50 STR ipp031 XY51 STR ipp032 - + XY52 STR ipp033 XY53 STR ipp034 XY54 STR ipp035 XY55 STR ipp036 - + XY56 STR ipp037 XY57 STR ipp038 XY60 STR ipp039 XY61 STR ipp040 - + XY62 STR ipp041 XY63 STR ipp042 XY64 STR ipp043 XY65 STR ipp044 - + XY66 STR ipp046 XY67 STR ipp047 XY71 STR ipp048 XY72 STR ipp049 - + XY73 STR ipp050 XY74 STR ipp051 @@ -363,81 +379,91 @@ ipphosts METADATA camera STR dist_skycell - count S32 76 - sky00 STR ipp005 + count S32 86 + sky00 STR ipp006 sky01 STR ipp006 sky02 STR ipp007 - sky03 STR ipp008 - sky04 STR ipp009 - sky05 STR ipp010 - sky06 STR ipp011 - sky07 STR ipp012 - sky08 STR ipp012 - sky09 STR ipp013 - sky10 STR ipp014 - sky11 STR ipp014 - sky12 STR ipp015 - sky13 STR ipp016 - sky14 STR ipp016 - sky15 STR ipp017 - sky16 STR ipp018 - sky17 STR ipp019 - sky18 STR ipp020 - sky19 STR ipp021 - sky20 STR ipp023 - sky21 STR ipp024 - sky22 STR ipp025 - sky23 STR ipp026 - sky24 STR ipp027 - sky25 STR ipp027 - sky26 STR ipp028 - sky27 STR ipp028 - sky28 STR ipp029 - sky29 STR ipp029 - sky30 STR ipp030 - sky31 STR ipp030 - sky32 STR ipp031 - sky33 STR ipp031 - sky34 STR ipp032 - sky35 STR ipp032 - sky36 STR ipp033 - sky37 STR ipp033 - sky38 STR ipp034 - sky39 STR ipp034 - sky40 STR ipp035 - sky41 STR ipp035 - sky42 STR ipp036 + sky03 STR ipp007 + sky04 STR ipp008 + sky05 STR ipp008 + sky06 STR ipp009 + sky07 STR ipp009 + sky08 STR ipp010 + sky09 STR ipp010 + sky10 STR ipp011 + sky11 STR ipp011 + sky12 STR ipp012 + sky13 STR ipp012 + sky14 STR ipp013 + sky15 STR ipp013 + sky16 STR ipp014 + sky17 STR ipp014 + sky18 STR ipp015 + sky19 STR ipp015 + sky20 STR ipp016 + sky21 STR ipp016 + sky22 STR ipp017 + sky23 STR ipp017 + sky24 STR ipp018 + sky25 STR ipp018 + sky26 STR ipp019 + sky27 STR ipp019 + sky28 STR ipp020 + sky29 STR ipp020 + sky30 STR ipp021 + sky31 STR ipp021 + sky32 STR ipp023 + sky33 STR ipp024 + sky34 STR ipp025 + sky35 STR ipp027 + sky36 STR ipp028 + sky37 STR ipp030 + sky38 STR ipp031 + sky39 STR ipp032 + sky40 STR ipp033 + sky41 STR ipp034 + sky42 STR ipp035 sky43 STR ipp036 sky44 STR ipp037 - sky45 STR ipp037 - sky46 STR ipp038 - sky47 STR ipp038 - sky48 STR ipp039 - sky49 STR ipp039 - sky50 STR ipp040 - sky51 STR ipp040 - sky52 STR ipp041 - sky53 STR ipp041 - sky54 STR ipp042 - sky55 STR ipp042 - sky56 STR ipp043 - sky57 STR ipp043 - sky58 STR ipp044 - sky59 STR ipp044 - sky60 STR ipp046 - sky61 STR ipp046 - sky62 STR ipp047 - sky63 STR ipp047 - sky64 STR ipp048 - sky65 STR ipp048 - sky66 STR ipp049 - sky67 STR ipp049 - sky68 STR ipp050 - sky69 STR ipp050 - sky70 STR ipp051 - sky71 STR ipp051 - sky72 STR ipp052 - sky73 STR ipp052 - sky74 STR ipp053 - sky75 STR ipp053 + sky45 STR ipp038 + sky46 STR ipp039 + sky47 STR ipp040 + sky48 STR ipp041 + sky49 STR ipp042 + sky50 STR ipp043 + sky51 STR ipp044 + sky52 STR ipp046 + sky53 STR ipp047 + sky54 STR ipp048 + sky55 STR ipp049 + sky56 STR ipp050 + sky57 STR ipp051 + sky58 STR ipp052 + sky59 STR ipp053 + sky60 STR ipp054 + sky61 STR ipp054 + sky62 STR ipp055 + sky63 STR ipp055 + sky64 STR ipp056 + sky65 STR ipp056 + sky66 STR ipp057 + sky67 STR ipp057 + sky68 STR ipp058 + sky69 STR ipp058 + sky70 STR ipp059 + sky71 STR ipp059 + sky72 STR ipp060 + sky73 STR ipp060 + sky74 STR ipp061 + sky75 STR ipp061 + sky76 STR ipp062 + sky77 STR ipp062 + sky78 STR ipp063 + sky79 STR ipp063 + sky80 STR ipp064 + sky81 STR ipp064 + sky82 STR ipp065 + sky83 STR ipp065 + sky84 STR ipp066 + sky85 STR ipp066 END Index: /branches/eam_branches/ipp-20111110/ippTasks/survey.pro =================================================================== --- /branches/eam_branches/ipp-20111110/ippTasks/survey.pro (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippTasks/survey.pro (revision 32694) @@ -302,10 +302,11 @@ # user functions to manipulate dist labels macro survey.add.dist - if ($0 != 3) - echo "USAGE: survey.add.dist (label) (workdir)" + if ($0 != 4) + echo "USAGE: survey.add.dist (label) (workdir) (muggle)" break end book newpage SURVEY_DIST $1 book setword SURVEY_DIST $1 WORKDIR $2 + book setword SURVEY_DIST $1 MUGGLE $3 book setword SURVEY_DIST $1 STATE PENDING end @@ -329,10 +330,11 @@ # user functions to manipulate chip_bg labels macro survey.add.chip.bg - if ($0 != 3) - echo "USAGE: survey.add.chip.bg (label) (dist_group)" + if ($0 != 4) + echo "USAGE: survey.add.chip.bg (label) (dist_group) (muggle)" break end book newpage SURVEY_CHIP_BG $1 book setword SURVEY_CHIP_BG $1 DIST_GROUP $2 + book setword SURVEY_CHIP_BG $1 MUGGLE $3 book setword SURVEY_CHIP_BG $1 STATE PENDING end @@ -356,10 +358,11 @@ # user functions to manipulate warp_bg labels macro survey.add.warp.bg - if ($0 != 3) - echo "USAGE: survey.add.warp.bg (label) (dist_group)" + if ($0 != 4) + echo "USAGE: survey.add.warp.bg (label) (dist_group) (muggle)" break end book newpage SURVEY_WARP_BG $1 book setword SURVEY_WARP_BG $1 DIST_GROUP $2 + book setword SURVEY_WARP_BG $1 MUGGLE $3 book setword SURVEY_WARP_BG $1 STATE PENDING end @@ -984,8 +987,13 @@ $workdir = $workdir_base/$label/$year/$month/$day + book getword SURVEY_DIST $label MUGGLE -var muggle + book setword SURVEY_DIST $label STATE DONE # note workdir is set by the script based on site.config $run = dist_defineruns.pl --label $label --workdir $workdir + if ($muggle != 0) + $run = $run --no_magic + end if ($DB:n == 0) @@ -1058,6 +1066,10 @@ book setword SURVEY_CHIP_BG $label STATE DONE book getword SURVEY_CHIP_BG $label DIST_GROUP -var dist_group + book getword SURVEY_CHIP_BG $label MUGGLE -var muggle $run = bgtool -definechip -label $label -set_dist_group $dist_group + if ($muggle == 0) + $run = $run -destreaked + end if ($DB:n == 0) @@ -1068,5 +1080,7 @@ end - # echo $run + if ($VERBOSE > 1) + echo $run + end command $run end @@ -1128,6 +1142,11 @@ book setword SURVEY_WARP_BG $label STATE DONE book getword SURVEY_WARP_BG $label DIST_GROUP -var dist_group + book getword SURVEY_WARP_BG $label MUGGLE -var muggle $run = bgtool -definewarp -warp_label $label -set_dist_group $dist_group + if ($muggle == 0) + $run = $run -destreaked + end + if ($DB:n == 0) @@ -1138,5 +1157,7 @@ end - # echo $run + if ($VERBOSE > 1) + echo $run + end command $run end @@ -1258,5 +1279,5 @@ book npages SURVEY_STATICSKYSINGLE -var N if ($N == 0) - echo "No STATICSKYSINGLE labels for processing" + # echo "No STATICSKYSINGLE labels for processing" break endif Index: /branches/eam_branches/ipp-20111110/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/eam_branches/ipp-20111110/ippToPsps/jython/ipptopspsdb.py (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippToPsps/jython/ipptopspsdb.py (revision 32694) @@ -125,35 +125,4 @@ self.logger.debug("Found %d processed but-not-on-datastore items" % len(ids)) - - return ids - - ''' - Returns a list of processed batch IDs that either not yet loaded to the ODM or not merge_worthy or not merged - NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to loading to ODM if there is a known problem with the batch - ''' - def getAllUnfinishedBatchIDs(self, epoch, dvoGpc1Label, batchType): - - sql = "SELECT batch_id \ - FROM batch \ - WHERE timestamp > '" + epoch + "' \ - AND dvo_db = '" + dvoGpc1Label + "' \ - AND loaded_to_datastore = 1 \ - AND batch_type = '" + batchType + "' \ - AND deleted_datastore = 0 \ - AND loaded_to_ODM != -1 \ - AND merge_worthy != -1 \ - AND merged = 0" - - ids = [] - try: - rs = self.executeQuery(sql) - while (rs.next()): - ids.append(rs.getInt(1)) - except: - self.logger.exception("Can't query for unloaded batch ids in ipptopsps Db") - - rs.close() - - self.logger.debug("Found %d unloaded items" % len(ids)) return ids Index: /branches/eam_branches/ipp-20111110/ippToPsps/jython/pollOdm.py =================================================================== --- /branches/eam_branches/ipp-20111110/ippToPsps/jython/pollOdm.py (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippToPsps/jython/pollOdm.py (revision 32694) @@ -17,26 +17,14 @@ Method to actually poll ODM ''' -def poll(batchType): +def poll(batchType, stage): + if stage == "unloaded": + ids = ippToPspsDb.getUnloadedBatchIDs(EPOCH, DVOLABEL, batchType) + elif stage == "unmergeworthy": + ids = ippToPspsDb.getUnmergeWorthyBatchIDs(EPOCH, DVOLABEL, batchType) + elif stage == "unmerged": + ids = ippToPspsDb.getUnmergedBatchIDs(EPOCH, DVOLABEL, batchType) - lookingForUnloaded = 0 - lookingForUnmergeworthy = 0 - lookingForUnmerged = 0 - lookingForAll = 0 - - if TYPE == "unloaded": - lookingForUnloaded = 1 - ids = ippToPspsDb.getUnloadedBatchIDs(EPOCH, DVOLABEL, batchType) - elif TYPE == "unmergeworthy": - lookingForUnmergeworthy = 1 - ids = ippToPspsDb.getUnmergeWorthyBatchIDs(EPOCH, DVOLABEL, batchType) - elif TYPE == "unmerged": - lookingForUnmerged = 1 - ids = ippToPspsDb.getUnmergedBatchIDs(EPOCH, DVOLABEL, batchType) - elif TYPE == "unfinished": - lookingForAll = 1 - ids = ippToPspsDb.getAllUnfinishedBatchIDs(EPOCH, DVOLABEL, batchType) - - logger.infoTitle("Checking %d %s %s batches" % (len(ids), TYPE, batchType)) + logger.infoTitle("Checking %d %s %s batches" % (len(ids), stage, batchType)) results = {} @@ -53,14 +41,14 @@ ippToPspsDb.updateOdmStatus(id, results) - if lookingForUnloaded: + if stage == "unloaded": if results['LOADEDTOODM'] == 1: newLoaded = newLoaded + 1 if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1 if results['MERGED'] == 1: newMerged = newMerged + 1 - elif lookingForUnmergeworthy: + elif stage == "unmergeworthy": if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1 if results['MERGED'] == 1: newMerged = newMerged + 1 - elif lookingForUnmerged: + elif stage == "unmerged": if results['MERGED'] == 1: newMerged = newMerged + 1 @@ -88,7 +76,7 @@ CONFIG = sys.argv[1] BATCHTYPE = sys.argv[2] - TYPE = sys.argv[3] + STAGE = sys.argv[3] else: - print "\n** Usage: " + sys.argv[0] + " [hours]\n" + print "\n** Usage: " + sys.argv[0] + " [hours]\n" sys.exit(1) @@ -113,11 +101,18 @@ EPOCH = configDoc.find("options/epoch").text +if STAGE == "all": stages = ['unloaded', 'unmergeworthy', 'unmerged'] +else: stages = [STAGE] + +if BATCHTYPE == "all": batchTypes = ['P2', 'ST'] +else: batchTypes = [BATCHTYPE] + +for stage in stages: logger.infoPair("Checking stage", stage) +for batchType in batchTypes: logger.infoPair("Checking batch type", batchType) + while True: - if BATCHTYPE == "all": - poll("P2") - poll("ST") - else: - poll(BATCHTYPE) + for stage in stages: + for batchType in batchTypes: + poll(batchType, stage) if SECONDS: Index: /branches/eam_branches/ipp-20111110/ippTools/share/bgtool_definechip.sql =================================================================== --- /branches/eam_branches/ipp-20111110/ippTools/share/bgtool_definechip.sql (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippTools/share/bgtool_definechip.sql (revision 32694) @@ -5,3 +5,2 @@ LEFT JOIN chipBackgroundRun USING(chip_id) WHERE chipRun.state = 'full' - AND chipRun.magicked > 0 -- require destreaked to stop race condition in destreaking Index: /branches/eam_branches/ipp-20111110/ippTools/share/chiptool_dropprocessedimfile_delete.sql =================================================================== --- /branches/eam_branches/ipp-20111110/ippTools/share/chiptool_dropprocessedimfile_delete.sql (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippTools/share/chiptool_dropprocessedimfile_delete.sql (revision 32694) @@ -2,5 +2,6 @@ USING magicDSRun, magicDSFile, chipRun, chipProcessedImfile WHERE chipRun.chip_id = chipProcessedImfile.chip_id - AND magicDSRun.stage ='chip' and magicDSRun.stage_id = chipRun.chip_id - AND magicDSFile.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSRun.stage ='chip' + AND magicDSRun.stage_id = chipRun.chip_id + AND magicDSRun.magic_ds_id = magicDSFile.magic_ds_id AND magicDSFile.component = chipProcessedImfile.class_id Index: /branches/eam_branches/ipp-20111110/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branches/ipp-20111110/ippTools/src/chiptool.c (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippTools/src/chiptool.c (revision 32694) @@ -885,4 +885,6 @@ } psFree(query); + int numDeleted = psDBAffectedRows(config->dbh); + psLogMsg("chiptool", PS_LOG_INFO, "Deleted %d chipProcessedImfiles", numDeleted); if (!fault) { @@ -898,4 +900,7 @@ } psFree(query_update); + int numUpdated = psDBAffectedRows(config->dbh); + psLogMsg("chiptool", PS_LOG_INFO, "Updated %d chipProcessedImfiles", numUpdated); + return true; Index: /branches/eam_branches/ipp-20111110/ippTools/src/disttool.c =================================================================== --- /branches/eam_branches/ipp-20111110/ippTools/src/disttool.c (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippTools/src/disttool.c (revision 32694) @@ -141,5 +141,4 @@ PXOPT_LOOKUP_BOOL(use_alternate, config->args, "-use_alternate", false); PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); - PXOPT_LOOKUP_BOOL(i_am_sure, config->args, "-i_am_sure", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); @@ -165,10 +164,4 @@ PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); - // queuing no_magic runs can be quite dangerous. Require confirmation - if (no_magic && !pretend && !i_am_sure) { - psError(PXTOOLS_ERR_SYS, true, "-i_am_sure required to queue no_magic distribution runs"); - return false; - } - if (use_alternate) { if (strcmp(stage, "raw")) { Index: /branches/eam_branches/ipp-20111110/ippTools/src/disttoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111110/ippTools/src/disttoolConfig.c (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippTools/src/disttoolConfig.c (revision 32694) @@ -49,5 +49,4 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-i_am_sure", 0, "confirmation needed when -no_magic is supplied", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-use_alternate", 0, "use alternate inputs", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); Index: /branches/eam_branches/ipp-20111110/ippTools/src/magicdstool.c =================================================================== --- /branches/eam_branches/ipp-20111110/ippTools/src/magicdstool.c (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippTools/src/magicdstool.c (revision 32694) @@ -53,4 +53,5 @@ static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state); +static bool validDSRunState(const char *state); # define MODECASE(caseName, func) \ @@ -1212,6 +1213,6 @@ psMetadata *where = psMetadataAlloc(); - // new state (required - PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", true, false); + // new state + PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", false, false); // old state (required) PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); @@ -1237,4 +1238,6 @@ if (!strcmp(state, "failed_revert")) { new_state = "new"; + } else if (!strcmp(state, "failed_revert_ud")) { + new_state = "update"; } else if (!strcmp(state, "failed_cleanup")) { new_state = "goto_cleaned"; @@ -1243,7 +1246,12 @@ return false; } + } else { + if (!validDSRunState(new_state)) { + psError(PS_ERR_UNKNOWN, true, "unexpected value for new state: %s", new_state); + return false; + } } if (!p_psDBRunQueryF(config->dbh, query, new_state)) { - psError(PS_ERR_UNKNOWN, false, "failed to revert"); + psError(PS_ERR_UNKNOWN, false, "failed to clear state faults"); psFree(query); return false; @@ -1403,4 +1411,5 @@ (strcmp(state, "wait") == 0) || (strcmp(state, "failed_revert") == 0) || + (strcmp(state, "failed_revert_ud") == 0) || (strcmp(state, "error_cleaned") == 0) || (strcmp(state, "restored") == 0) || Index: /branches/eam_branches/ipp-20111110/ippTools/src/magicdstoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111110/ippTools/src/magicdstoolConfig.c (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippTools/src/magicdstoolConfig.c (revision 32694) @@ -163,5 +163,5 @@ // -clearstatefaults psMetadata *clearstatefaultsArgs = psMetadataAlloc(); - psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL); + psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); psMetadataAddS64(clearstatefaultsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0); Index: /branches/eam_branches/ipp-20111110/ippconfig/recipes/ppSub.config =================================================================== --- /branches/eam_branches/ipp-20111110/ippconfig/recipes/ppSub.config (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippconfig/recipes/ppSub.config (revision 32694) @@ -219,4 +219,13 @@ END +SSTF_DIFF METADATA + DUAL BOOL FALSE # Dual convolution? + INVERSE BOOL FALSE # Generate inverse subtraction? + PHOTOMETRY BOOL TRUE # Perform photometry? + NOCONVOLVE BOOL TRUE + CONVOLVE.TARGET STR SINGLE2 # convolution direction +END + + # Gene's choice: multiplying by 1.75 #@ISIS.WIDTHS F32 1.8 3.3 5.8 10.2 17.9 # Gaussian FWHMs for ISIS kernels Index: /branches/eam_branches/ipp-20111110/ippconfig/recipes/psphot.config =================================================================== --- /branches/eam_branches/ipp-20111110/ippconfig/recipes/psphot.config (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippconfig/recipes/psphot.config (revision 32694) @@ -216,7 +216,7 @@ # @RADIAL.ANNULAR.BINS.UPPER F32 1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1 345.0 539.1 841.0 1305.0 # -# PS1 values? -@RADIAL.ANNULAR.BINS.LOWER F32 0.0 1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 # perl parser fails without a comment ticket #1476 -@RADIAL.ANNULAR.BINS.UPPER F32 1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1 # perl parser fails without a comment ticket #1476 +# PS1 values? (revised fpr PS1 @0.25 arcsec pixel scales) +@RADIAL.ANNULAR.BINS.LOWER F32 0.0 0.96 2.72 4.16 7.04 12.0 18.56 29.76 45.68 72.80 112.80 # perl parser fails without a comment ticket #1476 still? +@RADIAL.ANNULAR.BINS.UPPER F32 0.96 2.72 4.16 7.04 12.0 18.56 29.76 45.68 72.80 112.80 176.88 # perl parser fails without a comment ticket #1476 still? # Extended source fit parameters @@ -355,4 +355,7 @@ PETROSIAN_FOR_STARS BOOL FALSE + CONSTANT_PHOTOMETRIC_WEIGHTS BOOL TRUE + POISSON.ERRORS.PHOT.LMM BOOL TRUE + EXT_FIT_MAX_RADIUS F32 50.0 PSF_MODEL STR PS_MODEL_GAUSS @@ -505,4 +508,11 @@ END +SSTF_DIFF METADATA + PEAKS_NSIGMA_LIMIT_2 F32 4.0 # peak significance threshold + DIFF_STATS BOOL TRUE # Calculate diff-specific stats + OUTPUT.FORMAT STR PS1_DV2 # Format of output file +END + + # Recipe overrides for PR reduction PR METADATA Index: /branches/eam_branches/ipp-20111110/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /branches/eam_branches/ipp-20111110/ippconfig/recipes/reductionClasses.mdc (revision 32693) +++ /branches/eam_branches/ipp-20111110/ippconfig/recipes/reductionClasses.mdc (revision 32694) @@ -165,4 +165,6 @@ PSVIDEOPHOT STR PSVIDEOPHOT END + + # basic science analysis @@ -262,4 +264,5 @@ + DEFAULT_MASKTEST METADATA CHIP_PPIMAGE STR CHIP_MASKTEST @@ -282,4 +285,75 @@ BACKGROUND_PSWARP STR BACKGROUND END + +# SSTF tests +SSTF_NOCONV METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR SSTF_DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT +END + +SSTF_NOCONV_4SIG METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR SSTF_DIFF + DIFF_PSPHOT STR SSTF_DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT +END + +SSTF_4SIG METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR SSTF_DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT +END + # reduction class specifically for warpwarp diffs: Index: /branches/eam_branches/ipp-20111110/magic/remove/src/streaksio.c =================================================================== --- /branches/eam_branches/ipp-20111110/magic/remove/src/streaksio.c (revision 32693) +++ /branches/eam_branches/ipp-20111110/magic/remove/src/streaksio.c (revision 32694) @@ -632,4 +632,5 @@ if (!isImage(in)) { + psLogMsg("streaksremove", PS_LOG_INFO, "%s contains no image\n", in->name); return; } @@ -779,4 +780,9 @@ setupImageRefs(sFile *out, sFile *recoveryOut, sFile *in, int extnum, bool exciseAll) { + if (!in->image) { + // Input file is likely corrupt. XXX: Shouldn't we have caught this earlier? + psError(PS_ERR_UNKNOWN, true, "image is null for extension %d of %s file is likely corrupt", extnum, in->name); + streaksExit("", PS_EXIT_UNKNOWN_ERROR); + } if (!exciseAll) { // set output image to be the input image Index: /branches/eam_branches/ipp-20111110/ppSub/src/ppSubLoop.c =================================================================== --- /branches/eam_branches/ipp-20111110/ppSub/src/ppSubLoop.c (revision 32693) +++ /branches/eam_branches/ipp-20111110/ppSub/src/ppSubLoop.c (revision 32694) @@ -23,9 +23,9 @@ bool dumpout(pmConfig *config, char *name) { - pmFPAview *view = ppSubViewReadout(); // View to readout - pmReadout *out = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT"); - psphotSaveImage (NULL, out->image, name); - psFree(view); - return true; + pmFPAview *view = ppSubViewReadout(); // View to readout + pmReadout *out = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT"); + psphotSaveImage (NULL, out->image, name); + psFree(view); + return true; } @@ -33,13 +33,17 @@ bool ppSubLoop(ppSubData *data) { + bool mdok = false; + bool success = true; + psAssert(data, "Require processing data"); pmConfig *config = data->config; // Configuration psAssert(config, "Require configuration."); - bool success = true; + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim + psAssert(recipe, "We checked this earlier, so it should be here."); + bool noConvolve = psMetadataLookupBool(&mdok, recipe, "NOCONVOLVE"); // Do not use convolved images. pmConfigCamerasCull(config, NULL); pmConfigRecipesCull(config, "PPSUB,PPSTATS,PSPHOT,PSASTRO,MASKS,JPEG"); - pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT"); @@ -75,28 +79,27 @@ if (data->forcedPhot2) { // Change the recipe to use a higher nsigma limit and quit after pass1 - psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); - - psF32 nsigma_peak_save = psMetadataLookupF32 (NULL, recipe, "PEAKS_NSIGMA_LIMIT"); - char *breakPt_save = psMetadataLookupStr (NULL, recipe, "BREAK_POINT"); - - bool mdok; - psF32 pos2_nsigma_peak = psMetadataLookupF32 (&mdok, recipe, "PEAKS_POS2_NSIGMA_LIMIT"); + psMetadata *psphotRecipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + + psF32 nsigma_peak_save = psMetadataLookupF32 (NULL, psphotRecipe, "PEAKS_NSIGMA_LIMIT"); + char *breakPt_save = psMetadataLookupStr (NULL, psphotRecipe, "BREAK_POINT"); + + psF32 pos2_nsigma_peak = psMetadataLookupF32 (&mdok, psphotRecipe, "PEAKS_POS2_NSIGMA_LIMIT"); if (!mdok) { - psWarning("PEAKS_POS2_NSIGMA_LIMIT not found in recipe. Will use 25.\n"); + psWarning("PEAKS_POS2_NSIGMA_LIMIT not found in psphotRecipe. Will use 25.\n"); pos2_nsigma_peak = 25.; } - psMetadataAddF32(recipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", pos2_nsigma_peak); - psMetadataAddStr(recipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", "PASS1"); + psMetadataAddF32(psphotRecipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", pos2_nsigma_peak); + psMetadataAddStr(psphotRecipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", "PASS1"); bool foundDetections = false; if (!ppSubInputDetections(&foundDetections, "PPSUB.POS2.SOURCES", "PPSUB.REF", data)) { psError(psErrorCodeLast(), false, "Unable to measure positive detections (2)"); - psMetadataAddF32(recipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", nsigma_peak_save); - psMetadataAddStr(recipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", breakPt_save); - success = false; - goto ESCAPE; - } - psMetadataAddF32(recipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", nsigma_peak_save); - psMetadataAddStr(recipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", breakPt_save); + psMetadataAddF32(psphotRecipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", nsigma_peak_save); + psMetadataAddStr(psphotRecipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", breakPt_save); + success = false; + goto ESCAPE; + } + psMetadataAddF32(psphotRecipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", nsigma_peak_save); + psMetadataAddStr(psphotRecipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", breakPt_save); // if nothing was found, don't bother doing the forced photometry below if (!foundDetections) { @@ -107,16 +110,9 @@ // XXX if it exists, use the POS1, POS2 successs for the FWHMs - psMetadata *recipe = psMetadataLookupPtr(NULL, config->recipes, PPSUB_RECIPE); - bool noConvolve = psMetadataLookupBool(NULL, recipe, "NOCONVOLVE"); // Do not use convolved images. - if (noConvolve) { - psWarning("not matching PSFs because NOCONVOLVE is TRUE\n"); - } else { - if (!ppSubMatchPSFs(data)) { - psError(psErrorCodeLast(), false, "Unable to match PSFs."); - success = false; - goto ESCAPE; - } - } - + if (!ppSubMatchPSFs(data)) { + psError(psErrorCodeLast(), false, "Unable to match PSFs."); + success = false; + goto ESCAPE; + } if (data->quality) { @@ -138,11 +134,12 @@ psTimerClear("PPSUB_MATCH")); - // Close input files + // Close input files (freeing up space) : for the noConvolve case, we cannot close the + // inputs since we will use them for subtraction below. wait until later to do the work if (!noConvolve) { - if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) { - psError(PPSUB_ERR_IO, false, "Unable to close input files."); - success = false; - goto ESCAPE; - } + if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) { + psError(PPSUB_ERR_IO, false, "Unable to close input files."); + success = false; + goto ESCAPE; + } } if (!ppSubLowThreshold(data)) { @@ -165,5 +162,4 @@ } - if (!noConvolve) { if (!data->quality && !ppSubMakePSF(data)) { psError(psErrorCodeLast(), false, "Unable to generate PSF."); @@ -171,5 +167,5 @@ goto ESCAPE; } - } + // Now we've got a PSF, blow away detections in case they're confused with real output detections { @@ -197,9 +193,9 @@ // dumpout(config, "diff.2a.fits"); if (noConvolve) { - if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) { - psError(PPSUB_ERR_IO, false, "Unable to close input files."); - success = false; - goto ESCAPE; - } + if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) { + psError(PPSUB_ERR_IO, false, "Unable to close input files."); + success = false; + goto ESCAPE; + } } @@ -271,12 +267,12 @@ // Do Mask Stats { - pmFPAview *view = ppSubViewReadout(); // View to readout - if (!ppSubMaskStats(config, view,data->stats)) { - psError(psErrorCodeLast(), false, "Unable to generate mask statistics"); - success = false; - psFree(view); - goto ESCAPE; - } - psFree(view); + pmFPAview *view = ppSubViewReadout(); // View to readout + if (!ppSubMaskStats(config, view,data->stats)) { + psError(psErrorCodeLast(), false, "Unable to generate mask statistics"); + success = false; + psFree(view); + goto ESCAPE; + } + psFree(view); } // dumpout(config, "diff.4.fits"); @@ -344,5 +340,5 @@ } -ESCAPE: + ESCAPE: pmFPAfileDropInternal(config->files, "PSPHOT.BACKGND"); pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL"); Index: /branches/eam_branches/ipp-20111110/ppSub/src/ppSubMakePSF.c =================================================================== --- /branches/eam_branches/ipp-20111110/ppSub/src/ppSubMakePSF.c (revision 32693) +++ /branches/eam_branches/ipp-20111110/ppSub/src/ppSubMakePSF.c (revision 32694) @@ -46,23 +46,29 @@ bool noConvolve = psMetadataLookupBool(&mdok, recipe, "NOCONVOLVE"); // Do not use convolved images. - if (!noConvolve) { - printf("Using Convolved images because NOCONVOLVE is FALSE\n"); - if (reverse) { - minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); - minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.REF.CONV"); - } else { - minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); - minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.INPUT.CONV"); - } - } - else { - psWarning("Not using Convolved images because NOCONVOLVE is TRUE\n"); - if (reverse) { - minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF"); - minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.REF"); - } else { - minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT"); - minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.INPUT"); - } + if (noConvolve) { + // if we do not convolve, we need to copy the detections to the image for analysis + pmReadout *psfSourcesRO = NULL; // readout containing loaded sources for psf model + psWarning("Not using Convolved images because NOCONVOLVE is TRUE\n"); + if (reverse) { + minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF"); + minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.REF"); + psfSourcesRO = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.SOURCES"); + } else { + minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT"); + minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.INPUT"); + psfSourcesRO = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.SOURCES"); + } + psAssert (psfSourcesRO, "missing readout with sources"); + pmDetections *psfDetections = psMetadataLookupPtr(&mdok, psfSourcesRO->analysis, "PSPHOT.DETECTIONS"); + psMetadataAddPtr(minuend->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "Merged source list", psfDetections); + } else { + printf("Using Convolved images because NOCONVOLVE is FALSE\n"); + if (reverse) { + minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); + minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.REF.CONV"); + } else { + minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); + minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.INPUT.CONV"); + } } Index: /branches/eam_branches/ipp-20111110/ppSub/src/ppSubMatchPSFs.c =================================================================== --- /branches/eam_branches/ipp-20111110/ppSub/src/ppSubMatchPSFs.c (revision 32693) +++ /branches/eam_branches/ipp-20111110/ppSub/src/ppSubMatchPSFs.c (revision 32694) @@ -201,4 +201,6 @@ bool ppSubMatchPSFs(ppSubData *data) { + bool mdok = false; + psAssert(data, "Require processing data"); pmConfig *config = data->config; // Configuration @@ -208,4 +210,10 @@ psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim psAssert(recipe, "We checked this earlier, so it should be here."); + + bool noConvolve = psMetadataLookupBool(&mdok, recipe, "NOCONVOLVE"); // Do not use convolved images. + if (noConvolve) { + psWarning("not matching PSFs because NOCONVOLVE is TRUE\n"); + return true; + } pmFPAview *view = ppSubViewReadout(); // View to readout @@ -228,6 +236,4 @@ psFree(refConv); } - - bool mdok; // Status of MD lookup // Load pre-calculated kernel, if available Index: /branches/eam_branches/ipp-20111110/psphot/src/psphotSourceMatch.c =================================================================== --- /branches/eam_branches/ipp-20111110/psphot/src/psphotSourceMatch.c (revision 32693) +++ /branches/eam_branches/ipp-20111110/psphot/src/psphotSourceMatch.c (revision 32694) @@ -70,5 +70,5 @@ psVectorInit (foundSrc, 0); - psVector *foundObj = psVectorAlloc(sources->n, PS_TYPE_U8); + psVector *foundObj = psVectorAlloc(objects->n, PS_TYPE_U8); psVectorInit (foundObj, 0);