IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 32694


Ignore:
Timestamp:
Nov 17, 2011, 2:02:08 PM (15 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20111110
Files:
37 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20111110

  • branches/eam_branches/ipp-20111110/Nebulous-Server/bin/neb-admin

    r32524 r32694  
    534534{
    535535    my %destination;
     536    my %vol_id_list = ('ipp004.0' => 1,'ipp005.0' => 2, 'ipp006.0' => 3, 'ipp007.0' => 4,'ipp008.0' => 50,'ipp009.0' => 6,'ipp010.0' => 7,
     537                       'ipp011.0' => 8,'ipp012.0' => 15,'ipp013.0' => 16, 'ipp014.0' => 18,'ipp015.0' => 17,'ipp016.0' => 9,'ipp017.0' => 10,'ipp018.0' => 11,'ipp019.0' => 12,
     538                       'ipp020.0' => 52,'ipp021.0' => 14,'ipp023.0' => 19,'ipp024.0' => 20,'ipp025.0' => 32,'ipp026.0' => 21, 'ipp027.0' => 49,
     539                       'ipp028.0' => 23,'ipp029.0' => 24,'ipp030.0' => 25,'ipp031.0' => 26,'ipp032.0' => 27,'ipp033.0' => 28,'ipp034.0' => 29,
     540                       'ipp035.0' => 30,'ipp036.0' => 31,'ipp037.0' => 51,'ipp038.0' => 33,'ipp039.0' => 34,'ipp040.0' => 35,'ipp041.0' => 36,
     541                       'ipp042.0' => 37,'ipp043.0' => 38,'ipp044.0' => 39,'ipp045.0' => 40,'ipp046.0' => 41,'ipp047.0' => 42,'ipp048.0' => 43,
     542                       'ipp049.0' => 44,'ipp050.0' => 45,'ipp051.0' => 46,'ipp052.0' => 47,'ipp053.0' => 48,'ipp054.0' => 62,'ipp055.0' => 63,
     543                       'ipp056.0' => 64,'ipp057.0' => 65,'ipp058.0' => 66,'ipp059.0' => 67,'ipp060.0' => 68,'ipp061.0' => 69,'ipp062.0' => 70,
     544                       'ipp063.0' => 71,'ipp064.0' => 72,'ipp065.0' => 73,'ipp066.0' => 74);
    536545    %{ $destination{VOL} }  = ('ota01' => 7,       'ota02' => 8,       'ota03' => 3,       'ota04' => 3,       'ota05' => 4,       'ota06' => 4,
    537546                               'ota10' => 50,      'ota11' => 50,      'ota12' => 6,       'ota13' => 6,       'ota14' => 7,       'ota15' => 8,       'ota16' => 15,      'ota17' => 15,
    … …  
    542551                               'ota60' => 34,      'ota61' => 35,      'ota62' => 36,      'ota63' => 37,      'ota64' => 38,      'ota65' => 39,      'ota66' => 41,      'ota67' => 42,
    543552                               'ota71' => 43,      'ota72' => 44,      'ota73' => 45,      'ota74' => 46,      'ota75' => 47,      'ota76' => 48);
    544     %{ $destination{HOST} } = ('ota01' => 'ipp010','ota02' => 'ipp011','ota03' => 'ipp006','ota04' => 'ipp006','ota05' => 'ipp007','ota06' => 'ipp007',
    545                                'ota10' => 'ipp008','ota11' => 'ipp008','ota12' => 'ipp009','ota13' => 'ipp009','ota14' => 'ipp010','ota15' => 'ipp011','ota16' => 'ipp012','ota17' => 'ipp012',
    546                                'ota20' => 'ipp013','ota21' => 'ipp014','ota22' => 'ipp014','ota23' => 'ipp015','ota24' => 'ipp016','ota25' => 'ipp016','ota26' => 'ipp017','ota27' => 'ipp018',
    547                                'ota30' => 'ipp019','ota31' => 'ipp020','ota32' => 'ipp021','ota33' => 'ipp021','ota34' => 'ipp023','ota35' => 'ipp013','ota36' => 'ipp024','ota37' => 'ipp019',
     553    %{ $destination{HOST} } = ('ota01' => 'ipp010','ota02' => 'ipp011','ota03' => 'ipp006','ota04' => 'ipp054','ota05' => 'ipp007','ota06' => 'ipp055',
     554                               'ota10' => 'ipp008','ota11' => 'ipp056','ota12' => 'ipp009','ota13' => 'ipp057','ota14' => 'ipp058','ota15' => 'ipp059','ota16' => 'ipp012','ota17' => 'ipp060',
     555                               'ota20' => 'ipp013','ota21' => 'ipp014','ota22' => 'ipp061','ota23' => 'ipp015','ota24' => 'ipp016','ota25' => 'ipp062','ota26' => 'ipp064','ota27' => 'ipp065',
     556                               'ota30' => 'ipp066','ota31' => 'ipp020','ota32' => 'ipp063','ota33' => 'ipp021','ota34' => 'ipp023','ota35' => 'ipp013','ota36' => 'ipp024','ota37' => 'ipp019',
    548557                               'ota40' => 'ipp025','ota41' => 'ipp018','ota42' => 'ipp017','ota43' => 'ipp015','ota44' => 'ipp027','ota45' => 'ipp028','ota46' => 'ipp029','ota47' => 'ipp030',
    549558                               'ota50' => 'ipp031','ota51' => 'ipp032','ota52' => 'ipp033','ota53' => 'ipp034','ota54' => 'ipp035','ota55' => 'ipp036','ota56' => 'ipp037','ota57' => 'ipp038',
    550559                               'ota60' => 'ipp039','ota61' => 'ipp040','ota62' => 'ipp041','ota63' => 'ipp042','ota64' => 'ipp043','ota65' => 'ipp044','ota66' => 'ipp046','ota67' => 'ipp047',
    551560                               'ota71' => 'ipp048','ota72' => 'ipp049','ota73' => 'ipp050','ota74' => 'ipp051','ota75' => 'ipp052','ota76' => 'ipp053');
    552     %{ $destination{NAME} } = ('ota01' => 'ipp010.0','ota02' => 'ipp011.0','ota03' => 'ipp006.0','ota04' => 'ipp006.0','ota05' => 'ipp007.0','ota06' => 'ipp007.0',
    553                                'ota10' => 'ipp008.0','ota11' => 'ipp008.0','ota12' => 'ipp009.0','ota13' => 'ipp009.0','ota14' => 'ipp010.0','ota15' => 'ipp011.0','ota16' => 'ipp012.0','ota17' => 'ipp012.0',
    554                                'ota20' => 'ipp013.0','ota21' => 'ipp014.0','ota22' => 'ipp014.0','ota23' => 'ipp015.0','ota24' => 'ipp016.0','ota25' => 'ipp016.0','ota26' => 'ipp017.0','ota27' => 'ipp018.0',
    555                                'ota30' => 'ipp019.0','ota31' => 'ipp020.0','ota32' => 'ipp021.0','ota33' => 'ipp021.0','ota34' => 'ipp023.0','ota35' => 'ipp013.0','ota36' => 'ipp024.0','ota37' => 'ipp019.0',
     561    %{ $destination{NAME} } = ('ota01' => 'ipp010.0','ota02' => 'ipp011.0','ota03' => 'ipp006.0','ota04' => 'ipp054.0','ota05' => 'ipp007.0','ota06' => 'ipp055.0',
     562                               'ota10' => 'ipp008.0','ota11' => 'ipp056.0','ota12' => 'ipp009.0','ota13' => 'ipp057.0','ota14' => 'ipp058.0','ota15' => 'ipp059.0','ota16' => 'ipp012.0','ota17' => 'ipp060.0',
     563                               'ota20' => 'ipp013.0','ota21' => 'ipp014.0','ota22' => 'ipp061.0','ota23' => 'ipp015.0','ota24' => 'ipp016.0','ota25' => 'ipp062.0','ota26' => 'ipp064.0','ota27' => 'ipp065.0',
     564                               'ota30' => 'ipp066.0','ota31' => 'ipp020.0','ota32' => 'ipp063.0','ota33' => 'ipp021.0','ota34' => 'ipp023.0','ota35' => 'ipp013.0','ota36' => 'ipp024.0','ota37' => 'ipp019.0',
    556565                               'ota40' => 'ipp025.0','ota41' => 'ipp018.0','ota42' => 'ipp017.0','ota43' => 'ipp015.0','ota44' => 'ipp027.0','ota45' => 'ipp028.0','ota46' => 'ipp029.0','ota47' => 'ipp030.0',
    557566                               'ota50' => 'ipp031.0','ota51' => 'ipp032.0','ota52' => 'ipp033.0','ota53' => 'ipp034.0','ota54' => 'ipp035.0','ota55' => 'ipp036.0','ota56' => 'ipp037.0','ota57' => 'ipp038.0',
    558567                               'ota60' => 'ipp039.0','ota61' => 'ipp040.0','ota62' => 'ipp041.0','ota63' => 'ipp042.0','ota64' => 'ipp043.0','ota65' => 'ipp044.0','ota66' => 'ipp046.0','ota67' => 'ipp047.0',
    559568                               'ota71' => 'ipp048.0','ota72' => 'ipp049.0','ota73' => 'ipp050.0','ota74' => 'ipp051.0','ota75' => 'ipp052.0','ota76' => 'ipp053.0');
     569    foreach my $k (keys %{ $destination{NAME} }) {
     570        $destination{VOL}{$k} = $vol_id_list{$destination{NAME}{$k}};
     571    }
    560572    my %blocked_volume = ('ipp027' => 1,
    561573                          'ipp037' => 1,

  • branches/eam_branches/ipp-20111110/PS-IPP-Config/lib/PS/IPP/Config.pm

    r32506 r32694  
    398398    return $self->{nebulous} if defined $self->{nebulous}; # Already started
    399399
    400     my $server = metadataLookupStr( $self->{_siteConfig}, 'NEB_SERVER' ); # Nebulous server
     400    # For consistency with the rest of the IPP, don't use site.config
     401    # to set NEB_SERVER use environment variable.
     402    # my $server = metadataLookupStr( $self->{_siteConfig}, 'NEB_SERVER' ); # Nebulous server
     403    my $server = $ENV{NEB_SERVER};
    401404    unless (defined $server) {
    402405        carp "Unable to find NEB_SERVER in camera configuration file.";

  • branches/eam_branches/ipp-20111110/ippMonitor

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20111110/ippScripts/scripts/diff_skycell.pl

    r32637 r32694  
    289289    }
    290290
    291     if (!$ipprc->file_exists($configuration)) {
    292        print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n";
    293        $configuration = undef;
    294     }
    295291    # use WARPSTATS if we're updating, as we don't care about the new stuff from the STACK and DIFF STATS recipes
    296292    $recipe_ppstats = 'WARPSTATS';

  • branches/eam_branches/ipp-20111110/ippScripts/scripts/dist_defineruns.pl

    r30489 r32694  
    8888    push @stages, $stage;
    8989} else {
    90     @stages = qw( raw chip chip_bg camera fake warp warp_bg diff stack SSdiff sky);
     90    @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky);
    9191}
    9292

  • branches/eam_branches/ipp-20111110/ippScripts/scripts/ipp_apply_burntool_single.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20111110/ippScripts/scripts/magic_destreak.pl

    r32537 r32694  
    221221        my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells";
    222222        $command .= " $skycell_args";
     223        $command .= " -dbname $dbname" if $dbname;
    223224        my @diff_components;
    224225        if ($mismatched_tess) {

  • branches/eam_branches/ipp-20111110/ippScripts/scripts/magic_destreak_revert.pl

    r32356 r32694  
    511511    my $command = "$magicdstool -updaterun";
    512512    $command   .= " -magic_ds_id $magic_ds_id";
    513     $command   .= " -set_state failed_revert";
     513    if ($run_state ne 'update') {
     514        $command .= " -set_state failed_revert";
     515    } else {
     516        $command .= " -set_state failed_revert_ud";
     517    }
    514518    $command   .= " -dbname $dbname" if defined $dbname;
    515519

  • branches/eam_branches/ipp-20111110/ippTasks/diskbalance.pro

    r32390 r32694  
    318318$TARGET_LIMIT       = 500
    319319$TARGET_OFFSET      = 0
    320 $TARGET_TARGET      = "SOURCE"
     320$TARGET_TARGET      = SOURCE
    321321# Select Nebulous objects which should be shuffled
    322322task           target.load
    … …  
    423423    # Fix this with multihost restriction when possible.
    424424    if ("$TARGET_TARGET" == "DESTINATION")
    425        set.host.for.target $DEST_HOST
     425       set.host.for.balance $DEST_HOST
    426426    else
    427        set.host.for.target $SOURCE_HOST
     427       set.host.for.balance $SOURCE_HOST
    428428    end
    429429   

  • branches/eam_branches/ipp-20111110/ippTasks/ipphosts.mhpcc.config

    r32446 r32694  
    66# randomly.  the only exception is ipp004, which is the DVO db host
    77
    8 # NOTE: wave 1 machines have 10TB; wave 2 & 3 machines have 20TB
    9 # for balance, use 1 host per 10 TB in the partition
     8# I've allocated hosts based on disk size.
     9# 40TB => 2 units of skycells
     10# 20TB => 1 unit of skycells
     11# Each host gets one OTA/chip of data.
     12# These hosts are skipped:
     13# ipp004 : dvo
     14# ipp005 : dvo
     15# ipp026 : crashes
     16# ipp029 : crashes
     17# ipp045 : reference cat
    1018ipphosts METADATA
    1119  camera STR skycell
    12   count S32  76
    13   sky00 STR  ipp005
     20  count S32  86
     21  sky00 STR  ipp006
    1422  sky01 STR  ipp006
    1523  sky02 STR  ipp007
    16   sky03 STR  ipp008
    17   sky04 STR  ipp009
    18   sky05 STR  ipp010
    19   sky06 STR  ipp011
    20   sky07 STR  ipp012
    21   sky08 STR  ipp012
    22   sky09 STR  ipp013
    23   sky10 STR  ipp014
    24   sky11 STR  ipp014
    25   sky12 STR  ipp015
    26   sky13 STR  ipp016
    27   sky14 STR  ipp016
    28   sky15 STR  ipp017
    29   sky16 STR  ipp018
    30   sky17 STR  ipp019
    31   sky18 STR  ipp020
    32   sky19 STR  ipp021
    33   sky20 STR  ipp023
    34   sky21 STR  ipp024
    35   sky22 STR  ipp025
    36   sky23 STR  ipp026
    37   sky24 STR  ipp027
    38   sky25 STR  ipp027
    39   sky26 STR  ipp028
    40   sky27 STR  ipp028
    41   sky28 STR  ipp029
    42   sky29 STR  ipp029
    43   sky30 STR  ipp030
    44   sky31 STR  ipp030
    45   sky32 STR  ipp031
    46   sky33 STR  ipp031
    47   sky34 STR  ipp032
    48   sky35 STR  ipp032
    49   sky36 STR  ipp033
    50   sky37 STR  ipp033
    51   sky38 STR  ipp034
    52   sky39 STR  ipp034
    53   sky40 STR  ipp035
    54   sky41 STR  ipp035
    55   sky42 STR  ipp036
     24  sky03 STR  ipp007
     25  sky04 STR  ipp008
     26  sky05 STR  ipp008
     27  sky06 STR  ipp009
     28  sky07 STR  ipp009
     29  sky08 STR  ipp010
     30  sky09 STR  ipp010
     31  sky10 STR  ipp011
     32  sky11 STR  ipp011
     33  sky12 STR  ipp012
     34  sky13 STR  ipp012
     35  sky14 STR  ipp013
     36  sky15 STR  ipp013
     37  sky16 STR  ipp014
     38  sky17 STR  ipp014
     39  sky18 STR  ipp015
     40  sky19 STR  ipp015
     41  sky20 STR  ipp016
     42  sky21 STR  ipp016
     43  sky22 STR  ipp017
     44  sky23 STR  ipp017
     45  sky24 STR  ipp018
     46  sky25 STR  ipp018
     47  sky26 STR  ipp019
     48  sky27 STR  ipp019
     49  sky28 STR  ipp020
     50  sky29 STR  ipp020
     51  sky30 STR  ipp021
     52  sky31 STR  ipp021
     53  sky32 STR  ipp023
     54  sky33 STR  ipp024
     55  sky34 STR  ipp025
     56  sky35 STR  ipp027
     57  sky36 STR  ipp028
     58  sky37 STR  ipp030
     59  sky38 STR  ipp031
     60  sky39 STR  ipp032
     61  sky40 STR  ipp033
     62  sky41 STR  ipp034
     63  sky42 STR  ipp035
    5664  sky43 STR  ipp036
    5765  sky44 STR  ipp037
    58   sky45 STR  ipp037
    59   sky46 STR  ipp038
    60   sky47 STR  ipp038
    61   sky48 STR  ipp039
    62   sky49 STR  ipp039
    63   sky50 STR  ipp040
    64   sky51 STR  ipp040
    65   sky52 STR  ipp041
    66   sky53 STR  ipp041
    67   sky54 STR  ipp042
    68   sky55 STR  ipp042
    69   sky56 STR  ipp043
    70   sky57 STR  ipp043
    71   sky58 STR  ipp044
    72   sky59 STR  ipp044
    73   sky60 STR  ipp046
    74   sky61 STR  ipp046
    75   sky62 STR  ipp047
    76   sky63 STR  ipp047
    77   sky64 STR  ipp048
    78   sky65 STR  ipp048
    79   sky66 STR  ipp049
    80   sky67 STR  ipp049
    81   sky68 STR  ipp050
    82   sky69 STR  ipp050
    83   sky70 STR  ipp051
    84   sky71 STR  ipp051
    85   sky72 STR  ipp052
    86   sky73 STR  ipp052
    87   sky74 STR  ipp053
    88   sky75 STR  ipp053
    89 END
    90 
    91 # we have 18 10TB nodes and 30 20TB nodes (ipp037 is used for dvo)
    92 # 1 x W1 + 1 x W2 + 1 x W3 + 12 more nodes...
    93 
     66  sky45 STR  ipp038
     67  sky46 STR  ipp039
     68  sky47 STR  ipp040
     69  sky48 STR  ipp041
     70  sky49 STR  ipp042
     71  sky50 STR  ipp043
     72  sky51 STR  ipp044
     73  sky52 STR  ipp046
     74  sky53 STR  ipp047
     75  sky54 STR  ipp048
     76  sky55 STR  ipp049
     77  sky56 STR  ipp050
     78  sky57 STR  ipp051
     79  sky58 STR  ipp052
     80  sky59 STR  ipp053
     81  sky60 STR  ipp054
     82  sky61 STR  ipp054
     83  sky62 STR  ipp055
     84  sky63 STR  ipp055
     85  sky64 STR  ipp056
     86  sky65 STR  ipp056
     87  sky66 STR  ipp057
     88  sky67 STR  ipp057
     89  sky68 STR  ipp058
     90  sky69 STR  ipp058
     91  sky70 STR  ipp059
     92  sky71 STR  ipp059
     93  sky72 STR  ipp060
     94  sky73 STR  ipp060
     95  sky74 STR  ipp061
     96  sky75 STR  ipp061
     97  sky76 STR  ipp062
     98  sky77 STR  ipp062
     99  sky78 STR  ipp063
     100  sky79 STR  ipp063
     101  sky80 STR  ipp064
     102  sky81 STR  ipp064
     103  sky82 STR  ipp065
     104  sky83 STR  ipp065
     105  sky84 STR  ipp066
     106  sky85 STR  ipp066
     107END
     108
     109# These are ordered oddly to prevent excessive targetting shuffles.
    94110ipphosts METADATA
    95111  camera STR GPC1
    … …  
    98114  XY02  STR  ipp011
    99115  XY03  STR  ipp006
    100   XY04  STR  ipp006
     116  XY04  STR  ipp054
    101117
    102118  XY05  STR  ipp007
    103   XY06  STR  ipp007
     119  XY06  STR  ipp055
    104120  XY10  STR  ipp008
    105   XY11  STR  ipp008
     121  XY11  STR  ipp056
    106122
    107123  XY12  STR  ipp009
    108   XY13  STR  ipp009
    109   XY14  STR  ipp010
    110   XY15  STR  ipp011
     124  XY13  STR  ipp057
     125  XY14  STR  ipp058
     126  XY15  STR  ipp059
    111127
    112128  XY16  STR  ipp012
    113   XY17  STR  ipp012
     129  XY17  STR  ipp060
    114130  XY20  STR  ipp013
    115131  XY21  STR  ipp014
    116132
    117   XY22  STR  ipp014
     133  XY22  STR  ipp061
    118134  XY23  STR  ipp015
    119135  XY24  STR  ipp016
    120   XY25  STR  ipp016
    121 
    122   XY26  STR  ipp017
    123   XY27  STR  ipp018
    124   XY30  STR  ipp019
     136  XY25  STR  ipp062
     137
     138  XY26  STR  ipp064
     139  XY27  STR  ipp065
     140  XY30  STR  ipp066
    125141  XY31  STR  ipp020
    126142
    127   XY32  STR  ipp021
     143  XY32  STR  ipp063
    128144  XY33  STR  ipp021
    129145  XY34  STR  ipp023
    … …  
    177193  ota02  STR  ipp011
    178194  ota03  STR  ipp006
    179   ota04  STR  ipp006
    180                    
     195  ota04  STR  ipp054
     196
    181197  ota05  STR  ipp007
    182   ota06  STR  ipp007
     198  ota06  STR  ipp055
    183199  ota10  STR  ipp008
    184   ota11  STR  ipp008
    185                    
     200  ota11  STR  ipp056
     201
    186202  ota12  STR  ipp009
    187   ota13  STR  ipp009
    188   ota14  STR  ipp010
    189   ota15  STR  ipp011
    190                    
     203  ota13  STR  ipp057
     204  ota14  STR  ipp058
     205  ota15  STR  ipp059
     206
    191207  ota16  STR  ipp012
    192   ota17  STR  ipp012
     208  ota17  STR  ipp060
    193209  ota20  STR  ipp013
    194210  ota21  STR  ipp014
    195                    
    196   ota22  STR  ipp014
     211
     212  ota22  STR  ipp061
    197213  ota23  STR  ipp015
    198214  ota24  STR  ipp016
    199   ota25  STR  ipp016
    200                    
    201   ota26  STR  ipp017
    202   ota27  STR  ipp018
    203   ota30  STR  ipp019
     215  ota25  STR  ipp062
     216
     217  ota26  STR  ipp064
     218  ota27  STR  ipp065
     219  ota30  STR  ipp066
    204220  ota31  STR  ipp020
    205                    
    206   ota32  STR  ipp021
     221
     222  ota32  STR  ipp063
    207223  ota33  STR  ipp021
    208224  ota34  STR  ipp023
    209225  ota35  STR  ipp013
    210                    
     226
    211227  ota36  STR  ipp024
    212228  ota37  STR  ipp019
    213229  ota40  STR  ipp025
    214230  ota41  STR  ipp018
    215                    
     231
    216232  ota42  STR  ipp017
    217233  ota43  STR  ipp015
    218234  ota44  STR  ipp027
    219235  ota45  STR  ipp028
    220                    
     236
    221237  ota46  STR  ipp029
    222238  ota47  STR  ipp030
    223239  ota50  STR  ipp031
    224240  ota51  STR  ipp032
    225                    
     241
    226242  ota52  STR  ipp033
    227243  ota53  STR  ipp034
    228244  ota54  STR  ipp035
    229245  ota55  STR  ipp036
    230                    
     246
    231247  ota56  STR  ipp037
    232248  ota57  STR  ipp038
    233249  ota60  STR  ipp039
    234250  ota61  STR  ipp040
    235                    
     251
    236252  ota62  STR  ipp041
    237253  ota63  STR  ipp042
    238254  ota64  STR  ipp043
    239255  ota65  STR  ipp044
    240                    
     256
    241257  ota66  STR  ipp046
    242258  ota67  STR  ipp047
    243259  ota71  STR  ipp048
    244260  ota72  STR  ipp049
    245                    
     261
    246262  ota73  STR  ipp050
    247263  ota74  STR  ipp051
    … …  
    288304  XY02  STR  ipp011
    289305  XY03  STR  ipp006
    290   XY04  STR  ipp006
    291                    
     306  XY04  STR  ipp054
     307
    292308  XY05  STR  ipp007
    293   XY06  STR  ipp007
     309  XY06  STR  ipp055
    294310  XY10  STR  ipp008
    295   XY11  STR  ipp008
    296                    
     311  XY11  STR  ipp056
     312
    297313  XY12  STR  ipp009
    298   XY13  STR  ipp009
    299   XY14  STR  ipp010
    300   XY15  STR  ipp011
    301                    
     314  XY13  STR  ipp057
     315  XY14  STR  ipp058
     316  XY15  STR  ipp059
     317
    302318  XY16  STR  ipp012
    303   XY17  STR  ipp012
     319  XY17  STR  ipp060
    304320  XY20  STR  ipp013
    305321  XY21  STR  ipp014
    306                    
    307   XY22  STR  ipp014
     322
     323  XY22  STR  ipp061
    308324  XY23  STR  ipp015
    309325  XY24  STR  ipp016
    310   XY25  STR  ipp016
    311                    
    312   XY26  STR  ipp017
    313   XY27  STR  ipp018
    314   XY30  STR  ipp019
     326  XY25  STR  ipp062
     327
     328  XY26  STR  ipp064
     329  XY27  STR  ipp065
     330  XY30  STR  ipp066
    315331  XY31  STR  ipp020
    316                    
    317   XY32  STR  ipp021
     332
     333  XY32  STR  ipp063
    318334  XY33  STR  ipp021
    319335  XY34  STR  ipp023
    320336  XY35  STR  ipp013
    321                    
     337
    322338  XY36  STR  ipp024
    323339  XY37  STR  ipp019
    324340  XY40  STR  ipp025
    325341  XY41  STR  ipp018
    326                    
     342
    327343  XY42  STR  ipp017
    328344  XY43  STR  ipp015
    329345  XY44  STR  ipp027
    330346  XY45  STR  ipp028
    331                    
     347
    332348  XY46  STR  ipp029
    333349  XY47  STR  ipp030
    334350  XY50  STR  ipp031
    335351  XY51  STR  ipp032
    336                    
     352
    337353  XY52  STR  ipp033
    338354  XY53  STR  ipp034
    339355  XY54  STR  ipp035
    340356  XY55  STR  ipp036
    341                    
     357
    342358  XY56  STR  ipp037
    343359  XY57  STR  ipp038
    344360  XY60  STR  ipp039
    345361  XY61  STR  ipp040
    346                    
     362
    347363  XY62  STR  ipp041
    348364  XY63  STR  ipp042
    349365  XY64  STR  ipp043
    350366  XY65  STR  ipp044
    351                    
     367
    352368  XY66  STR  ipp046
    353369  XY67  STR  ipp047
    354370  XY71  STR  ipp048
    355371  XY72  STR  ipp049
    356                    
     372
    357373  XY73  STR  ipp050
    358374  XY74  STR  ipp051
    … …  
    363379ipphosts METADATA
    364380  camera STR dist_skycell
    365   count S32  76
    366   sky00 STR  ipp005
     381  count S32  86
     382  sky00 STR  ipp006
    367383  sky01 STR  ipp006
    368384  sky02 STR  ipp007
    369   sky03 STR  ipp008
    370   sky04 STR  ipp009
    371   sky05 STR  ipp010
    372   sky06 STR  ipp011
    373   sky07 STR  ipp012
    374   sky08 STR  ipp012
    375   sky09 STR  ipp013
    376   sky10 STR  ipp014
    377   sky11 STR  ipp014
    378   sky12 STR  ipp015
    379   sky13 STR  ipp016
    380   sky14 STR  ipp016
    381   sky15 STR  ipp017
    382   sky16 STR  ipp018
    383   sky17 STR  ipp019
    384   sky18 STR  ipp020
    385   sky19 STR  ipp021
    386   sky20 STR  ipp023
    387   sky21 STR  ipp024
    388   sky22 STR  ipp025
    389   sky23 STR  ipp026
    390   sky24 STR  ipp027
    391   sky25 STR  ipp027
    392   sky26 STR  ipp028
    393   sky27 STR  ipp028
    394   sky28 STR  ipp029
    395   sky29 STR  ipp029
    396   sky30 STR  ipp030
    397   sky31 STR  ipp030
    398   sky32 STR  ipp031
    399   sky33 STR  ipp031
    400   sky34 STR  ipp032
    401   sky35 STR  ipp032
    402   sky36 STR  ipp033
    403   sky37 STR  ipp033
    404   sky38 STR  ipp034
    405   sky39 STR  ipp034
    406   sky40 STR  ipp035
    407   sky41 STR  ipp035
    408   sky42 STR  ipp036
     385  sky03 STR  ipp007
     386  sky04 STR  ipp008
     387  sky05 STR  ipp008
     388  sky06 STR  ipp009
     389  sky07 STR  ipp009
     390  sky08 STR  ipp010
     391  sky09 STR  ipp010
     392  sky10 STR  ipp011
     393  sky11 STR  ipp011
     394  sky12 STR  ipp012
     395  sky13 STR  ipp012
     396  sky14 STR  ipp013
     397  sky15 STR  ipp013
     398  sky16 STR  ipp014
     399  sky17 STR  ipp014
     400  sky18 STR  ipp015
     401  sky19 STR  ipp015
     402  sky20 STR  ipp016
     403  sky21 STR  ipp016
     404  sky22 STR  ipp017
     405  sky23 STR  ipp017
     406  sky24 STR  ipp018
     407  sky25 STR  ipp018
     408  sky26 STR  ipp019
     409  sky27 STR  ipp019
     410  sky28 STR  ipp020
     411  sky29 STR  ipp020
     412  sky30 STR  ipp021
     413  sky31 STR  ipp021
     414  sky32 STR  ipp023
     415  sky33 STR  ipp024
     416  sky34 STR  ipp025
     417  sky35 STR  ipp027
     418  sky36 STR  ipp028
     419  sky37 STR  ipp030
     420  sky38 STR  ipp031
     421  sky39 STR  ipp032
     422  sky40 STR  ipp033
     423  sky41 STR  ipp034
     424  sky42 STR  ipp035
    409425  sky43 STR  ipp036
    410426  sky44 STR  ipp037
    411   sky45 STR  ipp037
    412   sky46 STR  ipp038
    413   sky47 STR  ipp038
    414   sky48 STR  ipp039
    415   sky49 STR  ipp039
    416   sky50 STR  ipp040
    417   sky51 STR  ipp040
    418   sky52 STR  ipp041
    419   sky53 STR  ipp041
    420   sky54 STR  ipp042
    421   sky55 STR  ipp042
    422   sky56 STR  ipp043
    423   sky57 STR  ipp043
    424   sky58 STR  ipp044
    425   sky59 STR  ipp044
    426   sky60 STR  ipp046
    427   sky61 STR  ipp046
    428   sky62 STR  ipp047
    429   sky63 STR  ipp047
    430   sky64 STR  ipp048
    431   sky65 STR  ipp048
    432   sky66 STR  ipp049
    433   sky67 STR  ipp049
    434   sky68 STR  ipp050
    435   sky69 STR  ipp050
    436   sky70 STR  ipp051
    437   sky71 STR  ipp051
    438   sky72 STR  ipp052
    439   sky73 STR  ipp052
    440   sky74 STR  ipp053
    441   sky75 STR  ipp053
     427  sky45 STR  ipp038
     428  sky46 STR  ipp039
     429  sky47 STR  ipp040
     430  sky48 STR  ipp041
     431  sky49 STR  ipp042
     432  sky50 STR  ipp043
     433  sky51 STR  ipp044
     434  sky52 STR  ipp046
     435  sky53 STR  ipp047
     436  sky54 STR  ipp048
     437  sky55 STR  ipp049
     438  sky56 STR  ipp050
     439  sky57 STR  ipp051
     440  sky58 STR  ipp052
     441  sky59 STR  ipp053
     442  sky60 STR  ipp054
     443  sky61 STR  ipp054
     444  sky62 STR  ipp055
     445  sky63 STR  ipp055
     446  sky64 STR  ipp056
     447  sky65 STR  ipp056
     448  sky66 STR  ipp057
     449  sky67 STR  ipp057
     450  sky68 STR  ipp058
     451  sky69 STR  ipp058
     452  sky70 STR  ipp059
     453  sky71 STR  ipp059
     454  sky72 STR  ipp060
     455  sky73 STR  ipp060
     456  sky74 STR  ipp061
     457  sky75 STR  ipp061
     458  sky76 STR  ipp062
     459  sky77 STR  ipp062
     460  sky78 STR  ipp063
     461  sky79 STR  ipp063
     462  sky80 STR  ipp064
     463  sky81 STR  ipp064
     464  sky82 STR  ipp065
     465  sky83 STR  ipp065
     466  sky84 STR  ipp066
     467  sky85 STR  ipp066
    442468END
    443469

  • branches/eam_branches/ipp-20111110/ippTasks/survey.pro

    r32290 r32694  
    302302# user functions to manipulate dist labels
    303303macro survey.add.dist
    304   if ($0 != 3)
    305     echo "USAGE: survey.add.dist (label) (workdir)"
     304  if ($0 != 4)
     305    echo "USAGE: survey.add.dist (label) (workdir) (muggle)"
    306306    break
    307307  end
    308308  book newpage SURVEY_DIST $1
    309309  book setword SURVEY_DIST $1 WORKDIR $2
     310  book setword SURVEY_DIST $1 MUGGLE $3
    310311  book setword SURVEY_DIST $1 STATE PENDING
    311312end
    … …  
    329330# user functions to manipulate chip_bg labels
    330331macro survey.add.chip.bg
    331   if ($0 != 3)
    332     echo "USAGE: survey.add.chip.bg (label) (dist_group)"
     332  if ($0 != 4)
     333    echo "USAGE: survey.add.chip.bg (label) (dist_group) (muggle)"
    333334    break
    334335  end
    335336  book newpage SURVEY_CHIP_BG $1
    336337  book setword SURVEY_CHIP_BG $1 DIST_GROUP $2
     338  book setword SURVEY_CHIP_BG $1 MUGGLE $3
    337339  book setword SURVEY_CHIP_BG $1 STATE PENDING
    338340end
    … …  
    356358# user functions to manipulate warp_bg labels
    357359macro survey.add.warp.bg
    358   if ($0 != 3)
    359     echo "USAGE: survey.add.warp.bg (label) (dist_group)"
     360  if ($0 != 4)
     361    echo "USAGE: survey.add.warp.bg (label) (dist_group) (muggle)"
    360362    break
    361363  end
    362364  book newpage SURVEY_WARP_BG $1
    363365  book setword SURVEY_WARP_BG $1 DIST_GROUP $2
     366  book setword SURVEY_WARP_BG $1 MUGGLE $3
    364367  book setword SURVEY_WARP_BG $1 STATE PENDING
    365368end
    … …  
    984987    $workdir = $workdir_base/$label/$year/$month/$day
    985988
     989    book getword SURVEY_DIST $label MUGGLE -var muggle
     990
    986991    book setword SURVEY_DIST $label STATE DONE
    987992 
    988993    # note workdir is set by the script based on site.config
    989994    $run = dist_defineruns.pl --label $label --workdir $workdir
     995    if ($muggle != 0)
     996        $run = $run --no_magic
     997    end
    990998
    991999    if ($DB:n == 0)
    … …  
    10581066    book setword SURVEY_CHIP_BG $label STATE DONE
    10591067    book getword SURVEY_CHIP_BG $label DIST_GROUP -var dist_group
     1068    book getword SURVEY_CHIP_BG $label MUGGLE -var muggle
    10601069 
    10611070    $run = bgtool -definechip -label $label -set_dist_group $dist_group
     1071    if ($muggle == 0)
     1072        $run = $run -destreaked
     1073    end
    10621074
    10631075    if ($DB:n == 0)
    … …  
    10681080    end
    10691081   
    1070     # echo $run
     1082    if ($VERBOSE > 1)
     1083        echo $run
     1084    end
    10711085    command $run
    10721086  end
    … …  
    11281142    book setword SURVEY_WARP_BG $label STATE DONE
    11291143    book getword SURVEY_WARP_BG $label DIST_GROUP -var dist_group
     1144    book getword SURVEY_WARP_BG $label MUGGLE -var muggle
    11301145 
    11311146    $run = bgtool -definewarp -warp_label $label -set_dist_group $dist_group
     1147    if ($muggle == 0)
     1148        $run = $run -destreaked
     1149    end
     1150
    11321151
    11331152    if ($DB:n == 0)
    … …  
    11381157    end
    11391158   
    1140     # echo $run
     1159    if ($VERBOSE > 1)
     1160        echo $run
     1161    end
    11411162    command $run
    11421163  end
    … …  
    12581279    book npages SURVEY_STATICSKYSINGLE -var N
    12591280    if ($N == 0)
    1260       echo "No STATICSKYSINGLE labels for processing"
     1281       # echo "No STATICSKYSINGLE labels for processing"
    12611282       break
    12621283    endif

  • branches/eam_branches/ipp-20111110/ippToPsps/jython/ipptopspsdb.py

    r32580 r32694  
    125125
    126126        self.logger.debug("Found %d processed but-not-on-datastore items" % len(ids))
    127 
    128         return ids
    129 
    130     '''
    131     Returns a list of processed batch IDs that either not yet loaded to the ODM or not merge_worthy or not merged
    132     NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to loading to ODM if there is a known problem with the batch
    133     '''
    134     def getAllUnfinishedBatchIDs(self, epoch, dvoGpc1Label, batchType):
    135 
    136         sql = "SELECT batch_id \
    137                FROM batch \
    138                WHERE timestamp > '" + epoch + "' \
    139                AND dvo_db = '" + dvoGpc1Label + "' \
    140                AND loaded_to_datastore = 1 \
    141                AND batch_type = '" + batchType + "' \
    142                AND deleted_datastore = 0 \
    143                AND loaded_to_ODM != -1 \
    144                AND merge_worthy != -1 \
    145                AND merged = 0"
    146 
    147         ids = []
    148         try:
    149             rs = self.executeQuery(sql)
    150             while (rs.next()):
    151                 ids.append(rs.getInt(1))
    152         except:
    153             self.logger.exception("Can't query for unloaded batch ids in ipptopsps Db")
    154 
    155         rs.close()
    156 
    157         self.logger.debug("Found %d unloaded items" % len(ids))
    158127
    159128        return ids

  • branches/eam_branches/ipp-20111110/ippToPsps/jython/pollOdm.py

    r32542 r32694  
    1717Method to actually poll ODM
    1818'''
    19 def poll(batchType):
     19def poll(batchType, stage):
    2020 
     21    if stage == "unloaded":
     22        ids = ippToPspsDb.getUnloadedBatchIDs(EPOCH, DVOLABEL, batchType)
     23    elif stage == "unmergeworthy":
     24        ids = ippToPspsDb.getUnmergeWorthyBatchIDs(EPOCH, DVOLABEL, batchType)
     25    elif stage == "unmerged":
     26        ids = ippToPspsDb.getUnmergedBatchIDs(EPOCH, DVOLABEL, batchType)
    2127
    22     lookingForUnloaded = 0
    23     lookingForUnmergeworthy = 0
    24     lookingForUnmerged = 0
    25     lookingForAll = 0
    26 
    27     if TYPE == "unloaded":
    28         lookingForUnloaded = 1
    29         ids = ippToPspsDb.getUnloadedBatchIDs(EPOCH, DVOLABEL, batchType)
    30     elif TYPE == "unmergeworthy":
    31         lookingForUnmergeworthy = 1
    32         ids = ippToPspsDb.getUnmergeWorthyBatchIDs(EPOCH, DVOLABEL, batchType)
    33     elif TYPE == "unmerged":
    34         lookingForUnmerged = 1
    35         ids = ippToPspsDb.getUnmergedBatchIDs(EPOCH, DVOLABEL, batchType)
    36     elif TYPE == "unfinished":
    37         lookingForAll = 1
    38         ids = ippToPspsDb.getAllUnfinishedBatchIDs(EPOCH, DVOLABEL, batchType)
    39 
    40     logger.infoTitle("Checking %d %s %s batches" % (len(ids), TYPE, batchType))
     28    logger.infoTitle("Checking %d %s %s batches" % (len(ids), stage, batchType))
    4129
    4230    results = {}
    … …  
    5341           ippToPspsDb.updateOdmStatus(id, results)
    5442
    55            if lookingForUnloaded:
     43           if stage == "unloaded":
    5644               if results['LOADEDTOODM'] == 1: newLoaded = newLoaded + 1
    5745               if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1
    5846               if results['MERGED'] == 1: newMerged = newMerged + 1
    5947
    60            elif lookingForUnmergeworthy:
     48           elif stage == "unmergeworthy":
    6149               if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1
    6250               if results['MERGED'] == 1: newMerged = newMerged + 1
    6351
    64            elif lookingForUnmerged:
     52           elif stage == "unmerged":
    6553               if results['MERGED'] == 1: newMerged = newMerged + 1
    6654
    … …  
    8876    CONFIG  = sys.argv[1]
    8977    BATCHTYPE = sys.argv[2]
    90     TYPE = sys.argv[3]
     78    STAGE = sys.argv[3]
    9179else:
    92     print "\n** Usage: " + sys.argv[0] + " <configPath> <P2|ST|all> <unloaded|unmergeworthy|unmerged|unfinished> [hours]\n"
     80    print "\n** Usage: " + sys.argv[0] + " <configPath> <P2|ST|all> <unloaded|unmergeworthy|unmerged|all> [hours]\n"
    9381    sys.exit(1)
    9482
    … …  
    113101EPOCH = configDoc.find("options/epoch").text
    114102
     103if STAGE == "all": stages = ['unloaded', 'unmergeworthy', 'unmerged']
     104else: stages = [STAGE]
     105
     106if BATCHTYPE == "all": batchTypes = ['P2', 'ST']
     107else: batchTypes = [BATCHTYPE]
     108
     109for stage in stages: logger.infoPair("Checking stage", stage)
     110for batchType in batchTypes: logger.infoPair("Checking batch type", batchType)
     111
    115112while True:
    116113
    117     if BATCHTYPE == "all":
    118         poll("P2")
    119         poll("ST")
    120     else:   
    121         poll(BATCHTYPE)
     114    for stage in stages:
     115        for batchType in batchTypes:
     116            poll(batchType, stage)
    122117
    123118    if SECONDS:

  • branches/eam_branches/ipp-20111110/ippTools/share/bgtool_definechip.sql

    r28568 r32694  
    55LEFT JOIN chipBackgroundRun USING(chip_id)
    66WHERE chipRun.state = 'full'
    7     AND chipRun.magicked > 0 -- require destreaked to stop race condition in destreaking

  • branches/eam_branches/ipp-20111110/ippTools/share/camtool_find_pendingimfile.sql

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20111110/ippTools/share/chiptool_dropprocessedimfile_delete.sql

    r32573 r32694  
    22    USING magicDSRun, magicDSFile, chipRun, chipProcessedImfile
    33WHERE chipRun.chip_id = chipProcessedImfile.chip_id
    4     AND magicDSRun.stage ='chip' and magicDSRun.stage_id = chipRun.chip_id
    5     AND magicDSFile.magic_ds_id = magicDSFile.magic_ds_id
     4    AND magicDSRun.stage ='chip'
     5    AND magicDSRun.stage_id = chipRun.chip_id
     6    AND magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    67    AND magicDSFile.component = chipProcessedImfile.class_id

  • branches/eam_branches/ipp-20111110/ippTools/share/pxadmin_create_tables.sql

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20111110/ippTools/src

  • branches/eam_branches/ipp-20111110/ippTools/src/chiptool.c

    r32577 r32694  
    885885    }
    886886    psFree(query);
     887    int numDeleted = psDBAffectedRows(config->dbh);
     888    psLogMsg("chiptool", PS_LOG_INFO, "Deleted %d chipProcessedImfiles", numDeleted);
    887889
    888890    if (!fault) {
    … …  
    898900    }
    899901    psFree(query_update);
     902    int numUpdated = psDBAffectedRows(config->dbh);
     903    psLogMsg("chiptool", PS_LOG_INFO, "Updated %d chipProcessedImfiles", numUpdated);
     904
    900905
    901906    return true;

  • branches/eam_branches/ipp-20111110/ippTools/src/disttool.c

    r32197 r32694  
    141141    PXOPT_LOOKUP_BOOL(use_alternate, config->args, "-use_alternate", false);
    142142    PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false);
    143     PXOPT_LOOKUP_BOOL(i_am_sure, config->args, "-i_am_sure", false);
    144143    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
    145144    PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false);
    … …  
    165164    PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false);
    166165
    167     // queuing no_magic runs can be quite dangerous. Require confirmation
    168     if (no_magic && !pretend && !i_am_sure) {
    169         psError(PXTOOLS_ERR_SYS, true, "-i_am_sure required to queue no_magic distribution runs");
    170         return false;
    171     }
    172    
    173166    if (use_alternate) {
    174167        if (strcmp(stage, "raw")) {

  • branches/eam_branches/ipp-20111110/ippTools/src/disttoolConfig.c

    r32197 r32694  
    4949
    5050    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false);
    51     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-i_am_sure", 0, "confirmation needed when -no_magic is supplied", false);
    5251    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-use_alternate", 0, "use alternate inputs", false);
    5352    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label",    0, "define label for run", NULL);

  • branches/eam_branches/ipp-20111110/ippTools/src/magicdstool.c

    r32479 r32694  
    5353
    5454static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state);
     55static bool validDSRunState(const char *state);
    5556
    5657# define MODECASE(caseName, func) \
    … …  
    12121213
    12131214    psMetadata *where = psMetadataAlloc();
    1214     // new state (required
    1215     PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", true, false);
     1215    // new state
     1216    PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", false, false);
    12161217    // old state (required)
    12171218    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
    … …  
    12371238        if (!strcmp(state, "failed_revert")) {
    12381239            new_state = "new";
     1240        } else if (!strcmp(state, "failed_revert_ud")) {
     1241            new_state = "update";
    12391242        } else if (!strcmp(state, "failed_cleanup")) {
    12401243            new_state = "goto_cleaned";
    … …  
    12431246            return false;
    12441247        }
     1248    } else {
     1249        if (!validDSRunState(new_state)) {
     1250            psError(PS_ERR_UNKNOWN, true, "unexpected value for new state: %s", new_state);
     1251            return false;
     1252        }
    12451253    }
    12461254    if (!p_psDBRunQueryF(config->dbh, query, new_state)) {
    1247         psError(PS_ERR_UNKNOWN, false, "failed to revert");
     1255        psError(PS_ERR_UNKNOWN, false, "failed to clear state faults");
    12481256        psFree(query);
    12491257        return false;
    … …  
    14031411          (strcmp(state, "wait") == 0) ||
    14041412          (strcmp(state, "failed_revert") == 0) ||
     1413          (strcmp(state, "failed_revert_ud") == 0) ||
    14051414          (strcmp(state, "error_cleaned") == 0) ||
    14061415          (strcmp(state, "restored") == 0) ||

  • branches/eam_branches/ipp-20111110/ippTools/src/magicdstoolConfig.c

    r32479 r32694  
    163163    // -clearstatefaults
    164164    psMetadata *clearstatefaultsArgs = psMetadataAlloc();
    165     psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL);
     165    psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL);
    166166    psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL);
    167167    psMetadataAddS64(clearstatefaultsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0);

  • branches/eam_branches/ipp-20111110/ippTools/src/magictool.c

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20111110/ippconfig

  • branches/eam_branches/ipp-20111110/ippconfig/recipes/ppSub.config

    r31435 r32694  
    219219END
    220220
     221SSTF_DIFF METADATA
     222        DUAL            BOOL    FALSE   # Dual convolution?
     223        INVERSE         BOOL    FALSE   # Generate inverse subtraction?
     224        PHOTOMETRY      BOOL    TRUE    # Perform photometry?
     225        NOCONVOLVE      BOOL    TRUE
     226        CONVOLVE.TARGET STR     SINGLE2 # convolution direction
     227END
     228 
     229
    221230# Gene's choice: multiplying by 1.75
    222231#@ISIS.WIDTHS   F32     1.8 3.3 5.8 10.2 17.9   # Gaussian FWHMs for ISIS kernels

  • branches/eam_branches/ipp-20111110/ippconfig/recipes/psphot.config

    r32349 r32694  
    216216# @RADIAL.ANNULAR.BINS.UPPER           F32      1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1 345.0 539.1 841.0 1305.0 #
    217217
    218 # PS1 values?
    219 @RADIAL.ANNULAR.BINS.LOWER           F32      0.0 1.2 3.4 5.2  8.8 15.0 23.2 37.2 57.1  91.0 141.0  # perl parser fails without a comment ticket #1476
    220 @RADIAL.ANNULAR.BINS.UPPER           F32      1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1  # perl parser fails without a comment ticket #1476
     218# PS1 values? (revised fpr PS1 @0.25 arcsec pixel scales)
     219@RADIAL.ANNULAR.BINS.LOWER           F32      0.0 0.96 2.72 4.16 7.04 12.0 18.56 29.76 45.68 72.80 112.80    # perl parser fails without a comment ticket #1476 still?
     220@RADIAL.ANNULAR.BINS.UPPER           F32      0.96 2.72 4.16 7.04 12.0 18.56 29.76 45.68 72.80 112.80 176.88 # perl parser fails without a comment ticket #1476 still?
    221221
    222222# Extended source fit parameters
    … …  
    355355  PETROSIAN_FOR_STARS                 BOOL  FALSE
    356356
     357  CONSTANT_PHOTOMETRIC_WEIGHTS        BOOL  TRUE
     358  POISSON.ERRORS.PHOT.LMM             BOOL  TRUE
     359
    357360  EXT_FIT_MAX_RADIUS                  F32   50.0
    358361  PSF_MODEL                           STR   PS_MODEL_GAUSS
    … …  
    505508END
    506509
     510SSTF_DIFF METADATA
     511  PEAKS_NSIGMA_LIMIT_2                F32   4.0             # peak significance threshold 
     512  DIFF_STATS                        BOOL TRUE          # Calculate diff-specific stats
     513  OUTPUT.FORMAT                     STR  PS1_DV2       # Format of output file
     514END
     515
     516
    507517# Recipe overrides for PR reduction
    508518PR      METADATA

  • branches/eam_branches/ipp-20111110/ippconfig/recipes/reductionClasses.mdc

    r32634 r32694  
    165165        PSVIDEOPHOT             STR     PSVIDEOPHOT
    166166END
     167
     168
    167169
    168170# basic science analysis
    … …  
    262264
    263265
     266
    264267DEFAULT_MASKTEST                METADATA
    265268        CHIP_PPIMAGE    STR     CHIP_MASKTEST
    … …  
    282285        BACKGROUND_PSWARP       STR     BACKGROUND
    283286END
     287
     288# SSTF tests
     289SSTF_NOCONV             METADATA
     290        CHIP_PPIMAGE      STR     CHIP
     291        CHIP_PSPHOT       STR     CHIP
     292        WARP_PSWARP       STR     WARP
     293        STACK_PPSTACK     STR     STACK
     294        STACK_PPSUB       STR     STACK
     295        STACK_PSPHOT      STR     STACK
     296        DIFF_PPSUB        STR     SSTF_DIFF
     297        DIFF_PSPHOT       STR     DIFF
     298        JPEG_BIN1         STR     PPIMAGE_J1
     299        JPEG_BIN2         STR     PPIMAGE_J2
     300        FAKEPHOT          STR     FAKEPHOT
     301        ADDSTAR           STR     ADDSTAR
     302        PSASTRO           STR     DEFAULT_RECIPE
     303        STACKPHOT_PSPHOT  STR     STACKPHOT
     304        STACKPHOT_PPSUB   STR     STACKPHOT
     305        STACKPHOT_PPSTACK STR     STACKPHOT
     306        STACKPHOT_SINGLE_PSPHOT  STR     STACKPHOT_SINGLE
     307        BACKGROUND_PPBACKGROUND STR     BACKGROUND
     308        BACKGROUND_PSWARP       STR     BACKGROUND
     309        PSVIDEOPHOT             STR     PSVIDEOPHOT
     310END
     311
     312SSTF_NOCONV_4SIG                METADATA
     313        CHIP_PPIMAGE      STR     CHIP
     314        CHIP_PSPHOT       STR     CHIP
     315        WARP_PSWARP       STR     WARP
     316        STACK_PPSTACK     STR     STACK
     317        STACK_PPSUB       STR     STACK
     318        STACK_PSPHOT      STR     STACK
     319        DIFF_PPSUB        STR     SSTF_DIFF
     320        DIFF_PSPHOT       STR     SSTF_DIFF
     321        JPEG_BIN1         STR     PPIMAGE_J1
     322        JPEG_BIN2         STR     PPIMAGE_J2
     323        FAKEPHOT          STR     FAKEPHOT
     324        ADDSTAR           STR     ADDSTAR
     325        PSASTRO           STR     DEFAULT_RECIPE
     326        STACKPHOT_PSPHOT  STR     STACKPHOT
     327        STACKPHOT_PPSUB   STR     STACKPHOT
     328        STACKPHOT_PPSTACK STR     STACKPHOT
     329        STACKPHOT_SINGLE_PSPHOT  STR     STACKPHOT_SINGLE
     330        BACKGROUND_PPBACKGROUND STR     BACKGROUND
     331        BACKGROUND_PSWARP       STR     BACKGROUND
     332        PSVIDEOPHOT             STR     PSVIDEOPHOT
     333END
     334
     335SSTF_4SIG               METADATA
     336        CHIP_PPIMAGE      STR     CHIP
     337        CHIP_PSPHOT       STR     CHIP
     338        WARP_PSWARP       STR     WARP
     339        STACK_PPSTACK     STR     STACK
     340        STACK_PPSUB       STR     STACK
     341        STACK_PSPHOT      STR     STACK
     342        DIFF_PPSUB        STR     DIFF
     343        DIFF_PSPHOT       STR     SSTF_DIFF
     344        JPEG_BIN1         STR     PPIMAGE_J1
     345        JPEG_BIN2         STR     PPIMAGE_J2
     346        FAKEPHOT          STR     FAKEPHOT
     347        ADDSTAR           STR     ADDSTAR
     348        PSASTRO           STR     DEFAULT_RECIPE
     349        STACKPHOT_PSPHOT  STR     STACKPHOT
     350        STACKPHOT_PPSUB   STR     STACKPHOT
     351        STACKPHOT_PPSTACK STR     STACKPHOT
     352        STACKPHOT_SINGLE_PSPHOT  STR     STACKPHOT_SINGLE
     353        BACKGROUND_PPBACKGROUND STR     BACKGROUND
     354        BACKGROUND_PSWARP       STR     BACKGROUND
     355        PSVIDEOPHOT             STR     PSVIDEOPHOT
     356END
     357
    284358
    285359# reduction class specifically for warpwarp diffs:

  • branches/eam_branches/ipp-20111110/magic/remove/src/streaksio.c

    r32252 r32694  
    632632
    633633    if (!isImage(in)) {
     634        psLogMsg("streaksremove", PS_LOG_INFO, "%s contains no image\n", in->name);
    634635        return;
    635636    }
    … …  
    779780setupImageRefs(sFile *out, sFile *recoveryOut, sFile *in, int extnum, bool exciseAll)
    780781{
     782    if (!in->image) {
     783        // Input file is likely corrupt. XXX: Shouldn't we have caught this earlier?
     784        psError(PS_ERR_UNKNOWN, true, "image is null for extension %d of %s file is likely corrupt", extnum, in->name);
     785        streaksExit("", PS_EXIT_UNKNOWN_ERROR);
     786    }
    781787    if (!exciseAll) {
    782788        // set output image to be the input image

  • branches/eam_branches/ipp-20111110/ppImage/src

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20111110/ppSub/src/ppSubLoop.c

    r31435 r32694  
    2323bool dumpout(pmConfig *config, char *name)
    2424{
    25   pmFPAview *view = ppSubViewReadout(); // View to readout
    26   pmReadout *out = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT");
    27   psphotSaveImage (NULL, out->image, name);
    28   psFree(view);
    29   return true;
     25    pmFPAview *view = ppSubViewReadout(); // View to readout
     26    pmReadout *out = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT");
     27    psphotSaveImage (NULL, out->image, name);
     28    psFree(view);
     29    return true;
    3030}
    3131
    … …  
    3333bool ppSubLoop(ppSubData *data)
    3434{
     35    bool mdok = false;
     36    bool success = true;
     37
    3538    psAssert(data, "Require processing data");
    3639    pmConfig *config = data->config;    // Configuration
    3740    psAssert(config, "Require configuration.");
    3841
    39     bool success = true;
     42    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim
     43    psAssert(recipe, "We checked this earlier, so it should be here.");
     44    bool noConvolve = psMetadataLookupBool(&mdok, recipe, "NOCONVOLVE"); // Do not use convolved images.
    4045
    4146    pmConfigCamerasCull(config, NULL);
    4247    pmConfigRecipesCull(config, "PPSUB,PPSTATS,PSPHOT,PSASTRO,MASKS,JPEG");
    43 
    4448
    4549    pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT");
    … …  
    7579    if (data->forcedPhot2) {
    7680        // Change the recipe to use a higher nsigma limit and quit after pass1
    77         psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE);
    78 
    79         psF32 nsigma_peak_save = psMetadataLookupF32 (NULL, recipe, "PEAKS_NSIGMA_LIMIT");
    80         char *breakPt_save =  psMetadataLookupStr (NULL, recipe, "BREAK_POINT");
    81 
    82         bool mdok;
    83         psF32 pos2_nsigma_peak = psMetadataLookupF32 (&mdok, recipe, "PEAKS_POS2_NSIGMA_LIMIT");
     81        psMetadata *psphotRecipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE);
     82
     83        psF32 nsigma_peak_save = psMetadataLookupF32 (NULL, psphotRecipe, "PEAKS_NSIGMA_LIMIT");
     84        char *breakPt_save =  psMetadataLookupStr (NULL, psphotRecipe, "BREAK_POINT");
     85
     86        psF32 pos2_nsigma_peak = psMetadataLookupF32 (&mdok, psphotRecipe, "PEAKS_POS2_NSIGMA_LIMIT");
    8487        if (!mdok) {
    85             psWarning("PEAKS_POS2_NSIGMA_LIMIT not found in recipe. Will use 25.\n");
     88            psWarning("PEAKS_POS2_NSIGMA_LIMIT not found in psphotRecipe. Will use 25.\n");
    8689            pos2_nsigma_peak = 25.;
    8790        }
    88         psMetadataAddF32(recipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", pos2_nsigma_peak);
    89         psMetadataAddStr(recipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", "PASS1");
     91        psMetadataAddF32(psphotRecipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", pos2_nsigma_peak);
     92        psMetadataAddStr(psphotRecipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", "PASS1");
    9093
    9194        bool foundDetections = false;
    9295        if (!ppSubInputDetections(&foundDetections, "PPSUB.POS2.SOURCES", "PPSUB.REF", data)) {
    9396            psError(psErrorCodeLast(), false, "Unable to measure positive detections (2)");
    94             psMetadataAddF32(recipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", nsigma_peak_save);
    95             psMetadataAddStr(recipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", breakPt_save);
    96             success = false;
    97             goto ESCAPE;
    98         }
    99         psMetadataAddF32(recipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", nsigma_peak_save);
    100         psMetadataAddStr(recipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", breakPt_save);
     97            psMetadataAddF32(psphotRecipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", nsigma_peak_save);
     98            psMetadataAddStr(psphotRecipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", breakPt_save);
     99            success = false;
     100            goto ESCAPE;
     101        }
     102        psMetadataAddF32(psphotRecipe, PS_LIST_TAIL, "PEAKS_NSIGMA_LIMIT", PS_META_REPLACE, "", nsigma_peak_save);
     103        psMetadataAddStr(psphotRecipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", breakPt_save);
    101104        // if nothing was found, don't bother doing the forced photometry below
    102105        if (!foundDetections) {
    … …  
    107110
    108111    // XXX if it exists, use the POS1, POS2 successs for the FWHMs
    109     psMetadata *recipe = psMetadataLookupPtr(NULL, config->recipes, PPSUB_RECIPE);
    110     bool noConvolve = psMetadataLookupBool(NULL, recipe, "NOCONVOLVE"); // Do not use convolved images.
    111     if (noConvolve) {
    112       psWarning("not matching PSFs because NOCONVOLVE is TRUE\n");
    113     } else {
    114       if (!ppSubMatchPSFs(data)) {
    115         psError(psErrorCodeLast(), false, "Unable to match PSFs.");
    116         success = false;
    117         goto ESCAPE;
    118       }
    119     }
    120 
     112    if (!ppSubMatchPSFs(data)) {
     113        psError(psErrorCodeLast(), false, "Unable to match PSFs.");
     114        success = false;
     115        goto ESCAPE;
     116    }
    121117
    122118    if (data->quality) {
    … …  
    138134                     psTimerClear("PPSUB_MATCH"));
    139135
    140     // Close input files
     136    // Close input files (freeing up space) : for the noConvolve case, we cannot close the
     137    // inputs since we will use them for subtraction below.  wait until later to do the work
    141138    if (!noConvolve) {
    142     if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) {
    143         psError(PPSUB_ERR_IO, false, "Unable to close input files.");
    144         success = false;
    145         goto ESCAPE;
    146     }
     139        if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) {
     140            psError(PPSUB_ERR_IO, false, "Unable to close input files.");
     141            success = false;
     142            goto ESCAPE;
     143        }
    147144    }
    148145    if (!ppSubLowThreshold(data)) {
    … …  
    165162    }
    166163
    167     if (!noConvolve) {
    168164    if (!data->quality && !ppSubMakePSF(data)) {
    169165        psError(psErrorCodeLast(), false, "Unable to generate PSF.");
    … …  
    171167        goto ESCAPE;
    172168    }
    173     }
     169
    174170    // Now we've got a PSF, blow away detections in case they're confused with real output detections
    175171    {
    … …  
    197193    // dumpout(config, "diff.2a.fits");
    198194    if (noConvolve) {
    199       if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) {
    200         psError(PPSUB_ERR_IO, false, "Unable to close input files.");
    201         success = false;
    202         goto ESCAPE;
    203       }
     195        if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) {
     196            psError(PPSUB_ERR_IO, false, "Unable to close input files.");
     197            success = false;
     198            goto ESCAPE;
     199        }
    204200    }
    205201   
    … …  
    271267    // Do Mask Stats
    272268    {
    273       pmFPAview *view = ppSubViewReadout(); // View to readout
    274       if (!ppSubMaskStats(config, view,data->stats)) {
    275         psError(psErrorCodeLast(), false, "Unable to generate mask statistics");
    276         success = false;
    277         psFree(view);
    278         goto ESCAPE;
    279       }
    280       psFree(view);
     269        pmFPAview *view = ppSubViewReadout(); // View to readout
     270        if (!ppSubMaskStats(config, view,data->stats)) {
     271            psError(psErrorCodeLast(), false, "Unable to generate mask statistics");
     272            success = false;
     273            psFree(view);
     274            goto ESCAPE;
     275        }
     276        psFree(view);
    281277    }
    282278    // dumpout(config, "diff.4.fits");
    … …  
    344340    }
    345341
    346 ESCAPE:
     342 ESCAPE:
    347343    pmFPAfileDropInternal(config->files, "PSPHOT.BACKGND");
    348344    pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL");

  • branches/eam_branches/ipp-20111110/ppSub/src/ppSubMakePSF.c

    r31435 r32694  
    4646    bool noConvolve = psMetadataLookupBool(&mdok, recipe, "NOCONVOLVE"); // Do not use convolved images.
    4747
    48     if (!noConvolve) {
    49       printf("Using Convolved images because NOCONVOLVE is FALSE\n");
    50       if (reverse) {
    51         minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV");
    52         minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.REF.CONV");
    53       } else {
    54         minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV");
    55         minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.INPUT.CONV");
    56       }
    57     }
    58     else {
    59       psWarning("Not using Convolved images because NOCONVOLVE  is TRUE\n");
    60       if (reverse) {
    61         minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF");
    62         minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.REF");
    63       } else {
    64         minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT");
    65         minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.INPUT");
    66       }
     48    if (noConvolve) {
     49        // if we do not convolve, we need to copy the detections to the image for analysis
     50        pmReadout *psfSourcesRO = NULL; // readout containing loaded sources for psf model
     51        psWarning("Not using Convolved images because NOCONVOLVE  is TRUE\n");
     52        if (reverse) {
     53            minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF");
     54            minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.REF");
     55            psfSourcesRO = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.SOURCES");
     56        } else {
     57            minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT");
     58            minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.INPUT");
     59            psfSourcesRO = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.SOURCES");
     60        }
     61        psAssert (psfSourcesRO, "missing readout with sources");
     62        pmDetections *psfDetections = psMetadataLookupPtr(&mdok, psfSourcesRO->analysis,  "PSPHOT.DETECTIONS");
     63        psMetadataAddPtr(minuend->analysis,  PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "Merged source list", psfDetections);
     64    } else {
     65        printf("Using Convolved images because NOCONVOLVE is FALSE\n");
     66        if (reverse) {
     67            minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV");
     68            minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.REF.CONV");
     69        } else {
     70            minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV");
     71            minuendFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.INPUT.CONV");
     72        }
    6773    }
    6874

  • branches/eam_branches/ipp-20111110/ppSub/src/ppSubMatchPSFs.c

    r31672 r32694  
    201201bool ppSubMatchPSFs(ppSubData *data)
    202202{
     203    bool mdok = false;
     204
    203205    psAssert(data, "Require processing data");
    204206    pmConfig *config = data->config;    // Configuration
    … …  
    208210    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim
    209211    psAssert(recipe, "We checked this earlier, so it should be here.");
     212
     213    bool noConvolve = psMetadataLookupBool(&mdok, recipe, "NOCONVOLVE"); // Do not use convolved images.
     214    if (noConvolve) {
     215        psWarning("not matching PSFs because NOCONVOLVE is TRUE\n");
     216        return true;
     217    }
    210218
    211219    pmFPAview *view = ppSubViewReadout(); // View to readout
    … …  
    228236        psFree(refConv);
    229237    }
    230 
    231     bool mdok;                          // Status of MD lookup
    232238
    233239    // Load pre-calculated kernel, if available

  • branches/eam_branches/ipp-20111110/psphot

  • branches/eam_branches/ipp-20111110/psphot/src/psphotSourceMatch.c

    r32633 r32694  
    7070    psVectorInit (foundSrc, 0);
    7171
    72     psVector *foundObj = psVectorAlloc(sources->n, PS_TYPE_U8);
     72    psVector *foundObj = psVectorAlloc(objects->n, PS_TYPE_U8);
    7373    psVectorInit (foundObj, 0);
    7474

  • branches/eam_branches/ipp-20111110/psvideophot

    • Property svn:mergeinfo changed (with no actual effect on merging)
Note: See TracChangeset for help on using the changeset viewer.