Index: /branches/eam_branches/ipp-20120805/DataStore/lib/DataStore/Utils.pm =================================================================== --- /branches/eam_branches/ipp-20120805/DataStore/lib/DataStore/Utils.pm (revision 34392) +++ /branches/eam_branches/ipp-20120805/DataStore/lib/DataStore/Utils.pm (revision 34393) @@ -38,5 +38,5 @@ ); -$STD_FIELD = qr/^[A-z0-9-_.: ]+$/; +$STD_FIELD = qr/^[A-z0-9-+_.: ]+$/; $TIME_FIELD = qr/^(\d{4})-(\d\d)-(\d\d) T (\d\d):(\d\d):(\d\d) Z$/x; $BYTE_FIELD = qr/^\d+$/; Index: /branches/eam_branches/ipp-20120805/ippMonitor/def/failedChipProcessedImfile.d =================================================================== --- /branches/eam_branches/ipp-20120805/ippMonitor/def/failedChipProcessedImfile.d (revision 34392) +++ /branches/eam_branches/ipp-20120805/ippMonitor/def/failedChipProcessedImfile.d (revision 34393) @@ -20,5 +20,5 @@ ARGS ARG3 class=$chipProcessedImfile.class_id -OP OP1 ( $chipProcessedImfile.fault == 2 ? "chip.revert.on and chip.revert.off" : "chiptool -revertprocessedimfile -fault $chipProcessedImfile.fault -label $chipRun.label -dbname ". $ID['proj'] ) +OP OP1 ( $chipProcessedImfile.fault == 2 ? "(chip.revert.on and chip.revert.off) or chiptool -revertprocessedimfile -fault $chipProcessedImfile.fault -label $chipRun.label -dbname ". $ID['proj'] : "chiptool -revertprocessedimfile -fault $chipProcessedImfile.fault -label $chipRun.label -dbname ". $ID['proj'] ) # field size format name show link to extras Index: /branches/eam_branches/ipp-20120805/ippMonitor/def/mopsStatus.d =================================================================== --- /branches/eam_branches/ipp-20120805/ippMonitor/def/mopsStatus.d (revision 34392) +++ /branches/eam_branches/ipp-20120805/ippMonitor/def/mopsStatus.d (revision 34393) @@ -1,3 +1,3 @@ -TABLE rawExp LEFT JOIN chipRun using(exp_id) LEFT JOIN camRun using(chip_id) LEFT JOIN fakeRun using(cam_id) LEFT JOIN warpRun using(fake_id) LEFT JOIN diffInputSkyfile AS diffInput ON diffInput.warp1 = warp_id LEFT JOIN diffRun AS diffInputRun ON diffInput.diff_id = diffInputRun.diff_id LEFT JOIN diffInputSkyfile AS diffTemplate ON diffTemplate.warp2 = warp_id LEFT JOIN diffRun AS diffTemplateRun ON diffTemplate.diff_id = diffTemplateRun.diff_id LEFT JOIN publishRun ON stage_id=diffInput.diff_id AND (client_id=1 OR client_id=5) LEFT JOIN publishRun AS pubRun2 ON pubRun2.stage_id=diffTemplate.diff_id AND (pubRun2.client_id=1 OR pubRun2.client_id=5) LEFT JOIN magicDSRun ON magicDSRun.magic_ds_id=camRun.magicked +TABLE rawExp LEFT JOIN chipRun using(exp_id) LEFT JOIN camRun using(chip_id) LEFT JOIN fakeRun using(cam_id) LEFT JOIN warpRun using(fake_id) LEFT JOIN diffInputSkyfile AS diffInput ON diffInput.warp1 = warp_id LEFT JOIN diffRun AS diffInputRun ON diffInput.diff_id = diffInputRun.diff_id LEFT JOIN diffInputSkyfile AS diffTemplate ON diffTemplate.warp2 = warp_id LEFT JOIN diffRun AS diffTemplateRun ON diffTemplate.diff_id = diffTemplateRun.diff_id LEFT JOIN publishRun ON stage_id=diffInput.diff_id AND (client_id=1 OR client_id=5) LEFT JOIN publishRun AS pubRun2 ON pubRun2.stage_id=diffTemplate.diff_id AND (pubRun2.client_id=1 OR pubRun2.client_id=5) TITLE MOPS Status FILE mopsStatus.php @@ -6,4 +6,5 @@ UNRESTRICTED WHERE dateobs>=DATE(NOW()) AND rawExp.obs_mode!='ENGINEERING' AND rawExp.obs_mode!='STS' AND rawExp.obs_mode!='M31' AND rawExp.obs_mode!='CNP' +RESTRICT_COUNT WHERE dateobs>=DATE(NOW()) AND rawExp.obs_mode!='ENGINEERING' AND rawExp.obs_mode!='STS' AND rawExp.obs_mode!='M31' AND rawExp.obs_mode!='CNP' # OP OP1 ($ddiff_id > 0 ? $ddiff_id : $tdiff_id) @@ -12,6 +13,6 @@ # $diffInput.diff_id+$diffTemplate.diff_id # PUB ID -OP OP2 ($row[12]>$row[13]?$row[12]. "(I)":($row[13]!=0?$row[13]. "(T)":"0")) -#($row[13]>$row[14]) +OP OP2 ($row[10]>$row[11]?$row[10]. "(I)":($row[11]!=0?$row[11]. "(T)":"0")) +#($row[11]>$row[12]) # field width format name show link to extras @@ -20,16 +21,14 @@ FIELD rawExp.exp_id, 5, %d, Exp ID FIELD chipRun.data_group, 5, %s, Data grp -FIELD chip_id, 5, %d, Chip ID (Stage 1) -FIELD camRun.cam_id AS camID, 5, %d, Cam ID (Stage 2) -FIELD fake_id, 5, %d, Fake ID (Stage 3) -FIELD warp_id, 5, %d, Warp ID (Stage 4) +FIELD chip_id, 5, %d, Chip ID (1/6) +FIELD camRun.cam_id AS camID, 5, %d, Cam ID (2/6) +FIELD fake_id, 5, %d, Fake ID (3/6) +FIELD warp_id, 5, %d, Warp ID (4/6) FIELD diffInput.diff_id AS ddiff_id, 5, %d, ddiff_id, none FIELD diffTemplate.diff_id AS tdiff_id, 5, %d, tdiff_id, none -FIELD *, 5, %s, Diff ID (Stage 5), op=OP1 -FIELD camRun.magicked, 5, %d, Magick ID (Stage 6) -FIELD magicDSRun.magic_ds_id, 5, %d, DestreakID (Stage 7) +FIELD *, 5, %s, Diff ID (5/6), op=OP1 FIELD publishRun.pub_id AS pubId1, 5, %d, pubId1, none FIELD pubRun2.pub_id AS pubId2, 5, %d, pubId2, none -FIELD *, 10, %s, Pub ID, op=OP2 +FIELD *, 10, %s, Pub ID (6/6), op=OP2 FIELD filter, 5, %s, Filter FIELD object, 15, %s, Object Index: /branches/eam_branches/ipp-20120805/ippMonitor/raw/czartool_labels.php =================================================================== --- /branches/eam_branches/ipp-20120805/ippMonitor/raw/czartool_labels.php (revision 34392) +++ /branches/eam_branches/ipp-20120805/ippMonitor/raw/czartool_labels.php (revision 34393) @@ -169,5 +169,5 @@ echo "Documentationhere"; echo "Cluster loadhere cpu_report mem_report"; -echo "Who's using the cluster?here"; +echo "Who's using the cluster?here new"; echo "Files monitorhere"; echo "IPP metricshere"; Index: /branches/eam_branches/ipp-20120805/ippMonitor/supervision/check_if_mysql_slave_is_running_fine.py =================================================================== --- /branches/eam_branches/ipp-20120805/ippMonitor/supervision/check_if_mysql_slave_is_running_fine.py (revision 34392) +++ /branches/eam_branches/ipp-20120805/ippMonitor/supervision/check_if_mysql_slave_is_running_fine.py (revision 34393) @@ -35,5 +35,5 @@ # Don't change anything after this - db = MySQLdb.connect('localhost', 'root') + db = MySQLdb.connect('localhost', 'root', connect_timeout=1000) cursor = db.cursor() cursor.execute('SHOW SLAVE STATUS') Index: /branches/eam_branches/ipp-20120805/ippScripts/scripts/lap_science.pl =================================================================== --- /branches/eam_branches/ipp-20120805/ippScripts/scripts/lap_science.pl (revision 34392) +++ /branches/eam_branches/ipp-20120805/ippScripts/scripts/lap_science.pl (revision 34393) @@ -280,27 +280,33 @@ $warptool_update_cmd .= " -dbname $dbname " if defined $dbname; - ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $chiptool_update_cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform chiptool -setimfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); - } -# if ($chip_magicDS_id != 0) { -# ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run(command => $magicDS_update_cmd, verbose => $verbose); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); -# } -# } - - ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $warptool_update_cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform warptool -setskyfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); - } - + # only need to update the data if the warps are not full + if ($warp->{state} ne 'full') { + if ($chip->{state} eq 'cleaned') { + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $chiptool_update_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -setimfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); + } + +# if ($chip_magicDS_id != 0) { +# ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $magicDS_update_cmd, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); +# } +# } + } + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $warptool_update_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -setskyfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); + } + } $exposure->{active} = 1; + return($exposure); } @@ -309,4 +315,10 @@ my $exposure = shift; + if (S64_IS_NOT_NULL($exposure->{warp_id})) { + if ($exposure->{warpRun_state} eq 'full') { + # this warp is in full state no need to update it or the chips + return 0; + } + } if (S64_IS_NOT_NULL($exposure->{chip_id})) { if (($exposure->{chip_state} eq 'cleaned')|| @@ -762,5 +774,5 @@ } print "STATUS: Made $Nstacks_made final stacks.\n"; - exit($Nstacks_made); + exit(0); # If this doesn't do anything, we may end up stuck here. Therefore, we need a more robust test. } Index: /branches/eam_branches/ipp-20120805/ppBackground/src/ppBackgroundRestore.c =================================================================== --- /branches/eam_branches/ipp-20120805/ppBackground/src/ppBackgroundRestore.c (revision 34392) +++ /branches/eam_branches/ipp-20120805/ppBackground/src/ppBackgroundRestore.c (revision 34393) @@ -59,7 +59,15 @@ psImage *patternImage = patternRO->image; // Image with pattern psImage *patternMask = patternRO->mask; // Mask for pattern + // The sign is flipped for the continuity correction. + bool mdok; + bool isContinuity = psMetadataLookupBool(&mdok, pattern->hdu->header, "PTRN_CON"); + for (int y = 0; y < numRows; y++) { for (int x = 0; x < numCols; x++) { - image->data.F32[y][x] -= patternImage->data.F32[y][x]; + if (isContinuity) { + image->data.F32[y][x] += patternImage->data.F32[y][x]; + } else { + image->data.F32[y][x] -= patternImage->data.F32[y][x]; + } mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= patternMask->data.PS_TYPE_IMAGE_MASK_DATA[y][x]; } Index: /branches/eam_branches/ipp-20120805/psastro/src/psastroChooseRefstars.c =================================================================== --- /branches/eam_branches/ipp-20120805/psastro/src/psastroChooseRefstars.c (revision 34392) +++ /branches/eam_branches/ipp-20120805/psastro/src/psastroChooseRefstars.c (revision 34393) @@ -62,4 +62,6 @@ int nMax = psMetadataLookupS32 (&status, recipe, "PSASTRO.MAX.NREF"); + psF32 clampMagMin = psMetadataLookupS32 (&status, recipe, "REFSTAR_CLAMP_MAG_MIN"); + if (!status) clampMagMin = -5; // de-activate all files except PSASTRO.REFSTARS @@ -103,4 +105,10 @@ for (int i = 0; i < refs->n; i++) { pmAstromObj *ref = pmAstromObjCopy(refs->data[index->data.S32[i]]); + + if (ref->Mag < clampMagMin) { + psWarning("Skipping refstar with abusrd magnintude %f ra: %f dec: %f\n", + ref->Mag, RAD_TO_DEG(ref->sky->r), RAD_TO_DEG(ref->sky->d)); + goto skip; + } psProject (ref->TP, ref->sky, fpa->toSky); Index: /branches/eam_branches/ipp-20120805/pstamp/scripts/dqueryparse.pl =================================================================== --- /branches/eam_branches/ipp-20120805/pstamp/scripts/dqueryparse.pl (revision 34392) +++ /branches/eam_branches/ipp-20120805/pstamp/scripts/dqueryparse.pl (revision 34393) @@ -132,5 +132,4 @@ print STDERR "skipping $command\n"; } - exit (0); } Index: /branches/eam_branches/ipp-20120805/tools/dlapstacks =================================================================== --- /branches/eam_branches/ipp-20120805/tools/dlapstacks (revision 34392) +++ /branches/eam_branches/ipp-20120805/tools/dlapstacks (revision 34393) @@ -13,5 +13,7 @@ # allow mask and variance images to be displayed as well -my $label = 'LAP.ThreePi.20110809'; +my $label; +my $label1 = 'LAP.ThreePi.20120706'; +my $label2 = 'LAP.ThreePi.20110809'; my $tess_id = 'RINGS.V3'; @@ -55,4 +57,12 @@ $ra = shift; $dec = shift; +# if neither label or data_group were supplied set default label +if (!$label and !$data_group) { + $label = $label1; +} else { + # drop label2 + $label2 = undef; +} + pod2usage( -msg => 'if ra is suplied must be supplied as well', -exitval => 2) @@ -90,5 +100,5 @@ # don't allow MD label unless it's a refstack. # Nightly stacks have too many images -if (($data_group && ($data_group =~ /MD/)) or ($label =~ /^MD/)) { +if (($data_group && ($data_group =~ /MD/)) or (defined $label and $label =~ /^MD/)) { unless ($data_group or uc($label) =~ /REF/) { die "cannot use this program for label $label without data_group because that will most likely select nightly stacks and we will match an unreasonably large number of matches. Talk to Bill\n"; @@ -124,4 +134,6 @@ if ($data_group) { $query .= " AND data_group = '$data_group'"; +} elsif ($label2) { + $query .= " AND (label = '$label' OR label = '$label2')"; } else { $query .= " AND label = '$label'"; Index: /branches/eam_branches/ipp-20120805/tools/fixburntool =================================================================== --- /branches/eam_branches/ipp-20120805/tools/fixburntool (revision 34393) +++ /branches/eam_branches/ipp-20120805/tools/fixburntool (revision 34393) @@ -0,0 +1,121 @@ +#!/usr/bin/env perl + +# script to regenrate the burntool table for a single exp_id/class_id +# Moves and then deletes existing file if found + +use strict; +use warnings; +use Carp; +use IPC::Cmd 0.36 qw( can_run run ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + + +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $burntool = can_run('ipp_apply_burntool_single.pl') or (warn "Can't find ipp_burntool_apply_single.pl" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $exp_id; +my $class_id; +my $dbname = 'gpc1'; +my $verbose = 1; +GetOptions( + 'exp_id|e=s' => \$exp_id, + 'class_id|c=s' => \$class_id, + 'verbose|v' => \$verbose, +) or pod2usage( 2 ); + +die "usage: $0 (exp_id) (class_id)\n" unless $class_id and $exp_id; + +$regtool .= " -dbname gpc1 -exp_id $exp_id -class_id $class_id"; +$burntool .= " --dbname gpc1"; + + +my $regData; +{ + my $command = "$regtool -processedimfile"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + carp("Unable to perform $command: $error_code"); + exit $error_code; + } + my $list = $mdcParser->parse(join "", @$stdout_buf) or die("Unable to parse regtool metadata"); + my $mdc = parse_md_list($list); + $regData = $mdc->[0]; +} +my $exp_name = $regData->{exp_name}; +my $uri = $regData->{uri}; + +die "failed to find exp_name in regtool output" unless $exp_name; +die "failed to find uri in regtool output" unless $uri; + +my $table; +($table = $uri) =~ s/\.fits/\.burn\.tbl/; +# print "$table\n"; +my $table_exists; +{ + my $command = "neb-stat $table"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + if ($success) { + $table_exists = 1; + } else { + $table_exists = 0; + } +} +if ($table_exists) { + # file exists + my $moved = "$table.bad"; + my $command = "neb-mv $table $moved"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + carp("Unable to perform $command: $error_code"); + exit $error_code; + } + # don't care if this fails + system "neb-rm --force $moved"; +} + +my $left = substr $exp_name, 0, 6; +my $num = substr $exp_name, 6, 4; +my $end = substr $exp_name, 10, 1; + +# print "$left $num $end\n"; + + +my $prev = $num - 1; +my $prev_name = sprintf "$left%04d$end", $prev; + +my $prev_uri = $uri; +$prev_uri =~ s/$exp_name/$prev_name/g; +die "failed" unless $prev_uri; +# print "$prev_name $prev_uri\n"; + + +{ + my $command = "$burntool --continue 10 --exp_id $exp_id --class_id $class_id --this_uri $uri --previous_uri $prev_uri"; + print "$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + carp("Unable to perform $command: $error_code"); + exit $error_code; + } +} +system "chiptool -dbname $dbname -revertprocessedimfile -exp_id $exp_id -class_id $class_id"; +print "Success\n"; +exit 0; Index: /branches/eam_branches/ipp-20120805/tools/listcmf =================================================================== --- /branches/eam_branches/ipp-20120805/tools/listcmf (revision 34392) +++ /branches/eam_branches/ipp-20120805/tools/listcmf (revision 34393) @@ -94,4 +94,6 @@ } +my $flags_col; +my $flags2_col; my %extensions; @@ -225,8 +227,10 @@ extend_incomplete_lines(\@lines, $total_cols, $ncols_previous); } else { - print STDERR "no xfit extension found in $cmf\n" unless $xfit_ext; + print STDERR "no xfit extension found in $cmf\n" if $verbose; } my $base = basename($cmf) ; +# data to be output for "missing" sources +my $null_line = ""; { print $oh "# cmf: $base FILTER: $filter ZPT: $zero_point EXPTIME: $exp_time\n"; @@ -235,17 +239,38 @@ print $oh "IPP_IDET "; + # Loop over columns and print out the header line and build $null_line for (my $i = 1; $i < scalar @psf_cols; $i++) { print $oh " $psf_cols[$i]_$filter"; + if ($i eq $flags_col) { + # for no source set flags to zero + $null_line .= ' 0'; + } elsif ($i eq $flags2_col) { + # for no source use PM_SOURCE_MODE2_DIFF_WITH_DOUBLE as a "no source" flag + # isnan(X_PSF_$filter) also works but this should be faster to process + $null_line .= ' 2'; + } elsif ($i > 1) { + # for other columns just print NULL which mana converts to nan + # Note data for column 1, IPP_IDET == $linenum, is added for the null line below + $null_line .= ' NULL'; + } } for (my $i = 0; $i < scalar @xsrc_cols; $i++) { - print $oh " $xsrc_cols[$i]_$filter" + print $oh " $xsrc_cols[$i]_$filter"; + $null_line .= ' NULL'; } for (my $i = 0; $i < scalar @xfit_cols; $i++) { print $oh " $xfit_cols[$i]_$filter"; + $null_line .= ' NULL'; } print $oh "\n"; } -foreach my $line (@lines) { +my $num_lines = scalar @lines; +for (my $i = 1; $i < $num_lines; $i++) { + my $line = $lines[$i]; + if (!$line) { + # Add line for missing source + $line = sprintf "%11d %s", $i, $null_line; + } print $oh "$line\n"; } @@ -323,4 +348,12 @@ # XXX TODO: check the format and insure that it is not a vector # because we currently don't support them + if ($name eq 'FLAGS') { + # print STDERR "found flags in column $col\n"; + $flags_col = $col; + } + if ($name eq 'FLAGS2') { + # print STDERR "found flags2 in column $col\n"; + $flags2_col = $col; + } } push @$r_cols, $name; @@ -336,18 +369,23 @@ # XXX: we are assuming here that the ids in a psf extension # are sequential - foreach my $line (@lines) { + # foreach my $line (@lines) { + my $num_lines = scalar @lines; + for (my $i = 0; $i < $num_lines; $i++) { + my $line = $lines[$i]; # id is first column chomp $line; my ($ipp_idet) = split " ", $line; - push @$r_ids, $ipp_idet; - push @$r_lines, $line; +# push @$r_ids, $ipp_idet; +# push @$r_lines, $line; + $r_ids->[$ipp_idet] = $ipp_idet; + $r_lines->[$ipp_idet] = $line; } } else { - my $num_objects = scalar @$r_lines; + my $num_lines = scalar @$r_lines; foreach my $line (@lines) { chomp $line; my ($id, $rest_of_line) = identify_line($line); - if ($id > $num_objects) { - die "TILT: id $id is greater than num objects\n"; + if ($id > $num_lines) { +# die "TILT: id $id is greater than num objects\n"; } # just append the line to this object's line @@ -374,4 +412,5 @@ my $nprevious = shift; for (my $i = 0; $i < scalar @$lines; $i++) { + next if !$lines->[$i]; my @words = split " ", $lines->[$i]; my $nwords = scalar @words; Index: /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/deploy.sh =================================================================== --- /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/deploy.sh (revision 34393) +++ /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/deploy.sh (revision 34393) @@ -0,0 +1,56 @@ +#!/bin/bash + +# Directory where the executables will be installed +BIN_TARGET_DIRECTORY=$HOME/operations/who_is_using_the_cluster +# Directory where the generated HTML pages will be written +# Must be accessible from each node and from apache +# On the IPP cluster, it's a symbolic link located in the ippm +# user home directory (hence nfs accessible) to /var/www/localhost/htdocs +# on ippmonitor.ipp.ifa.hawaii.edu +HTML_TARGET_DIRECTORY=$HOME/htdocs/clusterMonitor2 +# File containing the nodes names (one nore per line) +TARGET_NODENAMES_FILENAME=$HOME/common/share/ipp/AllNodesNames + +# No need to change anything from there unless you know what you're doing +echo "$BIN_TARGET_DIRECTORY" + +MKDIR="/bin/mkdir -p" +CP="/bin/cp -f" +CAT=/bin/cat +SED=/bin/sed +CHMOD=/bin/chmod + +echo "Getting SVN revision" +SVNREVISION=`/usr/bin/svn up . | cut -f3 -d' '` + +echo "Copying executables" +$MKDIR $BIN_TARGET_DIRECTORY +$CP update_all_nodes.py $BIN_TARGET_DIRECTORY/ +$CHMOD 755 $BIN_TARGET_DIRECTORY/update_all_nodes.py +$CP get_processes_information.py $BIN_TARGET_DIRECTORY/ +$CHMOD 755 $BIN_TARGET_DIRECTORY/get_processes_information.py +$CP documentation/README.deployed $BIN_TARGET_DIRECTORY/README +$CHMOD 644 $BIN_TARGET_DIRECTORY/README + +echo "Copying html and php files" +$MKDIR $HTML_TARGET_DIRECTORY/ +$CAT html/help.html | $SED "s/__SVN_REVISION__/$SVNREVISION/g" > $HTML_TARGET_DIRECTORY/help.html +$CP html/index.php $HTML_TARGET_DIRECTORY/ +$CP html/header.html $HTML_TARGET_DIRECTORY/ +$CP documentation/README $HTML_TARGET_DIRECTORY/ +$CHMOD 644 $HTML_TARGET_DIRECTORY/README + +echo "Creating html files for each node" +$MKDIR $HTML_TARGET_DIRECTORY/nodes +for node in `cat $TARGET_NODENAMES_FILENAME`; do + echo "$node not updated yet" > $HTML_TARGET_DIRECTORY/nodes/$node.html + $CHMOD 644 $HTML_TARGET_DIRECTORY/nodes/$node.html +done + +echo "Now you can add something like:" +echo "--" +echo "MAILTO=schastel@ifa.hawaii.edu" +echo "0-55/5 * * * * $HTML_TARGET_DIRECTORY/update_all_nodes.py >> /tmp/who_is_using_the_cluster.log 2>&1" +echo "--" +echo "to the crontab of a relevant user for a relevant node" + Index: /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/documentation/README =================================================================== --- /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/documentation/README (revision 34393) +++ /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/documentation/README (revision 34393) @@ -0,0 +1,78 @@ +Objective +========= + +The purpose of this software is to give details about how the IPP +cluster is being used. + +Dependencies +============ +psutil (0.6.0): http://code.google.com/p/psutil/ + +Installation +============ +1) Have a look at the installation script, read the associated +comment, and change the value of: +BIN_TARGET_DIRECTORY +HTML_TARGET_DIRECTORY +TARGET_NODENAMES_FILENAME + +2) Synchronize the contents of TARGET_NODENAMES_FILENAME and the nodes +array in the update_all_nodes.py script (I know it's awful). + +3) Run the deploy.sh script + +4) Add an entry to your crontab + +Constraints and Insights +======================== + +The update_all_nodes.py script is supposed to be run through a crontab +(or any scheduling mechanism resembling crontab) and produces a set of +HTML pages which are then rendered through a php script. + +The user running the script must then have the right to execute cron +jobs. This user must also be able to ssh to any node that needs to be +supervised. + +The products (that is html pages) are written to a location that has +to be writable to that user from every node where the (typically a +shared partition of some sort like a NFS one). If the pages are to be +seen thanks to a web browser, they also need to be accessible from +apache (see the "In the IPP context" paragraph for a practical case). + +Each instance of the get_processes_information.py generates its own +html page. It scans 5 times every .5s the process table on the host +and filters out some information according to the rules defined in the +Users class of this script, namely: +* ignore: the users who should always be ignored +* expected: the users who are expected to run something on the + cluster. Any other user that are not in 'ignore' and are not + 'expected' will be displayed as a potential problem (i.e. in red). +* always: the users for whom all processes must be displayed (even if + they don't use any resource) +* cmdline_ignore_re_progs_from_pattern: list of regular expressions of + some programs we don't want to be displayed (e.g. condor daemons) +* max_age_in_seconds: set to 6 hours. Any process older than that will + be displayed as a potential problem, i.e. in red. +* min_cpu (percentage): any process using more than this quantity will + be displayed as a potential problem, i.e. in red. +* min_memory: any process using more than this quantity will + be displayed as a potential problem, i.e. in red. + +In the IPP context +================== + +The update_all_nodes.py script is run from ippc18 by the ippm user +every 5 minutes. An independent thread is run for each node, +connecting through ssh to that node where the process analysis has to +be performed using the get_processes_information.py script. + +The nodes list is in update_all_nodes.py (but must be the same as the +list found in the file named TARGET_NODENAMES_FILENAME in the +deploy.sh script). + +TODO / Known bugs +================= +* Make the nodes names list unique +* There are likely hard-coded paths, values and that kind of stuff in + the scripts Index: /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/documentation/README.deployed =================================================================== --- /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/documentation/README.deployed (revision 34393) +++ /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/documentation/README.deployed (revision 34393) @@ -0,0 +1,12 @@ +Do not edit the executables in this directory + +Checkout the sources from +https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/trunk/tools/who_is_using_the_cluster + +Modify them, test them, commit them and use +./deploy.sh +as the ippm user on the cluster to deploy them on the cluster. + +-- +update_all_nodes.py is run from ippc18 crontab as the ippm user. + Index: /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/get_processes_information.py =================================================================== --- /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/get_processes_information.py (revision 34393) +++ /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/get_processes_information.py (revision 34393) @@ -0,0 +1,211 @@ +#!/usr/bin/env python + +import sys +sys.path.append('/home/panstarrs/ippm/common/lib/python2.7/site-packages') + +import psutil +import os +import pwd +import time +import datetime +import subprocess +import re +import socket + +class Callable: + """ + See http://code.activestate.com/recipes/52304-static-methods-aka-class-methods-in-python/ + """ + def __init__(self, anycallable): + self.__call__ = anycallable + +class Users: + """ + Class where some information related to users is cached. More a + container with convenient method actually. + """ + ignore = ['root', 'apache', 'bin', 'nobody', 'nagios', 'mysql', ] + expected = ['ipp', 'ippdor', 'ippdvo', 'iqanalysis', ] + always = ['ipp', 'ippdvo', ] + cmdline_ignore_re_progs_from_pattern = [ re.compile('.*condor_.*'), + re.compile('.*get_processes_information.py.*'), + ] + singleton = dict() # index is the (numeric) uid + def get_user_name(uid): + try: + return Users.singleton[uid] + except KeyError: + Users.singleton[uid] = pwd.getpwuid(uid).pw_name + return Users.singleton[uid] + get_user_name = Callable(get_user_name) + max_age_in_seconds = 6*3600 + min_cpu = 5. + min_memory = 5. + +class SmallProcess: + def __init__(self, pid, username, cmdline, cpu_percent, memory_percent, age, count = 1): + self.pid = pid + self.cmdline = cmdline + self.username = username + self.cpu_percent = cpu_percent + self.memory_percent = memory_percent + self.age = age + self.count = count + +class Processes: + processes = dict() + def add_process(process): + """Append the information relative to a process to the class + 'processes' container""" + try: + Processes.processes[process.pid].append(process) + except KeyError: + Processes.processes[process.pid] = [] + Processes.processes[process.pid].append(process) + add_process = Callable(add_process) + def statistics(): + data = [] + for key in sorted(Processes.processes.iterkeys()): + item = Processes.processes[key] + count = 0 + avg_cpu_perc = 0. + avg_mem_perc = 0. + for sprocess in item: + avg_cpu_perc += sprocess.cpu_percent + avg_mem_perc += sprocess.memory_percent + count += 1. + if count > 1: + sp = SmallProcess(sprocess.pid, + sprocess.username, + sprocess.cmdline, + avg_cpu_perc / count, + avg_mem_perc / count, + sprocess.age, + count) + data.append(sp) + return data + statistics = Callable(statistics) + +class ProcessesSnapshot: + """ + The container of all processes + """ + def __init__(self, min_cpu = Users.min_cpu, min_memory = Users.min_memory, max_age_in_seconds = Users.max_age_in_seconds): + processes = dict() # index is the pid + ages = get_correct_processes_age() + for process in psutil.process_iter(): + try: + age = ages[process.pid] + except KeyError: + age = 0 # Process died while we were playing... + try: + sp = SmallProcess(process.pid, + Users.get_user_name(process.uids.real), + process.cmdline, + process.get_cpu_percent(interval = 0), + process.get_memory_percent(), + age) + add_process = False + if sp.cpu_percent > min_cpu or sp.memory_percent > min_memory: + add_process = True + if sp.age > max_age_in_seconds and sp.username not in Users.ignore: + add_process = True + for prog in Users.cmdline_ignore_re_progs_from_pattern: + if prog.match(" ".join(sp.cmdline)) is not None: + add_process = False + if sp.username in Users.always: + add_process = True + if add_process: + Processes.add_process(sp) + except psutil.error.NoSuchProcess: + pass + +def get_correct_processes_age(): + p = subprocess.Popen('ps -eo pid,etime'.split(' '), + stderr = subprocess.PIPE, + stdout = subprocess.PIPE) + p.poll() + ages = dict() + for complex_line in p.stdout: + line = re.sub('[ \t][ \t]*', ' ', complex_line[:-1]) + line = re.sub('^ ', '', line) + if line.startswith('PID'): + continue + (pid, etime) = line.split(' ') + if '-' in etime: + (days, etime) = etime.split('-') + else: + days = 0 + age = int(days)*24*60*60 + fields = etime.split(':') + if len(fields) == 3: + (hours, minutes, seconds) = fields + age += int(seconds) + 60 * (int(minutes) + 60 * int(hours)) + elif len(fields) == 2: + (minutes, seconds) = fields + age += int(seconds) + 60 * int(minutes) + elif len(fields) == 2: + seconds = fields + age += int(seconds) + ages[int(pid)] = age + return ages + +if __name__ == '__main__': + nodename = socket.gethostbyaddr(socket.gethostname())[0].split('.')[0] + count = 0 + time_interval = .5 + iterations = 5 + while count < iterations: + ProcessesSnapshot() + count += 1 + time.sleep(time_interval) + + output = open('/home/panstarrs/ipp/htdocs/clusterMonitor2/nodes/%s.html' % nodename, 'w') + # output.write('\n') + # output.write('\n') + # output.write('\n') + # output.write('\n') + # output.write('\n') + # output.write('\n') + # output.write('\n') + now = datetime.datetime.now() + output.write('

%s

\n' % (nodename, nodename)) + output.write('Last refresh: %s / Time interval: %5.2f / #Iterations: %d
\n' % (now.isoformat().split('.')[0], + time_interval, + iterations)); + output.write('\n') + output.write('\n') + for sprocess in Processes.statistics(): + output.write(' \n') + output.write(' \n' % (sprocess.pid, + sprocess.count)) + if sprocess.username not in Users.expected: + alert_color = '#ff0000' + else: + alert_color = '#000000' + output.write(' \n' % (alert_color, + sprocess.username)) + if sprocess.cpu_percent >= Users.min_cpu: + alert_color = '#ff0000' + else: + alert_color = '#000000' + output.write(' \n' % (alert_color, + sprocess.cpu_percent) ) + if sprocess.memory_percent >= Users.min_memory: + alert_color = '#ff0000' + else: + alert_color = '#000000' + output.write(' \n' % (alert_color, + sprocess.memory_percent) ) + if sprocess.age >= Users.max_age_in_seconds: + alert_color = '#ff0000' + else: + alert_color = '#000000' + output.write(' \n' % (alert_color, + sprocess.age/3600.) ) + output.write(' \n' % (' '.join(sprocess.cmdline))) + output.write('\n') + output.write('
pid (#samples)ownercpu%mem%age (hrs)cmd line
%d (%d)%s%6.2f%6.2f%10.2f%s
\n') + # output.write('\n') + # output.write('\n') + output.close() Index: /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/html/header.html =================================================================== --- /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/html/header.html (revision 34393) +++ /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/html/header.html (revision 34393) @@ -0,0 +1,58 @@ +
+ Who is using the cluster? + + + + + + + +
+ Index: /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/html/help.html =================================================================== --- /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/html/help.html (revision 34393) +++ /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/html/help.html (revision 34393) @@ -0,0 +1,40 @@ +This page shows how the cluster is being used.

+ +Every 5 minutes, a script running on ippc18 (as the ippm user. See its +crontab) starts a process on each node of the cluster. There, five +(i.e. the number of iterations) snapshots of the node activity are +made at an interval of 0.5 second (i.e. time interval).
+ +Statistics about processes that have been observed at least +twice are then computed. They are displayed if at least one of +these conditions is satisfied (the corresponding column is then shown +in red): +

+ +The displayed columns are the following: + + +Feel free to correct this documentation and to send me your suggestions and comments.
+ +Serge
+
+SVN Revision: __SVN_REVISION__ + Index: /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/html/index.php =================================================================== --- /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/html/index.php (revision 34393) +++ /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/html/index.php (revision 34393) @@ -0,0 +1,67 @@ +\n"); + +print(file_get_contents("header.html")); + +print("\n"); + +$NODES = array( "ipp004", "ipp005", "ipp006", "ipp007", "ipp008", "ipp009", + "ipp010", "ipp011", "ipp012", "ipp013", "ipp014", "ipp015", "ipp016", "ipp017", "ipp018", "ipp019", + "ipp020", "ipp021", "ipp023", "ipp024", "ipp025", "ipp026", "ipp027", "ipp028", "ipp029", + "ipp030", "ipp031", "ipp032", "ipp033", "ipp034", "ipp035", "ipp036", "ipp037", "ipp038", "ipp039", + "ipp040", "ipp041", "ipp042", "ipp043", "ipp044", "ipp045", "ipp046", "ipp047", "ipp048", "ipp049", + "ipp050", "ipp051", "ipp052", "ipp053", "ipp054", "ipp055", "ipp056", "ipp057", "ipp058", "ipp059", + "ipp060", "ipp061", "ipp062", "ipp063", "ipp064", "ipp065", "ipp066", + "ippc01", "ippc02", "ippc03", "ippc04", "ippc05", "ippc06", "ippc07", "ippc08", "ippc09", + "ippc10", "ippc11", "ippc12", "ippc13", "ippc14", "ippc15", "ippc16", "ippc17", "ippc18", "ippc19", + "ippc20", "ippc21", "ippc22", "ippc23", "ippc24", "ippc25", "ippc26", "ippc27", "ippc28", "ippc29", + "ippc30", "ippc31", "ippc32", "ippc33", "ippc34", "ippc35", "ippc36", "ippc37", "ippc38", "ippc39", + "ippc40", "ippc41", "ippc42", "ippc43", "ippc44", "ippc45", "ippc46", "ippc47", "ippc48", "ippc49", + "ippc50", "ippc51", "ippc52", "ippc53", "ippc54", "ippc55", "ippc56", "ippc57", "ippc58", "ippc59", + "ippc60", "ippc61", "ippc62", "ippc63", + "stare00", "stare01", "stare02", "stare03", "stare04", + "ippdb00", "ippdb01", "ippdb02", "ippdb03", + "stsci00", "stsci01", "stsci02", "stsci03", "stsci04", + "stsci05", "stsci06", "stsci07", "stsci08", "stsci09", + ); + +print("\n
\n"); +print("\n \n"); +$current_index = 0; +$previous_node_index = 10; +foreach ($NODES as $node) { + $node_index = intval(substr($node, -1)); + if ($current_index > $node_index) { + $current_index = 0; + print(" \n"); + print(" \n"); + } + while ($current_index < $node_index) { + print(" \n"); + $current_index += 1; + } + print(" \n"); + $current_index += 1; + if ($current_index >= 10) { + $current_index = 0; + print(" \n"); + print(" \n"); + } +} +print(" \n
".$node."
\n"); +print("		 \n"); +print(file_get_contents("help.html")); +print("
\n"); + +print("

\n"); + +foreach ($NODES as $node) { + print(file_get_contents("nodes/$node.html")); + print("

\n"); +} + +print("\n"); + +print("\n"); +?> Index: /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/update_all_nodes.py =================================================================== --- /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/update_all_nodes.py (revision 34393) +++ /branches/eam_branches/ipp-20120805/tools/who_is_using_the_cluster/update_all_nodes.py (revision 34393) @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +log = open('/home/panstarrs/ippm/operations/who_is_using_the_cluster/update_all_nodes.log', 'w') + +import sys +sys.path.append('/home/panstarrs/ippm/common/lib/python2.7/site-packages') + +import threading +import socket +import subprocess + +nodes = [ "ipp004", "ipp005", "ipp006", "ipp007", "ipp008", "ipp009", + "ipp010", "ipp011", "ipp012", "ipp013", "ipp014", "ipp015", "ipp016", "ipp017", "ipp018", "ipp019", + "ipp020", "ipp021", "ipp023", "ipp024", "ipp025", "ipp026", "ipp027", "ipp028", "ipp029", + "ipp030", "ipp031", "ipp032", "ipp033", "ipp034", "ipp035", "ipp036", "ipp037", "ipp038", "ipp039", + "ipp040", "ipp041", "ipp042", "ipp043", "ipp044", "ipp045", "ipp046", "ipp047", "ipp048", "ipp049", + "ipp050", "ipp051", "ipp052", "ipp053", "ipp054", "ipp055", "ipp056", "ipp057", "ipp058", "ipp059", + "ipp060", "ipp061", "ipp062", "ipp063", "ipp064", "ipp065", "ipp066", + "ippc01", "ippc02", "ippc03", "ippc04", "ippc05", "ippc06", "ippc07", "ippc08", "ippc09", + "ippc10", "ippc11", "ippc12", "ippc13", "ippc14", "ippc15", "ippc16", "ippc17", "ippc18", "ippc19", + "ippc20", "ippc21", "ippc22", "ippc23", "ippc24", "ippc25", "ippc26", "ippc27", "ippc28", "ippc29", + "ippc30", "ippc31", "ippc32", "ippc33", "ippc34", "ippc35", "ippc36", "ippc37", "ippc38", "ippc39", + "ippc40", "ippc41", "ippc42", "ippc43", "ippc44", "ippc45", "ippc46", "ippc47", "ippc48", "ippc49", + "ippc50", "ippc51", "ippc52", "ippc53", "ippc54", "ippc55", "ippc56", "ippc57", "ippc58", "ippc59", + "ippc60", "ippc61", "ippc62", "ippc63", + "stare00", "stare01", "stare02", "stare03", "stare04", + "ippdb00", "ippdb01", "ippdb02", "ippdb03", + "stsci00", "stsci01", "stsci02", "stsci03", "stsci04", + "stsci05", "stsci06", "stsci07", "stsci08", "stsci09", + ] + +class SshThread(threading.Thread): + def __init__(self, node): + threading.Thread.__init__(self) + self.node = node + def run(self): + subprocess.Popen( [ 'ssh', + self.node, + '/home/panstarrs/ippm/operations/who_is_using_the_cluster/get_processes_information.py'] ) + +if __name__=='__main__': + ssh_threads = [] + for node in nodes: + try: + ssh_threads.append(SshThread(node)) + except socket.error, e: + print 'Exception caught for node %s' % node + print ' %s' % e + for t in ssh_threads: + try: + t.start() + except Exception, e: + print t.node, e + ssh_threads.remove(t) + for t in ssh_threads: + try: + t.join() + except RuntimeError, e: + print t.node, ": ", e +