Index: /branches/eam_branches/ipp-20111122/Nebulous-Server/bin/neb-admin =================================================================== --- /branches/eam_branches/ipp-20111122/Nebulous-Server/bin/neb-admin (revision 33637) +++ /branches/eam_branches/ipp-20111122/Nebulous-Server/bin/neb-admin (revision 33638) @@ -313,5 +313,5 @@ SELECT S.*,vol_id FROM ( -- Select objects that are in our so_id range, with two copies (and two requested copies) - SELECT MAX(ins_id) AS MAXins_id,MIN(ins_id) AS MINins_id,so_id,ext_id,value AS user_copies FROM + SELECT MAX(ins_id) AS MAXins_id,MIN(ins_id) AS MINins_id,so_id,ext_id FROM storage_object JOIN storage_object_xattr USING(so_id) RIGHT JOIN instance USING(so_id) WHERE name = 'user.copies' AND value >= 2 @@ -469,5 +469,5 @@ my $so_id = $obj->{so_id}; my $key = $obj->{ext_id}; - my $copies = $obj->{user_copies}; +# my $copies = $obj->{user_copies}; my $source_vol_id = $obj->{vol_id}; @@ -487,5 +487,5 @@ my %md = ( key => $key, - copies => $copies, +# copies => $copies, so_id => $so_id, source_vol_id => $source_vol_id, @@ -543,12 +543,13 @@ 'ipp056.0' => 64,'ipp057.0' => 65,'ipp058.0' => 66,'ipp059.0' => 67,'ipp060.0' => 68,'ipp061.0' => 69,'ipp062.0' => 70, 'ipp063.0' => 71,'ipp064.0' => 72,'ipp065.0' => 73,'ipp066.0' => 74); - %{ $destination{VOL} } = ('ota01' => 7, 'ota02' => 8, 'ota03' => 3, 'ota04' => 3, 'ota05' => 4, 'ota06' => 4, - 'ota10' => 50, 'ota11' => 50, 'ota12' => 6, 'ota13' => 6, 'ota14' => 7, 'ota15' => 8, 'ota16' => 15, 'ota17' => 15, - 'ota20' => 16, 'ota21' => 18, 'ota22' => 18, 'ota23' => 17, 'ota24' => 9, 'ota25' => 9, 'ota26' => 10, 'ota27' => 11, - 'ota30' => 12, 'ota31' => 52, 'ota32' => 14, 'ota33' => 14, 'ota34' => 19, 'ota35' => 16, 'ota36' => 20, 'ota37' => 12, - 'ota40' => 32, 'ota41' => 11, 'ota42' => 10, 'ota43' => 17, 'ota44' => 49, 'ota45' => 23, 'ota46' => 24, 'ota47' => 25, - 'ota50' => 26, 'ota51' => 27, 'ota52' => 28, 'ota53' => 29, 'ota54' => 30, 'ota55' => 31, 'ota56' => 51, 'ota57' => 33, - 'ota60' => 34, 'ota61' => 35, 'ota62' => 36, 'ota63' => 37, 'ota64' => 38, 'ota65' => 39, 'ota66' => 41, 'ota67' => 42, - 'ota71' => 43, 'ota72' => 44, 'ota73' => 45, 'ota74' => 46, 'ota75' => 47, 'ota76' => 48); +# This is replaced below. +# %{ $destination{VOL} } = ('ota01' => 7, 'ota02' => 8, 'ota03' => 3, 'ota04' => 3, 'ota05' => 4, 'ota06' => 4, +# 'ota10' => 50, 'ota11' => 50, 'ota12' => 6, 'ota13' => 6, 'ota14' => 7, 'ota15' => 8, 'ota16' => 15, 'ota17' => 15, +# 'ota20' => 16, 'ota21' => 18, 'ota22' => 18, 'ota23' => 17, 'ota24' => 9, 'ota25' => 9, 'ota26' => 10, 'ota27' => 11, +# 'ota30' => 12, 'ota31' => 52, 'ota32' => 14, 'ota33' => 14, 'ota34' => 19, 'ota35' => 16, 'ota36' => 20, 'ota37' => 12, +# 'ota40' => 32, 'ota41' => 11, 'ota42' => 10, 'ota43' => 17, 'ota44' => 49, 'ota45' => 23, 'ota46' => 24, 'ota47' => 25, +# 'ota50' => 26, 'ota51' => 27, 'ota52' => 28, 'ota53' => 29, 'ota54' => 30, 'ota55' => 31, 'ota56' => 51, 'ota57' => 33, +# 'ota60' => 34, 'ota61' => 35, 'ota62' => 36, 'ota63' => 37, 'ota64' => 38, 'ota65' => 39, 'ota66' => 41, 'ota67' => 42, +# 'ota71' => 43, 'ota72' => 44, 'ota73' => 45, 'ota74' => 46, 'ota75' => 47, 'ota76' => 48); %{ $destination{HOST} } = ('ota01' => 'ipp010','ota02' => 'ipp011','ota03' => 'ipp006','ota04' => 'ipp054','ota05' => 'ipp007','ota06' => 'ipp055', 'ota10' => 'ipp008','ota11' => 'ipp056','ota12' => 'ipp009','ota13' => 'ipp057','ota14' => 'ipp058','ota15' => 'ipp059','ota16' => 'ipp012','ota17' => 'ipp060', @@ -577,5 +578,11 @@ 'ipp023' => 1,'ipp024' => 1,'ipp025' => 1,'ipp026' => 1,'ipp028' => 1,'ipp029' => 1,'ipp030' => 1,'ipp031' => 1,'ipp032' => 1,'ipp033' => 1, 'ipp034' => 1,'ipp035' => 1,'ipp036' => 1,'ipp038' => 1,'ipp039' => 1,'ipp040' => 1,'ipp041' => 1,'ipp042' => 1,'ipp045' => 1,'ipp046' => 1, - 'ipp047' => 1,'ipp048' => 1,'ipp049' => 1,'ipp050' => 1,'ipp051' => 1,'ipp052' => 1); + 'ipp047' => 1,'ipp048' => 1,'ipp049' => 1,'ipp050' => 1,'ipp051' => 1,'ipp052' => 1, + # These hosts are blocked due to concern about their RAID arrays. +# 'ipp054' => 1,'ipp055' => 1,'ipp056' => 1,'ipp057' => 1,'ipp058' => 1,'ipp059' => 1, +# 'ipp060' => 1,'ipp061' => 1,'ipp062' => 1,'ipp063' => 1,'ipp064' => 1,'ipp065' => 1, 'ipp066' => 1 +# 'ipp064' => 1, 'ipp066' => 1 + 'ipp064' => 1 + ); if (not defined $so_id_start) { $so_id_start = 0; } if (not defined $so_id_range) { $so_id_range = 100000; } @@ -597,24 +604,53 @@ # Determine a list of first instances +# my $query = $dbh->prepare( +# " +# SELECT S.so_id,ext_id,ins_id,uri,substr(uri,-10,5) AS class_id,vol_id,name AS vol_name,host AS vol_host FROM ( +# select * FROM ( +# select MAX(ins_id) AS MAXins_id, MIN(ins_id) AS MINins_id, so_id,ext_id,value AS user_copies FROM storage_object +# JOIN storage_object_xattr USING(so_id) +# RIGHT JOIN instance USING(so_id) +# WHERE name = 'user.copies' +# AND value >= 2 +# AND ext_id LIKE '%ota%fits' +# AND so_id >= $so_id_start +# AND so_id < $so_id_end +# GROUP BY so_id +# ) AS V +# WHERE MAXins_id != MINins_id +# ) AS S +# JOIN instance ON (S.MINins_id = instance.ins_id AND S.so_id = instance.so_id) +# JOIN volume USING(vol_id) WHERE volume.available = 1 +# LIMIT $limit +# OFFSET $offset +# "); my $query = $dbh->prepare( " -SELECT S.so_id,ext_id,ins_id,uri,substr(uri,-10,5) AS class_id,vol_id,name AS vol_name,host AS vol_host FROM ( - select * FROM ( - select MAX(ins_id) AS MAXins_id, MIN(ins_id) AS MINins_id, so_id,ext_id,value AS user_copies FROM storage_object - JOIN storage_object_xattr USING(so_id) - RIGHT JOIN instance USING(so_id) - WHERE name = 'user.copies' - AND value >= 2 - AND ext_id LIKE '%ota%fits' - AND so_id >= $so_id_start - AND so_id < $so_id_end - GROUP BY so_id - ) AS V - WHERE MAXins_id != MINins_id - ) AS S - JOIN instance ON (S.MINins_id = instance.ins_id AND S.so_id = instance.so_id) JOIN volume USING(vol_id) WHERE volume.available = 1 - LIMIT $limit - OFFSET $offset +SELECT Z.so_id,ext_id,Z.ins_id,Z.uri,substr(Z.uri,-10,5) AS class_id,Z.vol_id,vol_name,vol_host, + instance.vol_id AS second_vol_id,volume.name AS second_name,volume.host AS second_host FROM ( + SELECT S.so_id,ext_id,ins_id,uri,substr(uri,-10,5) AS class_id,vol_id,name AS vol_name,host AS vol_host,MAXins_id FROM ( + select * FROM ( + select MAX(ins_id) AS MAXins_id, MIN(ins_id) AS MINins_id, so_id,ext_id FROM storage_object + JOIN storage_object_xattr USING(so_id) + RIGHT JOIN instance USING(so_id) + WHERE name = 'user.copies' + AND value >= 2 + AND ext_id LIKE '%ota%fits' + AND so_id >= $so_id_start + AND so_id < $so_id_end + GROUP BY so_id + ) AS V + WHERE MAXins_id != MINins_id + ) AS S + JOIN instance ON (S.MINins_id = instance.ins_id AND S.so_id = instance.so_id) + JOIN volume USING(vol_id) WHERE volume.available = 1 + AND volume.name AS $requested_source + ) AS Z +JOIN instance ON (MAXins_id = instance.ins_id AND Z.so_id = instance.so_id) +JOIN volume ON(instance.vol_id = volume.vol_id) WHERE volume.available = 1 +LIMIT $limit +OFFSET $offset "); + $query->execute(); @@ -650,4 +686,7 @@ my $source_host = $obj->{vol_host}; + my $second_vol_id = $obj->{second_vol_id}; + my $second_vol_name = $obj->{second_name}; + my $second_vol_host = $obj->{second_host}; my $destination_vol_id = $destination{VOL}{$class_id}; @@ -660,7 +699,5 @@ next; } - if (defined($blocked_volume{$destination_host})) { - next; - } + $destination_uri =~ s/$source_name/$destination_name/; @@ -680,5 +717,18 @@ destination_host => $destination_host, destination_uri => $destination_uri, + second_host => $second_vol_host, + second_vol_id => $second_vol_id +# blocked_host_bool => defined($blocked_volume{$destination_host}) ); + + if ((defined($blocked_volume{$destination_host}))|| + ($source_vol_id == $destination_vol_id)|| + ($second_vol_id == $destination_vol_id)|| + ($source_name eq $destination_name)|| + ($second_vol_name eq $destination_name)) { +# print_metadata("targetFailed", \%md); + next; + } + if ($source_vol_id != $destination_vol_id) { print_metadata("targetPending", \%md); Index: /branches/eam_branches/ipp-20111122/Nebulous-Server/bin/neb-host =================================================================== --- /branches/eam_branches/ipp-20111122/Nebulous-Server/bin/neb-host (revision 33637) +++ /branches/eam_branches/ipp-20111122/Nebulous-Server/bin/neb-host (revision 33638) @@ -18,5 +18,5 @@ my ($host, $state, $note, $db, $dbhost, $dbpass, $dbuser, - $allocate, $available, $debug); + $allocate, $available, $debug,$volume); # $db = $ENV{'NEB_DB'} unless $db; @@ -56,5 +56,5 @@ 'dbuser=s' => \$dbuser, 'dbpass=s' => \$dbpass, - + 'volume' => \$volume, ) || pod2usage( 2 ); @@ -98,5 +98,10 @@ my %constraint; - $constraint{'v.host'} = $host; + if (defined($volume)) { + $constraint{'v.name'} = $host; + } + else { + $constraint{'v.host'} = $host; + } my %set; $set{'v.allocate'} = $allocate if defined $allocate; Index: /branches/eam_branches/ipp-20111122/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /branches/eam_branches/ipp-20111122/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 33637) +++ /branches/eam_branches/ipp-20111122/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 33638) @@ -351,5 +351,5 @@ $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin; $command .= " -dateobs_end $dateobs_end" if $dateobs_end; - } else { + } elsif ($req_type ne "byid") { $command .= " -mjd_obs_begin $mjd_min" if $mjd_min; $command .= " -mjd_obs_end $mjd_max" if $mjd_max; @@ -438,4 +438,7 @@ } elsif ($stage eq "stack") { $stage_id = $out->{stack_id}; + # dquery wants these set + $out->{magicked} = 0; + $out->{data_state} = $out->{state}; } $out->{stage_id} = $stage_id; @@ -487,10 +490,10 @@ my $command = "$difftool -dbname $imagedb -pstamp_order"; - my $listrun = 0; + my $choose_components = 0; if ($byid) { if ($skycell_id and ($skycell_id ne 'all')) { $command .= " -diffskyfile -diff_id $id -skycell_id $skycell_id"; } else { - $listrun = 1; + $choose_components = 1; $command .= " -listrun -diff_id $id"; # the following is a work around for the problem reported in ticket #1394 @@ -512,5 +515,5 @@ my $images; - if ($listrun) { + if ($choose_components) { $images = selectComponents($ipprc, $imagedb, 'byid', 'diff', $rowList, $output, $verbose); } else { @@ -603,5 +606,5 @@ my @imageList = ($image); - setRowRefs($rowList, \@imageList); + setRowRefs($rowList, \@imageList) unless $choose_components; # the $image is going to be returned directly in this case so we need to duplicate # some of processing that lookup does for other img_types @@ -847,4 +850,5 @@ { my $command = "$whichimage $ra $dec"; + $command .= " --tess_id $requested_tess_id" if $requested_tess_id; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: /branches/eam_branches/ipp-20111122/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- /branches/eam_branches/ipp-20111122/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 33637) +++ /branches/eam_branches/ipp-20111122/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 33638) @@ -27,4 +27,6 @@ $PSTAMP_SELECT_PSF $PSTAMP_SELECT_BACKMDL + $PSTAMP_SELECT_JPEG + $PSTAMP_SELECT_UNCOMPRESSED $PSTAMP_SELECT_INVERSE $PSTAMP_SELECT_UNCONV @@ -64,4 +66,6 @@ our $PSTAMP_SELECT_PSF = 16; our $PSTAMP_SELECT_BACKMDL = 32; +our $PSTAMP_SELECT_JPEG = 64; +our $PSTAMP_SELECT_UNCOMPRESSED = 512; our $PSTAMP_SELECT_INVERSE = 1024; our $PSTAMP_SELECT_UNCONV = 2048; Index: /branches/eam_branches/ipp-20111122/dbconfig/changes.txt =================================================================== --- /branches/eam_branches/ipp-20111122/dbconfig/changes.txt (revision 33637) +++ /branches/eam_branches/ipp-20111122/dbconfig/changes.txt (revision 33638) @@ -2184,2 +2184,73 @@ alter table addRun add column stage_extra1 int after stage_id; + + +-- This adds new features to addRun, first add the tables: + +CREATE TABLE mergedvodbRun ( + merge_id BIGINT AUTO_INCREMENT, + minidvodb_id BIGINT, + mergedvodb VARCHAR(64) NOT NULL, + mergedvodb_path VARCHAR(255) NOT NULL, + state VARCHAR(64) NOT NULL, + creation_date TIMESTAMP, + PRIMARY KEY(merge_id), KEY(minidvodb_id), KEY(mergedvodb), INDEX(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +CREATE TABLE mergedvodbProcessed ( + merge_id BIGINT, + dtime_verify FLOAT, + dtime_merge FLOAT, + dtime_script FLOAT, + epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + fault SMALLINT NOT NULL, + UNIQUE KEY (merge_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbCopy ( + mergedvodbcopy_id BIGINT AUTO_INCREMENT, + merge_id BIGINT, + mergedvodb_rsync_path VARCHAR(255) NOT NULL, + destination_host VARCHAR(255) NOT NULL, + fault SMALLINT NOT NULL, + state VARCHAR(64) NOT NULL, + epoch TIMESTAMP, + dtime FLOAT, + PRIMARY KEY(mergedvodbcopy_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +alter table mergedvodbProcessed add column merge_order BIGINT after merge_id; + +-- then copy over the info into the 2 tables: + insert into mergedvodbRun ( minidvodb_id, mergedvodb,mergedvodb_path, + state, creation_date) select minidvodb_id, minidvodb_group,mergedvodb_path, +minidvodbRun.state, + minidvodbRun.creation_date from minidvodbRun where minidvodb_id >= 0; + +insert into mergedvodbProcessed (merge_id, merge_order, dtime_verify, +dtime_merge, dtime_script, epoch, fault) select merge_id, merge_order, +minidvodbProcessed.dtime_verify, minidvodbProcessed.dtime_merge, +minidvodbProcessed.dtime_script, minidvodbProcessed.epoch, +minidvodbProcessed.fault from minidvodbProcessed join mergedvodbRun using +(minidvodb_id) where merge_id > -1 ; + +--then clean up the tables: +-- once the tables have had the columns migrated over, then and only then do this: + +alter table minidvodbRun drop column mergedvodb_path; +alter table minidvodbProcessed drop column dtime_verify; +alter table minidvodbProcessed drop column merge_order; +alter table minidvodbProcessed drop column dtime_merge; +alter table minidvodbProcessed drop column mergedvodb_path; + + +--then to keep compatible with old camruns (stage_extra1 = negative means it used a destreaked smf) + +--it's commented out as it only needs to be run once (and only on gpc1) +--update addRun set stage_extra1 = -1 where stage = 'cam' <= 483624; + Index: /branches/eam_branches/ipp-20111122/dbconfig/ipp.m4 =================================================================== --- /branches/eam_branches/ipp-20111122/dbconfig/ipp.m4 (revision 33637) +++ /branches/eam_branches/ipp-20111122/dbconfig/ipp.m4 (revision 33638) @@ -25,4 +25,5 @@ include(magic.md) include(minidvodb.md) +include(mergedvodb.md) include(calibration.md) include(flatcorr.md) Index: /branches/eam_branches/ipp-20111122/dbconfig/mergedvodb.md =================================================================== --- /branches/eam_branches/ipp-20111122/dbconfig/mergedvodb.md (revision 33638) +++ /branches/eam_branches/ipp-20111122/dbconfig/mergedvodb.md (revision 33638) @@ -0,0 +1,29 @@ +mergedvodbRun METADATA + merge_id S64 0 + minidvodb_id S64 0 + mergedvodb STR 64 + mergedvodb_path STR 255 + state STR 64 + creation_date UTC 0001-01-01T00:00:00Z +END + +mergedvodbProcessed METADATA + merge_id S64 0 + merge_order S64 0 + dtime_verify F32 0.0 + dtime_merge F32 0.0 + dtime_script F32 0.0 + epoch UTC 0001-01-01T00:00:00Z + fault S16 0 +END + +mergedvodbCopy METADATA + mergevodbcopy_id S64 0 + merge_id S64 0 + mergedvodb_rsync_path STR 255 + destination_host STR 255 + fault S16 0 + state STR 64 + epoch UTC 0001-01-01T00:00:00Z + dtime F32 0.0 +END Index: /branches/eam_branches/ipp-20111122/dbconfig/minidvodb.md =================================================================== --- /branches/eam_branches/ipp-20111122/dbconfig/minidvodb.md (revision 33637) +++ /branches/eam_branches/ipp-20111122/dbconfig/minidvodb.md (revision 33638) @@ -4,5 +4,4 @@ minidvodb_group STR 64 minidvodb_path STR 255 - mergedvodb_path STR 255 state STR 64 creation_date UTC 0001-01-01T00:00:00Z @@ -11,12 +10,8 @@ minidvodbProcessed METADATA minidvodb_id S64 0 - merge_order S64 0 dtime_resort F32 0.0 dtime_relphot F32 0.0 - dtime_verify F32 0.0 - dtime_merge F32 0.0 dtime_script F32 0.0 epoch UTC 0001-01-01T00:00:00Z - mergedvodb_path STR 255 fault S16 0 END Index: /branches/eam_branches/ipp-20111122/dbconfig/pstamp.md =================================================================== --- /branches/eam_branches/ipp-20111122/dbconfig/pstamp.md (revision 33637) +++ /branches/eam_branches/ipp-20111122/dbconfig/pstamp.md (revision 33638) @@ -48,4 +48,6 @@ dep_id S64 0 fault_count S32 0 + parent_id S64 0 + is_parent BOOL f END Index: /branches/eam_branches/ipp-20111122/dbconfig/staticsky.md =================================================================== --- /branches/eam_branches/ipp-20111122/dbconfig/staticsky.md (revision 33637) +++ /branches/eam_branches/ipp-20111122/dbconfig/staticsky.md (revision 33638) @@ -31,2 +31,32 @@ quality S16 0 # Key END + +skycalRun METADATA + skycal_id S64 0 # Primary Key AUTO_INCREMENT + sky_id S64 0 # fkey(sky_id) ref staticskyRun(sky_id) + stack_id S64 0 # fkey(stack_id) ref stackRun(stack_id) + state STR 64 # Key + workdir STR 255 + label STR 64 # Key + data_group STR 64 # Key + dist_group STR 64 # Key + reduction STR 64 # Reduction class + registered TAI NULL + note STR 255 +END + +skycalResult METADATA + skycal_id S64 0 # Primary Key fkey(skycal_id) ref skycalRun(skycal_id) + path_base STR 255 + dtime_script F32 0.0 + dtime_astrom F32 0.0 + sigma_ra F32 0.0 + sigma_dec F32 0.0 + n_astrom S32 0 + zpt_obs F32 0.0 + zpt_stdev F32 0.0 + quality S16 0 # Key + software_ver STR 16 + hostname STR 64 + fault S16 0 # Key +END Index: /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarDbSchema.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarDbSchema.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarDbSchema.sql (revision 33638) @@ -0,0 +1,468 @@ +-- MySQL dump 10.11 +-- +-- Host: ippdb01 Database: czardb +-- ------------------------------------------------------ +-- Server version 5.0.77-log + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `burntool` +-- + +DROP TABLE IF EXISTS `burntool`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `burntool` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + KEY `burntoolIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `cam` +-- + +DROP TABLE IF EXISTS `cam`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `cam` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `camIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `chip` +-- + +DROP TABLE IF EXISTS `chip`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `chip` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `chipIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `cluster_space` +-- + +DROP TABLE IF EXISTS `cluster_space`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `cluster_space` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `total` float default NULL, + `available` float default NULL, + `used` float default NULL, + `hostsOver98` smallint(6) default NULL, + KEY `clusterSpaceIndex` (`timestamp`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `current_labels` +-- + +DROP TABLE IF EXISTS `current_labels`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `current_labels` ( + `server` varchar(128) default NULL, + `label` varchar(128) default NULL, + `priority` int(11) NOT NULL default '0' +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `deleted_datastore` +-- + +DROP TABLE IF EXISTS `deleted_datastore`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `deleted_datastore` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `ipptopsps_deleted_datastoreIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `deleted_dxlayer` +-- + +DROP TABLE IF EXISTS `deleted_dxlayer`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `deleted_dxlayer` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `ipptopsps_deleted_dxlayerIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `deleted_local` +-- + +DROP TABLE IF EXISTS `deleted_local`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `deleted_local` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `ipptopsps_deleted_local` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `diff` +-- + +DROP TABLE IF EXISTS `diff`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `diff` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `diffIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `dist` +-- + +DROP TABLE IF EXISTS `dist`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `dist` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `distIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `fake` +-- + +DROP TABLE IF EXISTS `fake`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `fake` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `fakeIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `hosts` +-- + +DROP TABLE IF EXISTS `hosts`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `hosts` ( + `host` varchar(128) default NULL, + `total` float default NULL, + `available` float default NULL, + `used` float default NULL, + `writable` tinyint(4) NOT NULL default '0', + `readable` tinyint(4) NOT NULL default '0' +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `loaded_to_ODM` +-- + +DROP TABLE IF EXISTS `loaded_to_ODM`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `loaded_to_ODM` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `ipptopsps_loaded_to_ODMIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `loaded_to_datastore` +-- + +DROP TABLE IF EXISTS `loaded_to_datastore`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `loaded_to_datastore` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `ipptopsps_loaded_to_datastoreIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `magic` +-- + +DROP TABLE IF EXISTS `magic`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `magic` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `magicIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `magicDS` +-- + +DROP TABLE IF EXISTS `magicDS`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `magicDS` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `magicDSIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `merge_worthy` +-- + +DROP TABLE IF EXISTS `merge_worthy`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `merge_worthy` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `ipptopsps_merge_worthyIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `merged` +-- + +DROP TABLE IF EXISTS `merged`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `merged` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `ipptopsps_mergedIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `nightlyscience` +-- + +DROP TABLE IF EXISTS `nightlyscience`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `nightlyscience` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `status` varchar(128) default 'NONE' +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `processed` +-- + +DROP TABLE IF EXISTS `processed`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `processed` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `ipptopsps_processedIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `pub` +-- + +DROP TABLE IF EXISTS `pub`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `pub` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + KEY `pubIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `reverts` +-- + +DROP TABLE IF EXISTS `reverts`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `reverts` ( + `stage` varchar(128) default NULL, + `reverting` tinyint(4) default NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `revision` +-- + +DROP TABLE IF EXISTS `revision`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `revision` ( + `revision` int(11) NOT NULL default '0', + `created` timestamp NOT NULL default CURRENT_TIMESTAMP, + PRIMARY KEY (`revision`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `server_dates` +-- + +DROP TABLE IF EXISTS `server_dates`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `server_dates` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `server` varchar(64) default 'NONE', + `date` varchar(64) default 'NONE' +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `servers` +-- + +DROP TABLE IF EXISTS `servers`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `servers` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `server` varchar(128) default NULL, + `alive` tinyint(4) default NULL, + `running` tinyint(4) default NULL, + KEY `serverIndex` (`timestamp`,`server`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `stack` +-- + +DROP TABLE IF EXISTS `stack`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `stack` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `stackIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `warp` +-- + +DROP TABLE IF EXISTS `warp`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `warp` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `label` varchar(128) default 'NONE', + `pending` bigint(20) NOT NULL, + `faults` bigint(20) NOT NULL, + `processed` bigint(20) NOT NULL default '0', + KEY `warpIndex` (`timestamp`,`label`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2012-03-06 6:00:30 Index: /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/czarconfig.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/czarconfig.xml (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/czarconfig.xml (revision 33638) @@ -21,5 +21,5 @@ gpc1 - ippdb03 + ippdb01 ippMonitor ippMonitor Index: /branches/eam_branches/ipp-20111122/ippMonitor/def/lapExposures.d =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/def/lapExposures.d (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippMonitor/def/lapExposures.d (revision 33638) @@ -6,54 +6,55 @@ UNRESTRICTED WHERE 0=1 -ARGS ARG1 rawImfile.exp_id=$lapExp.exp_id -ARGS ARG2 chipRun.chip_id=$lapExp.chip_id -ARGS ARG3 chipRun.chip_id=$lapExp.pair_id -ARGS ARG4 camRun.cam_id=$camRun1.cam_id -ARGS ARG5 camRun.cam_id=$camRun2.cam_id -ARGS ARG6 fakeRun.fake_id=$fakeRun1.fake_id -ARGS ARG7 fakeRun.fake_id=$fakeRun2.fake_id -ARGS ARG8 warpRun.warp_id=$warpRun1.warp_id -ARGS ARG9 warpRun.warp_id=$warpRun2.warp_id -ARGS ARG10 diffRun.diff_id=$diffRun.diff_id +ARGS ARG1 rawImfile.exp_id=$lapExp.exp_id +ARGS ARG2 chipRun.chip_id=$lapExp.chip_id +ARGS ARG3 chipRun.chip_id=$lapExp.pair_id +ARGS ARG4 camRun.cam_id=$camRun1.cam_id +ARGS ARG5 camRun.cam_id=$camRun2.cam_id +ARGS ARG6 fakeRun.fake_id=$fakeRun1.fake_id +ARGS ARG7 fakeRun.fake_id=$fakeRun2.fake_id +ARGS ARG8 warpRun.warp_id=$warpRun1.warp_id +ARGS ARG9 warpRun.warp_id=$warpRun2.warp_id +ARGS ARG10 diffRun.diff_id=$diffRun.diff_id -# field size format name show link to extras -FIELD lapRun.seq_id, 6, %d, seq_id -FIELD lapExp.lap_id, 6, %d, lap_id -FIELD lapExp.data_state, 6, %s, State (lapExp) -FIELD lapExp.exp_id, 6, %d, Exp ID, value, rawImfile.php, ARG1 -FIELD lapExp.chip_id, 6, %d, Chip1 (lapExp.chip_id), value, chipStageExp.php, ARG2 -FIELD chipRun1.state AS ch1state, 6, %s, Chip1 State -FIELD camRun1.cam_id AS cam_id_1, 6, %d, Cam1 ID, value, camStageExp.php, ARG4 -FIELD camRun1.state AS ca1state, 6, %s, Cam1 State -FIELD fakeRun1.fake_id AS fake_id_1, 6, %d, Fake1 ID, value, fakeStageExp.php, ARG6 -FIELD fakeRun1.state AS fa1state, 6, %s, Fake1 State -FIELD warpRun1.warp_id AS warp_id_1, 6, %d, Warp1 ID, value, warpStageExp.php, ARG8 -FIELD warpRun1.state AS wa1state, 6, %s, Warp1 State -FIELD lapExp.pair_id, 6, %d, Chip2 ID (lapExp.chip_id), value, chipStageExp.php, ARG3 -FIELD chipRun2.state AS ch2state, 6, %s, Chip2 State -FIELD camRun2.cam_id AS cam_id_2, 6, %d, Cam2 ID, value, camStageExp.php, ARG5 -FIELD camRun2.state AS ca2state, 6, %s, Cam2 State -FIELD fakeRun2.fake_id AS fake_id_2, 6, %d, Fake2 ID, value, fakeStageExp.php, ARG7 -FIELD fakeRun2.state AS fa2state, 6, %s, Fake2 State -FIELD warpRun2.warp_id AS warp_id_2, 6, %d, Warp2 ID, value, warpStageExp.php, ARG9 -FIELD warpRun2.state AS wa2state, 6, %s, Warp2 State -FIELD diffRun.diff_id, 6, %d, Diff ID, value, diffStageExp.php, ARG10 -FIELD diffRun.state AS diState, 6, %s, Diff State -FIELD lapRun.tess_id, 8, %s, tess_id -FIELD lapRun.projection_cell, 10, %s, ProjCell -FIELD lapRun.filter, 7, %s, Filter -FIELD lapRun.state, 15, %s, State (lapRun) -FIELD lapRun.label, 15, %s, Label -FIELD lapRun.dist_group, 15, %s, Dist Group -FIELD lapRun.registered, 19, %T, Registered -FIELD lapRun.fault, 3, %d, fault -FIELD lapRun.quick_sass_id, 6, %d, quick_sass_id -FIELD lapRun.final_sass_id, 6, %d, final_sass_id -FIELD lapExp.private, 2, %d, Private -FIELD lapExp.pairwise, 2, %d, Pairwise -FIELD lapExp.active, 2, %d, Active -FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.object, 8, %s, Object -FIELD rawExp.comment, 80, %s, Comment +# field size format name show link to extras +FIELD lapRun.seq_id, 6, %d, seq_id +FIELD lapExp.lap_id, 6, %d, lap_id +FIELD lapExp.data_state, 6, %s, State (lapExp) +FIELD lapExp.exp_id, 6, %d, Exp ID, value, rawImfile.php, ARG1 +FIELD lapExp.chip_id, 6, %d, Chip1 (lapExp.chip_id), value, chipStageExp.php, ARG2 +FIELD chipRun1.state AS ch1state, 6, %s, Chip1 State +FIELD camRun1.cam_id AS cam_id_1, 6, %d, Cam1 ID, value, camStageExp.php, ARG4 +FIELD camRun1.state AS ca1state, 6, %s, Cam1 State +FIELD fakeRun1.fake_id AS fake_id_1, 6, %d, Fake1 ID, value, fakeStageExp.php, ARG6 +FIELD fakeRun1.state AS fa1state, 6, %s, Fake1 State +FIELD warpRun1.warp_id AS warp_id_1, 6, %d, Warp1 ID, value, warpStageExp.php, ARG8 +FIELD warpRun1.state AS wa1state, 6, %s, Warp1 State +FIELD lapExp.pair_id, 6, %d, Chip2 ID (lapExp.chip_id), value, chipStageExp.php, ARG3 +FIELD chipRun2.state AS ch2state, 6, %s, Chip2 State +FIELD camRun2.cam_id AS cam_id_2, 6, %d, Cam2 ID, value, camStageExp.php, ARG5 +FIELD camRun2.state AS ca2state, 6, %s, Cam2 State +FIELD fakeRun2.fake_id AS fake_id_2, 6, %d, Fake2 ID, value, fakeStageExp.php, ARG7 +FIELD fakeRun2.state AS fa2state, 6, %s, Fake2 State +FIELD warpRun2.warp_id AS warp_id_2, 6, %d, Warp2 ID, value, warpStageExp.php, ARG9 +FIELD warpRun2.state AS wa2state, 6, %s, Warp2 State +FIELD diffRun.diff_id, 6, %d, Diff ID, value, diffStageExp.php, ARG10 +FIELD diffRun.state AS diState, 6, %s, Diff State +FIELD lapRun.tess_id, 8, %s, tess_id +FIELD lapRun.projection_cell, 10, %s, ProjCell +FIELD diffInputSkyfile.skycell_id, 10, %s, Skycell ID +FIELD lapRun.filter, 7, %s, Filter +FIELD lapRun.state, 15, %s, State (lapRun) +FIELD lapRun.label, 15, %s, Label +FIELD lapRun.dist_group, 15, %s, Dist Group +FIELD lapRun.registered, 19, %T, Registered +FIELD lapRun.fault, 3, %d, fault +FIELD lapRun.quick_sass_id, 6, %d, quick_sass_id +FIELD lapRun.final_sass_id, 6, %d, final_sass_id +FIELD lapExp.private, 2, %d, Private +FIELD lapExp.pairwise, 2, %d, Pairwise +FIELD lapExp.active, 2, %d, Active +FIELD rawExp.dateobs, 19, %T, Date/Time +FIELD rawExp.object, 8, %s, Object +FIELD rawExp.comment, 80, %s, Comment # MODE summary Index: /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_getplot.php =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_getplot.php (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_getplot.php (revision 33638) @@ -30,4 +30,6 @@ else if ($type=="st") $filePath = "$path/czarplot_cluster.png"; +else if ($type=="id") +$filePath = "$path/ippToPsps_density_".$label.".png"; Index: /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_ipptopsps.php =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_ipptopsps.php (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_ipptopsps.php (revision 33638) @@ -155,4 +155,8 @@ echo "
"; +echo "
"; + echo "
"; +echo "
"; + echo ""; Index: /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_labels.php =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_labels.php (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_labels.php (revision 33638) @@ -169,5 +169,5 @@ echo "Postage stamp server statushere"; echo "Documentationhere"; -echo "Cluster loadhere"; +echo "Cluster loadhere cpu_report"; echo "Who's using the cluster?here"; echo "IPP metricshere"; @@ -492,5 +492,5 @@ createFormattedTableCell(($selectedLabel != "all"), $link, "All labels", 0, null); - createFormattedTableCell(0, null, "Reverts:", 0, "$columnHeaderColor"); + createFormattedTableCell(0, null, "Reverting?", 0, "$columnHeaderColor"); foreach ($stages as &$stage) { @@ -498,18 +498,21 @@ if ($stage == "burntool") continue; $reverting = getRevertStatus($db, $stage); - $link = - "czartool_labels.php?pass=".$pass - ."&proj=".$proj - ."&mode=" .$selectedMode - ."&label=".$selectedLabel - ."&stage=".$selectedStage - ."&plottype=".$plotType - ."&revertstage=".$stage - ."&revertmode="; - - if(!$reverting) {$label = "Start";$link = $link . "on";} - if($reverting) {$label = "Stop";$link = $link . "off";} + #$link = + # "czartool_labels.php?pass=".$pass + # ."&proj=".$proj + # ."&mode=" .$selectedMode + # ."&label=".$selectedLabel + # ."&stage=".$selectedStage + # ."&plottype=".$plotType + # ."&revertstage=".$stage + # ."&revertmode="; +# + #if(!$reverting) {$label = "Start";$link = $link . "on";} + #if($reverting) {$label = "Stop";$link = $link . "off";} + if(!$reverting) {$label = "no";} + if($reverting) {$label = "yes";} unset($reverting); - createFormattedTableCell(1, $link, $label, 0, "$columnHeaderColor"); + #createFormattedTableCell(1, $link, $label, 0, "$columnHeaderColor"); + createFormattedTableCell(0, null, $label, 0, "$columnHeaderColor"); #echo "$label"; Index: /branches/eam_branches/ipp-20111122/ippMonitor/scripts/skyplot.dvo =================================================================== --- /branches/eam_branches/ipp-20111122/ippMonitor/scripts/skyplot.dvo (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippMonitor/scripts/skyplot.dvo (revision 33638) @@ -8,4 +8,5 @@ black gold + orange end @@ -17,4 +18,5 @@ 7 3 + 0 end @@ -26,4 +28,5 @@ 0.9 1.3 + 0.5 end Index: /branches/eam_branches/ipp-20111122/ippScripts/Build.PL =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/Build.PL (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/Build.PL (revision 33638) @@ -56,4 +56,5 @@ scripts/stack_skycell.pl scripts/staticsky.pl + scripts/destreak_restore_camera.pl scripts/magic_tree.pl scripts/magic_process.pl @@ -65,5 +66,9 @@ scripts/minidvodb_createdb.pl scripts/minidvodb_merge.pl + scripts/minidvodb_premerge.pl scripts/minidvodb_copy.pl + scripts/mergedvodb_copy.pl + scripts/mergedvodb_merge.pl + scripts/mergedvodb_queue.pl scripts/ippdb.pl scripts/ipp_cleanup.pl @@ -71,4 +76,5 @@ scripts/ipp_inject_expname.pl scripts/ipp_serial_inject.pl + scripts/ipp_serial_inject_into_nebulous.pl scripts/ipp_serial_inject_split.pl scripts/ipp_serial_inject_mosaic.pl @@ -94,4 +100,5 @@ scripts/dist_bundle.pl scripts/dist_advancerun.pl + scripts/dist_fix_file.pl scripts/dist_make_fileset.pl scripts/dist_defineruns.pl @@ -120,4 +127,5 @@ scripts/videophot_process.pl scripts/listvideocells.pl + scripts/skycalibration.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: /branches/eam_branches/ipp-20111122/ippScripts/queue_chips_by_night.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/queue_chips_by_night.pl (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippScripts/queue_chips_by_night.pl (revision 33638) @@ -0,0 +1,102 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $stdate = `date`; +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my @ARGS = @ARGV; + +my ( $date, $data_group_base, $label, $tess_id, $end_stage, $workdir_base, $dbname ); + +GetOptions( + 'date=s' => \$date, # date, like this 2011-02-12 + 'data_group_base=s' => \$data_group_base, # data_group_base, like this: isp + 'label=s' => \$label, # label + 'dbname|d=s' => \$dbname, # Database name + 'tess_id=s' => \$tess_id, # tess_id + 'end_stage=s' => \$end_stage, # end_stage + 'workdir_base=s' => \$workdir_base, # workdir + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dbname --label --tess_id --date \n Optional: --data_group_base, end_stage, workdir_base", + -exitval => 3) unless + defined $dbname and + defined $label and + defined $tess_id and + defined $date; + +my $truncdate = $date; + $truncdate =~ s/-//g; +if (defined $data_group_base) { + print "found data_group_base = $data_group_base\n"; +} else { + $data_group_base = $label; + print "using $label as data_group_base\n"; +} +if (defined $end_stage) { + print "found end_stage = $end_stage\n"; +} else { + $end_stage = "camera"; + print "using camera as end_stage\n"; +} +if (defined $workdir_base) { + print "found workdir_base = $workdir_base\n"; +} else { + $workdir_base = "neb://\@HOST\@.0/$dbname/$label"; + print "using $workdir_base as workdir\n"; +} + +# we have all the pieces, now build up the chiptool: + +my $command = "$chiptool -definebyquery -unique"; +$command .= " -dbname $dbname"; +$command .= " -dateobs_begin '$date"."T00:00:00'"; +$command .= " -dateobs_end '$date"."T23:59:59'"; +$command .= " -set_label $label"; +$command .= " -set_data_group $data_group_base.$truncdate"; +$command .= " -set_workdir $workdir_base/$truncdate"; +$command .= " -set_end_stage $end_stage"; +$command .= " -set_tess_id $tess_id"; + +print "\nrunning this command:\n\n$command\n\n"; + + +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform chiptool -definebyquery: $error_code\n"); + exit($error_code); + +} + + Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/addstar_run.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/addstar_run.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/addstar_run.pl (revision 33638) @@ -30,4 +30,5 @@ my $missing_tools; my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); @@ -37,5 +38,6 @@ } my $minidvodb_path; -my ( $add_id, $camera, $stage, $multi_num, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, +my $stage_extra1 ; +my ( $add_id, $camera, $stage, $stage_id, $multi_num, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( @@ -43,4 +45,5 @@ 'camera|c=s' => \$camera, # Camera 'stage|s=s' => \$stage, # Camera + 'stage_id|w=s' => \$stage_id, 'multi_num|w=s' => \$multi_num, # the number for a staticskymulti (for finding cmf) 'dbname|d=s' => \$dbname, # Database name @@ -71,4 +74,8 @@ defined $dvodb and defined $camera; +if ($stage =~ /cam/ && !defined $stage_id) { + my_die("cam stage requires -stage_id", $add_id, 3); + +} if ($minidvodb && !defined($minidvodb_group)) { my_die( "missing minidvodb_group", $add_id, 3 ); @@ -108,21 +115,68 @@ # it was PSASTRO.OUTPUT -my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - + +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); +my $fpaObjectsAlt = $fpaObjects; + + +if ($stage =~ /cam/) { + # if it is cam stage we need to be careful when grabbing the filename. + # This breaks down into a few steps: + + #get info about the cam_id + my $magicked; + { + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$camtool -processedexp -cam_id " . $stage_id; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get info on camRun", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + &my_die( "empty camRun", $add_id, $PS_EXIT_SYS_ERROR); + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $mparsed = $comp->{magicked}; + + if (!defined($mparsed)) { + &my_die("Unable to parse magicked", $add_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found a value for magicked:$mparsed\n"; + $magicked = $mparsed; + } + + #is this cam_id magicked or not? + if ($magicked) { + $stage_extra1 = $magicked; + $fpaObjects = $ipprc->destreaked_filename("$fpaObjects") or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + print "cam_id is magicked, using $fpaObjects for the cam smf\n"; + } else { + print "cam_id is NOT magicked, using $fpaObjects for the cam smf\n"; + } + } +} if (($stage =~/staticsky/) || ($stage =~/stack/)) { $fpaObjects =~ s/smf$/cmf/; - -} - -if ($stage =~/staticsky_multi/) { + $fpaObjectsAlt =~ s/smf$/cmf/; +} +my $checkalt = 0; +if ($stage =~/staticsky/) { $fpaObjects =~ s/smf$/cmf/; #this should do nothing because it was already modded above &my_die( "can't find the filter_num for staticsky_multi, giving up.", $add_id, $PS_EXIT_SYS_ERROR) unless (defined $multi_num); my $nice_num = sprintf ("%03d", $multi_num); $fpaObjects =~ s/cmf$/$nice_num.cmf/; #this make it look for .001.cmf, etc - + if ($multi_num == 0) { + $checkalt =1 ; #if it's staticsky and multi_num = 0 there's an alternate file to try if this one doesn't work + } } my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - + # convert supplied DVO database name to UNIX filename my $dvodbReal; @@ -131,84 +185,82 @@ $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $add_id, $PS_EXIT_CONFIG_ERROR); } - - - - my $dtime_addstar = 0; - if (defined $dvodbReal) { - if ($minidvodb) { - - #if (!defined($minidvodb_name)) { - #take the active one, if it's not defined on the command line - #reverts would have this already set, for example. - my $command = "addtool -listminidvodbrun "; - $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; - $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; - $command .= " -state 'active' -limit 1"; - $command .= " -dbname $dbname" if defined $dbname; - print $command; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); - - my $components = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); - my $comp = $$components[0]; - $minidvodb_path = $comp->{minidvodb_path}; - $minidvodb_name = $comp->{minidvodb_name}; - - if (!defined($minidvodb_path)) { - &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR); - } - if (!defined($minidvodb_name)) { - &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); - } - #} - # tack on the minidvodb part to the db. -# $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/'; -#we don't need this now that I fixed the paths - - } - else { - - $minidvodb_path = $dvodbReal; - } - -unless ($no_op) { + if ($minidvodb) { + my $command = "addtool -listminidvodbrun "; + $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; + $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; + $command .= " -state 'active' -limit 1"; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + $minidvodb_path = $comp->{minidvodb_path}; + $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_path)) { + &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR); + } + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); + } + } + else { + $minidvodb_path = $dvodbReal; + } + + unless ($no_op) { print $dvodbReal; - - ## addstar can either save the full set of detections, or just - ## the image metadata, in the dvodb. this is set in the - ## database table addRun - - # addstar requires the user to have a valid .ptolemyrc which - # in turn points at ippconfig/dvo.site - - # get the names for the camera and the real input file - my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar - my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR); - - # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) - my $command = "$addstar -update"; # XXX optionally set -update? - $command .= " -D CAMERA $camdir"; - $command .= " -D CATDIR $minidvodb_path"; - $command .= " $realFile"; - $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference + + ## addstar can either save the full set of detections, or just + ## the image metadata, in the dvodb. this is set in the + ## database table addRun + + # addstar requires the user to have a valid .ptolemyrc which + # in turn points at ippconfig/dvo.site + + # get the names for the camera and the real input file + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + + my $realFile = $ipprc->file_resolve($fpaObjects); + if (!defined($realFile)) { + print "can't find $fpaObjects\n"; + if ($checkalt) { + $realFile = $ipprc->file_resolve($fpaObjectsAlt) or &my_die("Unable to resolve $fpaObjectsAlt", $add_id, $PS_EXIT_SYS_ERROR); + $fpaObjects = $fpaObjectsAlt; # want to make sure the correct value is used. + } else { + # try the 'good one' again and fail i + #$realFile = $ipprc->file_resolve($fpaObjects) or + &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR); + } + + } + + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + my $command = "$addstar -update"; # XXX optionally set -update? + $command .= " -D CAMERA $camdir"; + $command .= " -D CATDIR $minidvodb_path"; + $command .= " $realFile"; + $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference $command .= " -image" if $image_only; if ($stage =~ /staticsky/) { $command .= " -accept-astrom "; } #careful here - this matches staticsky and staticsky_multi - my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); - } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script } } @@ -221,4 +273,5 @@ $fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path; $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; +$fpaCommand .= " -stage_extra1 $stage_extra1" if defined $stage_extra1; $fpaCommand .= " -dbname $dbname" if defined $dbname; print $fpaCommand; @@ -253,6 +306,8 @@ $command .= " -path_base $outroot" if defined $outroot; $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; + # $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; don't think we want it recorded (not sure) + $command .= " -stage_extra1 $stage_extra1" if defined $stage_extra1; $command .= " -dbname $dbname" if defined $dbname; + print $command; system ($command); } Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/background_chip.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/background_chip.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/background_chip.pl (revision 33638) @@ -42,5 +42,5 @@ # Parse the command-line arguments my ( $chip_bg_id, $class_id, $camera, $outroot, $dbname, $reduction, $verbose, - $threads, $no_update, $save_temps, $no_op, $redirect, $deburned ); + $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $magicked ); GetOptions( 'chip_bg_id=s' => \$chip_bg_id, # chipBackgroundRun identifier @@ -51,4 +51,6 @@ 'reduction=s' => \$reduction, # Reduction class 'threads=s' => \$threads, # Number of threads to use + 'chip_path_base=s' => \$chip_path_base, # optional chip_path_base + 'magicked=s' => \$magicked, # magicked status of input 'verbose' => \$verbose, # Print to stdout 'no-update' => \$no_update, # Don't update the database? @@ -88,6 +90,17 @@ # Get inputs my $in_path; # Input path -my $magicked; # Input is magicked? -{ +my $do_stats; +my $dump_config; +if ($chip_path_base and $no_update) { + # we are running outside of a chip_bg_run (perhaps by the postage stamp server) don't dump config + # or do stats. Get path to input from command line. + $in_path = $chip_path_base; + $magicked = 0 if !defined $magicked; + $do_stats = 0; + $dump_config = 0; +} else { + # normal operation. Get input parameters from the database + $do_stats = 1; + $dump_config = 1; my $command = "bgtool -chipinputs -chip_bg_id $chip_bg_id -class_id $class_id"; $command .= " -dbname $dbname" if defined $dbname; @@ -153,11 +166,11 @@ $command .= " -image $in_image"; $command .= " -mask $in_mask"; - $command .= " -stats $out_stats"; + $command .= " -stats $out_stats" if $do_stats; $command .= " -background $in_bg" if $apply_bg; $command .= " -pattern $in_pattern" if $apply_pattern; $command .= " -recipe PPBACKGROUND $recipe_ppBackground"; - $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -recipe PPSTATS CHIPSTATS" if $do_stats; $command .= " -dbname $dbname" if defined $dbname; - $command .= " -dumpconfig $out_config"; + $command .= " -dumpconfig $out_config" if $dump_config; $command .= " -tracedest $traceDest -log $logDest"; @@ -171,7 +184,7 @@ # Gather command-line arguments from statistics -my $cmdflags; # Command-line flags to add -my $quality; # Quality flag -{ +my $cmdflags = ""; # Command-line flags to add +my $quality = 0; # Quality flag +if ($do_stats) { &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats); @@ -191,7 +204,7 @@ if (!$quality and !$no_op) { - &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image); - &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_mask); - &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_config); + &my_die("Couldn't find expected output file: $out_image", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image); + &my_die("Couldn't find expected output file: $out_mask", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_mask); + &my_die("Couldn't find expected output file: $out_config", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$dump_config or $ipprc->file_exists($out_config); } Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/background_warp.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/background_warp.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/background_warp.pl (revision 33638) @@ -38,4 +38,5 @@ my ($warp_bg_id, $skycell_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $verbose, $no_update, $no_op, $redirect, $save_temps); +my ($astrometry, $imageName, $maskName, $magicked); GetOptions( 'warp_bg_id|i=s' => \$warp_bg_id, # Warp identifier @@ -46,4 +47,8 @@ 'reduction=s' => \$reduction, # Reduction class 'outroot=s' => \$outroot, # Output root name + 'image-list=s' => \$imageName, # list of input image files + 'mask-list=s' => \$maskName, # list of input mask files + 'astrometry=s' => \$astrometry, + 'magicked=s' => \$magicked, 'threads=s' => \$threads, # Number of threads to use for pswarp 'verbose' => \$verbose, # Print to stdout @@ -64,5 +69,20 @@ and defined $outroot; -my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_bg_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration +if ($imageName) { + # we have been invoked in a special mode where the caller tells us all about the inputs + # This is used by the postage stamp server to make non-background corrected images without + # the existence of a warpBackgroundRun (or chipBackgroundRun) + # Make sure all of the inputs were supplied + if (!(defined $maskName and defined $astrometry and defined $magicked)) { + print STDERR "mask-list, astrometry, and magicked must be supplied if image-list is used\n"; + if ($warp_bg_id) { + my_die( "invalid argument list.", $warp_bg_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); + } + exit $PS_EXIT_CONFIG_ERROR; + } +} + +my $ipprc = PS::IPP::Config->new( $camera ) or +my_die( "Unable to set up", $warp_bg_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id) or my_die( "Unable to get log filename", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR ); @@ -97,9 +117,9 @@ # Get list of filenames my $tempOutRoot = "/tmp/background.warp.$warp_bg_id.$skycell_id"; -my ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX", UNLINK => !$save_temps); -my ($maskFile, $maskName) = tempfile( "$tempOutRoot.mask.list.XXXX", UNLINK => !$save_temps); -my $astrometry; # Astrometry filename -my $magicked; # Magicked status -{ +if (!defined $imageName) { + # go find our inputs + my ($imageFile, $maskFile); + ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX", UNLINK => !$save_temps); + ($maskFile, $maskName) = tempfile( "$tempOutRoot.mask.list.XXXX", UNLINK => !$save_temps); my $command = "$bgtool -warpinputs"; $command .= " -warp_bg_id $warp_bg_id"; @@ -132,7 +152,7 @@ $astrometry = $astrom; } -} -close $imageFile; -close $maskFile; + close $imageFile; + close $maskFile; +} &my_die("Can't find input astrometry: $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($astrometry); Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/camera_exp.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/camera_exp.pl (revision 33638) @@ -41,7 +41,7 @@ my ( $exp_tag, $cam_id, $camera, $outroot, $dbname, $reduction, $dvodb, $verbose, $no_update, - $no_op, $redirect, $save_temps, $run_state); + $no_op, $redirect, $save_temps, $run_state, $skip_binned, $skip_masks); GetOptions( - 'exp_tag=s' => \$exp_tag, # Exposure identifier + 'exp_tag=s' => \$exp_tag, # Exposure identifier 'cam_id=s' => \$cam_id, # Camtool identifier 'camera|c=s' => \$camera, # Camera @@ -51,4 +51,6 @@ 'dvodb|w=s' => \$dvodb, # output DVO database 'run-state=s' => \$run_state, # 'new' or 'update' + 'skip-binned' => \$skip_binned, # override recipe - don't create binned images + 'skip-refmask' => \$skip_masks, # override recipe - don't create refmask 'verbose' => \$verbose, # Print to stdout 'no-update' => \$no_update, # Update the database? @@ -156,4 +158,10 @@ { # Determine if FWHM is too large to bother continuing. my $maxFWHM = 0; + # XXX: temporary hack if this is a CNP exposure set maxFWHM to a large + # value. + # Before the next tag we will put this into a recipe + if ($outroot =~ /CNP/) { + $maxFWHM = 25; + } else { my $command = "$ppConfigDump -camera $camera -get-key MAX_ALLOWED_FWHM"; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -168,4 +176,5 @@ $maxFWHM = metadataLookupStr($cameraConfig, 'MAX_ALLOWED_FWHM'); } + } if ($maxFWHM) { my $expFWHM; @@ -183,5 +192,5 @@ my $produceMasks; # Produce masks? -{ +if (!$skip_masks) { # Get the PSASTRO recipe my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -"; @@ -198,5 +207,4 @@ } - ### not needed to have such an extensive temp file name. my ($list1File, $list1Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b1.list.XXXX", UNLINK => !$save_temps ); # For binning 1 @@ -254,5 +262,5 @@ # Make the jpeg for binning 1 - if ($run_state eq 'new') { + if (!$skip_binned and ($run_state eq 'new')) { my $command = "$ppImage -list $list1Name $outroot"; # Command to run $command .= " -recipe PPIMAGE $recipe1"; @@ -269,5 +277,5 @@ # Make the jpeg for binning 2 - if ($run_state eq 'new') { + if (!$skip_binned and ($run_state eq 'new')) { my $command = "$ppImage -list $list2Name $outroot"; # Command to run $command .= " -recipe PPIMAGE $recipe2"; @@ -288,7 +296,8 @@ $command = "$psastro"; $command .= " -list $list3Name"; - $command .= " -masklist $list4Name"; + $command .= " -masklist $list4Name" if $produceMasks; $command .= " $outroot"; $command .= " -recipe PSASTRO $recipe_psastro"; + $command .= " -Db PSASTRO:REFSTAR_MASK F" if $skip_masks; $command .= " -tracedest $traceDest -log $logDest"; $command .= " -dbname $dbname" if defined $dbname; Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/chip_imfile.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/chip_imfile.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/chip_imfile.pl (revision 33638) @@ -147,9 +147,10 @@ my $outputBin2; my $dump_config = 1; +my $do_stats = 1; +$outputStats = prepare_output("PPIMAGE.STATS", $outroot, $class_id, 1); my $do_binned_images = 1; if ($run_state eq 'new') { # prepare the files that are only created for a new run $configuration = prepare_output("PPIMAGE.CONFIG", $outroot, $class_id, 1); - $outputStats = prepare_output("PPIMAGE.STATS", $outroot, $class_id, 1); } else { $configuration = $ipprc->filename('PPIMAGE.CONFIG', $outroot, $class_id) @@ -196,5 +197,4 @@ unless ($no_op) { my $command; - my $do_stats; ## get the ppImage recipe for this camera and CHIP reduction @@ -470,5 +470,5 @@ $command .= " -Db PPIMAGE:PHOTOM FALSE"; } - if ($run_state eq "new") { + if ($run_state eq "new" or $do_stats) { $command .= " -recipe PPSTATS CHIPSTATS"; $command .= " -stats $outputStats"; @@ -562,14 +562,23 @@ # Add the processed file to the database -unless ($no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to perform chiptool -addprocessedimfile: $error_code\n"); - exit($error_code); - } -} else { - print "skipping command: $command\n"; +my $tries = 0; +while (1) { + $tries++; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform chiptool -addprocessedimfile: $error_code\n"); + exit($error_code) if $tries >= 3; + warn("Waiting 10 seconds to try again\n"); + sleep 10; + } else { + last; + } + } else { + print "skipping command: $command\n"; + last; + } } Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/destreak_restore_camera.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/destreak_restore_camera.pl (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/destreak_restore_camera.pl (revision 33638) @@ -0,0 +1,549 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print STDERR "\n\n"; +print STDERR "Starting script $0 on $host at $date\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Digest::MD5::File qw( file_md5_hex ); + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Temp qw( tempfile ); +use File::Basename qw( basename dirname ); +use File::Copy; +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config 1.01 qw( :standard ); +use Nebulous::Client; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $checkfits = can_run('checkfits') or (warn "Can't find checkfits" and $missing_tools = 1); +my $ftable = can_run('ftable') or (warn "Can't find ftable" and $missing_tools = 1); +my $censorObjects = can_run('censorObjects') or (warn "Can't find censorObjects" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Parse the command-line arguments +my ($magic_ds_id, $camera, $streaks, $cam_id, $uri, $path_base); +my ($outroot, $release, $bytes, $md5sum); +my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); + +GetOptions( + 'magic_ds_id=s' => \$magic_ds_id,# Magic destreak run identifier + 'camera=s' => \$camera, # path_base of the input + 'cam_id=s' => \$cam_id, + 'path_base=s' => \$path_base, # path_base of the input + 'dbname=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print stuff? + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --magic_ds_id --camera --cam_id --path_base", + -exitval => 3) unless + defined $magic_ds_id and + defined $camera and + defined $cam_id and + defined $path_base; + +if ($camera ne 'GPC1') { + my_die("camera must be GPC1 not $camera ", $magic_ds_id, $PS_EXIT_PROG_ERROR); +} + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile; + +my $nebulousServer = $ENV{NEB_SERVER}; +&my_die("cannot find NEB_SERVER in environment", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer; + +my $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); }; +if ($@ or not defined $nebulous) { + &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $dirname = dirname($path_base); +my $basename = basename($path_base); +my $nebulousInput = inNebulous($dirname); + +&my_die ("$dirname is not in nebulous", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) unless $nebulousInput; + +my $backup_path_base = $dirname . "/SR_" . "$basename"; +my $recovery_path_base = $dirname . "/REC_" . "$basename"; + +my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $path_base); +my $bastrom = $ipprc->filename("PSASTRO.OUTPUT", $backup_path_base); + +restore_file($astrom, $bastrom, 1) or + &my_die("failed to restore smf file", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + +# Now restore the mask files + +my @otas = qw(XY01 XY02 XY03 XY04 XY05 XY06 XY10 XY11 XY12 XY13 XY14 XY15 XY16 XY17 XY20 XY21 XY22 XY23 XY24 XY25 XY26 XY27 XY30 XY31 XY32 XY33 XY34 XY35 XY36 XY37 XY40 XY41 XY42 XY43 XY44 XY45 XY46 XY47 XY50 XY51 XY52 XY53 XY54 XY55 XY56 XY57 XY60 XY61 XY62 XY63 XY64 XY65 XY66 XY67 XY71 XY72 XY73 XY74 XY75 XY76); + +foreach my $class_id (@otas) { + my $mask = "$path_base.$class_id.mk.fits"; + my $bmask = "$backup_path_base.$class_id.mk.fits"; + my $rmask = "$recovery_path_base.$class_id.mk.fits"; + if (!restore_file($mask, $bmask, 0)) { + # no good mask file. Mark any chips that need this mask file to be useful as bad + # XXX: what about chipRuns with multiple camRuns can't we preserve those? + my $command = "$chiptool -dropprocessedimfile -cam_id $cam_id -class_id $class_id -set_quality 45"; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool -addresult: $error_code", $magic_ds_id, + $error_code); + } + } else { + print "Skipping command: $command\n"; + } + } +} + +{ + # this sets the camRun's magicked value back to zero and deletes the magicDSFile + # row from the database + my $command = "$magicdstool -revertdestreakedfile -i_am_sure"; + $command .= " -state goto_restored"; + $command .= " -magic_ds_id $magic_ds_id"; + $command .= " -component exposure"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool -addresult: $error_code", $magic_ds_id, + $error_code); + } + } else { + print "Skipping command: $command\n"; + } +} +{ + # this sets the magicDSRun.state to restored + my $command = "$magicdstool -updaterun"; + $command .= " -set_state restored"; + $command .= " -magic_ds_id $magic_ds_id"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool -addresult: $error_code", $magic_ds_id, + $error_code); + } + } else { + print "Skipping command: $command\n"; + } +} +exit 0; + +### Pau. + +sub restore_file { + my $original_url = shift; + my $backup_url = shift; + my $smf = shift; + + print "\nStarting restore for $original_url\n"; + + # It is the backup that we expect to want to use, check it first. + my $backup = check_file($backup_url, 1, $smf); + + # Is backup a good file whether censored or not? + my $backup_ok = ($backup->{so_exists} and $backup->{num_instances}); + + # Check original, skip full check if backup is ok and uncensored since the backup + # is the one that we want to keep. + my $original = check_file($original_url, (!$backup_ok or $backup->{censored}), $smf); + + my $swap_urls = 0; + my $kill_censored = 0; + my $replicate = 0; + + if ($backup_ok and !$backup->{censored}) { + # + # Normal case. A good uncensored backup file is available + # + if ($backup->{inconsistent}) { + # At least one of the instances was bad. Repair. + repair_file($backup, $smf) or + &my_die( "failed to repair $backup_url", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + } + + $replicate = $backup->{num_instances} < 2; + + if ($original->{so_exists}) { + # + # Normal case. + # + $swap_urls = 1; + $kill_censored = 1; + } else { + # no file at original url rename the backup + # note: rename_file prints a message and dies on failure + rename_file($backup_url, $original_url); + } + } else { + # Something is wrong with the backup file, either not ok or censored + # check the original + my $original_ok = ($original->{so_exists} and $original->{num_instances}); + if ($original_ok) { + if ($original->{inconsistent}) { + repair_file($original, $smf) or + &my_die( "failed to repair $original_url", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + } + if (!$original->{censored}) { + # Note this is where we end up if this file has already been restored + print "Keeping uncensored file at original url: $original_url\n"; + } else { + print "No uncensored backup. Keeping censored file at original url: $original_url\n"; + } + $kill_censored = $backup->{so_exists}; + $replicate = $original->{num_instances} < 2; + } else { + # Original is bad. Is the backup at least a good file. If so swap it back in even if it is censored. + if ($backup_ok) { + print "No file found at $original_url\n"; + print "Swapping in $backup_url\n"; + if ($backup->{inconsistent}) { + repair_file($backup, $smf) or + &my_die( "failed to repair $backup_url", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + } + $replicate = $backup->{num_instances} < 2; + if ($original->{so_exists}) { + $swap_urls = 1; + $kill_censored = 1; + } else { + # note: rename_file prints a message and dies on failure + rename_file($backup_url, $original_url); + } + } else { + # we're out of luck + print "No good file for $original_url found\n"; + return 0; + } + } + } + + # Ok Now we know what we need to do so do it + + if (!$no_op) { + if ($swap_urls) { + if (! $nebulous->swap($backup_url, $original_url) ) { + print "failed to swap $backup_url\n"; + print " to $original_url\n"; + &my_die( "aborting", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + } else { + print "restored $original_url\n"; + } + } + if ($kill_censored) { + # destroy the backup file + $ipprc->kill_file($backup_url); + } + + if ($replicate) { + $ipprc->replicate_file($original_url) + or &my_die( "failed to replicate $original_url", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + } + } else { + print "Skipping:\n"; + print " Swap urls $backup_url $original_url\n" if $swap_urls; + print " Delete $backup_url\n" if $swap_urls; + print " Replicate $original_url\n" if $replicate; + } + + return 1; +} + +sub check_file { + my $url = shift; + my $full_check = shift; + my $smf = shift; + + my %file; + $file{url} = $url; + $file{so_exists} = $nebulous->storage_object_exists($url); + $file{total_instances} = 0; + $file{num_instances} = 0; + $file{censored} = 0; + $file{inconsistent} = 0; + $file{instances} = []; + $file{bad_instances} = []; + $file{size} = 0; + $file{md5} = 0; + + if ($file{so_exists}) { + my $stat = $nebulous->stat($url); + + &my_die("nebulous stat failed for $url", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) unless $stat; + + $file{total_instances} = $stat->[7]; + + if ($file{total_instances}) { + my $instances; + eval { + $instances = $nebulous->find_instances($url, 'any'); + }; + if (!$instances or !scalar @$instances) { + print "No instances found for $url\n"; + return \%file; + } + + my $good_inst; + my $good_size = 0; + my $good_md5 = ""; + my $good_censored ; + foreach my $inst (@$instances) { + $inst =~ s/file\:\/\///g; + + if ( ! -e $inst) { + print "instance file $inst does not exist\n"; + push @{$file{bad_instances}}, $inst; + next; + } + my $this_size = -s $inst; + if (!$this_size) { + print "instance file $inst has zero size\n"; + push @{$file{bad_instances}}, $inst; + next; + } + + my $this_censored = ($inst =~ /SR_/); + + my $this_md5 = ""; + if ($full_check) { + $this_md5 = file_md5_hex($inst); + if ($smf) { + my $command = "$ftable -list $inst | grep psf | wc"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + my $good = 0; + if ($success) { + my ($nchips) = split " ", join "", @$stdout_buf; + $nchips += 0; + if ($nchips and $nchips == 60) { + $good = 1; + } else { + print "$inst has less than 60 psf extensions: $nchips checking with censorObjects\n"; + my $command = "$censorObjects -checkinputonly -file $inst"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + if ($success) { + $good = 1; + } else { + print "$command failed with error_code: $error_code\n", + } + } + } else { + print "$command failed with error_code: $error_code\n", + } + if (!$good) { + push @{$file{bad_instances}}, $inst; + next; + } + } else { + my $command = "$checkfits $inst"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print "$command failed with error_code: $error_code\n", + push @{$file{bad_instances}}, $inst; + next; + } + } + } + if (!defined $good_inst) { + $good_inst = $inst; + $good_size = $this_size; + $good_md5 = $this_md5; + $good_censored = $this_censored; + + push @{$file{instances}}, $inst; + + # unless we're asked for a full all really care about is the censored status + last if !$full_check; + } elsif ($this_size != $good_size) { + print "size of $inst $this_size is not equal to size of $good_inst: $good_size\n"; + $file{inconsistent} = 1; + push @{$file{bad_instances}}, $inst; + } elsif ($this_md5 ne $good_md5) { + print "md5 of $inst $this_md5 is not equal to size of $good_inst: $good_md5\n"; + $file{inconsistent} = 1; + push @{$file{bad_instances}}, $inst; + } else { + # this matches the good instance + push @{$file{instances}}, $inst; + } + } + $file{num_instances} = scalar @{$file{instances}}; + $file{size} = $good_size; + $file{md5} = $good_md5; + if ($good_inst) { + $file{censored} = $good_censored; + } else { + # no good instances see if there was a bad one and check whether its url + # looks like a censored one + if (scalar @{$file{bad_instances}}) { + my $bad = ${$file{bad_instances}}->[0]; + $file{censored} = ($bad =~ /SR_/); + print "no good instances found setting censored based on $bad\n"; + } + } + } + } else { + print "No storage object found for $url \n"; + } + + return \%file; +} + +sub rename_file { + if ($no_op) { + return 1; + } + + my $original_url = shift; + my $backup_url = shift; + + # rename the backup as the orig + print "No storage object for original url: $original_url\n"; + print "Replacing with file at : $backup_url\n"; + + my $moved; + eval { + $moved = nebulous->move($backup_url, $original_url); + }; + if ($@ or not $moved) { + print "Failed to move $backup_url to $original_url\n"; + &my_die( "aborting", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); + } +} + +sub repair_file { + if ($no_op) { + return 1; + } + + my $file = shift; + my $smf = shift; + + if ($file->{num_instances} < 1) { + print "cannot repair file with only no good instances\n"; + return 0; + } + + # file wouldn't get into instances array unless we decided that it was good in check_file + my $good_inst = $file->{instances}->[0]; + + my $fixed = 0; + foreach my $bad_inst (@{$file->{bad_instances}}) { + print "Attempting to repair $bad_inst\n"; + + my $parent_directory = dirname($bad_inst); + if (!-d $parent_directory) { + if (! mkdir $parent_directory) { + print "failed to mkdir parent directory $parent_directory\n"; + $fixed = 0; + last; + } + if (!copy $good_inst, $bad_inst) { + print "failed to copy $good_inst to $bad_inst\n"; + } + $fixed = 1; + push @{$file->{instances}}, $bad_inst; + } + } + + return $fixed; +} + + + +sub delete_recovery_files +{ + foreach my $file (@_) { + next if !$file; + # don't care if this fails or not (it will fail if storage + # object doesn't exist + $ipprc->kill_file($file); + } +} + +sub inNebulous +{ + my $path = shift; + + my $scheme = file_scheme($path); + + return $scheme and ($scheme eq "neb"); +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $magic_ds_id = shift; # Magic DS identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + # fault the whole run if one of the components fails to revert + my $command = "$magicdstool -updaterun"; + $command .= " -magic_ds_id $magic_ds_id"; + $command .= " -set_state failed_restore"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + carp("failed to update database for $magic_ds_id"); + } + } else { + print "Skipping command: $command\n"; + } + + exit $exit_code; +} + +__END__ Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/dist_bundle.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/dist_bundle.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/dist_bundle.pl (revision 33638) @@ -194,4 +194,8 @@ # XXX: perhaps only do this for stages where we know that this happens next if $file_rule =~ /STATS/; + # don't fail on these "non-essential files" + next if $file_rule =~ /TRACE/; + next if $file_rule =~ /LOG/; + next if $file_rule =~ /BIN/; &my_die("failed to resolve $file_name", $component, $PS_EXIT_DATA_ERROR); Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/dist_component.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/dist_component.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/dist_component.pl (revision 33638) @@ -37,5 +37,5 @@ # Parse the command-line arguments my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean, $alt_path_base); -my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $exp_type); +my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $exp_type, $rerun); my ($dbname, $save_temps, $verbose, $no_update, $logfile); @@ -57,4 +57,5 @@ 'outdir=s' => \$outdir, # "directory" for outputs 'clean' => \$clean, # create clean distribution + 'rerun' => \$rerun, # recreate an existing component and update the database 'save-temps' => \$save_temps, # Save temporary files? 'dbname=s' => \$dbname, # Database name @@ -76,5 +77,5 @@ my $ipprc = PS::IPP::Config->new($camera); # IPP configuration -$ipprc->redirect_output($logfile) if $logfile; +$ipprc->redirect_to_logfile($logfile) if $logfile; my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR"); @@ -146,5 +147,12 @@ { - my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component -outdir $outdir"; + my $command; + if (!$rerun) { + $command = "$disttool -addprocessedcomponent"; + } else { + $command = "$disttool -updateprocessedcomponent"; + } + + $command .= " -dist_id $dist_id -component $component -outdir $outdir"; $command .= " -name $file_name -bytes $bytes -md5sum $md5sum"; $command .= " -dbname $dbname" if defined $dbname; @@ -175,5 +183,10 @@ $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; - my $command = "$disttool -addprocessedcomponent"; + my $command; + if (!$rerun) { + $command = "$disttool -addprocessedcomponent"; + } else { + $command = "$disttool -updateprocessedcomponent"; + } $command .= " -dist_id $dist_id"; $command .= " -component $component"; Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/dist_fix_file.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/dist_fix_file.pl (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/dist_fix_file.pl (revision 33638) @@ -0,0 +1,234 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +#print "\n\n"; +#print "Starting script $0 on $host at $date\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Temp qw( tempfile ); +use File::Basename qw( basename ); +use Digest::MD5::File qw( file_md5_hex ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use DBI; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $disttool = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1); +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $dist_component = can_run('dist_component.pl') or (warn "Can't find dist_component.pl" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Parse the command-line arguments +my ($dist_id, $camera, $component ); +my ($outdir, $run_state, $data_state, $magicked, $poor_quality, $exp_type, $rerun); +my ($dbname, $save_temps, $verbose, $no_update, $logfile); + +$camera = 'GPC1'; + +GetOptions( + 'dist_id=s' => \$dist_id, # distribution run identifier + 'component=s' => \$component, # the class_id or skycell_id + 'camera=s' => \$camera, # the class_id or skycell_id + 'save-temps' => \$save_temps, # Save temporary files? + 'dbname=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print stuff? + 'no-update' => \$no_update, # Don't update the database? + 'logfile=s' => \$logfile, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dist_id --camera --component", + -exitval => 3) unless + defined $dist_id and + defined $camera and + defined $component; + +my $ipprc = PS::IPP::Config->new($camera); # IPP configuration +$ipprc->redirect_to_logfile($logfile) if $logfile; + +my $dbh = getDBHandle() unless $no_update; + +my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR"); + +$temproot = "/tmp" if !defined $temproot; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +$disttool .= " -dbname $dbname" if $dbname; +$regtool .= " -dbname $dbname" if $dbname; + +my $comp = get_distcomponent($dist_id, $component); + +my $stage = $comp->{stage}; +my $stage_id = $comp->{stage_id}; +my $no_magic = $comp->{no_magic}; +my $path_base; +my $stage_args = ""; +if ($stage eq 'raw') { + # need path_base (get from uri) + # exp_type + # no_magic + # camera path base if !$no_magic + my $command = "$regtool -processedimfile -exp_id $stage_id -class_id $component"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $component, $outdir, $error_code); + } + + my $metadata = $mdcParser->parse (join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + + my $results = parse_md_list($metadata); + + if ((scalar @$results) != 1) { + my $n = scalar @$results; + &my_die("Unexected number of results from regtool -processedimfile: $n", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + } + my $rawfile = $results->[0]; + $path_base = $rawfile->{uri}; + $path_base =~ s/.fits//; + my $exp_type = $rawfile->{exp_type}; + if (!$no_magic) { + die "don't know how to create magicked raw bundles yet\n"; + } + $stage_args .= " --exp_type $exp_type --chip_path_base NULL"; + $stage_args .= " --no_magic" if $no_magic; +} else { + + die "not ready for stage $stage yet\n"; +} +my $dist_cmd = "$dist_component --rerun --camera $camera --dist_id $dist_id --stage $stage --stage_id $stage_id --component $component --outdir $comp->{outdir} --path_base $path_base $stage_args"; + +$dist_cmd .= " --dbname $dbname" if $dbname; +$dist_cmd .= " --verbose" if $verbose; +$dist_cmd .= " --no-update" if $no_update; + +print "$dist_cmd\n"; + +$comp = get_distcomponent($dist_id, $component); + +my $bytes = $comp->{bytes}; +my $md5sum = $comp->{md5sum}; +my $name = $comp->{name}; + + +my $query = "UPDATE dsFile join dsFileset using(fileset_id) join dsProduct USING(prod_id) SET dsFile.bytes =?, dsFile.md5sum = ? WHERE prod_name = ? AND fileset_name = ? AND file_name = ?"; + +my $stmt = $dbh->prepare($query); + +my $filesets = get_filesets($dist_id); + +foreach my $fs (@$filesets) { + my $product = $fs->{product}; + my $fileset = $fs->{fileset}; + $stmt->execute($bytes, $md5sum, $product, $fileset, $name) or + &my_die("failed to update dsFile for $product $fileset $name\n", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + + print "Updated dsFile for $product $fileset $name $bytes $md5sum\n"; +} + + +exit 0; + +sub getDBHandle { + my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DS_DBSERVER"); + my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DS_DBUSER"); + my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DS_DBPASSWORD"); + my $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DS_DBNAME"); + + &my_die("database configuration not set up", $dist_id, $component, $outdir, $PS_EXIT_UNKNOWN_ERROR) unless defined($dbserver) and defined($dbuser) + and defined($dbpassword) and defined($dbname); + + my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; + + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + or die "Cannot connect to database $dbname on $dbserver.\n"; + + return $dbh; +} + + + +sub get_distcomponent { + my $dist_id = shift; + my $component = shift; + my $command = "$disttool -processedcomponent -dist_id $dist_id -component $component"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $component, $outdir, $error_code); + } + + my $metadata = $mdcParser->parse (join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + + my $results = parse_md_list($metadata); + + if ((scalar @$results) != 1) { + my $n = scalar @$results; + &my_die("Unexected number of results from disttool -processedcomponent: $n", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + } + return $results->[0]; +} + +sub get_filesets { + my $dist_id = shift; + my $command = "$disttool -listfilesets -dist_id $dist_id"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $component, $outdir, $error_code); + } + + my $output = join "", @$stdout_buf; + if (!$output) { + return undef; + } + my $metadata = $mdcParser->parse ($output) or + &my_die("Unable to parse metadata config doc", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + + my $results = parse_md_list($metadata); + + return $results +} + + +### Pau. +sub my_die +{ + my $msg = shift; # Warning message on die + my $dist_id = shift; # distRun.dist_id + my $component = shift; # class_id, skycell_id, or exposure + my $outdir = shift; # output directory + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + exit $exit_code; +} + +__END__ Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/ipp_cleanup.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/ipp_cleanup.pl (revision 33638) @@ -218,4 +218,6 @@ $command .= " -dbname $dbname" if defined $dbname; +if (0) { + # XXX Don't set components to error cleaned anymore my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -224,4 +226,5 @@ &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); } +} # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/ipp_serial_inject_into_nebulous.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/ipp_serial_inject_into_nebulous.pl (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/ipp_serial_inject_into_nebulous.pl (revision 33638) @@ -0,0 +1,169 @@ +#!/usr/bin/env perl + +# this program injects a list of single-file exposures into the db, +# taking the filename (without .fits) as the exp_tag. the user +# supplies a temporary telescope and camera name. these are used +# for informational purposes only until the registration step can +# determine the true telescope and camera name from the image +# headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $dbname, $no_op, $help); +GetOptions('camera|i=s' => \$camera, # user-supplied camera name + 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'label=s' => \$label, # set chip label + 'dbname|d=s' => \$dbname, # Database name + 'no-op' => \$no_op, # pretend but don't actually inject + 'help' => \$help # give help listing +) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many files into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--label label] [--dbname dbname] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( -msg => "Required options: --telescope (name) --camera (name)", + -exitval => 3) unless + defined $telescope and + defined $camera; + +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# if workdir is not defined, assign the current path +# XXX we need to handle relative paths for workdir (not allowed) +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +my $num = 0; +foreach my $file ( @ARGV ) { + # check for file existence + if (! -e $file) { die "file $file not found\n"; } + my $absfile = File::Spec->rel2abs( $file ); + + #copy into nebulous + my $neb_part=$absfile; + $neb_part =~s/.*\///g; + my $base = $neb_part; + $base =~s/.fits//; + + print "$workdir/$base/$neb_part\n"; + + + `neb-touch $workdir/$base/$neb_part`; + my @paths = `neb-locate --path $workdir/$base/$neb_part`; + foreach my $path (@paths) { + chomp $path; + my @cperr = `cp $absfile $path 2>&1`; + if ($#cperr > 0) { + print "error copying $absfile $path\n"; + } + } + `neb-replicate $workdir/$base/$neb_part --set_copies 2`; + my $numofcopies = `neb-stat $workdir/$base/$neb_part | grep available`; + $numofcopies =~ s/available instances: //; + if ($numofcopies < 2) { + print "problem: less than 2 copies of $workdir/$base/$neb_part\n"; + } + + inject("$workdir/$base/$neb_part", $workdir, $dbname, $telescope, $camera); + $num ++; +} + +print "$num files injected.\n"; + +sub inject +{ + my $absfile = shift; # absolute path for this file + my $workdir = shift; # absolute path for output directory + my $dbname = shift; # IPP database to use + my $telescope = shift; # user-specified telescope + my $camera = shift; # user-specified camera + + # XXX provide an option for an alternative extension + my ( $vol, $path, $name ) = File::Spec->splitpath( $absfile ); + my ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/; + $exp_name =~ s|\s|_|g; + + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + # the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values + my $command_exp = "$pxinject -newExp"; + $command_exp .= " -tmp_exp_name $exp_name"; + $command_exp .= " -tmp_inst $camera"; + $command_exp .= " -tmp_telescope $telescope"; + $command_exp .= " -workdir $workdir"; + $command_exp .= " -reduction $reduction" if defined $reduction; + $command_exp .= " -dvodb $dvo_db" if defined $dvo_db; + $command_exp .= " -tess_id $tess_id" if defined $tess_id; + $command_exp .= " -end_stage $end_stage" if defined $end_stage; + $command_exp .= " -label $label" if defined $label; + $command_exp .= " -dbname $dbname" if defined $dbname; + + my $exp_id = 0; + unless ($no_op) { + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + $exp_id = $line[2]; # The exposure tag + } else { + print "skipping command: $command_exp\n"; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id fpa"; + $command_imfile .= " -uri '$relfile'"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } else { + print "skipping command: $command_imfile\n"; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_update = "$pxinject -updatenewExp"; + $command_update .= " -exp_id $exp_id"; + $command_update .= " -state run"; + $command_update .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; + } else { + print "skipping command: $command_update\n"; + } + + return 1; +} + + +__END__ Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/lap_science.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/lap_science.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/lap_science.pl (revision 33638) @@ -33,5 +33,5 @@ # Global configuration constants that probably should be read from elsewhere. my $qstack_threshold = 0.05; # Only make a quickstack if more than 5% of the exposures require it. -my $minimum_stack_inputs = 6; # We can avoid magicking stack inputs if we have more than this number. +my $minimum_stack_inputs = 4; # We can avoid magicking stack inputs if we have more than this number. GetOptions( @@ -286,12 +286,12 @@ &my_die("Unable to perform chiptool -setimfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); } - if ($chip_magicDS_id != 0) { - ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $magicDS_update_cmd, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); - } - } +# if ($chip_magicDS_id != 0) { +# ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $magicDS_update_cmd, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); +# } +# } ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -551,4 +551,8 @@ $complete_fstacks++; } + elsif (($stack->{final_state})&& + ($stack->{final_state} eq 'drop')) { + $complete_fstacks++; + } elsif (($stack->{final_state})&&($stack->{final_state} eq 'new')&& ($stack->{final_fault} >= 4)&&($stack->{final_fault} != 32767)) { @@ -616,5 +620,4 @@ my $are_warped = 0; my $are_magicked = 0; - foreach my $exposure (@$exposures) { $total_exposures++; @@ -706,17 +709,27 @@ $exposure->{data_state} = 'to_diff'; } - if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) { + if (($exposure->{diff_id})&& + (&S64_IS_NOT_NULL($exposure->{diff_id}))&& + (($exposure->{diffRun_state} eq 'full')|| + ($exposure->{diffRun_state} eq 'error_cleaned')|| + ($exposure->{diffRun_state} eq 'cleaned')) + ) { + $are_magicked++; $have_diff ++; - $exposure->{data_state} = 'to_magic'; - } - if (($exposure->{magicked})&& - ($exposure->{warpRun_state})&& - ($exposure->{warpRun_state} eq 'full')&& - (&S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. - $are_magicked++; +# $exposure->{data_state} = 'to_magic'; $exposure->{data_state} = 'full'; -# $can_fstack ++; - } - &update_this_exposure($exposure); + } + # We no longer need to care about magic +# if (($exposure->{magicked})&& +# ($exposure->{warpRun_state})&& +# ($exposure->{warpRun_state} eq 'full')&& +# (&S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. +# $are_magicked++; +# $exposure->{data_state} = 'full'; +# # $can_fstack ++; +# } + unless ($debug) { + &update_this_exposure($exposure); + } } @@ -744,5 +757,8 @@ if ($defined_fstack == 0) { # We have not yet made any stacks. print "STATUS: We can attempt to queue deep final stacks.\n"; - my $Nstacks_made = &queue_muggle_finalstack($exposures); + my $Nstacks_made; + unless ($debug) { + $Nstacks_made = &queue_muggle_finalstack($exposures); + } print "STATUS: Made $Nstacks_made final stacks.\n"; exit($Nstacks_made); @@ -752,4 +768,5 @@ if ($have_fstack == $defined_fstack) { # We have made all the stacks we have asked for (so far) if ($are_magicked < $total_exposures) { # But we have not yet made diffs +# if ($have_diff < $total_exposures) { # We have not yet made diffs if (($can_diff == $total_exposures)&& # And we have exposures that should be diffed. ($have_diff < $can_diff)) { # and we haven't made them all yet, either. @@ -759,5 +776,7 @@ # &update_this_exposure($exposure); # } - &queue_diffs($exposures); + unless ($debug) { + &queue_diffs($exposures); + } exit(0); } @@ -770,24 +789,30 @@ } elsif ($are_magicked == $total_exposures) { # We have made all of the diffs, and propagated through magic - print "STATUS: We can create all remaining stacks possible.\n"; - my $Nstacks_made = &queue_magic_finalstack($exposures); - if ($Nstacks_made == 0) { # We have made all the stacks that we will ever make. + print "STATUS: We would create all remaining stacks possible, but no longer require magic.\n"; +# my $Nstacks_made = -1; +# unless ($debug) { +# $Nstacks_made = &queue_muggle_finalstack($exposures); +# } +# if ($Nstacks_made == 0) { # We have made all the stacks that we will ever make. print "STATUS: No more stacks generated. Deactivating exposures for this complete lapRun.\n"; - &deactivate_exposures($exposures); - $command = "$laptool -updaterun -lap_id $lap_id"; - $command .= " -dbname $dbname " if defined $dbname; - $command .= " -set_state full "; - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); - } + unless ($debug) { + &deactivate_exposures($exposures); + + $command = "$laptool -updaterun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_state full "; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } +# } exit(0); } - else { - print "STATUS: $Nstacks_made stacks made.\n"; - exit(0); - } +# else { +# print "STATUS: $Nstacks_made stacks made.\n"; +# exit(0); +# } } else { # What? @@ -991,5 +1016,5 @@ foreach $exposure (@$exposures) { if (($exposure->{data_state} ne 'drop')&& - (S64_IS_NOT_NULL($exposure->{magicked}))&& +# (S64_IS_NOT_NULL($exposure->{magicked}))&& (S64_IS_NOT_NULL($exposure->{warp_id}))) { $warps .= " -warp_id $exposure->{warp_id} "; @@ -1286,4 +1311,14 @@ if (@$stdout_buf == 0) { # Nothing to do. + $command = "$laptool -updaterun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_state done "; + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + return(0); } @@ -1474,10 +1509,12 @@ $command .= " -set_data_state $exposure->{data_state} "; } - - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform laptool -updateexp: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + unless ($do_nothing) { + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updateexp: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } } } Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak.pl (revision 33638) @@ -99,5 +99,10 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile; + +if ($logfile) { + $ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Continuing script $0 on $host at $date\n\n"; +} $cam_reduction = 'DEFAULT' if !$cam_reduction or ($cam_reduction eq 'NULL'); @@ -180,5 +185,7 @@ } -my $statsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.stats"; +# XXX: create a DESTREAK file rule. For now just steal one that will work +my $error; +my $statsFile = $ipprc->prepare_output("PSASTRO.STATS", "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component", undef, 1, \$error) or &my_die("failed to prepare output for stats", $magic_ds_id, $component, $error); my $backup_path_base; @@ -252,5 +259,7 @@ foreach my $skycell (@$skycells) { my $skycell_uri; - if ($skycell->{data_state} eq "full" and $skycell->{quality} == 0) { + $skycell->{quality} = undef; + # XXXXXXXX remake all skycells to avoid dead instances (ipp064) + if (0 and $skycell->{data_state} eq "full" and $skycell->{quality} == 0) { $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base}); my $skycell_id = $skycell->{skycell_id}; Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak_cleanup.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak_cleanup.pl (revision 33638) @@ -61,5 +61,5 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile; +$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile; Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak_revert.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak_revert.pl (revision 33638) @@ -86,5 +86,5 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile; +$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile; &my_die("bytes and md5sum are is required for raw stage stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($replace and $stage eq 'raw' and (!$bytes or !$md5sum)); @@ -421,5 +421,7 @@ # files are in their original state. Just delete back up file # otherwise go to failed_revert for manual inspection. - if (!($o_basename =~ /SR_/) and ($b_basename =~ /SR_/)) { + # XXX: We probably can get by with just checking the basename of the original file + if (!($o_basename =~ /SR_/) ) { + # if (!($o_basename =~ /SR_/) and ($b_basename =~ /SR_/)) { print STDERR " basenames are as expected it is safe to delete backup file\n"; if (! $ipprc->file_delete($backup, $force)) { @@ -481,9 +483,7 @@ foreach my $file (@_) { next if !$file; - if ($ipprc->file_exists($file)) { - if (!$ipprc->file_delete($file)) { - &my_die("failed to delete recovery file $file", $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR); - } - } + # don't care if this fails or not (it will fail if storage + # object doesn't exist + $ipprc->kill_file($file); } } Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/mergedvodb_copy.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/mergedvodb_copy.pl (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/mergedvodb_copy.pl (revision 33638) @@ -0,0 +1,189 @@ +#!/usr/bin/env perl + +#This is essentially the same as minidvodb_copy.pl - the tables are essentially identical, and the task (to rsync a copy) is essentially identical. A new script was created for rsyncing the merges in case they should diverge in the future. + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; + +my $dtime; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $rsync = can_run('rsync') or (warn "Can't find rsync" and $missing_tools = 1); +my $mergedvodbtool = can_run('mergedvodbtool') or (warn "Can't find mergedvodbtool" and $missing_tools = 1); +my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); +my $df = can_run('df') or (warn "Can't find df" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + + + +my ( $mergedvodbcopy_id, $merge_id, $mergedvodbrun_path, $mergedvodb_rsync_path, $destination_host, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodbcopy_id|w=s' => \$mergedvodbcopy_id, #minidvodb database + 'merge_id|w=s' => \$mergedvodb_id, #minidvodb_id + 'mergedvodbrun_path|w=s' => \$mergedvodbrun_path, #minidvodb_id + 'mergedvodb_rsync_path|w=s' => \$mergedvodb_rsync_path, + 'destination_host|w=s' => \$destination_host, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodbcopy_id --mergedvodb_rsync_path --mergedvodbrun_path --destination_host", + -exitval => 3, + ) unless + defined $mergedvodbcopy_id and + defined $mergedvodb_rsync_path and + defined $mergedvodbrun_path and + defined $destination_host; + + + +my $ipprc = PS::IPP::Config->new(); + +my @df; + + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $mergedvodbcopy_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +{ #can we rsync? + + my $sizes = `du -sk $mergedvodbrun_path`; + my $total = 0; + for(split /[\r\n]+/,$sizes) # split on one or more newline characters + { + my($number,$file) = split /\t/,$_,2; # split on tab ($file not used here) + $total += $number; + } +#print 'Total: '.$total; +{ + print "Checking available diskspace on $destination_host\n\n"; + + my $command = "$ssh $destination_host df $mergedvodb_rsync_path"; + print "$command\n\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform '$ssh $destination_host df $mergedvodb_rsync_path' $error_code", $minidvodbcopy_id, $error_code); + } + + my @fullbuf= split(/\n/, @{$stdout_buf}[0]); + # print $#{$stdout_buf}; +# print "xx $#df xx"; +# print $fullbuf[0]; + if ($#fullbuf ==1 ) { +# print $fullbuf[1]; + @df = split /\s+/, $fullbuf[1]; + if ($#df == 5) { +# + #print $df[3], $df[4]; + + $df[4]=~ s/\%//; + + if ($df[4] < 97 && $df[3]-$total) { + print "\nThere is enough available space ($total < $df[3]) for the rsync\n"; + } else { + &my_die("Not enough available disk space: $df[4]% available, $df[3] on destination, $total free required for copy", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } +# + + } else { + &my_die("Cannot parse df:\n@{$full_buf}[0]\nCan't figure out %free or available", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } + } else { + &my_die("Cannot parse df:\n@{$full_buf}[0]\nToo many lines ($#fullbuf) vs 1", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } +} +} + +{#do the rsync + my $command = "$rsync -rvuaq"; + $command .= " $mergedvodbrun_path"; + $command .= " $destination_host"; + $command .= ":$mergedvodb_rsync_path"; + print "\nPerforming rsync:\n"; + print "$command\n\n"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform rsync: $error_code", $mergedvodbcopy_id, $error_code); + } + + print "Rsync complete..\n"; + +} + +{#update the database + $dtime = 86400.0*(DateTime->now->mjd - $mjd_start); + + my $command = "$mergedvodbtool -updatemergedvodbcopy -mergedvodbcopy_id $mergedvodbcopy_id"; + $command .= " -set_state full -set_dtime $dtime"; + $command .= " -dbname $dbname" if defined $dbname; + print "$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = 20; + &my_die("Unable to update database - rsync completed without error' $error_code", $mergedvodbcopy_id, $error_code); + } + +} +exit 0; + + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $mergedbvodbcopy_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodbcopy_id\n"; + +if (defined $mergedvodb_id ) { + my $command = "mergedvodbtool -updatemergedvodbcopy -mergedvodbcopy_id $mergedvodbcopy_id"; + $command .= " -set_fault $exit_code"; + $command .= " -set_dtime $dtime" if defined $dtime; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + system ($command); + } + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + exit $exit_code; +} + +__END__ Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/mergedvodb_merge.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/mergedvodb_merge.pl (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/mergedvodb_merge.pl (revision 33638) @@ -0,0 +1,237 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_merge; +my $dtime_verify; +my $dtime_script; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $mergetool = can_run('mergetool') or (warn "Can't find mergetool" and $missing_tools = 1); +my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1); + + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $mergedvodb_path, $merge_id, $mergedvodb, $minidvodb_path, $dbname,$camera,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodb_path|w=s' => \$mergedvodb_path, # output DVO database + 'minidvodb_path|w=s' => \$minidvodb_path, #minidvodb database + 'mergedvodb|w=s' => \$mergedvodb, #mergedvodb + 'merge_id|w=s' => \$merge_id, #minidvodb_id + 'dbname|d=s' => \$dbname, # Database name + 'camera|w=s' => \$camera, #camera name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --merge_id -- minidvodb_path --mergedvodb_path --mergedvodb --camera", + -exitval => 3, + ) unless + defined $mergedvodb_path and + defined $minidvodb_path and + defined $mergedvodb and + defined $merge_id and + defined $camera; + + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $merge_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $merge_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +# convert supplied DVO database name to UNIX filename + +unless ($no_op) { + + #this is chopped into several parts: firstcheck, addstar, relphot, dvoverify, merge + + #first check that there were no faulted merges before. If there are, fault this merge with PS_EXIT_UNKNOWN_ERROR + { + my $nothing_faulted = 0; + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$mergetool -listmerged -faulted -mergedvodb " . $mergedvodb; + $command .= " -dbname $dbname" if defined $dbname; + print "looking for faulted:\n\n$command\n\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of faulted mergedvodbs", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if nothign has faulted + $nothing_faulted =1; + print "previous merges are okay.\n"; + } + &my_die( "Previous merges faulted, do not proceed until they are investigated:", $merge_id, $PS_EXIT_UNKNOWN_ERROR) unless $nothing_faulted; + } + { + my $command = "$dvoverify -s $minidvodb_path"; + print "$command\n"; + my $mjd_dvoverify_start = DateTime->now->mjd; # MJD of starting script + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dvoverify: $error_code", $merge_id, $error_code); + } + $dtime_verify = 86400.0*(DateTime->now->mjd - $mjd_dvoverify_start); # MJD of starting script + print "dvoverify time $dtime_verify\n"; + } + #Merge + + my $this_is_the_first; + { + + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$mergetool -listmerged -mergedvodb " . $mergedvodb; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of mergedvodbruns", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + $this_is_the_first =1; + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $merge_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $merge_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $mergedvodb_path_tmp = $comp->{mergedvodb_path}; + + if (!defined($mergedvodb_path_tmp)) { + &my_die("Unable to parse mergedvodb_path", $merge_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found at least 1 mergedvodbrun in merged state\n"; + $this_is_the_first = 0; + } + } + print "$this_is_the_first $mergedvodb_path/Image.dat\n"; + if (-e "$mergedvodb_path/Image.dat") { + if ($this_is_the_first == 1) { + &my_die("refusing to merge, this is the first, but files already exist in dir", $merge_id, 4); + } + $this_is_the_first =0; + } + + print "$this_is_the_first $mergedvodb_path/Image.dat\n"; + { + my $merge_command; + my $mjd_merge_start = DateTime->now->mjd; # MJD of starting script + if ($this_is_the_first) { + $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path"; + } else { + $merge_command = "$dvomerge $minidvodb_path into $mergedvodb_path"; + } + print "\n$merge_command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $merge_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to merge: $error_code", $merge_id, $error_code); + } + $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start); # MJD of starting script + print "merge time $dtime_merge\n"; +# } + } + + { + + $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start); + my $command = "mergetool -merge_id $merge_id"; + $command .= " -addmerged"; + $command .= " -mergedvodb $mergedvodb" if defined $mergedvodb; +# $command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to mergedvodbprocessed: $error_code", $merge_id, $error_code); + } + } + +#} else { +# &my_die("dvodb: $mergedvodb_path not found", $merge_id, $PS_EXIT_UNKNOWN_ERROR); +} +else { + print "skipping processing for $mergedvodb_path\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $merge_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodb_path\n"; + +if (defined $merge_id ) { + + my $command = "mergetool -merge_id $merge_id"; + $command .= " -addmerged"; + $command .= " -mergedvodb $mergedvodb" if defined $mergedvodb; + $command .= " -fault $exit_code"; + #$command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify; + $command .= " -dbname $dbname" if defined $dbname; + + + + print $command; + system ($command); + } + + + + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + exit $exit_code; +} + +__END__ Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/mergedvodb_queue.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/mergedvodb_queue.pl (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/mergedvodb_queue.pl (revision 33638) @@ -0,0 +1,164 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; +use DateTime; + +# Ideally, we want to queue new runs with mergetool -definebyquery. We can't for the survey task, because we use the value in 'site.config' to find the mergedvodb_path. I do not know how to tell pantasks how to do this, but I know how to do it in perl. + + +use DateTime::Format::Strptime; +use DateTime::Duration; +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + + +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $mergetool = can_run('mergetool') or (warn "Can't find mergetool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $outroot, $dbname, $minidvodb_group, $mergedvodb , $camera, $verbose, $no_update, + $no_op, $redirect, $save_temps); + +# we just need any old camera to be able to read site.config. I'm using SIMPLE, unless it is overridden on the commandline (it makes no difference for this case) + +$camera = 'SIMPLE'; + + +GetOptions( + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'minidvodb_group|w=s' => \$minidvodb_group, # minidvodb_group. + 'outroot|w=s' => \$outroot, # output file base name + 'mergedvodb|w=s' => \$mergedvodb, # output miniDVODB + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodb --minidvodb_group --outroot", + -exitval => 3, + ) unless + defined $minidvodb_group and + # defined $camera and + defined $outroot and + defined $mergedvodb; + +print "camera is $camera\n"; + + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $mergedvodb , $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $minidvodb_group, $PS_EXIT_CONFIG_ERROR); +print "log file is $logDest\n"; +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $minidvodb_group, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +# Recipes to use based on reduction class + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs + my $create_new = 0; +# convert supplied DVO database name to UNIX filename +my $mergedvodbReal; +if (defined $mergedvodb) { + $mergedvodbReal = $ipprc->dvo_catdir( $mergedvodb ); # catdir for DVO + $mergedvodbReal = $ipprc->convert_filename_absolute( $mergedvodbReal ); +} else { + warn("mergedvodb undefined:\n"); + exit(4); +} +if (!defined $mergedvodbReal) { + warn("can't parse out mergedvodb_path\n"); + exit(4); +} + + + +print "mergedvodb_path = $mergedvodbReal\n"; + +unless ($no_op) { + + #create the mergedvodb entry (well, the command for it) + my $fpaCommand = "$mergetool -definebyquery"; + $fpaCommand .= " -mergedvodb $mergedvodb"; + $fpaCommand .= " -minidvodb_group $minidvodb_group"; + $fpaCommand .= " -set_mergedvodb_path $mergedvodbReal" ; + $fpaCommand .= " -dbname $dbname" if defined $dbname; + + print "running the following: \n\n"; + print "$fpaCommand\n\n"; + + unless ($no_update) { + + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } + } else { + print "skipping command: $fpaCommand\n"; + } +} + +sub my_die +{#complain if it doesn't work + my $msg = shift; # Warning message on die + my $mergedvodb = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/minidvodb_createdb.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/minidvodb_createdb.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/minidvodb_createdb.pl (revision 33638) @@ -226,5 +226,4 @@ $fpaCommand .= " -set_minidvodb_group $minidvodb_group"; $fpaCommand .= " -set_minidvodb_path $minidvodbReal" if defined $minidvodbReal; - $fpaCommand .= " -set_mergedvodb_path $dvodbReal"; $fpaCommand .= " -dbname $dbname" if defined $dbname; Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/minidvodb_premerge.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/minidvodb_premerge.pl (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/minidvodb_premerge.pl (revision 33638) @@ -0,0 +1,174 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_resort; +my $dtime_relphot; +my $dtime_script; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1); + + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'minidvodb|w=s' => \$minidvodb, #minidvodb database + 'minidvodb_id|w=s' => \$minidvodb_id, #minidvodb_id + 'minidvodb_group|w=s' => \$minidvodb_group, #minidvodb_id + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --minidvodb --minidvodb_id -- minidvodb_group --camera", + -exitval => 3, + ) unless + defined $minidvodb and + defined $minidvodb_id and + defined $minidvodb_group and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $minidvodb_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $minidvodb_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +my $dtime_addstar = 0; + +unless ($no_op) { + + #this is chopped into several parts: addstar, relphot + #addstar + { + my $command = "$addstar -resort"; + $command .= " -D CAMERA $camera"; + $command .= " -D CATDIR $minidvodb"; + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + print "\n$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $minidvodb_id, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); $dtime_resort = $dtime_addstar; + # MJD of starting script + $dtime_resort = $dtime_addstar; + print "addstar -resort time $dtime_addstar\n"; + } + + #relphot + + { + # relphot only takes lower case gpc1 + my $relphot_camera = lc($camera); + my $command = "$relphot -averages -update"; + $command .= " -D CAMERA $relphot_camera"; + $command .= " -D CATDIR $minidvodb"; + print "$command\n"; + my $mjd_relphot_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform relphot: $error_code", $minidvodb_id, $error_code); + } + $dtime_relphot = 86400.0*(DateTime->now->mjd - $mjd_relphot_start); # MJD of starting script + print "relphot time $dtime_relphot\n"; + } + + { + + $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start); + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to minidvodbprocessed: $error_code", $minidvodb_id, $error_code); + } + } +#} else { +# &my_die("dvodb: minidvodb_id = $minidvodb_id not found", $minidvodb_id, $PS_EXIT_UNKNOWN_ERROR); +#} +} else { + print "skipping processing for minidvodb_id = $minidvodb_id\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $minidbvodb_id = shift; + my $exit_code = shift; # Exit code to add + print STDERR "$msg $minidvodb_id\n"; + +if (defined $minidvodb_id ) { + + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -fault $exit_code"; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + exit $exit_code; +} + +__END__ Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/nightly_science.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/nightly_science.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/nightly_science.pl (revision 33638) @@ -996,5 +996,10 @@ $cmd .= " -min_num 4"; $cmd .= " -select_good_frac_min 0.05"; - $cmd .= " -set_reduction NIGHTLY_STACK "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + else { + $cmd .= " -set_reduction NIGHTLY_STACK "; + } $cmd .= " $select "; if ($debug == 1) { @@ -1410,4 +1415,10 @@ } } + if ((defined($science_config{$target}{SELF_WSDIFFS})) && ($science_config{$target}{SELF_WSDIFFS} == 1)) { + foreach my $filter (@filter_list) { + # This one needs to return a state to see if we need to wait on stacking before checking again. + $metadata_out{nsDiffState} = self_WS_diff_queue($date,$target,$filter,$pretend); + } + } } } @@ -1702,31 +1713,87 @@ my ($lunation_start,$lunation_end) = get_lunation_extent($date); - + + +# Get a list of exposures that could be diffed my $new_data_group = "${data_group}.offnight"; my $new_dist_group = "${dist_group}.offnight"; - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label "; - $cmd .= "-set_workdir $workdir -set_dist_group $new_dist_group -set_data_group $new_data_group "; - $cmd .= " -mintimediff 40000 "; - $cmd .= " -simple -set_label $label -filter $filter "; - $cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if (defined($debug)) { - print STDERR "ON_diffs wants to run this command: $cmd\n"; - } - - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - } -} + my $check_cmd = "$difftool -dbname $dbname -definewarpwarp "; + $check_cmd .= "-input_label $label -template_label $label "; + $check_cmd .= "-set_workdir $workdir -set_dist_group $new_dist_group -set_data_group $new_data_group "; + $check_cmd .= " -mintimediff 40000 "; + $check_cmd .= " -pretend -simple -rerun -set_label $label -filter $filter "; + $check_cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $check_cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + + # Parse results + my $diffs = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from difftool -definewarpwarp", 0, 0, $date, $PS_EXIT_PROG_ERROR); + foreach my $diff (@$diffs) { + unless (multi_date_verify_uniqueness_diff($diff->{input_warp_id},$diff->{template_warp_id},$date,$target)) { + # If we don't already have a diff with these inputs, make a diff with these inputs. + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label "; + $cmd .= "-warp_id $diff->{input_warp_id} -template_warp_id $diff->{template_warp_id} "; + $cmd .= "-set_workdir $workdir -set_dist_group $new_dist_group -set_data_group $new_data_group "; + $cmd .= " -mintimediff 40000 "; + $cmd .= " -simple -rerun -set_label $label -filter $filter "; + $cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + print STDERR "ON_diffs wants to run this command: $cmd\n"; + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + } + } + } +} + +sub intranight_WS_diff_queue { + my $date = shift; + my $target = shift; + my $filter = shift; + my $pretend = shift; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + my $db = init_gpc_db(); + + my ($stacks_to_finish,$stacks_total) = check_stack_count($label); + if (($stacks_to_finish == 0)&&($stacks_total != 0)) { + my $cmd = "$difftool -dbname $dbname -definewarpstack "; + $cmd .= " -good_frac 0.2 "; + $cmd .= " -warp_label $label -stack_label $label -set_label $label "; + $cmd .= " -set_workdir $workdir -available -set_reduction WARPSTACK -set_dist_group $dist_group "; + $cmd .= " -rerun "; + } +} + +sub check_stack_count { + my $label = shift; + + my $stacks_to_finish = 0; + my $stacks_total = 0; + + my $sth = "SELECT DISTINCT stack_id from stackRun where label = '$label' AND state = 'new'"; + my $data_ref = $db->selectall_arrayref( $sth ); + $stacks_to_finish = $#{ $data_ref } + 1; + + $sth = "SELECT DISTINCT stack_id from stackRun where label = '$label'"; + $data_ref = $db->selectall_arrayref( $sth ); + $stacks_total = $#{ $data_ref } + 1; + return($stacks_to_finish,$stacks_total); +} Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/queue_chips_by_night.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/queue_chips_by_night.pl (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/queue_chips_by_night.pl (revision 33638) @@ -0,0 +1,102 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $stdate = `date`; +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my @ARGS = @ARGV; + +my ( $date, $data_group_base, $label, $tess_id, $end_stage, $workdir_base, $dbname ); + +GetOptions( + 'date=s' => \$date, # date, like this 2011-02-12 + 'data_group_base=s' => \$data_group_base, # data_group_base, like this: isp + 'label=s' => \$label, # label + 'dbname|d=s' => \$dbname, # Database name + 'tess_id=s' => \$tess_id, # tess_id + 'end_stage=s' => \$end_stage, # end_stage + 'workdir_base=s' => \$workdir_base, # workdir + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dbname --label --tess_id --date \n Optional: --data_group_base, end_stage, workdir_base", + -exitval => 3) unless + defined $dbname and + defined $label and + defined $tess_id and + defined $date; + +my $truncdate = $date; + $truncdate =~ s/-//g; +if (defined $data_group_base) { + print "found data_group_base = $data_group_base\n"; +} else { + $data_group_base = $label; + print "using $label as data_group_base\n"; +} +if (defined $end_stage) { + print "found end_stage = $end_stage\n"; +} else { + $end_stage = "camera"; + print "using camera as end_stage\n"; +} +if (defined $workdir_base) { + print "found workdir_base = $workdir_base\n"; +} else { + $workdir_base = "neb://\@HOST\@.0/$dbname/$label"; + print "using $workdir_base as workdir\n"; +} + +# we have all the pieces, now build up the chiptool: + +my $command = "$chiptool -definebyquery -unique"; +$command .= " -dbname $dbname"; +$command .= " -dateobs_begin '$date"."T00:00:00'"; +$command .= " -dateobs_end '$date"."T23:59:59'"; +$command .= " -set_label $label"; +$command .= " -set_data_group $data_group_base.$truncdate"; +$command .= " -set_workdir $workdir_base/$truncdate"; +$command .= " -set_end_stage $end_stage"; +$command .= " -set_tess_id $tess_id"; + +print "\nrunning this command:\n\n$command\n\n"; + + +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform chiptool -definebyquery: $error_code\n"); + exit($error_code); + +} + + Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/skycalibration.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/skycalibration.pl (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/skycalibration.pl (revision 33638) @@ -0,0 +1,298 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use File::Temp qw( tempfile ); +use File::Basename; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); +my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); +my $fhead = can_run('fhead') or (warn "Can't find fhead" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +sub HelpMessage { + print "USAGE: skycalibration.pl --skycal_id (skycal_id) --outroot (root) --camera (camera) [options]\n"; + print " --skycal_id (skycal_id) : run identifier\n"; + print " --camera (camera) : Camera name\n"; + print " --dbname (dbname) : Database name\n"; + print " --threads (N) : Number of threads to use\n"; + print " --outroot (root) : Output root name\n"; + print " --path_base (path_base) : path_base of input\n"; + print " --filter (filter) : filter of input\n"; + print " --singlefilter : the input is the result of a single filter analysis"; + print " --reduction (class) : Reduction class\n"; + print " --verbose : Be extra verbose\n"; + print " --no-update : Don't update the database?\n"; + print " --no-op : Don't do any operations?\n"; + print " --redirect-output : send output to the log file,\n"; + print " --save-temps : Save temporary files?\n"; + print " --help : show this message\n"; + exit 2; +} + +my ($skycal_id, $stack_id, $camera, $dbname, $threads, $outroot, $path_base, $filter, $singlefilter, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps); +GetOptions( + 'skycal_id=s' => \$skycal_id, # sky calibration run id + 'stack_id=s' => \$stack_id, # static sky run id + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'path_base=s' => \$path_base, # path_base of input + 'filter=s' => \$filter, # filter for input stack + 'reduction=s' => \$reduction, # Reduction class + 'singlefilter' => \$singlefilter, # input was from single filter analysis + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + 'help' => sub { HelpMessage() }, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --skycal_id --stack_id --outroot --camera --path_base", + -exitval => 3, + ) unless + defined $skycal_id and + defined $stack_id and + defined $path_base and + defined $outroot and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $skycal_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot); +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $skycal_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); +my $stats = $ipprc->filename("PSASTRO.STATS", $outroot); + +my $outbase = basename($outroot); + +# Get list of input images to stack photometry +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + +# generate the input +my $cmdflags = ""; + +my $recipe_psastro = $ipprc->reduction($reduction, 'STATICSKY_CALIBRATION'); # Recipe to use for psastro +unless ($recipe_psastro) { + &my_die("Couldn't find selected reduction for STATICSKY_CALIBRATION: $reduction\n", $skycal_id, $PS_EXIT_CONFIG_ERROR); +} + +# XXX: We are making the bad assumption here that the staticksky run used the same reduction class +# as this skycalRun +my $recipe_psphot = $ipprc->reduction($reduction, "STACKPHOT_PSPHOT"); +unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for STATCKPHOT_PSPHOT: $reduction\n", $skycal_id, $PS_EXIT_CONFIG_ERROR); +} + +print "reduction: $reduction\n"; +print "recipe_psastro: $recipe_psastro\n"; +print "recipe_psphot: $recipe_psphot\n"; + +my $configuration = $ipprc->filename("PSASTRO.CONFIG", $outroot); + +# find input file +# First check the expected file name where the stack_id is the FILE_ID +my $file; +if (!$singlefilter) { + $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $stack_id); + if (! $ipprc->file_resolve($file)) { + # XXX: Beginning of section that can be removed eventually + + # no file with the expected name found + # assume that the input is from an early staticsky run that did not use stack_id as the FILE_ID + # but instead used FILE_ID = [0 .. num_filters-1] + print "\nfailed to resolve $file\n"; + if (!$filter) { + &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); + } + print "Trying old style FILE_ID\n"; + my $max_filters = 5; + for (my $i=0; $i < $max_filters; $i++) { + my $file_id = sprintf "%03d", $i; + $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id); + my $resolved = $ipprc->file_resolve($file); + if (!$resolved) { + # we fail here assumming that if file_id N doesn't exist, neither to N+1 + print "\nfailed to resolve $file\n\n"; + &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); + } + # Check the filter id for this file + my $command = "$fhead $resolved | grep FILTERID"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR); + } + # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name) + my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf); + my $this_filter = substr $filt, 1, 7; + + # if it matches we're done + last if $this_filter eq $filter; + + # nope loop around to try the next one + print "Input file for $filter is not $file ($this_filter)\n"; + $file = undef; + } + print "\nInput file for $stack_id filter: $filter: $file\n"; + + # XXX: End of section that can be removed eventually + } +} else { + # input is from a single filter static sky run use a different file rule + # XXX: can't we just make staticsky.pl use the same file rule? + $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base); + &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($file); +} + +if (!$file) { + &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR); +} + +# Perform psastro analysis +my $output_sources_filerule = 'PSASTRO.OUTPUT.CMF'; +{ + my $command = "$psastro $outroot"; + $command .= " -file $file"; +# $command .= " -threads $threads" if defined $threads; # psastro doesn't currently accept -threads + $command .= " -recipe PSASTRO $recipe_psastro"; + $command .= " -recipe PSPHOT $recipe_psphot"; + $command .= " -F PSASTRO.OUTPUT $output_sources_filerule"; + $command .= " -stats $stats"; + $command .= " -recipe PPSTATS CAMSTATS"; + $command .= " -dumpconfig $configuration" if $configuration; + $command .= " -tracedest $traceDest"; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psastro: $error_code", $skycal_id, $error_code); + } + + &my_die("Couldn't find stats file: $stats", $skycal_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($stats); + + { + my $resolved = $ipprc->file_resolve($stats) + or &my_die("failed to resolve stats file: $stats", $skycal_id, $PS_EXIT_SYS_ERROR) ; + my $command = "$ppStatsFromMetadata $resolved - STATICSKY_CAL"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psastro: $error_code", $skycal_id, $error_code); + } + $cmdflags = join "", @$stdout_buf; + chomp $cmdflags; + } + + my $quality; + ($quality) = $cmdflags =~ /-quality (\d+)/; + if (!$quality) { + my $outputSources = $ipprc->filename($output_sources_filerule, $outroot); + &my_die("Couldn't find expected output file: $outputSources", $skycal_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($outputSources); + + &my_die("Couldn't find expected output file: $configuration", $skycal_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($configuration); + } + } else { + print "Not executing: $command\n"; + } +} + + +# Add the result to the database +{ + my $command = "$staticskytool -skycal_id $skycal_id"; + $command .= " -addskycalresult -path_base $outroot"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + unless($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = "Unable to perform staticskytool -addresult"; + warn($err_message); + exit $error_code; + } + } else { + print "skipping $command\n"; + } +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $skycal_id = shift; # Diff identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $skycal_id and not $no_update) { + my $command = "$staticskytool -skycal_id $skycal_id -fault $exit_code"; + $command .= " -addskycalresult"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/videophot_process.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/videophot_process.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/videophot_process.pl (revision 33638) @@ -127,5 +127,7 @@ foreach my $cell_id (@video_cells) { my $path_base = "$outroot.$class_id.$cell_id"; - my $output = $ipprc->filename("PSVIDEOPHOT.OUTPUT", $path_base); + my $error; + my $output = $ipprc->prepare_output("PSVIDEOPHOT.OUTPUT", $path_base, undef, 1, \$error) + or &my_die("failed to prepare output file for PSVIDEOPHOT.OUTPUT", $vp_id, $error); my $command = "$psvideophot $output"; $command .= " -file $uri"; Index: /branches/eam_branches/ipp-20111122/ippScripts/scripts/warp_skycell.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippScripts/scripts/warp_skycell.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippScripts/scripts/warp_skycell.pl (revision 33638) @@ -373,17 +373,25 @@ # $run_state eq 'update' unless ($no_update) { - my $command = "$warptool -tofullskyfile"; - $command .= " -warp_id $warp_id"; - $command .= " -skycell_id $skycell_id"; - $command .= " -set_magicked $magicked" if $magicked; - $command .= " -set_quality $quality" if $quality; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to perform warptool -addwarped: $error_code\n"); - exit($error_code); + my $try = 0; + while (1) { + $try++; + my $command = "$warptool -tofullskyfile"; + $command .= " -warp_id $warp_id"; + $command .= " -skycell_id $skycell_id"; + $command .= " -set_magicked $magicked" if $magicked; + $command .= " -set_quality $quality" if $quality; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addwarped: $error_code\n"); + exit($error_code) if $try >= 2; + sleep 10; + warn("Trying again\n"); + } else { + last; + } } } Index: /branches/eam_branches/ipp-20111122/ippTasks/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/Makefile.am (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTasks/Makefile.am (revision 33638) @@ -21,4 +21,5 @@ destreak.cleanup.pro \ diff.pro \ + skycal.pro \ stack.pro \ staticsky.pro \ @@ -37,4 +38,5 @@ minidvodb.pro \ minidvodbcopy.pro \ + mergedvodb.pro \ nightly_science.pro \ burntool.pro \ Index: /branches/eam_branches/ipp-20111122/ippTasks/addstar.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/addstar.pro (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTasks/addstar.pro (revision 33638) @@ -26,7 +26,4 @@ active true end - task addstar.exp.load.staticsky_multi - active true - end task addstar.exp.run active true @@ -44,7 +41,4 @@ active false end - task addstar.exp.load.staticsky_multi - active false - end task addstar.exp.run active false @@ -53,10 +47,22 @@ macro addstar.revert.off - task addstar.revert + task addstar.revert.cam + active false + end + task addstar.revert.stack + active false + end + task addstar.revert.staticsky active false end end macro addstar.revert.on - task addstar.revert + task addstar.revert.cam + active true + end + task addstar.revert.stack + active true + end + task addstar.revert.staticsky active true end @@ -65,6 +71,10 @@ # this variable will cycle through the known database names -$addstar_DB = 0 -$addstar_revert_DB = 0 +$addstar_DB = 0 +#this may not work for more databases (addstar) will do that later) +$addstar_revert_DB_C = 0 +$addstar_revert_DB_S = 0 +$addstar_revert_DB_SS = 0 +$addstar_revert_DB_SSM = 0 @@ -237,59 +247,4 @@ end -task addstar.exp.load.staticsky_multi - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - - stdout NULL - stderr $LOGDIR/addstar.exp.log - - task.exec - # if ($LABEL:n == 0) break - $run = addtool -pendingexp -stage staticsky_multi - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$addstar_DB - $run = $run -dbname $DB:$addstar_DB - $addstar_DB ++ - if ($addstar_DB >= $DB:n) set addstar_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook addPendingExp - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup addPendingExp - end - - # default exit status - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - # run the addstar script on pending exposures @@ -318,5 +273,6 @@ book getword addPendingExp $pageName stage_extra1 -var STAGE_EXTRA1 book getword addPendingExp $pageName stageroot -var STAGEROOT - book getword addPendingExp $pageName stage -var STAGE + book getword addPendingExp $pageName stage -var STAGE + book getword addPendingExp $pageName stage_id -var STAGE_ID book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE book getword addPendingExp $pageName reduction -var REDUCTION @@ -350,7 +306,4 @@ sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID end - if ("$STAGE" == "staticsky_multi") - sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID - end if ("$STAGE" == "stack") sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID @@ -365,6 +318,9 @@ $run = $run --reduction $REDUCTION end - if ("$STAGE" == "staticsky_multi") + if ("$STAGE" == "staticsky") $run = $run --multi_num $STAGE_EXTRA1 + end + if ("$STAGE" == "cam") + $run = $run --stage_id $STAGE_ID end if ("$IMAGE_ONLY" == "T") @@ -411,5 +367,5 @@ end -task addstar.revert +task addstar.revert.cam host local @@ -425,13 +381,13 @@ task.exec if ($LABEL:n == 0) break - $run = addtool -revertprocessedexp + $run = addtool -revertprocessedexp -stage cam if ($DB:n == 0) option DEFAULT else # save the DB name for the exit tasks - option $DB:$addstar_revert_DB - $run = $run -dbname $DB:$addstar_revert_DB - $addstar_revert_DB ++ - if ($addstar_revert_DB >= $DB:n) set addstar_revert_DB = 0 + option $DB:$addstar_revert_DB_C + $run = $run -dbname $DB:$addstar_revert_DB_C + $addstar_revert_DB_C ++ + if ($addstar_revert_DB_C >= $DB:n) set addstar_revert_DB_C = 0 end add_poll_labels run @@ -457,2 +413,95 @@ end end +task addstar.revert.stack + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage stack + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_S + $run = $run -dbname $DB:$addstar_revert_DB_S + $addstar_revert_DB_S ++ + if ($addstar_revert_DB_S >= $DB:n) set addstar_revert_DB_S = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task addstar.revert.staticsky + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage staticsky + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_SSM + $run = $run -dbname $DB:$addstar_revert_DB_SSM + $addstar_revert_DB_SSM ++ + if ($addstar_revert_DB_SSM >= $DB:n) set addstar_revert_DB_SSM = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/eam_branches/ipp-20111122/ippTasks/fake.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/fake.pro (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTasks/fake.pro (revision 33638) @@ -146,5 +146,7 @@ # specify choice of local or remote host based on camera and fake (class_id) - set.host.for.camera $CAMERA $CLASS_ID + # XXX: this is not worth the trouble. use anyhost + # set.host.for.camera $CAMERA $CLASS_ID + host anyhost # set the WORKDIR variable Index: /branches/eam_branches/ipp-20111122/ippTasks/ipphosts.mhpcc.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/ipphosts.mhpcc.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTasks/ipphosts.mhpcc.config (revision 33638) @@ -471,77 +471,158 @@ camera STR STARE - ota01 STR ipp010 - ota02 STR ipp010 - ota03 STR ipp010 - ota04 STR ipp010 - - ota05 STR ipp011 - ota06 STR ipp011 - ota10 STR ipp011 - ota11 STR ipp011 - - ota12 STR ipp012 - ota13 STR ipp012 - ota14 STR ipp012 - ota15 STR ipp012 - - ota16 STR ipp013 - ota17 STR ipp013 - ota20 STR ipp013 - ota21 STR ipp013 - - ota22 STR ipp014 - ota23 STR ipp014 - ota24 STR ipp014 - ota25 STR ipp014 - - ota26 STR ipp015 - ota27 STR ipp015 - ota30 STR ipp015 - ota31 STR ipp015 - - ota32 STR ipp016 - ota33 STR ipp016 - ota34 STR ipp016 - ota35 STR ipp016 - - ota36 STR ipp017 - ota37 STR ipp017 - ota40 STR ipp017 - ota41 STR ipp017 - - ota42 STR ipp018 - ota43 STR ipp018 - ota44 STR ipp018 - ota45 STR ipp018 - - ota46 STR ipp019 - ota47 STR ipp019 - ota50 STR ipp019 - ota51 STR ipp019 - - ota52 STR ipp020 - ota53 STR ipp020 - ota54 STR ipp020 - ota55 STR ipp020 - - ota56 STR ipp021 - ota57 STR ipp021 - ota60 STR ipp021 - ota61 STR ipp021 - - ota62 STR ipp010 - ota63 STR ipp011 - ota64 STR ipp012 - ota65 STR ipp013 - - ota66 STR ipp014 - ota67 STR ipp015 - ota71 STR ipp016 - ota72 STR ipp017 - - ota73 STR ipp018 - ota74 STR ipp019 - ota75 STR ipp020 - ota76 STR ipp021 + ota01 STR stare00 + ota02 STR stare01 + ota03 STR stare02 + ota04 STR stare04 + + ota05 STR stare00 + ota06 STR stare01 + ota10 STR stare02 + ota11 STR stare04 + + ota12 STR stare00 + ota13 STR stare01 + ota14 STR stare02 + ota15 STR stare04 + + ota16 STR stare00 + ota17 STR stare01 + ota20 STR stare02 + ota21 STR stare04 + + ota22 STR stare00 + ota23 STR stare01 + ota24 STR stare02 + ota25 STR stare04 + + ota26 STR stare00 + ota27 STR stare01 + ota30 STR stare02 + ota31 STR stare04 + + ota32 STR stare00 + ota33 STR stare01 + ota34 STR stare02 + ota35 STR stare04 + + ota36 STR stare00 + ota37 STR stare01 + ota40 STR stare02 + ota41 STR stare04 + + ota42 STR stare00 + ota43 STR stare01 + ota44 STR stare02 + ota45 STR stare04 + + ota46 STR stare00 + ota47 STR stare01 + ota50 STR stare02 + ota51 STR stare04 + + ota52 STR stare00 + ota53 STR stare01 + ota54 STR stare02 + ota55 STR stare04 + + ota56 STR stare00 + ota57 STR stare01 + ota60 STR stare02 + ota61 STR stare04 + + ota62 STR stare00 + ota63 STR stare01 + ota64 STR stare02 + ota65 STR stare04 + + ota66 STR stare00 + ota67 STR stare01 + ota71 STR stare02 + ota72 STR stare04 + + ota73 STR stare00 + ota74 STR stare01 + ota75 STR stare02 + ota76 STR stare04 END + + +# Backup of old stare allocation setup. +# ipphosts METADATA +# camera STR STARE + +# ota01 STR ipp010 +# ota02 STR ipp010 +# ota03 STR ipp010 +# ota04 STR ipp010 + +# ota05 STR ipp011 +# ota06 STR ipp011 +# ota10 STR ipp011 +# ota11 STR ipp011 + +# ota12 STR ipp012 +# ota13 STR ipp012 +# ota14 STR ipp012 +# ota15 STR ipp012 + +# ota16 STR ipp013 +# ota17 STR ipp013 +# ota20 STR ipp013 +# ota21 STR ipp013 + +# ota22 STR ipp014 +# ota23 STR ipp014 +# ota24 STR ipp014 +# ota25 STR ipp014 + +# ota26 STR ipp015 +# ota27 STR ipp015 +# ota30 STR ipp015 +# ota31 STR ipp015 + +# ota32 STR ipp016 +# ota33 STR ipp016 +# ota34 STR ipp016 +# ota35 STR ipp016 + +# ota36 STR ipp017 +# ota37 STR ipp017 +# ota40 STR ipp017 +# ota41 STR ipp017 + +# ota42 STR ipp018 +# ota43 STR ipp018 +# ota44 STR ipp018 +# ota45 STR ipp018 + +# ota46 STR ipp019 +# ota47 STR ipp019 +# ota50 STR ipp019 +# ota51 STR ipp019 + +# ota52 STR ipp020 +# ota53 STR ipp020 +# ota54 STR ipp020 +# ota55 STR ipp020 + +# ota56 STR ipp021 +# ota57 STR ipp021 +# ota60 STR ipp021 +# ota61 STR ipp021 + +# ota62 STR ipp010 +# ota63 STR ipp011 +# ota64 STR ipp012 +# ota65 STR ipp013 + +# ota66 STR ipp014 +# ota67 STR ipp015 +# ota71 STR ipp016 +# ota72 STR ipp017 + +# ota73 STR ipp018 +# ota74 STR ipp019 +# ota75 STR ipp020 +# ota76 STR ipp021 +# END Index: /branches/eam_branches/ipp-20111122/ippTasks/mergedvodb.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/mergedvodb.pro (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTasks/mergedvodb.pro (revision 33638) @@ -0,0 +1,226 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + + +#There is a book for each task, because I don't use labels. + +if (not($?havemergedvodbBooks)) + book create MERGEDVODB_MERGE + $havemergedvodbBooks = TRUE +end + +$MERGEDVODB_DB = 0 +$MERGEDVODB_MERGE_DB = 0 + +book init mergedvodbMergelist + +macro mergedvodb.merge.status + book listbook mergedvodbMergelist +end + +macro mergedvodb.merge.reset + book init mergedvodbMergelist +end + +macro mergedvodb.merge.on + task mergedvodb.merge.load + active true + end + task mergedvodb.merge.run + active true + end +end +macro mergedvodb.merge.off + task mergedvodb.merge.load + active false + end + task mergedvodb.merge.run + active false + end +end + +## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. +macro add.mergedvodb + if ($0 != 2) + echo "USAGE: add.mergedvodb (mergedvodb)" + break + end + #merge + book newpage MERGEDVODB_MERGE $1 + book setword MERGEDVODB_MERGE $1 MERGEDVODB $1 + book setword MERGEDVODB_MERGE $1 STATE PENDING +end + +macro del.mergedvodb + if ($0 != 2) + echo "USAGE: del.mergedvodb (mergedvodb)" + break + end + book delpage MERGEDVODB_MERGE $1 +end + +macro show.mergedvodb + if ($0 != 1) + echo "USAGE: show.mergedvodb" + break + end + echo "mergedvodb merge" + book listbook MERGEDVODB_MERGE + echo "n pages" + book npages MERGEDVODB_MERGE -var N + echo $N +end + + +task mergedvodb.merge.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/mergedvodb.merge.load.log + + task.exec + book npages MERGEDVODB_MERGE -var N + if ($N == 0) + echo "No labels for processing" + break + endif + book getpage MERGEDVODB_MERGE 0 -var mergedvodb -key STATE NEW + if ("$mergedvodb" == "NULL") + # All labels have been done --- reset + for i 0 $N + book getpage MERGEDVODB_MERGE $i -var mergedvodb + book setword MERGEDVODB_MERGE $mergedvodb STATE NEW + end + book getpage MERGEDVODB_MERGE 0 -var mergedvodb -key STATE NEW + + # Select different database + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + book setword MERGEDVODB_MERGE $mergedvodb STATE DONE + + $run = mergetool -pendingmerge -limit 1 + $run = $run -mergedvodb $mergedvodb + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MERGEDVODB_DB + $run = $run -dbname $DB:$MERGEDVODB_DB + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + #add_poll_args run + #echo $run + command $run + end + + # success + task.exit 0 + #echo "success" + # convert 'stdout' to book format + ipptool2book stdout mergedvodbMergelist -key merge_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook mergedvodbMergelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup mergedvodbMergelist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task mergedvodb.merge.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60000 + + #we only want one running at a time + + host local + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + book npages mergedvodbMergelist -var N + if ($N == 0) break + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage mergedvodbMergelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + book setword mergedvodbMergelist $pageName pantaskState RUN + book getword mergedvodbMergelist $pageName merge_id -var MERGE_ID + book getword mergedvodbMergelist $pageName mergedvodb -var MERGEDVODB + book getword mergedvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH + book getword mergedvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword mergedvodbMergelist $pageName state -var STATE + stdout $LOGDIR/mergedvodb.merge.run.log + stderr $LOGDIR/mergedvodb.merge.run.log + $run = mergedvodb_merge.pl --camera GPC1 + $run = $run --mergedvodb_path $MERGEDVODB_PATH + $run = $run --minidvodb_path $MINIDVODB_PATH + $run = $run --merge_id $MERGE_ID --mergedvodb $MERGEDVODB + echo $run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MERGEDVODB_DB + $run = $run --dbname $DB:$MERGEDVODB_DB + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit mergedvodbMergelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + + echo "hostname: $JOB_HOSTNAME" + + # Set a fault code in the database + + process_exit mergedvodbMergelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword mergedvodbMergelist $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/ipp-20111122/ippTasks/minidvodb.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/minidvodb.pro (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTasks/minidvodb.pro (revision 33638) @@ -11,5 +11,5 @@ if (not($?haveminidvodbBooks)) book create MINIDVODB - book create MINIDVODB_MERGE + book create MINIDVODB_PREMERGE book create MINIDVODB_CREATE book create MINIDVODB_ACTIVE @@ -20,10 +20,10 @@ $MINIDVODB_WAIT_DB = 0 -$MINIDVODB_MERGE_DB = 0 +$MINIDVODB_PREMERGE_DB = 0 $MINIDVODB_CREATE_DB = 0 $MINIDVODB_ACTIVE_DB = 0 book init minidvodbWaitlist -book init minidvodbMergelist +book init minidvodbPreMergelist book init minidvodbCreatelist book init minidvodbActivelist @@ -54,10 +54,10 @@ end -macro minidvodb.merge.status - book listbook minidvodbMergelist -end - -macro minidvodb.merge.reset - book init minidvodbMergelist +macro minidvodb.premerge.status + book listbook minidvodbPreMergelist +end + +macro minidvodb.premerge.reset + book init minidvodbPreMergelist end @@ -112,17 +112,17 @@ ##these merge the dbs -macro minidvodb.merge.on - task minidvodb.merge.load +macro minidvodb.premerge.on + task minidvodb.premerge.load active true end - task minidvodb.merge.run + task minidvodb.premerge.run active true end end -macro minidvodb.merge.off - task minidvodb.merge.load +macro minidvodb.premerge.off + task minidvodb.premerge.load active false end - task minidvodb.merge.run + task minidvodb.premerge.run active false end @@ -142,8 +142,8 @@ book setword MINIDVODB $1 STATE PENDING #merge - book newpage MINIDVODB_MERGE $1 - book setword MINIDVODB_MERGE $1 MINIDVODB_GROUP $1 - book setword MINIDVODB_MERGE $1 DVODB $3 - book setword MINIDVODB_MERGE $1 STATE PENDING + book newpage MINIDVODB_PREMERGE $1 + book setword MINIDVODB_PREMERGE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_PREMERGE $1 DVODB $3 + book setword MINIDVODB_PREMERGE $1 STATE PENDING #active book newpage MINIDVODB_ACTIVE $1 @@ -168,5 +168,5 @@ end book delpage MINIDVODB $1 - book delpage MINIDVODB_MERGE $1 + book delpage MINIDVODB_PREMERGE $1 book delpage MINIDVODB_CREATE $1 book delpage MINIDVODB_ACTIVE $1 @@ -180,6 +180,6 @@ echo "minidvodb wait" book listbook MINIDVODB - echo "minidvodb merge" - book listbook MINIDVODB_MERGE + echo "minidvodb premerge" + book listbook MINIDVODB_PREMERGE echo "minidvodb create" book listbook MINIDVODB_CREATE @@ -339,5 +339,5 @@ -task minidvodb.merge.load +task minidvodb.premerge.load host local @@ -348,8 +348,8 @@ stdout NULL - stderr $LOGDIR/minidvodb.merge.load.log + stderr $LOGDIR/minidvodb.premerge.load.log task.exec - book npages MINIDVODB_MERGE -var N + book npages MINIDVODB_PREMERGE -var N if ($N == 0) # echo "No labels for processing" @@ -357,13 +357,13 @@ endif - book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW + book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW if ("$minidvodb_group" == "NULL") # All labels have been done --- reset # echo "Resetting labels" for i 0 $N - book getpage MINIDVODB_MERGE $i -var minidvodb_group - book setword MINIDVODB_MERGE $minidvodb_group STATE NEW + book getpage MINIDVODB_PREMERGE $i -var minidvodb_group + book setword MINIDVODB_PREMERGE $minidvodb_group STATE NEW end - book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW + book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW # Select different database @@ -372,5 +372,5 @@ end #finds the minidvodbs in a state of 'to_be_merged' - book setword MINIDVODB_MERGE $minidvodb_group STATE DONE + book setword MINIDVODB_PREMERGE $minidvodb_group STATE DONE $run = addtool -listminidvodbrun -state to_be_merged -limit 1 $run = $run -minidvodb_group $minidvodb_group @@ -392,10 +392,10 @@ task.exit 0 # convert 'stdout' to book format - ipptool2book stdout minidvodbMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout minidvodbPreMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT if ($VERBOSE > 2) - book listbook minidvodbMergelist + book listbook minidvodbPreMergelist end # delete existing entries in the appropriate pantaskStates - process_cleanup minidvodbMergelist + process_cleanup minidvodbPreMergelist end @@ -418,5 +418,5 @@ -task minidvodb.merge.run +task minidvodb.premerge.run periods -poll $RUNPOLL periods -exec $RUNEXEC @@ -432,21 +432,20 @@ periods -exec $RUNEXEC - book npages minidvodbMergelist -var N + book npages minidvodbPreMergelist -var N if ($N == 0) break # look for new images in minidvodbWaitlist (pantaskState == INIT) - book getpage minidvodbMergelist 0 -var pageName -key pantaskState INIT + book getpage minidvodbPreMergelist 0 -var pageName -key pantaskState INIT if ("$pageName" == "NULL") break - book setword minidvodbMergelist $pageName pantaskState RUN - book getword minidvodbMergelist $pageName minidvodb_id -var MINIDVODB_ID - book getword minidvodbMergelist $pageName minidvodb_group -var MINIDVODB_GROUP - book getword minidvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH - book getword minidvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH - book getword minidvodbMergelist $pageName state -var STATE - stdout $LOGDIR/minidvodb.merge.run.log - stderr $LOGDIR/minidvodb.merge.run.log - - $run = minidvodb_merge.pl --camera GPC1 --mergedvodb $MERGEDVODB_PATH --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID + book setword minidvodbPreMergelist $pageName pantaskState RUN + book getword minidvodbPreMergelist $pageName minidvodb_id -var MINIDVODB_ID + book getword minidvodbPreMergelist $pageName minidvodb_group -var MINIDVODB_GROUP + book getword minidvodbPreMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword minidvodbPreMergelist $pageName state -var STATE + stdout $LOGDIR/minidvodb.premerge.run.log + stderr $LOGDIR/minidvodb.premerge.run.log + + $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID if ($DB:n == 0) @@ -472,5 +471,5 @@ # default exit status task.exit default - process_exit minidvodbMergelist $options:0 $JOB_STATUS + process_exit minidvodbPreMergelist $options:0 $JOB_STATUS end @@ -482,5 +481,5 @@ # Set a fault code in the database - process_exit minidvodbMergelist $options:0 $EXIT_CRASH_ERR + process_exit minidvodbPreMergelist $options:0 $EXIT_CRASH_ERR end @@ -488,5 +487,5 @@ task.exit timeout showcommand timeout - book setword minidvodbMergelist $options:0 pantaskState TIMEOUT + book setword minidvodbPreMergelist $options:0 pantaskState TIMEOUT end end Index: /branches/eam_branches/ipp-20111122/ippTasks/nightly_stacks.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/nightly_stacks.pro (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTasks/nightly_stacks.pro (revision 33638) @@ -290,7 +290,7 @@ task ns.dqstats.load host local - periods -poll 3600 + periods -poll 10 periods -exec $LOADEXEC - periods -timeout 30 + periods -timeout 600 trange 22:00:00 23:59:59 trange 00:00:00 10:00:00 Index: /branches/eam_branches/ipp-20111122/ippTasks/pstamp.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/pstamp.pro (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTasks/pstamp.pro (revision 33638) @@ -626,5 +626,5 @@ periods -poll $LOADPOLL # XXX: create a macro for this time - periods -exec 20 + periods -exec 15 # periods -exec $LOADEXEC periods -timeout 300 @@ -676,5 +676,5 @@ task pstamp.dependent.run periods -poll $RUNPOLL - periods -exec 5 + periods -exec 0.5 periods -timeout 300 Index: /branches/eam_branches/ipp-20111122/ippTasks/rcserver.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/rcserver.pro (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTasks/rcserver.pro (revision 33638) @@ -108,4 +108,5 @@ periods -exec $RUNEXEC periods -timeout 60 + npending 5 task.exec Index: /branches/eam_branches/ipp-20111122/ippTasks/skycal.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/skycal.pro (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTasks/skycal.pro (revision 33638) @@ -0,0 +1,246 @@ +## skycal.pro : tasks for static sky calibration analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the +## skycal analysis. After a skycalRun entry (with +## associated skycal_id) is defined, psastro is performed (via script +## skycalibration.pl) (tasks in skycalRun). + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init skycalRun + +### Database lists +$skycal_DB = 0 +$skycal_revert_DB = 0 + +### Check status of skycal tasks +macro skycal.status + book listbook skycalRun +end + +### Reset skycal tasks +macro skycal.reset + book init skycalRun +end + +### Turn skycal tasks on +macro skycal.on + task skycal.load + active true + end + task skycal.run + active true + end +end + +### Turn skycal tasks off +macro skycal.off + task skycal.load + active false + end + task skycal.run + active false + end +end + +macro skycal.revert.on + task skycal.revert + active true + end +end + +macro skycal.revert.off + task skycal.revert + active false + end +end + +### Load tasks for skycal +### Tasks are loaded into skycalRun. +task skycal.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/skycal.log + + task.exec + if ($LABEL:n == 0) break + $run = staticskytool -pendingskycalrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$skycal_DB + $run = $run -dbname $DB:$skycal_DB -limit 40 + $skycal_DB ++ + if ($skycal_DB >= $DB:n) set skycal_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout skycalRun -key skycal_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook skycalRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup skycalRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the skycal analysis +### Tasks are taken from skycalRun. +task skycal.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + book npages skycalRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new entries in skycalRun (pantaskState == INIT) + book getpage skycalRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword skycalRun $pageName pantaskState RUN + book getword skycalRun $pageName skycal_id -var SKYCAL_ID + book getword skycalRun $pageName stack_id -var STACK_ID + book getword skycalRun $pageName tess_id -var TESS_DIR + book getword skycalRun $pageName skycell_id -var SKYCELL_ID + book getword skycalRun $pageName camera -var CAMERA + book getword skycalRun $pageName filter -var FILTER + book getword skycalRun $pageName workdir -var WORKDIR_TEMPLATE + book getword skycalRun $pageName path_base -var PATH_BASE + book getword skycalRun $pageName reduction -var REDUCTION + book getword skycalRun $pageName singlefilter -var SINGLEFILTER + book getword skycalRun $pageName dbname -var DBNAME + book getword skycalRun $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.stk.%s.skycal.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $STACK_ID $SKYCAL_ID + + stdout $LOGDIR/skycal.log + stderr $LOGDIR/skycal.log + + $run = skycalibration.pl --skycal_id $SKYCAL_ID --outroot $outroot --redirect-output --camera $CAMERA --path_base $PATH_BASE --stack_id $STACK_ID --filter $FILTER + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$SINGLEFILTER" != "0") + $run = $run --singlefilter + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit skycalRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword skycalRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword skycalRun $options:0 pantaskState TIMEOUT + end +end + +task skycal.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = staticskytool -revertskycal -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$skycal_revert_DB + $run = $run -dbname $DB:$skycal_revert_DB + $skycal_revert_DB ++ + if ($skycal_revert_DB >= $DB:n) set skycal_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /branches/eam_branches/ipp-20111122/ippTasks/summit.copy.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/summit.copy.pro (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTasks/summit.copy.pro (revision 33638) @@ -29,4 +29,8 @@ # list of pzDownloadExps that have completed downloading book init pzPendingAdvance + +macro copy.status + book listbook pzPendingImfile +end macro copy.on @@ -446,6 +450,12 @@ substr $EXP_NAME 10 1 EXPTYPE_KEY if ("$EXPTYPE_KEY" == "a") + if ($NEBULOUS == 1) + $workdir_template = neb://@HOST@.1 + end set.workdir.by.camera STARE $CLASS_ID $workdir_template $default_host workdir_base else + if ($NEBULOUS == 1) + $workdir_template = neb://@HOST@.0 + end set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base end Index: /branches/eam_branches/ipp-20111122/ippTasks/survey.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/survey.pro (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTasks/survey.pro (revision 33638) @@ -3,22 +3,4 @@ # test for required global variables check.globals - -# there is a bug in 'getbook', use a different method below -# book getbook SURVEY_DIFF -var isBook -# if ("$isBook" != "SURVEY_DIFF") -# book create SURVEY_DIFF -# end -# book getbook SURVEY_MAGIC -var isBook -# if ("$isBook" == "NULL") -# book create SURVEY_MAGIC -# end -# book getbook SURVEY_DESTREAK -var isBook -# if ("$isBook" == "NULL") -# book create SURVEY_DESTREAK -# end -# book getbook SURVEY_DIST -var isBook -# if ("$isBook" == "NULL") -# book create SURVEY_DIST -# end if (not($?haveSurveyBooks)) @@ -30,4 +12,5 @@ book create SURVEY_DIST book create SURVEY_ADDSTAR + book create SURVEY_MERGEDVODB book create SURVEY_CHIP_BG book create SURVEY_WARP_BG @@ -44,4 +27,5 @@ $SURVEY_DIST_DB = 0 $SURVEY_ADDSTAR_DB = 0 +$SURVEY_MERGEDVODB_DB = 0 $SURVEY_CHIP_BG_DB = 0 $SURVEY_WARP_BG_DB = 0 @@ -68,4 +52,7 @@ active true end + task survey.mergedvodb + active true + end task survey.destreak active true @@ -104,4 +91,7 @@ active false end + task survey.mergedvodb + active false + end task survey.destreak active false @@ -270,4 +260,31 @@ book listbook SURVEY_ADDSTAR end + +macro survey.add.mergedvodb + if ($0 != 4) + echo "USAGE: survey.add.mergedvodb (tag) (mergedvodb) (minidvodb_group)" + break + end + book newpage SURVEY_MERGEDVODB $1 + book setword SURVEY_MERGEDVODB $1 MERGEDVODB $2 + book setword SURVEY_MERGEDVODB $1 MINIDVODB_GROUP $3 +end + +macro survey.del.mergedvodb + if ($0 != 2) + echo "USAGE: survey.del.mergedvodb (tag)" + break + end + book delpage SURVEY_MERGEDVODB $1 +end + +macro survey.show.mergedvodb + if ($0 != 1) + echo "USAGE: survey.show.mergedvodb" + break + end + book listbook SURVEY_MERGEDVODB +end + # user functions to manipulate destreak labels @@ -617,5 +634,8 @@ periods -exec $SURVEY_EXEC periods -timeout $SURVEY_TIMEOUT - trange 13:00:00 14:00:00 +# 13:00 was being done in early morning rather than afternoon HST (so UT value), want both times to have SSdiff ASAP? +# changing earlier for QUB to fetch @10pm for nightly processing there, should be enough time to finish MD stacks +# trange 13:00:00 14:00:00 + trange 20:00:00 21:00:00 npending 1 @@ -826,11 +846,12 @@ if ("$stage" == "cam") #only queue destreaked cams. stacks and staticsky don't need this - if ("$DB:$SURVEY_ADDSTAR_DB" == "isp") + # if ("$DB:$SURVEY_ADDSTAR_DB" == "isp") #this is the only way I can think of how to handle this (but it is messy). If it is a database that we KNOW doesn't use magicked (ie, isp), then do not queue destreaked + # now with no magic, all are uncensored $run = $run -uncensored - else +# else #if not isp, then run destreaked version only - $run = $run -destreaked - end +# $run = $run -destreaked +# end end if ("$stage" == "staticsky") @@ -838,9 +859,4 @@ $run = $run -uncensored end - if ("$stage" == "staticsky_multi") - #only queue uncensored staticsky (multi filter) - $run = $run -uncensored - end - if ("$stage" == "stack") #only queue uncensored stacks @@ -875,4 +891,82 @@ end end + + +task survey.mergedvodb + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.mergedvodb.log + stderr $LOGDIR/survey.mergedvodb.log + + task.exec + book npages SURVEY_MERGEDVODB -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW + if ("$tag" == "NULL") + # All labels have been done --- reset + # echo "Resetting labels" + for i 0 $N + book getpage SURVEY_MERGEDVODB $i -var tag + book setword SURVEY_MERGEDVODB $tag STATE NEW + end + book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW + + # Select different database + $SURVEY_MERGEDVODB_DB ++ + if ($SURVEY_MERGEDVODB_DB >= $DB:n) set SURVEY_MERGEDVODB_DB = 0 + end + book setword SURVEY_MERGEDVODB $tag STATE DONE + book getword SURVEY_MERGEDVODB $tag MERGEDVODB -var mergedvodb + book getword SURVEY_MERGEDVODB $tag MINIDVODB_GROUP -var minidvodb_group + + # $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb + #$run = mergetool + $run = mergedvodb_queue.pl --outroot $mergedvodb --mergedvodb $mergedvodb --minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + $run = $run --dbname $DB:$SURVEY_MERGEDVODB_DB + option $DB:$SURVEY_MERGEDVODB_DB + end + #this run doesn't work because there's no workdir + #$run = $run -definebyquery -mergedvodb $mergedvodb -minidvodb_group $minidvodb_group + + + # echo $run + + echo $run + command $run + end + + # success + + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /branches/eam_branches/ipp-20111122/ippTasks/vp.pro =================================================================== --- /branches/eam_branches/ipp-20111122/ippTasks/vp.pro (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTasks/vp.pro (revision 33638) @@ -13,4 +13,6 @@ $vp_DB = 0 $vp_revert_DB = 0 + +$VP_POLL_LIMIT = 20 ### Check status of vping tasks @@ -32,7 +34,4 @@ active true end - task vp.revert - active true - end end @@ -45,7 +44,4 @@ active false end - task vp.revert - active false - end end @@ -60,4 +56,11 @@ active false end +end + +macro set.vp.poll + $VP_POLL_LIMIT = $1 +end +macro get.vp.poll + echo $VP_POLL_LIMIT end @@ -90,4 +93,5 @@ add_poll_args run add_poll_labels run + $run = $run -limit $VP_POLL_LIMIT command $run end @@ -159,4 +163,6 @@ stdout $LOGSUBDIR/vp.log stderr $LOGSUBDIR/vp.log + + host anyhost $run = videophot_process.pl --vp_id $VP_ID --outroot $outroot --redirect-output --camera $CAMERA Index: /branches/eam_branches/ipp-20111122/ippToPsps/config/OB/tables.vot =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/config/OB/tables.vot (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/config/OB/tables.vot (revision 33638) @@ -0,0 +1,298 @@ + + + + + + VOTable description of PSPS table Object + + + ODM object identifier index + + + IPP object number + + + projection cell identifier at discovery time + + + HTM index + + + Zone index + + + static stack version number + + + flag indicating odd properties + + + variable in difference image detections + + + Survey ID + + + Right Ascension + + + Declination + + + Right Ascension error + + + Declination error + + + Cartesian x + + + Cartesian y + + + Cartesian z + + + ecliptic longitude + + + ecliptic latitude + + + galactic longitude + + + galactic latitude + + + supergalactic longitude + + + supergalactic latitude + + + total number of detection measurements in all filters + + + mean RA of object + + + mean DEC of object + + + estimated error in raMean + + + estimated error in decMean + + + number of measures in g filter + + + mean g magnitude + + + mean g flux + + + estimated error of g magnitude mean + + + estimated error of g flux mean + + + minimum g magnitide + + + maximum g magnitude + + + number of measures in r filter + + + mean r magnitude + + + mean r flux + + + estimated error of r magnitude mean + + + estimated error of r flux mean + + + minimum r magnitude + + + maximum r magnitude + + + number of measures in i filter + + + mean i magnitude + + + mean i flux + + + estimated error of i magitude mean + + + estimated error of i flux mean + + + minimum i magniutde + + + maximum i magnitude + + + number of measures in z filter + + + mean z magnitude + + + mean z flux + + + estimated error of z magnitude mean + + + estimated error of z flux mean + + + minimum z magnitude + + + maximum z magnitude + + + number of measures in y filter + + + mean y magnitude + + + mean y flux + + + estimated error of y magnitude mean + + + estimated error of y flux mean + + + minimum y magnitude + + + maximum y magnitude + + + number of measures in the wide (w) solar system filter + + + mean w magnitude + + + mean w flux + + + estimated error of w magnitude mean + + + estimated error of w flux mean + + + minimum w magnitude + + + maximum w magnitude + + + g-r color from gMean &amp; rMean + + + r-i color from rMean &amp; iMean + + + i-z color from iMean &amp; zMean + + + z-y color from zMean &amp; yMean + + + Estimated error in g-r color from gMean &amp; rMean + + + Estimated error in r-i color from rMean &amp; iMean + + + Estimated error in i-z color from iMean &amp; zMean + + + Estimated error in z-y color from zMean &amp; yMean + + + g magnitude from stack frames + + + index of which g model magnitude was used from stack + + + r magnitude from stack frames + + + index of which r model magnitude was used from stack + + + i magnitude from stack frames + + + index of which i model magnitude was used from stack + + + z magnitude from stack frames + + + index of which z model magnitude was used from stack + + + y magnitude from stack frames + + + index of which y model magnitude was used from stack + + + g-r color from gmag and rmag + + + r-i color from rmag and imag + + + i-z color from imag and zmag + + + z-y color from zmag and ymag + + + star/galaxy separator - true if any one color is + + + Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation + + + Data release + + + MJD of last update + + + + +
+ + Index: /branches/eam_branches/ipp-20111122/ippToPsps/config/ippToPspsDbSchema.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/config/ippToPspsDbSchema.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/config/ippToPspsDbSchema.sql (revision 33638) @@ -0,0 +1,201 @@ +-- MySQL dump 10.11 +-- +-- Host: ippdb01 Database: ipptopsps +-- ------------------------------------------------------ +-- Server version 5.0.77-log + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `batch` +-- + +DROP TABLE IF EXISTS `batch`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `batch` ( + `batch_id` bigint(20) unsigned NOT NULL auto_increment, + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `batch_type` varchar(10) NOT NULL, + `survey` varchar(10) default NULL, + `dvo_db` varchar(100) default NULL, + `datastore_product` varchar(100) default NULL, + `min_obj_id` bigint(20) default NULL, + `max_obj_id` bigint(20) default NULL, + `processed` tinyint(4) default '0', + `loaded_to_datastore` tinyint(4) default '0', + `loaded_to_ODM` tinyint(4) default '0', + `merge_worthy` tinyint(4) default '0', + `merged` tinyint(4) default '0', + `deleted_local` tinyint(4) default '0', + `deleted_datastore` tinyint(4) default '0', + `deleted_dxlayer` tinyint(4) default '0', + `stage_id` bigint(20) default NULL, + `total_detections` bigint(20) default NULL, + `comment` varchar(400) default NULL, + `purged` tinyint(4) NOT NULL default '0', + PRIMARY KEY (`batch_id`) +) ENGINE=InnoDB AUTO_INCREMENT=326479 DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `box` +-- + +DROP TABLE IF EXISTS `box`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `box` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `id` bigint(20) unsigned NOT NULL auto_increment, + `config` varchar(30) default NULL, + `ra_center` float default NULL, + `dec_center` float default NULL, + `box_side` float default NULL, + PRIMARY KEY (`id`), + KEY `fk_config` (`config`), + CONSTRAINT `fk_config` FOREIGN KEY (`config`) REFERENCES `config` (`name`) ON DELETE CASCADE +) ENGINE=InnoDB AUTO_INCREMENT=25457 DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `clients` +-- + +DROP TABLE IF EXISTS `clients`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `clients` ( + `started` timestamp NOT NULL default CURRENT_TIMESTAMP, + `type` varchar(50) default NULL, + `pid` bigint(20) unsigned NOT NULL, + `host` varchar(20) NOT NULL, + `config` varchar(30) default NULL, + `paused` tinyint(1) default '0', + `killed` tinyint(1) default '0', + `timestamp` timestamp NOT NULL default '0000-00-00 00:00:00', + `id` bigint(20) unsigned NOT NULL auto_increment, + PRIMARY KEY (`id`), + UNIQUE KEY `unique_key` (`host`,`pid`) +) ENGINE=InnoDB AUTO_INCREMENT=961 DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `config` +-- + +DROP TABLE IF EXISTS `config`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `config` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `name` varchar(30) NOT NULL, + `datastore_product` varchar(30) default NULL, + `datastore_type` varchar(30) default 'IPP_PSPS', + `datastore_publish` tinyint(1) default '0', + `dvo_label` varchar(100) default NULL, + `dvo_location` varchar(1000) default NULL, + `min_ra` double default '0', + `max_ra` double default '360', + `min_dec` double default '-30', + `max_dec` double default '90', + `box_size` double default '4', + `base_path` varchar(1000) default NULL, + `data_release` smallint(6) default '0', + `delete_local` tinyint(1) default '0', + `delete_datastore` tinyint(1) default '1', + `delete_dxlayer` tinyint(1) default '1', + `epoch` timestamp NOT NULL default '0000-00-00 00:00:00', + `survey` varchar(10) default NULL, + `psps_survey` varchar(10) default NULL, + `queue_P2` tinyint(1) default '1', + `queue_ST` tinyint(1) default '0', + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `detection` +-- + +DROP TABLE IF EXISTS `detection`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `detection` ( + `batch_id` bigint(20) unsigned NOT NULL, + `exp_id` bigint(20) default NULL, + `filter` varchar(1) default NULL, + KEY `batch_id` (`batch_id`), + CONSTRAINT `detection_ibfk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `pending` +-- + +DROP TABLE IF EXISTS `pending`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `pending` ( + `box_id` bigint(20) unsigned default NULL, + `batch_type` varchar(10) default NULL, + `stage_id` bigint(20) default NULL, + `ra_bore` float default NULL, + `dec_bore` float default NULL, + UNIQUE KEY `a_key` (`box_id`,`stage_id`,`batch_type`), + CONSTRAINT `box_key` FOREIGN KEY (`box_id`) REFERENCES `box` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `stack` +-- + +DROP TABLE IF EXISTS `stack`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `stack` ( + `batch_id` bigint(20) unsigned NOT NULL, + `filter` varchar(1) default NULL, + `stack_type` varchar(20) default NULL, + KEY `batch_id` (`batch_id`), + CONSTRAINT `stack_ibfk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- +-- Table structure for table `stripe` +-- + +DROP TABLE IF EXISTS `stripe`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `stripe` ( + `client_id` bigint(20) unsigned default NULL, + `config` varchar(30) default NULL, + `ra_center` float default NULL, + UNIQUE KEY `unique_key` (`client_id`), + CONSTRAINT `stripe_ibfk_1` FOREIGN KEY (`client_id`) REFERENCES `clients` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2012-03-22 7:59:34 Index: /branches/eam_branches/ipp-20111122/ippToPsps/config/settings.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/config/settings.xml (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/config/settings.xml (revision 33638) @@ -0,0 +1,44 @@ + + + + + + + /data/ipp005.0/rhenders + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + czardb + ippdb01 + ipp + ipp + + + + /data/ipp009.0/czartool_plots + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/batch.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/batch.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/batch.py (revision 33638) @@ -21,5 +21,5 @@ ''' -Base class of all batch types. +Abstract base class of all batch types. ''' class Batch(object): @@ -27,25 +27,22 @@ ''' Constructor - - >>> batch = Batch(1,2,3,4,5,6,7) - >>> print batch.pspsVoTableFilePath - "../config/2/tables.vot" ''' def __init__(self, logger, - configPath, - doc, + config, gpc1Db, ippToPspsDb, + scratchDb, id, batchID, batchType, - fits): + fits, + useFullTables): self.everythingOK = False self.readHeader = False - self.configPath = configPath - self.doc = doc + self.config = config self.fits = fits + self.useFullTables = useFullTables # set up logging @@ -59,4 +56,5 @@ self.gpc1Db = gpc1Db self.ippToPspsDb = ippToPspsDb + self.scratchDb = scratchDb self.batchType = batchType; self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot" @@ -77,41 +75,28 @@ return - # get info from config - self.survey = self.doc.find("options/survey").text - self.pspsSurvey = self.doc.find("options/pspsSurvey").text - self.dvoGpc1Label = self.doc.find("dvo/gpc1Label").text - self.dvoLocation = self.doc.find("dvo/location").text - self.useFullTables = int(self.doc.find("dvo/useFullTables").text) - self.scratchDb = ScratchDb(logger, self.doc, self.useFullTables) - - if not self.scratchDb.everythingOK: return + self.scratchDb.setUseFullTables(self.useFullTables) # TODO self.tablesToExport = [] - if self.survey != "": - self.surveyID = self.scratchDb.getSurveyID(self.survey) + if self.config.survey != "": + self.surveyID = self.scratchDb.getSurveyID(self.config.survey) else: self.surveyID = -1; - # get some options from the config - self.testMode = int(self.doc.find("options/testMode").text) - self.dataRelease = int(self.doc.find("metadata/dataRelease").text) - # create datastore object - self.datastore = Datastore(self.logger, self.doc, self.ippToPspsDb) + self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) # get local storage location from config self.batchName = Batch.getNameFromID(self.batchID) - self.basePath = self.doc.find("localOutPath").text self.subDir = Batch.getSubDir( - self.basePath, + self.config.basePath, self.batchType, - self.dvoGpc1Label) + self.config.dvoLabel) self.localOutPath = Batch.getOutputPath( - self.basePath, + self.config.basePath, self.batchType, - self.dvoGpc1Label, + self.config.dvoLabel, self.batchID) @@ -128,8 +113,5 @@ self.logger.infoTitle("New " + self.batchType + " batch") self.logger.infoPair("Batch name", self.batchName) - self.logger.infoPair("Survey", self.survey) self.logger.infoPair("Survey ID", "%d" % self.surveyID) - self.logger.infoPair("Publishing to PSPS as survey", self.pspsSurvey) - self.logger.infoPair("DVO location", self.dvoLocation) self.logger.infoBool("Use full DVO tables?", self.useFullTables) @@ -230,5 +212,5 @@ if value != "NULL": return value else: - if not self.testMode: return "NULL" + if not self.config.test: return "NULL" header[key] = default self.logger.infoPair("Hardcoding " + key + " to", header[key]) @@ -259,5 +241,7 @@ root.attrib['type'] = self.batchType root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - if self.batchType != "IN": root.attrib['survey'] = self.pspsSurvey + root.attrib['survey'] = self.config.pspsSurvey + + # min/max object IDs try: self.minObjID except: pass @@ -278,7 +262,15 @@ # clunky way to prettify XML - p = Popen("xmllint --format " + tmpPath + " > " + outPath, shell=True, stdout=PIPE) - p.wait() - os.remove(tmpPath) + try: + cmd = "xmllint --format " + tmpPath + " > " + outPath + p = Popen(cmd, shell=True, stdout=PIPE) + p.wait() + except: + self.logger.errorPair("Popen failed", cmd) + return False + finally: + os.remove(tmpPath) + + return True @@ -290,7 +282,7 @@ # set up filenams and paths tarFile = Batch.getTarFile(self.batchID) - tarPath = Batch.getTarPath(self.basePath, self.batchType, self.dvoGpc1Label, self.batchID) + tarPath = Batch.getTarPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID) tarballFile = Batch.getTarballFile(self.batchID) - tarballPath = Batch.getTarballPath(self.basePath, self.batchType, self.dvoGpc1Label, self.batchID) + tarballPath = Batch.getTarballPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID) # tar directory @@ -395,5 +387,10 @@ self.logger.infoPair("Importing tables with filter", filter) - tables = stilts.treads(self.fits.getPath()) + + try: + tables = stilts.treads(self.fits.getPath()) + except: + self.logger.errorPair("STILTS could not import from", self.fits.getPath()) + return False count = 0 @@ -496,10 +493,10 @@ def getIDsFromDVO(self): - if self.scratchDb.getRowCount("dvoMeta") < 1: - self.logger.error("No DVO IDs found in dvoMeta") + if self.scratchDb.getRowCount(self.scratchDb.dvoImagesTable) < 1: + self.logger.errorInfo("No DVO images found in", self.scratchDb.dvoImagesTable) return False # TODO path to DVO prog hardcoded temporarily - cmd = "../src/dvograbber " + self.configPath + " " + self.dvoLocation + cmd = "../src/dvograbber " + self.config.settingsPath + " " + self.scratchDb.dbName + " " + self.config.dvoLocation self.logger.infoPair("Running DVO", cmd) p = Popen(cmd, shell=True, stdout=PIPE) @@ -539,13 +536,16 @@ self.logger.errorPair("Aborting this batch", "unable to export tables to FITS file") else: - self.writeBatchManifest() - if int(self.doc.find("options/publishToDatastore").text): - # tar and zip ready for publication to datastore - if self.tarAndZip(): - # now publish to the datastore - tarballFile = Batch.getTarballFile(self.batchID) - Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile) - - if int(self.doc.find("options/reportNulls").text): self.reportNullsInAllPspsTables(False) + if self.writeBatchManifest(): + if self.config.datastorePublishing: + # tar and zip ready for publication to datastore + if self.tarAndZip(): + # now publish to the datastore + tarballFile = Batch.getTarballFile(self.batchID) + Batch.publishToDatastore( + self.datastore, + self.batchID, + self.batchName, + self.subDir, + tarballFile) from datastore import Datastore Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/cleanup.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/cleanup.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/cleanup.py (revision 33638) @@ -9,114 +9,122 @@ import logging.config -from pslogger import PSLogger -from ipptopspsdb import IppToPspsDb +from ipptopsps import IppToPsps from batch import Batch from datastore import Datastore from dxlayer import DXLayer -''' -Finds stuff to delete for this batch type, then: - -- removes from the datastore -- deletes from local disk -- deletes files from DXLayer ''' -def clean(batchType): +Cleanup class +''' +class Cleanup(IppToPsps): - logger.infoSeparator() - logger.infoTitle("Cleaning up batch type %s" % batchType) + ''' + Constructor + ''' + def __init__(self, argv): + super(Cleanup, self).__init__(argv) - localIDs = ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType, EPOCH, DVOLABEL) - datastoreIDs = ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType, EPOCH, DVOLABEL) - dxlayerIDs = ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType, EPOCH, DVOLABEL) - logger.infoPair("Merged but still on local disk", "%d" % len(localIDs)) - logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(datastoreIDs)) - logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(dxlayerIDs)) + self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + self.dxlayer = DXLayer(self.logger) + if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) - if DELETELOCAL: - count = 0 - for id in localIDs: + + ''' + Runs... + ''' + def run(self): + + if self.SECONDS: self.logger.infoPair("Performing clean-up every", "%f hours" % (self.SECONDS/60.0/60.0)) + while True: + + self.config.printDeletionPolicy() + self.clean("P2") + self.clean("ST") + + self.checkClientStatus() + if not self.waitForPollTime(): break + + ''' + Finds stuff to delete for this batch type, then: + + - removes from the datastore + - deletes from local disk + - deletes files from DXLayer + + ''' + def clean(self, batchType): + + self.logger.infoSeparator() + self.logger.infoTitle("Cleaning up batch type %s" % batchType) + + loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel) + loadedDxlayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType, self.config.epoch, self.config.dvoLabel) + mergedLocalIDs = self.ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel) + + purgedDatastoreIDs = self.ippToPspsDb.getPurgedButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel) + purgedLocalIDs = self.ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel) + + self.logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(loadedDatastoreIDs)) + self.logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(loadedDxlayerIDs)) + self.logger.infoPair("Merged but still on local disk", "%d" % len(mergedLocalIDs)) + self.logger.infoPair("Purged but still on datastore", "%d" % len(purgedDatastoreIDs)) + self.logger.infoPair("Purged but still on local disk", "%d" % len(purgedLocalIDs)) - if Batch.deleteFromDisk(logger, BASEPATH, batchType, DVOLABEL, id): - ippToPspsDb.updateDeletedLocal(id, 1) - count = count + 1 + if self.config.deleteLocal: + self.logger.infoSeparator() + count = 0 + ids = mergedLocalIDs + purgedLocalIDs + for id in ids: + + if Batch.deleteFromDisk(self.logger, self.config.basePath, batchType, self.config.dvoLabel, id): + self.ippToPspsDb.updateDeletedLocal(id, 1) + count = count + 1 + + self.logger.infoSeparator() + self.logger.infoPair("Removed from local disk", "%d" % count) + self.logger.infoPair("Remaining on local disk", "%d" % (len(ids) - count)) + + + if self.config.deleteDatastore: + self.logger.infoSeparator() + count = 0 + ids = loadedDatastoreIDs + purgedDatastoreIDs + for id in ids: + + batchName = Batch.getNameFromID(id) + + if self.datastore.remove(batchName): + count = count + 1 + + self.logger.infoSeparator() + self.logger.infoPair("Removed from datastore", "%d" % count) + self.logger.infoPair("Remaining on datastore", "%d" % (len(ids) - count)) + + if self.config.deleteDxLayer: + self.logger.infoSeparator() + count = 0 + for id in loadedDxlayerIDs: + + if self.dxlayer.deleteBatch(id): + self.ippToPspsDb.updateDeletedDxlayer(id, 1) + count = count + 1 + + self.logger.infoSeparator() + self.logger.infoPair("Removed from DXLayer", "%d" % count) + self.logger.infoPair("Remaining on DXLayer", "%d" % (len(loadedDxlayerIDs) - count)) + - logger.infoPair("Removed from local disk", "%d" % count) - logger.infoPair("Remaining on local disk", "%d" % (len(localIDs) - count)) - + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Cleanup, self).printUsage("[]") - if DELETEDATASTORE: - count = 0 - for id in datastoreIDs: - - batchName = Batch.getNameFromID(id) - - if datastore.remove(batchName): - count = count + 1 - - logger.infoPair("Removed from datastore", "%d" % count) - logger.infoPair("Remaining on datastore", "%d" % (len(datastoreIDs) - count)) - - if DELETEDXLAYER: - count = 0 - for id in dxlayerIDs: - - if dxlayer.deleteBatch(id): - ippToPspsDb.updateDeletedDxlayer(id, 1) - count = count + 1 - - logger.infoPair("Removed from DXLayer", "%d" % count) - logger.infoPair("Remaining on DXLayer", "%d" % (len(dxlayerIDs) - count)) - ''' Start of program ''' - -if len(sys.argv) > 1: CONFIG = sys.argv[1] -else: - print "\n** Usage: " + sys.argv[0] + " [hours]\n" - sys.exit(1) - -if len(sys.argv) > 2: - HOURS = float(sys.argv[2]) - SECONDS = HOURS * 60.0 * 60.0 -else: - SECONDS = None - -# open config file -configDoc = ElementTree(file=CONFIG) - -# set up logging -logging.setLoggerClass(PSLogger) -logger = logging.getLogger(sys.argv[0]) -logger.setup(configDoc, sys.argv[0]) - -ippToPspsDb = IppToPspsDb(logger, configDoc) -datastore = Datastore(logger, configDoc, ippToPspsDb) -dxlayer = DXLayer(logger) - -DVOLABEL = configDoc.find("dvo/gpc1Label").text -EPOCH = configDoc.find("options/epoch").text -BASEPATH = configDoc.find("localOutPath").text -DELETELOCAL = int(configDoc.find("deletion/local").text) -DELETEDATASTORE = int(configDoc.find("deletion/datastore").text) -DELETEDXLAYER = int(configDoc.find("deletion/dxlayer").text) - -logger.infoTitle("ippToPsps clean-up") -if SECONDS: logger.infoPair("Performing clean-up every", "%f hours" % (SECONDS/60.0/60.0)) -logger.infoBool("Deleting from local disk?", DELETELOCAL) -logger.infoBool("Deleting from datastore?", DELETEDATASTORE) -logger.infoBool("Deleting from DXLayer?", DELETEDXLAYER) - -while True: - - clean("P2") - clean("ST") - - if SECONDS: - logger.infoPair("Waiting for", "%f hours" % HOURS) - time.sleep(SECONDS) - else: break - +cleanup = Cleanup(sys.argv) +cleanup.run() +cleanup.exitProgram("finished") Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/config.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/config.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/config.py (revision 33638) @@ -0,0 +1,135 @@ +#!/usr/bin/env jython + +import logging +from xml.etree.ElementTree import ElementTree, Element, tostring + +from pslogger import PSLogger + + +''' +A class encapsulating a ippToPsps configuration. This information is stored in the 'config' table +of the ipptopsps database, but there are some higher level config details in the 'settings.xml' file +''' +class Config(object): + + + ''' + Constructor + + Basically reads the entire config and stores values to class variables + ''' + def __init__(self, programName, name): + + self.programName = programName + self.name = name + self.settingsPath = "../config/settings.xml" # TODO + self.logger = None + + self.settingsDoc = ElementTree(file=self.settingsPath) + self.logPath = self.settingsDoc.find("logPath").text + self.czarPlotsPath = self.settingsDoc.find("czarPlotsPath").text + + # this is the border (in degrees) that we place around any loading box of + # PS1 pointings to ensure we pull a large enough area out of DVO + self.BORDER = 1.65 + self.isLoaded = False + + ''' + Prints everything for this config + ''' + def printAll(self): + + try: + self.logger.infoSeparator() + self.logger.infoPair("Config name", self.name) + self.logger.infoPair("Survey", self.survey) + self.logger.infoPair("Publishing to PSPS as survey", self.pspsSurvey) + self.logger.infoPair("Loading epoch", self.epoch) + self.logger.infoPair("Data release", "%d" % self.dataRelease) + for batchType in self.batchTypes: self.logger.infoPair("Queuing batch type", batchType) + except: + pass + + self.printDvoInfo() + self.printDatastoreInfo() + self.printBoxCoords() + self.printDeletionPolicy() + try: self.logger.infoSeparator() + except: pass + + ''' + Queuing this batch type? + ''' + def queuingThisBatchType(self, batchType): + if batchType in self.batchTypes: return 1 + return 0 + + ''' + Creates a logger object and returns it + ''' + def getLogger(self, host, pid, stdout=1, sendToFile=0): + + logging.setLoggerClass(PSLogger) + self.logger = logging.getLogger(self.programName) + self.logger.setup(self.programName, self.logPath, self.name, host, pid, stdout, sendToFile) + + return self.logger + + ''' + Prints the currently set DVO info + ''' + def printDvoInfo(self): + + try: + self.logger.infoPair("DVO label", self.dvoLabel) + self.logger.infoPair("DVO location", self.dvoLocation) + except: + pass + + ''' + Prints datastore info + ''' + def printDatastoreInfo(self): + + try: + self.logger.infoBool("Datastore publishing?", self.datastorePublishing) + if self.datastorePublishing: + self.logger.infoPair("Datastore type", self.datastoreProduct) + self.logger.infoPair("Datastore product", self.datastoreType) + except: + pass + + ''' + Prints the current deletion policy + ''' + def printDeletionPolicy(self): + + try: + self.logger.infoBool("Deleting local?", self.deleteLocal) + self.logger.infoBool("Deleting datastore?", self.deleteDatastore) + self.logger.infoBool("Deleting DXLayer?", self.deleteDxLayer) + except: + pass + + ''' + Prints the currently set RA/Dec bounding box + ''' + def printBoxCoords(self): + + try: + self.logger.infoPair("RA limits", "%.1f -> %.1f" % (self.minRa, self.maxRa)) + self.logger.infoPair("Dec limits", "%.1f -> %.1f" % (self.minDec, self.maxDec)) + self.logger.infoPair("Loading box size", "%.1f degrees" % self.boxSize) + except: + pass + + + ''' + Various db metadata getters + ''' + def getDbName(self, dbType): return self.settingsDoc.find(dbType +"/name").text + def getDbHost(self, dbType): return self.settingsDoc.find(dbType +"/host").text + def getDbUser(self, dbType): return self.settingsDoc.find(dbType +"/user").text + def getDbPassword(self, dbType): return self.settingsDoc.find(dbType +"/password").text + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/lap20110809.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/lap20110809.xml (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/lap20110809.xml (revision 33638) @@ -59,5 +59,5 @@ - 0 + 1 1 1 @@ -69,5 +69,5 @@ 3PI 1 - 1 + 0 0 0 Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/lap20110809_2.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/lap20110809_2.xml (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/lap20110809_2.xml (revision 33638) @@ -0,0 +1,82 @@ + + + + + + + + + ipptopsps_scratch2 + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + LAP.ThreePi.20110809 + /data/ipp005.0/gpc1/catdirs/LAP.ThreePi.20110809 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-08-16 + 3PI + 3PI + 1 + 0 + 0 + 0 + 1 + 0 + + + + + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/lap20110809_3.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/lap20110809_3.xml (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/lap20110809_3.xml (revision 33638) @@ -0,0 +1,82 @@ + + + + + + + + + ipptopsps_scratch3 + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + LAP.ThreePi.20110809 + /data/ipp005.0/gpc1/catdirs/LAP.ThreePi.20110809 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-08-16 + 3PI + 3PI + 1 + 0 + 0 + 0 + 1 + 0 + + + + + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/newmd4.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/newmd4.xml (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/newmd4.xml (revision 33638) @@ -0,0 +1,90 @@ + + + + + + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + czardb + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 1 + MD04.20111128 + /data/ipp005.0/gpc1/catdirs/MD04.20111128 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 1 + 1 + 1 + + + + 2011-12-03 + MD04 + MDF + 1 + 1 + 0 + 0 + 1 + 0 + + + + + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/newsas.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/newsas.xml (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/newsas.xml (revision 33638) @@ -46,6 +46,6 @@ 0 - SAS.20111125 - /data/ipp005.0/gpc1/catdirs/SAS.20111125 + SAS.20111201 + /data/ipp005.0/gpc1/catdirs/SAS.20111201 @@ -59,5 +59,5 @@ - 0 + 1 1 1 Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi.xml (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi.xml (revision 33638) @@ -48,8 +48,8 @@ ThreePi.V3 /data/ipp005.0/gpc1/catdirs/ThreePi.V3 - 105 + 90 270 - -100 - 100 + -30 + 90 Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi2.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi2.xml (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi2.xml (revision 33638) @@ -0,0 +1,86 @@ + + + + + + + + + ipptopsps_scratch2 + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + ThreePi.V3 + /data/ipp005.0/gpc1/catdirs/ThreePi.V3 + 90 + 270 + -100 + 100 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-10-27 + 3PI + OLD + 1 + 0 + 0 + 0 + 1 + 0 + + + + + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi3.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi3.xml (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepi3.xml (revision 33638) @@ -0,0 +1,86 @@ + + + + + + + + + ipptopsps_scratch3 + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + ThreePi.V3 + /data/ipp005.0/gpc1/catdirs/ThreePi.V3 + 90 + 270 + -100 + 100 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-10-27 + 3PI + OLD + 1 + 0 + 0 + 0 + 1 + 0 + + + + + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepiTest.xml =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepiTest.xml (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/configs/oldthreepiTest.xml (revision 33638) @@ -0,0 +1,95 @@ + + + + + + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps_test + ippdb01 + ipp + ipp + + + + + czardb + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + ThreePi.V3 + /data/ipp005.0/gpc1/catdirs/ThreePi.V3 + 90 + 270 + -30 + 90 + + + + /data/ipp005.0/rhenders/test + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-10-27 + 3PI + OLD + 1 + 0 + 1 + 0 + 0 + 0 + + + 72203 + + + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/czardb.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/czardb.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/czardb.py (revision 33638) @@ -17,6 +17,6 @@ Constructor ''' - def __init__(self, logger, doc): - super(CzarDb, self).__init__(logger, doc, "czardatabase") + def __init__(self, logger, config): + super(CzarDb, self).__init__(logger, config, "czardatabase") Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastore.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastore.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastore.py (revision 33638) @@ -19,17 +19,10 @@ ''' - def __init__(self, logger, doc, ippToPspsDb): + def __init__(self, logger, config, ippToPspsDb): # setup logger self.logger = logger - self.doc = doc + self.config = config self.ippToPspsDb = ippToPspsDb - self.logger.debug("Datastore constructor") - - # open config - self.product = doc.find("datastore/product").text - self.type = doc.find("datastore/type").text - - self.logger.debug("Using product: '" + self.product + "' and type: '" + self.type + "'") ''' @@ -49,6 +42,6 @@ command = "dsreg --add " + name + "\ --link --datapath " + path + "\ - --type " + self.type + "\ - --product " + self.product + "\ + --type " + self.config.datastoreType + "\ + --product " + self.config.datastoreProduct + "\ --list " + tempFile.name @@ -82,5 +75,5 @@ --del " + name + " \ --rm \ - --product " + self.product + --product " + self.config.datastoreProduct p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE) Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastoreRemover.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastoreRemover.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/datastoreRemover.py (revision 33638) @@ -0,0 +1,51 @@ +#!/usr/bin/env jython + +from datastore import Datastore +import logging +import sys +import getopt + +from ipptopsps import IppToPsps +from datastore import Datastore + +''' +DatastoreRemover class +''' +class DatastoreRemover(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(DatastoreRemover, self).__init__(argv) + + if len(sys.argv) < 3: + self.printUsage() + self.exitProgram("incorrect args") + + self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + + ''' + Runs... + ''' + def run(self): + + if len(sys.argv) < 4: + self.datastore.remove(str(sys.argv[2])) + else: + self.datastore.removeRange(str(sys.argv[2]), str(sys.argv[3])) + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(DatastoreRemover, self).printUsage(" []") + + +''' +Start of program +''' +datastoreRemover = DatastoreRemover(sys.argv) +datastoreRemover.run() +datastoreRemover.exitProgram("finished") + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/detectionbatch.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/detectionbatch.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/detectionbatch.py (revision 33638) @@ -33,21 +33,23 @@ def __init__(self, logger, - configPath, - configDoc, + config, gpc1Db, ippToPspsDb, + scratchDb, camID, - batchID): + batchID, + useFullTables): super(DetectionBatch, self).__init__( logger, - configPath, - configDoc, + config, gpc1Db, ippToPspsDb, + scratchDb, camID, batchID, "P2", - gpc1Db.getCameraStageSmf(camID)) + gpc1Db.getCameraStageSmf(camID), + useFullTables) if not self.everythingOK: return @@ -73,5 +75,5 @@ # if test mode - if self.testMode: + if self.config.test: self.startX = 3 self.endX = 4 @@ -219,5 +221,5 @@ self.scratchDb.updateFilterID("FrameMeta", self.filter) self.scratchDb.updateAllRows("FrameMeta", "calibModNum", str(self.calibModNum)) - self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.config.dataRelease)) ''' @@ -322,5 +324,5 @@ ," + self.safeDictionaryAccess(header, 'DAPMIFIT') + " \ ,'" + self.safeDictionaryAccess(header, 'DETECTOR') + "' \ - ," + str(self.scratchDb.getDvoImageFlags(header['SOURCEID'], header['IMAGEID'])) + " \ + ," + str(self.scratchDb.getDvoImageFlags(header['IMAGEID'])) + " \ ,'" + self.safeDictionaryAccess(header, 'DETREND.MASK') + "' \ ,'" + self.safeDictionaryAccess(header, 'DETREND.DARK') + "' \ @@ -366,5 +368,5 @@ self.scratchDb.updateFilterID(tableName, self.filter) self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO']) self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0") @@ -416,6 +418,4 @@ ,momentYY \ ,apMag \ - ,kronFlux \ - ,kronFluxErr \ ,infoFlag \ ,sky \ @@ -448,6 +448,4 @@ ,MOMENTS_YY \ ,AP_MAG \ - ,KRON_FLUX \ - ,KRON_FLUX_ERR \ ,FLAGS\ ,SKY \ @@ -457,5 +455,5 @@ , '" + self.dateStr + "' \ , 0 \ - , " + str(self.dataRelease) + "\ + , " + str(self.config.dataRelease) + "\ FROM " + ippTableName self.scratchDb.execute(sql) @@ -501,5 +499,5 @@ self.scratchDb.execute(sql) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) ''' @@ -530,5 +528,5 @@ self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) @@ -545,6 +543,5 @@ self.populateFrameMeta() - # dictionary objects to hold sourceIDs and imageIDs for later - self.sourceIDs = {} + # dictionary objects to hold imageIDs for later self.imageIDs = {} @@ -578,5 +575,4 @@ # store these for later - self.sourceIDs[ota] = header['SOURCEID'] self.imageIDs[ota] = header['IMAGEID'] @@ -614,7 +610,7 @@ Updates provided table with DVO IDs from DVO table ''' - def updateDvoIDs(self, table, sourceID, externID): - - imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID) + def updateDvoIDs(self, table, externID): + + imageID = self.scratchDb.getImageIDFromExternID(externID) self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID) sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ @@ -624,5 +620,4 @@ a.infoFlag = b.flags << 32 | a.infoFlag \ WHERE a.ippDetectID = b.ippDetectID \ - AND b.sourceID = " + str(sourceID) + " \ AND b.imageID = " + str(imageID) @@ -655,5 +650,5 @@ ota = "XY%d%d" % (x, y) - if ota not in self.sourceIDs: continue + if ota not in self.imageIDs: continue #self.logger.infoTitle("Processing " + ota) @@ -666,5 +661,5 @@ # now add DVO IDs - self.updateDvoIDs("Detection_" + ota, self.sourceIDs[ota], self.imageIDs[ota]) + self.updateDvoIDs("Detection_" + ota, self.imageIDs[ota]) results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID") self.updateImageID("Detection_" + ota, x, y) @@ -693,5 +688,5 @@ # update ImageMeta with count of detections for this OTA and photoCodeID sql = "UPDATE ImageMeta_" + ota + " \ - SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.sourceIDs[ota], self.imageIDs[ota])) + SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.imageIDs[ota])) self.scratchDb.execute(sql) @@ -757,8 +752,8 @@ def importIppTables(self, filter=""): - if self.testMode: regex = "XY33.psf" + if self.config.test: regex = "XY33.psf" else : regex = ".*.psf" - columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS SKY SKY_SIGMA EXT_NSIGMA" + columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG FLAGS SKY SKY_SIGMA EXT_NSIGMA" return super(DetectionBatch, self).importIppTables(columns, regex) Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/dvo.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/dvo.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/dvo.py (revision 33638) @@ -0,0 +1,429 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob +from subprocess import call, PIPE, Popen + +from pslogger import PSLogger +from scratchdb import ScratchDb + +import time +from java.lang import * +from java.sql import * + + +''' +A class that encapsulates a DVO database + +Pulls relavant bits and pieces into the scratch MySQL database and then provides access queries +''' +class Dvo(object): + + ''' + Constructor + + - sets up logging + - checks we are using the correct DVO + - checks we have up-to-date Images.dat + - checks we have up-to-date SkyTable.fits + + ''' + def __init__(self, logger, config, scratchDbName): + + # set up logging + self.logger = logger + self.config = config + + # set up empty lists + self.regionsToIngest = [] + self.regionsToPurge = [] + self.allPopulatedRegions = [] + self.regionsAlreadyIngested = [] + self.regionsIngestedButOutOfDate = [] + + # connect to the specified scratch database + self.scratchDb = ScratchDb(self.logger, self.config, scratchDbName) + self.scratchDb.setUseFullTables(True) + + # decide if we are using the right DVO for this scratchDb + self.correctDvo = self.scratchDb.isCorrectDvo(self.config.dvoLocation) + + # this scratch Db either has no DVO info ingested, or a different DVO ingested. Prompt user + if not self.correctDvo: + print "*******************************************************************************" + self.logger.warning("Wrong or no DVO in use. Do you want to reset and use '" + self.config.dvoLocation + "'?") + response = raw_input("(y/n) ") + if response == "y": + self.logger.warning("Are you ABSOLUTELY sure you want to do this?") + response = raw_input("(y/n) ") + if response == "y": + self.resetAllTables() + + + ''' + Destructor + ''' + def __del__(self): + self.logger.debug("Dvo destructor") + + ''' + Check we are importing the DVO currenly ingested + ''' + def isThisDvoCurrentlyIngested(self): + return self.correctDvo + + ''' + Resets all dvo tables. Be very careful.... + ''' + def resetAllTables(self): + + self.logger.infoPair("Resetting", "all tables") + if not self.scratchDb.resetAllDvoTables(): return + self.correctDvo = True + if not self.loadSkyTable(): + self.correctDvo = False + return + + + ''' + Loads the DVO Images.dat FITS file + ''' + def loadImages(self): + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: return + + # check if we have up-to-date version + path = self.config.dvoLocation + "/Images.dat" + if self.scratchDb.alreadyImportedThisDvoTable(path): + self.logger.debugPair("DVO Images.dat file", "up-to-date") + return + + self.logger.infoSeparator() + self.logger.infoPair("DVO Images.dat file", "NOT up-to-date") + + self.importFits( + path, + "SOURCE_ID IMAGE_ID EXTERN_ID FLAGS PHOTCODE", + self.scratchDb.dvoImagesTable) + self.logger.infoPair("Adding primary key to", self.scratchDb.dvoImagesTable) + self.scratchDb.execute("ALTER TABLE " + self.scratchDb.dvoImagesTable + " ADD PRIMARY KEY (IMAGE_ID, EXTERN_ID)") + + self.scratchDb.setImportedThisDvoTable(path) + + ''' + Loads the DVO SkyTable into MySQL and then applies an index + ''' + def loadSkyTable(self): + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: return False + + path = self.config.dvoLocation + "/SkyTable.fits" + if self.scratchDb.alreadyImportedThisDvoTable(path): + self.logger.debugPair("DVO SkyTable.fits file", "up-to-date") + return False + + self.logger.infoSeparator() + self.logger.infoPair("DVO SkyTable.fits file", "NOT up-to-date") + self.importFits( + path, + "R_MIN R_MAX D_MIN D_MAX INDEX NAME", + self.scratchDb.dvoSkyTable) + + self.logger.infoPair("Adding index to", self.scratchDb.dvoSkyTable) + + self.scratchDb.createIndex(self.scratchDb.dvoSkyTable, "INDEX") + + self.scratchDb.setImportedThisDvoTable(path) + self.logger.infoPair("Finished importing SkyTable at", self.config.dvoLocation) + + return True + + ''' + Sets a bunch of region files from DVO for a stripe starting at this RA and with this width + ''' + def setSkyAreaAsStripe(self, ra=0., width=4.): + + self.setSkyArea(ra, ra+width, -91., 91.) + + ''' + Sets a bunch of region files from DVO for a RA/Dec box centered with these coords and with this side length + ''' + def setSkyAreaAsBox(self, ra=180., dec=0., side=181.0): + + halfSide = side/2.0 + self.setSkyArea(ra-halfSide, ra+halfSide, dec-halfSide, dec+halfSide) + + ''' + Determines what DVO files need to be ingested to the defines region + ''' + def setSkyArea(self, minRa=-1., maxRa=361., minDec=-91., maxDec=91): + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: return + + if (maxRa - minRa <= 0) or (maxDec - minDec) <= 0: + self.logger.errorPair("Zero range in either RA or Dec", + "RA: %.2f -> %.2f, Dec: %.2f -> %.2f" % (minRa, maxRa, minDec, maxDec)) + return False + + # update the two meta tables, Images and SkyTable + self.loadSkyTable() + + # set bounding-box coords + self.minRa = minRa + self.maxRa = maxRa + self.minDec = minDec + self.maxDec = maxDec + + # reset all lists + allRegions = self.scratchDb.getDvoRegions(minRa, maxRa, minDec, maxDec) + allIngestedRegions = self.scratchDb.getIngestedDvoRegions() + self.regionsToIngest = [] + self.regionsToPurge = [] + self.allPopulatedRegions = [] + self.regionsAlreadyIngested = [] + self.regionsIngestedButOutOfDate = [] + + # loop through all regions in DVO inside this bounding-box + for region in allRegions: + + # get paths for this region for the file types we are interested in + paths = [] + for fileType in self.ingestFileTypes: + paths.append(self.config.dvoLocation + "/" + region + "." + fileType) + + # check for the existence of all interested file types + skipThisRegion = False + for path in paths: + if not os.path.isfile(path): + skipThisRegion = True + break + + if skipThisRegion: continue + + # ok, have real, populated files so add this region to the list + self.allPopulatedRegions.append(region) + + # if we have already imported up-to-date versions of all interested files, then we skip this region + alreadyIngested = True + for path in paths: + if not self.scratchDb.alreadyImportedThisDvoTable(path): + alreadyIngested = False + break + + if alreadyIngested: + self.regionsAlreadyIngested.append(region) + continue + + # check if we have out-of-date versions of any interested files, if so, add region to the purge list + outOfDate = False + for path in paths: + if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(path): + outOfDate = True + break + + if outOfDate: self.regionsIngestedButOutOfDate.append(region) + + self.regionsToIngest.append(region) + + # add out-of-range regions for purge list + items = list(set(allIngestedRegions) - set(self.allPopulatedRegions)) + for item in items: self.regionsToPurge.append(item) + + # now add out-of-date items to purge list + for item in self.regionsIngestedButOutOfDate: self.regionsToPurge.append(item) + + self.sizeOfAllPopulatedRegions = self.getDiskSizeOfDvoRegions(self.allPopulatedRegions) + self.sizeOfRegionsAlreadyIngested = self.getDiskSizeOfDvoRegions(self.regionsAlreadyIngested) + self.sizeOfRegionsIngestedButOutOfDate = self.getDiskSizeOfDvoRegions(self.regionsIngestedButOutOfDate) + self.sizeOfRegionsToPurge = self.getDiskSizeOfDvoRegions(self.regionsToPurge) + self.sizeOfRegionsToIngest = self.getDiskSizeOfDvoRegions(self.regionsToIngest) + + ''' + Prints out a summary of the currently set regions + ''' + def printSummary(self): + + self.logger.infoTitle("Summary for currently set DVO region") + self.logger.infoPair("DVO ingest type", self.__class__.__name__) + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: + self.logger.infoPair("Wrong DVO in use", self.config.dvoLocation) + return + + for fileType in self.ingestFileTypes: + self.logger.infoPair("Ingesting files of type", fileType) + + self.logger.infoPair("RA range", "%.2f -> %.2f" % (self.minRa, self.maxRa)) + self.logger.infoPair("Dec range", "%.2f -> %.2f" % (self.minDec, self.maxDec)) + + self.logger.info("+----------------+----------------+----------------+----------------+----------------+") + self.logger.info("| populated | ingested | out-of-date | to-purge | to ingest |") + self.logger.info("+----------------+----------------+----------------+----------------+----------------+") + self.logger.info("| %5d (%6s) | %5d (%6s) | %5d (%6s) | %5d (%6s) | %5d (%6s) |" % ( + len(self.allPopulatedRegions), self.getFileSizeStr(self.sizeOfAllPopulatedRegions), + len(self.regionsAlreadyIngested), self.getFileSizeStr(self.sizeOfRegionsAlreadyIngested), + len(self.regionsIngestedButOutOfDate), self.getFileSizeStr(self.sizeOfRegionsIngestedButOutOfDate), + len(self.regionsToPurge), self.getFileSizeStr(self.sizeOfRegionsToPurge), + len(self.regionsToIngest), self.getFileSizeStr(self.sizeOfRegionsToIngest))) + self.logger.info("+----------------+----------------+----------------+----------------+----------------+") + + ''' + Returns the size on disk of the regions that need to be ingested in GB + ''' + def getDiskSizeOfRegionsToBeIngested(self): + + try: + return self.sizeOfRegionsToIngest/1073741824.0 + except: + return 0.0 + + ''' + Purges the provided regions from the database + ''' + def purgeRegions(self): + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: return False + + if len(self.regionsToPurge) < 1: return False + + self.logger.infoPair("Purging", "%d regions" % len(self.regionsToPurge)) + + detectionDeleteCount = self.scratchDb.purgeTheseDvoRegions(self.regionsToPurge, self.ingestFileTypes) + if detectionDeleteCount > 0: self.logger.infoPair("Deleted", "%d detections" % detectionDeleteCount) + + return True + + ''' + Formats a nice file size string + Accepts a size in bytes + ''' + def getFileSizeStr(self, byteSize): + + # format a string for the user -smaller than one MB + if byteSize < 1048576: return "%.1f K" % (byteSize/1024.0) + # smaller than one GB + elif byteSize < 1073741824: return "%.1fM" % (byteSize/1048576.0) + # more than a GB + else: return "%.1fG" % (byteSize/1073741824.0) + + ''' + Returns the disk size of this file + ''' + def getDiskSize(self, path): + + return os.stat(path).st_size + + ''' + Returns the disk size of these DVO regions, i.e. the size on disk of all relevant DVO FITS files + ''' + def getDiskSizeOfDvoRegions(self, regions): + + size = 0.0 + for region in regions: + + # get combined size of all interested files + for fileType in self.ingestFileTypes: + size = size + self.getDiskSize(self.config.dvoLocation + "/" + region + "." + fileType) + + return size + + ''' + Abstract method for ingesting a particular reegion. Must be implemented in all subclasses + ''' + def ingestRegion(self, region): + self.logger.errorPair("ingestRegion()", "not implemented for this class") + + + ''' + Synchronises MySQL database with the sky area currently set from DVO + Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this + includes purging detections outside sky area + ''' + def sync(self): + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: return False + + # check we have something to do + if len(self.regionsToPurge) < 1 and len(self.regionsToIngest) < 1: + self.logger.infoPair("Nothing to ingest or purge", "finished ingest") + return True + + # make sure we have an up-to-date Images table + self.loadImages() + + # purge stuff + self.purgeRegions() + + totalRegions = len(self.regionsToIngest) + if totalRegions > 0: self.logger.infoSeparator() + + # now loop through all regions and ingest to MySQL + ingestedRegionsCount = 0 + regionCount = 1 + for region in self.regionsToIngest: + + self.logger.infoTitle("Region %d of %d: %s" % (regionCount, totalRegions, region)) + if self.ingestRegion(region): ingestedRegionsCount += 1 + regionCount += 1 + + self.logger.infoSeparator() + self.logger.infoPair("Ingested", "%d DVO regions (out of %d)" % (ingestedRegionsCount, len(self.regionsToIngest))) + + # update lists after attempted sync + self.setSkyArea(self.minRa, self.maxRa, self.minDec, self.maxDec) + self.printSummary() + + return True + + ''' + Imports a FITS file. A regex filter lets you choose which tables to pull from the file + An optional final argument lets you name the resulatant database table + ''' + def importFits(self, path, columns, tableName): + + self.logger.infoPair("Importing file", "%s (%s)" % (path, self.getFileSizeStr(self.getDiskSize(path)))) + self.logger.debugPair("Writing to database table", tableName) + + tables = stilts.treads(path) + + count = 0 + for table in tables: + + self.logger.debugPair("Reading IPP table", table.name) + table = stilts.tpipe(table, cmd='explodeall') + + # IPP FITS files are littered with infinities, so remove these + self.logger.debugPair("Removing", "infinity values") + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + + attempts = 0 + self.logger.debugPair("Writing table to database", table.name) + while True: + + try: + if attempts > 0: self.logger.infoPair("Attempt %d to write" % attempts, table.name) + table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName) + break + except: + # wait a few secs then try again + time.sleep(2) + attempts = attempts + 1 + + count = count + 1 + + self.logger.debugPair("Finished importing", tableName) + + return tableName + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/dvodetections.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/dvodetections.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/dvodetections.py (revision 33638) @@ -0,0 +1,118 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob + +from dvo import Dvo + + +''' +A class for ingesting DVO detections into MySQL +''' +class DvoDetections(Dvo): + + ''' + Constructor + ''' + def __init__(self, logger, config, scratchDbName): + + super(DvoDetections, self).__init__(logger, config, scratchDbName) + self.ingestFileTypes = ['cpm','cpt'] + + ''' + Ingests detections for this region + ''' + def ingestRegion(self, region): + + cpmPath = self.config.dvoLocation + "/" + region + ".cpm" + cptPath = self.config.dvoLocation + "/" + region + ".cpt" + + cpmTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpm") + cptTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpt") + + # if we have not already ingested the cpm table, then do it now + if not self.scratchDb.alreadyImportedThisDvoTable(cpmPath): + + # import cpm table and index + self.importFits( + cpmPath, + "IMAGE_ID DET_ID OBJ_ID CAT_ID EXT_ID DB_FLAGS", + cpmTableName) + + self.scratchDb.createIndex(cpmTableName, "IMAGE_ID") + self.scratchDb.createIndex(cpmTableName, "CAT_ID") + self.scratchDb.createIndex(cpmTableName, "OBJ_ID") + + # we can now report that we have imported this table + self.scratchDb.setImportedThisDvoTable(cpmPath, region) + + # get the id for this cpm file so we can insert it in the dvoDetection table + fileId = self.scratchDb.getDvoFileId(cpmPath) + + # import cpt table and index + self.importFits( + cptPath, + "OBJ_ID CAT_ID EXT_ID", + cptTableName) + + self.scratchDb.createIndex(cptTableName, "IMAGE_ID") + self.scratchDb.createIndex(cptTableName, "CAT_ID") + self.scratchDb.createIndex(cptTableName, "OBJ_ID") + + self.logger.infoPair("Adding","PSPS objIDs") + # first try to add a column for PSPS_OBJ_ID. catch failure and continue + try: + sql = "ALTER TABLE " + cpmTableName + " ADD COLUMN (PSPS_OBJ_ID BIGINT)" + self.scratchDb.execute(sql) + except: + self.logger.infoPair("Already added PSPS_OBJ_ID column to", cpmTableName) + + # shove PSPS objIDs from cpt table into measurement table + sql = "UPDATE " + cpmTableName + " AS a, " + cptTableName + " AS b \ + SET a.PSPS_OBJ_ID = b.EXT_ID \ + WHERE a.CAT_ID = b.CAT_ID \ + AND a.OBJ_ID = b.OBJ_ID" + self.scratchDb.execute(sql) + + # NB we use an INSERT IGNORE here. This is because of a known issue where multiple DVO cpm + # files can include the same detection, with the same IMAGE_ID/DET_ID pairing, but different + # PSPS object IDs assigned in the corresponding cpt file. This is believed to be a chip-boundary + # issue within DVO. So, for now, we take the first IMAGE_ID/DET_ID detection we find, and ignore the rest + self.logger.infoPair("Merging into", self.scratchDb.dvoDetectionTable) + sql = "INSERT IGNORE INTO " + self.scratchDb.dvoDetectionTable + " (\ + imageID \ + ,ippDetectID \ + ,detectID \ + ,ippObjID \ + ,objID \ + ,flags \ + ,fileID \ + ) SELECT \ + IMAGE_ID \ + ,DET_ID \ + ,EXT_ID \ + ,CAT_ID * 1000000000 + OBJ_ID \ + ,PSPS_OBJ_ID \ + ,DB_FLAGS \ + , " + str(fileId) + " \ + FROM " + cpmTableName + try: + self.scratchDb.execute(sql) + except: + self.logger.error("FAILED: " + sql) + return False + finally: + self.logger.infoPair("Dropping tables", "%s and %s" % (cpmTableName, cptTableName)) + self.scratchDb.dropTable(cpmTableName) + self.scratchDb.dropTable(cptTableName) + + self.scratchDb.setImportedThisDvoTable(cptPath, region) + + return True + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/dvoobjects.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/dvoobjects.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/dvoobjects.py (revision 33638) @@ -0,0 +1,51 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob + +from dvo import Dvo + + +''' +A class for ingesting DVO detections into MySQL +''' +class DvoObjects(Dvo): + + ''' + Constructor + ''' + def __init__(self, logger, config, scratchDbName): + + super(DvoObjects, self).__init__(logger, config, scratchDbName) + self.ingestFileTypes = ['cps','cpt'] + + ''' + Ingests detections for this region + ''' + def ingestRegion(self, region): + + cpsPath = self.config.dvoLocation + "/" + region + ".cps" + cptPath = self.config.dvoLocation + "/" + region + ".cpt" + + cpsTableName = self.scratchDb.getDbFriendlyTableName(region + ".cps") + cptTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpt") + + self.importFits(cpsPath, "*", cpsTableName) + self.importFits(cptPath, "*", cptTableName) + + # we can now report that we have imported this table + self.scratchDb.setImportedThisDvoTable(cpsPath, region) + self.scratchDb.setImportedThisDvoTable(cptPath, region) + + #self.logger.infoPair("Dropping tables", "%s and %s" % (cpsTableName, cptTableName)) + #self.scratchDb.dropTable(cpsTableName) + #self.scratchDb.dropTable(cptTableName) + + return True + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/dxlayer.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/dxlayer.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/dxlayer.py (revision 33638) @@ -34,5 +34,5 @@ stdout=PIPE) p.wait() - + if p.returncode != 0: self.logger.debugPair("DXLayer webservice", "failed") @@ -58,4 +58,5 @@ doc = ElementTree(file=dxLayerResponseFile) + ret = False try: MESSAGE = doc.find("Rows/Row/Value").text Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/fits.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/fits.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/fits.py (revision 33638) @@ -15,12 +15,11 @@ ''' - def __init__(self, logger, doc, originalPath): + def __init__(self, logger, config, originalPath): # set class variables self.originalPath = originalPath self.logger = logger - self.doc = doc + self.config = config self.header = None - self.localDir = self.doc.find("localOutPath").text # does this file even exist? @@ -31,5 +30,5 @@ # ok, we have a file, now copy it locally to save on NFS overhead self.logger.debugPair("FITS file", self.originalPath) - #self.localCopyPath = self.localDir + "/temp.fits" + #self.localCopyPath = self.config.basePath + "/temp.fits" #shutil.copy2(self.originalPath, self.localCopyPath) Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/gpc1db.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/gpc1db.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/gpc1db.py (revision 33638) @@ -20,6 +20,6 @@ Constructor ''' - def __init__(self, logger, doc): - super(Gpc1Db, self).__init__(logger, doc, "gpc1database") + def __init__(self, logger, config): + super(Gpc1Db, self).__init__(logger, config, "gpc1database") ''' @@ -32,12 +32,26 @@ ''' - Gets a list of ids in this DVO database for this stage, could be cam or staticsky, and a box can be defined with equatorial coordinates - ''' - def getIDsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999): - - if batchType == "P2": # TODO define these someplace + Gets a list of ids, ra and dec in this DVO database for this stage, could be cam or staticsky, for a box with defined center and side length + ''' + def getItemsInThisDVODbForThisStageInThisBox(self, dvoDb, batchType, ra=180., dec=0., side=181.): + + halfSide = side/2.0 + return self.getItemsInThisDVODbForThisStage(dvoDb, batchType, ra-halfSide, ra+halfSide, dec-halfSide, dec+halfSide) + + ''' + Gets a list of ids, ra and dec in this DVO database for this stage, could be cam or staticsky, and a box can be defined with equatorial coordinates + NB this uses Bill's potentially temporary 'skycells' database when finding RA/Dec for stacks. + ''' + def getItemsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999): + + self.logger.debugPair("GPC1 RA range", "%.2f -> %.2f" % (minRA, maxRA)) + self.logger.debugPair("GPC1 Dec range", "%.2f -> %.2f" % (minDec, maxDec)) + + rows = [] + + if batchType == "P2": # TODO define these some place stage = "cam" - sql = "SELECT DISTINCT stage_id \ + sql = "SELECT DISTINCT stage_id, DEGREES(ra), DEGREES(decl) \ FROM addRun \ JOIN minidvodbRun USING(minidvodb_name) \ @@ -46,4 +60,22 @@ JOIN chipRun USING (chip_id) \ JOIN rawExp USING (exp_id) \ + JOIN mergedvodbRun using (minidvodb_id) \ + WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \ + AND minidvodbRun.state = 'merged' \ + AND mergedvodbRun.state = 'full' \ + AND minidvodbProcessed.fault = 0 \ + AND addRun.stage = '" + stage + "' \ + AND addRun.state = 'full' \ + AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \ + AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ")" + + elif batchType == "ST": + + stage = "staticsky_multi" + sql = "SELECT DISTINCT stack_id, ra, decl FROM staticskyInput \ + JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \ + JOIN minidvodbRun USING(minidvodb_name) \ + JOIN minidvodbProcessed USING(minidvodb_id) \ + JOIN stackRun USING(stack_id) JOIN skycells.skycell USING(skycell_id) \ WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \ AND minidvodbRun.state = 'merged' \ @@ -51,34 +83,19 @@ AND addRun.stage = '" + stage + "' \ AND addRun.state = 'full' \ - AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \ - AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ")" - - elif batchType == "ST": - - stage = "staticsky" - sql = "SELECT DISTINCT stack_id FROM staticskyInput \ - JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \ - JOIN minidvodbRun USING(minidvodb_name) \ - JOIN minidvodbProcessed USING(minidvodb_id) \ - WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \ - AND minidvodbRun.state = 'merged' \ - AND minidvodbProcessed.fault = 0 \ - AND addRun.stage = '" + stage + "' \ - AND addRun.state = 'full'" - - try: - rs = self.executeQuery(sql) + AND decl BETWEEN " + str(minDec) + " AND " + str(maxDec) + " \ + AND ra BETWEEN " + str(minRA) + " AND " + str(maxRA) + + try: + rs = self.executeQuery(sql) + while (rs.next()): + rows.append([rs.getInt(1), rs.getFloat(2), rs.getFloat(3)]) + rs.close() except: self.logger.exception("Can't query for ids in DVO") - - ids = [] - while (rs.next()): - ids.append(rs.getInt(1)) - - rs.close() - - self.logger.debug("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage)) - - return ids + return rows + + self.logger.debug("Found %d items in DVO database '%s' for stage='%s'" % (len(rows), dvoDb, stage)) + + return rows ''' @@ -193,7 +210,6 @@ try: - # get path to base dir of cmf files + # get path to base dir of cmf files path = rs.getString(1) - path = path[0:path.rfind("/")] except: self.logger.errorPair("No smf files found for cam_id", str(camID) ) @@ -204,5 +220,5 @@ if path.startswith("neb"): - f=os.popen("neb-ls -p "+path+"/%smf") + f=os.popen("neb-ls -p "+path+"%smf") for i in f.readlines(): files.append(i.rstrip()) @@ -210,24 +226,64 @@ # or not a neb path else: - files = glob.glob(path + "/*.smf") + files = glob.glob(path + ".smf") if len(files) < 1: return None - return Fits(self.logger, self.doc, files[0]) # TODO just returning first file - check + return Fits(self.logger, self.config, files[0]) # TODO just returning first file - check ''' Gets the path to the cmf file for this stack_id - ''' - def getStackStageCmf(self, stackID): - - sql = "SELECT staticskyResult.path_base \ - FROM staticskyRun \ - JOIN staticskyInput USING(sky_id) \ + + The logic here is fairly ridiculous due to issues with how GPC1 stores info regarding multi-filter stacks. In partcular, it + is impossible to find a full path for a cmf file for a given stack_id + + Instead, we get a path_base to a directory with a collection of cmfs. neb-ls will list 5 files, but a number < 5 may actually exist + meaning that neb-ls -p will fail. + + our logic is as follows: + + - use stack_id and the DVO label to find a path base for all cmfs + - loop through all the potential cmfs, run neb-ls -p on each + - if neb-ls -p fails, then the file does not exists and we ignore it + - if neb-ls -p succeeds, we open that cmf file and check the STK_ID value in its header to see if it is the one we want + - if it is, great, exit + - if not, try the next one in the list + + ''' + def getStackStageCmf(self, dvoDb, stackID): + + ''' + sql = "SELECT DISTINCT staticskyResult.path_base \ + FROM staticskyRun \ + JOIN staticskyInput USING(sky_id) \ JOIN staticskyResult USING(sky_id) \ - JOIN stackRun USING(stack_id) \ + JOIN stackRun USING(stack_id) \ JOIN stackSumSkyfile USING(stack_id) \ - WHERE stack_id = %s" % stackID - + JOIN addRun ON sky_id = stage_id \ + JOIN minidvodbRun USING (minidvodb_name) \ + WHERE stack_id = " + str(stackID) + " \ + AND minidvodbRun.minidvodb_group = '" + dvoDb + "' \ + AND minidvodbRun.state = 'merged' \ + AND stage = 'staticsky_multi'" + ''' + + sql = "SELECT DISTINCT staticskyResult.path_base \ + FROM staticskyRun \ + JOIN staticskyInput USING(sky_id) \ + JOIN staticskyResult USING(sky_id) \ + JOIN stackRun USING(stack_id) \ + JOIN stackSumSkyfile USING(stack_id) \ + JOIN addRun ON sky_id = stage_id \ + JOIN minidvodbRun USING (minidvodb_name) \ + JOIN mergedvodbRun using (minidvodb_id) \ + WHERE stack_id = " + str(stackID) + " \ + AND minidvodbRun.state = 'merged' \ + AND mergedvodbRun.state = 'full' \ + AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \ + AND stage = 'staticsky_multi'" + + + # get single path base for the directory containing all cmf files, one of which is hopefully for our stack_id try: rs = self.executeQuery(sql) @@ -235,19 +291,38 @@ pathBase = rs.getString(1) - # now find ALL cmf files at this path_base as there can be more than one - f=os.popen("neb-ls -p " + pathBase + "%cmf") - for i in f.readlines(): - - path = i.rstrip() - fits = Fits(self.logger, self.doc, path) - - # we need to check if this is the correct cmf file, and if so then break and return - if fits.getPrimaryHeaderValue("STK_ID") == str(stackID): - return fits - - except: - self.logger.exception("Can't query for stack cmfs") - - self.logger.error("Could not find stack cmf") + # now find and loop through all cmf files at this path_base + files=os.popen("neb-ls " + pathBase + "%cmf") + for i in files.readlines(): + + nebPath = i.rstrip() + + # now attempt to run 'neb-ls -p' to actually read this cmf file + try: + self.logger.debug("Trying to neb-ls -p on '" + nebPath + "'") + paths=os.popen("neb-ls -p " + nebPath) + path = paths.readline().rstrip() + if len(path) < 1: + self.logger.debug("zero length path - skipping") + raise + + # if we get here, then the cmf is readable, now check the stack_id + fits = Fits(self.logger, self.config, path) + if fits.getPrimaryHeaderValue("STK_ID") == str(stackID): + # we have the right file! + self.logger.debug("SUCCESS!: %s == %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID))) + return fits + else: + # this is not the correct file + self.logger.debug("STACK ID: %s != %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID))) + + # an exception here means that nebulous could not read this file, so we just skip it + except: + self.logger.debug("NEB FAILED SKIPPING") + pass + + except: + self.logger.errorPair("Can't query for stack cmfs", "stack_id = %d" % stackID) + + self.logger.errorPair("Couldn't find stack cmf for", "stack_id = %d" % stackID) return None Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/initbatch.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/initbatch.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/initbatch.py (revision 33638) @@ -27,18 +27,19 @@ def __init__(self, logger, - configPath, - configDoc, + config, gpc1Db, ippToPspsDb, + scratchDb, batchID): super(InitBatch, self).__init__(logger, - configPath, - configDoc, + config, gpc1Db, ippToPspsDb, + scratchDb, 0, batchID, "IN", - None) + None, + 0) self.outputFitsFile = "00000000.FITS"; Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/ipptopsps.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/ipptopsps.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/ipptopsps.py (revision 33638) @@ -0,0 +1,157 @@ +#!/usr/bin/env jython + +import signal +import sys +import os +import time +import logging +import socket + +from config import Config +from pslogger import PSLogger +from ipptopspsdb import IppToPspsDb + +''' +Abstract base class for all ippToPsps programs +Handles command-line args, setting up of Config and logger objects, clean exiting etc +''' +class IppToPsps(object): + + ''' + Constructor + ''' + def __init__(self, argv, logToStdout=1, logToFile=0): + + # are the arsg ok? all programs require a config name + if len(sys.argv) < 2: + self.printUsage() + self.exitProgram("Wrong args") + + # some class constants + self.PROGNAME = sys.argv[0] + self.CONFIGNAME = sys.argv[1] + self.HOST = socket.gethostname() + self.PID = os.getpid() + self.PAUSEPERIOD = 60 + self.HOURS = None + self.MINUTES = None + self.SECONDS = None + + # set up config object + self.config = Config(self.PROGNAME, self.CONFIGNAME) + self.logger = self.config.getLogger(self.HOST, self.PID, logToStdout, logToFile) + + # create connection to databases database + self.ippToPspsDb = IppToPspsDb(self.logger, self.config) + + self.checkClientStatus() + + # catch Ctrl-C signal + signal.signal(signal.SIGINT, self.signal_handler) + + # title for log + self.logger.infoSeparator() + self.logger.infoTitle("ippToPsps '" + self.PROGNAME + "' started") + self.logger.infoPair("Host", self.HOST) + self.logger.infoPair("PID", "%d" % self.PID) + self.logger.infoPair("Config", self.CONFIGNAME) + + ''' + Parses a poll-time argument string and stored as class variables + ''' + def parsePollTimeArg(self, hours): + + try: + self.HOURS = float(hours) + self.MINUTES = self.HOURS * 60.0 + self.SECONDS = self.MINUTES * 60.0 + except: + pass + + ''' + Waits for user defined poll time + ''' + def waitForPollTime(self): + + # if not POLL time set, then return False + if not self.SECONDS: return False + + if self.HOURS < 1.0: + str = "%.1f minutes" % self.MINUTES + else: + str = "%.1f hours" % self.HOURS + + self.logger.infoPair("Waiting for", str) + time.sleep(self.SECONDS) + + return True + + ''' + Prints usage to stdout + ''' + def printUsage(self, extra=""): + + # write message to log or stdout if no logger set up + msg = sys.argv[0] + " " + extra + try: + self.logger.errorPair("Usage:", msg) + except: + print "*** Usage: " + msg + + + ''' + Refreshes the config then recreates objects with new settings + ''' + def refreshConfig(self): + return self.ippToPspsDb.readConfig(self.HOST, self.PID) + + ''' + All implementing subclasses (programs) should call this method once in a while to see if it should be paused or killed, otherwise this updates the clients table with current time and reloads the config data + ''' + def checkClientStatus(self): + + self.ippToPspsDb.updateClient(self.PROGNAME, self.HOST, self.PID) + + if self.ippToPspsDb.isKilled(self.HOST, self.PID): + self.exitProgram("killed via Db") + + # this loop pauses the process if we have no config or we have set it to pause + firstTimeIn = True + while not self.refreshConfig() or self.ippToPspsDb.isPaused(self.HOST, self.PID): + + self.ippToPspsDb.updateClient(self.PROGNAME, self.HOST, self.PID) + if self.ippToPspsDb.isKilled(self.HOST, self.PID): + self.exitProgram("killed while paused") + if firstTimeIn: self.logger.infoPair("Paused. Will check again every", "%d secs" % self.PAUSEPERIOD) + firstTimeIn = False + time.sleep(self.PAUSEPERIOD) + + + ''' + Handler for Ctrl-C signal + ''' + def signal_handler(self, signal, frame): + + self.exitProgram("Ctrl-C") + + + ''' + Exits the program cleanly, removes info from ipptopsps database + ''' + def exitProgram(self, exitReason="quit"): + + # write message to log or stdout if no logger set up + try: + self.logger.infoPair("Program exited", exitReason) + except: + print "*** Program exited: " + exitReason + + # remove from database if possible + try: + self.ippToPspsDb.removeClient(self.HOST, self.PID) + except: + pass + + sys.exit(0) + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/ipptopspsdb.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/ipptopspsdb.py (revision 33638) @@ -17,8 +17,8 @@ Constructor ''' - def __init__(self, logger, doc): - super(IppToPspsDb, self).__init__(logger, doc, "ipptopspsdatabase") - - self.MAX_FAILS = 4 + def __init__(self, logger, config): + super(IppToPspsDb, self).__init__(logger, config, "ipptopspsdatabase") + + self.MAX_FAILS = 5 ''' @@ -38,10 +38,34 @@ try: rs = self.executeQuery(sql) - while (rs.next()): - ids.append(rs.getInt(1)) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() except: self.logger.exception("Can't query for merged batch ids in ipptopsps Db") - rs.close() + self.logger.debug("Found %d merged but un-deleted items" % len(ids)) + + return ids + + ''' + Returns a list of batch IDs marked as 'purged' but not yet deleted + ''' + def getPurgedButNotDeletedBatchIDs(self, batchType, epoch, dvoGpc1Label, column): + + sql = "SELECT DISTINCT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND batch_type = '" + batchType + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND purged = 1 \ + AND " + column + " = 0" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() + except: + self.logger.exception("Can't query for merged batch ids in ipptopsps Db") + self.logger.debug("Found %d merged but un-deleted items" % len(ids)) @@ -65,10 +89,8 @@ try: rs = self.executeQuery(sql) - while (rs.next()): - ids.append(rs.getInt(1)) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() except: self.logger.exception("Can't query for merged batch ids in ipptopsps Db") - - rs.close() self.logger.debug("Found %d merged but un-deleted items" % len(ids)) @@ -99,4 +121,16 @@ def getLoadedToODMButNotDeletedFromDXLayer(self, batchType, epoch, dvoGpc1Label): return self.getLoadedToODMButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_dxlayer") + + ''' + Returns a list of purged batch IDs that not deleted from local disk + ''' + def getPurgedButNotDeletedFromLocalDisk(self, batchType, epoch, dvoGpc1Label): + return self.getPurgedButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_local") + + ''' + Returns a list of purged batch IDs that are not deleted from datastore + ''' + def getPurgedButNotDeletedFromDatastore(self, batchType, epoch, dvoGpc1Label): + return self.getPurgedButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_datastore") @@ -107,5 +141,5 @@ want to load a test batch to PSPS - We also check that these batches are more than 4 HOURs old, so that we don't attempt to + We also check that these batches are more than 12 HOURs old, so that we don't attempt to publish something simulataneously with the client that it producing it ''' @@ -120,5 +154,5 @@ AND loaded_to_datastore != 1 \ AND timestamp > '2011-10-27' \ - AND timestamp < now() - INTERVAL 4 HOUR" + AND timestamp < now() - INTERVAL 12 HOUR" ids = [] @@ -127,8 +161,7 @@ while (rs.next()): ids.append(rs.getInt(1)) + rs.close() except: self.logger.exception("Can't query for processed batch ids not loaded to datastore ipptopsps Db") - - rs.close() self.logger.debug("Found %d processed but-not-on-datastore items" % len(ids)) @@ -240,6 +273,6 @@ except: self.logger.exception("Can't query for batch ids with " + column + " = " + str(value) + " in ipptopsps Db") - - rs.close() + finally: + rs.close() self.logger.debug("Found %d batches with %s = %d" % (len(ids), column, value)) @@ -250,5 +283,5 @@ Returns the total detections published to the datastore for this epoch, dvo label and batch type ''' - def getTotalDetectionsPublished(self, batchType, epoch, dvoGpc1Label): + def getTotalDetectionsPublished(self, batchType, epoch, dvoGpc1Label, interval="10 YEAR"): sql = "SELECT SUM(total_detections) \ @@ -258,5 +291,6 @@ AND dvo_db = '" + dvoGpc1Label + "' \ AND processed = 1 \ - AND loaded_to_datastore = 1" + AND loaded_to_datastore = 1 \ + AND timestamp BETWEEN now() - INTERVAL " + interval + " AND now()" total = -1 @@ -267,6 +301,6 @@ except: self.logger.exception("Can't query for total detections published") - - rs.close() + finally: + rs.close() self.logger.debug("Found %d detections" % total) @@ -338,10 +372,18 @@ ''' def getStages(self): - return ['processed', 'loaded_to_datastore', 'loaded_to_ODM', 'merge_worthy', 'deleted_datastore', 'deleted_dxlayer', 'merged', 'deleted_local' ] + return [ + 'processed', + 'loaded_to_datastore', + 'loaded_to_ODM', + 'merge_worthy', + 'deleted_datastore', + 'deleted_dxlayer', + 'merged', + 'deleted_local' ] ''' Returns a count of batches processed in the specified time period. ''' - def countBatchesInLastPeriod(self, batchType, epoch, dvoGpc1Label, interval): + def countBatches(self, batchType, epoch, dvoGpc1Label, interval="10 YEAR"): sql = "SELECT COUNT(batch_id) \ @@ -359,4 +401,6 @@ self.logger.exception("Unable to count batches in interval") return 0 + finally: + rs.close() ''' @@ -454,8 +498,6 @@ def updateOdmStatus(self, batchID, odmStatus): - if odmStatus['LOADFAILED'] == 1: loadedToODM = -1 - else: loadedToODM = odmStatus['LOADEDTOODM'] sql = "UPDATE batch \ - SET loaded_to_ODM = " + str(loadedToODM) + ", \ + SET loaded_to_ODM = " + str(odmStatus['LOADEDTOODM']) + ", \ merge_worthy = " + str(odmStatus['MERGEWORTHY']) + ", \ merged = " + str(odmStatus['MERGED']) + " \ @@ -465,7 +507,7 @@ ''' - Is someone processing this item right now? Checks whether this was attempted within the last 2 HOUR - ''' - def processingNow(self, batchType, stage_id, epoch, dvoGpc1Label): + Is someone processing this item right now? Checks whether this was attempted within the last 12 HOUR + ''' + def processingNow(self, batchType, stage_id): sql = "SELECT COUNT(*) \ @@ -473,6 +515,7 @@ WHERE stage_id = " + str(stage_id) + " \ AND batch_type = '" + batchType + "' \ - AND dvo_db = '" + dvoGpc1Label + "' \ - AND timestamp BETWEEN now() - INTERVAL 4 HOUR AND now()" + AND dvo_db = '" + self.config.dvoLabel + "' \ + AND batch_type != 'IN' \ + AND timestamp BETWEEN now() - INTERVAL 12 HOUR AND now()" try: @@ -480,5 +523,5 @@ rs.first() if rs.getInt(1) > 0: - self.logger.errorPair(str(stage_id) + " is already being processed", "skipping") + self.logger.debugPair(str(stage_id) + " is already being processed", "skipping") return True else: @@ -509,5 +552,5 @@ Have we already processed this stage_id? ''' - def alreadyProcessed(self, batchType, stage_id, epoch, dvoGpc1Label): + def alreadyProcessed(self, batchType, stage_id): sql = "SELECT COUNT(*) \ @@ -515,6 +558,7 @@ WHERE stage_id = " + str(stage_id) + " \ AND batch_type = '" + batchType + "' \ - AND timestamp > '" + epoch + "' \ - AND dvo_db = '" + dvoGpc1Label + "' \ + AND timestamp > '" + self.config.epoch + "' \ + AND dvo_db = '" + self.config.dvoLabel + "' \ + AND batch_type != 'IN' \ AND processed = 1" @@ -523,5 +567,5 @@ rs.first() if rs.getInt(1) > 0: - self.logger.errorPair(str(stage_id) + " has already been published", "skipping") + self.logger.debugPair(str(stage_id) + " has already been published", "skipping") return True else: @@ -533,5 +577,5 @@ Has this stage_id consistently failed to process? ''' - def consistentlyFailed(self, batchType, stage_id, epoch, dvoGpc1Label): + def consistentlyFailed(self, batchType, stage_id): sql = "SELECT COUNT(*) \ @@ -539,6 +583,6 @@ WHERE stage_id = " + str(stage_id) + " \ AND batch_type = '" + batchType + "' \ - AND timestamp > '" + epoch + "' \ - AND dvo_db = '" + dvoGpc1Label + "' \ + AND timestamp > '" + self.config.epoch + "' \ + AND dvo_db = '" + self.config.dvoLabel + "' \ AND processed = -1" @@ -588,14 +632,14 @@ ''' - Creates a new batch. This is done with a locked table in order that no two clients can start work on the same item - ''' - def createNewBatch(self, batchType, stageID, survey, epoch, dvoDb, datastoreProduct, force): + Creates a new batch. This is done with a locked table in order that no two clients can start work on the same item, i.e. this is a crital section + ''' + def createNewBatch(self, batchType, stageID): batchID = -1; - if force or \ - (not self.alreadyProcessed(batchType, stageID, epoch, dvoDb) \ - and not self.processingNow(batchType, stageID, epoch, dvoDb) \ - and not self.consistentlyFailed(batchType, stageID, epoch, dvoDb)): + if self.config.force or \ + (not self.alreadyProcessed(batchType, stageID) \ + and not self.processingNow(batchType, stageID) \ + and not self.consistentlyFailed(batchType, stageID)): sql = "INSERT INTO batch ( \ @@ -608,7 +652,7 @@ '" + batchType + "', \ " + str(stageID) + ", \ - '" + survey + "', \ - '" + dvoDb + "', \ - '" + datastoreProduct + "' \ + '" + self.config.survey + "', \ + '" + self.config.dvoLabel + "', \ + '" + self.config.datastoreProduct + "' \ )" @@ -670,4 +714,6 @@ loaded_to_datastore = 0, \ loaded_to_ODM = 0, \ + merge_worthy = 0, \ + merged = 0, \ deleted_datastore = 0, \ deleted_dxlayer = 0, \ @@ -678,9 +724,573 @@ ''' + Pause loading clients + ''' + def pauseLoaders(self, bool, ids): + self.setColForLoaders("paused", bool, ids) + + ''' + Kill loading clients + ''' + def killLoaders(self, bool, ids): + self.setColForLoaders("killed", bool, ids) + + ''' + Sets the config field for a set of loader clients + ''' + def setConfigForLoaders(self, config, ids): + + for id in ids: + self.execute("UPDATE clients SET config = '" + config + "' \ + WHERE type = 'loader.py' \ + AND id = " + str(id)) + + ''' + Sets the value for a given column for a set of loader clients + ''' + def setColForLoaders(self, col, bool, ids): + + if bool: value = 1 + else: value = 0 + + for id in ids: + self.execute("UPDATE clients SET " + col + " = " + str(value) + " \ + WHERE type = 'loader.py' \ + AND id = " + str(id)) + + ''' + Remove client by id + ''' + def removeClientsByIds(self, ids): + for id in ids: + self.execute("DELETE FROM clients WHERE id = " + str(id)) + + ''' + Remove client + ''' + def removeClient(self, host, pid): + self.execute("DELETE FROM clients WHERE host = '" + host + "' AND pid = " + str(pid)) + + ''' + Insert client + ''' + def insertClient(self, type, host, pid): + self.execute("INSERT INTO clients (timestamp, type, host, pid, config) VALUES (now(), '" + type + "', '" + host + "', " + str(pid) + ", '" + self.config.name + "')") + + ''' + Update clients, or inserts it for the first time + ''' + def updateClient(self, type, host, pid): + + try: + self.insertClient(type, host, pid) + except: + self.execute("UPDATE clients SET timestamp = now() WHERE host = '" + host + "' AND pid = " + str(pid)) + + ''' + Is this client paused? + ''' + def isPaused(self, host, pid): + return self.getClientsColumnBoolean("paused", host, pid) + + ''' + Is this client killed? + ''' + def isKilled(self, host, pid): + return self.getClientsColumnBoolean("killed", host, pid) + + ''' + Returns boolean value for this column in clients table + ''' + def getClientsColumnBoolean(self, col, host, pid): + + sql = "SELECT " + col + " \ + FROM clients \ + WHERE host = '" + host + "' AND pid = " + str(pid) + + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) == 1: return True + else: return False + except: + self.logger.exception("Unable to check whether " + col + " is true or false in clients table") + + return False + + ''' + Returns a list of available configs + ''' + def getConfigList(self): + + sql = "SELECT DISTINCT name FROM config" + + configs = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): configs.append(rs.getString(1)) + except: + self.logger.errorPair("Can't get config names", sql) + finally: + rs.close() + + return configs + + + ''' + Writes current Config object to Db + ''' + def writeConfig(self): + + sql = "UPDATE config \ + SET \ + datastore_product = '" + self.config.datastoreProduct + "' \ + ,datastore_type = '" + self.config.datastoreType + "' \ + ,datastore_publish = " + self.config.datastorePublishing + " \ + ,dvo_label = '" + self.config.dvoLabel + "' \ + ,dvo_location = '" + self.config.dvoLocation + "' \ + ,min_ra = " + self.config.minRa + " \ + ,max_ra = " + self.config.maxRa + " \ + ,min_dec = " + self.config.minDec + " \ + ,max_dec = " + self.config.maxDec + " \ + ,box_size = " + self.config.boxSize + " \ + ,base_path = '" + self.config.basePath + "' \ + ,data_release = " + self.config.dataRelease + " \ + ,delete_local = " + self.config.deleteLocal + " \ + ,delete_datastore = " + self.config.deleteDatastore + " \ + ,delete_dxlayer = " + self.config.deleteDxLayer + " \ + ,epoch = '" + self.config.epoch + "' \ + ,survey = '" + self.config.survey + "' \ + ,psps_survey = '" + self.config.pspsSurvey + "' \ + ,queue_P2 = " + self.config.queuingThisBatchType("P2") + " \ + ,queue_ST = " + self.config.queuingThisBatchType("ST") + " \ + WHERE name = '" + self.config.name + "'" + + self.execute(sql) + + ''' + Reads config from the database and populates Config object + ''' + def readConfig(self, host, pid): + + # first get config defined for this client + sql = "SELECT config \ + FROM clients \ + WHERE host = '" + host + "' \ + AND pid = " + str(pid) + + try: + rs = self.executeQuery(sql) + rs.first() + self.config.name = rs.getString(1) + except: + self.logger.errorPair("No config set for", "%s (pid=%d)" % (host, pid)) + self.config.isLoaded = False + return False + + # now load that config + sql = "SELECT \ + datastore_product \ + ,datastore_type \ + ,datastore_publish \ + ,dvo_label \ + ,dvo_location \ + ,min_ra \ + ,max_ra \ + ,min_dec \ + ,max_dec \ + ,box_size \ + ,base_path \ + ,data_release \ + ,delete_local \ + ,delete_datastore \ + ,delete_dxlayer \ + ,epoch \ + ,survey \ + ,psps_survey \ + ,queue_P2 \ + ,queue_ST \ + FROM config \ + WHERE name = '" + self.config.name + "'" + + self.config.batchTypes = [] + try: + rs = self.executeQuery(sql) + rs.first() + self.config.datastoreProduct = rs.getString(1) + self.config.datastoreType = rs.getString(2) + if rs.getInt(3) == 1: self.config.datastorePublishing = True + else: self.config.datastorePublishing = False + self.config.dvoLabel = rs.getString(4) + self.config.dvoLocation = rs.getString(5) + self.config.minRa = rs.getDouble(6) + self.config.maxRa = rs.getDouble(7) + self.config.minDec = rs.getDouble(8) + self.config.maxDec = rs.getDouble(9) + + self.config.boxSize = rs.getDouble(10) + self.config.halfBox = self.config.boxSize/2.0 + self.config.boxSizeWithBorder = self.config.boxSize + (self.config.BORDER * 2) + + self.config.basePath = rs.getString(11) + self.config.dataRelease = rs.getInt(12) + self.config.deleteLocal = rs.getInt(13) + self.config.deleteDatastore = rs.getInt(14) + self.config.deleteDxLayer = rs.getInt(15) + self.config.epoch = rs.getString(16) + self.config.survey = rs.getString(17) + self.config.pspsSurvey = rs.getString(18) + if rs.getInt(19) == 1: self.config.batchTypes.append("P2") + if rs.getInt(20) == 1: self.config.batchTypes.append("ST") + self.config.force = False # TODO + self.config.test = False # TODO + self.config.isLoaded = True + except: + self.logger.errorPair("Could not read config with name", self.config.name) + self.config.isLoaded = False + + return self.config.isLoaded + + ''' + Gets id of box with these coords + ''' + def getBoxId(self, ra, dec): + + sql = "SELECT id FROM box \ + WHERE config = '" + self.config.name + "' \ + AND ra_center = " + str(ra) + " \ + AND dec_center = " + str(dec) + " \ + AND box_side = " + str(self.config.boxSize) + + id = -1 + try: + rs = self.executeQuery(sql) + rs.first() + id = rs.getInt(1) + except: + pass + + return id + + ''' + Remove all boxes and associated ids in pending table + ''' + def removeAllBoxes(self): + self.execute("DELETE FROM box WHERE config = '" + self.config.name + "'") + + ''' + Inserts new box into box table. If it already exisits, it returns existing box id + ''' + def insertBox(self, ra, dec): + + id = self.getBoxId(ra, dec) + + # does not already exists so insert + if id < 0: + + sql = "INSERT INTO box \ + (config, ra_center, dec_center, box_side) \ + VALUES \ + ('" + self.config.name + "', " + str(ra) + ", " + str(dec) + ", " + str(self.config.boxSize) + ")" + + self.execute(sql) + id = self.getBoxId(ra, dec) + + return id + + ''' + Inserts new box ids into the pending table + ''' + def insertPending(self, box_id, batchType, ids): + + # first delete old pending items + self.execute("DELETE FROM pending \ + WHERE box_id = " + str(box_id) + " \ + AND batch_type = '" + batchType + "'") + + for id in ids: + sql = "INSERT INTO pending \ + (box_id, batch_type, stage_id) \ + VALUES \ + (" + str(box_id) + ", '" + batchType + "', " + str(id) + ")" + + self.execute(sql) + + ''' + Returns ids for all boxes with pending items in a stripe of RA not already being processed + by another client. If none are available, then it uses the stripe already used by another + client that has the most pending items + ''' + def getStripeBoxIds(self, host, pid, batchType): + + self.execute("LOCK TABLES box READ, pending READ, stripe WRITE, clients WRITE") + + ids = [] + raCenter = -1.0 + sql = "SELECT ra_center \ + FROM box \ + JOIN pending ON (id = box_id) \ + WHERE config = '" + self.config.name + "' \ + AND batch_type = '" + batchType + "' \ + AND ra_center NOT IN \ + (SELECT ra_center \ + FROM stripe \ + WHERE config = '" + self.config.name + "') \ + GROUP BY ra_center LIMIT 1" + + try: + rs = self.executeQuery(sql) + rs.first() + raCenter = rs.getFloat(1) + rs.close() + except: + self.logger.errorPair("No free stripes", "trying to share with other client...") + sql = "SELECT ra_center, COUNT(*) AS numPending\ + FROM box \ + JOIN pending ON (id = box_id) \ + WHERE config = '" + self.config.name + "' \ + AND batch_type = '" + batchType + "' \ + GROUP BY ra_center \ + ORDER BY numPending \ + DESC LIMIT 1" + + try: + rs = self.executeQuery(sql) + rs.first() + raCenter = rs.getFloat(1) + rs.close() + except: + self.logger.errorPair("No available stripes to share", "no stripe obtained") + + self.logger.infoPair("Using RA stripe", "%f" % raCenter) + sql = "SELECT DISTINCT id \ + FROM box \ + JOIN pending ON (id = box_id) \ + WHERE ra_center = " + str(raCenter) + " \ + AND batch_type = '" + batchType + "' \ + AND config = '" + self.config.name + "' \ + ORDER BY dec_center DESC" + + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() + except: + self.logger.errorPair("Can't get stripe boxes", sql) + + self.registerStripe(host, pid, raCenter) + + self.unlockTables() + + return ids + + ''' + Inserts row into strip table + ''' + def registerStripe(self, host, pid, raCenter): + + # first, delete any existing record + try: + self.execute("DELETE FROM stripe \ + WHERE client_id = \ + (SELECT id FROM clients \ + WHERE host = '" + host + "' AND pid = " + str(pid) + ")") + except: + self.logger.errorPair("Can't delete from stripe table", sql) + + + # now insert new stripe entry + sql = "INSERT INTO stripe \ + (client_id, config, ra_center) \ + SELECT clients.id, clients.config, " + str(raCenter) + " \ + FROM clients \ + WHERE host = '" + host + "' \ + AND pid = " + str(pid) + try: + self.execute(sql) + except: + self.logger.errorPair("Can't insert into stripe", sql) + + + ''' + Returns dimensions of this box + ''' + def getBoxDimensions(self, boxId): + + sql = "SELECT ra_center, dec_center, box_side FROM box WHERE id = " + str(boxId) + + boxDim = {} + rs = self.executeQuery(sql) + rs.first() + + boxDim['RA'] = rs.getFloat(1) + boxDim['DEC'] = rs.getFloat(2) + boxDim['SIDE'] = rs.getFloat(3) + + return boxDim + + ''' + Returns ids for pending items for this config + ''' + def getPendingIds(self): + + sql = "SELECT DISTINCT stage_id FROM pending JOIN box ON (box_id = id) WHERE config = '" + self.config.name + "'" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() + except: + self.logger.errorPair("Can't get pending ids for this config", sql) + + return ids + + ''' + Returns ids for pending items in this box for this batch type + ''' + def getPendingIdsForThisBox(self, boxId, batchType): + + sql = "SELECT stage_id FROM pending WHERE batch_type = '" + batchType + "' AND box_id = " + str(boxId) + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() + except: + self.logger.errorPair("Can't get ids from this box", sql) + + return ids + + ''' + Deletes an item from the pending table + ''' + def deletePendingItem(self, batchType, stageID): + + sql = "DELETE pending \ + FROM pending \ + JOIN box ON (id = box_id) \ + WHERE stage_id = " + str(stageID) + " \ + AND batch_type = '" + batchType + "' \ + AND config = '" + self.config.name + "'" + + self.execute(sql) + + ''' + Writes density-plot data to file for pending stuff over sky for given config + ''' + def createPendingDensityPlotData(self, batchType, DATFILE): + + sql = "SELECT id, ra_center, dec_center \ + FROM box \ + WHERE config = '" + self.config.name + "' \ + ORDER BY ra_center, dec_center" + + maxPending = 0 + try: + rs = self.executeQuery(sql) + while (rs.next()): + + boxId = rs.getInt(1) + + sql = "SELECT COUNT(stage_id) \ + FROM pending \ + WHERE box_id = " + str(boxId) + " \ + AND batch_type = '" + batchType + "'" + rs2 = self.executeQuery(sql) + rs2.first() + max = rs2.getInt(1) + rs2.close() + print >> DATFILE, rs.getFloat(2), rs.getFloat(3), max + if max > maxPending: maxPending = max + rs.close() + except: + self.logger.errorPair("Can't get density plot data", sql) + + return maxPending + + ''' + Creates a temporary table and shoves it full of stage_ids and ra/dec coords + ''' + def storeAllItems(self, rows): + + try: + self.execute("DROP TABLE all_pending") + except: + pass + + self.execute("CREATE TEMPORARY TABLE all_pending (stage_id bigint(20), ra_bore float, dec_bore float)") + count = 0 + for row in rows: + + try: + sql = "INSERT INTO all_pending (stage_id, ra_bore, dec_bore) \ + VALUES (%d, %f, %f)" % (row[0], row[1], row[2]) + self.execute(sql) + count += 1 + except: continue + + self.logger.infoPair("All items written to Db", "%d" % count) + + ''' + Gets all items in the all_pending temporary table within the bounds of this box + ''' + def getItemsInThisThisBox(self, ra, dec): + + halfSide = self.config.boxSize/2.0 + minRa = ra-halfSide + maxRa = ra+halfSide + minDec = dec-halfSide + maxDec = dec+halfSide + + ids = [] + + sql = "SELECT DISTINCT stage_id \ + FROM all_pending \ + WHERE ra_bore BETWEEN " + str(minRa) + " AND " + str(maxRa) + " \ + AND dec_bore BETWEEN " + str(minDec) + " AND " + str(maxDec) + + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + except: + self.logger.errorPair("Can't get items in this box", sql) + + return ids + + ''' + Gets client info + ''' + def getClientInfo(self): + + sql = "SELECT id, host, started, timestamp, ra_center, paused, killed \ + FROM stripe \ + JOIN clients ON (client_id = id) \ + WHERE clients.config = '" + self.config.name + "' \ + ORDER BY host, started" + + rows = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): + rows.append([rs.getInt(1), rs.getString(2), rs.getString(3), rs.getString(4), rs.getFloat(5), rs.getInt(6), rs.getInt(7)]) + rs.close() + except: + self.logger.errorPair("Can't query for client data", sql) + + return rows + + ''' + Purges old clients that may have die, i.e. those with a timestamp older than n hours + ''' + def purgeDeadClients(self): + self.execute("DELETE FROM clients WHERE timestamp < now() - INTERVAL 6 HOUR") + + ''' Destructor ''' def __del__(self): - self.logger.debug("IppToPspsDb Desstructor") + self.logger.debug("IppToPspsDb Destructor") self.stmt.close() self.con.close() Index: anches/eam_branches/ipp-20111122/ippToPsps/jython/load.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/load.py (revision 33637) +++ (revision ) @@ -1,254 +1,0 @@ -#!/usr/bin/env jython - -# -# The main loading program for ippToPsps. See usage below. -# - -import time -import sys -import os -import socket -import logging.config -from xml.etree.ElementTree import ElementTree, Element, tostring - -from pslogger import PSLogger -from gpc1db import Gpc1Db -from ipptopspsdb import IppToPspsDb -from datastore import Datastore -from batch import Batch - -from initbatch import InitBatch -from stackbatch import StackBatch -from detectionbatch import DetectionBatch - -''' -Finds batches that processed ok, but did not get loaded to the datastore, then tries to load them -''' -def publishAnyUnpublishedBatches(batchType): - - batchIDs = ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(EPOCH, DVOGPC1LABEL, batchType) - logger.infoPair("%s batches" % batchType, "%d" % len(batchIDs)) - - for batchID in batchIDs: - - batchName = Batch.getNameFromID(batchID) - subDir = Batch.getSubDir(BASEPATH, batchType, DVOGPC1LABEL) - tarballFile = Batch.getTarballFile(batchID) - logger.infoPair("Batch name", batchName) - - Batch.publishToDatastore(datastore, batchID, batchName, subDir, tarballFile) - -''' -Queues up all available stuff for this batch type -''' -def queueItems(batchType): - - logger.infoTitle("Queuing " + batchType + " items") - - allIDs = [] - - # see if there are any IDs listed in the config - try: - for element in configDoc.findall('options/ids/id'): - allIDs.append(int(element.text)) - except: - logger.errorPair("Problem loading IDs from config under", "options/ids/id") - - if len(allIDs) > 0: - logger.infoPair("Using config IDs (not DVO)", "%d" % len(allIDs)) - # no config dictated IDs, so get from DVO instead - else: - - if MINRA and MAXRA and MINDEC and MAXDEC: - allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType, MINRA, MAXRA, MINDEC, MAXDEC) - else: - allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType) - - logger.infoPair("All %s items in DVO" % batchType, "%d" % len(allIDs)) - - # if in FORCE mode, then queue full list - if FORCE: ids = allIDs - - # if not in TEST mode, then only queue un-processed items - else: - processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOGPC1LABEL) - consistaentlyFailedIDs = ippToPspsDb.getConsistentlyFailedIDs(batchType, EPOCH, DVOGPC1LABEL) - ids = list(set(allIDs) - set(processedIDs) - set(consistaentlyFailedIDs)) - logger.infoPair("Processed items", "%d" % len(processedIDs)) - logger.infoPair("Consistently failing items", "%d" % len(consistaentlyFailedIDs)) - logger.infoPair("Unprocessed items", "%d" % len(ids)) - - logger.infoPair("%s items queued" % batchType, "%d" % len(ids)) - - # loop round IDs of all items to be processed - ippToPspsDb.lockBatchTable() - for id in ids: - - batchID = ippToPspsDb.createNewBatch( - batchType, - id, - SURVEY, - EPOCH, - DVOGPC1LABEL, - datastore.product, - FORCE) - - if batchID < 0: continue; - - ippToPspsDb.unlockTables() - - if batchType == "P2": - batch = DetectionBatch(logger, - CONFIG, - configDoc, - gpc1Db, - ippToPspsDb, - id, - batchID) - elif batchType == "ST": - batch = StackBatch(logger, - CONFIG, - configDoc, - gpc1Db, - ippToPspsDb, - id, - batchID) - - batch.run() - - ippToPspsDb.lockBatchTable() - - logger.infoSeparator() - - # if in TEST mode, then quit after one batch - if TEST: break - - ippToPspsDb.unlockTables() - -''' -Start of program. -''' - -if len(sys.argv) > 1: CONFIG = sys.argv[1] -else: - print "** Usage: " + sys.argv[0] + " [init]" - sys.exit(1) - -if len(sys.argv) > 2 and sys.argv[2] == "init": - QUEUE_IN = 1 -else: - QUEUE_IN = 0 - -# open config file -configDoc = ElementTree(file=CONFIG) -TEST = int(configDoc.find("options/testMode").text) - -# set up logging -logging.setLoggerClass(PSLogger) -logger = logging.getLogger("ippToPspsLog") - -# get hostnamee and PID for unique log naming -HOST = socket.gethostname() -PID = os.getpid() - -# if in test mode, print log to screen, otherwise, only to file -logger.setup(configDoc, "ippToPsps", TEST, not TEST, HOST, PID) - -# create database objects -gpc1Db = Gpc1Db(logger, configDoc) -ippToPspsDb = IppToPspsDb(logger, configDoc) -datastore = Datastore(logger, configDoc, ippToPspsDb) - -# check we connected ok -if not gpc1Db.everythingOK: sys.exit(1) -if not ippToPspsDb.everythingOK: sys.exit(1) - -# get values from the configutaion file -POLLPERIOD = 600 -DVOGPC1LABEL = configDoc.find("dvo/gpc1Label").text -FORCE = int(configDoc.find("options/force").text) -EPOCH = configDoc.find("options/epoch").text -PUBLISH = int(configDoc.find("options/publishToDatastore").text) -BASEPATH = configDoc.find("localOutPath").text -QUEUE_P2 = int(configDoc.find("options/queueP2").text) -QUEUE_ST = int(configDoc.find("options/queueST").text) -SURVEY = configDoc.find("options/survey").text - -# get equatorial coord limits, if any -MINRA = None -MAXRA = None -MINDEC = None -MAXDEC = None -try: - MINRA = float(configDoc.find("dvo/minRA").text) - MAXRA = float(configDoc.find("dvo/maxRA").text) - MINDEC = float(configDoc.find("dvo/minDec").text) - MAXDEC = float(configDoc.find("dvo/maxDec").text) -except: - pass - -# prompt user: FORCE and PUBLISH is a dangerous combination -if FORCE and PUBLISH and not QUEUE_IN: - response = raw_input("\n*** Are you sure you want to publish data with the 'force' option enabled (y/n)? ") - if response != "y": sys.exit(1) - -# prompt user: TEST and PUBLISH is a dangerous combination -if TEST and PUBLISH and not QUEUE_IN: - response = raw_input("\n*** Are you sure you want to publish data with the 'testMode' option enabled (y/n)? ") - if response != "y": sys.exit(1) - -# print info to log -logger.infoSeparator() -logger.infoTitle("ippToPsps loading started") -logger.infoPair("Configuration file", CONFIG) -logger.infoPair("Loading epoch", EPOCH) -logger.infoPair("DVO gpc1 label", DVOGPC1LABEL) -logger.infoBool("Queuing P2 batches?", QUEUE_P2) -logger.infoBool("Queuing ST batches?", QUEUE_ST) -logger.infoBool("Forcing?", FORCE) -logger.infoBool("Test mode?", TEST) -logger.infoBool("Publishing?", PUBLISH) - -''' -Main processing loop: - - 1) creates an IN batch if requested - 2) queues available P2 batches (if requested) - 3) queues available ST batches (if requested) - 4) if in test mode, quits - 5) otherwise, waits then repeats from 2 - -''' -while True: - - # queue all items for all batch types, sequentially - if QUEUE_IN: - batchID = ippToPspsDb.createNewBatch( - "IN", - 0, - SURVEY, - EPOCH, - DVOGPC1LABEL, - datastore.product, - 1) - if batchID > 0: - batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb, batchID) - batch.run() - break; - - # queue up batches that are processed but not loaded to datastore - logger.infoTitle("Previous failed datastore loads") - publishAnyUnpublishedBatches("P2") - publishAnyUnpublishedBatches("ST") - - if QUEUE_P2: queueItems("P2") - if QUEUE_ST: queueItems("ST") - - # in the TEST mode, we quit after submitting one batch - if TEST: break - - # wait for the POLLPERIOD before checking for new ids - logger.infoPair("Finished. Waiting for", "%.1f minutes before checking DVO for new items" % (POLLPERIOD/60.0)) - time.sleep(POLLPERIOD) - - Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/loader.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/loader.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/loader.py (revision 33638) @@ -0,0 +1,250 @@ +#!/usr/bin/env jython + +# +# The main loading program for ippToPsps. See usage below. +# + +import time +import sys +import os +import traceback +import logging.config + +from ipptopsps import IppToPsps +from scratchdb import ScratchDb +from gpc1db import Gpc1Db +from datastore import Datastore +from batch import Batch +from dvo import Dvo + +from initbatch import InitBatch +from stackbatch import StackBatch +from detectionbatch import DetectionBatch + +''' +Loader class +''' +class Loader(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(Loader, self).__init__(argv, 1, 1) + + # create gpc1 database objects + self.gpc1Db = Gpc1Db(self.logger, self.config) + + # connect to scratch database + scratchDbs = ['ipptopsps_scratch', 'ipptopsps_scratch2', 'ipptopsps_scratch3'] + for dbName in scratchDbs: + self.scratchDb = ScratchDb(self.logger, self.config, dbName) + if self.scratchDb.anyOtherConnections(): + self.logger.errorPair("This scratch Db is already in use", dbName) + self.scratchDb.disconnect() + continue + self.logger.infoPair("Using scratch Db", self.scratchDb.dbName) + break + + if not self.scratchDb.connected: + self.logger.errorPair("Cannot connect to a", "scratch database") + + # create Dvo and Datastore objects + self.dvo = Dvo(self.logger, self.config, self.scratchDb.dbName) + self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + + # if an IN batch is requested, create and quit + if len(sys.argv) > 2 and sys.argv[2] == "init": + batchID = self.ippToPspsDb.createNewBatch("IN", 0) + if batchID > 0: + batch = InitBatch(self.logger, + self.config, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + batchID) + batch.run() + else: + self.logger.errorPair("Could not create batch ID", "%d" % batchID) + self.exitProgram("failed to create init batch") + + self.exitProgram("init batch created") + + + # set a poll time of about 1 minutes + self.parsePollTimeArg("0.0166") + + self.config.printAll() + + ''' + Overrides base-class version so we can ensure our scratch Db is using the right DVO Db + ''' + def refreshConfig(self): + + ret = super(Loader, self).refreshConfig() + try: self.dvo.checkDvoLocation() + except: + self.logger.errorPair("can't check", "dvo.checkDvoLocation()") + return ret + + return ret + + ''' + Overrides base_class version so we can break out of processing loops if the config changes + ''' + def checkClientStatus(self): + + oldConfigName = self.config.name + super(Loader, self).checkClientStatus() + if oldConfigName != self.config.name: + self.logger.infoPair("Config changed", "from '" + oldConfigName + "' to '" + self.config.name + "'") + self.config.printAll() + return False + + return True + + ''' + Main processing loop + ''' + def run(self): + + numAttempts = 0 + while True: + + abort = False + for batchType in self.config.batchTypes: + + # get a stripe's worth of box IDs + boxIds = self.ippToPspsDb.getStripeBoxIds(self.HOST, self.PID, batchType) + + numAttempts += 1 + + self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds)) + + if len(boxIds) > 0: numAttempts = 0 + + for boxId in boxIds: + + if not self.checkClientStatus(): + abort = True + break + + # get box info. if boxes have changed, break and start again + try: + boxDim = self.ippToPspsDb.getBoxDimensions(boxId) + except: + self.logger.infoPair("Can't get details for this box", "%d" % boxId) + break + + ids = self.ippToPspsDb.getPendingIdsForThisBox(boxId, batchType) + + if len(ids) < 1: + self.logger.debugPair("No " + batchType + " items found in this box", "skipping") + continue + + self.logger.infoSeparator() + self.logger.infoTitle("New box") + self.logger.infoPair("Box dimensions", "%.1f / %.1f / %.1f" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE'])) + self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids)) + boxSizeWithBorder = boxDim['SIDE'] + (self.config.BORDER * 2) + + # look in DVO for this box (with extra border) + #self.dvo.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder) + #sizeToBeIngested = self.dvo.getDiskSizeOfRegionsToBeIngested() + #if sizeToBeIngested == 0.0: smfsPerGB = 999999999 + #else: smfsPerGB = len(ids)/sizeToBeIngested + #self.logger.infoPair("DVO to be ingested", "%.1f GB" % sizeToBeIngested) + #self.logger.infoPair("smfs-per-GB", "%.1f" % smfsPerGB) + smfsPerGB = 0 + # should do we pre-ingest stuff from DVO? + if smfsPerGB > 40: + if not self.dvo.sync(): + self.logger.errorPair("Could not sync DVO with MySQL", "skipping") + continue + + useFullTables = 1 + else: useFullTables = 0 + + self.logger.infoBool("Using pre-ingested DVO data?", useFullTables) + if not self.processTheseItems(batchType, ids, useFullTables): + abort = True + break + + if abort or not self.checkClientStatus(): abort = True + elif numAttempts > 1 and not self.waitForPollTime(): break + + + ''' + Actually loops through items and processes them + ''' + def processTheseItems(self, batchType, ids, useFullTables): + + self.logger.infoPair("%s items queued" % batchType, "%d" % len(ids)) + + # loop round IDs of all items to be processed + self.ippToPspsDb.lockBatchTable() + unattemptedCount = 0 + for id in ids: + + batchID = self.ippToPspsDb.createNewBatch(batchType, id) + + if batchID < 0: + unattemptedCount += 1 + continue + + self.ippToPspsDb.unlockTables() + + self.ippToPspsDb.deletePendingItem(batchType, id) + + if batchType == "P2": + batch = DetectionBatch(self.logger, + self.config, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID, + useFullTables) + elif batchType == "ST": + batch = StackBatch(self.logger, + self.config, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID, + useFullTables) + + batch.run() + + if not self.checkClientStatus(): return False + + self.ippToPspsDb.lockBatchTable() + + self.logger.infoSeparator() + + # if in test mode, then quit after one batch # TODO remove + if self.config.test: break + + self.ippToPspsDb.unlockTables() + self.logger.infoPair("Unattempted batches", "%d" % unattemptedCount) + + return True + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Cleanup, self).printUsage("<[init]>") + + +''' +Start of program. +''' +try: + loader = Loader(sys.argv) + loader.run() + loader.exitProgram("finished") +except Exception, e: + print str(e) + traceback.print_exc() Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/metrics.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/metrics.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/metrics.py (revision 33638) @@ -7,209 +7,198 @@ import datetime -from xml.etree.ElementTree import ElementTree, Element, tostring - -from pslogger import PSLogger -from ipptopspsdb import IppToPspsDb +from ipptopsps import IppToPsps from czardb import CzarDb from gpc1db import Gpc1Db - -''' -Makes a gnuplot of the data -''' -def plotMe(batchType, file): - - OUTPUTFILE = "plots/" + DVOLABEL + "_" + batchType + ".png" - f=os.popen('gnuplot', 'w') - - if 0: - TERM = "X11" - else: - TERM = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" - - print >> f, "set term " + TERM + "; \ - set output \"" + OUTPUTFILE + "\"; \ - set title \"ippToPsps : " + batchType + " loading for " + DVOLABEL + "\"; \ - set boxwidth; \ - set xtic rotate by -90 scale 0; \ - set style data histogram; \ - set style histogram rowstacked; \ - set style fill solid border -1; \ - set ylabel \"Items processed per loading stage\"; \ - set boxwidth 0.75; \ - plot '" + file + "' using 2:xtic(1) notitle lt 2, '' using 3 notitle lt 1, '' using 4 notitle lt rgb \"#808080\"" - - f.flush() - -''' -Formats an int to a string -''' -def getIntAsString(value): - if value == 0: return "" - return str(value) - -''' -Prints (to the log) one row of data (one row being one batch-type) -''' -def printTableRow(stages, batchType): - - # get a master list of IDs in DVO for this batch type - if MINRA and MAXRA and MINDEC and MAXDEC: - allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType, MINRA, MAXRA, MINDEC, MAXDEC) - else: - allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType) - prevList = allIDs - numEverything = len(allIDs) - - sys.stdout.write("| %2s |%6d " % (batchType, numEverything)) - - tempFilename = batchType + '.dat' - DATFILE = open(tempFilename,'w') - print >> DATFILE, "DVO", numEverything, 0, 0 - for stage in stages: - - # get lists. Use unions with prev list to make sure the right stuff is included - success = list(set(ippToPspsDb.getStageIDs(batchType, EPOCH, DVOLABEL, stage, 1)) & set(prevList)) - fail = list(set(ippToPspsDb.getStageIDs(batchType, EPOCH, DVOLABEL, stage, -1)) & set(prevList) - set(success)) - pending = list(set(prevList) - set(success) - set(fail)) - - # count stuff - numSuccess = len(success) - numFail = len(fail) - numPending = len(pending) - - #if stage == 'loaded_to_datastore' and batchType == "P2": - # for n in fail: print n - - str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m " % (getIntAsString(numPending), getIntAsString(numSuccess), getIntAsString(numFail)) - sys.stdout.write("|%-19s" % str) - - prevList = success - print >> DATFILE, stage, numSuccess, numFail, numPending - - czarDb.insertStats(stage, DVOLABEL, batchType, numPending, numSuccess, numFail) - - if stage == 'processed': numPendingProcessed = numPending - - sys.stdout.write("|\n") - plotMe(batchType, tempFilename) - DATFILE.close() - - return numPendingProcessed - -''' -Writes table separator -''' -def writetableSeparator(colCount): - - sys.stdout.write("+----+-------") - for i in range(0, colCount): sys.stdout.write("+-------------------") - sys.stdout.write("+\n") - -''' -Prints various timing stats for this batch type -''' -def printStats(batchType): - - rate = ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "1 HOUR") / 1.0 - - logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" % - (batchType, - rate, - ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "24 HOUR"), - ippToPspsDb.countBatchesInLastPeriod(batchType, EPOCH, DVOLABEL, "1 WEEK") / 7.0, - ippToPspsDb.getTotalDetectionsPublished(batchType, EPOCH, DVOLABEL), - ippToPspsDb.getTimeOfLastBatchPublished(batchType, EPOCH, DVOLABEL))) - - return rate +from plot import Plot + +''' +Metrics class +''' +class Metrics(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(Metrics, self).__init__(argv) + + if len(sys.argv) > 2: self.parsePollTimeArg(sys.argv[2]) + + # create database objects + self.czarDb = CzarDb(self.logger, self.config) + self.gpc1Db = Gpc1Db(self.logger, self.config) + self.plot = Plot(self.logger, self.config, self.ippToPspsDb) + + ''' + Run... + ''' + def run(self): + + while True: + + self.logger.infoTitle("ippToPsps loading summary") + now = datetime.datetime.now() + self.logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S")) + self.logger.infoPair("Loading epoch", self.config.epoch) + self.logger.infoPair("DVO label", self.config.dvoLabel) + self.config.printBoxCoords() + + self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + self.logger.info("|Type| batches per hour | last 24 hours | per day this week | detections/batch | last published |") + self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + rateP2 = self.printStats("P2") + rateST = self.printStats("ST") + self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + + self.logger.info("") + + stages = self.ippToPspsDb.getStages() + colCount = len(stages) + self.writeTableSeparator(colCount) + sys.stdout.write("|Type| DVO ") + + for stage in stages: sys.stdout.write("|%19s" % stage) + sys.stdout.write("|\n") + sys.stdout.write("| | ") + for stage in stages: sys.stdout.write("| Pend Succ Fail ") + sys.stdout.write("|\n") + self.writeTableSeparator(colCount) + pendP2Processed = self.printTableRow(stages, "P2") + pendSTProcessed = self.printTableRow(stages, "ST") + self.writeTableSeparator(colCount) + + self.logger.info("") + try: self.logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2)) + except: pass + try: self.logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST)) + except: pass + + self.checkClientStatus() + + # create a density plot of pending stuff for czartool + self.plot.createDensityPlot("P2", True) + self.plot.createDensityPlot("ST", True) + + if not self.waitForPollTime(): break + + + ''' + Formats an int to a string + ''' + def getIntAsString(self, value): + if value == 0: return "" + return str(value) + + ''' + Prints (to the log) one row of data (one row being one batch-type) + ''' + def printTableRow(self, stages, batchType): + + # get a master list of IDs in DVO for this batch type + rows = self.gpc1Db.getItemsInThisDVODbForThisStage(self.config.dvoLabel, + batchType, + self.config.minRa, + self.config.maxRa, + self.config.minDec, + self.config.maxDec) + + # we only want to IDs + allIDs = [] + for row in rows: + try: allIDs.append(row[0]) + except: pass + + #allIDs = self.ippToPspsDb.getPendingIdsForThisConfig(self.config.name) + + prevList = allIDs + numEverything = len(allIDs) + + sys.stdout.write("| %2s |%6d " % (batchType, numEverything)) + + tempFilename = batchType + '.dat' + for stage in stages: + + # get lists. Use unions with prev list to make sure the right stuff is included + success = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, 1)) & set(prevList)) + fail = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, -1)) & set(prevList) - set(success)) + pending = list(set(prevList) - set(success) - set(fail)) + + # count stuff + numSuccess = len(success) + numFail = len(fail) + numPending = len(pending) + + #if stage == 'loaded_to_datastore' and batchType == "P2": + # for n in fail: print n + + str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m " % ( + self.getIntAsString(numPending), + self.getIntAsString(numSuccess), + self.getIntAsString(numFail)) + sys.stdout.write("|%-19s" % str) + + prevList = success + + self.czarDb.insertStats(stage, self.config.dvoLabel, batchType, numPending, numSuccess, numFail) + + if stage == 'processed': numPendingProcessed = numPending + + sys.stdout.write("|\n") + + return numPendingProcessed + + ''' + Writes table separator + ''' + def writeTableSeparator(self, colCount): + + sys.stdout.write("+----+-------") + for i in range(0, colCount): sys.stdout.write("+-------------------") + sys.stdout.write("+\n") + + ''' + Safe divide function + ''' + def divide(self, num, denom): + if denom == 0: return 0 + else: return num / denom + + ''' + Prints various timing stats for this batch type + ''' + def printStats(self, batchType): + + rate = self.ippToPspsDb.countBatches(batchType, self.config.epoch, self.config.dvoLabel, "1 HOUR") / 1.0 + + + batchesInLast24Hours = self.ippToPspsDb.countBatches(batchType, self.config.epoch, self.config.dvoLabel, "24 HOUR") + detectionsInLast24Hours = self.ippToPspsDb.getTotalDetectionsPublished(batchType, self.config.epoch, self.config.dvoLabel, "24 HOUR") + + self.logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" % + (batchType, + rate, + batchesInLast24Hours, + self.ippToPspsDb.countBatches(batchType, self.config.epoch, self.config.dvoLabel, "1 WEEK") / 7.0, + self.divide(detectionsInLast24Hours, batchesInLast24Hours), + self.ippToPspsDb.getTimeOfLastBatchPublished(batchType, self.config.epoch, self.config.dvoLabel))) + + return rate + + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Metrics, self).printUsage("[]") + + ''' Start of program. ''' -if len(sys.argv) > 1: CONFIG = sys.argv[1] -else: - print "** Usage: " + sys.argv[0] + " [hours]" - sys.exit(1) - -if len(sys.argv) > 2: - HOURS = float(sys.argv[2]) - SECONDS = HOURS * 60.0 * 60.0 -else: - SECONDS = None - - - -# open config file -configDoc = ElementTree(file=CONFIG) - -logging.setLoggerClass(PSLogger) -logger = logging.getLogger("metrics") -logger.setup(configDoc, "metrics") - -# create database objects -ippToPspsDb = IppToPspsDb(logger, configDoc) -czarDb = CzarDb(logger, configDoc) -gpc1Db = Gpc1Db(logger, configDoc) - -DVOLABEL = configDoc.find("dvo/gpc1Label").text -EPOCH = configDoc.find("options/epoch").text - -while True: - - logger.infoTitle("ippToPsps loading summary") - now = datetime.datetime.now() - logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S")) - logger.infoPair("Loading epoch", EPOCH) - logger.infoPair("DVO label", DVOLABEL) - - # get equatorial coord constraints, if any - MINRA = None - MAXRA = None - MINDEC = None - MAXDEC = None - - try: - MINRA = float(configDoc.find("dvo/minRA").text) - MAXRA = float(configDoc.find("dvo/maxRA").text) - MINDEC = float(configDoc.find("dvo/minDec").text) - MAXDEC = float(configDoc.find("dvo/maxDec").text) - logger.infoPair("Coordinates", "%.1f to %.1f RA, %.1f to %.1f Dec" % (MINRA, MAXRA, MINDEC, MAXDEC)) - except: - pass - - - logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") - logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |") - logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") - rateP2 = printStats("P2") - rateST = printStats("ST") - logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") - - logger.info("") - - stages = ippToPspsDb.getStages() - colCount = len(stages) - writetableSeparator(colCount) - sys.stdout.write("|Type| DVO ") - - for stage in stages: sys.stdout.write("|%19s" % stage) - sys.stdout.write("|\n") - sys.stdout.write("| | ") - for stage in stages: sys.stdout.write("| Pend Succ Fail ") - sys.stdout.write("|\n") - writetableSeparator(colCount) - pendP2Processed = printTableRow(stages, "P2") - pendSTProcessed = printTableRow(stages, "ST") - writetableSeparator(colCount) - - logger.info("") - try: logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2)) - except: pass - try: logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST)) - except: pass - - if SECONDS: - logger.infoPair("Waiting for", "%f hours" % HOURS) - time.sleep(SECONDS) - else: break - - +metrics = Metrics(sys.argv) +metrics.run() +metrics.exitProgram("finished") + + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/mysql.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/mysql.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/mysql.py (revision 33638) @@ -19,18 +19,18 @@ ''' Constructor - - ''' - def __init__(self, logger, doc, dbType): + ''' + def __init__(self, logger, config, dbType, dbName=None): # set up logging self.logger = logger - self.doc = doc + self.config = config self.logger.debug("MySql class constructor") # open config and grab database parameters - self.dbName = self.doc.find(dbType +"/name").text - self.dbHost = self.doc.find(dbType +"/host").text - self.dbUser = self.doc.find(dbType +"/user").text - self.dbPass = self.doc.find(dbType +"/password").text + if not dbName: self.dbName = config.getDbName(dbType) + else: self.dbName = dbName + self.dbHost = config.getDbHost(dbType) + self.dbUser = config.getDbUser(dbType) + self.dbPass = config.getDbPassword(dbType) # set up JDBC connection @@ -42,8 +42,10 @@ except: self.logger.error("Unable to connect to " + self.url) - self.everythingOK = False + self.everythingOK = False # TODO need this? + self.connected = False return self.everythingOK = True + self.connected = True ''' @@ -52,4 +54,5 @@ def disconnect(self): self.con.close() + self.connected = False ''' @@ -80,9 +83,28 @@ connectionID = self.getLastConnectionID() if connectionID == self.connectionID: - self.logger.error("NOT going to kill THIS connection ID") + self.logger.debug("NOT going to kill THIS connection ID") return sql = "KILL %d" % connectionID self.execute(sql) + + ''' + Checks whether there are any other connections to this database + ''' + def anyOtherConnections(self): + + sql = "SELECT COUNT(*) \ + FROM INFORMATION_SCHEMA.PROCESSLIST \ + WHERE DB='" + self.dbName + "'" + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) > 1: return True + else: return False + except: + self.logger.errorPair("Could not check for connection", sql) + pass + + return True ''' @@ -108,4 +130,24 @@ ''' + OPTIMIZEs a table + ''' + def optimizeTable(self, table): + + sql = "OPTIMIZE TABLE " + table + try: self.execute(sql) + except: return + + ''' + Creates a MySQL-friendly table name from the provided string + ''' + def getDbFriendlyTableName(self, origName): + + name = origName + name = name.replace('.', '_') + name = name.replace('/', '_') + + return name + + ''' Drops a table ''' @@ -141,4 +183,24 @@ except: pass #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'") + ''' + Changes the database engine for this table to InnoDb + ''' + def changeEngineToInnoDB(self, table): + self.changeEngine(table, "InnoDB") + + ''' + Changes the database engine for this table + ''' + def changeEngine(self, table, engine): + + sql = "ALTER TABLE " + table + " ENGINE=" + engine + try: + self.execute(sql) + except: + self.logger.errorPair("Could not change table engine", sql) + return False + + return True + ''' Adds an index to the supplied table and column @@ -244,5 +306,5 @@ sql="DELETE FROM " + tableName + " WHERE " + columnName + " IS NULL" self.execute(sql) - self.logger.debugPair("NULL %s values" % columnName, "%d deleted" % nBad) + self.logger.debugPair("NULL %s values in %s" % (columnName, tableName), "%d deleted" % nBad) return nBad Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/odm.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/odm.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/odm.py (revision 33638) @@ -2,4 +2,6 @@ import re +import os +import signal import sys import tempfile @@ -17,8 +19,7 @@ Constructor ''' - def __init__(self, logger, doc): + def __init__(self, logger): self.logger = logger - self.doc = doc ''' @@ -33,7 +34,8 @@ p = Popen("../perl/checkOdmStatus.pl -b " + batch + " -f " + tempFile.name, shell=True, - stdout=PIPE) + stdout=PIPE, + close_fds=True) p.wait() - + if p.returncode != 0: self.logger.debugPair("ODM polling", "failed") @@ -46,4 +48,5 @@ tempFile.close() p.stdout.close() + if not ret: self.logger.debugPair("Cannot read ODM file for batch", "%d" % batchID) @@ -59,5 +62,4 @@ loadedToODM = 0 - loadFailed = 0 mergeWorthy = 0 merged = 0 @@ -73,6 +75,5 @@ # batches to have there state changed if re.search("failed", STATE) or re.search("OnHold", MESSAGE): - loadedToODM = 1 - loadFailed = 1 + loadedToODM = -1 DETAILS = doc.find("{%s}OdmBatchState/{%s}Details" % (NAMESPACE, NAMESPACE)).text results['DETAILS'] = DETAILS @@ -87,8 +88,7 @@ merged = 1 except: - return 0 + pass results['LOADEDTOODM'] = loadedToODM - results['LOADFAILED'] = loadFailed results['MERGEWORTHY'] = mergeWorthy results['MERGED'] = merged Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/plot.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/plot.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/plot.py (revision 33638) @@ -0,0 +1,65 @@ +#!/usr/bin/env jython + +import tempfile +import os +import time + +from ipptopsps import IppToPsps + +''' +Plot class +''' +class Plot(object): + + ''' + Constructor + ''' + def __init__(self, logger, config, ippToPspsDb): + + self.logger = logger + self.config = config + self.ippToPspsDb = ippToPspsDb + + ''' + Creates a density plot of pending exposures for this config + ''' + def createDensityPlot(self, batchType, forCzartool=False): + + tempFilename = "./" + self.config.name + "_" + batchType + "_plotData.dat" + DATFILE = open(tempFilename,'w') + max = self.ippToPspsDb.createPendingDensityPlotData(batchType, DATFILE) + DATFILE.close() + + if max == 0: max = 1 + + timestamp = time.strftime('%Y_%m%d_%H%M%S') + + if forCzartool: + OUTPUTFILE = self.config.czarPlotsPath + "/ippToPsps_density_" + self.config.dvoLabel + "_" + batchType + ".png" + else: + OUTPUTFILE = self.config.name + "_" + batchType + "_" + timestamp + ".png" + + f = os.popen('gnuplot', 'w') + + if 0: + TERM = "X11" + else: + TERM = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" + + print >> f, "set term " + TERM + "; \ + set output \"" + OUTPUTFILE + "\"; \ + set title \"Unprocessed " + batchType + " items for " + self.config.dvoLabel + " as of " + timestamp + "\"; \ + set grid; \ + set xrange [" + str(self.config.minRa) + ":" + str(self.config.maxRa) + "] reverse; \ + set yrange [" + str(self.config.minDec) + ":" + str(self.config.maxDec) + "]; \ + unset key; \ + set palette rgb 22,13,10; \ + set ylabel \"Dec\"; \ + set ylabel \"RA\"; \ + set cbrange [0:" + str(max) + "]; \ + set datafile missing \"NaN\"; \ + plot '" + tempFilename + "' using 1:2:3 with image" + + f.flush() + #os.remove(tempFilename) + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/plotter.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/plotter.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/plotter.py (revision 33638) @@ -0,0 +1,41 @@ +#!/usr/bin/env jython + +import sys + +from ipptopsps import IppToPsps +from plot import Plot + + +''' +Plotter class +''' +class Plotter(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(Plotter, self).__init__(argv) + + self.plot = Plot(self.logger, self.config, self.ippToPspsDb) + + if len(argv) < 3: + self.printUsage() + self.exitProgram("No plot type specified") + + if argv[2] == "density": + self.plot.createDensityPlot("P2") + self.plot.createDensityPlot("ST") + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Plotter, self).printUsage("[density]") + + +''' +Start of program +''' +plotter = Plotter(sys.argv) +plotter.exitProgram("finished") Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/pollOdm.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/pollOdm.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/pollOdm.py (revision 33638) @@ -8,116 +8,124 @@ import time -import logging.config -from pslogger import PSLogger -from ipptopspsdb import IppToPspsDb +from ipptopsps import IppToPsps +from config import Config from odm import Odm from batch import Batch + ''' -Method to actually poll ODM +PollOdm class ''' -def poll(batchType, stage): - - if stage == "unloaded": - ids = ippToPspsDb.getUnloadedBatchIDs(EPOCH, DVOLABEL, batchType) - elif stage == "unmergeworthy": - ids = ippToPspsDb.getUnmergeWorthyBatchIDs(EPOCH, DVOLABEL, batchType) - elif stage == "unmerged": - ids = ippToPspsDb.getUnmergedBatchIDs(EPOCH, DVOLABEL, batchType) +class PollOdm(IppToPsps): - logger.infoTitle("Checking %d %s %s batches" % (len(ids), stage, batchType)) + ''' + Constructor + ''' + def __init__(self, argv): + super(PollOdm, self).__init__(argv) - results = {} - newLoaded = 0 - newMergeWorthy = 0 - newMerged = 0 + # get batch and stage args + if len(sys.argv) > 3: + self.BATCHTYPE = sys.argv[2] + self.STAGE = sys.argv[3] + else: + self.printUsage() + self.exitProgram("incorrect args") - logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+") - logger.info("| Batch | Loaded to ODM | Load failed | Merge worthy | Merged | New loaded |New merge worthy| New merged |") - logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+") - for id in ids: + # create Odm object + self.odm = Odm(self.logger) - if odm.checkBatch(id, results): - ippToPspsDb.updateOdmStatus(id, results) + # get poll time arg + if len(sys.argv) > 4: self.parsePollTimeArg(sys.argv[4]) - if stage == "unloaded": - if results['LOADEDTOODM'] == 1: newLoaded = newLoaded + 1 - if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1 - if results['MERGED'] == 1: newMerged = newMerged + 1 + if self.STAGE == "all": self.stages = ['unloaded', 'unmergeworthy', 'unmerged'] + else: self.stages = [self.STAGE] - elif stage == "unmergeworthy": - if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1 - if results['MERGED'] == 1: newMerged = newMerged + 1 + if self.BATCHTYPE == "all": self.batchTypes = ['P2', 'ST'] + else: self.batchTypes = [self.BATCHTYPE] - elif stage == "unmerged": - if results['MERGED'] == 1: newMerged = newMerged + 1 + for stage in self.stages: self.logger.infoPair("Checking stage", stage) + for batchType in self.batchTypes: self.logger.infoPair("Checking batch type", batchType) - logger.info("| %9s | %14d | %14d | %14d | %14d | %14d | %14d | %14d |" % - (Batch.getNameFromID(id), - results['LOADEDTOODM'], - results['LOADFAILED'], + ''' + Run... + ''' + def run(self): + + while True: + + for stage in self.stages: + for batchType in self.batchTypes: + self.poll(batchType, stage) + + self.checkClientStatus() + if not self.waitForPollTime(): break + + ''' + Method to actually poll ODM + ''' + def poll(self, batchType, stage): + + if stage == "unloaded": + ids = self.ippToPspsDb.getUnloadedBatchIDs(self.config.epoch, self.config.dvoLabel, batchType) + elif stage == "unmergeworthy": + ids = self.ippToPspsDb.getUnmergeWorthyBatchIDs(self.config.epoch, self.config.dvoLabel, batchType) + elif stage == "unmerged": + ids = self.ippToPspsDb.getUnmergedBatchIDs(self.config.epoch, self.config.dvoLabel, batchType) + + self.logger.infoTitle("Checking %d %s %s batches" % (len(ids), stage, batchType)) + + results = {} + newLoaded = 0 + newMergeWorthy = 0 + newMerged = 0 + + self.logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+") + self.logger.info("| Batch | Loaded to ODM | Merge worthy | Merged | New loaded |New merge worthy| New merged |") + self.logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+") + for id in ids: + + if self.odm.checkBatch(id, results): + self.ippToPspsDb.updateOdmStatus(id, results) + + if stage == "unloaded": + if results['LOADEDTOODM'] == 1: newLoaded = newLoaded + 1 + if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1 + if results['MERGED'] == 1: newMerged = newMerged + 1 + + elif stage == "unmergeworthy": + if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1 + if results['MERGED'] == 1: newMerged = newMerged + 1 + + elif stage == "unmerged": + if results['MERGED'] == 1: newMerged = newMerged + 1 + + self.logger.info("| %9s | %14d | %14d | %14d | %14d | %14d | %14d |" % + (Batch.getNameFromID(id), + results['LOADEDTOODM'], results['MERGEWORTHY'], - results['MERGED'], - newLoaded, - newMergeWorthy, - newMerged)) + results['MERGED'], + newLoaded, + newMergeWorthy, + newMerged)) + + + else: + self.logger.info("| %9s | failed to poll ODM |" % Batch.getNameFromID(id)) + + self.logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+") + - - else: - logger.info("| %9s | failed to poll ODM |" % Batch.getNameFromID(id)) - - logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+") - - + ''' + Overrides base-class version + ''' + def printUsage(self): + super(PollOdm, self).printUsage(" [hours]") + + ''' Program starts here ''' -if len(sys.argv) > 3: - CONFIG = sys.argv[1] - BATCHTYPE = sys.argv[2] - STAGE = sys.argv[3] -else: - print "\n** Usage: " + sys.argv[0] + " [hours]\n" - sys.exit(1) - -if len(sys.argv) > 4: - HOURS = float(sys.argv[4]) - SECONDS = HOURS * 60.0 * 60.0 -else: - SECONDS = None - -# open config file -configDoc = ElementTree(file=CONFIG) - -# set up logging -logging.setLoggerClass(PSLogger) -logger = logging.getLogger(sys.argv[0]) -logger.setup(configDoc, sys.argv[0]) - -ippToPspsDb = IppToPspsDb(logger, configDoc) -odm = Odm(logger, configDoc) - -DVOLABEL = configDoc.find("dvo/gpc1Label").text -EPOCH = configDoc.find("options/epoch").text - -if STAGE == "all": stages = ['unloaded', 'unmergeworthy', 'unmerged'] -else: stages = [STAGE] - -if BATCHTYPE == "all": batchTypes = ['P2', 'ST'] -else: batchTypes = [BATCHTYPE] - -for stage in stages: logger.infoPair("Checking stage", stage) -for batchType in batchTypes: logger.infoPair("Checking batch type", batchType) - -while True: - - for stage in stages: - for batchType in batchTypes: - poll(batchType, stage) - - if SECONDS: - logger.infoPair("Waiting for", "%f hours" % HOURS) - time.sleep(SECONDS) - else: break - - +pollodm = PollOdm(sys.argv) +pollodm.run() +pollodm.exitProgram("finished") Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/pslogger.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/pslogger.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/pslogger.py (revision 33638) @@ -3,11 +3,13 @@ import os import sys +import socket class PSLogger(logging.getLoggerClass()): ''' - Sets up logging using values from the ippToPsps config file + Sets up logging using provided values and this process hostname and PID if sending to file + This will default to stout and no file output ''' - def setup(self, doc, name, stdout=1, sendToFile=0, host="host", pid=1234): + def setup(self, programName, basePath, configName, host, pid, stdout=1, sendToFile=0): formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S') @@ -15,9 +17,9 @@ if sendToFile: - PATH = doc.find("localOutPath").text + "/log" + + # generate path + PATH = basePath + "/log" if not os.path.exists(PATH): os.makedirs(PATH) - DVOLABEL = doc.find("dvo/gpc1Label").text - FULLPATH = PATH + "/" + name + "_" + DVOLABEL + "_" + host + "_" + str(pid) + ".log" - + FULLPATH = PATH + "/" + programName + "_" + configName + "_" + host + "_" + str(pid) + ".log" # opens file to be appended hdlr = logging.FileHandler(FULLPATH, "a") @@ -27,5 +29,5 @@ if stdout: stdout = logging.StreamHandler(sys.__stdout__) - #stdout.setFormatter(formatter) + stdout.setFormatter(formatter) self.addHandler(stdout) Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/queue.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/queue.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/queue.py (revision 33638) @@ -0,0 +1,152 @@ +#!/usr/bin/env jython + +# +# The main loading program for ippToPsps. See usage below. +# + +import time +import sys +import os +import logging.config + +from ipptopsps import IppToPsps +from gpc1db import Gpc1Db +from datastore import Datastore +from batch import Batch + +''' +Queue class +''' +class Queue(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(Queue, self).__init__(argv) + + + # create various objects + self.gpc1Db = Gpc1Db(self.logger, self.config) + self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + + self.config.printAll() + + if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) + + + ''' + Main processing loop. + Tiled area is defined in config and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing + by one or more loading clients that may be running on any host + ''' + def run(self): + + # this outer while loop simply waits a few minutes then starts queuing again (as more stuff may appear in DVO over time) + while True: + + self.ippToPspsDb.removeAllBoxes() + + # queue up batches that are processed but not loaded to datastore + for batchType in self.config.batchTypes: + self.logger.infoTitle("Previous failed datastore loads") + self.publishAnyUnpublishedBatches(batchType) + + # get totals for whole area to check if there is anything to do + processedIDs = self.ippToPspsDb.getProcessedIDs(batchType, self.config.epoch, self.config.dvoLabel) + consistantlyFailedIDs = self.ippToPspsDb.getConsistentlyFailedIDs(batchType, self.config.epoch, self.config.dvoLabel) + rows = self.gpc1Db.getItemsInThisDVODbForThisStage( + self.config.dvoLabel, + batchType, + self.config.minRa, + self.config.maxRa, + self.config.minDec, + self.config.maxDec) + + # first report total stuff + allIDs = [] + for row in rows: + try: allIDs.append(row[0]) + except: pass + + ids = list(set(allIDs) - set(processedIDs) - set(consistantlyFailedIDs)) + self.logger.infoPair("All unprocessed items", "%d" % len(ids)) + + # now trim full list down to only pending stuff + pending = [[], [], []] + for row in rows: + try: + findId = row[0] + except: continue + if findId in ids: pending.append([row[0], row[1], row[2]]) + + # store all items in temp table + self.logger.infoPair("Inserting all pending items into", "ippToPsps database") + self.ippToPspsDb.storeAllItems(pending) + + # loop through full range of RA/Dec queueing stuff in boxes of size self.config.boxSize + if len(ids) > 0: + + self.logger.info("+-------------+-------------+-------------+-------------+") + self.logger.info("| RA | Dec | box size | unprocessed |") + self.logger.info("+-------------+-------------+-------------+-------------+") + + # starting positions + ra = self.config.minRa + self.config.halfBox + dec = self.config.minDec + self.config.halfBox + + while ra <= self.config.maxRa: + while dec <= self.config.maxDec: + + self.checkClientStatus() + box_id = self.ippToPspsDb.insertBox(ra, dec) + + ids = self.ippToPspsDb.getItemsInThisThisBox(ra, dec) + + self.logger.info("| %5.1f | %5.1f | %9d | %9d |" % ( + ra, + dec, + self.config.boxSize, + len(ids))) + + if len(ids) > 0: self.ippToPspsDb.insertPending(box_id, batchType, ids) + + dec = dec + self.config.boxSize + dec = self.config.minDec + self.config.halfBox + ra = ra + self.config.boxSize + + self.logger.info("+-------------+-------------+-------------+-------------+") + + if not self.waitForPollTime(): break + + ''' + Finds batches that processed ok, but did not get loaded to the datastore, then tries to load them + ''' + def publishAnyUnpublishedBatches(self, batchType): + + batchIDs = self.ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(self.config.epoch, self.config.dvoLabel, batchType) + self.logger.infoPair("%s batches" % batchType, "%d" % len(batchIDs)) + + for batchID in batchIDs: + + batchName = Batch.getNameFromID(batchID) + subDir = Batch.getSubDir(self.config.basePath, batchType, self.config.dvoLabel) + tarballFile = Batch.getTarballFile(batchID) + self.logger.infoPair("Batch name", batchName) + + Batch.publishToDatastore(self.datastore, batchID, batchName, subDir, tarballFile) + + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Cleanup, self).printUsage("[]") + + +''' +Start of program. +''' +queue = Queue(sys.argv) +queue.run() +queue.exitProgram("finished") Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/regionTest.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/regionTest.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/regionTest.py (revision 33638) @@ -0,0 +1,96 @@ +from pslogger import PSLogger +from ipptopspsdb import IppToPspsDb +from gpc1db import Gpc1Db +from dvo import Dvo + +import sys +from xml.etree.ElementTree import ElementTree, Element, tostring +import logging + +''' +Start of program. +''' + +if len(sys.argv) > 1: CONFIG = sys.argv[1] +else: + print "** Usage: " + sys.argv[0] + " " + sys.exit(1) + +# open config file +configDoc = ElementTree(file=CONFIG) + +logging.setLoggerClass(PSLogger) +logger = logging.getLogger("regionTest") +logger.setup(configDoc, "regionTest") + + +gpc1Db = Gpc1Db(logger, configDoc) +dvo = Dvo(logger, configDoc) +#dvo.resetAllTables() + +DVOLABEL = configDoc.find("dvo/gpc1Label").text + +# get equatorial coord constraints, if any +try: + MINRA = float(configDoc.find("dvo/minRA").text) + MAXRA = float(configDoc.find("dvo/maxRA").text) + MINDEC = float(configDoc.find("dvo/minDec").text) + MAXDEC = float(configDoc.find("dvo/maxDec").text) +except: + MINRA = 0.0 + MAXRA = 360.0 + MINDEC = -30.0 + MAXDEC = 90.0 + +BORDER = 1.60 +#BOXSIZE = 4 +BOXSIZE = 1.0 +HALFBOX = BOXSIZE/2.0 + + + +ra = MINRA + HALFBOX +dec = MINDEC + HALFBOX + +while ra <= MAXRA: + while dec <= MAXDEC: + + logger.infoSeparator() + logger.infoPair("BOXSIZE", "%d" % BOXSIZE) + allIDs = gpc1Db.getIDsInThisDVODbForThisStageInThisBox(DVOLABEL, "P2", ra, dec, BOXSIZE) + logger.infoPair("Found", "%d exposures" % len(allIDs)) + + dvo.loadRegionInThisBoxWithBorder(ra, dec, BOXSIZE, BORDER) + + ''' + regionsToIngest = [] + regionsToPurge = [] + + dvo.getRegionsInThisBoxWithBorder(regionsToIngest, regionsToPurge, ra, dec, BOXSIZE, BORDER) + + size = dvo.getDiskSizeOfDvoRegions(regionsToIngest)/1073741824.0 + if size == 0: smfsPerGB = 0 + else: smfsPerGB = len(allIDs)/size + logger.infoPair("smf files per GB", "%f" % smfsPerGB) + ''' + dec = dec + BOXSIZE + break + ra = ra + BOXSIZE + break + +#regionsToIngest = [] +#regionsToPurge = [] +#dvo.getRegions(regionsToIngest, regionsToPurge) +#dvo.getRegions(regionsToIngest, regionsToPurge, MINRA-BORDER, MAXRA+BORDER, MINDEC-BORDER, MAXDEC+BORDER) +#size = dvo.getSizeOfDvoRegionFromFiles(files) +#logger.infoPair("Size of region in DVO", dvo.getFileSizeStr(size)) +#logger.infoPair("Average region file size", dvo.getFileSizeStr(size/len(files))) + +#dvo.loadRegion(329, 331.1, 1.4, 2) +#dvo.loadRegion(MINRA-BORDER, MAXRA+BORDER, MINDEC-BORDER, MAXDEC+BORDER) +#dvo.loadRegion() + +logger.infoPair("Program...", "complete") + + + Index: anches/eam_branches/ipp-20111122/ippToPsps/jython/removeFromDatastore.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/removeFromDatastore.py (revision 33637) +++ (revision ) @@ -1,33 +1,0 @@ -#!/usr/bin/env jython - -from datastore import Datastore -import logging -import sys -import getopt - -from xml.etree.ElementTree import ElementTree, Element, tostring - -from pslogger import PSLogger -from ipptopspsdb import IppToPspsDb - - - -if len(sys.argv) < 3: - print "** Usage: " + sys.argv[0] + " []" - sys.exit(1) - -CONFIG = sys.argv[1] -configDoc = ElementTree(file=CONFIG) - -logging.setLoggerClass(PSLogger) -logger = logging.getLogger("removeFromDatastore") -logger.setup(configDoc, "removeFromDatastore") - -ippToPspsDb = IppToPspsDb(logger, configDoc) -datastore = Datastore(logger, configDoc, ippToPspsDb) - -if len(sys.argv) < 4: - datastore.remove(str(sys.argv[2])) -else: - datastore.removeRange(str(sys.argv[2]), str(sys.argv[3])) - Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/scratchdb.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/scratchdb.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/scratchdb.py (revision 33638) @@ -19,24 +19,33 @@ Constructor ''' - def __init__(self, logger, doc, useFull=0): - super(ScratchDb, self).__init__(logger, doc, "localdatabase") + def __init__(self, logger, config, dbName=None): + super(ScratchDb, self).__init__(logger, config, "localdatabase", dbName) + + self.dvoDoneTable = "dvoDone" + self.dvoSkyTable = "dvoSkyTable" + + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("ScratchDb destructor") + + + ''' + Sets whether we are using normal or pre-ingested DVO tables + ''' + def setUseFullTables(self, useFull): if useFull: - self.dvoMetaTable = "dvoMetaFull" + self.dvoImagesTable = "dvoImagesFull" self.dvoDetectionTable = "dvoDetectionFull" - self.dvoDoneTable = "dvoDone" else: - self.dvoMetaTable = "dvoMeta" + self.dvoImagesTable = "dvoImages" self.dvoDetectionTable = "dvoDetection" - self.logger.debugPair("Using DVO meta table", self.dvoMetaTable) + self.logger.debugPair("Using DVO images table", self.dvoImagesTable) self.logger.debugPair("Using DVO detection table", self.dvoDetectionTable) - - ''' - Destructor - ''' - def __del__(self): - - self.logger.debug("ScratchDb destructor") ''' @@ -112,9 +121,9 @@ Gets imageID from extern ID ''' - def getImageIDFromExternID(self, sourceID, externID): + def getImageIDFromExternID(self, externID): imageID = -1 - sql = "SELECT imageID FROM " + self.dvoMetaTable + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + sql = "SELECT IMAGE_ID FROM " + self.dvoImagesTable + " WHERE EXTERN_ID = %s" % (externID) try: rs = self.executeQuery(sql) @@ -129,9 +138,9 @@ Gets DVO image flags ''' - def getDvoImageFlags(self, sourceID, externID): + def getDvoImageFlags(self, externID): flags = 0 - sql = "SELECT flags FROM " + self.dvoMetaTable + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + sql = "SELECT FLAGS FROM " + self.dvoImagesTable + " WHERE EXTERN_ID = %s" % (externID) try: rs = self.executeQuery(sql) @@ -145,9 +154,9 @@ Gets photcode (aka photoCalID from dvo table) ''' - def getPhotoCalID(self, sourceID, externID): + def getPhotoCalID(self, externID): photcode = -1 - sql = "SELECT photcode FROM " + self.dvoMetaTable + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + sql = "SELECT PHOTCODE FROM " + self.dvoImagesTable + " WHERE EXTERN_ID = %s" % (externID) try: rs = self.executeQuery(sql) @@ -168,11 +177,11 @@ ''' - Inserts a new sourceID/imageID combo into dvoMeta + Inserts a new sourceID/imageID combo into dvoImages ''' def insertNewDvoExternID(self, sourceID, externID): - sql = "INSERT INTO dvoMeta ( \ - sourceID, \ - externID \ + sql = "INSERT INTO " + self.dvoImagesTable + " ( \ + SOURCE_ID, \ + EXTERN_ID \ ) VALUES (\ " + str(sourceID) + ", \ @@ -182,11 +191,154 @@ ''' + Do we have an out-of-date version of this file? + ''' + def haveOutOfDateVersionOfThisDvoTable(self, path): + + fileStat = os.stat(path) + + sql = "SELECT COUNT(*) FROM " + self.dvoDoneTable + " WHERE path = '" + path + "' AND modifiedDate != " + str(fileStat.st_mtime) + + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) > 0: + self.logger.debugPair("Have out-to-date version of", path) + return True + else: + return False + except: + self.logger.exception("Unable to check whether this DVO table is out-of-date") + + ''' + Have we already imported this DVO table? + Checks path and modified date to be sure + ''' + def alreadyImportedThisDvoTable(self, path): + + fileStat = os.stat(path) + + sql = "SELECT COUNT(*) FROM " + self.dvoDoneTable + " WHERE path = '" + path + "' AND modifiedDate = " + str(fileStat.st_mtime) + + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) > 0: + self.logger.debugPair("Already imported up-to-date version of", path) + return True + else: + return False + except: + self.logger.exception("Unable to check whether this DVO table has been imported") + + + ''' Updates dvoDone table with this DVO table ''' - def setImportedThisDvoTable(self, name): - - sql = "INSERT INTO dvoDone (name) VALUES ('" + name + "')" + def setImportedThisDvoTable(self, path, region="null"): + + fileStat = os.stat(path) + + # first delete any old version we have ingested + sql = "DELETE FROM " + self.dvoDoneTable + " WHERE path = '" + path + "'" self.execute(sql) - + + # now insert new version with up-to-date size and date + sql = "INSERT INTO " + self.dvoDoneTable + " (region, path, modifiedDate, size) VALUES ('" + region + "', '" + path + "', " + str(str(fileStat.st_mtime)) + ", " + str(str(fileStat.st_size)) + ")" + self.execute(sql) + + + ''' + Gets the id for the cpm file for the provided region + ''' + def getDvoCpmFileIdForRegion(self, region): + + id = -1 + sql = "SELECT id FROM " + self.dvoDoneTable + " WHERE region = '" + region + "' AND path LIKE '%.cpm'"; + try: + rs = self.executeQuery(sql) + rs.first() + id = rs.getInt(1) + except: + self.logger.errorPair("Could not get file id from " + self.dvoDoneTable, "for region: " + region) + + return id + + + ''' + Gets the id for the provided file + ''' + def getDvoFileId(self, path): + + id = -1 + sql = "SELECT id FROM " + self.dvoDoneTable + " WHERE path = '" + path + "'" + try: + rs = self.executeQuery(sql) + rs.first() + id = rs.getInt(1) + except: + self.logger.errprPair("Could not get file id from " + self.dvoDoneTable + "for", path) + + return id + + ''' + Returns a list of DVO regions currently ingested + ''' + def getIngestedDvoRegions(self): + + regions = [] + + sql = "SELECT DISTINCT region FROM dvoDone WHERE region != 'null'" + try: + rs = self.executeQuery(sql) + while (rs.next()): + regions.append(rs.getString(1)) + rs.close() + except: + self.logger.errorPai("Can't query for ingested regions", sql) + + return regions + + + ''' + Gets total size of DVO files imported to this database + Counts only cpm and cpt files + ''' + def getTotalSizeOfIngestedDvoFiles(self): + + size = 0 + sql = "SELECT SUM(size) FROM " + self.dvoDoneTable + " WHERE path LIKE '%cpm' OR path LIKE '%cpt'" + try: + rs = self.executeQuery(sql) + rs.first() + size = rs.getLong(1) + except: + self.logger.errorPair("Unable to get total ingested DVO size", sql) + + return size + + ''' + Checks whether DVO files we have ingested are from the DVO path provided + Does this by checking the original path of the SkyTable.fits file + ''' + def isCorrectDvo(self, path): + + sql = "SELECT path FROM " + self.dvoDoneTable + " WHERE path LIKE '%SkyTable.fits'" + try: + rs = self.executeQuery(sql) + rs.first() + + head,tail = os.path.split(rs.getString(1)) + + if head == path: + self.logger.debugPair("Already ingested stuff from DVO at", path) + return True + else: + self.logger.errorPair("DVO does not match. We have", head) + return False + + except: + self.logger.errorPair("No DVO files ingested for this path", path) + return False + ''' Checks whether the astrometric solution is ok for this chip @@ -210,22 +362,4 @@ return True - ''' - Have we already imported this DVO table? - ''' - def alreadyImportedThisDvoTable(self, name): - - sql = "SELECT COUNT(*) FROM dvoDone WHERE name = '" + name + "'" - - try: - rs = self.executeQuery(sql) - rs.first() - if rs.getInt(1) > 0: - self.logger.errorPair("Already imported DVO tables for", name) - return True - else: - return False - except: - self.logger.exception("Unable to check whether this DVO table has been imported") - ''' @@ -233,22 +367,18 @@ ''' def createDvoTables(self): - - self.logger.debugPair("Creating DVO table", "dvoMeta") - - sql = "DROP TABLE dvoMeta" + + # drop and create Images table + self.logger.debugPair("Creating DVO table", self.dvoImagesTable) + sql = "DROP TABLE " + self.dvoImagesTable try: self.execute(sql) except: pass - sql = "DROP TABLE dvoDetection" - try: self.execute(sql) - except: pass - - sql = "CREATE TABLE dvoMeta ( \ - sourceID INT, \ - imageID INT, \ - externID INT, \ - flags INT, \ - photcode INT, \ - PRIMARY KEY (sourceID, imageID, externID) \ + sql = "CREATE TABLE " + self.dvoImagesTable + " ( \ + SOURCE_ID SMALLINT, \ + IMAGE_ID INT, \ + EXTERN_ID INT, \ + FLAGS INT, \ + PHOTCODE SMALLINT, \ + PRIMARY KEY (IMAGE_ID, EXTERN_ID) \ )" @@ -257,7 +387,12 @@ self.logger.error("Unable to create DVO meta-data database table") + # now detection table + self.logger.debugPair("Creating DVO table", self.dvoDetectionTable) + sql = "DROP TABLE " + self.dvoDetectionTable + try: self.execute(sql) + except: pass + self.logger.debugPair("Creating DVO table", "dvoDetection") sql = "CREATE TABLE dvoDetection ( \ - sourceID INT, \ imageID INT, \ ippDetectID BIGINT, \ @@ -266,5 +401,5 @@ objID BIGINT, \ flags INT, \ - PRIMARY KEY (sourceID, imageID, ippDetectID) \ + PRIMARY KEY (imageID, ippDetectID) \ )" @@ -275,10 +410,41 @@ ''' + Removes the provided DVO FITS files from from dvoDone and dvoDetection tables + Returns a count of the number of detections deleted + ''' + def purgeTheseDvoRegions(self, regions): + + detectionCount = self.getRowCount(self.dvoDetectionTable) + for region in regions: + + fileId = self.getDvoCpmFileIdForRegion(region) + + if fileId < 0: + self.logger.errorPair("Unable to get file ID to delete " + region + " from", self.dvoDoneTable) + continue + + sql = "DELETE FROM " + self.dvoDoneTable + " WHERE region = '" + region + "'" + try: self.execute(sql) + except: + self.logger.errorPair("Unable to delete " + region + " from", self.dvoDoneTable) + + finalCount = detectionCount - self.getRowCount(self.dvoDetectionTable) + + # after a big delete, we should OPTIMIZE the table + self.logger.infoPair("Running OPTIMIZE on ", self.dvoDetectionTable) + if finalCount > 0: self.optimizeTable(self.dvoDetectionTable) + + return finalCount + + ''' Drops and recreates tables necessary for dvoToMySQL program. Be very careful before using this... ''' - def resetDvoToMysqlTables(self): - - self.logger.infoPair("Dropping table", self.dvoMetaTable) - self.dropTable(self.dvoMetaTable) + def resetAllDvoTables(self): + + # drop all dvo tables - NB detections have to be deleted prior to dvoDone due to foreign key constraint + self.logger.infoPair("Dropping table", self.dvoImagesTable) + self.dropTable(self.dvoImagesTable) + self.logger.infoPair("Dropping table", self.dvoSkyTable) + self.dropTable(self.dvoSkyTable) self.logger.infoPair("Dropping table", self.dvoDetectionTable) self.dropTable(self.dvoDetectionTable) @@ -286,19 +452,79 @@ self.dropTable(self.dvoDoneTable) - self.logger.infoPair("Creating table", self.dvoMetaTable) - sql = "CREATE TABLE " + self.dvoMetaTable + " LIKE dvoMeta" + # create dvoDone table + self.logger.infoPair("Creating table", self.dvoDoneTable) + sql = "CREATE TABLE " + self.dvoDoneTable + " (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, region VARCHAR(100), path VARCHAR(1000), modifiedDate BIGINT, size BIGINT)" try: self.execute(sql) - except: self.logger.errorPair("Unable to create table", self.dvoMetaTable) - + except: + self.logger.errorPair("Unable to create table", self.dvoDoneTable) + return False + + self.changeEngineToInnoDB(self.dvoDoneTable) + + # create detections table self.logger.infoPair("Creating table", self.dvoDetectionTable) sql = "CREATE TABLE " + self.dvoDetectionTable + " LIKE dvoDetection" try: self.execute(sql) - except: self.logger.errorPair("Unable to create table", self.dvoDetectionTable) - - self.logger.infoPair("Creating table", self.dvoDoneTable ) - sql = "CREATE TABLE " + self.dvoDoneTable + " (name VARCHAR(100))" + except: + self.logger.errorPair("Unable to create table", self.dvoDetectionTable) + return False + self.changeEngineToInnoDB(self.dvoDetectionTable) + + # add fileID column + sql = "ALTER TABLE " + self.dvoDetectionTable + " ADD fileID INT NOT NULL" try: self.execute(sql) - except: self.logger.errorPair("Unable to create table", self.dvoDoneTable) - - - + except: + self.logger.errorPair("Unable to add fileID column to ", self.dvoDetectionTable) + return False + + # add an index to the fileID column + self.createIndex(self.dvoDetectionTable, "fileID") + + # now add a delete cascading foreign key constraint on fileID + sql = "ALTER TABLE dvoDetectionFull \ + ADD CONSTRAINT fk_fileID \ + FOREIGN KEY (fileID) \ + REFERENCES dvoDone(id) \ + ON DELETE CASCADE" + try: self.execute(sql) + except: + self.logger.errorPair("Unable to create foreign key on", self.dvoDoneTable) + return False + + return True + + ''' + Gets a list of DVO regions that overlap with the defined box + ''' + def getDvoRegions(self, minRa, maxRa, minDec, maxDec): + + self.logger.debug("Querying DVO SkyTable for FITS files in this region") + + regions = [] + self.getDvoRegionsForThisBox(regions, minRa, maxRa, minDec, maxDec) + + # deal with 0/360 boundary + if minRa < 0: self.getDvoRegionsForThisBox(regions, minRa + 360, 361 ,minDec, maxDec) + if maxRa > 360: self.getDvoRegionsForThisBox(regions, 0, maxRa - 360 ,minDec, maxDec) + + return regions + + ''' + Gets a list of DVO regions that overlap with the defined box + ''' + def getDvoRegionsForThisBox(self, regions, minRa, maxRa, minDec, maxDec): + + sql = "SELECT name FROM " + self.dvoSkyTable + " \ + WHERE NOT ((" + str(maxRa) + " <= R_MIN) OR \ + (" + str(minRa) + " >= R_MAX) OR \ + (" + str(maxDec) + " <= D_MIN) OR \ + (" + str(minDec) + " >= D_MAX)) " + + try: + rs = self.executeQuery(sql) + while (rs.next()): regions.append(rs.getString(1)) + rs.close() + except: + self.logger.exception("Can't query for DVO regions") + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/setupScratchDb.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/setupScratchDb.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/setupScratchDb.py (revision 33638) @@ -1,49 +1,58 @@ #!/usr/bin/env jython -import os.path import sys -import time import stilts from java.lang import * from java.sql import * -import logging.config -from java.lang import Math -from org.apache.commons.math.special import Erf -from xml.etree.ElementTree import ElementTree, Element, tostring -from pslogger import PSLogger +from ipptopsps import IppToPsps from scratchdb import ScratchDb ''' -Start of program. +SetupScratchDb class ''' +class SetupScratchDb(IppToPsps): -if len(sys.argv) > 1: CONFIG = sys.argv[1] -else: - print "** Usage: " + sys.argv[0] + " [init]" - sys.exit(1) + ''' + Constructor + ''' + def __init__(self, argv): + super(SetupScratchDb, self).__init__(argv) -# open config file -configDoc = ElementTree(file=CONFIG) + if len(argv) < 3: + self.printUsage() + self.exitProgram("incorrect args") -# set up logging -logging.setLoggerClass(PSLogger) -logger = logging.getLogger("setupScratchDb") -logger.setup(configDoc, "setupScratchDb") + self.scratchDb = ScratchDb(self.logger, self.config, argv[2]) -scratchDb = ScratchDb(logger, configDoc) -# install stored procedure -logger.info("Installing psfLikelihood() stored procedure") -scratchDb.createPsfLikelihoodFunction() + def run(self): -# install IN tables -logger.info("Installing initialization tables") -tables = stilts.treads("../config/IN/tables.vot") -for table in tables: - logger.debug("Creating IN table: " + table.name) - table.write(scratchDb.url + '#' + table.name) + self.logger.infoSeparator() -logger.info("Done") + # install stored procedure + self.logger.infoPair("Installing", "psfLikelihood() stored procedure") + self.scratchDb.createPsfLikelihoodFunction() + # install IN tables + self.logger.infoPair("Installing", "initialization tables") + tables = stilts.treads("../config/IN/tables.vot") + for table in tables: + self.logger.debug("Creating IN table: " + table.name) + table.write(self.scratchDb.url + '#' + table.name) + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(SetupScratchDb, self).printUsage("") + +''' +Program starts here +''' +setupScratchDb = SetupScratchDb(sys.argv) +setupScratchDb.run() +setupScratchDb.exitProgram("finished") + + Index: /branches/eam_branches/ipp-20111122/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/jython/stackbatch.py (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/jython/stackbatch.py (revision 33638) @@ -30,21 +30,23 @@ def __init__(self, logger, - configPath, - configDoc, + config, gpc1Db, ippToPspsDb, + scratchDb, stackID, - batchID): + batchID, + useFullTables): super(StackBatch, self).__init__( logger, - configPath, - configDoc, + config, gpc1Db, ippToPspsDb, + scratchDb, stackID, batchID, "ST", - gpc1Db.getStackStageCmf(stackID)) + gpc1Db.getStackStageCmf(config.dvoLabel, stackID), + useFullTables) if not self.everythingOK: return @@ -60,4 +62,5 @@ self.filter = meta[0]; self.filter = self.filter[0:1] + self.filterID = self.scratchDb.getFilterID(self.filter) self.skycell = meta[1]; @@ -77,7 +80,6 @@ self.analysisVer = meta[2]; - self.expTime = gpc1Db.getStackExpTime(self.id) - - self.logger.debug("Got exp time of %d" % self.expTime) + #self.expTime = gpc1Db.getStackExpTime(self.id) + self.expTime = self.header['EXPTIME'] # delete PSPS tables @@ -281,7 +283,7 @@ " + str(self.id) + " \ ," + self.skycell + " \ - ," + str(self.scratchDb.getPhotoCalID(self.header['SOURCEID'], self.header['IMAGEID'])) + " \ + ," + str(self.scratchDb.getPhotoCalID(self.header['IMAGEID'])) + " \ ," + self.header['FPA.ZP'] + " \ - ," + str(self.expTime) + " \ + ," + self.expTime + " \ ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \ ,'" + self.safeDictionaryAccess(self.header, 'FWHM_MAJ') + "' \ @@ -304,5 +306,5 @@ self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackMeta", self.filter) - self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.config.dataRelease)) self.updateStackTypeID("StackMeta") @@ -400,5 +402,5 @@ self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackDetection", self.filter) - self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.config.dataRelease)) self.scratchDb.updateAllRows("StackDetection", "primaryF", "0") self.scratchDb.updateAllRows("StackDetection", "activeFlag", "0") @@ -442,5 +444,5 @@ # TODO temporarily loading 1st convolved fluxes into unconvolved fields self.logger.infoPair("Adding fluxes", "un-convolved") - self.updateApFlxs("", "< 7.0") + self.updateApFlxs("", "IS NULL") self.logger.infoPair("Adding fluxes", "1st convolved") self.updateApFlxs("c1", "< 7.0") @@ -458,4 +460,5 @@ ,petR90=b.PETRO_RADIUS_90 \ ,petR90Err=b.PETRO_RADIUS_90_ERR \ + ,petCf=b.PETRO_FILL \ WHERE a.ippDetectID=b.IPP_IDET" self.scratchDb.execute(sql) @@ -463,5 +466,5 @@ self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackApFlx", self.filter) - self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.config.dataRelease)) self.scratchDb.updateAllRows("StackApFlx", "primaryF", "0") self.scratchDb.updateAllRows("StackApFlx", "activeFlag", "0") @@ -497,5 +500,5 @@ self.scratchDb.updateAllRows("StackModelFit", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackModelFit", self.filter) - self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.config.dataRelease)) self.scratchDb.updateAllRows("StackModelFit", "primaryF", "0") self.scratchDb.updateAllRows("StackModelFit", "activeFlag", "0") @@ -515,5 +518,4 @@ rs.first() nMissing = rs.getInt(1) - self.logger.infoPair("Detections deleted from %s that are not in StackDetection" + table, "%5d" % nMissing) if nMissing < 1: return @@ -521,4 +523,5 @@ sql = "DELETE FROM " + table + " WHERE ippDetectID NOT IN (SELECT ippDetectID FROM StackDetection)" self.scratchDb.execute(sql) + self.logger.infoPair("Detections not in StackDetection", "%d deleted from %s" % (nMissing, table)) @@ -562,5 +565,5 @@ self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID)) - self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.config.dataRelease)) ''' @@ -582,5 +585,5 @@ self.scratchDb.updateFilterID("ObjectCalColor", self.filter) - self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.config.dataRelease)) @@ -616,12 +619,11 @@ def updateDvoIDs(self, table): - imageID = self.scratchDb.getImageIDFromExternID(self.header['SOURCEID'], self.header['IMAGEID']) + imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) self.logger.debug("Updating table '" + table + "' with DVO IDs...") sql = "UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ a.ippObjID = b.ippObjID, \ - a.stackDetectID = b.detectID, \ + a.stackDetectID = 10 * b.detectID + " + str(self.filterID) + ", \ a.objID = b.objID \ WHERE a.ippDetectID = b.ippDetectID \ - AND b.sourceID = " + self.header['SOURCEID'] + "\ AND b.imageID = " + str(imageID) self.scratchDb.execute(sql) Index: /branches/eam_branches/ipp-20111122/ippToPsps/perl/deleteFromDXLayer.pl =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/perl/deleteFromDXLayer.pl (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/perl/deleteFromDXLayer.pl (revision 33638) @@ -19,4 +19,5 @@ my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; +#my $dxWebServiceUrl = "http://166.122.172.121/cgi-bin/dxlayer/dxwebservice.cgi"; my $ua = LWP::UserAgent->new; $ua->timeout(15); @@ -28,5 +29,5 @@ my $paramNode = $doc->createElement('Param'); -$paramNode->setAttribute('name', "BatchID"); # default FirstRow is 1 +$paramNode->setAttribute('Name', "BatchID"); # default FirstRow is 1 my $text=XML::LibXML::Text->new($batchName); $paramNode->appendChild($text); #append @@ -35,4 +36,5 @@ my $xmlParams = $doc->toString; +print $xmlParams; my $response = $ua->post($dxWebServiceUrl, Index: /branches/eam_branches/ipp-20111122/ippToPsps/src/Dvo.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/src/Dvo.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/src/Dvo.c (revision 33638) @@ -35,5 +35,5 @@ mysql_query(this->mysql, "DELETE FROM dvoDetection"); - mysql_query(this->mysql, "SELECT sourceID, externID FROM dvoMeta"); + mysql_query(this->mysql, "SELECT SOURCE_ID, EXTERN_ID FROM dvoImages"); MYSQL_RES* result = mysql_store_result(this->mysql); @@ -47,7 +47,7 @@ externID = atoi(row[1]); - this->logger->print(this->logger, MSG_INFO, "Dvo", "---------------------------------------------------------------------\n"); - this->logger->print(this->logger, MSG_INFO, "Dvo", "Getting skylist from DVO database for source ID = %d and extern ID = %d\n", - sourceID, externID); + //this->logger->print(this->logger, MSG_INFO, "Dvo", "---------------------------------------------------------------------\n"); + //this->logger->print(this->logger, MSG_INFO, "Dvo", "Getting skylist from DVO database for source ID = %d and extern ID = %d\n", + // sourceID, externID); skyList = dvoSkyListByExternID(this->dvoConfig, sourceID, externID, &image); if (!skyList) { @@ -61,5 +61,5 @@ sprintf(sql, - "UPDATE dvoMeta SET imageID = %d,flags = %d, photcode = %d WHERE sourceID = %d AND externID = %d", + "UPDATE dvoImages SET IMAGE_ID = %d, FLAGS = %d, PHOTCODE = %d WHERE SOURCE_ID = %d AND EXTERN_ID = %d", image->imageID, image->flags, @@ -70,16 +70,16 @@ // now get detections - this->logger->print(this->logger, MSG_INFO, "Dvo", "Getting detections...\n"); + //this->logger->print(this->logger, MSG_INFO, "Dvo", "Getting detections...\n"); dvoDetection* dvoDetections = NULL; int32_t maxDetectionId = -1; - int32_t numDetections = + //int32_t numDetections = dvoGetDetections(skyList, image->imageID, &dvoDetections, &maxDetectionId); - this->logger->print(this->logger, MSG_INFO, "Dvo", - "Found %d detections with a max detection ID of %d\n", - numDetections, maxDetectionId); - - - this->logger->print(this->logger, MSG_INFO, "Dvo", "Inserting IDs into dvoDetection table...\n"); + //this->logger->print(this->logger, MSG_INFO, "Dvo", + // "Found %d detections with a max detection ID of %d\n", + // numDetections, maxDetectionId); + + + //this->logger->print(this->logger, MSG_INFO, "Dvo", "Inserting IDs into dvoDetection table...\n"); uint32_t zeroDvoCount = 0; @@ -91,6 +91,5 @@ else { sprintf(sql, - "INSERT INTO dvoDetection (sourceID, imageID, ippDetectID, detectID, ippObjID, objID, flags) VALUES (%d, %d, %u, %lu, %lu, %lu, %u)", - sourceID, // sourceID + "INSERT INTO dvoDetection (imageID, ippDetectID, detectID, ippObjID, objID, flags) VALUES (%d, %u, %lu, %lu, %lu, %u)", image->imageID, // imageID dvoDetections[i].meas.detID, // ippDetectID @@ -117,5 +116,5 @@ Constructor. */ -Dvo* new_Dvo(Logger* logger, const char* configFile, const char* dvoDatabase) { +Dvo* new_Dvo(Logger* logger, const char* configFile, const char* dbname, const char* dvoDatabase) { Dvo* this = (Dvo*)calloc(1, sizeof(Dvo)); @@ -130,5 +129,4 @@ char user[20]; char passwd[20]; - char dbname[20]; @@ -157,6 +155,5 @@ sprintf(name, "%s", dbParams->name); value = xmlNodeListGetString(doc, dbParams->xmlChildrenNode, 1); - if (strcmp(name, "name") == 0) sprintf(dbname, "%s", value); - else if (strcmp(name, "user") == 0) sprintf(user, "%s", value); + if (strcmp(name, "user") == 0) sprintf(user, "%s", value); else if (strcmp(name, "host") == 0) sprintf(host, "%s", value); else if (strcmp(name, "password") == 0) sprintf(passwd, "%s", value); @@ -225,13 +222,12 @@ Logger* logger = new_Logger(NULL, true); - if (argc < 3) { + if (argc < 4) { logger->print(logger, MSG_ERROR, "Dvo", "Incorrect arguments\n"); - logger->print(logger, MSG_INFO, "Dvo", "Usage: ./dvo \n"); + logger->print(logger, MSG_INFO, "Dvo", "Usage: ./dvo \n"); } else { - //Dvo * dvo = new_Dvo(logger, "/data/ipp004.0/gpc1/catdirs/ThreePi.V1"); - Dvo * dvo = new_Dvo(logger, argv[1], argv[2]); - //dvo->getDetections(dvo, 33, 8345290); + + Dvo * dvo = new_Dvo(logger, argv[1], argv[2], argv[3]); dvo->getDetections(dvo); dvo->destroy(dvo); Index: /branches/eam_branches/ipp-20111122/ippToPsps/src/Dvo.h =================================================================== --- /branches/eam_branches/ipp-20111122/ippToPsps/src/Dvo.h (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippToPsps/src/Dvo.h (revision 33638) @@ -42,5 +42,5 @@ // constructor -Dvo* new_Dvo(Logger* logger, const char* configFile, const char* dvoDatabase); +Dvo* new_Dvo(Logger* logger, const char* configFile, const char* dbname, const char* dvoDatabase); #endif // IPPTOPSPS_DVO_H Index: /branches/eam_branches/ipp-20111122/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/Makefile.am (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/Makefile.am (revision 33638) @@ -8,6 +8,4 @@ addtool_find_cam_id_dvo.sql \ addtool_find_cam_id.sql \ - addtool_find_sky_id_dvo.sql \ - addtool_find_sky_id.sql \ addtool_find_sky_id_multi_dvo.sql \ addtool_find_sky_id_multi.sql \ @@ -17,10 +15,8 @@ addtool_find_pendingexp_cam.sql \ addtool_find_pendingexp_stack.sql \ - addtool_find_pendingexp_staticsky.sql \ addtool_find_pendingexp_staticsky_multi.sql \ addtool_find_pendingmergeprocess.sql \ addtool_find_processedexp_cam.sql \ addtool_find_processedexp_stack.sql \ - addtool_find_processedexp_staticsky.sql \ addtool_find_processedexp_staticsky_multi.sql \ addtool_find_minidvodbprocessed.sql \ @@ -30,5 +26,4 @@ addtool_queue_cam_id.sql \ addtool_queue_stack_id.sql \ - addtool_queue_sky_id.sql \ addtool_queue_sky_id_multi.sql \ addtool_queue_minidvodbrun.sql \ @@ -310,4 +305,14 @@ minidvodbtool_queue_minidvodb_id.sql \ minidvodbtool_revertminidvodbcopy.sql \ + mergetool_find_minidvodb_id.sql \ + mergetool_find_mergedvodbcopy.sql \ + mergetool_find_merge_id_dvo.sql \ + mergetool_find_pendingmergeprocess.sql \ + mergetool_find_pending_merge.sql \ + mergetool_find_processed.sql \ + mergetool_queue_mergecopy_id.sql \ + mergetool_queue_merge_id.sql \ + mergetool_revertmergedvodbcopy.sql \ + mergetool_revertmergedvodbprocessed.sql \ pstamptool_completedreq.sql \ pstamptool_datastore.sql \ @@ -385,4 +390,8 @@ staticskytool_definebyquery_select_by_dg.sql \ staticskytool_definebyquery_inputs.sql \ + staticskytool_defineskycalrun.sql \ + staticskytool_pendingskycalrun.sql \ + staticskytool_revertskycal.sql \ + staticskytool_skycalresult.sql \ staticskytool_export_run.sql \ staticskytool_export_input.sql \ Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_checkminidvodbaddrun.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 33638) @@ -1,4 +1,4 @@ -- this selects the minidvodbs that have complete addRuns -SELECT minidvodb_id, minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg +SELECT minidvodb_id, minidvodb_name, minidvodb_path, state, minidbg AS minidvodb_group, (cnt - cnt2) as not_full, cnt as addRun_count FROM minidvodbRun @@ -10,5 +10,5 @@ (SELECT minidvodb_group as minidbg, minidvodb_name as minidbn, count(state) AS cnt - FROM addRun + FROM addRun where minidvodb_group = '%s' GROUP BY minidvodb_name ) AS foo1 ON minidvodb_name = minidbn Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_minidvodbrun2.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_minidvodbrun2.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_minidvodbrun2.sql (revision 33638) @@ -1,4 +1,4 @@ -- this selects the minidvodbs that have complete addRuns -SELECT minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg +SELECT minidvodb_name, minidvodb_path, state, minidbg AS minidvodb_group FROM minidvodbRun Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_pendingexp_staticsky_multi.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_pendingexp_staticsky_multi.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_pendingexp_staticsky_multi.sql (revision 33638) @@ -25,5 +25,5 @@ WHERE staticskyRun.state = 'full' - AND stage = 'staticsky_multi' + AND stage = 'staticsky' AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') AND addRun.dvodb IS NOT NULL Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id.sql (revision 33638) @@ -2,6 +2,6 @@ staticskyRun.* FROM - staticskyResult -join staticskyRun using (sky_id) + staticskyRun +join staticskyResult using (sky_id) join staticskyInput using(sky_id) JOIN stackRun USING(stack_id) @@ -19,4 +19,4 @@ staticskyRun.state = 'full' AND staticskyResult.quality = 0 - AND added_exp_id IS NULL + AND added_sky_id IS NULL -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_dvo.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_dvo.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_dvo.sql (revision 33638) @@ -1,12 +1,13 @@ -SELECT staticskyRun.* FROM staticskyResult -JOIN staticskyRun USING(sky_id) +SELECT staticskyRun.* FROM staticskyRun +JOIN staticskyResult USING(sky_id) join staticskyInput using(sky_id) JOIN stackRun USING(stack_id) WHERE staticskyRun.state = 'full' and staticskyResult.quality = 0 - AND sky_id NOT IN (SELECT sky_id + AND stack_id NOT IN (SELECT stack_id FROM addRun JOIN staticskyResult on staticskyResult.sky_id = addRun.stage_id JOIN staticskyRun USING(sky_id) + JOIN staticskyInput USING(sky_id) WHERE addRun.stage = 'staticsky' AND %s ) Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi.sql (revision 33638) @@ -4,16 +4,16 @@ staticskyResult.num_inputs FROM - staticskyResult -join staticskyRun using (sky_id) + staticskyRun +join staticskyResult using (sky_id) join staticskyInput using(sky_id) JOIN stackRun USING(stack_id) -LEFT JOIN (SELECT sky_id AS added_sky_id, +LEFT JOIN (SELECT stack_id AS added_stack_id, addRun.dvodb AS previous_dvodb FROM addRun JOIN staticskyRun on sky_id = stage_id ) as foo - ON sky_id = added_sky_id - AND stage = 'staticsky_multi' + ON stack_id = added_stack_id + AND stage = 'staticsky' -- hook for qualifying the join on the previous_dvodb AND %s @@ -21,4 +21,4 @@ staticskyRun.state = 'full' AND staticskyResult.quality = 0 - AND added_exp_id IS NULL + AND added_stack_id IS NULL -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi_dvo.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi_dvo.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_find_sky_id_multi_dvo.sql (revision 33638) @@ -5,8 +5,9 @@ WHERE staticskyRun.state = 'full' and staticskyResult.quality = 0 - AND sky_id NOT IN (SELECT sky_id + AND stack_id NOT IN (SELECT stack_id FROM addRun JOIN staticskyResult on staticskyResult.sky_id = addRun.stage_id JOIN staticskyRun USING(sky_id) - WHERE addRun.stage = 'staticsky_multi' AND %s + JOIN staticskyInput USING(sky_id) + WHERE addRun.stage = 'staticsky' AND %s ) Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_minidvodbrun.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_minidvodbrun.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_minidvodbrun.sql (revision 33638) @@ -4,5 +4,4 @@ minidvodb_name, minidvodb_path, - mergedvodb_path, 'new' Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_sky_id_multi.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_sky_id_multi.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_sky_id_multi.sql (revision 33638) @@ -2,5 +2,5 @@ SELECT 0, -- add_id - 'staticsky_multi', -- stage + 'staticsky', -- stage sky_id, -- stage_id %d, -- stage_extra1 Index: /branches/eam_branches/ipp-20111122/ippTools/share/addtool_revertprocessedexp_staticsky_multi.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/addtool_revertprocessedexp_staticsky_multi.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/addtool_revertprocessedexp_staticsky_multi.sql (revision 33638) @@ -5,3 +5,3 @@ AND addProcessedExp.fault != 0 AND addRun.state = 'new' - AND addRun.stage = 'staticsky_multi' + AND addRun.stage = 'staticsky' Index: /branches/eam_branches/ipp-20111122/ippTools/share/camtool_find_pendingimfile.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/camtool_find_pendingimfile.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/camtool_find_pendingimfile.sql (revision 33638) @@ -2,6 +2,7 @@ camRun.cam_id, chipProcessedImfile.*, - (IF(exp_time IS NOT NULL AND deteff_magref IS NOT NULL, - (2.5 * LOG10(exp_time) + chipProcessedImfile.deteff_magref), NULL)) + (IF(rawExp.exp_time IS NOT NULL AND deteff_magref IS NOT NULL, + (2.5 * LOG10(rawExp.exp_time) + chipProcessedImfile.deteff_magref), + NULL)) AS deteff_inst, rawExp.exp_name, @@ -17,5 +18,5 @@ USING(exp_id) JOIN rawImfile - USING(exp_id, class_id, exp_time) + USING(exp_id, class_id) LEFT JOIN camProcessedExp USING(cam_id) Index: /branches/eam_branches/ipp-20111122/ippTools/share/camtool_find_processedexp.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/camtool_find_processedexp.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/camtool_find_processedexp.sql (revision 33638) @@ -3,4 +3,5 @@ camRun.workdir, camRun.label, + camRun.data_group, camRun.state, camRun.magicked, Index: /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_change_exp_state.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_change_exp_state.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_change_exp_state.sql (revision 33638) @@ -14,4 +14,5 @@ WHERE chipProcessedImfile.chip_id = chipRun.chip_id + AND chipProcessedImfile.quality = 0 AND data_state != '%s' ) = 0 Index: /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_definecopy.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_definecopy.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_definecopy.sql (revision 33638) @@ -1,5 +1,8 @@ SELECT rawExp.*, - newExp.label + newExp.label, + chipOld.workdir AS old_workdir, + chipOld.data_group AS old_data_group, + chipOld.state AS old_state FROM chipRun AS chipOld JOIN rawExp USING(exp_id) Index: /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_dropprocessedimfile_update.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_dropprocessedimfile_update.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_dropprocessedimfile_update.sql (revision 33638) @@ -1,5 +1,5 @@ -UPDATE chipProcessedImfile - SET quality = %d, - data_state = 'full', - magicked = 0, - fault = 0 +UPDATE chipProcessedImfile LEFT JOIN camRun USING(chip_id) + SET chipProcessedImfile.quality = %d, + chipProcessedImfile.data_state = 'full', + chipProcessedImfile.magicked = 0, + chipProcessedImfile.fault = 0 Index: /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_pendingcleanupimfile.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_pendingcleanupimfile.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_pendingcleanupimfile.sql (revision 33638) @@ -14,5 +14,5 @@ WHERE ((chipRun.state = 'goto_cleaned' AND (chipProcessedImfile.data_state = 'full' - OR chipProcessedImfile.data_state = 'update')) + OR chipProcessedImfile.data_state = 'update') AND chipProcessedImfile.quality = 0) OR (chipRun.state = 'goto_scrubbed' AND chipProcessedImfile.data_state != 'scrubbed') Index: /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_setimfiletoupdate.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_setimfiletoupdate.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/chiptool_setimfiletoupdate.sql (revision 33638) @@ -1,7 +1,7 @@ UPDATE chipRun JOIN chipProcessedImfile USING(chip_id) - LEFT JOIN magicDSRun ON (stage_id = chip_id AND stage = 'chip') - LEFT JOIN magicDSFile ON (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND component = class_id) + -- LEFT JOIN magicDSRun ON (stage_id = chip_id AND stage = 'chip') + -- LEFT JOIN magicDSFile ON (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + -- AND component = class_id) SET chipRun.state = 'update', chipProcessedImfile.data_state = 'update', @@ -11,7 +11,8 @@ AND (chipRun.state = 'cleaned' OR chipRun.state = 'update') AND (chipProcessedImfile.data_state = 'cleaned') + AND (chipProcessedImfile.quality = 0) -- don't queue update if the associated magicDSFile exists and isn't cleaned - AND (chipRun.magicked = 0 - OR ((magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') - AND (magicDSFile.data_state = 'cleaned' OR magicDSFile.data_state = 'update')) - ) + -- AND (chipRun.magicked = 0 + -- OR ((magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') + -- AND (magicDSFile.data_state = 'cleaned' OR magicDSFile.data_state = 'update')) + -- ) Index: /branches/eam_branches/ipp-20111122/ippTools/share/difftool_change_run_state.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/difftool_change_run_state.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/difftool_change_run_state.sql (revision 33638) @@ -14,3 +14,4 @@ diffSkyfile.diff_id = diffRun.diff_id AND data_state != '%s' + AND diffSkyfile.quality = 0 ) = 0 Index: /branches/eam_branches/ipp-20111122/ippTools/share/difftool_pendingcleanupskyfile.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/difftool_pendingcleanupskyfile.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/difftool_pendingcleanupskyfile.sql (revision 33638) @@ -9,5 +9,5 @@ USING(diff_id) WHERE - ((diffRun.state = 'goto_cleaned' AND (diffSkyfile.data_state = 'full' OR diffSkyfile.data_state = 'update')) + ((diffRun.state = 'goto_cleaned' AND (diffSkyfile.data_state = 'full' OR diffSkyfile.data_state = 'update') AND diffSkyfile.quality = 0) OR (diffRun.state = 'goto_scrubbed' AND diffSkyfile.data_state != 'scrubbed') Index: /branches/eam_branches/ipp-20111122/ippTools/share/difftool_setskyfiletoupdate.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/difftool_setskyfiletoupdate.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/difftool_setskyfiletoupdate.sql (revision 33638) @@ -8,2 +8,3 @@ AND (diffRun.state = 'cleaned' OR diffRun.state = 'update') AND (diffSkyfile.data_state = 'cleaned' OR diffSkyfile.data_state = 'update') + AND (diffSkyfile.quality = 0) Index: /branches/eam_branches/ipp-20111122/ippTools/share/difftool_todiffskyfile.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/difftool_todiffskyfile.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/difftool_todiffskyfile.sql (revision 33638) @@ -77,5 +77,5 @@ OR (warpSkyfile.data_state = 'full' AND warpSkyfile.fault = 0 - AND warpSkyfile.magicked >= 0 +-- AND warpSkyfile.magicked >= 0 AND warpSkyfile.quality = 0)) -- Ensure input stacks are available @@ -88,5 +88,5 @@ OR (warpTemplateSkyfile.data_state = 'full' AND warpTemplateSkyfile.fault = 0 - AND warpSkyfile.magicked >= 0 +-- AND warpSkyfile.magicked >= 0 AND warpTemplateSkyfile.quality = 0)) -- Ensure template stacks are available Index: /branches/eam_branches/ipp-20111122/ippTools/share/disttool_definebyquery_raw_no_magic.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/disttool_definebyquery_raw_no_magic.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/disttool_definebyquery_raw_no_magic.sql (revision 33638) @@ -14,5 +14,4 @@ JOIN rcInterest USING(target_id) LEFT JOIN distRun ON distRun.stage_id = exp_id AND distRun.target_id = distTarget.target_id -WHERE distRun.dist_id IS NULL -- no existing distRun for this exposure - AND distTarget.state = 'enabled' -- target and intrest are enabled +WHERE distTarget.state = 'enabled' -- target and intrest are enabled AND rcInterest.state = 'enabled' Index: /branches/eam_branches/ipp-20111122/ippTools/share/disttool_pending_chip_bg.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/disttool_pending_chip_bg.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/disttool_pending_chip_bg.sql (revision 33638) @@ -14,5 +14,5 @@ chipBackgroundRun.state, chipBackgroundRun.state AS data_state, - 0 AS quality, + chipBackgroundImfile.quality AS quality, distRun.no_magic, chipBackgroundImfile.magicked, Index: /branches/eam_branches/ipp-20111122/ippTools/share/disttool_processedcomponent.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/disttool_processedcomponent.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/disttool_processedcomponent.sql (revision 33638) @@ -10,4 +10,5 @@ name, distComponent.state, + no_magic, distComponent.fault FROM distRun Index: /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_merge_id_dvo.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_merge_id_dvo.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_merge_id_dvo.sql (revision 33638) @@ -0,0 +1,9 @@ +SELECT mergedvodbRun.* FROM mergedvodbRun +JOIN mergedvodbProcessed USING(merge_id) +WHERE ( mergedvodbRun.state = 'merged' OR mergedvodbRun.state = 'full') + AND mergedvodbProcessed.fault = 0 + AND merge_id NOT IN (SELECT merge_id + FROM mergedvodbCopy + JOIN mergedvodbRun USING(mergedvodb_id) + WHERE %s + ) Index: /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_mergedvodbcopy.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_mergedvodbcopy.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_mergedvodbcopy.sql (revision 33638) @@ -0,0 +1,7 @@ +SELECT + mergedvodbCopy.*, + mergedvodbRun.mergedvodb_path +FROM mergedvodbCopy + JOIN mergedvodbRun using (mergedvodb_id) + + Index: /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_minidvodb_id.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_minidvodb_id.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_minidvodb_id.sql (revision 33638) @@ -0,0 +1,8 @@ +SELECT minidvodbRun.* FROM minidvodbRun +JOIN minidvodbProcessed USING(minidvodb_id) +WHERE minidvodbRun.state = 'merged' and minidvodbProcessed.fault = 0 + AND minidvodb_id NOT IN (SELECT minidvodb_id + FROM mergedvodbRun + JOIN minidvodbRun USING(minidvodb_id) + WHERE %s + ) Index: /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_pending_merge.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_pending_merge.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_pending_merge.sql (revision 33638) @@ -0,0 +1,14 @@ +SELECT + mergedvodbRun.*, + minidvodbRun.minidvodb_path +FROM minidvodbRun +JOIN mergedvodbRun + USING(minidvodb_id) +LEFT JOIN mergedvodbProcessed + USING(merge_id) +WHERE + (mergedvodbRun.state = 'new' AND mergedvodbProcessed.merge_id IS NULL) + AND mergedvodbRun.mergedvodb IS NOT NULL + AND mergedvodbRun.mergedvodb_path IS NOT NULL + + Index: /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_pendingmergeprocess.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_pendingmergeprocess.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_pendingmergeprocess.sql (revision 33638) @@ -0,0 +1,5 @@ +SELECT mergedvodbRun.* +FROM mergedvodbRun +LEFT JOIN mergedvodbProcessed +USING(merge_id) +WHERE (mergedvodbRun.state = 'new' AND mergedvodbProcessed.merge_id IS NULL) Index: /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_processed.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_processed.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_find_processed.sql (revision 33638) @@ -0,0 +1,7 @@ +SELECT + mergedvodbProcessed.*, + mergedvodbRun.mergedvodb_path +FROM mergedvodbProcessed +JOIN mergedvodbRun + USING(merge_id) + Index: /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_queue_merge_id.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_queue_merge_id.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_queue_merge_id.sql (revision 33638) @@ -0,0 +1,11 @@ +INSERT INTO mergedvodbRun + SELECT + 0, -- merge_id + minidvodb_id, -- minidvodb_id + '%s', -- mergedvodb + '%s', -- mergedvodb_path + '%s', -- state + FROM minidvodbRun join minidvodbProcessed using (minidvodb_id) + WHERE + minidvodbRun.state = 'merged' and minidvodbProcessed.fault = 0 + AND minidvodbRun.minidvodb_id = %lld Index: /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_queue_mergecopy_id.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_queue_mergecopy_id.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_queue_mergecopy_id.sql (revision 33638) @@ -0,0 +1,16 @@ +INSERT INTO mergedvodbCopy + SELECT + 0, -- mergedvodbcopy_id + merge_id, -- merge_id + '%s', -- mergedvodb_rsync_path + '%s', -- destination_host + %d, -- fault + '%s', -- state + %s, -- epoch + %f -- dtime + + FROM mergedvodbRun + WHERE + (mergedvodbRun.state = 'merged' + OR mergedvodbRun.state = 'full') + AND mergedvodbRun.merge_id = %lld Index: /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_queue_minidvodb_id.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_queue_minidvodb_id.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_queue_minidvodb_id.sql (revision 33638) @@ -0,0 +1,15 @@ +INSERT INTO minidvodbCopy + SELECT + 0, -- minidvodbcopy_id + minidvodb_id, -- minidvodb_id + '%s', -- minidvodb_rsync_path + '%s', -- destination_host + %d, -- fault + '%s', -- state + %s, -- epoch + %f -- dtime + + FROM minidvodbRun + WHERE + minidvodbRun.state = 'merged' + AND minidvodbRun.minidvodb_id = %lld Index: /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_revertmergedvodbcopy.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_revertmergedvodbcopy.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_revertmergedvodbcopy.sql (revision 33638) @@ -0,0 +1,7 @@ +UPDATE mergedvodbCopy + -- SET minidvodbCopy.state = 'new', + SET mergedvodbCopy.fault = 0 +WHERE + mergedvodbCopy.state = 'new' + AND mergedvodbCopy.fault >0 + Index: /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_revertmergedvodbprocessed.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_revertmergedvodbprocessed.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/mergetool_revertmergedvodbprocessed.sql (revision 33638) @@ -0,0 +1,6 @@ +DELETE FROM mergedvodbProcessed +USING mergedvodbProcessed, mergedvodbRun +WHERE + mergedvodbProcessed.merge_id = mergedvodbRun.merge_id + AND mergedvodbProcessed.fault != 0 + AND mergedvodbRun.state = 'new' Index: /branches/eam_branches/ipp-20111122/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql (revision 33638) @@ -1,5 +1,6 @@ SELECT minidvodbRun.* FROM minidvodbRun JOIN minidvodbProcessed USING(minidvodb_id) -WHERE minidvodbRun.state = 'merged' +WHERE minidvodbRun.state = 'merged' + AND minidvodbProcessed.fault = 0 AND minidvodb_id NOT IN (SELECT minidvodb_id FROM minidvodbCopy Index: /branches/eam_branches/ipp-20111122/ippTools/share/pstamptool_pendingjob.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/pstamptool_pendingjob.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/pstamptool_pendingjob.sql (revision 33638) @@ -1,4 +1,7 @@ -SELECT pstampJob.*, - IFNULL(Label.priority, 10000) AS priority +SELECT * FROM +(SELECT pstampJob.*, + IFNULL(Label.priority, 10000) AS priority, + 0 AS myParent, + 0 AS runningChildren FROM pstampJob JOIN pstampRequest USING(req_id) @@ -9,4 +12,37 @@ AND pstampJob.state = 'run' AND pstampJob.fault = 0 + AND NOT is_parent AND (dep_id = 0 OR pstampDependent.state = 'full') AND (Label.active OR Label.active IS NULL) + -- where hook 1 %s +UNION +SELECT * FROM + (SELECT pstampJob.*, + IFNULL(Label.priority, 10000) AS priority + FROM pstampJob + JOIN pstampRequest USING(req_id) + LEFT JOIN pstampDependent USING(dep_id) + LEFT JOIN Label ON pstampRequest.label = Label.label + WHERE pstampRequest.state = 'run' + AND pstampRequest.fault = 0 + AND pstampJob.state = 'run' + AND pstampJob.fault = 0 + AND is_parent + AND (dep_id = 0 OR pstampDependent.state = 'full') + AND (Label.active OR Label.active IS NULL) + -- where hook 2 %s + ) as parentJob + LEFT JOIN + (SELECT parent_id AS myParent, + count(job_id) as runningChildren + FROM pstampRequest JOIN pstampJob USING(req_id) + WHERE pstampRequest.state ='run' + AND pstampJob.state = 'run' + AND pstampJob.parent_id + GROUP BY parent_id + ) as childJobs + ON childJobs.myParent = parentJob.job_id + WHERE runningChildren IS NULL +) AS theQuery + + Index: /branches/eam_branches/ipp-20111122/ippTools/share/pstamptool_stopdependentjob.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/pstamptool_stopdependentjob.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/pstamptool_stopdependentjob.sql (revision 33638) @@ -5,5 +5,6 @@ SET pstampJob.state = 'stop', pstampJob.fault = %d, - pstampDependent.fault = %d + pstampDependent.fault = %d, + pstampDependent.state = '%s' WHERE pstampJob.state = 'run' AND pstampDependent.state = 'new' Index: /branches/eam_branches/ipp-20111122/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/pxadmin_create_tables.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/pxadmin_create_tables.sql (revision 33638) @@ -190,7 +190,8 @@ magicked BIGINT, PRIMARY KEY(exp_id), + KEY(end_stage), + KEY(fault), + KEY(state), KEY(exp_name), - KEY(end_stage), - KEY(fault), FOREIGN KEY(exp_id) REFERENCES newExp(exp_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1441,5 +1442,8 @@ fault SMALLINT, PRIMARY KEY(req_id), - KEY(req_id) + KEY(req_id), + KEY(state), + KEY(fault), + KEY(label) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1455,6 +1459,15 @@ options BIGINT, dep_id BIGINT, + fault_count INT, + parent_id BIGINT DEFAULT NULL, + is_parent TINYINT DEFAULT 0, PRIMARY KEY(job_id, req_id), KEY(job_id), + KEY(req_id), + KEY(state), + KEY(fault), + KEY(dep_id), + KEY(parent_id), + KEY(is_parent), FOREIGN KEY(req_id) REFERENCES pstampRequest(req_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1467,7 +1480,7 @@ component VARCHAR(64), imagedb VARCHAR(64), + outdir VARCHAR(255), rlabel VARCHAR(64), need_magic TINYINT, - outdir VARCHAR(255), fault SMALLINT, fault_count INT, @@ -1476,4 +1489,6 @@ KEY(stage), KEY(stage_id), + KEY(component), + KEY(imagedb), KEY(fault) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1963,5 +1978,4 @@ minidvodb_group VARCHAR(64) NOT NULL, minidvodb_path VARCHAR(255) NOT NULL, - mergedvodb_path VARCHAR(255) NOT NULL, state VARCHAR(64) NOT NULL, creation_date TIMESTAMP, @@ -1971,12 +1985,8 @@ CREATE TABLE minidvodbProcessed ( minidvodb_id BIGINT, - merge_order BIGINT, dtime_resort FLOAT, dtime_relphot FLOAT, - dtime_verify FLOAT, - dtime_merge FLOAT, dtime_script FLOAT, epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - mergedvodb_path VARCHAR(255) NOT NULL, fault SMALLINT NOT NULL, UNIQUE KEY (minidvodb_id), @@ -1999,4 +2009,42 @@ FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbRun ( + merge_id BIGINT AUTO_INCREMENT, + minidvodb_id BIGINT, + mergedvodb VARCHAR(64) NOT NULL, + mergedvodb_path VARCHAR(255) NOT NULL, + state VARCHAR(64) NOT NULL, + creation_date TIMESTAMP, + PRIMARY KEY(merge_id), KEY(minidvodb_id), KEY(mergedvodb), INDEX(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbProcessed ( + merge_id BIGINT, + merge_order BIGINT, + dtime_verify FLOAT, + dtime_merge FLOAT, + dtime_script FLOAT, + epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + fault SMALLINT NOT NULL, + UNIQUE KEY (merge_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbCopy ( + mergedvodbcopy_id BIGINT AUTO_INCREMENT, + merge_id BIGINT, + mergedvodb_rsync_path VARCHAR(255) NOT NULL, + destination_host VARCHAR(255) NOT NULL, + fault SMALLINT NOT NULL, + state VARCHAR(64) NOT NULL, + epoch TIMESTAMP, + dtime FLOAT, + PRIMARY KEY(mergedvodbcopy_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE vpRun ( vp_id BIGINT AUTO_INCREMENT, @@ -2035,4 +2083,49 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE skycalRun ( + skycal_id BIGINT AUTO_INCREMENT, -- unique identifier + sky_id BIGINT, + stack_id BIGINT, + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + note VARCHAR(255), -- note + PRIMARY KEY(skycal_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + KEY(sky_id), + KEY(stack_id), + FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Result of sky calibration analysis +CREATE TABLE skycalResult ( + skycal_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + dtime_astrom FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + n_astrom INT, + zpt_obs FLOAT, + zpt_stdev FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(skycal_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + -- This comment line is here to avoid empty query error. -- Another way to avoid that problem is to omit the semicolon above but I think that is untidy. Index: /branches/eam_branches/ipp-20111122/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/pxadmin_drop_tables.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/pxadmin_drop_tables.sql (revision 33638) @@ -86,4 +86,5 @@ DROP TABLE IF EXISTS publishDone; DROP TABLE IF EXISTS pstampDependent; +DROP TABLE IF EXISTS pstampFile; DROP TABLE IF EXISTS staticskyInput; DROP TABLE IF EXISTS staticskyResult; @@ -105,4 +106,6 @@ DROP TABLE IF EXISTS vpProcessedCell; DROP TABLE IF EXISTS vpRun; +DROP TABLE IF EXISTS skycalRun; +DROP TABLE IF EXISTS skycalResult; SET FOREIGN_KEY_CHECKS=1 Index: /branches/eam_branches/ipp-20111122/ippTools/share/pztool_pendingimfile.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/pztool_pendingimfile.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/pztool_pendingimfile.sql (revision 33638) @@ -1,5 +1,6 @@ SELECT * FROM ( - SELECT + SELECT * FROM + (SELECT summitImfile.*, summitExp.dateobs @@ -19,3 +20,27 @@ AND pzDownloadImfile.class IS NULL AND pzDownloadImfile.class_id IS NULL + AND summitImfile.exp_name NOT LIKE '%a' + ORDER BY dateobs) AS partA +UNION + SELECT * FROM + (SELECT + summitImfile.*, + summitExp.dateobs + FROM summitImfile + JOIN pzDownloadExp + USING(summit_id) + LEFT JOIN pzDownloadImfile + USING(summit_id, class, class_id) + JOIN summitExp + USING(summit_id) + WHERE + pzDownloadExp.state = 'run' + AND pzDownloadImfile.summit_id IS NULL + AND pzDownloadImfile.exp_name IS NULL + AND pzDownloadImfile.camera IS NULL + AND pzDownloadImfile.telescope IS NULL + AND pzDownloadImfile.class IS NULL + AND pzDownloadImfile.class_id IS NULL + AND summitImfile.exp_name LIKE '%a' + ORDER BY dateobs) AS partB ) as Foo Index: /branches/eam_branches/ipp-20111122/ippTools/share/stacktool_inputskyfile.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/stacktool_inputskyfile.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/stacktool_inputskyfile.sql (revision 33638) @@ -1,5 +1,10 @@ SELECT warpSkyfile.*, - rawExp.camera + rawExp.camera, + rawExp.exp_id, + rawExp.exp_name, + rawExp.object, + rawExp.dateobs, + rawExp.comment FROM stackRun JOIN stackInputSkyfile USING(stack_id) Index: /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_defineskycalrun.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_defineskycalrun.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_defineskycalrun.sql (revision 33638) @@ -0,0 +1,16 @@ +SELECT + sky_id, + stack_id, + staticskyRun.workdir, + staticskyRun.label, + staticskyRun.data_group, + staticskyRun.dist_group +FROM staticskyRun + JOIN staticskyResult USING(sky_id) + JOIN staticskyInput USING(sky_id) + JOIN stackRun USING(stack_id) + JOIN stackSumSkyfile USING(stack_id) + LEFT JOIN skycalRun USING(sky_id, stack_id) +WHERE staticskyRun.state = 'full' + AND staticskyResult.quality = 0 + Index: /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_inputs.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_inputs.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_inputs.sql (revision 33638) @@ -2,6 +2,8 @@ staticskyRun.*, staticskyInput.stack_id, - stackSumSkyfile.path_base + stackSumSkyfile.path_base, + stackRun.filter FROM staticskyRun JOIN staticskyInput USING(sky_id) JOIN stackSumSkyfile USING (stack_id) +JOIN stackRun using(stack_id) Index: /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_pendingskycalrun.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_pendingskycalrun.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_pendingskycalrun.sql (revision 33638) @@ -0,0 +1,21 @@ +SELECT + skycalRun.*, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + staticskyResult.path_base, + rawExp.camera, + staticskyResult.num_inputs = 1 AS singlefilter +FROM skycalRun +JOIN staticskyRun USING (sky_id) +JOIN staticskyResult USING(sky_id) +JOIN stackRun USING (stack_id) +JOIN stackInputSkyfile USING(stack_id) +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING (chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN skycalResult USING(skycal_id) +WHERE + (skycalRun.state = 'new' AND skycalResult.skycal_id IS NULL) Index: /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_result.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_result.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_result.sql (revision 33638) @@ -1,3 +1,3 @@ -SELECT DISTINCT +SELECT staticskyResult.*, staticskyRun.state, @@ -6,5 +6,5 @@ stackRun.tess_id, stackRun.skycell_id, - stackRun.filter + count(stackRun.filter) AS num_filters FROM staticskyRun JOIN staticskyResult USING(sky_id) Index: /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_revertskycal.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_revertskycal.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_revertskycal.sql (revision 33638) @@ -0,0 +1,6 @@ +DELETE FROM skycalResult +USING skycalResult, skycalRun, stackRun +WHERE skycalRun.skycal_id = skycalResult.skycal_id + AND stackRun.stack_id = skycalRun.stack_id + AND skycalRun.state = 'new' + AND skycalResult.fault != 0 Index: /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_skycalresult.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_skycalresult.sql (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_skycalresult.sql (revision 33638) @@ -0,0 +1,14 @@ +SELECT + skycalResult.*, + skycalRun.stack_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + skycalRun.state, + skycalRun.label, + skycalRun.data_group, + skycalRun.sky_id +FROM skycalRun +JOIN skycalResult USING(skycal_id) +JOIN stackRun USING(stack_id) + Index: /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_todo.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_todo.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/staticskytool_todo.sql (revision 33638) @@ -6,12 +6,17 @@ staticskyRun.state, stackRun.tess_id, - stackRun.skycell_id + stackRun.skycell_id, + IFNULL(Label.priority, 10000) AS priority FROM staticskyRun JOIN staticskyInput USING (sky_id) JOIN stackRun USING (stack_id) LEFT JOIN staticskyResult USING(sky_id) +LEFT JOIN Label ON staticskyRun.label = Label.label WHERE (staticskyRun.state = 'new' AND staticskyResult.sky_id IS NULL) + AND (Label.active OR Label.active IS NULL) + -- WHERE hook %s GROUP BY sky_id HAVING SUM(IF(stackRun.state = 'full', 1, 0)) = COUNT(staticskyInput.sky_id) +ORDER BY priority DESC, sky_id Index: /branches/eam_branches/ipp-20111122/ippTools/share/warptool_change_run_state.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/warptool_change_run_state.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/warptool_change_run_state.sql (revision 33638) @@ -17,3 +17,4 @@ warpSkyfile.warp_id = warpRun.warp_id AND warpSkyfile.data_state != '%s' + AND warpSkyfile.quality = 0 ) = 0 Index: /branches/eam_branches/ipp-20111122/ippTools/share/warptool_pendingcleanupskyfile.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 33638) @@ -11,5 +11,6 @@ WHERE ((warpRun.state = 'goto_cleaned' AND (warpSkyfile.data_state = 'full' - OR warpSkyfile.data_state = 'update')) + OR warpSkyfile.data_state = 'update') + AND warpSkyfile.quality = 0) OR (warpRun.state = 'goto_scrubbed' AND warpSkyfile.data_state != 'scrubbed') Index: /branches/eam_branches/ipp-20111122/ippTools/share/warptool_setskyfiletoupdate.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/warptool_setskyfiletoupdate.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/warptool_setskyfiletoupdate.sql (revision 33638) @@ -8,2 +8,3 @@ AND (warpRun.state = 'cleaned' OR warpRun.state = 'update') AND (warpSkyfile.data_state = 'cleaned' OR warpSkyfile.data_state = 'update') + AND (warpSkyfile.quality = 0) Index: /branches/eam_branches/ipp-20111122/ippTools/share/warptool_towarped.sql =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/share/warptool_towarped.sql (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/share/warptool_towarped.sql (revision 33638) @@ -106,7 +106,5 @@ -- where hook 2 %s GROUP BY warp_id, skycell_id - -- if warpSkyfile was magicked previously require inputs to be magicked - -- this blocks processing until all the chip inputs have been destreaked - HAVING COUNT(warpSkyCellMap.class_id) = SUM(IF(chipProcessedImfile.data_state ='full' and (chipRun.magicked = 0 OR chipProcessedImfile.magicked > 0), 1, 0)) + HAVING COUNT(warpSkyCellMap.class_id) = SUM(IF(chipProcessedImfile.data_state ='full', 1, 0)) -- limit hook 2 %s ) Index: /branches/eam_branches/ipp-20111122/ippTools/src/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/Makefile.am (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/Makefile.am (revision 33638) @@ -29,4 +29,5 @@ diffphottool \ minidvodbtool \ + mergetool \ laptool \ vptool @@ -43,4 +44,5 @@ pxmagic.h \ pxminidvodb.h \ + pxmerge.h \ pxregister.h \ pxspace.h \ @@ -79,4 +81,5 @@ diffphottool.h \ minidvodbtool.h \ + mergetool.h \ laptool.h \ vptool.h @@ -98,4 +101,5 @@ pxmagic.c \ pxminidvodb.c \ + pxmerge.c \ pxregister.c \ pxspace.c \ @@ -294,4 +298,10 @@ minidvodbtoolConfig.c +mergetool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +mergetool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +mergetool_SOURCES = \ + mergetool.c \ + mergetoolConfig.c + laptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) laptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la Index: /branches/eam_branches/ipp-20111122/ippTools/src/addtool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/addtool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/addtool.c (revision 33638) @@ -141,11 +141,4 @@ pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label PXOPT_COPY_STR(config->args, where, "-reduction", "staticskyyRun.reduction", "=="); - pxAddLabelSearchArgs (config, where, "-filter", "stackRun.filter", "=="); // define using camRun label - } - if (strcmp(stage, "staticsky_multi")== 0) { - - pxAddLabelSearchArgs (config, where, "-label", "staticskyRun.label", "=="); // define using camRun label - pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label - PXOPT_COPY_STR(config->args, where, "-reduction", "staticskyyRun.reduction", "=="); //no filter here } @@ -227,21 +220,6 @@ } } + if (strcmp(stage,"staticsky") == 0) { - if (dvodb ) { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_dvo.sql\n%s\n", dvodb,stage); - // find the cam_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_sky_id_dvo.sql"); - // user supplied dvodb - psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); - } else { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_sky_id.sql\n%s\n",stage); - // find the cam_id of all the exposures that we want to queue up. - bare_query = pxDataGet("addtool_find_sky_id.sql"); - // inherit dvodb from camRun, avoid matching NULL - psStringAppend(&dvodb_string, "(staticskyRun.dvodb IS NOT NULL AND previous_dvodb = staticskyRun.dvodb)"); - } - } - - if (strcmp(stage,"staticsky_multi") == 0) { if (dvodb ) { psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_multi_dvo.sql\n%s\n", dvodb,stage); @@ -292,7 +270,8 @@ } else { //This picks only the unmagicked/uncensored ones - if (strcmp(stage,"cam") == 0) { - psStringAppend(&query, " AND (camRun.magicked = 0)"); - } + //if (strcmp(stage,"cam") == 0) { + //we can now properly handle the magicked case in ippScripts, so we queue camRuns in any magic state now if -uncensored. + //psStringAppend(&query, " AND (camRun.magicked = 0)"); + //} if (strcmp(stage,"stack") == 0) { psStringAppend(&query, " AND (stackRun.magicked = 0)"); @@ -309,8 +288,8 @@ psStringAppend(&query, " GROUP BY stack_id"); } + // if (strcmp(stage,"staticsky") == 0) { + // psStringAppend(&query, " GROUP BY stack_id"); + // } if (strcmp(stage,"staticsky") == 0) { - psStringAppend(&query, " GROUP BY stack_id"); - } - if (strcmp(stage,"staticsky_multi") == 0) { psStringAppend(&query, " GROUP BY sky_id"); //some reason it needs this } @@ -401,31 +380,6 @@ } } + if (strcmp(stage,"staticsky") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - - staticskyRunRow *row = staticskyRunObjectFromMetadata(md); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); - psFree(output); - return false; - } - - if (!dvodb) { //there's no staticsky.dvodb - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, sky_id %" PRId64, row->label, row->sky_id); - psFree(output); - return false; - } - if (!workdir && !row->workdir) { - psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, sky_id %" PRId64, row->label, row->sky_id); - psFree(output); - return false; - } - - psFree(row); - } - } - if (strcmp(stage,"staticsky_multi") == 0) { for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; @@ -565,48 +519,6 @@ } } + if (strcmp(stage,"staticsky") == 0) { - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *md = output->data[i]; - psS64 stage_id =0; - - staticskyRunRow *row = staticskyRunObjectFromMetadata(md); - stage_id = row->sky_id; - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); - psFree(output); - return false; - } - - // queue the exp - if (!pxaddQueueByCamID(config, - stage, - stage_id, - 0, - workdir ? workdir : row->workdir, - reduction ? reduction : row->reduction, - label ? label : row->label, - data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), - dvodb ? dvodb : NULL, - note ? note : NULL, - image_only, - minidvodb, - minidvodb_group, - minidvodb_name - )) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error sfg"); - } - psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue stage %s %" PRId64,stage, stage_id); - psFree(row); - psFree(output); - return false; - } - psFree(row); - } - } - - if (strcmp(stage,"staticsky_multi") == 0) { for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; @@ -704,7 +616,5 @@ query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); } - if (strcmp(stage, "staticsky_multi")==0) { - query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); - } + // pxUpdateRun gets parameters from config->args and runs the update query @@ -744,7 +654,4 @@ } if (strcmp(stage, "staticsky")==0) { - query = pxDataGet("addtool_find_pendingexp_staticsky.sql"); - } - if (strcmp(stage, "staticsky_multi")==0) { query = pxDataGet("addtool_find_pendingexp_staticsky_multi.sql"); } @@ -767,9 +674,9 @@ psStringAppend(&query, " GROUP BY %s", "stack_id"); } + //if (strcmp(stage, "staticsky") == 0) { + // //this group by is needed to join against all the warps (to get camera) + // psStringAppend(&query, " GROUP BY %s", "sky_id"); + //} if (strcmp(stage, "staticsky") == 0) { - //this group by is needed to join against all the warps (to get camera) - psStringAppend(&query, " GROUP BY %s", "sky_id"); - } - if (strcmp(stage, "staticsky_multi") == 0) { //this group by is needed to join against all the warps (to get camera) psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1"); @@ -824,5 +731,5 @@ PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-minidvodb_name", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - + PXOPT_LOOKUP_S64(stage_extra1, config->args, "-stage_extra1", false, false); // generate restrictions psMetadata *where = psMetadataAlloc(); @@ -902,5 +809,19 @@ } } - + //if there is a stage_extra1, set it in addRun (it's not known until it is processed) + if (stage_extra1) { + psString setExtra = NULL; + psStringAppend (&setExtra, "UPDATE addRun set stage_extra1 = %" PRId64, stage_extra1); + psStringAppend (&setExtra, " where add_id = %" PRId64, row->add_id); + + if (!p_psDBRunQuery(config->dbh, setExtra)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + + return false; + } + } @@ -962,7 +883,4 @@ } if (strcmp (stage,"staticsky") == 0) { - query = pxDataGet("addtool_find_processedexp_staticsky.sql"); - } - if (strcmp (stage,"staticsky_multi") == 0) { query = pxDataGet("addtool_find_processedexp_staticsky_multi.sql"); } @@ -1049,5 +967,5 @@ PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); PXOPT_COPY_S64(config->args, where, "-stage_id", "addRun.stage_id", "=="); - PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); @@ -1075,8 +993,5 @@ query = pxDataGet("addtool_revertprocessedexp_stack.sql"); } - if (strcmp(stage, "staticsky") == 0) { - query = pxDataGet("addtool_revertprocessedexp_staticsky.sql"); - } - if (strcmp(stage, "staticsky_multi") == 0) { + if (strcmp(stage, "staticsky") == 0) { query = pxDataGet("addtool_revertprocessedexp_staticsky_multi.sql"); } @@ -1221,7 +1136,5 @@ PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", true, false); - - //optional + //optional PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); @@ -1238,4 +1151,7 @@ if (minidvodb_path) { minidvodbpath = minidvodb_path; + } else { + psError(PS_ERR_UNKNOWN, false, "require minidvodb_path"); + return false; } @@ -1245,6 +1161,5 @@ minidvodb_group, minidvodbpath, - mergedvodb_path, - "new", + "new", 0 )) { @@ -1268,9 +1183,4 @@ psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id); } - - if (!minidvodb_path) { - psStringAppend(&minidvodb_path,"%s/%s",mergedvodb_path,minidvodb_name); - } - if (minidvodb_path) { psStringAppend(&minidvodb_path,"/%s",minidvodb_name); @@ -1304,5 +1214,4 @@ PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_STR(config->args, where, "-minidvodb_path", "minidvodb_path", "=="); - PXOPT_COPY_STR(config->args, where, "-mergedvodb_path", "mergedvodb_path", "=="); PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodb_group", "=="); @@ -1310,5 +1219,4 @@ PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", false, false); @@ -1345,12 +1253,4 @@ } - if (mergedvodb_path) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); - cnt++; - } - if (minidvodb_group) { if (cnt) { @@ -1396,5 +1296,5 @@ } - psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL"); + psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL"); psString firstwhereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -1444,5 +1344,5 @@ //now flip new -> active - psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL "); + psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL "); psStringAppend(&query2, " AND minidvodb_group = '%s' limit 1;", minidvodb_group); if (!p_psDBRunQuery(config->dbh, query2)) { @@ -1480,6 +1380,6 @@ PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "=="); PXOPT_COPY_STR(config->args, where, "-state", "minidvodbRun.state", "=="); - - + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(all_addrun_states, config->args, "-all_addrun_states", false); @@ -1493,18 +1393,33 @@ } - psString query = pxDataGet("addtool_checkminidvodbaddrun.sql"); - - if (!query) { + if (!minidvodb_group) { + psError(PXTOOLS_ERR_CONFIG, false, "minidvodb_group required"); + return false; + } + + + psString bare_query = pxDataGet("addtool_checkminidvodbaddrun.sql"); + + if (!bare_query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; } - + psString where_string = NULL; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&where_string, " WHERE %s", whereClause); psFree(whereClause); - } - + //search params are defined above + } + + psString query = NULL; + psStringAppend(&query, bare_query, minidvodb_group); + psStringAppend(&query, where_string); + + + + + if (!all_addrun_states) { psStringAppend(&query, " AND (cnt2 = cnt) "); @@ -1519,5 +1434,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error,\n\n%s", query); psFree(query); return false; @@ -1621,13 +1536,9 @@ // required PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-mergedvodb_path", true, false); PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", true, false); // optional - PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-dtime_relphot", false, false); PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false); - PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); - PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); @@ -1680,12 +1591,8 @@ minidvodbProcessedRow *row = minidvodbProcessedRowAlloc( pendingRow->minidvodb_id, - merge_order, dtime_resort, dtime_relphot, - dtime_verify, - dtime_merge, dtime_script, epoch, - mergedvodb_path, fault ); @@ -1702,67 +1609,4 @@ } - - - - //this finds the # of merged things (for the merge order) - psString query3 = NULL; - psStringAppend(&query3, "select count(*) from minidvodbRun join minidvodbProcessed using (minidvodb_id) where state = 'merged' and minidvodb_group = '%s';", minidvodb_group); - - if (!p_psDBRunQuery(config->dbh, query3)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query3); - return false; - } - psArray *output2 = p_psDBFetchResult(config->dbh); - if (!output2) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output2)) { - psTrace("addtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - bool status; - psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); - if (!status) { - - psAbort("failed to lookup value for count column"); - return false; - } - psString final = NULL; - psStringAppend(&final, "%" PRIu64, m_order); - //return false; - psFree(query3); - psFree(output2); - - - - //update the merge_order - - psString query4 = NULL; - psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order); - psStringAppend(&query4," where minidvodb_id = %" PRIu64, minidvodb_id); - //printf("%s", query4); - if (!p_psDBRunQuery(config->dbh, query4)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query4); - return false; - } - // -psFree(query4); - - - -// since there is only one exp per 'new' set mindvodbRun.state = 'merged' - psString query2 = NULL ; psStringAppend(&query2, "UPDATE minidvodbRun SET state = 'merged' WHERE minidvodb_id = %'" PRIu64, row->minidvodb_id); @@ -1777,12 +1621,4 @@ return false; } - - - - - - - - psFree(row); psFree(pendingRow); @@ -1797,10 +1633,4 @@ return false; } - - - //print the merge_order (why not!) - printf("%s", final); - psFree(final); - return true; } @@ -1961,10 +1791,7 @@ PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); - PXOPT_LOOKUP_U64(merge_order, config->args, "-set_merge_order", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-set_dtime_relphot", false, false); PXOPT_LOOKUP_F32(dtime_resort, config->args, "-set_dtime_resort", false, false); - PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); -PXOPT_LOOKUP_F32(dtime_verify, config->args, "-set_dtime_verify", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); @@ -1986,13 +1813,4 @@ } - if (merge_order) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - - psStringAppend(&query, " merge_order = %" PRId64, merge_order); - cnt++; - } - if (dtime_relphot) { if (cnt) { @@ -2011,19 +1829,5 @@ } - if (dtime_merge) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " dtime_merge = %f", dtime_merge); - cnt++; - } -if (dtime_verify) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " dtime_verify = %f", dtime_verify); - cnt++; - } -if (dtime_script) { + if (dtime_script) { if (cnt) { psStringAppend(&query, "%s", comma); Index: /branches/eam_branches/ipp-20111122/ippTools/src/addtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/addtoolConfig.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/addtoolConfig.c (revision 33638) @@ -115,4 +115,6 @@ psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); +psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-stage_extra1", 0, "set stage_extra1", 0); + // -processedexp @@ -165,5 +167,4 @@ psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define minidvodb_group (required)", NULL); psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); - psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for the merged dvodb (required)", NULL); psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -174,9 +175,7 @@ psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name (LIKE)", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_path", 0, "search by path for minidvodb", NULL); - psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-mergedvodb_path", 0, "search by path for the merged dvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0, "define minidvodb_name", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); - psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for minidvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define path for the merged dvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -205,12 +204,8 @@ psMetadata *addminidvodbprocessedArgs = psMetadataAlloc(); psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "define minidvodb_id (required)", 0); - psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-merge_order", 0, "define merge order", 0); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_relphot", 0, "define elapsed time for relphot (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_resort", 0, "define elapsed time for resort (seconds)", NAN); - psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_verify", 0, "define elapsed time for DVO verify (seconds)", NAN); - psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time for script (seconds)", NAN); psMetadataAddTime(addminidvodbprocessedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); - psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-mergedvodb_path",0, "path of merged dvodb", NULL); psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",0, "minidvodb_group", NULL); psMetadataAddS16(addminidvodbprocessedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); @@ -232,9 +227,6 @@ psMetadataAddStr(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name", NULL); psMetadataAddS16(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); - psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_merge_order", 0, "define merge order", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_relphot", 0, "define elapsed time for relphot (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_resort", 0, "define elapsed time for resort (seconds)", 0); - psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_verify", 0, "define elapsed time for DVO verify (seconds)", 0); - psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); psMetadata *argSets = psMetadataAlloc(); Index: /branches/eam_branches/ipp-20111122/ippTools/src/camtool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/camtool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/camtool.c (revision 33638) @@ -112,4 +112,5 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "chipRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "=="); if (!psListLength(where->list) && Index: /branches/eam_branches/ipp-20111122/ippTools/src/camtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/camtoolConfig.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/camtoolConfig.c (revision 33638) @@ -55,4 +55,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by chipRun reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-data_group", 0, "search by chipRun data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by rawExp obs_mode", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); Index: /branches/eam_branches/ipp-20111122/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/chiptool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/chiptool.c (revision 33638) @@ -198,4 +198,11 @@ GET_VALUE (psS64, S64, exp_id, "exp_id"); GET_VALUE (psString, Str, raw_workdir, "workdir"); +#ifdef notyet + GET_VALUE (psString, Str, old_workdir, "old_workdir"); + GET_VALUE (psString, Str, old_data_group, "old_data_group"); +#else + const char *old_workdir = NULL; + const char *old_data_group = NULL; +#endif GET_VALUE (psString, Str, raw_label, "label"); GET_VALUE (psString, Str, raw_reduction, "reduction"); @@ -213,7 +220,7 @@ if (!pxchipQueueByExpTag(config, exp_id, - workdir ? workdir : raw_workdir, + workdir ? workdir : (old_workdir ? old_workdir : raw_workdir), label ? label : raw_label, - data_group ? data_group : (label ? label : raw_label), + data_group ? data_group : (old_data_group ? old_data_group : (label ? label : raw_label)), dist_group, reduction ? reduction : raw_reduction, @@ -345,4 +352,5 @@ pxAddLabelSearchArgs (config, where, "-label", "chipOld.label", "LIKE"); pxAddLabelSearchArgs (config, where, "-data_group", "chipOld.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "chipOld.state", "=="); // psListLength(where->list) is at least 1 because exp_type defaults to "object" @@ -354,5 +362,5 @@ } - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); @@ -468,4 +476,5 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); psString query = pxDataGet("chiptool_pendingimfile.sql"); @@ -1006,5 +1015,9 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chipProcessedImfile.chip_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); - PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", false, false); + // if cam_id is not supplied chip_id is required + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", cam_id ? false : true, false); PXOPT_LOOKUP_S64(class_id, config->args, "-class_id", true, false); PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", true, false); @@ -1017,5 +1030,5 @@ if (psListLength(where->list) == 0) { // won't get here - psError(PXTOOLS_ERR_CONFIG, true, "-chip_id and -class_id are required"); + psError(PXTOOLS_ERR_CONFIG, true, "-chip_id (or -cam_id) and -class_id are required"); return false; } @@ -2024,7 +2037,16 @@ return false; } - psFree(setHook); psFree(query); + + // if we are updating a whole chipRun set lingering chips with poor quality to full data_state + if (!class_id) { + query = "UPDATE chipProcessedImfile SET data_state ='full', fault = 0 WHERE chip_id = %" PRId64 " AND quality != 0 AND (data_state ='cleaned' OR data_state = 'update')"; + + if (!p_psDBRunQueryF(config->dbh, query, chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } return true; Index: /branches/eam_branches/ipp-20111122/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/chiptoolConfig.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/chiptoolConfig.c (revision 33638) @@ -72,6 +72,7 @@ psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-exp_type", PS_META_REPLACE, "search by exp_type", "object"); - - psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-state", 0, "search by chipRun.state", NULL); + + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); @@ -111,4 +112,5 @@ psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", 0); pxchipSetSearchArgs(pendingimfileArgs); psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -240,5 +242,6 @@ // -dropprocessedimfile psMetadata *dropprocessedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(dropprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID (required)", 0); + psMetadataAddS64(dropprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddS64(dropprocessedimfileArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam ID", 0); psMetadataAddStr(dropprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID (required)", NULL); psMetadataAddS16(dropprocessedimfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality (required)", 0); Index: /branches/eam_branches/ipp-20111122/ippTools/src/difftool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/difftool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/difftool.c (revision 33638) @@ -1981,5 +1981,5 @@ psString finishQuery = NULL; psStringAppend( &finishQuery, - "UPDATE diffRun set state ='full', note = 'dummy run - no overlap' WHERE diff_id = %" PRId64, + "UPDATE diffRun set state ='full', dist_group = NULL, note = 'dummy run - no overlap' WHERE diff_id = %" PRId64, run->diff_id); if (!p_psDBRunQuery(config->dbh, finishQuery)) { Index: /branches/eam_branches/ipp-20111122/ippTools/src/disttool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/disttool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/disttool.c (revision 33638) @@ -34,7 +34,9 @@ static bool updaterunMode(pxConfig *config); static bool revertrunMode(pxConfig *config); -static bool rerunMode(pxConfig *config); +static bool startoverMode(pxConfig *config); static bool pendingcomponentMode(pxConfig *config); static bool addprocessedcomponentMode(pxConfig *config); +static bool updateprocessedcomponentMode(pxConfig *config); +static bool revertcomponentMode(pxConfig *config); static bool revertcomponentMode(pxConfig *config); static bool processedcomponentMode(pxConfig *config); @@ -84,7 +86,8 @@ MODECASE(DISTTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(DISTTOOL_MODE_REVERTRUN, revertrunMode); - MODECASE(DISTTOOL_MODE_RERUN, rerunMode); + MODECASE(DISTTOOL_MODE_STARTOVER, startoverMode); MODECASE(DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentMode); MODECASE(DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentMode); + MODECASE(DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT, updateprocessedcomponentMode); MODECASE(DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentMode); MODECASE(DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentMode); @@ -165,4 +168,5 @@ PXOPT_LOOKUP_BOOL(single, config->args, "-singlefilter", false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); if (use_alternate) { @@ -404,4 +408,7 @@ psStringAppend(&query, " AND (camRun.state = 'full')"); } + } + if (!rerun) { + psStringAppend(&query, " AND (distRun.dist_id IS NULL)"); } @@ -643,5 +650,5 @@ } -static bool rerunMode(pxConfig *config) +static bool startoverMode(pxConfig *config) { psMetadata *where = psMetadataAlloc(); @@ -653,5 +660,5 @@ // require data_group or dist_id to be supplied - PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", true, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", false, false); if (!data_group && !dist_id) { @@ -969,4 +976,60 @@ return true; } +static bool updateprocessedcomponentMode(pxConfig *config) +{ + + // required values + PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-outdir", false, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + + bool setfault = clearfault || fault; + + if (!bytes && !md5sum && !outdir && !name && !setfault && !fault) { + psError(PS_ERR_UNKNOWN, true, "at least one of bytes md5sum outdir name fault or setfault is required"); + return false; + } + + char *sep = ""; + psString query = psStringCopy("UPDATE distComponent SET "); + if (setfault) { + psStringAppend(&query, "%s fault = %d", sep, fault ? 1 : 0); + sep = ", "; + } + if (bytes) { + psStringAppend(&query, "%s bytes = %d", sep, bytes); + sep = ", "; + } + if (md5sum) { + psStringAppend(&query, "%s md5sum = '%s'", sep, md5sum); + sep = ", "; + } + if (outdir) { + psStringAppend(&query, "%s outdir = '%s'", sep, outdir); + sep = ", "; + } + if (name) { + psStringAppend(&query, "%s name = '%s'", sep, name); + sep = ", "; + } + + psStringAppend(&query, "\nWHERE dist_id = %"PRId64 " AND component = '%s'", + dist_id, component); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} static bool toadvanceMode(pxConfig *config) @@ -1048,4 +1111,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -2185,4 +2249,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); if (!psListLength(where->list)) { Index: /branches/eam_branches/ipp-20111122/ippTools/src/disttool.h =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/disttool.h (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/disttool.h (revision 33638) @@ -28,9 +28,10 @@ DISTTOOL_MODE_UPDATERUN, DISTTOOL_MODE_REVERTRUN, - DISTTOOL_MODE_RERUN, + DISTTOOL_MODE_STARTOVER, DISTTOOL_MODE_PENDINGCOMPONENT, DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, DISTTOOL_MODE_REVERTCOMPONENT, DISTTOOL_MODE_PROCESSEDCOMPONENT, + DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT, DISTTOOL_MODE_TOADVANCE, DISTTOOL_MODE_PENDINGCLEANUP, Index: /branches/eam_branches/ipp-20111122/ippTools/src/disttoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/disttoolConfig.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/disttoolConfig.c (revision 33638) @@ -65,4 +65,5 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-singlefilter", 0, "select single filter runs (sky stage only)", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new distRun even if one already exists", false); psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -87,15 +88,15 @@ psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); - // -rerun - psMetadata *rerunArgs = psMetadataAlloc(); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-data_group", 0, "define data_group (required)", NULL); - psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-stage", 0, "search by stage", NULL); - psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-set_label", 0, "define new label", NULL); - psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false); - psMetadataAddU64(rerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -startover + psMetadata *startoverArgs = psMetadataAlloc(); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-data_group", 0, "define data_group", NULL); + psMetadataAddS64(startoverArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-stage", 0, "search by stage", NULL); + psMetadataAddS64(startoverArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-set_label", 0, "define new label", NULL); + psMetadataAddBool(startoverArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false); + psMetadataAddU64(startoverArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(startoverArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -revertrun @@ -151,6 +152,18 @@ psMetadata *processedcomponentArgs = psMetadataAlloc(); psMetadataAddS64(processedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(processedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); psMetadataAddU64(processedcomponentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(processedcomponentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updateprocessedcomponentArgs + psMetadata *updateprocessedcomponentArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component (required)", NULL); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-outdir", 0, "define output directory", NULL); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL); + psMetadataAddS32(updateprocessedcomponentArgs, PS_LIST_TAIL, "-bytes", 0, "define file size", 0); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-md5sum", 0, "define stage for bundle", NULL); + psMetadataAddS32(updateprocessedcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + psMetadataAddBool(updateprocessedcomponentArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", false); // -toadvance @@ -180,5 +193,5 @@ psMetadata *updatefilesetArgs = psMetadataAlloc(); psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); -// psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-set_state",0, "new value for state", NULL); psMetadataAddS32(updatefilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); @@ -343,8 +356,9 @@ PXOPT_ADD_MODE("-definebyquery", "define distribution run", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); PXOPT_ADD_MODE("-updaterun", "update distRun", DISTTOOL_MODE_UPDATERUN, updaterunArgs); - PXOPT_ADD_MODE("-rerun", "reprocess a cleaned distribution run", DISTTOOL_MODE_RERUN, rerunArgs); + PXOPT_ADD_MODE("-startover", "reprocess a cleaned distribution run", DISTTOOL_MODE_STARTOVER, startoverArgs); PXOPT_ADD_MODE("-revertrun", "revert distRun.faults", DISTTOOL_MODE_REVERTRUN, revertrunArgs); PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); + PXOPT_ADD_MODE("-updateprocessedcomponent", "", DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT, updateprocessedcomponentArgs); PXOPT_ADD_MODE("-revertcomponent", "revert faulted components", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs); PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs); Index: /branches/eam_branches/ipp-20111122/ippTools/src/faketool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/faketool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/faketool.c (revision 33638) @@ -335,4 +335,5 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "fakeRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "fakeRun.data_group", "=="); PXOPT_COPY_STR(config->args, where, "-dist_group", "fakeRun.dist_group", "=="); Index: /branches/eam_branches/ipp-20111122/ippTools/src/laptool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/laptool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/laptool.c (revision 33638) @@ -390,4 +390,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); psMetadata *where = psMetadataAlloc(); @@ -411,4 +412,9 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n %s", limitString); + psFree(limitString); } Index: /branches/eam_branches/ipp-20111122/ippTools/src/laptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/laptoolConfig.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/laptoolConfig.c (revision 33638) @@ -74,4 +74,5 @@ ADD_OPT(Str, pendingrunArgs, "-fault", "search by LAP run fault", NULL); ADD_OPT(Bool,pendingrunArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, pendingrunArgs, "-limit", "limit result set to N items", 0); // -listrun @@ -86,4 +87,5 @@ ADD_OPT(Str, listrunArgs, "-fault", "search by LAP run fault", NULL); ADD_OPT(Bool,listrunArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, listrunArgs, "-limit", "limit result set to N items", 0); // -updaterun Index: /branches/eam_branches/ipp-20111122/ippTools/src/magicdstool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/magicdstool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/magicdstool.c (revision 33638) @@ -779,5 +779,5 @@ case IPP_STAGE_CAMERA: query = NULL; - clearRunQuery = "UPDATE chipBackgroundRun SET magicked = 0 where chip_bg_id = %" PRId64; + clearRunQuery = "UPDATE camRun SET magicked = 0 where cam_id = %" PRId64; break; case IPP_STAGE_WARP: @@ -1242,4 +1242,6 @@ } else if (!strcmp(state, "failed_cleanup")) { new_state = "goto_cleaned"; + } else if (!strcmp(state, "failed_restore")) { + new_state = "goto_restored"; } else { psError(PS_ERR_UNKNOWN, true, "unexpected value for state: %s", state); @@ -1410,4 +1412,5 @@ (strcmp(state, "drop") == 0) || (strcmp(state, "wait") == 0) || + (strcmp(state, "failed_restore") == 0) || (strcmp(state, "failed_revert") == 0) || (strcmp(state, "failed_revert_ud") == 0) || Index: /branches/eam_branches/ipp-20111122/ippTools/src/mergetool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/mergetool.c (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/src/mergetool.c (revision 33638) @@ -0,0 +1,1227 @@ +/* + * addtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmerge.h" +#include "mergetool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingmergeMode(pxConfig *config); +static bool addmergedMode(pxConfig *config); +static bool listmergedMode(pxConfig *config); +static bool revertmergedMode(pxConfig *config); +static bool updatemergedMode(pxConfig *config); +static bool definebyquerymergecopyMode(pxConfig *config); +static bool listmergedvodbcopyMode(pxConfig *config); +static bool revertmergedvodbcopyMode(pxConfig *config); +static bool updatemergedvodbcopyMode(pxConfig *config); + + + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = mergetoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(MERGETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(MERGETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(MERGETOOL_MODE_PENDINGMERGE, pendingmergeMode); + MODECASE(MERGETOOL_MODE_ADDMERGED, addmergedMode); + MODECASE(MERGETOOL_MODE_LISTMERGED, listmergedMode); + MODECASE(MERGETOOL_MODE_REVERTMERGED, revertmergedMode); + MODECASE(MERGETOOL_MODE_UPDATEMERGED, updatemergedMode); + MODECASE(MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, definebyquerymergecopyMode); + MODECASE(MERGETOOL_MODE_LISTMERGEDVODBCOPY, listmergedvodbcopyMode); + MODECASE(MERGETOOL_MODE_REVERTMERGEDVODBCOPY, revertmergedvodbcopyMode); + MODECASE(MERGETOOL_MODE_UPDATEMERGEDVODBCOPY, updatemergedvodbcopyMode); + + + + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); + // PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + PXOPT_LOOKUP_STR(mergedvodb, config->args, "-mergedvodb", true, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + //find the items to queue (uniq minidvodb_id) + psString dvodb_string = NULL; + psString bare_query = NULL; + +bare_query = pxDataGet("mergetool_find_minidvodb_id.sql"); + psStringAppend(&dvodb_string, "mergedvodbRun.mergedvodb = '%s'", mergedvodb); + +if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return false; + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); + + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + psFree(where); + + + psStringAppend(&query, " GROUP BY minidvodb_id"); + + + psTrace("mergetool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query); + +if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error, \n%s\n", query); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + +for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + minidvodbRunRow *row = minidvodbRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into minidvodbRun"); + psFree(output); + return false; + } + + if (!mergedvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue merge run without a defined mergedvodb: minidvodb_id %" PRId64,row->minidvodb_id); + psFree(output); + return false; + } + if (!mergedvodb_path) { + psError(PS_ERR_UNKNOWN, false, "cannot queue merge run without a defined mergedvodb_path: minidvodb_id %" PRId64,row->minidvodb_id); + psFree(output); + return false; + } + + psFree(row); + +} + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } +for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 minidvodb_id =0; + + minidvodbRunRow *row = minidvodbRunObjectFromMetadata(md); + minidvodb_id = row->minidvodb_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into minidvodbRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxmergeQueueByMinidvodbID(config, + minidvodb_id, + mergedvodb, + mergedvodb_path + + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue mergedvodb %" PRId64, minidvodb_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + + } + +psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + psString query = NULL; + + query = psStringCopy("UPDATE mergedvodbRun SET"); + + int cnt = 0; + psString comma = ","; + if (state) { + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + if (mergedvodb_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); + cnt++; + } + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + + return true; +} + +static bool pendingmergeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "mergedvodbRun.minidvodb_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = NULL; + query = pxDataGet("mergetool_find_pending_merge.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbPendingMerge", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_U64(merge_id, config->args, "-merge_id", true, false); + PXOPT_LOOKUP_STR(mergedvodb, config->args, "-mergedvodb", true, false); + + // optional + +//PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); + PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + + PXOPT_LOOKUP_TIME(epoch, config->args, "-epoch", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psString query = pxDataGet("mergetool_find_pendingmergeprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + psTrace("mergetool.c", PS_LOG_INFO, "query is:\n%s", query); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + +psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("mergetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + mergedvodbRunRow *pendingRow = mergedvodbRunObjectFromMetadata(output->data[0]); + psFree(output); + mergedvodbProcessedRow *row = mergedvodbProcessedRowAlloc( + pendingRow->merge_id, + 0, + dtime_verify ? dtime_verify : 0, + dtime_merge ? dtime_merge : 0, + dtime_script ? dtime_script : 0, + epoch, + fault + ); + + if (!mergedvodbProcessedInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + +//this finds the # of merged things (for the merge order) + psString query3 = NULL; + psStringAppend(&query3, "select count(*) from mergedvodbRun join mergedvodbProcessed using (merge_id) where state = 'full' and mergedvodb = '%s';", mergedvodb); + + if (!p_psDBRunQuery(config->dbh, query3)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query3); + return false; + } + psArray *output2 = p_psDBFetchResult(config->dbh); + if (!output2) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output2)) { + psTrace("mergetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + bool status; + psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); + if (!status) { + + psAbort("failed to lookup value for count column"); + return false; + } + psString final = NULL; + psStringAppend(&final, "%" PRIu64, m_order); + //return false; + psFree(query3); + psFree(output2); + + + + //update the merge_order + + psString query4 = NULL; + psStringAppend(&query4, "update mergedvodbProcessed set merge_order = %"PRIu64,m_order); + psStringAppend(&query4," where merge_id = %" PRIu64, merge_id); + //printf("%s", query4); + if (!p_psDBRunQuery(config->dbh, query4)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query4); + return false; + } + // +psFree(query4); + +// since there is only one exp per 'new' set mergedvodbRun.state = 'full' + + psString query2 = NULL ; + + if (!fault) { + psStringAppend(&query2, "UPDATE mergedvodbRun SET state = 'full' WHERE merge_id = %'" PRIu64, row->merge_id); + + if (!p_psDBRunQuery(config->dbh, query2)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query2); + return false; + } + + } + + + + + + + psFree(row); + psFree(pendingRow); + + + + + + //commit the changes + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + + //print the merge_order (why not!) + printf("%s", final); + psFree(final); + + + + + + + + + + + + return true; +} + + +static bool listmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "mergedvodbRun.minidvodb_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = NULL; + query = pxDataGet("mergetool_find_processed.sql"); + +if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + + + // we either add AND (condition) or WHERE (condition): + if (psListLength(where->list) && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND fault != 0"); + } + if (psListLength(where->list) && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND fault = 0"); + } + if (!psListLength(where->list) && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE fault != 0"); + } + if (!psListLength(where->list) && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE fault = 0"); + } + +psFree(where); + + // order by merge_id + psStringAppend(&query, " ORDER BY merge_id"); + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + + + return true; +} + + +static bool revertmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "mergedvodbRun.fault", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = NULL; + query = pxDataGet("mergetool_revertmergedvodbprocessed.sql"); + +if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + return true; +} + +static bool updatemergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-merge_id", true, false); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbProcessed.merge_id", "=="); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); + PXOPT_LOOKUP_F32(dtime_verify, config->args, "-set_dtime_verify", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = psStringCopy("UPDATE mergedvodbProcessed JOIN mergedvodbRun USING (merge_id) SET "); +int cnt = 0; + psString comma = ","; + if (fault) { + psStringAppend(&query, " fault = %d", fault); + cnt++; + } + + + + if (dtime_verify) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_verify = %f", dtime_verify); + cnt++; + } + + if (dtime_merge) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_merge = %f", dtime_merge); + cnt++; + } + +if (dtime_script) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_script = %f", dtime_script); + cnt++; + } +if (state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error \n%s\n",query ); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + +static bool definebyquerymergecopyMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + pxAddLabelSearchArgs (config, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); // define using camRun label + + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(destination_host, config->args, "-set_destination_host", false, false); + PXOPT_LOOKUP_STR(mergedvodb_rsync_path, config->args, "-set_mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(last_merged, config->args, "-last_merged", false); + // prevent queueing an addRun if a given exposure has already been added to + // the given dvo database + psString dvodb_string = NULL; + psString bare_query = NULL; + if (destination_host) { + psTrace("mergedvodbtool.c", PS_LOG_INFO, "destination_host argument found (%s) using mergedvodbtool_find_merge_id_dvo.sql\n", destination_host); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("mergedvodbtool_find_merge_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "mergedvodbCopy.destination_host = '%s'", destination_host); + } else { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined destination_host"); + + return false; + } + + if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return false; + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); + + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + if (last_merged) { + psStringAppend(&query, " ORDER BY merge_order DESC LIMIT 1"); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbCopy", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // loop over our list of camRun rows to check the supplied and selected dvodb and workdir values: + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + mergedvodbRunRow *row = mergedvodbRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into mergedvodbRun"); + psFree(output); + return false; + } + + if (!mergedvodb_rsync_path) { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined mergedvodb_rsync_path: merge_id %" PRId64, row->merge_id); + psFree(output); + return false; + } + if (!destination_host) { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined destination_host: merge_id %" PRId64, row->merge_id); + psFree(output); + return false; + } + + psFree(row); + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + mergedvodbRunRow *row = mergedvodbRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByMergeID(config, + row->merge_id, + destination_host ? destination_host : "NULL", + mergedvodb_rsync_path ?mergedvodb_rsync_path : "NULL" + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue merge_id: %" PRId64, row->merge_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool listmergedvodbcopyMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_STR(config->args, where, "-destination_host", "mergedvodbCopy.destination_host", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pending, config->args, "-pending", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("mergedvodbtool_find_mergedvodbcopy.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + +if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + +// we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault = 0"); + } + if (!where->list && faulted) { + + // list only faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE mergedvodbCopy.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE mergedvodbCopy.fault = 0"); + } + psFree(where); + + if (pending) { + //add the cuts for pending (state new, no faults) + psStringAppend(&query, " %s", " AND mergedvodbCopy.state = 'new' AND mergedvodbCopy.fault = 0 AND mergedvodbCopy.destination_host IS NOT NULL AND mergedvodbCopy.mergedvodb_rsync_path IS NOT NULL"); + } + + + // order by epoch + psStringAppend(&query, " ORDER BY mergedvodbcopy_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s ", query); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbCopy", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + +return true; +} + +static bool revertmergedvodbcopyMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_COPY_S64(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-destination_host", "destination_host", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "mergedvodbCopy.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("mergedvodbtool_revertmergedvodbcopy.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + } + psError(PS_ERR_UNKNOWN, false, "database error %s", query ); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updatemergedvodbcopyMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_U64(mergedvodbcopy_id, config->args, "-mergedvodbcopy_id", false, false); + PXOPT_LOOKUP_U64(merge_id, config->args, "-merge_id", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(host, config->args, "-host", false, false); + PXOPT_LOOKUP_STR(rsync_path, config->args, "-mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + PXOPT_LOOKUP_S16(set_fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_STR(set_rsync_path, config->args, "-set_mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_STR(set_host, config->args, "-set_destination_host", false, false); + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_F32(dtime, config->args, "-set_dtime", false, false); + PXOPT_COPY_S64(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-state", "mergedvodbCopy.state", "=="); + PXOPT_COPY_STR(config->args, where, "-host", "mergedvodbCopy.destination_host", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb_rsync_path", "mergedvodbCopy.mergedvodb_rsync_path", "=="); + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE mergedvodbCopy JOIN mergedvodbRun USING (merge_id) SET "); + int cnt = 0; + psString comma = ","; + if (set_fault) { + psStringAppend(&query, " mergedvodbCopy.fault = %d", set_fault); + cnt++; + } + if (set_rsync_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.mergedvodb_rsync_path = '%s'" , set_rsync_path); + cnt++; + } + + if (set_host) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.destination_host = '%s'" , set_host); + cnt++; + } + + + if (set_state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.state = '%s'" , set_state); + cnt++; + } + + if (dtime) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodbCopy.dtime = %f", dtime); + cnt++; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + + + Index: /branches/eam_branches/ipp-20111122/ippTools/src/mergetool.h =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/mergetool.h (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/src/mergetool.h (revision 33638) @@ -0,0 +1,41 @@ +/* + * mergetool.h + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MERGETOOL_H +#define MERGETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + MERGETOOL_MODE_NONE = 0x0, + MERGETOOL_MODE_DEFINEBYQUERY, + MERGETOOL_MODE_UPDATERUN, + MERGETOOL_MODE_PENDINGMERGE, + MERGETOOL_MODE_ADDMERGED, + MERGETOOL_MODE_LISTMERGED, + MERGETOOL_MODE_REVERTMERGED, + MERGETOOL_MODE_UPDATEMERGED, + MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, + MERGETOOL_MODE_LISTMERGEDVODBCOPY, + MERGETOOL_MODE_REVERTMERGEDVODBCOPY, + MERGETOOL_MODE_UPDATEMERGEDVODBCOPY +} mergetoolMode; + +pxConfig *mergetoolConfig(pxConfig *config, int argc, char **argv); + +#endif // MERGETOOL_H Index: /branches/eam_branches/ipp-20111122/ippTools/src/mergetoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/mergetoolConfig.c (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/src/mergetoolConfig.c (revision 33638) @@ -0,0 +1,196 @@ +/* + * addtoolConfig.c + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxmerge.h" +#include "mergetool.h" + +pxConfig *mergetoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + // pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-mergedvodb", PS_META_DUPLICATE_OK, "search by mergedvodbRun.mergedvodb and use as mergedvodb", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-minidvodb_group", PS_META_DUPLICATE_OK, "search by minidvodbRun.minidvodb_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define workdir", NULL); + // psMetadataAddStr(definebyqueryArge, PS_LIST_TAIL, "-set_mergedvodb", 0, "define mergedvodb"); + + // psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadata *updaterunArgs = psMetadataAlloc(); + + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", 0); + psMetadataAddU64(updaterunArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define workdir", NULL); + + + psMetadata *pendingmergeArgs = psMetadataAlloc(); + psMetadataAddU64(pendingmergeArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(pendingmergeArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", NULL); + psMetadataAddStr(pendingmergeArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", NULL); + + psMetadataAddU64(pendingmergeArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(pendingmergeArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + + + psMetadata *addmergedArgs = psMetadataAlloc(); + psMetadataAddU64(addmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id (required)", 0); + psMetadataAddStr(addmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb (required)", 0); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_verify", 0, "define elapsed time for DVO verify (seconds)", NAN); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time for script (seconds)", NAN); + psMetadataAddTime(addmergedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); + psMetadataAddS16(addmergedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + + + + psMetadata *listmergedArgs = psMetadataAlloc(); + psMetadataAddU64(listmergedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(listmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", NULL); + psMetadataAddStr(listmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by minidvodbRun.minidvodb_group", NULL); + psMetadataAddU64(listmergedArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listmergedArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listmergedArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + + psMetadata *revertmergedArgs = psMetadataAlloc(); + psMetadataAddU64(revertmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddS16(revertmergedArgs, PS_LIST_TAIL, "-fault", 0, "search by minidvodbCopyfault", 0); + psMetadataAddStr(revertmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", NULL); + + psMetadata *updatemergedArgs = psMetadataAlloc(); + + psMetadataAddU64(updatemergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddS16(updatemergedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_verify", 0, "define elapsed time for DVO verify (seconds)", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); + psMetadataAddStr(updatemergedArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + // -definebyquerymergecopy + psMetadata *definebyquerymergecopyArgs = psMetadataAlloc(); + psMetadataAddS64(definebyquerymergecopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-mergedvodb", PS_META_DUPLICATE_OK, "search by mergedvodb", NULL); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-set_mergedvodb_rsync_path", 0, "define workdir", NULL); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-set_destination_host", 0, "set destination host", NULL); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-last_merged", 0, "if multiple minidvodbs have been dvomerged between rsyncs, then only queue the most recent merged dvodb for copying", false); + + psMetadata *listmergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(listmergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by mergedvodb_id", 0); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", NULL); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodbRun.mergedvodb", NULL); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-destination_host", 0, "search by mergedvodbCopy.destination_host", NULL); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-pending", 0, "limit to pending items", false); + psMetadataAddU64(listmergedvodbcopyArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + psMetadata *revertmergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", 0); + psMetadataAddU64(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-destination_host", 0, "search by destination_host", NULL); + psMetadataAddS16(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-fault", 0, "search by mergedvodbCopyfault", 0); + psMetadata *updatemergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", 0); + psMetadataAddU64(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-host", 0, "search by host", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodb_rsync_path", 0, "search by rsync", NULL); + psMetadataAddS16(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + + + psMetadataAddS16(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + + psMetadataAddF32(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_dtime", 0, "set elapsed time for transfer", 0); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_mergedvodb_rsync_path", 0, "change the mergedvodb_rsync_path", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_destination_host", 0, "change the host", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + + + + + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from minindvodbRun", MERGETOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "update existing mergedvodbRuns", MERGETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingmerge", "list pending mergedvodbRuns", MERGETOOL_MODE_PENDINGMERGE, pendingmergeArgs); + PXOPT_ADD_MODE("-addmerged", "add processed mergedvodbRun", MERGETOOL_MODE_ADDMERGED, addmergedArgs); + PXOPT_ADD_MODE("-listmerged","list merged dvodbs", MERGETOOL_MODE_LISTMERGED, listmergedArgs); + PXOPT_ADD_MODE("-revertmerged","revert merged dvodbs", MERGETOOL_MODE_REVERTMERGED, revertmergedArgs); + PXOPT_ADD_MODE("-updatemerged","change merged dvodb properties",MERGETOOL_MODE_UPDATEMERGED, updatemergedArgs); + PXOPT_ADD_MODE("-definebyquerymergecopy", "create runs from mergedvodbRun", MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, definebyquerymergecopyArgs); + PXOPT_ADD_MODE("-listmergedvodbcopy","list copy mergedvodbs", MERGETOOL_MODE_LISTMERGEDVODBCOPY, listmergedvodbcopyArgs); + PXOPT_ADD_MODE("-revertmergedvodbcopy","revert copy mergedvobs",MERGETOOL_MODE_REVERTMERGEDVODBCOPY, revertmergedvodbcopyArgs); + PXOPT_ADD_MODE("-updatemergedvodbcopy","change mergedvodb copy properties",MERGETOOL_MODE_UPDATEMERGEDVODBCOPY, updatemergedvodbcopyArgs); + + + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/eam_branches/ipp-20111122/ippTools/src/minidvodbtool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/minidvodbtool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/minidvodbtool.c (revision 33638) @@ -89,5 +89,5 @@ psMetadata *where = psMetadataAlloc(); pxcamGetSearchArgs (config, where); - PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodbRun_id", "=="); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); pxAddLabelSearchArgs (config, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); // define using camRun label Index: /branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c (revision 33638) @@ -704,6 +704,8 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); - PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); + PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", false, false); + PXOPT_LOOKUP_S64(parent_id, config->args, "-parent_id", false, false); + PXOPT_LOOKUP_BOOL(is_parent, config->args, "-is_parent", false); // unless the job is being inserted with stop state require outputBase @@ -716,4 +718,19 @@ if (!strcmp(job_type, "get_image") || !strcmp(job_type, "detect_query") || !strcmp(job_type, "none")) { stampJob = false; + } else if (!strcmp(job_type, "child")) { + // job_type child's only action is to resolve a dependent + // XXX: IS this necessary? + if (!dep_id) { + psError(PS_ERR_UNKNOWN, true, "dep_id required for child job\n"); + return false; + } + if (is_parent) { + psError(PS_ERR_UNKNOWN, true, "job type child can not be a parent job\n"); + return false; + } + stampJob = false; + } else if (!strcmp(job_type, "mosaic")) { + stampJob = false; + is_parent = true; } else if (!strcmp(job_type, "stamp")) { stampJob = true; @@ -723,5 +740,5 @@ } if (!pstampJobInsert(config->dbh, - 0, // job_id + 0, // job_id req_id, rownum, @@ -733,5 +750,7 @@ options, dep_id, - 0 // fault_count + 0, // fault_count + parent_id, + is_parent )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -761,4 +780,5 @@ PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + PXOPT_COPY_STR(config->args, where, "-jobType", "jobType", "=="); PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); @@ -842,8 +862,11 @@ // use psDBGenerateWhereSQL because the SQL yields an intermediate table + psString whereStr = NULL; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereStr, "\n AND %s", whereClause); psFree(whereClause); + } else { + psStringAppend(&whereStr, "%s", ""); } psFree(where); @@ -858,9 +881,11 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } + if (!p_psDBRunQueryF(config->dbh, query, whereStr, whereStr)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereStr); + psFree(query); + return false; + } + psFree(whereStr); psFree(query); @@ -956,4 +981,5 @@ PXOPT_LOOKUP_S32(fault, config->args, "-set_fault", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); psMetadata *where = psMetadataAlloc(); @@ -986,5 +1012,5 @@ psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, fault, fault)) { + if (!p_psDBRunQueryF(config->dbh, query, fault, fault, state)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); Index: /branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c (revision 33638) @@ -144,4 +144,6 @@ psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0, "define options", 0); psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-dep_id", 0, "define job dep_id", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-parent_id", 0, "define parent's job_id", 0); + psMetadataAddBool(addjobArgs, PS_LIST_TAIL, "-is_parent", 0, "define whether job has children", false); psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "define job result", 0); @@ -152,4 +154,5 @@ psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-dep_id", 0, "select by dependent ID", 0); psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0, "select by fault", 0); + psMetadataAddStr(listjobArgs, PS_LIST_TAIL, "-jobType", 0, "select by jobType", 0); psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -186,4 +189,5 @@ psMetadataAddS32(stopdependentjobArgs, PS_LIST_TAIL, "-fault_count", 0, "select by fault_count (>=)", 0); psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL, "-set_fault", 0, "new fault value for job and dependent (required)", 0); + psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-set_state", 0, "new pstampDependent.state", "new"); psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); psMetadataAddU64(stopdependentjobArgs, PS_LIST_TAIL, "-limit", 0, "not used", 0); Index: /branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c (revision 33638) @@ -157,8 +157,4 @@ } if (strcmp(stage,"staticsky") == 0) { - query = pxDataGet("addtool_queue_sky_id.sql"); - psMemSetPersistent(query, true); - } - if (strcmp(stage,"staticsky_multi") == 0) { query = pxDataGet("addtool_queue_sky_id_multi.sql"); psMemSetPersistent(query, true); Index: /branches/eam_branches/ipp-20111122/ippTools/src/pxchip.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pxchip.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pxchip.c (revision 33638) @@ -71,5 +71,5 @@ psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); - psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object (LIKE comparison)", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); Index: /branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.c (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.c (revision 33638) @@ -0,0 +1,97 @@ +/* + * pxadd.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmerge.h" + +bool pxmergeQueueByMinidvodbID(pxConfig *config, + psS64 minidvodb_id, + char *mergedvodb, + char *mergedvodb_path + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("mergetool_queue_merge_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + } + if (!p_psDBRunQueryF(config->dbh,"INSERT INTO mergedvodbRun SELECT 0, minidvodb_id, '%s', '%s', 'new', NULL FROM minidvodbRun WHERE minidvodbRun.minidvodb_id = %lld and minidvodbRun.state = 'merged'", mergedvodb, mergedvodb_path, (long long) minidvodb_id)) + { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + return false; + } + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row \n%s", query); + return false; + } + return true; +} + +bool pxaddQueueByMergeID(pxConfig *config, + psS64 merge_id, + char *destination_host, + char *mergedvodb_rsync_path + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("mergetool_queue_mergecopy_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + } + + if (!p_psDBRunQueryF(config->dbh,"INSERT INTO mergedvodbCopy SELECT 0, merge_id, '%s', '%s', 0, 'new', NULL, 0 FROM mergedvodbRun WHERE mergedvodbRun.merge_id = %lld and (mergedvodbRun.state = 'merged' or mergedvodbRun.state = 'full')", mergedvodb_rsync_path, destination_host, (long long) merge_id)) + + { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row \n%s", query); + return false; + } + + return true; +} Index: /branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.h =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.h (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pxmerge.h (revision 33638) @@ -0,0 +1,41 @@ +/* + * pxadd.h + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXMERGE_H +#define PXMERGE_H 1 + +#include + +#include "pxtools.h" + + +bool pxmergeQueueByMinidvodbID(pxConfig *config, + psS64 minidvodb_id, + char *mergedvodbb, + char *mergedvodb_path + ); + +bool pxaddQueueByMergeID(pxConfig *config, + psS64 merge_id, + char *destination_host, + char *mergedvodb_rsync_path + ); + + +#endif // PXMERGE_H Index: /branches/eam_branches/ipp-20111122/ippTools/src/pztool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/pztool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/pztool.c (revision 33638) @@ -338,8 +338,8 @@ } - psStringAppend(&query, " ORDER BY dateobs"); - if (desc) { - psStringAppend(&query, " DESC"); - } +/* psStringAppend(&query, " ORDER BY dateobs"); */ +/* if (desc) { */ +/* psStringAppend(&query, " DESC"); */ +/* } */ // request the full "limit" from each known camera and throw away any Index: /branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.c (revision 33638) @@ -38,7 +38,15 @@ static bool resultMode(pxConfig *config); static bool revertMode(pxConfig *config); -static bool updateresult(pxConfig *config); +static bool updateresultMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); + +static bool defineskycalrunMode(pxConfig *config); +static bool updateskycalrunMode(pxConfig *config); +static bool pendingskycalrunMode(pxConfig *config); +static bool addskycalresultMode(pxConfig *config); +static bool skycalresultMode(pxConfig *config); +static bool revertskycalresultMode(pxConfig *config); +static bool updateskycalresultMode(pxConfig *config); static bool setstaticskyRunState(pxConfig *config, psS64 sky_id, const char *state); @@ -69,7 +77,14 @@ MODECASE(STATICSKYTOOL_MODE_RESULT, resultMode); MODECASE(STATICSKYTOOL_MODE_REVERT, revertMode); - MODECASE(STATICSKYTOOL_MODE_UPDATERESULT, updateresult); + MODECASE(STATICSKYTOOL_MODE_UPDATERESULT, updateresultMode); MODECASE(STATICSKYTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(STATICSKYTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(STATICSKYTOOL_MODE_DEFINESKYCALRUN, defineskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_UPDATESKYCALRUN, updateskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_PENDINGSKYCALRUN, pendingskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_ADDSKYCALRESULT, addskycalresultMode); + MODECASE(STATICSKYTOOL_MODE_UPDATESKYCALRESULT,updateskycalresultMode); + MODECASE(STATICSKYTOOL_MODE_REVERTSKYCALRESULT,revertskycalresultMode); + MODECASE(STATICSKYTOOL_MODE_SKYCALRESULT, skycalresultMode); default: psAbort("invalid option (this should not happen)"); @@ -97,10 +112,8 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - - // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); // optional - PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); @@ -111,5 +124,6 @@ psMetadata *whereMD = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-select_stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); @@ -621,5 +635,8 @@ PXOPT_COPY_STR(config->args, where, "-label", "staticskyRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "staticskyRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + PXOPT_LOOKUP_S32(num_filters, config->args, "-num_filters", false, false); PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); @@ -645,8 +662,14 @@ psFree(where); + psStringAppend(&query, "\nGROUP BY sky_id"); + if (num_filters) { + psStringAppend(&query, "\nHAVING COUNT(filter) >= %d", num_filters); + } + + // treat limit == 0 as "no limit" if (limit) { psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); + psStringAppend(&query, "\n%s", limitString); psFree(limitString); } @@ -735,5 +758,5 @@ } -static bool updateresult(pxConfig *config) +static bool updateresultMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -978,2 +1001,477 @@ # endif +static bool defineskycalrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + // none required. We get workdir, etc from staticskyRun if not provided + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + + psMetadata *whereMD = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, whereMD, "-select_sky_id", "staticskyRun.sky_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-select_stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); + pxAddLabelSearchArgs(config, whereMD, "-select_label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "stackRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psString query = pxDataGet("staticskytool_defineskycalrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(whereMD); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + if (!rerun) { + if (label) { + psStringAppend(&query, "\nAND (skycalRun.label IS NULL OR skycalRun.label = '%s')", label); + } + psStringAppend(&query, "\nAND skycal_id IS NULL"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // we now have a list of (tess_id, skycell_id) that (potentially) meet out needs + // we now need to loop over all of these and for each pair, select the best set of + // inputs + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return false; + } + if (!psArrayLength(output)) { + psWarning("staticskytool: no rows found"); + psFree(output); + return true; + } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "skycalRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + psS64 sky_id = psMetadataLookupS64(&status, row, "sky_id"); + psS64 stack_id = psMetadataLookupS64(&status, row, "stack_id"); + psString sky_workdir = psMetadataLookupStr(&status, row, "workdir"); + psString sky_label = psMetadataLookupStr(&status, row, "label"); + psString sky_data_group = psMetadataLookupStr(&status, row, "data_group"); + + // create a staticskyRun + if (!skycalRunInsert(config->dbh, + 0x0, // skycal_id + sky_id, + stack_id, + "new", // state + workdir ? workdir : sky_workdir, + label ? label : sky_label, + data_group ? data_group : sky_data_group, + dist_group, + reduction, + registered, + note + ) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + } + psFree(output); + return true; +} + +static bool updateskycalrunMode(pxConfig *config) +{ +#ifdef notyet + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE staticskyRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "staticskyRun", "sky_id", "staticskyResult", true, false); + psFree(query); + psFree(where); + + return result; +#endif + return false; +} +static bool pendingskycalrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-skycal_id", "skycal_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-sky_id", "sky_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "skycalRun.label", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-filter", "stackRun.filter", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("staticskytool_pendingskycalrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + // the where clause is required and matches the WHERE hook format string + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("staticskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pendingskycalRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addskycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(skycal_id, config->args, "-skycal_id", true, false); + + // optional + PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); + PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); + + PXOPT_LOOKUP_F32(zpt_obs, config->args, "-zpt_obs", false, false); + PXOPT_LOOKUP_F32(zpt_stdev, config->args, "-zpt_stdev", false, false); + + + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); + + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); +// PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); + PXOPT_LOOKUP_S32(n_astrom, config->args, "-n_astrom", false, false); + + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_psastro, config->args, "-ver_psastro", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + + psString software_ver = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + software_ver = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + software_ver = pxMergeCodeVersions(software_ver,ver_psphot); + } + if (ver_psastro) { + software_ver = pxMergeCodeVersions(software_ver,ver_psastro); + } + if (ver_ppstats) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstats); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + skycalResultRow *row = skycalResultRowAlloc( + skycal_id, + path_base, + dtime_script, + dtime_astrom, + sigma_ra, + sigma_dec, + n_astrom, + zpt_obs, + zpt_stdev, + quality, + software_ver, + hostname, + fault + ); + + if (!skycalResultInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + return false; + } + + psFree(row); + + if (fault) { + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + psString query = "UPDATE skycalRun SET state = 'full' WHERE skycal_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, skycal_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool updateskycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id", "=="); + + if (!pxSetFaultCode(config->dbh, "skycalResult", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} +static bool skycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "skycalRun.sky_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "skycalRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "skycalRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "skycalRun.data_group", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "skycalResult.fault", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("staticskytool_skycalresult.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("staticskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "staticskyResult", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +static bool revertskycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "staticskyResult.sky_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "staticskyRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-data_group", "staticskyRun.data_group", "=="); + pxAddLabelSearchArgs(config, where, "-filter", "stackRun.filter", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("staticskytool_revertskycal.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("staticskytool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} Index: /branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.h =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.h (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/staticskytool.h (revision 33638) @@ -34,5 +34,13 @@ STATICSKYTOOL_MODE_UPDATERESULT, STATICSKYTOOL_MODE_EXPORTRUN, - STATICSKYTOOL_MODE_IMPORTRUN + STATICSKYTOOL_MODE_IMPORTRUN, + + STATICSKYTOOL_MODE_DEFINESKYCALRUN, + STATICSKYTOOL_MODE_PENDINGSKYCALRUN, + STATICSKYTOOL_MODE_UPDATESKYCALRUN, + STATICSKYTOOL_MODE_ADDSKYCALRESULT, + STATICSKYTOOL_MODE_UPDATESKYCALRESULT, + STATICSKYTOOL_MODE_SKYCALRESULT, + STATICSKYTOOL_MODE_REVERTSKYCALRESULT, } staticskytoolMode; Index: /branches/eam_branches/ipp-20111122/ippTools/src/staticskytoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/staticskytoolConfig.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/staticskytoolConfig.c (revision 33638) @@ -48,4 +48,5 @@ // XXX need a 'ra_min,max', 'dec_min,max' : to do this, we need a table of skycell boundaries psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_stack_id", 0, "search for stack_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); @@ -107,8 +108,9 @@ psMetadata *resultArgs= psMetadataAlloc(); psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); - // psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); - // psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs (>=)", 0); psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -138,5 +140,102 @@ psMetadata *importrunArgs = psMetadataAlloc(); psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - + + // -defineskycalrun + psMetadata *defineskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(defineskycalrunArgs, PS_LIST_TAIL, "-select_sky_id", 0, "search for sky_id", 0); + psMetadataAddS64(defineskycalrunArgs, PS_LIST_TAIL, "-select_stack_id", 0, "search for stack_id", 0); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddF32(defineskycalrunArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddTime(defineskycalrunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-rerun", 0, "queue new run even if one already exists for inputs", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-check_inputs", 0, "list inputs, do not modify database", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updateskycalrun + psMetadata *updateskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by stack ID", 0); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -pendingskycalrun + psMetadata *pendingskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(pendingskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(pendingskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(pendingskycalrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddStr(pendingskycalrunArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddU64(pendingskycalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + + // -addskycalresult + psMetadata *addskycalresultArgs = psMetadataAlloc(); + psMetadataAddS64(addskycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0, "define camtool ID (required)", 0); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_ra", 0, "define exposure E ra", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_dec", 0, "define exposure E dec", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_obs", 0, "define observed zero point", 0); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_stdev", 0, "define observed zero point stdandard deviation", 0); + + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-dtime_astrom", 0, "define elapsed time for astrometry (seconds)", NAN); + + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + // psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); + psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_astrom", 0, "define number of astrometry reference objects", 0); + + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addskycalresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + psMetadataAddS16(addskycalresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psastro", 0, "define psastro version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + + // -revertskycal + psMetadata *revertskycalArgs= psMetadataAlloc(); + psMetadataAddS64(revertskycalArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", 0); + psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-filter", PS_META_DUPLICATE_OK, "search by filter", 0); + psMetadataAddS16(revertskycalArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertskycalArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false); + + // -updateskycalresult + psMetadata *updateskycalresultArgs = psMetadataAlloc(); + psMetadataAddS64(updateskycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0, "define staticksky ID (required)", 0); + psMetadataAddS16(updateskycalresultArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updateskycalresultArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality code", 0); + + // -skycalresult + psMetadata *skycalresultArgs= psMetadataAlloc(); + psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); + psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison)", NULL); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(skycalresultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(skycalresultArgs, PS_LIST_TAIL, "-limit", 0, "limit skycalresult set to N items", 0); + psMetadataAddBool(skycalresultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -155,4 +254,11 @@ PXOPT_ADD_MODE("-exportrun", "Export run", STATICSKYTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "Import run", STATICSKYTOOL_MODE_IMPORTRUN, importrunArgs); + PXOPT_ADD_MODE("-defineskycalrun", "Define a new skycalrun", STATICSKYTOOL_MODE_DEFINESKYCALRUN, defineskycalrunArgs); + PXOPT_ADD_MODE("-updateskycalrun", "Update a skycalrun", STATICSKYTOOL_MODE_UPDATESKYCALRUN, updateskycalrunArgs); + PXOPT_ADD_MODE("-pendingskycalrun", "Get skcal runs to do", STATICSKYTOOL_MODE_PENDINGSKYCALRUN, pendingskycalrunArgs); + PXOPT_ADD_MODE("-addskycalresult", "add skycal result", STATICSKYTOOL_MODE_ADDSKYCALRESULT, addskycalresultArgs); + PXOPT_ADD_MODE("-revertskycal", "revert faulted skycal run", STATICSKYTOOL_MODE_REVERTSKYCALRESULT, revertskycalArgs); + PXOPT_ADD_MODE("-updateskycal", "revert faulted skycal run", STATICSKYTOOL_MODE_UPDATESKYCALRESULT, updateskycalresultArgs); + PXOPT_ADD_MODE("-skycalresult", "Get result of skycal run", STATICSKYTOOL_MODE_SKYCALRESULT, skycalresultArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /branches/eam_branches/ipp-20111122/ippTools/src/warptool.c =================================================================== --- /branches/eam_branches/ipp-20111122/ippTools/src/warptool.c (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippTools/src/warptool.c (revision 33638) @@ -2581,4 +2581,13 @@ psFree(query); - return true; -} + if (!skycell_id) { + // If we are updateing a whole warpRun set skycells with bad quality to 'full' + query = "UPDATE warpSkyfile SET data_state ='full', fault = 0 WHERE warp_id = %" PRId64 " AND quality != 0 AND (data_state != 'full')"; + if (!p_psDBRunQueryF(config->dbh, query, warp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + return true; +} Index: /branches/eam_branches/ipp-20111122/ippconfig/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/Makefile.am (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/Makefile.am (revision 33638) @@ -18,4 +18,5 @@ lbc_red \ lulin \ + ssp \ uh8k Index: /branches/eam_branches/ipp-20111122/ippconfig/configure.ac =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/configure.ac (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/configure.ac (revision 33638) @@ -29,4 +29,5 @@ lbc_red/Makefile lulin/Makefile + ssp/Makefile uh8k/Makefile ]) Index: /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20090220.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20090220.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20090220.config (revision 33638) @@ -590,4 +590,6 @@ XY76 STR 1111111111111111111111111111111111111111111111111111111111111111 END +PATTERN.CELL BOOL FALSE +PATTERN.CONTINUITY BOOL TRUE PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction XY01 BOOL FALSE Index: /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20091209.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20091209.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20091209.config (revision 33638) @@ -525,7 +525,135 @@ # PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? +# PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction +# XY01 BOOL FALSE +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 STR 0011100100000000000000000000000000000000000000000000000000000000 +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000000100000000000000000000000000000000000000000000 # xy23 +# XY15 STR 1111111100000000000000000000000000000000000000000000000000000000 # cols: 0 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 BOOL FALSE +# XY25 BOOL FALSE +# XY26 BOOL FALSE +# XY27 STR 1111111111111111111111111111111100000000000000000000000011111111 # cols: 0,1,2,3,7 +# XY30 BOOL FALSE +# XY31 STR 0000000000000000000000000000000000000000000000000000000011111111 # cols: 7 +# XY32 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY33 BOOL FALSE +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# XY36 BOOL FALSE +# XY37 BOOL FALSE +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 BOOL FALSE +# XY44 BOOL FALSE +# XY45 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY46 BOOL FALSE +# XY47 STR 1111111100000000000000001111111100000000111111110000000011111111 # cols: 0,3,5,7 +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 STR 1111111111111111111111110000000000000000000000001111111111111111 # cols: 0,1,2,6,7 +# XY60 STR 0000000000000000000000000000000000000000000001000000000000000000 # cell xy55 +# XY61 BOOL FALSE +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 BOOL FALSE +# XY73 BOOL FALSE +# XY74 STR 0000000000000000111111110000000000000000000000000000000011111111 # cols: 2,7 +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END + +# # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 +# XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 +# XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY35 BOOL FALSE +# XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 +# XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 +# XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END +# PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction XY01 BOOL FALSE - XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 + XY02 BOOL FALSE XY03 BOOL FALSE XY04 BOOL FALSE @@ -536,5 +664,5 @@ XY12 BOOL FALSE XY13 BOOL FALSE - XY14 STR 0000000000000000000100000000000000000000000000000000000000000000 # xy23 + XY14 STR 0000000000000000000100000000000000000000000000000000000000000000 # xy23 XY15 STR 1111111100000000000000000000000000000000000000000000000000000000 # cols: 0 XY16 BOOL FALSE @@ -578,5 +706,6 @@ XY64 BOOL FALSE XY65 BOOL FALSE - XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 + XY66 BOOL FALSE XY67 BOOL FALSE XY71 BOOL FALSE @@ -588,65 +717,82 @@ END -# PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction -PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction - XY01 BOOL FALSE - XY02 BOOL FALSE - XY03 BOOL FALSE - XY04 BOOL FALSE - XY05 BOOL FALSE - XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 - XY10 BOOL FALSE - XY11 BOOL FALSE - XY12 BOOL FALSE - XY13 BOOL FALSE - XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 - XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 - XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 - XY17 BOOL FALSE - XY20 BOOL FALSE - XY21 BOOL FALSE - XY22 BOOL FALSE - XY23 BOOL FALSE - XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 - XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 - XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 - XY27 BOOL FALSE - XY30 BOOL FALSE - XY31 BOOL FALSE - XY32 BOOL FALSE - XY33 BOOL FALSE - XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 - XY35 BOOL FALSE - XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 - XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 - XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 - XY41 BOOL FALSE - XY42 BOOL FALSE - XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 - XY44 BOOL FALSE - XY45 BOOL FALSE - XY46 BOOL FALSE - XY47 BOOL FALSE - XY50 BOOL FALSE - XY51 BOOL FALSE - XY52 BOOL FALSE - XY53 BOOL FALSE - XY54 BOOL FALSE - XY55 BOOL FALSE - XY56 BOOL FALSE - XY57 BOOL FALSE - XY60 BOOL FALSE - XY61 BOOL FALSE - XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 - XY63 BOOL FALSE - XY64 BOOL FALSE - XY65 BOOL FALSE - XY66 BOOL FALSE - XY67 BOOL FALSE - XY71 BOOL FALSE - XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 - XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 - XY74 BOOL FALSE - XY75 BOOL FALSE - XY76 BOOL FALSE -END +PATTERN.CELL BOOL FALSE +PATTERN.CONTINUITY BOOL TRUE + +# # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# # XY06 STR 0101000000010000000110000100000000000000000000000000000000000000 +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000000000000000000000 +# XY15 STR 1111111110000000100000000000000000000000000000000000000000000000 +# # XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# # XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY24 BOOL FALSE +# # XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY25 BOOL FALSE +# # XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY26 BOOL FALSE +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# # XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# # XY36 STR 0001000000000000000000000000000000010000000000000000000000000000 +# XY36 BOOL FALSE +# # XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 ? +# XY37 BOOL FALSE +# # XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# # XY42 BOOL FALSE +# XY42 STR 0000000000000000000100000001000000000000000000000000000000000000 +# # XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY43 STR 0000000000000000000000000000000000000000000010000000000000000000 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# # XY62 STR 0000000000000000000000000000000000000000000000000000000000000010 +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# # XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY72 BOOL FALSE +# # XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY73 BOOL FALSE +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END Index: /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20100122.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20100122.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20100122.config (revision 33638) @@ -527,8 +527,136 @@ # Pattern correction information: +# # PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? +# PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction +# XY01 BOOL FALSE +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 STR 0011100100000000000000000000000000000000000000000000000000000000 +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000000100000000000000000000000000000000000000000000 # xy23 +# XY15 STR 1111111100000000000000000000000000000000000000000000000000000000 # cols: 0 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 BOOL FALSE +# XY25 BOOL FALSE +# XY26 BOOL FALSE +# XY27 STR 1111111111111111111111111111111100000000000000000000000011111111 # cols: 0,1,2,3,7 +# XY30 BOOL FALSE +# XY31 STR 0000000000000000000000000000000000000000000000000000000011111111 # cols: 7 +# XY32 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY33 BOOL FALSE +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# XY36 BOOL FALSE +# XY37 BOOL FALSE +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 BOOL FALSE +# XY44 BOOL FALSE +# XY45 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY46 BOOL FALSE +# XY47 STR 1111111100000000000000001111111100000000111111110000000011111111 # cols: 0,3,5,7 +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 STR 1111111111111111111111110000000000000000000000001111111111111111 # cols: 0,1,2,6,7 +# XY60 STR 0000000000000000000000000000000000000000000001000000000000000000 # cell xy55 +# XY61 BOOL FALSE +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 BOOL FALSE +# XY73 BOOL FALSE +# XY74 STR 0000000000000000111111110000000000000000000000000000000011111111 # cols: 2,7 +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END + +# # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 +# XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 +# XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY35 BOOL FALSE +# XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 +# XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 +# XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END # PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction XY01 BOOL FALSE - XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 + XY02 BOOL FALSE XY03 BOOL FALSE XY04 BOOL FALSE @@ -581,5 +709,6 @@ XY64 BOOL FALSE XY65 BOOL FALSE - XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 + XY66 BOOL FALSE XY67 BOOL FALSE XY71 BOOL FALSE @@ -591,65 +720,82 @@ END -# PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction -PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction - XY01 BOOL FALSE - XY02 BOOL FALSE - XY03 BOOL FALSE - XY04 BOOL FALSE - XY05 BOOL FALSE - XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 - XY10 BOOL FALSE - XY11 BOOL FALSE - XY12 BOOL FALSE - XY13 BOOL FALSE - XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 - XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 - XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 - XY17 BOOL FALSE - XY20 BOOL FALSE - XY21 BOOL FALSE - XY22 BOOL FALSE - XY23 BOOL FALSE - XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 - XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 - XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 - XY27 BOOL FALSE - XY30 BOOL FALSE - XY31 BOOL FALSE - XY32 BOOL FALSE - XY33 BOOL FALSE - XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 - XY35 BOOL FALSE - XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 - XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 - XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 - XY41 BOOL FALSE - XY42 BOOL FALSE - XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 - XY44 BOOL FALSE - XY45 BOOL FALSE - XY46 BOOL FALSE - XY47 BOOL FALSE - XY50 BOOL FALSE - XY51 BOOL FALSE - XY52 BOOL FALSE - XY53 BOOL FALSE - XY54 BOOL FALSE - XY55 BOOL FALSE - XY56 BOOL FALSE - XY57 BOOL FALSE - XY60 BOOL FALSE - XY61 BOOL FALSE - XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 - XY63 BOOL FALSE - XY64 BOOL FALSE - XY65 BOOL FALSE - XY66 BOOL FALSE - XY67 BOOL FALSE - XY71 BOOL FALSE - XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 - XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 - XY74 BOOL FALSE - XY75 BOOL FALSE - XY76 BOOL FALSE -END +PATTERN.CELL BOOL FALSE +PATTERN.CONTINUITY BOOL TRUE + +# # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# # XY06 STR 0101000000010000000110000100000000000000000000000000000000000000 +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000000000000000000000 +# XY15 STR 1111111110000000100000000000000000000000000000000000000000000000 +# # XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# # XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY24 BOOL FALSE +# # XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY25 BOOL FALSE +# # XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY26 BOOL FALSE +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# # XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# # XY36 STR 0001000000000000000000000000000000010000000000000000000000000000 +# XY36 BOOL FALSE +# # XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 ? +# XY37 BOOL FALSE +# # XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# # XY42 BOOL FALSE +# XY42 STR 0000000000000000000100000001000000000000000000000000000000000000 +# # XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY43 STR 0000000000000000000000000000000000000000000010000000000000000000 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# # XY62 STR 0000000000000000000000000000000000000000000000000000000000000010 +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# # XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY72 BOOL FALSE +# # XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY73 BOOL FALSE +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END Index: /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20100228.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20100228.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20100228.config (revision 33638) @@ -526,8 +526,136 @@ # Pattern correction information: +# # PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? +# PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction +# XY01 BOOL FALSE +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 STR 0011100100000000000000000000000000000000000000000000000000000000 +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000000100000000000000000000000000000000000000000000 # xy23 +# XY15 STR 1111111100000000000000000000000000000000000000000000000000000000 # cols: 0 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 BOOL FALSE +# XY25 BOOL FALSE +# XY26 BOOL FALSE +# XY27 STR 1111111111111111111111111111111100000000000000000000000011111111 # cols: 0,1,2,3,7 +# XY30 BOOL FALSE +# XY31 STR 0000000000000000000000000000000000000000000000000000000011111111 # cols: 7 +# XY32 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY33 BOOL FALSE +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# XY36 BOOL FALSE +# XY37 BOOL FALSE +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 BOOL FALSE +# XY44 BOOL FALSE +# XY45 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY46 BOOL FALSE +# XY47 STR 1111111100000000000000001111111100000000111111110000000011111111 # cols: 0,3,5,7 +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 STR 1111111111111111111111110000000000000000000000001111111111111111 # cols: 0,1,2,6,7 +# XY60 STR 0000000000000000000000000000000000000000000001000000000000000000 # cell xy55 +# XY61 BOOL FALSE +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 BOOL FALSE +# XY73 BOOL FALSE +# XY74 STR 0000000000000000111111110000000000000000000000000000000011111111 # cols: 2,7 +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END + +# # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 +# XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 +# XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY35 BOOL FALSE +# XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 +# XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 +# XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END # PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction XY01 BOOL FALSE - XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 + XY02 BOOL FALSE XY03 BOOL FALSE XY04 BOOL FALSE @@ -580,5 +708,6 @@ XY64 BOOL FALSE XY65 BOOL FALSE - XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 + XY66 BOOL FALSE XY67 BOOL FALSE XY71 BOOL FALSE @@ -590,65 +719,82 @@ END +PATTERN.CELL BOOL FALSE +PATTERN.CONTINUITY BOOL TRUE + # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction -PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction - XY01 BOOL FALSE - XY02 BOOL FALSE - XY03 BOOL FALSE - XY04 BOOL FALSE - XY05 BOOL FALSE - XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 - XY10 BOOL FALSE - XY11 BOOL FALSE - XY12 BOOL FALSE - XY13 BOOL FALSE - XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 - XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 - XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 - XY17 BOOL FALSE - XY20 BOOL FALSE - XY21 BOOL FALSE - XY22 BOOL FALSE - XY23 BOOL FALSE - XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 - XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 - XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 - XY27 BOOL FALSE - XY30 BOOL FALSE - XY31 BOOL FALSE - XY32 BOOL FALSE - XY33 BOOL FALSE - XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 - XY35 BOOL FALSE - XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 - XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 - XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 - XY41 BOOL FALSE - XY42 BOOL FALSE - XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 - XY44 BOOL FALSE - XY45 BOOL FALSE - XY46 BOOL FALSE - XY47 BOOL FALSE - XY50 BOOL FALSE - XY51 BOOL FALSE - XY52 BOOL FALSE - XY53 BOOL FALSE - XY54 BOOL FALSE - XY55 BOOL FALSE - XY56 BOOL FALSE - XY57 BOOL FALSE - XY60 BOOL FALSE - XY61 BOOL FALSE - XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 - XY63 BOOL FALSE - XY64 BOOL FALSE - XY65 BOOL FALSE - XY66 BOOL FALSE - XY67 BOOL FALSE - XY71 BOOL FALSE - XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 - XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 - XY74 BOOL FALSE - XY75 BOOL FALSE - XY76 BOOL FALSE -END +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# # XY06 STR 0101000000010000000110000100000000000000000000000000000000000000 +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000000000000000000000 +# XY15 STR 1111111110000000100000000000000000000000000000000000000000000000 +# # XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# # XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY24 BOOL FALSE +# # XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY25 BOOL FALSE +# # XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY26 BOOL FALSE +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# # XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# # XY36 STR 0001000000000000000000000000000000010000000000000000000000000000 +# XY36 BOOL FALSE +# # XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 ? +# XY37 BOOL FALSE +# # XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# # XY42 BOOL FALSE +# XY42 STR 0000000000000000000100000001000000000000000000000000000000000000 +# # XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY43 STR 0000000000000000000000000000000000000000000010000000000000000000 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# # XY62 STR 0000000000000000000000000000000000000000000000000000000000000010 +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# # XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY72 BOOL FALSE +# # XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY73 BOOL FALSE +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END Index: /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20100723.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20100723.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/gpc1/format_20100723.config (revision 33638) @@ -529,5 +529,6 @@ PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction XY01 BOOL FALSE - XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 + XY02 BOOL FALSE XY03 BOOL FALSE XY04 BOOL FALSE @@ -580,5 +581,6 @@ XY64 BOOL FALSE XY65 BOOL FALSE - XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 + XY66 BOOL FALSE XY67 BOOL FALSE XY71 BOOL FALSE @@ -590,65 +592,80 @@ END -# PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction -PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction - XY01 BOOL FALSE - XY02 BOOL FALSE - XY03 BOOL FALSE - XY04 BOOL FALSE - XY05 BOOL FALSE - XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 - XY10 BOOL FALSE - XY11 BOOL FALSE - XY12 BOOL FALSE - XY13 BOOL FALSE - XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 - XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 - XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 - XY17 BOOL FALSE - XY20 BOOL FALSE - XY21 BOOL FALSE - XY22 BOOL FALSE - XY23 BOOL FALSE - XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 - XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 - XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 - XY27 BOOL FALSE - XY30 BOOL FALSE - XY31 BOOL FALSE - XY32 BOOL FALSE - XY33 BOOL FALSE - XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 - XY35 BOOL FALSE - XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 - XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 - XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 - XY41 BOOL FALSE - XY42 BOOL FALSE - XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 - XY44 BOOL FALSE - XY45 BOOL FALSE - XY46 BOOL FALSE - XY47 BOOL FALSE - XY50 BOOL FALSE - XY51 BOOL FALSE - XY52 BOOL FALSE - XY53 BOOL FALSE - XY54 BOOL FALSE - XY55 BOOL FALSE - XY56 BOOL FALSE - XY57 BOOL FALSE - XY60 BOOL FALSE - XY61 BOOL FALSE - XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 - XY63 BOOL FALSE - XY64 BOOL FALSE - XY65 BOOL FALSE - XY66 BOOL FALSE - XY67 BOOL FALSE - XY71 BOOL FALSE - XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 - XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 - XY74 BOOL FALSE - XY75 BOOL FALSE - XY76 BOOL FALSE -END +PATTERN.CONTINUITY BOOL TRUE + PATTERN.CELL BOOL FALSE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# # XY06 STR 0101000000010000000110000100000000000000000000000000000000000000 +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000000000000000000000 +# XY15 STR 1111111110000000100000000000000000000000000000000000000000000000 +# # XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# # XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY24 BOOL FALSE +# # XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY25 BOOL FALSE +# # XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY26 BOOL FALSE +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# # XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# # XY36 STR 0001000000000000000000000000000000010000000000000000000000000000 +# XY36 BOOL FALSE +# # XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 ? +# XY37 BOOL FALSE +# # XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# # XY42 BOOL FALSE +# XY42 STR 0000000000000000000100000001000000000000000000000000000000000000 +# # XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY43 STR 0000000000000000000000000000000000000000000010000000000000000000 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# # XY62 STR 0000000000000000000000000000000000000000000000000000000000000010 +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# # XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY72 BOOL FALSE +# # XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY73 BOOL FALSE +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END Index: /branches/eam_branches/ipp-20111122/ippconfig/gpc1/ppImage.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/gpc1/ppImage.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/gpc1/ppImage.config (revision 33638) @@ -159,4 +159,5 @@ BIAS BOOL FALSE # Bias subtraction DARK BOOL TRUE # Dark subtraction + NOISEMAP BOOL TRUE REMNANCE BOOL FALSE # Remnance masking SHUTTER BOOL FALSE # Shutter correction @@ -166,5 +167,36 @@ MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails PATTERN.ROW BOOL TRUE # Row pattern correction - PATTERN.CELL BOOL TRUE # Cell pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE # Cell pattern correction using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL TRUE # Subtract model background? +END + +# Chip processing for testing pattern correction +CHIP_PATTERNTEST METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL TRUE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL FALSE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -196,5 +228,6 @@ MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails PATTERN.ROW BOOL TRUE # Row pattern correction - PATTERN.CELL BOOL TRUE # Cell pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -280,4 +313,36 @@ END END + +CHIP_NOISEMAP METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL TRUE # apply non-linearity correction + NOISEMAP BOOL TRUE + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE # Cell pattern correction using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL TRUE # Subtract model background? +END + # Standard chip processing @@ -1001,4 +1066,40 @@ END +PPIMAGE_NOISEMAP METADATA + BASE.FITS BOOL TRUE # Save base image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + CHECK.CTE BOOL FALSE # measure CTE errors? + CHECK.NOISE BOOL TRUE # measure cell level noise. + +# DETREND.CONSTRAINTS METADATA +# DARK METADATA +# DETTYPE STR DARK_PREMASK +# EXPTIME STR FPA.EXPOSURE +# END +# FLAT METADATA +# DETTYPE STR FLAT_PREMASK +# FILTER STR FPA.FILTERID +# END +# END +END + PR_DARKTEST_NOSUB METADATA @@ -1129,6 +1230,6 @@ MASK.BUILD BOOL TRUE # Build internal mask? MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails - PATTERN.ROW BOOL FALSE # Row pattern correction - PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL TRUE # Cell pattern correction FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? Index: /branches/eam_branches/ipp-20111122/ippconfig/gpc1/psastro.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/gpc1/psastro.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/gpc1/psastro.config (revision 33638) @@ -344,4 +344,6 @@ PSASTRO.MATCH.RADIUS.N7 F32 5 PSASTRO.MATCH.FIT.NITER S32 5 + + PSASTRO.CATDIR STR STS.PP5.REFCAT END Index: /branches/eam_branches/ipp-20111122/ippconfig/lbc_red/filerules.mdc =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/lbc_red/filerules.mdc (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/lbc_red/filerules.mdc (revision 33638) @@ -136,4 +136,5 @@ PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE NONE +PSASTRO.OUTPUT.CMF OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE NONE PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE NONE PSASTRO.OUT.ASTROM OUTPUT {OUTPUT}.ast ASTROM NONE FPA TRUE NONE Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/filerules-mef.mdc =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/filerules-mef.mdc (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/filerules-mef.mdc (revision 33638) @@ -42,4 +42,5 @@ PSASTRO.INPUT STR PSASTRO.INPUT.CMF PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF +#PSASTRO.OUTPUT.CMF STR PSASTRO.OUT.CMF DVOCORR.OUTPUT STR DVOCORR.MEF.OUTPUT DVOFLAT.OUTPUT STR DVOFLAT.MEF.OUTPUT @@ -84,4 +85,5 @@ PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF PSPHOT.INPUT.TEXT INPUT @FILES CHIP SRCTEXT +PSPHOT.EXPNUM INPUT @FILES CHIP MASK ## files used by psastro @@ -202,4 +204,5 @@ PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.NOISEMAP OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.fits FRINGE NONE CHIP TRUE NONE PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.sigma.fits IMAGE NONE CHIP TRUE NONE @@ -243,4 +246,6 @@ PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE SPLIT PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUTPUT.CMF OUTPUT {OUTPUT}.cmf CMF NONE CHIP TRUE NONE +#PSASTRO.OUT.CMF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE NONE PSASTRO.OUT.MODEL OUTPUT {OUTPUT}.asm ASTROM.MODEL NONE FPA TRUE NONE PSASTRO.OUT.REFSTARS OUTPUT {OUTPUT}.aref.fits ASTROM.REFSTARS NONE FPA TRUE NONE @@ -304,9 +309,14 @@ PPSTACK.CONFIG OUTPUT {OUTPUT}.ppStack.mdc TEXT NONE FPA TRUE NONE -PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE -PPSTAMP.OUTPUT.DIFF OUTPUT {OUTPUT}.fits IMAGE COMP_SUB FPA TRUE NONE -PPSTAMP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE -PPSTAMP.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits VARIANCE COMP_WT FPA TRUE NONE -PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PPSTAMP.OUTPUT.NOCOMP OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.OUTPUT.DIFF OUTPUT {OUTPUT}.fits IMAGE COMP_SUB FPA TRUE NONE +PPSTAMP.OUTPUT.DIFF.NOCOMP OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PPSTAMP.OUTPUT.MASK.NOCMOP OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE +PPSTAMP.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits VARIANCE COMP_WT FPA TRUE NONE +PPSTAMP.OUTPUT.VARIANCE.NOCOMP OUTPUT {OUTPUT}.wt.fits VARIANCE NONE FPA TRUE NONE +PPSTAMP.OUTPUT.JPEG OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE +PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF PPSIM.OUTPUT.MEF OUTPUT {OUTPUT}.fits IMAGE NONE CHIP TRUE NONE Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/filerules-simple.mdc =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/filerules-simple.mdc (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/filerules-simple.mdc (revision 33638) @@ -56,4 +56,5 @@ PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF PSPHOT.INPUT.TEXT INPUT @FILES CHIP SRCTEXT +PSPHOT.EXPNUM INPUT @FILES CHIP MASK ## files used by psphotStack @@ -179,4 +180,5 @@ PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE SIMPLE PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE SIMPLE +PPMERGE.OUTPUT.NOISEMAP OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE SIMPLE PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.fits FRINGE NONE FPA TRUE SIMPLE PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.sigma.fits IMAGE NONE FPA TRUE SIMPLE @@ -287,9 +289,14 @@ PPSTACK.CONFIG OUTPUT {OUTPUT}.ppStack.mdc TEXT NONE FPA TRUE NONE -PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE -PPSTAMP.OUTPUT.DIFF OUTPUT {OUTPUT}.fits IMAGE COMP_SUB FPA TRUE NONE -PPSTAMP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE -PPSTAMP.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits VARIANCE COMP_WT FPA TRUE NONE -PPSTAMP.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE NONE +PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PPSTAMP.OUTPUT.NOCOMP OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.OUTPUT.DIFF OUTPUT {OUTPUT}.fits IMAGE COMP_SUB FPA TRUE NONE +PPSTAMP.OUTPUT.DIFF.NOCOMP OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE +PPSTAMP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PPSTAMP.OUTPUT.MASK.NOCMOP OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE +PPSTAMP.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits VARIANCE COMP_WT FPA TRUE NONE +PPSTAMP.OUTPUT.VARIANCE.NOCOMP OUTPUT {OUTPUT}.wt.fits VARIANCE NONE FPA TRUE NONE +PPSTAMP.OUTPUT.JPEG OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE +PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF PPSIM.OUTPUT OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE SIMPLE Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/filerules-split.mdc (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/filerules-split.mdc (revision 33638) @@ -13,5 +13,5 @@ SOURCE.PLOT.APRESID STR SOURCE.PLOT.RAW.APRESID -# PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.MEF +PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.MEF # PSASTRO.OUTPUT.SPL STR PSASTRO.OUT.CMF.SPL # PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF @@ -74,4 +74,5 @@ PSPHOT.INPUT.CMF INPUT @FILES CHIP CMF PSPHOT.INPUT.TEXT INPUT @FILES CHIP SRCTEXT +PSPHOT.EXPNUM INPUT @FILES CHIP MASK ## files used by psphotStack @@ -197,4 +198,5 @@ PPMERGE.OUTPUT.SHUTTER OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE PPMERGE.OUTPUT.FLAT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE +PPMERGE.OUTPUT.NOISEMAP OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE PPMERGE.OUTPUT.FRINGE OUTPUT {OUTPUT}.{CHIP.NAME}.fits FRINGE NONE CELL TRUE NONE PPMERGE.OUTPUT.SIGMA OUTPUT {OUTPUT}.{CHIP.NAME}.sigma.fits IMAGE NONE CELL TRUE NONE @@ -245,8 +247,8 @@ # PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE # PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE NONE -# PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF PSASTRO.OUT.MODEL OUTPUT {OUTPUT}.asm ASTROM.MODEL NONE FPA TRUE NONE PSASTRO.OUT.REFSTARS OUTPUT {OUTPUT}.aref.fits ASTROM.REFSTARS NONE FPA TRUE NONE -PSASTRO.OUTPUT OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUTPUT.CMF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE PSASTRO.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE PSASTRO.CONFIG OUTPUT {OUTPUT}.psastro.mdc TEXT NONE FPA TRUE NONE @@ -319,10 +321,14 @@ PPSTAMP.OUTPUT OUTPUT {OUTPUT}.fits IMAGE COMP_IMG FPA TRUE NONE +PPSTAMP.OUTPUT.NOCOMP OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE PPSTAMP.OUTPUT.DIFF OUTPUT {OUTPUT}.fits IMAGE COMP_SUB FPA TRUE NONE +PPSTAMP.OUTPUT.DIFF.NOCOMP OUTPUT {OUTPUT}.fits IMAGE NONE FPA TRUE NONE PPSTAMP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE +PPSTAMP.OUTPUT.MASK.NOCOMP OUTPUT {OUTPUT}.mk.fits MASK NONE FPA TRUE NONE PPSTAMP.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits VARIANCE COMP_WT FPA TRUE NONE +PPSTAMP.OUTPUT.VARIANCE.NOCOMP OUTPUT {OUTPUT}.wt.fits VARIANCE NONE FPA TRUE NONE +PPSTAMP.OUTPUT.JPEG OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE # mosaic image used for making stamps of raw images. Not saved. PPSTAMP.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF -PPSTAMP.OUTPUT.JPEG1 OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE PPSIM.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/jpeg.mdc =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/jpeg.mdc (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/jpeg.mdc (revision 33638) @@ -236,4 +236,21 @@ END END +NOISEMAP METADATA + PPIMAGE.JPEG1 METADATA + COLORMAP STR rainbow + SCALE.MODE STR VALUE + SCALE.MIN F32 0.0 + SCALE.MAX F32 10.0 + OPTIONS STR +SB,-X,+Y + END + + PPIMAGE.JPEG2 METADATA + COLORMAP STR rainbow + SCALE.MODE STR VALUE + SCALE.MIN F32 0.0 + SCALE.MAX F32 10.0 + OPTIONS STR -SB,-X,+Y + END +END @@ -254,2 +271,17 @@ END +PPSTAMP.OUTPUT.JPEG METADATA + COLORMAP STR greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 +10.0 + OPTIONS STR -SB,+X,+Y +END + +PPSTAMP.OUTPUT.JPEG.FLIP METADATA + COLORMAP STR greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 +10.0 + OPTIONS STR -SB,-X,+Y +END Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/nightly_science.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/nightly_science.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/nightly_science.config (revision 33638) @@ -2,15 +2,15 @@ CLEAN_MODES METADATA MODE STR CHIP - COMMAND STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked + COMMAND STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME U16 21 END CLEAN_MODES METADATA MODE STR WARP - COMMAND STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked + COMMAND STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME U16 7 END CLEAN_MODES METADATA MODE STR DIFF - COMMAND STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked + COMMAND STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME S16 21 END @@ -159,5 +159,5 @@ NAME STR MD08 DISTRIBUTION STR MD08 - TESS STR MD08.V2 + TESS STR MD08.V3 OBSMODE STR MD OBJECT STR MD08% @@ -170,5 +170,5 @@ NAME STR MD09 DISTRIBUTION STR MD09 - TESS STR MD09.V2 + TESS STR MD09.V3 OBSMODE STR MD OBJECT STR MD09% @@ -192,5 +192,5 @@ NAME STR MD11 DISTRIBUTION STR MD11 - TESS STR MD11.V2 + TESS STR MD11.V3 OBSMODE STR MD OBJECT STR MD11% @@ -219,7 +219,8 @@ TESS STR RINGS.V3 OBSMODE STR SS - STACKABLE BOOL FALSE + STACKABLE BOOL TRUE DIFFABLE BOOL TRUE - REDUCTION STR SSTF_4SIG + REDUCTION STR SWEETSPOT +# REDUCTION STR SSTF_4SIG # WARP S16 60 END @@ -229,7 +230,8 @@ TESS STR RINGS.V3 OBSMODE STR OSS - STACKABLE BOOL FALSE - DIFFABLE BOOL TRUE - REDUCTION STR SSTF_4SIG + STACKABLE BOOL TRUE + DIFFABLE BOOL TRUE + REDUCTION STR SWEETSPOT +# REDUCTION STR SSTF_4SIG END TARGETS METADATA @@ -238,7 +240,8 @@ TESS STR RINGS.V3 OBSMODE STR ESS - STACKABLE BOOL FALSE - DIFFABLE BOOL TRUE - REDUCTION STR SSTF_4SIG + STACKABLE BOOL TRUE + DIFFABLE BOOL TRUE + REDUCTION STR SWEETSPOT +# REDUCTION STR SSTF_4SIG END TARGETS METADATA @@ -247,7 +250,8 @@ TESS STR RINGS.V3 OBSMODE STR MSS - STACKABLE BOOL FALSE - DIFFABLE BOOL TRUE - REDUCTION STR SSTF_4SIG + STACKABLE BOOL TRUE + DIFFABLE BOOL TRUE + REDUCTION STR SWEETSPOT +# REDUCTION STR SSTF_4SIG END TARGETS METADATA @@ -266,4 +270,6 @@ STACKABLE BOOL FALSE DIFFABLE BOOL FALSE + CHIP S16 800 + WARP S16 800 END TARGETS METADATA Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppImage.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppImage.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppImage.config (revision 33638) @@ -27,4 +27,7 @@ PATTERN.SUBSET METADATA END +PATTERN.CONTINUITY BOOL FALSE +PATTERN.CONTINUITY.WIDTH S32 50 +PATTERN.CONTINUITY.SUBSET BOOL TRUE FRINGE BOOL FALSE # Fringe subtraction PHOTOM BOOL FALSE # Source identification and photometry @@ -33,4 +36,5 @@ BACKGROUND BOOL FALSE # Subtract model background? CHECK.CTE BOOL FALSE # measure CTE errors? +CHECK.NOISE BOOL FALSE USE.DEBURNED.IMAGE BOOL FALSE # use burntool-repaired image? USE.BEST.BURNTOOL BOOL FALSE # require the best burntooled data @@ -107,5 +111,5 @@ PATTERN.ROW.MEAN STR SAMPLE_MEAN # Statistic for mean for row pattern PATTERN.ROW.STDEV STR SAMPLE_STDEV # Statistic for standard deviation for row pattern -PATTERN.CELL.BG STR ROBUST_MEDIAN # Statistic for background for cell pattern +PATTERN.CELL.BG STR FITTED_MEAN # Statistic for background for cell pattern PATTERN.CELL.MEAN STR SAMPLE_MEDIAN # Statistic for mean for cell pattern @@ -127,4 +131,7 @@ CTE.XBIN S32 20 CTE.YBIN S32 20 + +NOISE.XBIN S32 20 +NOISE.YBIN S32 20 PPIMAGE.JPEG1 METADATA @@ -177,4 +184,31 @@ BACKGROUND BOOL TRUE # Subtract background? MASK.STATS BOOL TRUE # Calculate Mask statistics. +END + +# Standard chip processing +CHIP_PATTERNTEST METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + MASK.STATS BOOL TRUE # Calculate Mask statistics. + PATTERN.CELL BOOL FALSE + PATTERN.ROW BOOL FALSE END @@ -214,4 +248,28 @@ END +CHIP_NOISEMAP METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + NOISEMAP BOOL TRUE + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + MASK.STATS BOOL TRUE # Calculate Mask statistics. +END @@ -1842,4 +1900,65 @@ END END + +PPIMAGE_NOISEMAP METADATA + BASE.FITS BOOL TRUE # Save base image? + BASE.MASK.FITS BOOL TRUE # Save base detrended image? + BASE.VARIANCE.FITS BOOL TRUE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + CHECK.CTE BOOL FALSE # measure CTE errors? + CHECK.NOISE BOOL TRUE # measure cell level noise. + +# DETREND.CONSTRAINTS METADATA +# DARK METADATA +# DETTYPE STR DARK_PREMASK +# EXPTIME STR FPA.EXPOSURE +# END +# FLAT METADATA +# DETTYPE STR FLAT_PREMASK +# FILTER STR FPA.FILTERID +# END +# END +END +PPIMAGE_NM METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + OVERSCAN BOOL FALSE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + NOISEMAP BOOL TRUE + CHECK.NOISE BOOL TRUE + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? +END + # For SDSS cameras Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppMerge.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppMerge.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppMerge.config (revision 33638) @@ -123,4 +123,12 @@ END + +PPMERGE_NOISEMAP METADATA + REJ F32 3.0 + ITER S32 2 + VARIANCES BOOL FALSE + COMBINE STR MEAN +END + # Shutter generation --- already included in default, above PPMERGE_SHUTTER METADATA Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppStack.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppStack.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppStack.config (revision 33638) @@ -19,5 +19,5 @@ POOR.FRACTION F32 0.25 # Maximum fraction of bad variance for poor pixels THRESHOLD.MASK F32 0.5 # Threshold for mask deconvolution (0..1) -DECONV.LIMIT F32 2.0 # Deconvolution fraction for rejecting entire image +DECONV.LIMIT F32 3.0 # Deconvolution fraction for rejecting entire image IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image MATCH.REJ F32 3.0 # Rejection threshold for chi^2 values from matching Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppStatsFromMetadata.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppStatsFromMetadata.config (revision 33638) @@ -536,2 +536,24 @@ # ENTRY VAL REPLICATE_OUTPUTS F32 CONSTANT -dtime_replicate # time to replicate the outputs END + +STATICSKY_CAL METADATA + ENTRY MULTI + TYPE VAL KEYWORD TYPE STATISTIC FLAG + ENTRY VAL CERROR F32 CONSTANT -sigma_ra + ENTRY VAL CERROR F32 CONSTANT -sigma_dec + ENTRY VAL DT_ASTR F32 CONSTANT -dtime_astrom + ENTRY VAL NASTRO S32 CONSTANT -n_astrom + + ENTRY VAL ZPT_OBS F32 CONSTANT -zpt_obs + ENTRY VAL ZPT_ERR F32 CONSTANT -zpt_stdev + + ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag + + # Revision values + ENTRY VAL PSLIB_V STR CONSTANT -ver_pslib + ENTRY VAL MODULE_V STR CONSTANT -ver_psmodules + ENTRY VAL PHOT_V STR CONSTANT -ver_psphot + ENTRY VAL ASTRO_V STR CONSTANT -ver_psastro + ENTRY VAL STATS_V STR CONSTANT -ver_ppstats +# ENTRY VAL IMAGE_V STR CONSTANT -ver_ppimage +END Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppSub.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppSub.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/ppSub.config (revision 33638) @@ -85,4 +85,5 @@ # Recipe overrides for STACK STACK METADATA + SYS.ERR F32 0.0 # current use of SYS.ERR not appropriate for stacks END Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/psastro.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/psastro.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/psastro.config (revision 33638) @@ -258,2 +258,9 @@ END +STATICSKY_CAL METADATA + REFSTAR_MASK BOOL FALSE + PSASTRO.SAVE.REFMATCH BOOL FALSE + PSASTRO.FIX.CHIPS BOOL FALSE + PSASTRO.USE.MODEL BOOL FALSE + PSASTRO.MOSAIC.MODE BOOL FALSE +END Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/reductionClasses.mdc (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/reductionClasses.mdc (revision 33638) @@ -140,4 +140,11 @@ FRINGE_JPEG_IMAGE STR FRINGE FRINGE_JPEG_RESID STR FRINGE_RESID + + NOISEMAP_PROCESS STR PPIMAGE_NOISEMAP + NOISEMAP_RESID STR PPIMAGE_NM + NOISEMAP_VERIFY STR PPIMAGE_NM + NOISEMAP_STACK STR PPMERGE_NOISEMAP + NOISEMAP_JPEG_IMAGE STR NOISEMAP + NOISEMAP_JPEG_RESID STR NOISEMAP END @@ -164,4 +171,5 @@ BACKGROUND_PSWARP STR BACKGROUND PSVIDEOPHOT STR PSVIDEOPHOT + STATICSKY_CALIBRATION STR STATICSKY_CAL END @@ -218,4 +226,29 @@ END +# Sweetspot reduction class +SWEETSPOT METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR QUICKSTACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR SSTF_DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT +END + + CONSTANT_WEIGHTPHOT METADATA CHIP_PPIMAGE STR CHIP @@ -285,4 +318,30 @@ BACKGROUND_PSWARP STR BACKGROUND END + +DEFAULT_NMTEST METADATA + CHIP_PPIMAGE STR CHIP_NOISEMAP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT + STATICSKY_CALIBRATION STR STATICSKY_CAL +END + + # SSTF tests @@ -310,4 +369,27 @@ END +SSTF_4SIG METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR SSTF_DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT +END + SSTF_NOCONV_4SIG METADATA CHIP_PPIMAGE STR CHIP @@ -333,26 +415,4 @@ END -SSTF_4SIG METADATA - CHIP_PPIMAGE STR CHIP - CHIP_PSPHOT STR CHIP - WARP_PSWARP STR WARP - STACK_PPSTACK STR STACK - STACK_PPSUB STR STACK - STACK_PSPHOT STR STACK - DIFF_PPSUB STR DIFF - DIFF_PSPHOT STR SSTF_DIFF - JPEG_BIN1 STR PPIMAGE_J1 - JPEG_BIN2 STR PPIMAGE_J2 - FAKEPHOT STR FAKEPHOT - ADDSTAR STR ADDSTAR - PSASTRO STR DEFAULT_RECIPE - STACKPHOT_PSPHOT STR STACKPHOT - STACKPHOT_PPSUB STR STACKPHOT - STACKPHOT_PPSTACK STR STACKPHOT - STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE - BACKGROUND_PPBACKGROUND STR BACKGROUND - BACKGROUND_PSWARP STR BACKGROUND - PSVIDEOPHOT STR PSVIDEOPHOT -END Index: /branches/eam_branches/ipp-20111122/ippconfig/recipes/rejections.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/recipes/rejections.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/recipes/rejections.config (revision 33638) @@ -459,4 +459,28 @@ END +NOISEMAP METADATA + FILTER STR * + EXPECTED F32 0.0 + IMFILE.MEAN F32 0.0 + IMFILE.STDEV F32 0.0 + IMFILE.MEANSTDEV F32 0.0 + IMFILE.SKEWNESS F32 0.0 + IMFILE.KURTOSIS F32 0.0 + IMFILE.SNR F32 0.0 + IMFILE.BIN.STDEV F32 0.0 + IMFILE.BIN.SNR F32 0.0 + IMFILE.FLUX F32 0.0 + EXP.MEAN F32 0.0 + EXP.STDEV F32 0.0 + EXP.MEANSTDEV F32 0.0 + EXP.SNR F32 0.0 + EXP.BIN.STDEV F32 0.0 + EXP.BIN.SNR F32 0.0 + EXP.FLUX F32 0.0 + ENSEMBLE.MEAN F32 0.0 + ENSEMBLE.STDEV F32 0.0 + ENSEMBLE.MEANSTDEV F32 0.0 +END + # FILTER is an additional qualifier, and may be "*" (or absent!), in which case it matches everything # EXPECTED is the expected mean value Index: /branches/eam_branches/ipp-20111122/ippconfig/ssp/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/ssp/Makefile.am (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippconfig/ssp/Makefile.am (revision 33638) @@ -0,0 +1,18 @@ + +installdir = $(datadir)/ippconfig/ssp + +install_files = \ + camera.config \ + format.config \ + ppImage.config \ + psastro.config \ + psphot.config + +install_DATA = $(install_files) + +install-data-hook: + chmod 0755 $(installdir) + +EXTRA_DIST = $(install_files) + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/eam_branches/ipp-20111122/ippconfig/ssp/camera.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/ssp/camera.config (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippconfig/ssp/camera.config (revision 33638) @@ -0,0 +1,68 @@ +# Camera configuration file for drop-dead boring camera, consisting of a single chip with single cell. + +# File formats that we know about +FORMATS METADATA + SSP STR ssp/format.config +END + +# Description of camera --- all the chips and the cells that comprise them +FPA METADATA + Chip STR Cell +END + +# move this elsewhere? we need a lookup table to go from filter ID to abstract name +FILTER.ID METADATA + B STR B + V STR V + R STR R + I STR I + z STR z +END + +# Table of strings to use for the class identifier (see ippdb) according to the file level. +CLASSID METADATA + FPA STR fpa +END + +DVO.CAMERADIR STR megacam # Camera directory for DVO + +# convert supplied FPA.OBSTYPE values to abstract exptype names +OBSTYPE.TABLE METADATA + bias STR BIAS + zero STR BIAS + dark STR DARK + flat STR SKYFLAT + skyflat STR SKYFLAT + domeflat STR DOMEFLAT + object STR OBJECT + science STR OBJECT +END + +# Recipe options +RECIPES METADATA + PPIMAGE STR ssp/ppImage.config # Default: all (normal) options on + PSPHOT STR ssp/psphot.config # psphot details +END + +PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.NAME} # Rule for generating photcode + +# reduction classes (recipes which are grouped together) +REDUCTION STR recipes/reductionClasses.mdc + +FITSTYPES STR recipes/fitstypes.mdc + +FILERULES STR recipes/filerules-simple.mdc + +EXTNAME.RULES METADATA + CMF.HEAD STR hdr + CMF.DATA STR psf + CMF.XSRC STR xsrc + CMF.XFIT STR xfit + CMF.DETEFF STR deteff + + PSF.HEAD STR hdr + PSF.TABLE STR psf_model + PSF.RESID STR psf_resid +END + +MASK.FORCE BOOL TRUE # Force reliance on MASKS recipe if headers not present Index: /branches/eam_branches/ipp-20111122/ippconfig/ssp/format.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/ssp/format.config (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippconfig/ssp/format.config (revision 33638) @@ -0,0 +1,80 @@ +# Drop-dead simple camera with single chip with single cell + +# How to identify this type +RULE METADATA + SIMPLE BOOL TRUE +### NAXIS S32 2 +END + +# How to read this data +FILE METADATA + PHU STR FPA # The FITS file represents an entire FPA + EXTENSIONS STR NONE # There are no extensions + FPA.OBS STR NAXIS # A PHU keyword for unique identifier within the hierarchy level +END + +# What's in the FITS file? +CONTENTS STR Chip:Cell:amplifier + +# Specify the cell data +CELLS METADATA + amplifier METADATA + CELL.TRIMSEC.SOURCE STR VALUE + #CELL.TRIMSEC STR [0:0,0:0] + + # alternatives for getting these from the headers + # CELL.TRIMSEC.SOURCE STR HEADER + CELL.BIASSEC.SOURCE STR VALUE + CELL.TRIMSEC STR DATASEC + CELL.BIASSEC STR BIASSEC + END +END + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + #FPA.TELESCOPE STR TELESCOP + FPA.INSTRUMENT STR INSTRUME + FPA.OBSTYPE STR OBSTYPE + FPA.AIRMASS STR AIRMASS + FPA.FILTER STR FILTER + FPA.POSANGLE STR POSANGLE + FPA.RA STR RA + FPA.DEC STR DEC + FPA.OBJECT STR OBJECT + FPA.TIME STR DATE-OBS # UTC-OBS # Date and time + FPA.EXPOSURE STR EXPTIME + CELL.EXPOSURE STR EXPTIME + CELL.DARKTIME STR DARKTIME + FPA.FOCUS STR FOCUS + FPA.DETECTOR STR DETECTOR + CELL.GAIN STR CCDGAIN +# CELL.TIME STR DATE-OBS TIME-OBS # Date and time +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + FPA.TELESCOPE STR SSP + CELL.XBIN S32 1 + CELL.YBIN S32 1 + FPA.TIMESYS STR UTC + FPA.RADECSYS STR ICRS + #CELL.GAIN F32 1.0 + CELL.READNOISE F32 2.0 + CELL.BAD F32 0 + CELL.SATURATION F32 65535 +# CELL.BAD F32 -65535 + CELL.READDIR S32 1 + CELL.TIMESYS STR UTC + CELL.XPARITY S32 1 + CELL.YPARITY S32 1 + CHIP.XPARITY S32 1 + CHIP.YPARITY S32 1 + CELL.X0 S32 0 + CELL.Y0 S32 0 +END + +FORMATS METADATA + FPA.RA STR HOURS + FPA.DEC STR DEGREES + FPA.TIME STR SEPARATE,YEAR.FIRST +END Index: /branches/eam_branches/ipp-20111122/ippconfig/ssp/ppImage.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/ssp/ppImage.config (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippconfig/ssp/ppImage.config (revision 33638) @@ -0,0 +1,120 @@ +### ppImage recipe configuration file for drop-dead simple camera + +# List of tasks to perform +SHUTTER BOOL FALSE # Shutter correction +DARK BOOL TRUE # Dark correction + +BASE.FITS BOOL TRUE # Save base detrended image? +BIN1.FITS BOOL TRUE # Save 1st binned chip image? +BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + +# binned output image options +BIN1.XBIN S32 8 +BIN1.YBIN S32 8 +BIN2.XBIN S32 64 +BIN2.YBIN S32 64 + +# Overscan subtraction +OVERSCAN.SINGLE BOOL FALSE # Reduce overscan to a single value? +OVERSCAN.FIT STR POLYNOMIAL # NONE | POLYNOMIAL | SPLINE +OVERSCAN.ORDER S32 3 # Order of polynomial fit +OVERSCAN.STAT STR MEAN # MEAN | MEDIAN + +# How to select the appropriate detrend image +DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + MASK METADATA + END + DARK METADATA + END + FLAT METADATA + FILTER STR FPA.FILTER + END + FRINGE METADATA + FILTER STR FPA.FILTER + END + SHUTTER METADATA + END +END + +######################################################################################## +# Need the following in order to turn off overscan and shutter for the site-level recipe +######################################################################################## + +# Overscan subtraction only +PPIMAGE_O METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Dark subtraction only +PPIMAGE_D METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Shutter correction only +PPIMAGE_S METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Overscan, bias, dark +PPIMAGE_OBD METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Overscan, bias, dark, shutter +PPIMAGE_OBDS METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Overscan, bias, dark, shutter, flat-field +PPIMAGE_OBDSF METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Overscan, bias, dark, shutter, flat-field, fringe +PPIMAGE_OBDSFR METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction +END Index: /branches/eam_branches/ipp-20111122/ippconfig/ssp/psastro.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/ssp/psastro.config (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippconfig/ssp/psastro.config (revision 33638) @@ -0,0 +1,36 @@ + +# astrometry matching parameters + +# nominal plate scale (microns / pixel) +# PSASTRO.PLATE.SCALE F32 13.5 +PSASTRO.PLATE.SCALE F32 1.0 + +# pmAstromGridMatch: +PSASTRO.GRID.MIN.ANGLE F32 -0.0851 +PSASTRO.GRID.MAX.ANGLE F32 +0.0851 +PSASTRO.GRID.DEL.ANGLE F32 0.0170 + +# pmAstromGridAngle +# max grid offset in FP units (microns) +# use plate-scale to make this in pixels? +# PSASTRO.GRID.OFFSET F32 10000. +# PSASTRO.GRID.SCALE F32 500. +PSASTRO.GRID.OFFSET F32 1000. +PSASTRO.GRID.SCALE F32 50 + +# these tweak are in FP units (pixels, currently) +PSASTRO.TWEAK.SCALE F32 1 +PSASTRO.TWEAK.RANGE F32 75 +PSASTRO.TWEAK.SMOOTH F32 2 +PSASTRO.TWEAK.NSIGMA F32 3 + +# pmAstromRadiusMatch +# use plate-scale to make this in pixels? +PSASTRO.MATCH.RADIUS F32 8 + +# pmAstromMatchFit +PSASTRO.CHIP.NX S32 1 # fit order +PSASTRO.CHIP.NY S32 1 # fit order +PSASTRO.CHIP.NITER S32 3 # fit clipping iterations +PSASTRO.CHIP.NSIGMA F32 3 # fit clipping sigmas + Index: /branches/eam_branches/ipp-20111122/ippconfig/ssp/psphot.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/ssp/psphot.config (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippconfig/ssp/psphot.config (revision 33638) @@ -0,0 +1,2 @@ + +SAVE.RESID BOOL TRUE Index: /branches/eam_branches/ipp-20111122/ippconfig/system.config =================================================================== --- /branches/eam_branches/ipp-20111122/ippconfig/system.config (revision 33637) +++ /branches/eam_branches/ipp-20111122/ippconfig/system.config (revision 33638) @@ -16,4 +16,5 @@ VYSOS5 STR vysos5/camera.config # VYSOS 5in VYSOS20 STR vysos20/camera.config # VYSOS 5in + SSP STR ssp/camera.config # Pan-STARRS spectroscopic sky probe SIMMOSAIC STR simmosaic/camera.config # Simulated mosaic, for testing SIMTEST STR simtest/camera.config # Simulation test camera Index: /branches/eam_branches/ipp-20111122/ippdor/IppSiteTest.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/IppSiteTest.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/IppSiteTest.py (revision 33638) @@ -0,0 +1,37 @@ +""" +This module contains the definitions relative to the installation and +the configuration of the IPP-Condor framework. +""" +class IppSite: + """ + This module contains the definitions relative to the installation + and the configuration of the IPP-Condor framework. + """ + def __init__(self): + """Not to be instantiated + """ + raise Exception('Not to be instantiated') + + SVN_SOURCE_URL = 'https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/trunk/ippdor/src' + TARGET_DIRECTORY = '/home/schastel/dev/ippdor/test/ipp' + TARGET_BINARY_DIRECTORY = '/home/schastel/dev/ippdor/test/bin' + + LOCAL_TMP_DIR = '/tmp/ippdor_processing' + NFS_TMP_DIR = '~/ippdor_processing' + IPP_BIN_PATH = '~/ipp/default.linux' + +class Gpc1: + """ + Information relative to the Gpc1 database + + NOT INTENDED TO BE USABLE + """ + def __init__(self): + """Not to be instantiated + """ + raise Exception('Not to be instantiated') + HOST = 'somehost.ipp.ifa.hawaii.edu' + NAME = 'gpc1' + USER = 'username' + PASSWORD = 'its_password' + Index: /branches/eam_branches/ipp-20111122/ippdor/README =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/README (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/README (revision 33638) @@ -0,0 +1,10 @@ +Installation: + ./deployment/deploy.py -h + +Documentation: + cd ./doc && make + +Code Quality Metrics (pylint 0.25 required): + ./quality_metrics.sh + + Index: /branches/eam_branches/ipp-20111122/ippdor/conf/pylintrc =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/conf/pylintrc (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/conf/pylintrc (revision 33638) @@ -0,0 +1,310 @@ +# lint Python modules using external checkers. +# +# This is the main checker controlling the other ones and the reports +# generation. It is itself both a raw checker and an astng checker in order +# to: +# * handle message activation / deactivation at the module level +# * handle some basic but necessary stats'data (number of classes, methods...) +# +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +profile=no + +# Add to the black list. It should be a base name, not a +# path. You may set this option multiple times. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# Set the cache size for astng objects. +cache-size=500 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + + +[MESSAGES CONTROL] + +# Enable only checker(s) with the given id(s). This option conflicts with the +# disable-checker option +#enable-checker= + +# Enable all checker(s) except those with the given id(s). This option +# conflicts with the enable-checker option +#disable-checker= + +# Enable all messages in the listed categories (IRCWEF). +#enable-msg-cat= + +# Disable all messages in the listed categories (IRCWEF). +disable-msg-cat=I + +# Enable the message(s) with the given id(s). +#enable-msg= + +# Disable the message(s) with the given id(s). +disable-msg=W0704 + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Include message's id in output +include-ids=no + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (R0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (R0004). +comment=no + +# Enable the report(s) with the given id(s). +#enable-report= + +# Disable the report(s) with the given id(s). +#disable-report= + + +# checks for : +# * doc strings +# * modules / classes / functions / methods / arguments / variables name +# * number of arguments, local variables, branches, returns and statements in +# functions, methods +# * required module attributes +# * dangerous default values as arguments +# * redefinition of function / method / class +# * uses of the global statement +# +[BASIC] + +# Required attributes for module, separated by a comma +required-attributes= + +# Regular expression which should only match functions or classes name which do +# not require a docstring +no-docstring-rgx=__.*__ + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + + +# try to find bugs in the code using type inference +# +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. +generated-members=REQUEST,acl_users,aq_parent + + +# checks for +# * unused variables / imports +# * undefined variables +# * redefinition of variable from builtins or from an outer scope +# * use of variable before assignment +# +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching names used for dummy variables (i.e. not used). +dummy-variables-rgx=_|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +# checks for : +# * methods without self as first argument +# * overridden methods signature +# * access only to existent members via self +# * attributes not defined in the __init__ method +# * supported interfaces implementation +# * unreachable code +# +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + + +# checks for +# * external modules dependencies +# * relative / wildcard imports +# * cyclic imports +# * uses of deprecated modules +# +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,string,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report R0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report R0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report R0402 must +# not be disabled) +int-import-graph= + + +# checks for sign of poor/misdesign: +# * number of methods, attributes, local variables... +# * size, complexity of functions, methods +# +[DESIGN] + +# Maximum number of arguments for function / method +max-args=8 + +# Maximum number of locals for function / method body +max-locals=25 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branchs=15 + +# Maximum number of statements in function / method body +max-statements=150 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=0 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=30 + + +# checks for: +# * warning notes in the code like FIXME, XXX +# * PEP 263: source code with non ascii character but no encoding declaration +# +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +# checks for similarities and duplicated code. This computation may be +# memory / CPU intensive, so you should disable it if you experiments some +# problems. +# +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=10 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + + +# checks for : +# * unauthorized constructions +# * strict indentation +# * line length +# * use of <> instead of != +# +[FORMAT] + +# Maximum number of characters on a single line. +#max-line-length=80 +max-line-length=120 + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' Index: /branches/eam_branches/ipp-20111122/ippdor/deployment/Deployment.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/deployment/Deployment.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/deployment/Deployment.py (revision 33638) @@ -0,0 +1,226 @@ +#!/usr/bin/env python + +"""Utility class for Condor IPP software deployent +""" + +import os +import subprocess +import tempfile +import shutil +import logging + +class Deployment: + """Utility class for Condor IPP software deployent + """ + logger = logging.getLogger('ippdor') + + CLASSES_FILENAMES = [ '__init__.py', + 'ChipManager.py', + 'CamManager.py', + 'FakeManager.py', + 'WarpManager.py', + 'StackManager.py', + 'Gpc1Manager.py', + 'helpers.py', + 'constants/__init__.py', + 'constants/Globals.py', + 'constants/Chip.py', + 'constants/Camera.py', + 'constants/Fake.py', + 'constants/Warp.py', + 'constants/Stack.py', + 'exceptions/__init__.py', + 'exceptions/IppException.py', + 'exceptions/UninstantiableIppException.py', + ] + EXECUTABLES_FILENAMES = [ 'generate_chip_jobs.py', + 'generate_cam_jobs.py', + 'generate_fake_jobs.py', + 'generate_warp_jobs.py', + 'generate_stack_jobs.py', + 'generate_exposure_dag.py', + 'ippdor_chip_to_warp.py', + # 'ippdor_chip_to_stack.py', + 'undo_dag_exposure.py', + 'send_email.py', + 'encapsulate_chip.sh', + 'encapsulate_camera.sh', + 'encapsulate_fake.sh', + 'encapsulate_warp.sh', + 'encapsulate_stack.sh', + ] + + def __init__(self, + svn_source_url, + target_directory, + target_binary_directory, + logger = None): + """ + Constructor: + + svn_source_url: SVN Source URL + target_directory: Target Python Directory + target_binary_directory: Target Executables Directory + logger: optional argument (if None: logging to logging.getLogger('ippdor')) + """ + self.svn_source_url = svn_source_url + self.target_directory = target_directory + self.target_binary_directory = target_binary_directory + self.temporary_directory = None # Set when necessary + if logger is not None: + Deployment.logger = logger + + def copy_to_targetdir(self, path_to_file, filename): + """ + Copy a file from the SVN temporary directory to the target + directory. + """ + Deployment.logger.info('...... Installing %s' % filename) + shutil.copyfile('%s/%s/%s' % (self.temporary_directory, + path_to_file, + filename), + '%s/%s' % (self.target_directory, filename)) + + def copy_to_bindir(self, path_to_file, filename): + """ + Copy a file from the SVN temporary directory to the target + binary directory. + """ + Deployment.logger.info('...... Installing %s' % filename) + shutil.copyfile('%s/%s/%s' % (self.temporary_directory, + path_to_file, + filename), + '%s/%s' % (self.target_binary_directory, filename)) + + def check(self): + """ + Check different elements + """ + Deployment.logger.info('Checking Condor-IPP Softwares Configuration') + # Configuration + from IppSite import IppSite, Gpc1 + Deployment.logger.info('Checking definition of IppSite.SVN_SOURCE_URL: %s' + % IppSite.SVN_SOURCE_URL) + Deployment.logger.info('Checking definition of IppSite.TARGET_DIRECTORY: %s' + % IppSite.TARGET_DIRECTORY) + Deployment.logger.info('Checking definition of IppSite.TARGET_BINARY_DIRECTORY: %s' + % IppSite.TARGET_BINARY_DIRECTORY) + Deployment.logger.info('Checking definition of IppSite.LOCAL_TMP_DIR: %s' + % IppSite.LOCAL_TMP_DIR) + Deployment.logger.info('Checking definition of IppSite.NFS_TMP_DIR: %s' + % IppSite.NFS_TMP_DIR) + Deployment.logger.info('Checking definition of IppSite.IPP_BIN_PATH: %s' + % IppSite.IPP_BIN_PATH) + Deployment.logger.info('Checking definition of Gpc1.HOST: %s' + % Gpc1.HOST) + Deployment.logger.info('Checking definition of Gpc1.NAME: %s' + % Gpc1.NAME) + Deployment.logger.info('Checking definition of Gpc1.USER: %s' + % Gpc1.USER) + Deployment.logger.info('Checking definition of Gpc1.PASSWORD: %s[Hidden]' + % Gpc1.PASSWORD[:0]) + # target_directory + Deployment.logger.info('... Checking write permission to %s' % self.target_directory) + extra = '' + if not os.path.exists(self.target_directory): + os.makedirs(self.target_directory) + extra = ' (directory was created)' + if not os.access(self.target_directory, os.W_OK): + raise IOError('Directory [%s] is not writable' % self.target_directory) + Deployment.logger.info('...... OK%s' % extra) + # target_binary_directory + Deployment.logger.info('... Checking write permission to %s' % self.target_binary_directory) + extra = '' + if not os.path.exists(self.target_binary_directory): + os.makedirs(self.target_binary_directory) + extra = ' (directory was created)' + if not os.access(self.target_binary_directory, os.W_OK): + raise IOError('Directory [%s] is not writable' % self.target_binary_directory) + Deployment.logger.info('...... OK%s' % extra) + # Check deployment software version? + # Anything else? + Deployment.logger.info('... Check complete') + + def svn_export(self): + """ + Export the software under configuration into the SVN temporary directory + """ + Deployment.logger.info('Updating Condor-IPP Softwares SVN temporary directory') + Deployment.logger.info('... Creating temporary directory:') + self.temporary_directory = tempfile.mkdtemp() + Deployment.logger.info('...... OK [%s]' % self.temporary_directory) + Deployment.logger.info('... Exporting Condor source code to temporary directory') + process = subprocess.Popen( ('svn export --force %s %s' % (self.svn_source_url, + self.temporary_directory)).split(' '), + stdout = open('/dev/null', 'w'), + stderr = open('/dev/null', 'w') ) + status = process.wait() + if status != 0: + Deployment.logger.error('Error: svn export failed') + raise Exception('svn export failed') + Deployment.logger.info('...... OK') + Deployment.logger.info('... SVN Update complete') + + def generate(self): + """ + Generate any file depending on the installation target + """ + Deployment.logger.info('Generating Condor-IPP Softwares') + Deployment.logger.debug('--- Nothing to generate yet from %s' % self.svn_source_url) + Deployment.logger.info('... Generation complete') + + def deploy(self): + """ + Deploy files on the target + """ + Deployment.logger.info('Deploying Condor-IPP Softwares') + Deployment.logger.info('... Installing Site definition class to [%s]' + % self.target_directory) + shutil.copyfile('.tmp_site/IppSite.py', + '%s/IppSite.py' % (self.target_directory)) + Deployment.logger.info('... Installing Python classes to [%s]' % self.target_directory) + for filename in Deployment.CLASSES_FILENAMES: + path_elements = filename.split('/') + if len(path_elements) > 1: + try: + os.makedirs('%s/%s' % (self.target_directory, + path_elements[0])) + except OSError, exc: + Deployment.logger.warn(exc) + Deployment.logger.info('Ignored') + for filename in Deployment.CLASSES_FILENAMES: + self.copy_to_targetdir('ipp', filename) + Deployment.logger.info('... Installing executables to [%s]' % self.target_binary_directory) + for filename in Deployment.EXECUTABLES_FILENAMES: + self.copy_to_bindir('executables', filename) + Deployment.logger.info('... Deployment complete') + + def clean(self, doit = False): + """ + Clean the installation, i.e. remove files and subdirectories + if doit is True (default is False). Otherwise, doesn't do + anything... + """ + Deployment.logger.info('Cleaning') + if not doit: + Deployment.logger.warn('!!! NO DELETION WILL BE PERFORMED SINCE doit IS False') + try: + Deployment.logger.info('... Deleting temporary directory [%s]' % self.temporary_directory) + if doit: + for root, dirs, files in os.walk(self.temporary_directory, topdown=False): + Deployment.logger.debug('...... Deleting [%s]' % root) + for name in files: + Deployment.logger.debug('...... Deleting [%s] (in files)' % name) + os.remove(os.path.join(root, name)) + for name in dirs: + Deployment.logger.debug('...... Deleting [%s] (in dirs)' % name) + os.rmdir(os.path.join(root, name)) + os.rmdir(self.temporary_directory) + except AttributeError: + Deployment.logger.info('... Nothing to be cleaned') + Deployment.logger.info('... Cleaning complete') + +if __name__ == '__main__': + print "Running doctest" + import doctest + doctest.testmod() Index: /branches/eam_branches/ipp-20111122/ippdor/deployment/__init__.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/deployment/__init__.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/deployment/__init__.py (revision 33638) @@ -0,0 +1,3 @@ +""" +This is the deployment module: a module to deploy Condor-IPP software. +""" Index: /branches/eam_branches/ipp-20111122/ippdor/deployment/deploy.py =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/deployment/deploy.py (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/deployment/deploy.py (revision 33638) @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +""" +Utility executable for IPP Condor deployment +""" +import sys +import os +import logging +#optparse Deprecated in python 2.7. Will be replaced by argparse +from optparse import OptionParser +from Deployment import Deployment + +if __name__ == '__main__': + if os.getenv('PYTHONPATH') is not None: + print 'Error: Unset PYTHONPATH environment variable to run this program' + sys.exit(1) + USAGE = """usage: %prog [options] + +Deploy the Condor-IPP softwares +""" + PARSER = OptionParser(usage = USAGE) + PARSER.set_defaults(verbose=True) + PARSER.add_option( '-v', action = 'store_true', + dest = 'verbose', + help = 'Be verbose', + default = False) + PARSER.add_option('-q', action='store_true', + dest = 'quiet', + help = 'Be quiet', + default = False) + (OPTIONS, ARGS) = PARSER.parse_args() # ... the values are stored in options + if len(ARGS) != 1: + PARSER.print_help() + sys.exit(1) + IPP_SITE_FILE = ARGS[0] + # Logging setup + LOGGER = logging.getLogger('ippdor') + FORMATTER = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', + '%Y-%m-%dT%H:%M:%S') + STDOUT = logging.StreamHandler(sys.stdout) + STDOUT.setFormatter(FORMATTER) + LOGGER.addHandler(STDOUT) + if OPTIONS.verbose: + LOGGER.setLevel(logging.DEBUG) + LOGGER.debug('Logging level set to DEBUG') + elif OPTIONS.quiet: + LOGGER.setLevel(logging.ERROR) + LOGGER.info('Logging level set to ERROR') + else: + LOGGER.setLevel(logging.INFO) + # Import IppSite definition + LOGGER.info('Importing local site definitions') + try: + os.makedirs('.tmp_site') + except OSError: + LOGGER.warn('Existing .tmp_site directory') + import shutil + shutil.copy(IPP_SITE_FILE, '.tmp_site/IppSite.py') + INITFILE = open('.tmp_site/__init__.py', 'w') + INITFILE.close() + sys.path.append('.tmp_site') + from IppSite import IppSite + + # Deployment + try: + DEPLOYMENT = Deployment(IppSite.SVN_SOURCE_URL, + IppSite.TARGET_DIRECTORY, + IppSite.TARGET_BINARY_DIRECTORY) + DEPLOYMENT.check() + DEPLOYMENT.svn_export() + DEPLOYMENT.generate() + DEPLOYMENT.deploy() + DEPLOYMENT.clean(doit = True) + except Exception, e: + LOGGER.info('Exception caught') + DEPLOYMENT.clean(doit = False) + LOGGER.error(e) + import traceback + traceback.print_exc() + # Clean temporary site + LOGGER.info('Cleaning temporary site definitions') + os.remove('.tmp_site/IppSite.py') + try: + os.remove('.tmp_site/IppSite.pyc') + except OSError: + pass # Ignore if not compiled + os.remove('.tmp_site/__init__.py') + os.rmdir('.tmp_site') Index: /branches/eam_branches/ipp-20111122/ippdor/doc/Chip2Stack/stack.tex =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/doc/Chip2Stack/stack.tex (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/doc/Chip2Stack/stack.tex (revision 33638) @@ -0,0 +1,97 @@ +\documentclass[10pt]{article} +\usepackage[hmargin=0.5in,vmargin=0.5in]{geometry} + +\begin{document} + +\textbf{\Large FOR EACH EXPOSURE} + +\begin{description} +\item[Prerequisites] The exposures to be processed are + \textbf{labelled} externally in the chipRun table. +\item[A. Chip stage] Detail prerequisites \mbox{ } + \begin{description} + \item[PRE A010] (Script) Generate the Condor job file containing the + \verb+chip_imfile.pl+ commands to be run on the cluster + \hfill{}\mbox{Equivalent to chip.imfile.load} + \item[A010] (Condor) Queue that Condor job file (exactly 60 + commands/jobs: one for each ota) \hfill{}\mbox{Equivalent to + chip.imfile.run} + \item[POST A010] (Script) Generate the Condor job file containing the + appropriate jobs for reverting \hfill{}\mbox{Equivalent to + chip.revert} + \item[A020] (Condor) Queue that Condor job file (various, possibly + nothing to do) + \item[POST A020] (Script) Generate the Condor job file containing the + appropriate jobs for reverting \hfill{}\mbox{Equivalent to + chip.revert} + \item[A030] (Condor) Queue that Condor job file (various, possibly + nothing to do) + \item[POST A030] (Script) Generate the Condor job file containing the + appropriate jobs for reverting \hfill{}\mbox{Equivalent to + chip.revert} + \item[A040] (Condor) Queue that Condor job file (various, possibly + nothing to do) + \item[POST A040] (Script) If there is failing otas, send e-mail to + operator and stop processing. Otherwise generate stage 050 jobs file. + \item[A050] (Script) Advance the exposure with + \verb+chiptool -advanceexp+ \hfill{}\mbox{Equivalent to + chip.advanceexp} + \end{description} +\item[B. Camera stage] Detail prerequisites \mbox{ } + \begin{description} + \item[PRE B010] (Script) Generate the Condor job file containing the + \verb+camera_exp.pl+ command(s?) \hfill{}\mbox{Equivalent to + camera.exp.load} + \item[B010] (Condor) Queue that Condor job file + \hfill{}\mbox{Equivalent to camera.exp.run} + \item[PRE B020] (Script) Generate the Condor job file possibly + containing the revert command (possibly nothing to revert) + \hfill{}\mbox{Equivalent to camera.revert} + \item[B020] (Condor) Queue that Condor job file + \item[PRE B030] (Script) Generate the Condor job file possibly + containing the revert command (possibly nothing to revert) + \hfill{}\mbox{Equivalent to camera.revert} + \item[B030] (Condor) Queue that Condor job file + \item[PRE B040] (Script) Generate the Condor job file possibly + containing the revert command (possibly nothing to revert) + \hfill{}\mbox{Equivalent to camera.revert} + \item[B040] (Condor) Queue that Condor job file + \item[POST B040] (Script) If the camera entry could be reverted, send + e-mail to operator and stop processing + \end{description} +\item[C. Fake stage] Detail prerequisites \mbox{ } + \begin{description} + \item[PRE C010] (Script) Generate the Condor job file containing the + \verb+fake_imfile.pl+ command(s?) \hfill{}\mbox{Equivalent to + fake.imfile.load} + \item[C010] (Condor) Queue that Condor job file + \hfill{}\mbox{Equivalent to fake.imfile.run} + \item[C020] (Script (or Script+Condor)) Run the + \verb+faketool -advanceexp+ command(s?) (or: PRE C020: Generate + the Condor job file containing the \verb+faketool -advanceexp+ + command(s?); C020: Queue that Condor job file) + \hfill{}\mbox{Equivalent to fake.advanceexp} + \end{description} +\item[D. Warp stage] Detail prerequisites \hrulefill{}\textbf{BARF}\mbox{ } + \begin{description} + \item[TODO] \verb+warptool -tooverlap+ \hfill{}\mbox{Equivalent to warp.exp.load} + \item[TODO] \verb+warp_overlap.pl+ \hfill{}\mbox{Equivalent to warp.exp.run} + \item[TODO] \hfill{}\mbox{Equivalent to warp.revert.overlap} + \item[TODO] \verb+warptool -towarp+ \hfill{}\mbox{Equivalent to warp.skycell.load} + \item[TODO] \verb+warp_skycell.pl+ \hfill{}\mbox{Equivalent to warp.skycell.run} + \item[TODO] \verb++ \hfill{}\mbox{Equivalent to warp.revert.warped} + \item[TODO] \verb+ warptool -advancerun+ \hfill{}\mbox{Equivalent to warp.advancerun} + \item[Check if needed?] \verb++ \hfill{}\mbox{Equivalent to + warp.summary.load and/or warp.summary.run} + \end{description} +\item[E. Stack stage] Detail prerequisites? \mbox{ } + \begin{description} + \item[TODO] \verb+stacktool -tosum+ \hfill{}\mbox{Equivalent to stack.skycell.load} + \item[TODO] \verb+stack_skycell.pl + \hfill{}\mbox{Equivalent to stack.skycell.run} + \item[TODO] \verb++ \hfill{}\mbox{Equivalent to stack.revert} + \item[Check if needed?] \verb++ \hfill{}\mbox{Equivalent to + stack.summary.load and/or stack.summary.run} + \end{description} +\end{description} + +\end{document} Index: /branches/eam_branches/ipp-20111122/ippdor/doc/Full/Makefile =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/doc/Full/Makefile (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/doc/Full/Makefile (revision 33638) @@ -0,0 +1,10 @@ +MAIN = ippdor + +all: + pdflatex $(MAIN).tex + makeindex $(MAIN) + pdflatex $(MAIN).tex + +clean: + /bin/rm -f $(MAIN).pdf + CleanTex.sh Index: /branches/eam_branches/ipp-20111122/ippdor/doc/Full/administration.tex =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/doc/Full/administration.tex (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/doc/Full/administration.tex (revision 33638) @@ -0,0 +1,34 @@ + +\chapter{Administration} +\label{chap-adm} + +\section{The \texttt{ippdor} user} +\label{sec-adm-ippdor} + +\texttt{ippdor} is the dedicated user for any processing happening in +Condor. This user uses the \texttt{bash} shell. The ippdor has no +particular privilege on the system. + +The \texttt{bashrc} file contains some definitions and expansions: +\begin{itemize} +\item \texttt{PATH} adds \texttt{\$HOME/local/bin}; +\item \texttt{LD\_LIBRARY\_PATH} adds \texttt{\$HOME/local/lib}; +\item \texttt{PYTHONPATH} adds \texttt{\$HOME/local/share/python} and + \texttt{\$HOME/local/lib/python2.6/site-packages}; +\item The \texttt{PSCONFIG} value is fixed to \texttt{current}. See + section~\ref{sec-link-ippTODO} for details. The + \texttt{psconfig.bash} utility is called using that value. +\end{itemize} + +\texttt{cmake} was locally installed to allow Condor build and +install. It is now the only external software. It was built using the +\texttt{--prefix=\$HOME/local} option and can be found in the +\texttt{\$HOME/softwares} directory. + +\section{Deploying Condor on a new host} +\label{sec-adm-newnode} + +\section{Updating Condor} +\label{sec-adm-update} + +To be written when it's happening... Index: /branches/eam_branches/ipp-20111122/ippdor/doc/Full/annex.tex =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/doc/Full/annex.tex (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/doc/Full/annex.tex (revision 33638) @@ -0,0 +1,103 @@ +\chapter{Annex} + +\section{Held Jobs} +\label{annex-heldjobs} + +\null\hfill{\parbox{5in}{\scriptsize This example is related to the + DAG shown in section~\ref{sec-oper-tasks}. The A.condor and + B.condor jobs are defined as "echo A" and "echo B" commands.}} + +If \texttt{condor\_submit\_dag} is executed from a local directory +(e.g. \textbf{/export/ippc01.0/ippdor/simple\_dag}), the jobs will be +indefinitely held by Condor. + +\begin{verbatim} +ippdor@ippc01 /export/ippc01.0/ippdor/simple_dag $ condor_submit_dag AB.dag +----------------------------------------------------------------------- +File for submitting this DAG to Condor : AB.dag.condor.sub +Log of DAGMan debugging messages : AB.dag.dagman.out +Log of Condor library output : AB.dag.lib.out +Log of Condor library error messages : AB.dag.lib.err +Log of the life of condor_dagman itself : AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71204. +----------------------------------------------------------------------- +\end{verbatim} + +This can be seen with the +\texttt{condor\_q} command: + +\begin{verbatim} +$ condor_q + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu + ID OWNER SUBMITTED RUN_TIME ST PRI SIZE CMD +71204.0 ippdor 11/25 11:41 0+00:01:27 R 0 12.2 condor_dagman +71205.0 ippdor 11/25 11:42 0+00:00:01 H 0 0.0 echo A + +2 jobs; 0 idle, 1 running, 1 held +\end{verbatim} + +``In-depth'' analysis can be performed with \texttt{condor\_q -analyze} command: + +\begin{verbatim} +$ condor_q -analyze 71205.0 + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu +--- +71205.000: Request is held. + +Hold reason: Error from slot7@ipp029.ifa.hawaii.edu: Failed to execute +'/bin/echo' with arguments A: Cannot access specified iwd +"/export/ippc01.0/ippdor/simple_dag" (errno=2: 'No such file or directory') +\end{verbatim} + +The hold reason makes things obvious as soon as you know that +\texttt{iwd} stands for "Initial Working Directory". There is no way +that the Execute node \texttt{ipp029} can access the contents of what is +on the \texttt{ippc01} local file system. + +To fix the problem, stop processing with: +\begin{verbatim} +$ condor_rm 71204.0 +\end{verbatim} + +Note the creation of the \texttt{rescue} file in +\texttt{/export/ippc01.0/ippdor/simple\_dag} + +Now submit the DAG from any NFS directory: +\begin{verbatim} +ippdor@ippc01 ~ $ condor_submit_dag /export/ippc01.0/ippdor/simple_dag/AB.dag + +Running rescue DAG 1 +----------------------------------------------------------------------- +File for submitting this DAG to Condor : /export/ippc01.0/ippdor/simple_dag/AB.dag.condor.sub +Log of DAGMan debugging messages : /export/ippc01.0/ippdor/simple_dag/AB.dag.dagman.out +Log of Condor library output : /export/ippc01.0/ippdor/simple_dag/AB.dag.lib.out +Log of Condor library error messages : /export/ippc01.0/ippdor/simple_dag/AB.dag.lib.err +Log of the life of condor_dagman itself : /export/ippc01.0/ippdor/simple_dag/AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71208. +----------------------------------------------------------------------- +\end{verbatim} + +Note that the "rescue DAG 1" is used. + +... and the DAG should terminate correctly from the \texttt{condor\_q} +command and the contents of the \texttt{AB.dag.dagman.out} file: +\begin{verbatim} +[...] +11/25/11 12:13:58 Done Pre Queued Post Ready Un-Ready Failed +11/25/11 12:13:58 === === === === === === === +11/25/11 12:13:58 2 0 0 0 0 0 0 +11/25/11 12:13:58 0 job proc(s) currently held +11/25/11 12:13:58 All jobs Completed! +11/25/11 12:13:58 Note: 0 total job deferrals because of -MaxJobs limit (0) +11/25/11 12:13:58 Note: 0 total job deferrals because of -MaxIdle limit (0) +11/25/11 12:13:58 Note: 0 total job deferrals because of node category throttles +11/25/11 12:13:58 Note: 0 total PRE script deferrals because of -MaxPre limit (0) +11/25/11 12:13:58 Note: 0 total POST script deferrals because of -MaxPost limit (0) +11/25/11 12:13:58 **** condor_scheduniv_exec.71208.0 (condor_DAGMAN) pid 18455 EXITING WITH STATUS 0 +\end{verbatim} Index: /branches/eam_branches/ipp-20111122/ippdor/doc/Full/design.tex =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/doc/Full/design.tex (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/doc/Full/design.tex (revision 33638) @@ -0,0 +1,4 @@ +\chapter{Software Design} +\label{chap-software} + +TODO Index: /branches/eam_branches/ipp-20111122/ippdor/doc/Full/heldjobs.tex =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/doc/Full/heldjobs.tex (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/doc/Full/heldjobs.tex (revision 33638) @@ -0,0 +1,66 @@ +\chapter{Annex} + +\section{Held Jobs} +\label{annex-heldjobs} + +\nul\hfill{\scriptsize This example is related to the DAG shown in + section~\ref{sec-oper-tasks}. The A.condor and B.condor jobs are + defined as "echo A" and "echo B" commands.} + +If \texttt{condor\_submit\_dag} is executed from a local directory +(e.g. \textbf{/export/ippc01.0/ippdor/simple_dag}), the jobs will be +indefinitely held by Condor. + +\begin{verbatim} +ippdor@ippc01 /export/ippc01.0/ippdor/simple_dag $ condor_submit_dag AB.dag +----------------------------------------------------------------------- +File for submitting this DAG to Condor : AB.dag.condor.sub +Log of DAGMan debugging messages : AB.dag.dagman.out +Log of Condor library output : AB.dag.lib.out +Log of Condor library error messages : AB.dag.lib.err +Log of the life of condor_dagman itself : AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71204. +----------------------------------------------------------------------- +\end{verbatim} + +This can be seen with the +\texttt{condor\_q} command: + +\begin{verbatim} +$ condor_q + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu + ID OWNER SUBMITTED RUN_TIME ST PRI SIZE CMD +71204.0 ippdor 11/25 11:41 0+00:01:27 R 0 12.2 condor_dagman +71205.0 ippdor 11/25 11:42 0+00:00:01 H 0 0.0 echo A + +2 jobs; 0 idle, 1 running, 1 held +\end{verbatim} + +``In-depth'' analysis can be performed with \texttt{condor\_q -analyze} command: + +\begin{verbatim} +$ condor_q -analyze 71205.0 + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu +--- +71205.000: Request is held. + +Hold reason: Error from slot7@ipp029.ifa.hawaii.edu: Failed to execute +'/bin/echo' with arguments A: Cannot access specified iwd +"/export/ippc01.0/ippdor/simple_dag" (errno=2: 'No such file or directory') +\end{verbatim} + +The hold reason makes things obvious (as soon as you know that +\texttt{iwd} stands for "Initial Working Directory")... There is no +way that the Execute node \texttt{ipp029} can get the contents of what +is on \texttt{ippc01} local file system. + +To fix the problem, stop processing with: +\begin{verbatim} +$ condor_rm 71204.0 +\end{verbatim} + +Note the creation of the \texttt{rescue} file in \texttt{/export/ippc01.0/ippdor/simple\_dag} Index: /branches/eam_branches/ipp-20111122/ippdor/doc/Full/ippdor.tex =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/doc/Full/ippdor.tex (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/doc/Full/ippdor.tex (revision 33638) @@ -0,0 +1,100 @@ +\documentclass[10pt]{report} + +\usepackage{vmargin} +\setpapersize{USletter} +\setmarginsrb{0.5in}{0.5in}{1.25in}{0.5in}{0pt}{0mm}{0pt}{0mm} +\usepackage{longtable} +\usepackage{hyperref} +\usepackage{makeidx} +\usepackage{enumitem} + +\title{Condor in the IPP} +\author{S.~Chastel\\Version 0.1} +\date{2011-11-23} + +\makeindex + +\newcommand{\CondorSubmit}{\texttt{condor\_submit}} +\newcommand{\CondorSubmitDag}{\texttt{condor\_submit\_dag}} + +\newcommand{\IppPython}[1]{\texttt{\~{ }ippdor/local/share/python/ipp/{#1}}} + +\newcommand{\definition}[3]{% + %#1: What is displayed in the text + %#2: What is displayed in the margin + %#3: Entry in the index + \emph{#1}\marginpar{\scriptsize #2}% + \index{#3}% +} + +\begin{document} +\maketitle + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\chapter*{Introduction} + +This is the documentation detailing how the Condor HTC system is used +to perform processing for the IPP. Note that it is (currently) not +supposed to replace the \texttt{pantasks} scheduling system (mainly +because we choose not to use Condor as a scheduler). + +This documentation consists of five parts: +\begin{enumerate} +\item This introduction; +\item Operational Users Manual (page~\pageref{chap-operation}): its + targets are the operators of the IPP, that is, persons who want to + use the Condor framework to process data for the IPP; +\item Administration Manual (page~\pageref{chap-adm}): for system + administrators. Mainly how to install and upgrade the Condor + framework; +\item Link with the IPP (page~\pageref{chap-ipp}): TODO: Should be + part of Software Organization; +\item Software Organization (page~\pageref{chap-software}): Design, + Classes. How to add new features. +\end{enumerate} + +\emph{Note:} Be aware that this documentation is not a substitute to +the Condor documentation\definition{}{Condor + documentation}{Condor!Documentation} which is available from the +Condor Project Homepage\footnote{when this document was released, it + was: + \href{http://research.cs.wisc.edu/condor/}{http://research.cs.wisc.edu/condor/}}. + +\subsection*{Document History} +\begin{tabular}{|c|c|p{5in}|} + \hline + \textbf{Version} & \textbf{Date} & \textbf{Comment} \\ + \hline + 0.1 & 2011-11-23 & First draft \\ + \hline +\end{tabular} + +\subsection*{Acronyms} +\begin{longtable}{lp{5in}} + IPP & Image Processing Pipeline \\ + IWD & Initial Working Directory \\ + RTCM & Read the Condor Manual \\ +\end{longtable} + +\setcounter{tocdepth}{2} +\tableofcontents + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{operations.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{administration.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{link_ipp.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{design.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{annex.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\printindex + +\end{document} Index: /branches/eam_branches/ipp-20111122/ippdor/doc/Full/link_ipp.tex =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/doc/Full/link_ipp.tex (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/doc/Full/link_ipp.tex (revision 33638) @@ -0,0 +1,4 @@ +\chapter{Link with the IPP} +\label{chap-ipp} + +TODO Index: /branches/eam_branches/ipp-20111122/ippdor/doc/Full/operations.tex =================================================================== --- /branches/eam_branches/ipp-20111122/ippdor/doc/Full/operations.tex (revision 33638) +++ /branches/eam_branches/ipp-20111122/ippdor/doc/Full/operations.tex (revision 33638) @@ -0,0 +1,622 @@ +\newcommand{\QueueExposuresWarping}{% + \texttt{queue\_exposures\_for\_warping.py}} +\newcommand{\QueueExposuresStacking}{% + \texttt{queue\_exposures\_for\_stacking.py}} + +\chapter{Operators Manual} +\label{chap-operation} + +\section{Condor Framework Availability on the IPP Production Cluster} + +The Condor framework is not available on all hosts of the IPP +production cluster. The purpose of this section is to give some +details about the different nodes which are used and their roles in +Condor. The set of all machines on which Condor is installed is called +the \definition{Condor pool}{Condor pool}{Condor!Pool}. + +The Condor \definition{central manager}{Central + manager}{Condor!Central manager} is \texttt{ippdb03}. From the +Condor documentation (section~3.1), the role of the central manager is +to "[collect] information, and [negotiate] between resources and +resource requests". It "plays a very important part in the Condor pool +and should be reliable" since "[if it] crashes, no further matchmaking +can be performed within the Condor system". Moreover "[all] queries go +to the central manager". + +The other nodes of the IPP production cluster where Condor is +installed are all both Execute and Submit Condor nodes. The complete +list of those nodes is available in the file +\texttt{\~{ }ippdor/local/share/CondorNodes}. Note that a node on which +Condor is installed is not necessary available for processing in +Condor since it can be deactivated (see section~\ref{sec-oper-hosts} +for details). + +An \definition{Execute}{Execute node}{Condor!Execute node} node is a +node where processing can be performed. From the Condor manual +section~3.1, "the only resource that might matter is disk +space"... Until now, it has never been a matter. + +A \definition{Submit}{Submit node}{Condor!Submit node} node is a node +where Condor jobs can be submitted (see +section~\ref{sec-ippdor-run}). It is suggested (again this is from the +Condor manual, section~3.1), that since "every job that you submit +that is currently running on a remote machine generates another +process on your submit machine [, ...] if you have lots of jobs +running, you will need a fair amount of swap space and/or real +memory". Until now, jobs were generally submitted from one of the +\texttt{ippc\emph{xy}} nodes. + +Note: For Condor gurus, there is no Checkpoint Server since all +processing is performed in the \emph{vanilla} Condor universe. + +\section{Condor Dedicated User: \texttt{ippdor}} +\label{sec-oper-login} + +To use the Condor framework you need to log into the IPP production +cluster as the user \texttt{ippdor}. Note that since Condor is not +installed on \texttt{ippc18}, you will have to login again into +another node for the condor commands to be usable. \texttt{ipp004} or +\texttt{ippdb03} are other "open" nodes where you can connect. + +Note: The \texttt{ippdor} account uses \texttt{bash}. + +Theoretically it should be possible to use the Condor framework with +your own login but 1) It has not been tested; 2) No support will be +provided for that; 3) So... RTCM. + +As detailed in section~\ref{sec-adm-ippdor}, the \texttt{ippdor} +account uses files that may be localized on the Condor node. If you +get a \texttt{command not found} error, make sure you are on one of +the node where Condor is installed\footnote{that is, the command + \texttt{"grep \`{ }hostname\`{ } \~{ + }ippdor/local/share/CondorNodes"} should display the host name. If + you don't see anything, read the first paragraph of this section.}. + +\section{Managing Condor Execution on the IPP Production Cluster + Nodes} +\label{sec-oper-hosts} + +Note 1: To install Condor on a new node, see +section~\ref{sec-adm-newnode}. + +Note 2: Except for a few home-grown commands, information was +extracted from the Condor Manual section~3.10. + +\subsection{Node Availability} +\label{sec-oper-hosts-availability} +\definition{}{Node availability}{Node availability}% +\definition{}{}{Condor!Node availability}% +To check if a node can be used by Condor, check the result of the +\texttt{ipp\_condor\_status } command. You should get some +information about the host from the Condor point of view. For +instance, \texttt{ipp060} potentially has 12 potential processors for +Condor execution\footnote{To get more information about what the + columns mean, RTCM}: +\begin{verbatim} +ippdor@ippc20 ~ $ ipp_condor_status ipp060 +Name OpSys Arch State Activity LoadAv Mem ActvtyTime + +slot10@ipp060.ifa. LINUX X86_64 Unclaimed Idle 0.000 4032 0+05:54:29 +slot11@ipp060.ifa. LINUX X86_64 Owner Idle 0.400 4032 0+00:00:06 +slot12@ipp060.ifa. LINUX X86_64 Owner Idle 1.000 4032 0+00:00:07 +slot1@ipp060.ifa.h LINUX X86_64 Owner Idle 0.840 4032 0+00:00:04 +slot2@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+14:34:09 +slot3@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+06:56:32 +slot4@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+14:25:44 +slot5@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+14:33:57 +slot6@ipp060.ifa.h LINUX X86_64 Claimed Idle 0.000 4032 3+11:22:07 +slot7@ipp060.ifa.h LINUX X86_64 Owner Idle 1.000 4032 0+00:00:29 +slot8@ipp060.ifa.h LINUX X86_64 Owner Idle 0.790 4032 0+00:00:03 +slot9@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+06:55:39 + Total Owner Claimed Unclaimed Matched Preempting Backfill + + X86_64/LINUX 12 5 1 6 0 0 0 + + Total 12 5 1 6 0 0 0 +\end{verbatim} + +If you don't get anything, the node has been deactivated for Condor +processing). For instance (while the author writes these lines), +\texttt{ipp065} will not execute any Condor jobs: +\begin{verbatim} +ippdor@ippc20 ~ $ ipp_condor_status ipp065 +ippdor@ippc20 ~ $ +\end{verbatim} + +\emph{Note:} \texttt{ipp\_condor\_status} is an alias of +\texttt{condor\_status} defined in \texttt{\~{ }ippdor/.bashrc}. + +\subsection{Make a Node Unavailable} +\label{sec-oper-hosts-remove} + +To remove a node from the list of execution nodes, execute the command +\texttt{ipp\_condor\_off } from any node where Condor is +installed. No job will be started/executed and submitted from that +host. \texttt{ipp\_condor\_off} is a function defined in \texttt{\~{ + }ippdor/.bashrc}. + +To stop Condor execution and submission from \definition{all}{Stopping + Condor on stare nodes}{Stopping Condor on stare nodes}stare nodes, +the \texttt{stare\_nodes\_off} and \texttt{stare\_nodes\_on} scripts +are provided. As it can be expected, those scripts just call +\texttt{ipp\_condor\_off} or \texttt{ipp\_condor\_on} for each of the +stares nodes. + +\subsection{Make a Node Available} +\label{sec-oper-hosts-available} + +To make a node available for Condor after it has been stopped, execute +the command \texttt{ipp\_condor\_on } from any node where +Condor is installed. + +\emph{Note:} If the following message is displayed, +\begin{verbatim} +bash: condor_master: command not found +\end{verbatim} +it simply means that Condor is not installed on the node where you try +to start it. + +\section{Condor Jobs and DAG} + +\subsection{Running Jobs in Condor} +\label{sec-oper-jobs} +We will call \definition{job}{job}{job} an elementary unit of +processing. It consists of the execution of one instance of an +executable (we will assume that an executable is a program that can be +called directly from the command line). For instance, "/bin/echo One" +and "/bin/echo Two" are two jobs, each job being an instance of the +same executable "/bin/echo". + +In our case, defining a correct Condor job is as simple (not to say +simplistic) as defining: +\begin{itemize} +\item The name of the executable; +\item The Condor universe of execution (in the IPP case, to avoid + recompilation of programs we use the \emph{vanilla} universe); +\item The file where the standard output of the executable will go (on + an NFS directory since we don't know where the executable will be + run); +\item The file where the error output of the executable will go (on + an NFS directory since we don't know where the executable will be + run); +\item The file where Condor will write the log of the job (on a local + directory); +\item The requirements about the node where execution will take place; +\item Some argument to tell that we want to keep the environment + variables; +\item The arguments for each execution of the executable. +\end{itemize} + +With the example given on \texttt{/bin/echo}, the contents of the +condor job file (let's call this file \texttt{echo\_test.condor}) are: +\begin{verbatim} +Executable = /bin/echo +Universe = vanilla +Output = /home/panstarrs/ippdor/tmp/tests/out.echo_test +Error = /home/panstarrs/ippdor/tmp/tests/err.echo_test +Log = /tmp/echo_test.log +Requirements = LoadAvg < 0.3 + +getenv = True + +Arguments = One +Queue + +Arguments = Two +Queue +\end{verbatim} + +To execute those jobs, they just need to be \definition{submitted}{Job + submission}{Job submission} from any Submit node to the Condor +system with: + +\texttt{condor\_submit echo\_test.condor} + +The jobs are (usually) not executed immediately and there is no way of +guessing the order on the jobs execution (if the order of jobs +execution is a matter, look at paragraph~\ref{sec-oper-tasks}). + +Note: From what the author experienced, \texttt{condor\_submit} and +the condor jobs definition file can be executed and placed on any kind +of file system (local and/or NFS) which is different from +\texttt{condor\_submit\_dag}. + +\subsection{Running tasks} +\label{sec-oper-tasks} + +\subsubsection{DAGMan} + +We will call \definition{task}{task}{task} a set of jobs where jobs +chaining depend on each other. Condor provides a mechanism of +organizing jobs according to their dependencies: +\definition{DAGMan}{DAGMan}{DAGMan} (which stands for Directed Acyclic +Graph Manager). Exhaustive documentation about DAGMan can be found in +the Condor documentation section 2.10. In this section I will just +give details about what is used for the IPP. + +\subsubsection{DAG Description File} + +To make it simple, a DAG is a high-level description of how jobs are +sequenced. If the job B has to be performed after the job A, the +dependency is formalized this way: +\begin{verbatim} +JOB A /some/directory/A.condor +JOB B /some/directory/B.condor +PARENT A CHILD B +\end{verbatim} + +Assuming the previous text is saved in the file named \texttt{AB.dag}, +the execution of the DAG can be \definition{submitted}{DAG + submission}{DAG submission} from any Submit node to the Condor +system using: + +\texttt{condor\_submit\_dag AB.dag} + +The output of that command will look like: +\begin{verbatim} +----------------------------------------------------------------------- +File for submitting this DAG to Condor : AB.dag.condor.sub +Log of DAGMan debugging messages : AB.dag.dagman.out +Log of Condor library output : AB.dag.lib.out +Log of Condor library error messages : AB.dag.lib.err +Log of the life of condor_dagman itself : AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71201. +----------------------------------------------------------------------- +\end{verbatim} + +All those files are created in the directory of the \texttt{AB.dag} +file. The \texttt{AB.dag.dagman.out} file is the most interesting log +file. + +\subsubsection{Files and Directories Locations} + +The DAG file can be localized on any file system: local or +NFS. However for performance concerns, I recommend to put it on the +local file system of the submit node. Monitor to the log growth rate +(the logs are not huge though... For instance, full processing of 2300 +MD06 exposures from Chip to Warp stages in the IPP created a log of 15 +MB for the \texttt{.out} file and of a total of about 50 kB for the 4 +other files). + +The \texttt{/some/directory} directory (where the condor jobs are +defined) can be placed on any file system. Again, I suggest to put +them on the local file system of the Submit node (mostly for +performance concerns). + +\begin{center} + \fbox{\parbox{6in}{\textbf{Important note:} The + \texttt{condor\_submit\_dag} command \textbf{must} be executed + \textbf{from a NFS file system} or the jobs will be indefinitely + \texttt{Held} (shown with \texttt{H} status in + \texttt{condor\_q}. See section~\ref{annex-heldjobs} for more + details.}} +\end{center} + +One restriction is that there cannot be cycles, i.e. circular +dependencies. For instance, job Egg depends on job Chicken, which +depend on job Egg. If you submit a DAG with a cycle, looking at the +\texttt{dagman.out} file, you will see a message like: +\begin{verbatim} +[...] +11/25/11 12:22:02 ERROR: DAG finished but not all nodes are complete -- + checking for a cycle... +11/25/11 12:22:02 ... ERROR: a cycle exists in the dag, plese check input +[...] +11/25/11 12:22:03 **** condor_scheduniv_exec.71211.0 (condor_DAGMAN) + pid 20996 EXITING WITH STATUS 1 +\end{verbatim} + +\subsubsection{Advanced DAGMan Features} +\label{sec-oper-dagman} + +DAGMan offers other convenient functions for a DAG description. As for +the other Condor features the reader is invited to check the Condor +documentation for in-depth details about them. + +\paragraph{SubDAG:} A graph node of a DAG can be itself a DAG making +the development and the analysis of a processing chain easier. A DAG +in a DAG is called a \definition{SubDAG}{SubDAG}{SubDAG} and can be +added to a DAG using this syntax: +\begin{verbatim} +SUBDAG EXTERNAL +\end{verbatim} + +\paragraph{Script:} Scripts canbe executed before or after a node +execution. They can be added using the following syntax: +\begin{verbatim} +SCRIPT PRE [] +SCRIPT POST [] +\end{verbatim} + +From the Condor documentation, "the \texttt{} specifies the +node to which the script is attached. The \texttt{} +specifies the script to be executed, and it may be followed by any +command line \texttt{} to that script". In the IPP context +we currently use only the \texttt{SCRIPT POST} feature. + +\subsection{Condor Interrupted} +\label{sec-oper-condor-interrupted} + +If the execution of Condor DAG is interrupted, for instance, because +of an operator \texttt{condor\_rm} or because of errors, Condor marks +the jobs or subdags that were successfully completed by adding the +\texttt{DONE} keyword to the DAG file. The resulting DAG file is saved +in a \definition{rescue DAG file}{Rescue DAG file}{Rescue DAG file}, +which is simply a copy of the original DAG file with a +\texttt{rescue} file extension (\texttt{} is +an auto-incrementing integer coded on three digits, starting at +\texttt{001}). + +To \definition{rescue processing}{Rescuing processing}{Rescuing + processing} you just need to execute \textbf{exactly} the same +submission command: Condor will use the last rescue file that was +produced. + +Note: If you know what you are doing you can create you rown rescue +file or add/remove \texttt{DONE} keywords to/from a rescue file. Some +utility programs can do that for you for some IPP stages +(see~\ref{TODO}). + +\section{IPP Tasks} +\label{sec-oper-ipp} + +The IPP defines complex tasks. The purpose of this section is to +provide a user manual about the programs that need to be run to +prepare the set of Condor files which are needed to perform some of +the IPP tasks. Developers should look at chapter~\ref{chap-software} +for in-depth details. + +Each subsection of this section details: 1) The software role; 2) Its +required inputs; 3) Its outputs; 4) Its IPP products. + +Currently the IPP parts that are implemented are: +\begin{itemize}% + \setlength{\itemsep}{0pt}\setlength{\topsep}{0pt}\setlength{\partopsep}{0pt}\setlength{\parsep}{0pt}% +\item Chip to Warp stages (paragraph~\ref{sec-oper-ipp-chip2warp}); +\item Chip to Stack stages (paragraph~\ref{sec-oper-ipp-chip2stack}); +\end{itemize} + +\subsection{Chip to Warp Processing: \QueueExposuresWarping{}} +\label{sec-oper-ipp-chip2warp} + +Since this section is supposed to belong to the operators manual, no +detail about the software will be provided. The developer will read +section~\ref{sec-TODO} for such details. + +\subsubsection{Role} +The role of \QueueExposuresWarping{} is to prepare the set of Condor +and IPP elements which are required to process a set of exposures from +the Chip to the Warp stage. + +The \QueueExposuresWarping{} script has four roles: +\begin{enumerate}[topsep=0pt, partopsep=0pt, itemsep=0pt, parsep=0pt, itemindent=10pt] +\item Assert that the label has not already been used and that the + input exposures exist; +\item Create entries in the \texttt{chipRun} table; +\item Create the DAG files which are necessary for the processing of + all exposures; +\item Display the \texttt{submit\_condor\_dag} command that needs to + be run by the operator. +\end{enumerate} + +Note that the operator has to run the \texttt{submit\_condor\_dag} +command. + +\subsubsection{Submit Node} + +A node needs to be chosen as Submit node. TODO: add details. + +\subsubsection{Inputs} +The inputs are: +\begin{itemize} +\item A \textbf{label} The label used for processing. If the label has + already been used, the program is aborted. As for the processing in + IPP, don't use spaces or non-ASCII characters for the label (for + instance, \texttt{My.New\_Label}); +\item An \textbf{e-mail address} who gets the e-mails (for instance, + \texttt{someone@mail.com}); +\item A \textbf{tessellation ID} (e.g. \texttt{MD04.V3}); +\item A \textbf{list of exposures} to be processed (for instance, + \texttt{o4926g0064o}, \texttt{o4926g0068o}, \texttt{o4926g0069o}, + \texttt{o4926g0071o}, and \texttt{o4940g0237o}). +\end{itemize} + +The \QueueExposuresWarping{} script can be called in two ways: +\begin{itemize} +\item Either they are provided in a file which has the format of a + Properties-style file. With the previous parameters: +\begin{verbatim} +label = My.New_Label +tessId = MD04.V3 +recipient = someone@mail.com +exposure = o4926g0064o +exposure = o4926g0068o +exposure = o4926g0069o +exposure = o4926g0071o +# Comment line +exposure = o4940g0237o # Line followed by a comment +\end{verbatim} + Assume that the file is stored under the name + \texttt{~{ }ippdor/examples/ChipsToWarps.ippdor} + the script is called with (note the \texttt{-f} option flag): +\begin{verbatim} +queue_exposures_for_warping.py -f ~ippdor/examples/01/ChipsToWarps.ippdor +\end{verbatim} +\item Or the input parameters are provided as arguments, that is (the + arguments have to be in this order): +\begin{verbatim} +queue_exposures_for_warping.py My.New_Label MD04.V3 someone@mail.com \ + o4926g0064o o4926g0068o o4926g0069o o4926g0071o o4940g0237o +\end{verbatim} +\end{itemize} + +\subsubsection{Outputs. Entries on GPC1 Database} +The script writes its outputs to the directory named +\texttt{/