IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 27710


Ignore:
Timestamp:
Apr 19, 2010, 12:15:08 PM (16 years ago)
Author:
rhenders
Message:

Now using dist_group from IPP Db to set 'survey type' in PSPS; better auditing

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/scripts/ippToPsps_run.pl

    r27699 r27710  
    7171#######################################################################################
    7272#
     73# Generates a PSPS-suitable survey 'type' from the IPP distribution group
     74#
     75#######################################################################################
     76sub getSurveyTypeFromDistGroup {
     77    my ($distGroup) = @_;
     78
     79    if ($distGroup eq "MD01") {return $distGroup;}
     80    if ($distGroup eq "MD02") {return $distGroup;}
     81    if ($distGroup eq "MD03") {return $distGroup;}
     82    if ($distGroup eq "MD04") {return $distGroup;}
     83    if ($distGroup eq "MD05") {return $distGroup;}
     84    if ($distGroup eq "MD06") {return $distGroup;}
     85    if ($distGroup eq "MD07") {return $distGroup;}
     86    if ($distGroup eq "MD08") {return $distGroup;}
     87    if ($distGroup eq "MD09") {return $distGroup;}
     88    if ($distGroup eq "MD10") {return $distGroup;}
     89    if ($distGroup eq "M31") {return $distGroup;}
     90    if ($distGroup eq "sts") {return "STS";}
     91    if ($distGroup eq "SweetSpot") {return "SS";}
     92    if ($distGroup eq "3PI") {return $distGroup;}
     93    if ($distGroup eq "ThreePi") {return "3PI";}
     94    if ($distGroup eq "SAS") {return "3PI";}
     95
     96    print "* Do not understand distribution group: '$distGroup'\n";
     97    return undef;
     98}
     99
     100#######################################################################################
     101#
    73102# Finds all exposures for the provided label and generates PSPS FITS data for them
    74103#
    … …  
    91120
    92121        # query to retrieve all exposures with the provided label
    93         $query =
    94             "SELECT DISTINCT rawExp.exp_id ".
    95             "FROM camRun, chipRun, rawExp ".
    96             "WHERE camRun.chip_id = chipRun.chip_id ".
    97             "AND chipRun.exp_id = rawExp.exp_id ".
    98             "AND camRun.label like '%$label%' ".
    99             #    "AND rawExp.decl >= '-0.157079633' AND rawExp.decl <= '0.157079633' ". # TODO Jim's decl range
    100             "ORDER BY rawExp.exp_id ASC";
     122        $query = $db->prepare(<<SQL);
     123
     124        SELECT DISTINCT rawExp.exp_id, camRun.dist_group
     125            FROM camRun, chipRun, rawExp
     126            WHERE camRun.chip_id = chipRun.chip_id
     127            AND chipRun.exp_id = rawExp.exp_id
     128            AND camRun.label like '%$label%'
     129            ORDER BY rawExp.exp_id ASC
     130SQL
    101131    }
    102132    else {
    103133
    104         $query = "SELECT DISTINCT exp_id FROM rawExp WHERE rawExp.exp_id = $singleExpId"
    105 
    106     }
    107 
    108     if ($verbose) { print"$query\n";}
     134        $query = $db->prepare(<<SQL);
     135
     136        SELECT DISTINCT exp_id FROM rawExp WHERE rawExp.exp_id = $singleExpId
     137SQL
     138    }
     139
     140    if ($verbose) { print"* $query\n";}
    109141
    110142    my $batchId = 0;
    … …  
    114146    my $jobId = getJobId();
    115147    $jobId++;
    116     if ($jobId < 0) { print "Cannot find sensible jobId ($jobId)\n";
     148    if ($jobId < 0) { print "* Cannot find sensible jobId ($jobId)\n";
    117149        return 0;
    118150    }
    119151
    120     print "------------------------------------------------------------------------------------------\n\n";
     152    print "*******************************************************************************\n*\n";
    121153
    122154    my $anyBatches = 0;
    123155
     156
     157    $query->execute;
     158
    124159    # loop round exposures
    125     my $expResults = $db->selectall_arrayref( $query );
    126 
    127     for my $expRow (@$expResults) {
    128 
    129         my ($expId) = @$expRow;
    130 
    131         print "      *** Exposure ID = $expId ***\n";
     160    while (my @row = $query->fetchrow_array()) {
     161        my ($expId, $distGroup) = @row;
     162
     163        print "* Exposure ID = $expId with dist group = '$distGroup'\n";
     164
     165        my $surveyType = getSurveyTypeFromDistGroup($distGroup);
     166
     167        if (!$surveyType) {next;}
     168        print "* Using survey type of $surveyType\n";
    132169
    133170        # generate batch and job paths from job and batch IDs
    … …  
    137174        my $batchOutputPath = sprintf("$jobOutputPath/%s", $batchDir);
    138175
    139         print "Preparing job '$job'...\n";
     176        print "* Preparing exposure as job '$job'...\n";
    140177
    141178        # make directories
    … …  
    161198            elsif ($anyBatches) {
    162199
    163                 if (writeJobManifest($output, $job)) {
     200                if (writeJobManifest($output, $job, $surveyType)) {
    164201
    165202                    if (!$no_publish && !publishToDatastore($output, $job)) {
    … …  
    175212            else {
    176213
    177                 print "No batches to publish\n"
     214                print "* No batches to publish\n"
    178215            }
    179216        }
    180217
    181         if ($product eq "init" ) {print "Wrote initialisation batch\n"; last;}
    182 
    183 
    184         print "------------------------------------------------------------------------------------------\n\n";
     218        if ($product eq "init" ) {print "* Wrote initialisation batch\n"; last;}
     219
     220
     221        print "*\n*******************************************************************************\n\n";
    185222    }
    186223
    … …  
    204241    close (MYFILE);
    205242
    206     if ($verbose) { print "Current jobId = $jobId\n"; }
     243    if ($verbose) { print "* Current jobId = $jobId\n"; }
    207244
    208245    return $jobId;
    … …  
    221258    close (MYFILE);
    222259
    223     if ($verbose) { print "New jobId = $jobId\n"; }
     260    if ($verbose) { print "* New jobId = $jobId\n"; }
    224261}
    225262
    … …  
    265302        run(command => $command, verbose => $verbose);
    266303
    267     if (!$success) { print "Unable to tar up dir: $batch\n" and return 0 };
     304    if (!$success) { print "* Unable to tar up dir: $batch\n" and return 0 };
    268305
    269306    # zip
    … …  
    272309        run(command => $command, verbose => $verbose);
    273310
    274     if (!$success) { print "Unable to gzip: $batchTar\n" and return 0 };
     311    if (!$success) { print "* Unable to gzip: $batchTar\n" and return 0 };
    275312
    276313    $command = "rm $batchTar";
    … …  
    278315        run(command => $command, verbose => $verbose);
    279316
    280     if (!$success) { print "Unable to remove: $batchTar\n" };
     317    if (!$success) { print "* Unable to remove: $batchTar\n" };
    281318
    282319    $command = "rm -r $batchDir";
    … …  
    284321        run(command => $command, verbose => $verbose);
    285322
    286     if (!$success) { print "Unable to remove: $batch\n"};
     323    if (!$success) { print "* Unable to remove: $batch\n"};
    287324
    288325    return 1;
    … …  
    343380
    344381    # open directory to hunt for FITS files
    345     opendir(DIR, $path) or print "Cannot open '$path' to write batch manifest\n" and return 0;
     382    opendir(DIR, $path) or print "* Cannot open '$path' to write batch manifest\n" and return 0;
    346383    my @thefiles= readdir(DIR);
    347384    closedir(DIR);
    … …  
    353390        if ($f =~ /\.FITS/) {
    354391
    355             if ($foundFits) { print "More than one FITS file found for this batch\n"; return 0;}
     392            if ($foundFits) { print "* More than one FITS file found for this batch\n"; return 0;}
    356393
    357394            # get md5sum of file
    … …  
    404441
    405442
    406     if (!$foundFits) { print "Could not find any FITS files for this batch\n"; return 0;}
     443    if (!$foundFits) { print "* Could not find any FITS files for this batch\n"; return 0;}
    407444    return 1;
    408445}
    … …  
    414451#######################################################################################
    415452sub writeJobManifest {
    416     my ($path, $job) = @_;
     453    my ($path, $job, $surveyType) = @_;
    417454
    418455    use XML::Writer;
    … …  
    433470    $writer->startTag('manifest',
    434471            "name" => "$job",
    435             "type" => "3PI", # TODO survey ID
     472            "type" => $surveyType,
    436473            "timestamp" => "$timeStamp");
    437474
    … …  
    439476
    440477    # open directory to hunt for FITS files
    441     opendir(DIR, $jobPath) or print "Cannot open '$path' to write job manifest\n" and return 0;
     478    opendir(DIR, $jobPath) or print "* Cannot open '$path' to write job manifest\n" and return 0;
    442479    my @thefiles= readdir(DIR);
    443480    closedir(DIR);
    … …  
    472509    $writer->end();
    473510
    474     if (!$foundBatch) { print "Could not find any batches for this job\n"; return 0;}
     511    if (!$foundBatch) { print "* Could not find any batches for this job\n"; return 0;}
    475512    return 1;
    476513}
    … …  
    489526
    490527    # loop through all batch files in this job directory
    491     opendir(DIR, $fullJobPath) or print "Cannot open '$fullJobPath' in order to publish to datastore\n" and return 0;
     528    opendir(DIR, $fullJobPath) or print "* Cannot open '$fullJobPath' in order to publish to datastore\n" and return 0;
    492529    my @thefiles= readdir(DIR);
    493530    closedir(DIR);
    … …  
    519556        run(command => $command, verbose => $verbose);
    520557
    521     if (!$success) { print "Unable to publish $job to datastore\n" and return 0 };
    522 
    523     print "Successfully published $job to datastore\n";
     558    if (!$success) { print "* Unable to publish $job to datastore\n" and return 0 };
     559
     560    print "* Successfully published $job to datastore\n";
    524561
    525562    close($tempFile);
    … …  
    558595        "WHERE rawExp.exp_id = $expId AND camRun.magicked";
    559596
    560     if ($verbose) { print"$query\n";}
     597    if ($verbose) { print"* $query\n";}
    561598
    562599
    … …  
    564601    my $results = $db->selectall_arrayref( $query );
    565602    my $size = @$results;
    566     if ($size < 1) {print "No smf files found for this exposure\n"; return 0;}
     603    if ($size < 1) {print "* No smf files found for this exposure\n"; return 0;}
    567604
    568605    # loop round db results TODO should only be one result
    … …  
    604641        "WHERE rawExp.exp_id = $expId AND camRun.magicked";
    605642
    606     if ($verbose) { print"$query\n";}
     643    if ($verbose) { print"* $query\n";}
    607644
    608645    # execute query and check results
    609646    my $results = $db->selectall_arrayref( $query );
    610647    my $size = @$results;
    611     if ($size < 1) {print "No cmf files found for this exposure\n"; return 0;}
     648    if ($size < 1) {print "* No cmf files found for this exposure\n"; return 0;}
    612649
    613650    my ($tempFile, $tempName) = tempfile( "inputFileList.XXXX", UNLINK => !$save_temps);
Note: See TracChangeset for help on using the changeset viewer.