IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 28849


Ignore:
Timestamp:
Aug 5, 2010, 12:26:50 PM (16 years ago)
Author:
rhenders
Message:

Changed SQL to query for exposures against data known to be loaded to the provided DVO database

Location:
trunk/ippToPsps/perl
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/perl/ippToPsps/Gpc1Db.pm

    r28841 r28849  
    2828        JOIN diffSkyfile USING (diff_id)
    2929        WHERE rawExp.exp_id = $expId AND camRun.magicked
    30 
    3130SQL
    3231
    … …  
    3736###########################################################################
    3837#
    39 # Returns camera-stage smf files for this exposure
     38# Returns camera-stage smf file for this exposure as used in the provided DVO Db
     39#
     40###########################################################################
     41sub getCameraStageSmfForThisDvoDb {
     42    my ($self, $dvoDb, $expId) = @_;
     43
     44    my $query = $self->{_db}->prepare(<<SQL);
     45    SELECT path_base
     46        FROM camProcessedExp
     47        JOIN addRun USING(cam_id)   
     48        JOIN camRun USING(cam_id) 
     49        JOIN chipRun USING(chip_id)   
     50        JOIN rawExp USING(exp_id)   
     51        WHERE addRun.dvodb LIKE '$dvoDb'
     52        AND exp_id = $expId;
     53SQL
     54
     55    $query->execute;
     56    return $query->fetchrow_array();
     57}
     58
     59###########################################################################
     60#
     61# Returns latest processed camera-stage smf file for this exposure ID
    4062#
    4163###########################################################################
    … …  
    6890###########################################################################
    6991sub getExposureListFromDvoDb {
    70     my ($self, $dvoDb) = @_;
     92    my ($self, $dvoDb, $exposures) = @_;
    7193
    7294    my $query = $self->{_db}->prepare(<<SQL);
    … …  
    80102SQL
    81103
    82     $query->execute;
    83     return $query->fetchall_arrayref();
    84 }
    85 
    86 ###########################################################################
    87 #
    88 # Returns a list of exposure IDs for this survey
    89 #
    90 ###########################################################################
    91 sub getExposureListFromSurvey {
    92     my ($self, $survey) = @_;
    93 
    94     my $query = $self->{_db}->prepare(<<SQL);
    95 
    96     SELECT rawExp.exp_id, rawExp.exp_name, camRun.dist_group
    97         FROM camRun, chipRun, rawExp
    98         WHERE camRun.state = 'full'
    99         AND camRun.chip_id = chipRun.chip_id
    100         AND chipRun.exp_id = rawExp.exp_id
    101         AND camRun.dist_group = '$survey'   
    102         GROUP BY  rawExp.exp_id
    103         ORDER BY rawExp.exp_id ASC;
    104 SQL
    105 
    106 
    107104    #AND rawExp.exp_id > 133887
    108105    #AND filter = 'r.00000'
    … …  
    115112
    116113    $query->execute;
    117     return $query->fetchall_arrayref();
     114    ${$exposures} = $query->fetchall_arrayref();
     115    my $numOfExposures = scalar @{${$exposures}};
     116    if ($numOfExposures > 0) {
     117   
     118        print "* Found $numOfExposures exposures in DVO Db '$dvoDb'\n";
     119        return 1;
     120    }
     121
     122    print "* No exposures found in DVO Db '$dvoDb'\n";
     123    return 0;
    118124}
    119125
    120126###########################################################################
    121127#
    122 # Returns info for this exposure ID
     128# Returns info for this exposure ID for given DVO Db
    123129#
    124130###########################################################################
    125 sub getSingleExposureInfo {
    126     my ($self, $expId) = @_;
     131sub getSingleExposureFromDvoDb {
     132    my ($self, $dvoDb, $expId, $exposures) = @_;
    127133
    128134    my $query = $self->{_db}->prepare(<<SQL);
    129     SELECT DISTINCT rawExp.exp_id,  rawExp.exp_name, dist_group
    130         FROM rawExp, chipRun
    131         WHERE chipRun.exp_id = rawExp.exp_id
    132         AND rawExp.exp_id = $expId
     135    SELECT exp_id, exp_name, camRun.dist_group
     136        FROM addRun
     137        JOIN camRun USING(cam_id)
     138        JOIN chipRun USING(chip_id)
     139        JOIN rawExp USING(exp_id)
     140        WHERE addRun.dvodb LIKE '$dvoDb'
     141        AND exp_id = $expId;
    133142SQL
    134143
    135144    $query->execute;
    136     return $query->fetchall_arrayref();
     145
     146    ${$exposures} = $query->fetchall_arrayref();
     147
     148    if (scalar @{${$exposures}} > 0) {
     149   
     150        print "* Found exposure $expId in DVO Db '$dvoDb'\n";
     151        return 1;
     152    }
     153
     154    print "* Exposure $expId NOT found in DVO Db '$dvoDb'\n";
     155    return 0;
    137156}
    138157
    139 
    140 
    1411581;

  • trunk/ippToPsps/perl/ippToPsps_run.pl

    r28842 r28849  
    1818my $camera = 'GPC1';
    1919my $batchType = undef;
    20 my $survey = undef;
    21 my $dvodb = undef;
     20my $dvoPath = undef;
     21my $dvoDb = undef;
     22my $fullDvoPath = undef;
    2223my $verbose = undef;
    2324my $save_temps = undef;
    … …  
    3435        'output|o=s' => \$output,
    3536        'batch|b=s' => \$batchType,
    36         'dvodb|d=s' => \$dvodb,
    37         'survey|s=s' => \$survey,
     37        'dvopath=s' => \$dvoPath,
     38        'dvodb|d=s' => \$dvoDb,
    3839        'expid|e=s' => \$singleExpId,
    3940        'product|p=s' => \$datastoreProduct,
    … …  
    4950print "* \n";
    5051if (@ARGV) {
    51     print "* UNKNKOWN: option                          @ARGV\n"; $quit=1;}
     52    $quit=1;
     53    print "* UNKNKOWN: option                          @ARGV\n";
     54}
    5255if (!defined $output) {
    53     print "* REQUIRED: need to provide an output path  -o <path>\n"; $quit=1;}
     56    $quit=1;
     57    print "* REQUIRED: need to provide an output path  -o <path>\n";
     58}
    5459if (!defined $batchType) {
    55     print "* REQUIRED: need to define a batch type     -b <init|det|diff|stack>\n"; $quit=1;}
    56 if (!defined $dvodb) {
    57     print "* REQUIRED: need to provide a DVO Db path   -d <path>\n"; $quit=1;}
    58 if (!defined $survey and !defined $singleExpId) {
    59     print "* REQUIRED: need to provide a survey type   -s <ThreePi|STS|SAS|M31|MD01|MD02|etc> ***OR***\n";
    60     print "*           an exposure ID                  -e <expID>\n"; $quit=1;}
     60    $quit=1;
     61    print "* REQUIRED: need to define a batch type     -b <init|det|diff|stack>\n";
     62}
     63if (!defined $dvoDb) {
     64    $quit=1;
     65    print "* REQUIRED: need to provide a DVO Db name   -d <name>\n";
     66}
     67if (!defined $dvoPath) {
     68    $dvoPath="/data/ipp037.0/gpc1/catdirs";
     69    print "* OPTIONAL: path to DVO Db                  -dvopath <path>      default = '$dvoPath'\n";
     70}
     71if (!defined $singleExpId) {
     72
     73    print "* OPTIONAL: an single exposure ID           -e <expID>           default = none\n";
     74}
    6175if (!defined $datastoreProduct) {
    62     print "* OPTIONAL: datastore product               -p <product>\n";}
     76
     77    print "* OPTIONAL: datastore product               -p <product>         default = none (i.e. data will not be published)\n";
     78}
    6379if (!defined $verbose) {
    64     print "* OPTIONAL: run in verbose mode             -v\n"; $verbose = 0;}
     80    $verbose = 0;
     81    print "* OPTIONAL: run in verbose mode             -v                   default = $verbose\n";
     82}
    6583if (!defined $save_temps) {
    66     print "* OPTIONAL: keep temp files                 -t\n"; $save_temps = 0;}
     84    $save_temps = 0;
     85    print "* OPTIONAL: keep temp files                 -t                   default = $save_temps\n";
     86}
    6787if (!defined $no_update) {
    68     print "* OPTIONAL: don't update database           -u\n"; $no_update = 0;}
     88    $no_update = 0;
     89    print "* OPTIONAL: don't update database           -u                   default = $no_update\n";
     90}
    6991if (!defined $force) {
    70     print "* OPTIONAL: force if already processed      -f\n"; $force = 0;}
     92    $force = 0;
     93    print "* OPTIONAL: force if already processed      -f                   default = $force\n";
     94}
    7195if (!defined $dontTarNZip) {
    72     print "* OPTIONAL: don't tar and zip output        -z\n"; $dontTarNZip = 0;}
    73     print "*\n*******************************************************************************\n";
     96    $dontTarNZip = 0;
     97    print "* OPTIONAL: don't tar and zip output        -z                   default = $dontTarNZip\n";
     98}
     99print "*\n*******************************************************************************\n";
    74100
    75101if ($quit) { exit; }
     102
     103$fullDvoPath = $dvoPath . "/" . $dvoDb;
    76104
    77105my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps);
    … …  
    85113if ($batchType eq "init") {$initBatch = 1;}
    86114else {$initBatch = 0;}
    87 process();
     115
     116if (!process()) {print "* Finished unsuccessfully\n";}
     117else {print "* Finished successfully\n";}
    88118
    89119#######################################################################################
    … …  
    114144    my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps);
    115145    my $lastExpId = $ippToPspsDb->getLastExpId();
    116     my $rows = undef;
     146    my $exposures;
    117147
    118148    my $query;
    119149    if ($initBatch) {
    120    
    121         my @row = (0, 'NULL', 'ThreePi');
    122         @{$rows} = (\@row);
    123     }
    124     elsif ($singleExpId) {$rows = $gpc1Db->getSingleExposureInfo($singleExpId);}
    125     else {$rows = $gpc1Db->getExposureListFromSurvey($survey);}
    126 
    127 # TODO check if there are no exposures and give a warning
     150
     151        my @exposure = (0, 'NULL', 'ThreePi');
     152        @{$exposures} = (\@exposure);
     153    }
     154    # get single exposure
     155    elsif ($singleExpId) {
     156       
     157        if (!$gpc1Db->getSingleExposureFromDvoDb($dvoDb, $singleExpId, \$exposures)) {return 0;}
     158    }
     159    # get all exposures in this DVO Db
     160    else {
     161       
     162        if (!$gpc1Db->getExposureListFromDvoDb($dvoDb, \$exposures)) {return 0;}
     163    }
    128164
    129165    my $batchId = 0;
    … …  
    132168
    133169    #my $batchId = -1;
    134     my $row;
    135     foreach $row ( @{$rows} ) {
    136         my ($expId, $expName, $distGroup) = @{$row};
    137     #        print "JHGHGHGHGH2 $expId, $expName, $distGroup\n";
    138 
    139     # loop round exposures
    140     #while (my @row = $query->fetchrow_array()) {
    141      #   my ($expId, $expName, $distGroup) = @row;
     170    my $exposure;
     171    foreach $exposure ( @{$exposures} ) {
     172        my ($expId, $expName, $distGroup) = @{$exposure};
    142173
    143174        if (!$initBatch && $ippToPspsDb->isExposureAlreadyProcessed($expId)) {
    144                
    145                 if ($force) {print "* Forcing....\n";}
    146                 else {next};
     175
     176            if ($force) {print "* Forcing....\n";}
     177            else {next};
    147178        }
    148179
    … …  
    457488    my ($expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_;
    458489
    459     my $nebPath = $gpc1Db->getCameraStageSmf($expId);
     490    my $nebPath = $gpc1Db->getCameraStageSmfForThisDvoDb($dvoDb, $expId);
    460491    if (!$nebPath) { return 0; }
    461492
    … …  
    527558    $command .= " -input $input";
    528559    $command .= " -output $output";
    529     $command .= " -D CATDIR $dvodb";
     560    $command .= " -D CATDIR $fullDvoPath";
    530561    $command .= " -config ../config"; # TODO
    531     $command .= " -expid $expid";
     562        $command .= " -expid $expid";
    532563    $command .= " -expname $expName";
    533564    $command .= " -survey $surveyType";
Note: See TracChangeset for help on using the changeset viewer.