IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 31253


Ignore:
Timestamp:
Apr 8, 2011, 6:31:08 PM (15 years ago)
Author:
watersc1
Message:

merge from trunk

Location:
branches/czw_branch/20110406
Files:
48 edited
4 copied

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/20110406

  • branches/czw_branch/20110406/Ohana/src/addstar/include/skycells.h

    r31027 r31253  
    6565int    NMAX;
    6666int    NX_SUB, NY_SUB;
     67int    X_PARITY;
    6768double SCALE;
    6869double PADDING;

  • branches/czw_branch/20110406/Ohana/src/addstar/src/args_skycells.c

    r31177 r31253  
    188188    remove_argument (N, &argc, argv);
    189189    OVERLAP_DEC  = atof (argv[N]);
     190    remove_argument (N, &argc, argv);
     191  }
     192  X_PARITY = 1;
     193  if ((N = get_argument (argc, argv, "-skyparity"))) {
     194    X_PARITY = -1;
    190195    remove_argument (N, &argc, argv);
    191196  }
    … …  
    224229  fprintf (stderr, "  -nx Nx                      : subdivide skycell projection in x by Nx\n");
    225230  fprintf (stderr, "  -ny Ny                      : subdivide skycell projection in y by Ny\n");
     231  fprintf (stderr, "  -overlap Or Od              : overlap between skycells (arcseconds)\n");
    226232  fprintf (stderr, "  -projection-number Np       : set projection-number (local mode only)\n");
     233  fprintf (stderr, "  -skyparity                  : set wcs for skycells so that east is to the left\n");
    227234  fprintf (stderr, "  -help                       : this list\n");
    228235  fprintf (stderr, "  -h                          : this list\n\n");

  • branches/czw_branch/20110406/Ohana/src/addstar/src/sky_tessalation.c

    r31027 r31253  
    318318  memset (image, 0, sizeof(Image));
    319319  image[0].coords = *refcoords;
    320   image[0].coords.pc1_1 = image[0].coords.pc2_2 = 1.0;
     320  image[0].coords.pc1_1 = 1.0 * X_PARITY;
     321  image[0].coords.pc2_2 = 1.0;
    321322  image[0].coords.pc1_2 = image[0].coords.pc2_1 = 0.0;
    322323
    … …  
    468469
    469470  if (FIX_NS) {
    470     rectangle[0].coords.pc1_1 = +1.0;
     471    rectangle[0].coords.pc1_1 = +1.0 * X_PARITY;
    471472    rectangle[0].coords.pc1_2 = +0.0;
    472473    rectangle[0].coords.pc2_1 = -0.0;
    … …  
    475476    ycr = yc*cos(angle) + xc*sin(angle);
    476477  } else {
    477     rectangle[0].coords.pc1_1 = +cos(angle);
     478    rectangle[0].coords.pc1_1 = +cos(angle) * X_PARITY;
    478479    rectangle[0].coords.pc1_2 = +sin(angle);
    479     rectangle[0].coords.pc2_1 = -sin(angle);
     480    rectangle[0].coords.pc2_1 = -sin(angle) * X_PARITY;
    480481    rectangle[0].coords.pc2_2 = +cos(angle);
    481482    xcr = xc;
    … …  
    514515  angle = 0.0;
    515516  if (FIX_NS) {
    516     rectangle[0].coords.pc1_1 = +1.0;
     517    rectangle[0].coords.pc1_1 = +1.0 * X_PARITY;
    517518    rectangle[0].coords.pc1_2 = +0.0;
    518519    rectangle[0].coords.pc2_1 = -0.0;
    519520    rectangle[0].coords.pc2_2 = +1.0;
    520521  } else {
    521     rectangle[0].coords.pc1_1 = +cos(angle);
     522    rectangle[0].coords.pc1_1 = +cos(angle) * X_PARITY;
    522523    rectangle[0].coords.pc1_2 = +sin(angle);
    523     rectangle[0].coords.pc2_1 = -sin(angle);
     524    rectangle[0].coords.pc2_1 = -sin(angle) * X_PARITY;
    524525    rectangle[0].coords.pc2_2 = +cos(angle);
    525526  }
    … …  
    626627    ring[i].coords.crval2 = dec;
    627628
    628     ring[i].coords.pc1_1 = +1.0;
     629    ring[i].coords.pc1_1 = +1.0 * X_PARITY;
    629630    ring[i].coords.pc1_2 = +0.0;
    630631    ring[i].coords.pc2_1 = -0.0;

  • branches/czw_branch/20110406/Ohana/src/getstar/include/dvoImagesAtCoords.h

    r27435 r31253  
    77
    88int       VERBOSE;
     9int       LISTCHIPCOORDS;
    910
    1011char OUTPUT[256];
    … …  
    1920
    2021typedef struct {
     22    int     n;
     23    double  x;
     24    double  y;
     25} Match;
     26
     27typedef struct {
    2128    int     id;
    2229    double  ra;
    2330    double  dec;
    2431    int     Nmatches;
    25     int     *matches;
     32    Match   *matches;
    2633    int     arrayLength;
    2734} Point;

  • branches/czw_branch/20110406/Ohana/src/getstar/src/MatchCoords.c

    r30791 r31253  
    9393            totalMatches++;
    9494
    95             pt->matches[pt->Nmatches] = i;
     95            pt->matches[pt->Nmatches].n = i;
     96            pt->matches[pt->Nmatches].x = x;
     97            pt->matches[pt->Nmatches].y = y;
    9698            pt->Nmatches ++;
    9799
    98100            if (pt->Nmatches == pt->arrayLength) {
    99101                pt->arrayLength += 20;
    100                 REALLOCATE (pt->matches, int, pt->arrayLength);
     102                REALLOCATE (pt->matches, Match, pt->arrayLength);
    101103            }
    102104        }

  • branches/czw_branch/20110406/Ohana/src/getstar/src/args_coords.c

    r26288 r31253  
    4141
    4242  /* check for command line options */
     43  LISTCHIPCOORDS = FALSE;
     44  if ((N = get_argument (argc, argv, "-listchipcoords"))) {
     45    LISTCHIPCOORDS = TRUE;
     46    remove_argument (N, &argc, argv);
     47  }
    4348  WITH_PHU = FALSE;
    4449  if ((N = get_argument (argc, argv, "+phu"))) {

  • branches/czw_branch/20110406/Ohana/src/getstar/src/dvoImagesAtCoords.c

    r29001 r31253  
    9191    while ((Nread = fscanf(f, "%d %lf %lf\n", &pts[Npoints].id, &pts[Npoints].ra, &pts[Npoints].dec)) == 3) {
    9292        pts[Npoints].Nmatches = 0;
    93         ALLOCATE(pts[Npoints].matches, int, 20);
     93        ALLOCATE(pts[Npoints].matches, Match, 20);
    9494        pts[Npoints].arrayLength = 20;
    9595
    … …  
    118118    pts[0].dec = dec;
    119119    pts[0].Nmatches = 0;
    120     ALLOCATE(pts[0].matches, int, 20);
     120    ALLOCATE(pts[0].matches, Match, 20);
    121121    pts[0].arrayLength = 20;
    122122
    … …  
    133133      Point *pt = points + j;
    134134      for (i = 0; i < pt->Nmatches; i++) {
    135         int N = pt->matches[i];
     135        int N = pt->matches[i].n;
     136        double x = pt->matches[i].x;
     137        double y = pt->matches[i].y;
    136138
    137139        char *name;
    … …  
    150152            name = dbImages[N].name;
    151153        }
    152         fprintf (stdout, "%d %lf %lf %s\n", pt->id, pt->ra, pt->dec, name);
     154        if (LISTCHIPCOORDS) {
     155            fprintf (stdout, "%d %lf %lf %s %8.2lf %8.2lf\n", pt->id, pt->ra, pt->dec, name, x, y);
     156        } else {
     157            fprintf (stdout, "%d %lf %lf %s\n", pt->id, pt->ra, pt->dec, name);
     158        }
    153159        if  (copy) {
    154160            free(copy);

  • branches/czw_branch/20110406/ippScripts/scripts/ipp_apply_burntool_single.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/czw_branch/20110406/ippScripts/scripts/stack_skycell.pl

    r30842 r31253  
    7373PPSTACK.OUTPUT.EXPNUM       
    7474PPSTACK.OUTPUT.EXPWT
    75 PPSTACK.TARGET.PSF         
    7675PPSTACK.OUTPUT.JPEG1       
    7776PPSTACK.OUTPUT.JPEG2       
    … …  
    8483# extra outputs when convolving
    8584my @outputListUnconv = qw(
     85PPSTACK.TARGET.PSF         
    8686PPSTACK.UNCONV
    8787PPSTACK.UNCONV.MASK

  • branches/czw_branch/20110406/ippScripts/scripts/staticsky.pl

    r31164 r31253  
    3232my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
    3333my $psphotStack = can_run('psphotStack') or (warn "Can't find psphotStack" and $missing_tools = 1);
     34my $psphot = can_run('psphot') or (warn "Can't find psphot" and $missing_tools = 1);
    3435my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
     36my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
    3537if ($missing_tools) {
    3638    warn("Can't find required tools.");
    … …  
    9698}
    9799
    98 # generate the input
    99 print $listFile "INPUT   MULTI\n";
    100 my $nInputs = @$files;
    101 
    102 my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot);
    103 
    104 foreach my $file (@$files) {
    105     print $listFile "INPUT   METADATA\n";
    106 
    107     # XXX if we take the input from 'warp', we will need to make different selections here
    108     my $path_base = $file->{path_base};
    109     print "input: $path_base\n";
    110 
    111     my $imageCnv  = $ipprc->filename("PPSTACK.OUTPUT",          $path_base ); # Image name
    112     my $maskCnv   = $ipprc->filename("PPSTACK.OUTPUT.MASK",     $path_base ); # Mask name
    113     my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name
    114 
    115     my $imageRaw  = $ipprc->filename("PPSTACK.UNCONV",          $path_base ); # Image name
    116     my $maskRaw   = $ipprc->filename("PPSTACK.UNCONV.MASK",     $path_base ); # Mask name
    117     my $weightRaw = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $path_base ); # Weight name
    118 
    119     my $sources   = $ipprc->filename("PSPHOT.OUT.CMF.MEF",      $path_base ); # Sources name
    120 
    121     # XXX is this the correct PSF file?
    122     my $psfCnv    = $ipprc->filename("PPSTACK.TARGET.PSF",      $path_base ); # PSF name
    123 
    124     # XXX we could make some different choices if some inputs do not exist...
    125     &my_die("Couldn't find input: $imageRaw",  $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageRaw");
    126     &my_die("Couldn't find input: $maskRaw",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskRaw");
    127     &my_die("Couldn't find input: $weightRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightRaw");
    128     &my_die("Couldn't find input: $imageCnv",  $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv");
    129     &my_die("Couldn't find input: $maskCnv",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv");
    130     &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv");
    131     &my_die("Couldn't find input: $psfCnv",    $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv");
    132     &my_die("Couldn't find input: $sources",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources");
    133 
    134     print $listFile "  RAW:IMAGE     STR  " . $imageRaw  . "\n";
    135     print $listFile "  RAW:MASK      STR  " . $maskRaw   . "\n";
    136     print $listFile "  RAW:VARIANCE  STR  " . $weightRaw . "\n";
    137 
    138     print $listFile "  CNV:IMAGE     STR  " . $imageCnv  . "\n";
    139     print $listFile "  CNV:MASK      STR  " . $maskCnv   . "\n";
    140     print $listFile "  CNV:VARIANCE  STR  " . $weightCnv . "\n";
    141     print $listFile "  CNV:PSF       STR  " . $psfCnv    . "\n";
    142 
    143     print $listFile "  SOURCES       STR  " . $sources   . "\n";
    144 
    145     print $listFile "END\n\n";
    146 }
    147 
    148100# Recipes to use based on reduction class
    149101$reduction = 'DEFAULT' unless defined $reduction;
    … …  
    160112print "recipe_ppstack: $recipe_ppstack\n";
    161113
    162 # my $cmdflags;
    163 
    164 # Perform stack photometry analysis
    165 {
    166     my $command = "$psphotStack $outroot";
    167     $command .= " -input $listName";
    168     $command .= " -threads $threads" if defined $threads;
    169     $command .= " -recipe PSPHOT  $recipe_psphot";
    170     $command .= " -recipe PPSUB   $recipe_ppsub";
    171     $command .= " -recipe PPSTACK $recipe_ppstack";
    172     $command .= " -dumpconfig $configuration";
    173     $command .= " -tracedest $traceDest -log $logDest";
    174     # $command .= " -dbname $dbname" if defined $dbname;
    175     # $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id;
    176     # $command .= " -source_id $source_id" if defined $source_id;
    177 
    178     unless ($no_op) {
    179         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    180         unless ($success) {
    181             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    182             &my_die("Unable to perform psphotStack: $error_code", $sky_id, $error_code);
    183         }
    184 
    185         # my $outputStatsReal = $ipprc->file_resolve($outputStats);
    186         # &my_die("Couldn't find expected output file: $outputStats", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
    187 
    188         # measure chip stats
    189         # $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL";
    190         # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    191         #     run(command => $command, verbose => $verbose);
    192         # unless ($success) {
    193         #     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    194         #     &my_die("Unable to perform ppStatsFromMetadata: $error_code", $sky_id, $error_code);
    195         # }
    196         # foreach my $line (@$stdout_buf) {
    197         #     $cmdflags .= " $line";
    198         # }
    199         # chomp $cmdflags;
    200 
    201         # my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
    202 
    203         my $quality = 0;
    204         if (!$quality) {
    205 
    206             # Get the output filenames
    207             # we have one set of output files per input file set
    208             for (my $i = 0; $i < @$files; $i++) {
    209                 my $outputName     = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot, $i);
    210                 my $outputMask     = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot, $i);
    211                 my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot, $i);
    212                 my $outputSources  = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot, $i);
    213 
    214                 # XXX these are optional and not generated by default
    215                 # &my_die("Couldn't find expected output file: $outputName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
    216                 # &my_die("Couldn't find expected output file: $outputMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
    217                 # &my_die("Couldn't find expected output file: $outputVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance);
    218                 &my_die("Couldn't find expected output file: $outputSources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
    219             }
    220 
    221             #my $configuration  = $ipprc->filename("PPSUB.CONFIG", $outroot);
    222             #my $outputStats    = $ipprc->filename("SKYCELL.STATS", $outroot);
    223             #my $traceDest      = $ipprc->filename("TRACE.EXP", $outroot);
    224 
    225             my $chisqName     = $ipprc->filename("PSPHOT.CHISQ.IMAGE", $outroot);
    226             my $chisqMask     = $ipprc->filename("PSPHOT.CHISQ.MASK", $outroot);
    227             my $chisqVariance = $ipprc->filename("PSPHOT.CHISQ.VARIANCE", $outroot);
    228 
    229             # XXX check the recipe -- should we expect these to exist?
    230             # &my_die("Couldn't find expected output file: $chisqName",           $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqName);
    231             # &my_die("Couldn't find expected output file: $chisqMask",           $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqMask);
    232             # &my_die("Couldn't find expected output file: $chisqVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqVariance);
    233         }
    234     } else {
    235         print "Not executing: $command\n";
    236     }
     114
     115# generate the input
     116print $listFile "INPUT   MULTI\n";
     117my $nInputs = @$files;
     118
     119if ($nInputs > 1) {
     120        my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot);
     121
     122        foreach my $file (@$files) {
     123            print $listFile "INPUT   METADATA\n";
     124
     125            # XXX if we take the input from 'warp', we will need to make different selections here
     126            my $path_base = $file->{path_base};
     127            print "input: $path_base\n";
     128
     129            my $imageCnv  = $ipprc->filename("PPSTACK.OUTPUT",          $path_base ); # Image name
     130            my $maskCnv   = $ipprc->filename("PPSTACK.OUTPUT.MASK",     $path_base ); # Mask name
     131            my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name
     132
     133            my $imageRaw  = $ipprc->filename("PPSTACK.UNCONV",          $path_base ); # Image name
     134            my $maskRaw   = $ipprc->filename("PPSTACK.UNCONV.MASK",     $path_base ); # Mask name
     135            my $weightRaw = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $path_base ); # Weight name
     136
     137            my $sources   = $ipprc->filename("PSPHOT.OUT.CMF.MEF",      $path_base ); # Sources name
     138
     139            # XXX is this the correct PSF file?
     140            my $psfCnv    = $ipprc->filename("PPSTACK.TARGET.PSF",      $path_base ); # PSF name
     141
     142            # XXX we could make some different choices if some inputs do not exist...
     143            &my_die("Couldn't find input: $imageRaw",  $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageRaw");
     144            &my_die("Couldn't find input: $maskRaw",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskRaw");
     145            &my_die("Couldn't find input: $weightRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightRaw");
     146            &my_die("Couldn't find input: $imageCnv",  $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv");
     147            &my_die("Couldn't find input: $maskCnv",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv");
     148            &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv");
     149            &my_die("Couldn't find input: $psfCnv",    $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv");
     150            &my_die("Couldn't find input: $sources",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources");
     151
     152            print $listFile "  RAW:IMAGE     STR  " . $imageRaw  . "\n";
     153            print $listFile "  RAW:MASK      STR  " . $maskRaw   . "\n";
     154            print $listFile "  RAW:VARIANCE  STR  " . $weightRaw . "\n";
     155
     156            print $listFile "  CNV:IMAGE     STR  " . $imageCnv  . "\n";
     157            print $listFile "  CNV:MASK      STR  " . $maskCnv   . "\n";
     158            print $listFile "  CNV:VARIANCE  STR  " . $weightCnv . "\n";
     159            print $listFile "  CNV:PSF       STR  " . $psfCnv    . "\n";
     160
     161            print $listFile "  SOURCES       STR  " . $sources   . "\n";
     162
     163            print $listFile "END\n\n";
     164        }
     165
     166        # my $cmdflags;
     167
     168        # Perform stack photometry analysis
     169        {
     170            my $command = "$psphotStack $outroot";
     171            $command .= " -input $listName";
     172            $command .= " -threads $threads" if defined $threads;
     173            $command .= " -recipe PSPHOT  $recipe_psphot";
     174            $command .= " -recipe PPSUB   $recipe_ppsub";
     175            $command .= " -recipe PPSTACK $recipe_ppstack";
     176            $command .= " -dumpconfig $configuration";
     177            $command .= " -tracedest $traceDest -log $logDest";
     178            # $command .= " -dbname $dbname" if defined $dbname;
     179            # $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id;
     180            # $command .= " -source_id $source_id" if defined $source_id;
     181
     182            unless ($no_op) {
     183                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     184                unless ($success) {
     185                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     186                    &my_die("Unable to perform psphotStack: $error_code", $sky_id, $error_code);
     187                }
     188
     189                # my $outputStatsReal = $ipprc->file_resolve($outputStats);
     190                # &my_die("Couldn't find expected output file: $outputStats", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
     191
     192                # measure chip stats
     193                # $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL";
     194                # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     195                #     run(command => $command, verbose => $verbose);
     196                # unless ($success) {
     197                #     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     198                #     &my_die("Unable to perform ppStatsFromMetadata: $error_code", $sky_id, $error_code);
     199                # }
     200                # foreach my $line (@$stdout_buf) {
     201                #     $cmdflags .= " $line";
     202                # }
     203                # chomp $cmdflags;
     204
     205                # my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
     206
     207                my $quality = 0;
     208                if (!$quality) {
     209
     210                    # Get the output filenames
     211                    # we have one set of output files per input file set
     212                    for (my $i = 0; $i < @$files; $i++) {
     213                        my $outputName     = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot, $i);
     214                        my $outputMask     = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot, $i);
     215                        my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot, $i);
     216                        my $outputSources  = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot, $i);
     217
     218                        # XXX these are optional and not generated by default
     219                        # &my_die("Couldn't find expected output file: $outputName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
     220                        # &my_die("Couldn't find expected output file: $outputMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
     221                        # &my_die("Couldn't find expected output file: $outputVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance);
     222                        &my_die("Couldn't find expected output file: $outputSources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
     223                    }
     224
     225                    #my $configuration  = $ipprc->filename("PPSUB.CONFIG", $outroot);
     226                    #my $outputStats    = $ipprc->filename("SKYCELL.STATS", $outroot);
     227                    #my $traceDest      = $ipprc->filename("TRACE.EXP", $outroot);
     228
     229                    my $chisqName     = $ipprc->filename("PSPHOT.CHISQ.IMAGE", $outroot);
     230                    my $chisqMask     = $ipprc->filename("PSPHOT.CHISQ.MASK", $outroot);
     231                    my $chisqVariance = $ipprc->filename("PSPHOT.CHISQ.VARIANCE", $outroot);
     232
     233                    # XXX check the recipe -- should we expect these to exist?
     234                    # &my_die("Couldn't find expected output file: $chisqName",           $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqName);
     235                    # &my_die("Couldn't find expected output file: $chisqMask",           $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqMask);
     236                    # &my_die("Couldn't find expected output file: $chisqVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqVariance);
     237                }
     238            } else {
     239                print "Not executing: $command\n";
     240            }
     241        }
     242} else {
     243        # single input. Run psphot
     244        my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot);
     245
     246        my $file = $files->[0];
     247
     248        # XXX if we take the input from 'warp', we will need to make different selections here
     249        my $path_base = $file->{path_base};
     250        print "input: $path_base\n";
     251
     252        # examine the recipe to determine whether to analyze the "raw" or convolved images
     253        my $command = "$ppConfigDump -camera $camera -dump-recipe PSPHOT -recipe PSPHOT $recipe_psphot -";
     254        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     255        run(command => $command, verbose => 0);
     256        unless ($success) {
     257            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     258            &my_die("Unable to perform ppConfigDump: $error_code", $sky_id, $PS_EXIT_SYS_ERROR);
     259        }
     260        my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
     261            &my_die("Unable to parse metadata config doc", $sky_id, $PS_EXIT_SYS_ERROR);
     262
     263        my $use_raw = metadataLookupBool($recipeData, 'PSPHOT.STACK.USE.RAW');
     264
     265        my ($image, $mask, $variance);
     266        if ($use_raw) {
     267            $image  = $ipprc->filename("PPSTACK.UNCONV",          $path_base ); # Image name
     268            $mask   = $ipprc->filename("PPSTACK.UNCONV.MASK",     $path_base ); # Mask name
     269            $variance = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $path_base ); # Weight name
     270        } else {
     271            $image  = $ipprc->filename("PPSTACK.OUTPUT",          $path_base ); # Image name
     272            $mask   = $ipprc->filename("PPSTACK.OUTPUT.MASK",     $path_base ); # Mask name
     273            $variance = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name
     274        }
     275
     276        my $sources   = $ipprc->filename("PSPHOT.OUT.CMF.MEF",      $path_base ); # Sources name
     277
     278        # XXX is this the correct PSF file?
     279        my $psfCnv    = $ipprc->filename("PPSTACK.TARGET.PSF",      $path_base ); # PSF name
     280
     281        # XXX we could make some different choices if some inputs do not exist...
     282        &my_die("Couldn't find input: $image",  $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$image");
     283        &my_die("Couldn't find input: $mask",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$mask");
     284        &my_die("Couldn't find input: $variance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$variance");
     285        &my_die("Couldn't find input: $psfCnv",    $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv");
     286        &my_die("Couldn't find input: $sources",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources");
     287
     288        # my $cmdflags;
     289
     290        # Perform stack photometry analysis
     291        {
     292            my $command = "$psphot $outroot";
     293            $command .= " -file $image";
     294            $command .= " -mask $mask";
     295            $command .= " -variance $variance";
     296            $command .= " -threads $threads" if defined $threads;
     297            $command .= " -recipe PSPHOT  $recipe_psphot";
     298            $command .= " -dumpconfig $configuration" if $configuration;
     299            $command .= " -tracedest $traceDest -log $logDest";
     300
     301            unless ($no_op) {
     302                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     303                unless ($success) {
     304                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     305                    &my_die("Unable to perform psphotStack: $error_code", $sky_id, $error_code);
     306                }
     307
     308                # my $outputStatsReal = $ipprc->file_resolve($outputStats);
     309                # &my_die("Couldn't find expected output file: $outputStats", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
     310
     311                # measure chip stats
     312                # $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL";
     313                # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     314                #     run(command => $command, verbose => $verbose);
     315                # unless ($success) {
     316                #     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     317                #     &my_die("Unable to perform ppStatsFromMetadata: $error_code", $sky_id, $error_code);
     318                # }
     319                # foreach my $line (@$stdout_buf) {
     320                #     $cmdflags .= " $line";
     321                # }
     322                # chomp $cmdflags;
     323
     324                # my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
     325
     326                my $quality = 0;
     327                if (0 and !$quality) {
     328
     329                    # Get the output filenames
     330                    # we have one set of output files per input file set
     331                    for (my $i = 0; $i < @$files; $i++) {
     332                        my $outputName     = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot, $i);
     333                        my $outputMask     = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot, $i);
     334                        my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot, $i);
     335                        my $outputSources  = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot, $i);
     336
     337                        # XXX these are optional and not generated by default
     338                        # &my_die("Couldn't find expected output file: $outputName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
     339                        # &my_die("Couldn't find expected output file: $outputMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
     340                        # &my_die("Couldn't find expected output file: $outputVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance);
     341                        &my_die("Couldn't find expected output file: $outputSources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
     342                    }
     343
     344                    #my $configuration  = $ipprc->filename("PPSUB.CONFIG", $outroot);
     345                    #my $outputStats    = $ipprc->filename("SKYCELL.STATS", $outroot);
     346                    #my $traceDest      = $ipprc->filename("TRACE.EXP", $outroot);
     347
     348                    my $chisqName     = $ipprc->filename("PSPHOT.CHISQ.IMAGE", $outroot);
     349                    my $chisqMask     = $ipprc->filename("PSPHOT.CHISQ.MASK", $outroot);
     350                    my $chisqVariance = $ipprc->filename("PSPHOT.CHISQ.VARIANCE", $outroot);
     351
     352                    # XXX check the recipe -- should we expect these to exist?
     353                    # &my_die("Couldn't find expected output file: $chisqName",           $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqName);
     354                    # &my_die("Couldn't find expected output file: $chisqMask",           $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqMask);
     355                    # &my_die("Couldn't find expected output file: $chisqVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqVariance);
     356                }
     357            } else {
     358                print "Not executing: $command\n";
     359            }
     360        }
    237361}
    238362

  • branches/czw_branch/20110406/ippToPsps/jython/batch.py

    r31201 r31253  
    77import os
    88import md5
    9 
     9import shutil
     10import logging
     11from subprocess import call, PIPE, Popen
     12
     13from datastore import Datastore
    1014from gpc1db import Gpc1Db
    1115from ipptopspsdb import IppToPspsDb
    … …  
    1418from java.sql import *
    1519from xml.etree.ElementTree import ElementTree, Element, tostring
    16 from xml.dom import minidom
    1720
    1821'''
    … …  
    3033    "../config/2/tables.vot"
    3134    '''
    32     def __init__(self, batchType, inputFitsPath="", survey=""):
     35    def __init__(self, logger, batchType, inputFitsPath="", survey=""):
     36
     37        # set up logging
     38        self.logger = logger
     39        self.logger.debug("Batch class constructor")
    3340
    3441        # set up class variables
    … …  
    5158
    5259        # create Gpc1Db object
    53         self.gpc1Db = Gpc1Db()
     60        self.gpc1Db = Gpc1Db(self.logger)
    5461
    5562        # get survey ID from init table
    … …  
    6067            self.surveyID = rs.getInt(1)
    6168        except:
    62             self.log("No survey ID found for this survey: '" + survey + "'")
     69            self.logger.exception("No survey ID found for this survey: '" + survey + "'")
    6370            self.surveyID = -1;
    6471   
    6572        # get dvo info from config
    66         dvoName = doc.find(survey+"dvo/name").text
    67         dvoLocation = doc.find(survey+"dvo/location").text
     73        if survey != "":
     74            dvoName = doc.find(survey+"dvo/name").text
     75            dvoLocation = doc.find(survey+"dvo/location").text
     76        else:
     77            dvoName = ""
     78            dvoLocation = ""
    6879         
    6980        # get datastore info from config
    70         datastoreProduct = doc.find("datastore/product").text
     81        self.datastore = Datastore(self.logger)
    7182
    7283        # create IppToPspsDb object and create a new batch
    73         self.ippToPspsDb = IppToPspsDb()
    74         self.batchID = self.ippToPspsDb.createNewBatch(66, survey, self.getPspsBatchType(), dvoName, datastoreProduct)
     84        self.ippToPspsDb = IppToPspsDb(logger)
     85        self.batchID = self.ippToPspsDb.createNewBatch(66,
     86                survey,
     87                self.getPspsBatchType(),
     88                dvoName,
     89                self.datastore.product)
    7590
    7691        # get local storage location from config
    7792        self.batchName = "B%08d" % self.batchID
    78         self.localOutPath = doc.find("localOutPath").text + "/" + self.getPspsBatchType() + "/" + dvoName + "/%s" % self.batchName
     93        self.subDir = doc.find("localOutPath").text + "/" + self.getPspsBatchType() + "/" + dvoName
     94        self.localOutPath = self.subDir + "/" + self.batchName
    7995        if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath)
    8096
    … …  
    94110    def __del__(self):
    95111
    96         self.log("Batch destructor")
     112        self.logger.debug("Batch destructor")
    97113        self.localStmt.close()
    98114        self.localCon.close()
    … …  
    105121        outPath = self.localOutPath + "/BatchManifest.xml"
    106122        tmpPath = "./tmp.xml"
    107         self.log("Creating batch manifest file here: " + outPath)
     123        self.logger.info("Creating batch manifest file here: " + outPath)
    108124        root = Element('manifest')
    109125
    … …  
    117133
    118134        # get md5sum
    119         fits=os.popen("md5sum " + self.outputFitsPath)
    120         for f in fits.readlines(): md5sum = f[0:f.rfind(" ")]
     135        p = Popen("md5sum " + self.outputFitsPath, shell=True, stdout=PIPE)
     136        p.wait()
     137        out = p.stdout.read()
     138        md5sum = out[0:out.rfind(" ")]
    121139
    122140        # get file size
    … …  
    136154
    137155        # clunky way to prettify XML
    138         os.popen("xmllint --format " + tmpPath + " > " + outPath)
    139 #        os.remove(tmpPath)
    140 
     156        p = Popen("xmllint --format " + tmpPath + " > " + outPath, shell=True, stdout=PIPE)
     157        p.wait()
     158        os.remove(tmpPath)
     159
     160
     161    '''
     162    tar and zips batch directory
     163    '''
     164    def createTarball(self):
     165     
     166        # set up filenams and paths
     167        tarFile = self.batchName + ".tar"
     168        tarPath = self.subDir + "/" + tarFile
     169
     170        self.tarballFile = tarFile + ".gz"
     171        tarballPath = self.subDir + "/" + self.tarballFile
     172
     173        # tar directory
     174        p = Popen("tar -cvf " + tarPath + "\
     175                -C " + self.subDir + " \
     176                " + self.batchName, shell=True, stdout=PIPE)
     177        p.wait()
     178
     179        # zip tar archive
     180        p = Popen("gzip -c " + tarPath + " > " + tarballPath, shell=True, stdout=PIPE)
     181        p.wait()
     182
     183        # delete tar file and original directory
     184        os.remove(tarPath)
     185        shutil.rmtree(self.localOutPath)
     186
     187    '''
     188    Publishes this batch to the datastore
     189    '''
     190    def publishToDatastore(self):
     191
     192        self.datastore.publish(self.batchName, self.subDir, self.tarballFile, "tgz")
     193        # TODO update ippToPsps Db here
    141194
    142195    '''
    … …  
    147200        if self.survey == "ThreePi": return "3PI"
    148201        elif self.survey == "MD04": return "MD04"
    149         else: self.log("ERROR: Don't know this survey: " + self.survey)
     202        else: self.logger.error("Don't know this survey: " + self.survey)
    150203
    151204    '''
    … …  
    157210        elif self.batchType == "detection": return "P2"
    158211        elif self.batchType == "stack": return "ST"
    159         else: self.log("ERROR: Don't know this batch type: " + self.survey)
     212        else: self.logger.error("Don't know this batch type: " + self.survey)
    160213
    161214
    … …  
    244297         self.pspsTables = stilts.treads(self.pspsVoTableFilePath)
    245298         for table in self.pspsTables:
    246              self.log("Creating PSPS table: " + table.name)
     299             self.logger.info("Creating PSPS table: " + table.name)
    247300             table.write(self.localUrl + '#' + table.name)
    248301
    … …  
    253306    '''   
    254307    def indexIppTables(self):
    255         self.log("indexIppTables not implemented")
     308        self.logger.warn("indexIppTables not implemented")
    256309
    257310
    … …  
    261314    def createIndex(self, table, column):
    262315
    263         self.log("Creating index on column '"+column+"' for table '"+table+"'")
     316        self.logger.info("Creating index on column '"+column+"' for table '"+table+"'")
    264317
    265318        sql = "CREATE INDEX "+table+"_index ON "+table+" ("+column+")"
    … …  
    267320            self.localStmt.execute(sql)
    268321        except:
    269             self.log("Index already in place on '" + column + "' for table '" + table + "'")
     322            self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'")
    270323
    271324
    … …  
    274327    '''   
    275328    def indexPspsTables(self):
    276         self.log("indexPspsTables not implemented")
     329        self.logger.warn("indexPspsTables not implemented")
    277330
    278331    '''
    … …  
    290343          match = re.match(filter, table.name)
    291344          if not match: continue
    292           self.log("Creating IPP table " + table.name)
     345          self.logger.info("Creating IPP table " + table.name)
    293346          table = stilts.tpipe(table, cmd='explodeall')
    294347          try:
    295348              table.write(self.localUrl + '#' + table.name)
    296349          except:
    297               self.log("ERROR problem writing table '" + table.name + "' to the database")
     350              self.logger.exception("Problem writing table '" + table.name + "' to the database")
    298351
    299352          count = count + 1
    300353
    301       self.log("Imported %d tables from '%s' " % (count, self.inputFitsPath))
     354      self.logger.info("Imported %d tables from '%s' " % (count, self.inputFitsPath))
    302355
    303356      self.indexIppTables()
    … …  
    308361    def exportPspsTablesToFits(self):
    309362
    310         self.log("Exporting all PSPS tables to FITS")
     363        self.logger.info("Exporting all PSPS tables to FITS")
    311364        _tables = []
    312365
    313         self.log("    Selecting database tables")
     366        self.logger.info("    Selecting database tables")
    314367        for table in self.pspsTables:
    315368           _table = stilts.tread(self.localUrl + '#SELECT * FROM ' + table.name)
    … …  
    317370           _tables.append(_table)
    318371
    319         self.log("    Writing to FITS file '" + self.outputFitsPath + "'...")
     372        self.logger.info("    Writing to FITS file '" + self.outputFitsPath + "'...")
    320373        stilts.twrites(_tables, self.outputFitsPath, fmt='fits')
    321         self.log("    ...done")
     374        self.logger.info("    ...done")
    322375
    323376    '''
    … …  
    394447    def replaceAllPspsNulls(self, sub):
    395448
    396         self.log("Replacing all NULL values in PSPS tables with '" + sub + "'...")
     449        self.logger.info("Replacing all NULL values in PSPS tables with '" + sub + "'...")
    397450        for table in self.pspsTables:
    398451            self.replaceNulls(table.name, sub)
    399         self.log("...done")
     452        self.logger.info("...done")
    400453
    401454    '''
    … …  
    403456    '''
    404457    def populatePspsTables(self):
    405         self.log("Not implemented yet")
     458        self.logger.warn("Not implemented yet")
    406459
    407460    '''
    … …  
    410463    def updateDvoIDs(self, table):
    411464
    412         self.log("Updating table '" + table + "' with DVO IDs...")
     465        self.logger.info("Updating table '" + table + "' with DVO IDs...")
    413466        sql = "UPDATE " + table + " AS a, dvo AS b SET \
    414467               a.ippObjID = b.ippObjID, \
    … …  
    416469               WHERE a.ippDetectID = b.ippDetectID"
    417470        self.localStmt.execute(sql)
    418         self.log("...done")
     471        self.logger.info("...done")
    419472
    420473
    … …  
    424477    def createDvoTable(self):
    425478
    426         self.log("Creating DVO table for ID matching")
     479        self.logger.info("Creating DVO table for ID matching")
    427480        sql = "DROP TABLE dvo"
    428481        self.localStmt.execute(sql)

  • branches/czw_branch/20110406/ippToPsps/jython/config.xml

    r31199 r31253  
    3232  <datastore>
    3333    <product>PSPS_JHU</product>
     34    <type>IPP_PSPS</type>
    3435  </datastore>
    3536

  • branches/czw_branch/20110406/ippToPsps/jython/gpc1db.py

    r31200 r31253  
    55import glob
    66import os
     7import logging
    78
    89from java.sql import *
    … …  
    2122
    2223    '''
    23     def __init__(self):
     24    def __init__(self, logger):
     25
     26        # setup logging
     27        self.logger = logger
     28        self.logger.debug("Gpc1Db constructor")
    2429
    2530        # open config
    … …  
    4045    def __del__(self):
    4146
    42         print "Gpc1Db destructor"
     47        self.logger.debug("Gpc1Db destructor")
    4348        self.stmt.close()
    4449        self.con.close()
    … …  
    4954    '''
    5055    def getStackStageCmfs(self, skyID):
     56
     57        self.logger.debug("Querying for stack cmf files")
     58
    5159        sql = "SELECT path_base, num_inputs \
    5260               FROM staticskyResult \
    … …  
    5664            rs.first()
    5765        except:
    58             print "No worky"
     66            self.logger.exception("Can't query for stack cmfs")
    5967
    6068        # get path to base dir of cmf files

  • branches/czw_branch/20110406/ippToPsps/jython/initbatch.py

    r31180 r31253  
    22
    33import stilts
     4import logging
     5import logging.config
    46from java.lang import *
    57from java.sql import *
    … …  
    1416    Constructor
    1517    '''
    16     def __init__(self):
    17        super(InitBatch, self).__init__("init")
     18    def __init__(self, logger):
     19       super(InitBatch, self).__init__(logger, "init")
    1820
    19        self.outputFitsPath = "00000000.FITS";
     21       self.outputFitsFile = "00000000.FITS";
     22       self.outputFitsPath = self.localOutPath + "/" + self.outputFitsFile
    2023
    21 
    22 initBatch = InitBatch()
     24logging.config.fileConfig("logging.conf")
     25logger = logging.getLogger("initbatch")
     26initBatch = InitBatch(logger)
    2327initBatch.createEmptyPspsTables()
    2428initBatch.exportPspsTablesToFits()

  • branches/czw_branch/20110406/ippToPsps/jython/ipptopspsdb.py

    r31198 r31253  
    44import sys
    55import os
     6import logging
    67
    78from java.sql import *
    … …  
    1920
    2021    '''
    21     def __init__(self):
     22    def __init__(self, logger):
    2223
     24        # setup logging
     25        self.logger = logger
     26        self.logger.debug("IppToPspsDb Constructor")
    2327        # open config
    2428        doc = ElementTree(file="config.xml")
    … …  
    3842    def createNewBatch(self, expID, surveyType, batchType, dvoDb, datastoreProduct):
    3943
    40         print "Creating nw batch in ippToPsps database"
    4144        sql = "SELECT batch_id FROM batches ORDER BY batch_id DESC LIMIT 1"
    4245
    … …  
    4851            batchID = rs.getInt(1)
    4952        except:
    50             print "Unable to get batch ID"
     53            self.logger.exception("Unable to get batch ID")
    5154
    5255        if batchID > 0:
    … …  
    7073            self.stmt.execute(sql)
    7174
     75        self.logger.info("Creating new batch in ippToPsps database with batchID = %d" % batchID)
     76
    7277        return batchID;
    7378
    … …  
    7782    def __del__(self):
    7883
    79         print "IppToPspsDb destructor"
     84        self.logger.debug("IppToPspsDb Desstructor")
    8085        self.stmt.close()
    8186        self.con.close()

  • branches/czw_branch/20110406/ippToPsps/jython/stackbatch.py

    r31202 r31253  
    77from gpc1db import Gpc1Db
    88from batch import Batch
     9import logging.config
    910
    1011class StackBatch(Batch):
    … …  
    1314    Constructor
    1415    '''
    15     def __init__(self, skyID, inputFile):
     16    def __init__(self, logger, skyID, inputFile):
    1617       super(StackBatch, self).__init__(
     18               logger,
    1719               "stack",
    1820               inputFile,
    1921               "MD04") # TODO
    2022
    21        self.log("StackBatch constructor. Creating batch from input file here: ''" + inputFile + "'")
     23       self.logger.info("StackBatch constructor. Creating batch from: ''" + inputFile + "'")
    2224
    2325       # obs time makes no sense except for nightly stacks
    … …  
    185187    '''
    186188    def populateStackMeta(self):
    187         self.log("Procesing StackMeta table")
     189        self.logger.info("Procesing StackMeta table")
    188190
    189191        sql = "INSERT INTO StackMeta (\
    … …  
    233235    '''
    234236    def populateStackDetection(self):
    235         self.log("Procesing StackDetection table")
     237        self.logger.info("Procesing StackDetection table")
    236238
    237239        # insert all the detections
    … …  
    297299    '''
    298300    def populateStackApFlx(self):
    299         self.log("Procesing StackApFlx table")
     301        self.logger.info("Procesing StackApFlx table")
    300302 
    301303        sql = "INSERT INTO StackApFlx \
    … …  
    307309
    308310        # TODO temporarily loading 1st convolved fluxes into unconvolved fields
    309         self.log("    Adding un-convolved fluxes")
     311        self.logger.info("    Adding un-convolved fluxes")
    310312        self.updateApFlxs("", "< 7.0")
    311         self.log("    Adding 1st convolved fluxes")
     313        self.logger.info("    Adding 1st convolved fluxes")
    312314        self.updateApFlxs("c1", "< 7.0")
    313         self.log("    Adding 2nd convolved fluxes")
     315        self.logger.info("    Adding 2nd convolved fluxes")
    314316        self.updateApFlxs("c2", "> 7.0")
    315317
    316         self.log("    Adding petrosians for extended sources")
     318        self.logger.info("    Adding petrosians for extended sources")
    317319        sql = "UPDATE StackApFlx AS a, SkyChip_xsrc AS b SET \
    318320        petRadius=b.PETRO_RADIUS \
    … …  
    337339    '''
    338340    def populateStackModelFit(self):
    339         self.log("Procesing StackModelFit table")
     341        self.logger.info("Procesing StackModelFit table")
    340342
    341343        # insert all the detections
    … …  
    344346
    345347        # populate model parameters
    346         self.log("    Adding deVaucouleurs fit")
     348        self.logger.info("    Adding deVaucouleurs fit")
    347349        self.updateModelFit("deV", "PS_MODEL_DEV")
    348         self.log("    Adding exponential fit")
     350        self.logger.info("    Adding exponential fit")
    349351        self.updateModelFit("exp", "PS_MODEL_EXP")
    350         self.log("    Adding sersic fit")
     352        self.logger.info("    Adding sersic fit")
    351353        self.updateModelFit("ser", "PS_MODEL_SERSIC")
    352354
    … …  
    361363    '''
    362364    def populateStackToImage(self):
    363         self.log("Procesing StackToImage table")
     365        self.logger.info("Procesing StackToImage table")
    364366
    365367        sql = "INSERT INTO StackToImage (stackMetaID) VALUES (" + self.header['STK_ID'] + ")"
    … …  
    371373    def indexPspsTables(self):
    372374
    373         self.log("Creating indexes on PSPS tables")
     375        self.logger.info("Creating indexes on PSPS tables")
    374376        self.createIndex("StackDetection", "ippDetectID")
    375377        self.createIndex("StackApFlx", "ippDetectID")
    … …  
    381383    def indexIppTables(self):
    382384
    383         self.log("Creating indexes on IPP tables")
     385        self.logger.info("Creating indexes on IPP tables")
    384386        self.createIndex("SkyChip_psf", "IPP_IDET")
    385387        self.createIndex("SkyChip_xfit", "IPP_IDET")
    … …  
    391393    def fudgeIDs(self):
    392394
    393         self.log("Creating bogus IDs on all detections")
     395        self.logger.info("Creating bogus IDs on all detections")
    394396
    395397        sql = "INSERT INTO dvo (ippDetectID) SELECT IPP_IDET FROM SkyChip_psf";
    … …  
    430432
    431433
    432 sky_id = 299
    433 gpc1Db = Gpc1Db()
     434logging.config.fileConfig("logging.conf")
     435logger = logging.getLogger("stackbatch")
     436sky_id = 299 # TODO
     437gpc1Db = Gpc1Db(logger)
    434438cmfFiles = gpc1Db.getStackStageCmfs(sky_id)
    435439
    … …  
    437441for file in cmfFiles:
    438442
    439     stackBatch = StackBatch(sky_id, file)
     443    stackBatch = StackBatch(logger, sky_id, file)
    440444   
    441445    stackBatch.createEmptyPspsTables()
    442     #stackBatch.importIppTables("")
    443     stackBatch.populatePspsTables()
    444     stackBatch.fudgeIDs()
    445     #stackBatch.reportNullsInAllPspsTables(False)
    446     #stackBatch.replaceAllPspsNulls("-999") 
    447     stackBatch.exportPspsTablesToFits()
    448     stackBatch.writeBatchManifest()
     446    stackBatch.importIppTables("")
     447#    stackBatch.populatePspsTables()
     448 #   stackBatch.fudgeIDs()
     449  #  stackBatch.reportNullsInAllPspsTables(False)
     450  #  stackBatch.replaceAllPspsNulls("-999") 
     451  #  stackBatch.exportPspsTablesToFits()
     452  #  stackBatch.writeBatchManifest()
     453  #  stackBatch.createTarball()
     454  #  stackBatch.publishToDatastore()
    449455
    450456    i = i + 1

  • branches/czw_branch/20110406/ippTools/share/Makefile.am

    r30373 r31253  
    164164        disttool_definebyquery_fake.sql \
    165165        disttool_definebyquery_raw.sql \
     166        disttool_definebyquery_raw_no_magic.sql \
    166167        disttool_definebyquery_sky.sql \
    167168        disttool_definebyquery_stack.sql \

  • branches/czw_branch/20110406/ippTools/share/difftool_listrun.sql

    r29172 r31253  
    2020    camProcessedInput.zpt_lq,
    2121    camProcessedInput.zpt_uq,
     22    camProcessedInput.fwhm_major,
    2223    rawInput.comment,
    2324    rawInput.exp_time,
    … …  
    2526    rawInput.exp_name AS exp_name_1,
    2627    rawInput.exp_id AS exp_id_1,
     28    rawInput.filter AS filter_1,
    2729    rawInput.comment AS comment_1,
    2830    rawInput.dateobs AS dateobs_1,
    … …  
    3436    rawTemplate.exp_name AS exp_name_2,
    3537    rawTemplate.exp_id AS exp_id_2,
     38    rawInput.filter AS filter_2,
    3639    rawTemplate.comment AS comment_2,
    3740    rawTemplate.dateobs AS dateobs_2,

  • branches/czw_branch/20110406/ippTools/share/difftool_skyfile.sql

    r27743 r31253  
    2424    rawInput.exp_name AS exp_name_1,
    2525    rawInput.exp_id AS exp_id_1,
     26    rawInput.filter AS filter_1,
    2627    chipInput.chip_id AS chip_id_1,
    2728    camInput.cam_id AS cam_id_1,
    … …  
    3132    rawTemplate.exp_name AS exp_name_2,
    3233    rawTemplate.exp_id AS exp_id_2,
     34    rawTemplate.filter AS filter_2,
    3335    chipTemplate.chip_id AS chip_id_2,
    3436    camTemplate.cam_id AS cam_id_2,

  • branches/czw_branch/20110406/ippTools/share/disttool_pending_SSdiff.sql

    r30387 r31253  
    66    stage_id,
    77    diffSkyfile.skycell_id AS component,
     8    exp_type,
    89    clean,
    910    rawExp.camera,

  • branches/czw_branch/20110406/ippTools/share/disttool_pending_camera.sql

    r30387 r31253  
    66    stage_id,
    77    'exposure' AS component,
     8    exp_type,
    89    clean,
    910    rawExp.camera,

  • branches/czw_branch/20110406/ippTools/share/disttool_pending_chip.sql

    r30387 r31253  
    66    stage_id,
    77    chipProcessedImfile.class_id AS component,
     8    exp_type,
    89    distRun.clean,
    910    rawExp.camera,

  • branches/czw_branch/20110406/ippTools/share/disttool_pending_chip_bg.sql

    r30387 r31253  
    66    stage_id,
    77    chipBackgroundImfile.class_id AS component,
     8    exp_type,
    89    distRun.clean,
    910    rawExp.camera,

  • branches/czw_branch/20110406/ippTools/share/disttool_pending_diff.sql

    r30387 r31253  
    66    stage_id,
    77    diffSkyfile.skycell_id AS component,
     8    exp_type,
    89    clean,
    910    rawExp.camera,

  • branches/czw_branch/20110406/ippTools/share/disttool_pending_fake.sql

    r30387 r31253  
    66    stage_id,
    77    fakeProcessedImfile.class_id AS component,
     8    exp_type,
    89    clean,
    910    rawExp.camera,

  • branches/czw_branch/20110406/ippTools/share/disttool_pending_raw.sql

    r30653 r31253  
     1SELECT * FROM (
     2-- rawExp magicked with re_place
    13SELECT
    24    distRun.dist_id,
    … …  
    68    rawExp.exp_id AS stage_id,
    79    rawImfile.class_id AS component,
     10    rawExp.exp_type,
    811    clean,
    912    rawExp.camera,
    … …  
    3841    AND distRun.stage = 'raw'
    3942    AND distComponent.dist_id IS NULL
    40     AND (rawExp.magicked OR distRun.no_magic)
     43    -- AND (rawExp.magicked OR distRun.no_magic)
     44    AND rawExp.magicked
    4145    -- need to have magicked the chip image which makes the camera mask
    4246    AND chipProcessedImfile.magicked != 0
    4347    AND camRun.magicked > 0
     48    AND (Label.active OR Label.active IS NULL)
     49UNION
     50SELECT
     51    -- raw images no_magic required
     52    distRun.dist_id,
     53    distRun.label,
     54    distTarget.dist_group,
     55    'raw' AS stage,
     56    rawExp.exp_id AS stage_id,
     57    rawImfile.class_id AS component,
     58    rawExp.exp_type,
     59    clean,
     60    rawExp.camera,
     61    CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
     62    -- XXX: replace this with rawImfile.path_base once it exists
     63    TRIM(TRAILING '.fits' FROM rawImfile.uri) AS path_base,
     64    CAST(NULL AS CHAR(255)) AS alt_path_base,
     65    -- pass camera stage path base since we want the camera mask file. The script knows what to do
     66    CAST(NULL AS CHAR(255)) AS chip_path_base,
     67    CAST(NULL AS CHAR(255)) AS state,
     68    CAST(NULL AS CHAR(255)) AS data_state,
     69    0 as quality,
     70    distRun.no_magic,
     71    rawImfile.magicked,
     72    IFNULL(Label.priority, 10000) AS priority
     73FROM distRun
     74JOIN distTarget USING(target_id, stage, clean)
     75JOIN rawExp ON rawExp.exp_id = stage_id
     76            AND distTarget.stage = 'raw' AND distRun.alternate = 0
     77JOIN rawImfile USING(exp_id)
     78LEFT JOIN distComponent
     79    ON distRun.dist_id = distComponent.dist_id
     80    AND rawImfile.class_id = distComponent.component
     81LEFT JOIN Label ON distRun.label = Label.label
     82WHERE
     83    distRun.state = 'new'
     84    AND distRun.clean = 0
     85    AND distRun.stage = 'raw'
     86    AND distComponent.dist_id IS NULL
     87    AND (distRun.no_magic)
    4488    AND (Label.active OR Label.active IS NULL)
    4589UNION
    … …  
    5296    rawExp.exp_id AS stage_id,
    5397    rawImfile.class_id AS component,
     98    rawExp.exp_type,
    5499    clean,
    55100    rawExp.camera,
    … …  
    96141    rawExp.exp_id AS stage_id,
    97142    'exposure' AS component,
     143    rawExp.exp_type,
    98144    clean,
    99145    rawExp.camera,
    … …  
    120166    AND distComponent.dist_id IS NULL
    121167    AND (Label.active OR Label.active IS NULL)
     168) AS distRun
     169WHERE 1

  • branches/czw_branch/20110406/ippTools/share/disttool_pending_sky.sql

    r30387 r31253  
    66    stage_id,
    77    stackRun.skycell_id AS component,
     8    exp_type,
    89    clean,
    910    rawExp.camera,

  • branches/czw_branch/20110406/ippTools/share/disttool_pending_stack.sql

    r30387 r31253  
    66    stage_id,
    77    stackRun.skycell_id AS component,
     8    exp_type,
    89    clean,
    910    rawExp.camera,

  • branches/czw_branch/20110406/ippTools/share/disttool_pending_warp.sql

    r30387 r31253  
    66    stage_id,
    77    warpSkyfile.skycell_id AS component,
     8    exp_type,
    89    clean,
    910    rawExp.camera,

  • branches/czw_branch/20110406/ippTools/share/disttool_pending_warp_bg.sql

    r30387 r31253  
    66    stage_id,
    77    warpBackgroundSkyfile.skycell_id AS component,
     8    exp_type,
    89    clean,
    910    rawExp.camera,

  • branches/czw_branch/20110406/ippTools/src/difftool.c

    r30907 r31253  
    29672967    pxAddLabelSearchArgs (config, where, "-data_group", "diffRun.data_group", "LIKE");
    29682968    pxAddLabelSearchArgs (config, where, "-dist_group", "diffRun.dist_group", "LIKE");
     2969
     2970    PXOPT_COPY_S64(config->args, where, "-template_exp_id", "rawTemplate.exp_id", "==");
    29692971   
    29702972    PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);

  • branches/czw_branch/20110406/ippTools/src/difftoolConfig.c

    r30907 r31253  
    182182    psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_end", 0,      "search for exposures by time (<=)", NULL);
    183183    psMetadataAddStr(listrunArgs, PS_LIST_TAIL,  "-filter", 0,           "search for filter", NULL);
     184    psMetadataAddS64(listrunArgs, PS_LIST_TAIL,  "-template_exp_id",  0, "search by exposure ID of template", 0);
    184185    psMetadataAddStr(listrunArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL);
    185186    psMetadataAddStr(listrunArgs,  PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by diffRun data_group (LIKE comparison)", NULL);

  • branches/czw_branch/20110406/ippTools/src/stacktool.c

    r30945 r31253  
    243243    PXOPT_COPY_STR(config->args,  where, "-select_exp_type",           "rawExp.exp_type", "==");
    244244    PXOPT_COPY_F32(config->args,  where, "-select_good_frac_min",      "warpSkyfile.good_frac", ">=");
    245     PXOPT_COPY_STR(config->args,  where, "-select_skycell_id",         "warpSkyfile.skycell_id", "==");
     245    PXOPT_COPY_STR(config->args,  where, "-select_skycell_id",         "warpSkyfile.skycell_id", "LIKE");
    246246    PXOPT_COPY_STR(config->args,  where, "-select_data_group",         "warpRun.data_group", "==");
    247247    pxAddLabelSearchArgs (config, where, "-select_label",              "warpRun.label", "LIKE"); // define using warp label

  • branches/czw_branch/20110406/ippTools/src/staticskytool.c

    r30907 r31253  
    123123    psAssert (filters->data.list->n, "-select_filter should at least have a place-holder");
    124124    int num_filter = filters->data.list->n;
    125     if (num_filter < 2) {
    126         psError(PXTOOLS_ERR_CONFIG, false, "invalid request: only 1 filter selected");
    127         psFree(whereMD);
    128         return false;
    129     }
    130125
    131126    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

  • branches/czw_branch/20110406/ippTools/src/staticskytoolConfig.c

    r28154 r31253  
    5252    psMetadataAddF32(definebyqueryArgs,  PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0);
    5353    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL);
    54     psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple required)", NULL);
     54    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiplei OK)", NULL);
    5555    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL);
    5656    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-set_label", 0, "define label", NULL);

  • branches/czw_branch/20110406/ippconfig/recipes/nightly_science.config

    r31110 r31253  
    203203  NAME      STR STS
    204204  DISTRIBUTION STR STS
    205   TESS      STR STS
     205  TESS      STR STS.V3
    206206  OBSMODE   STR STS%
    207207  OBJECT    STR STS%

  • branches/czw_branch/20110406/ippconfig/recipes/ppStack.config

    r31163 r31253  
    123123END
    124124
     125QUICKSTACK    METADATA
     126        CONVOLVE        BOOL   FALSE
     127        COMBINE.REJ     F32    3.0
     128        COMBINE.DISCARD F32    0.3
     129        MASK.VAL        STR    MASK.VALUE,CONV.BAD,GHOST,BURNTOOL # Mask value of input bad pixels
     130END
    125131
    126132#

  • branches/czw_branch/20110406/ippconfig/recipes/psphot.config

    r31163 r31253  
    203203
    204204# define the annuli (in pixels) for surface brightness measurements
    205 # @RADIAL.ANNULAR.BINS.LOWER           F32     0.0   5.0  10.0  20.0  40.0  80.0
    206 # @RADIAL.ANNULAR.BINS.UPPER           F32     5.0  10.0  20.0  40.0  80.0 160.0
     205# @RADIAL.ANNULAR.BINS.LOWER           F32     0.0   5.0  10.0  20.0  40.0  80.0 # comment due to ticket #1476
     206# @RADIAL.ANNULAR.BINS.UPPER           F32     5.0  10.0  20.0  40.0  80.0 160.0 #
    207207
    208208# SDSS values (in SDSS pixels)
    209 #@RADIAL.ANNULAR.BINS.LOWER           F32     0.00 0.56 1.69 2.59 4.41  7.51 11.58 18.58 28.55 45.50  70.51 110.53 172.49 269.52 420.51
    210 #@RADIAL.ANNULAR.BINS.UPPER           F32     0.56 1.69 2.59 4.41 7.51 11.58 18.58 28.55 45.50 70.51 110.53 172.49 269.52 420.51 652.50
     209#@RADIAL.ANNULAR.BINS.LOWER           F32     0.00 0.56 1.69 2.59 4.41  7.51 11.58 18.58 28.55 45.50  70.51 110.53 172.49 269.52 420.51 #
     210#@RADIAL.ANNULAR.BINS.UPPER           F32     0.56 1.69 2.59 4.41 7.51 11.58 18.58 28.55 45.50 70.51 110.53 172.49 269.52 420.51 652.50 #
    211211
    212212# PS1 values? (all SDSS @ PS1 0.2 arcsec pixel scales)
    213 # @RADIAL.ANNULAR.BINS.LOWER           F32      0.0 1.2 3.4 5.2  8.8 15.0 23.2 37.2 57.1  91.0 141.0 221.1 345.0 539.1  841.0
    214 # @RADIAL.ANNULAR.BINS.UPPER           F32      1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1 345.0 539.1 841.0 1305.0
     213# @RADIAL.ANNULAR.BINS.LOWER           F32      0.0 1.2 3.4 5.2  8.8 15.0 23.2 37.2 57.1  91.0 141.0 221.1 345.0 539.1  841.0 #
     214# @RADIAL.ANNULAR.BINS.UPPER           F32      1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1 345.0 539.1 841.0 1305.0 #
    215215
    216216# PS1 values?
    217 @RADIAL.ANNULAR.BINS.LOWER           F32      0.0 1.2 3.4 5.2  8.8 15.0 23.2 37.2 57.1  91.0 141.0
    218 @RADIAL.ANNULAR.BINS.UPPER           F32      1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1
     217@RADIAL.ANNULAR.BINS.LOWER           F32      0.0 1.2 3.4 5.2  8.8 15.0 23.2 37.2 57.1  91.0 141.0  # perl parser fails without a comment ticket #1476
     218@RADIAL.ANNULAR.BINS.UPPER           F32      1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1  # perl parser fails without a comment ticket #1476
    219219
    220220# Extended source fit parameters

  • branches/czw_branch/20110406/ippconfig/recipes/reductionClasses.mdc

    r31175 r31253  
    239239END
    240240
     241# quick stacks
     242QUICKSTACK            METADATA
     243      STACK_PPSTACK   STR      QUICKSTACK
     244      STACK_PPSUB     STR      STACK
     245      STACK_PSPHOT    STR      STACK
     246END
     247
    241248# basic science analysis
    242249PHOTFEST.SINGLE1        METADATA
    … …  
    632639END
    633640
     641STS_DATASET METADATA
     642   CHIP_PPIMAGE STR CHIP
     643   CHIP_PSPHOT STR CHIP
     644   WARP_PSWARP STR WARP
     645   STACK_PPSTACK   STR STACK
     646   STACK_PPSUB STR STACK
     647   STACK_PSPHOT    STR STACK
     648   DIFF_PPSUB  STR DIFF
     649   DIFF_PSPHOT STR DIFF
     650   JPEG_BIN1   STR PPIMAGE_J1
     651   JPEG_BIN2   STR PPIMAGE_J2
     652   FAKEPHOT    STR FAKEPHOT
     653   ADDSTAR     STR ADDSTAR
     654   PSASTRO    STR STS_DATASET
     655   STACKPHOT       STR     STACKPHOT
     656END
     657

  • branches/czw_branch/20110406/ppSub/src/ppSub.c

    r30874 r31253  
    9494
    9595        if (data) {
    96             psString dump_file = psMetadataLookupStr(NULL, data->config->arguments, "-dumpconfig");
     96            psString dump_file = psMetadataLookupStr(NULL, data->config->arguments, "DUMP_CONFIG");
    9797            if (dump_file) {
    9898                if (!pmConfigDump(data->config, dump_file)) {

  • branches/czw_branch/20110406/psModules/src/camera/pmFPAWrite.c

    r27992 r31253  
    690690    pmHDU *hdu = pmHDUFromCell(cell);   // HDU for cell
    691691    psMetadata *header = psMetadataCopy(NULL, hdu->header); // Header to write
     692
    692693    psMetadataAddS32(header, PS_LIST_TAIL, "COVARIANCE.CENTRE.X", PS_META_REPLACE,
    693694                     "Centre of covariance matrix in x", -xMinCovar);
    … …  
    697698    // Turn off compression
    698699    int bitpix = fits->options ? fits->options->bitpix : 0; // Desired bits per pixel
     700    psFitsScaling scaling = fits->options ? fits->options->scaling : 0; // Current scaling method.
    699701    psFitsCompression *compress = psFitsCompressionGet(fits); // Current compression options
     702   
     703/*     fprintf(stderr,"Attempting to write chip %s cell %s extension %s with scaling %d\n", */
     704/*          chipName,cellName,extname,fits->options->scaling); */
    700705    if (!psFitsSetCompression(fits, PS_FITS_COMPRESS_NONE, NULL, 0, 0, 0)) {
    701706        psError(PS_ERR_UNKNOWN, false, "Unable to set FITS compression to NONE");
    … …  
    708713    if (fits->options) {
    709714        fits->options->bitpix = 0;
     715    }
     716    if (fits->options) {
     717        fits->options->scaling = psFitsScalingFromString("STDEV_POSITIVE"); // This is a bit of a hack. We don't really have a default value.
    710718    }
    711719
    … …  
    732740        fits->options->bitpix = bitpix;
    733741    }
     742    if (fits->options) {
     743        fits->options->scaling = scaling;
     744    }
    734745    if (!psFitsCompressionApply(fits, compress)) {
    735746        psError(PS_ERR_UNKNOWN, false, "Unable to restore FITS compression");

  • branches/czw_branch/20110406/tools/mysql-dump

    • Property svn:ignore

      •  

        old new  
        11password.sh
         2*.log
         3.curr_gpc1_name

  • branches/czw_branch/20110406/tools/mysql-dump/gpc1_dump.sh

    r31210 r31253  
    1515. /home/panstarrs/ipp/mysql-dump/functions.sh
    1616. /home/panstarrs/ipp/mysql-dump/password.sh
     17
     18EMAILTO=schastel@ifa.hawaii.edu
    1719
    1820HOST=ippc02
    … …  
    4345if [ "$dump_status" -ne "0" ]; then
    4446        log ERROR "Sending warning e-mail"
    45         email "Gpc1 dump failed" schastel@ifa.hawaii.edu "Check log file: /export/$HOST.0/mysql-dumps/gpc1_dump.log"
     47        email "Gpc1 dump failed" $EMAILTO "Check log file: /export/$HOST.0/mysql-dumps/gpc1_dump.log"
    4648        log INFO "End of gpc1 dump (Error)"
    4749        exit 1

  • branches/czw_branch/20110406/tools/mysql-dump/gpc1_install.sh

    r31211 r31253  
    99#      iterates up to $MAX_ITERATIONS times.
    1010# 2) Check the gpc1 dump file against its checksum.
    11 # 3) Import gpc1 into gpc1_0 or gpc1_1 (ensured by gpc1Import.py utility)
    12 # 4) Distribute the validated files, that is:
     11# 3) Distribute the validated files, that is:
    1312#    - Hard-links the gzipped MySQL dump in the distribution directory
    1413#    - Moves the MD5 checksum to the distribution directory
     14# 4) Import gpc1 into gpc1_0 or gpc1_1 (ensured by gpc1Import.py utility)
    1515. /home/panstarrs/ipp/mysql-dump/functions.sh
    1616
    … …  
    3030
    3131ITERATION=1
    32 if [ ! -s $TARGET/$MD5FILE ]; then
    33         log WARNING "MD5 file not found [$TARGET/$MD5FILE]. Waiting $SLEEPING_TIME seconds (Attempt: $ITERATION out of $MAX_ITERATIONS)"
    34         sleep $SLEEPING_TIME
    35         let ITERATION=ITERATION+1
    36         if [ "$ITERATION" -ge "$MAX_ITERATIONS" ]; then
    37                 log ERROR "Copy of $TARGET/$MD5FILE failed after $ITERATION iterations. Giving up"
    38                 email "Too many failures when attempting to copy $TARGET/$MD5FILE" $EMAILTO "Check log"
    39                 exit 1
     32MD5_STATUS=1
     33while [ "$MD5_STATUS" -ne "0" ]; do
     34        if [ ! -s $TARGET/$MD5FILE ]; then
     35                log WARNING "MD5 file not found [$TARGET/$MD5FILE]. Waiting $SLEEPING_TIME seconds (Attempt: $ITERATION out of $MAX_ITERATIONS)"
     36                sleep $SLEEPING_TIME
     37                let ITERATION=ITERATION+1
     38                if [ "$ITERATION" -ge "$MAX_ITERATIONS" ]; then
     39                        log ERROR "Copy of $TARGET/$MD5FILE failed after $ITERATION iterations. Giving up"
     40                        email "Too many failures when attempting to copy $TARGET/$MD5FILE" $EMAILTO "Check log"
     41                        log INFO "End of gpc1_install.sh (Failure)"
     42                        exit 1
     43                fi
     44        else
     45                MD5_STATUS=0
    4046        fi
    41 fi
     47done
    4248
    4349log INFO "Checking gpc1 backup file checksum"
    … …  
    4551BACKUP_FILE=`cat $TARGET/$MD5FILE | sed 's/^.* \//\//'`
    4652BACKUP_FILE=`basename $BACKUP_FILE`
    47 
     53log INFO "Backup file is [$TARGET/$BACKUP_FILE]"
    4854ACTUAL_MD5SUM=`/usr/bin/md5sum $TARGET/$BACKUP_FILE | sed 's/ .*$//'`
    4955if [ "$ACTUAL_MD5SUM" != "$EXPECTED_MD5SUM" ]; then
    5056        log ERROR "Checksums are different: actual=[$ACTUAL_MD5SUM], expected=[$EXPECTED_MD5SUM]"
    5157        email "Invalid checksum for gpc1 backup" $EMAILTO "Checksums are different: actual=[$ACTUAL_MD5SUM], expected=[$EXPECTED_MD5SUM]"
     58        log INFO "End of gpc1_install.sh (Failure)"
    5259        exit 2
    5360fi
    54 
    55 log INFO "Importing gpc1 backup file to gpc1_0 or gpc1_1"
    56 /usr/bin/python /home/panstarrs/ipp/mysql-dump/gpc1Import.py
    5761
    5862log INFO "Symlinking backup file [$TARGET/$BACKUP_FILE] to distribution [$DISTRIBUTION_TARGET]"
    … …  
    6064/usr/bin/mv -f $TARGET/$MD5FILE $DISTRIBUTION_MD5
    6165
     66# if it is between 0 and 4, ingest into gpc1_0 or gpc1_1
     67HOUR=`date +"%H"`
     68if [ "$HOUR" -lt "4" ]; then
     69        log INFO "Importing gpc1 backup file to gpc1_0 or gpc1_1"
     70        /usr/bin/python /home/panstarrs/ipp/mysql-dump/gpc1Import.py
     71else
     72        log DEBUG "No gpc1 backup importation"
     73fi
     74
     75log INFO "End of gpc1_install.sh (Success)"
     76

  • branches/czw_branch/20110406/tools/mysql-dump/neb_dump.sh

    r31213 r31253  
    1212. /home/panstarrs/ipp/mysql-dump/password.sh
    1313
     14EMAILTO=schastel@ifa.hawaii.edu
    1415HOST=ippdb02
    1516
    … …  
    3637if [ "$dump_status" -ne "0" ]; then
    3738        log ERROR "Sending warning e-mail"
    38         email "Nebulous dump failed" schastel@ifa.hawaii.edu "Check log file: /export/ippdb02.0/mysql-dumps/neb-dump.log"
     39        email "Nebulous dump failed" $EMAILTO "Check log file: /export/ippdb02.0/mysql-dumps/neb-dump.log"
    3940        log INFO "End of nebulous dump (Error)"
    4041        exit 1

  • branches/czw_branch/20110406/tools/mysql-dump/ops_dump.csh

    r31209 r31253  
    22
    33echo "## `/bin/date +%FT%T`"
     4
     5set EMAILTO = schastel@ifa.hawaii.edu
    46
    57# dump ippRequestServer from ippc19
    … …  
    79mysqldump --compress -hippc19 -uipp -pipp --flush-logs --single-transaction --log-error=RS-err ippRequestServer | gzip -9 > $DUMP_PATH/mysql-RS-ippc19-`date +%FT%T`.dump.gz
    810if ($status != 0) then
    9    /bin/mail -s "Warning RS dump failed" schastel@ifa.hawaii.edu < msg
     11   /bin/mail -s "Warning ippRequestServer/RS dump failed" $EMAILTO < msg
    1012endif
    1113
    … …  
    1618mysqldump --compress -hippdb01 -uipp -pipp --flush-logs --single-transaction ippadmin | gzip -9 > $FILENAME
    1719if ($status != 0) then
    18    /bin/mail -s "Warning ippadmin dump failed" schastel@ifa.hawaii.edu < msg 
     20   /bin/mail -s "Warning ippadmin dump failed" $EMAILTO < msg 
    1921endif
    2022
    … …  
    3133mysqldump --compress -hippdb01 -uipp -pipp --flush-logs --single-transaction isp | gzip -9 > $FILENAME
    3234if ($status != 0) then
    33    /bin/mail -s "Warning isp dump failed" schastel@ifa.hawaii.edu < msg 
     35   /bin/mail -s "Warning isp dump failed" $EMAILTO < msg 
    3436endif
    3537
    … …  
    3941ln -f  ../mysql-isp-ippdb01-$DATEVAR.dump.gz ippdb01-isp.dump.gz
    4042
    41 # dump gpc1 from ippdb01
    42 #set DUMP_PATH = /data/ipp001.0/ipp/mysql-dumps               #path(SAB)
    43 #set DATEVAR=`date +%FT%T`                                    #date
    44 #set FILENAME = $DUMP_PATH/mysql-gpc1-ippc02-$DATEVAR.dump.gz #dump file name
    45 #mysqldump --compress -hippc -P3302 -uipp -pipp --flush-logs --single-transaction gpc1 | gzip -9 > $FILENAME
    46 #if ($status != 0) then
    47 #   /bin/mail -s "Warning gpc1 dump failed" schastel@ifa.hawaii.edu < msg 
    48 #endif
    49 
    50 # simlink to gpc1 ippdb01 dump
    51 #set DUMP_PATH = /data/ipp001.0/ipp/mysql-dumps       #path(SAB)
    52 #cd $DUMP_PATH/distribution/                          #link dir
    53 #ln -f  ../mysql-gpc1-ippc02-$DATEVAR.dump.gz ippdb01-gpc1.dump.gz   #hard link
    54 
    55 
    56 # dump the ippdb02 database
    57 #set DUMP_PATH = /data/ipp001.0/ipp/mysql-dumps             #path
    58 #set DATEVAR = `date +%FT%T`                                #date       
    59 #set FILENAME = $DUMP_PATH/mysql-neb-ippdb02-$DATEVAR.dump.gz   #dump file name
    60 #mysqldump --compress -hippdb02 -ujhipp -pjhipp --flush-logs --single-transaction nebulous | gzip -9 > $FILENAME
    61 #if ($status != 0) then
    62 #   /bin/mail -s "Warning nebulous dump failed" schastel@ifa.hawaii.edu < msg 
    63 #endif
    64 
    65 # simlink to latest ippdb02 dump
    66 #set DUMP_PATH = /data/ipp001.0/ipp/mysql-dumps            #path (SAB)
    67 #cd $DUMP_PATH/distribution/                                #link dir
    68 #ln -f  ../mysql-neb-ippdb02-$DATEVAR.dump.gz ippdb02.dump.gz   #hard link
    69 
    70 # dump the ippdb00 database
    71 #set DUMP_PATH = /data/ipp001.0/ipp/mysql-dumps             #path
    72 #set DATEVAR = `date +%FT%T`                                #date       
    73 #set FILENAME = $DUMP_PATH/mysql-neb-ippdb00-$DATEVAR.dump.gz   #dump file name
    74 #mysqldump --compress -hippdb00 -ujhipp -pjhipp --flush-logs --single-transaction nebulous | gzip -9 > $FILENAME
    75 #if ($status != 0) then
    76 #   /bin/mail -s "Warning nebulous dump failed" schastel@ifa.hawaii.edu < msg 
    77 #endif
    78 

  • branches/czw_branch/20110406/tools/runcameraexp.pl

    r29419 r31253  
    1616my $dbname = "gpc1";
    1717my ($cam_id, $update, $redirect, $save_temps, $pretend, $no_verbose);
     18my ($outdir);
    1819
    1920GetOptions(
    … …  
    2223    'redirect-output'   => \$redirect,
    2324    'save-temps'        => \$save_temps,
     25    'outdir=s'          => \$outdir,
    2426    'update'            => \$update,
    2527    'dbname=s'          => \$dbname,
    … …  
    5557
    5658die "Cannot update when run is not faulted\n" if $update and ! $fault;
     59
     60if ($outdir) {
     61    my $base = basename($path_base);
     62    $path_base = "$outdir/$base";
     63    print STDERR "changing path_base to $path_base";
     64    $update = 0;
     65}
    5766
    5867my  $run_state;
Note: See TracChangeset for help on using the changeset viewer.