IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29495


Ignore:
Timestamp:
Oct 20, 2010, 11:53:36 AM (16 years ago)
Author:
bills
Message:

Add path_base to magicMask. Convert destreaking to get streaks file from there.
Also deleted magicInputSkyfile.diff_id from pxadmin_create_tables.sql. The column has been
gone for ages in gpc1 and the rest of the code but new databases were getting it set

Location:
trunk
Files:
12 edited

Legend:

Unmodified
Added
Removed

  • trunk/dbconfig/changes.txt

    r29493 r29495  
    19461946ALTER TABLE stackSummary DROP PRIMARY KEY, ADD PRIMARY KEY (sass_id, projection_cell);
    19471947
     1948ALTER TABLE stackSumSkyfile ADD COLUMN mjd_obs double AFTER good_frac;
     1949
    19481950-- changes to support cleanup of the background preserved images
    19491951ALTER TABLE chipBackgroundImfile ADD COLUMN data_state VARCHAR(64) AFTER path_base;
    … …  
    19521954ALTER TABLE warpBackgroundSkyfile  ADD KEY(data_state);
    19531955
    1954 ALTER TABLE stackSumSkyfile ADD COLUMN mjd_obs double AFTER good_frac;
     1956ALTER TABLE magicMask ADD COLUMN path_base VARCHAR(255) AFTER uri;
     1957
     1958-- diff_id was removed from magicInputSkyfile ages ago, but
     1959-- pxadmin_create_tables.sql still had it this command didn't work due to
     1960-- a mysql error.
     1961-- ALTER TABLE magicInputSkyfile DROP COLUMN diff_id;

  • trunk/dbconfig/magic.md

    r27450 r29495  
    4242    magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
    4343    uri         STR         255
     44    path_base   STR         255
    4445    streaks     S32         0
    4546    fault       S16         0       # Key

  • trunk/ippScripts/scripts/magic_destreak.pl

    r28931 r29495  
    4242# Parse the command-line arguments
    4343my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base, $cam_reduction);
     44my ($streaks_path_base, $inv_streaks_path_base);
    4445my ($outroot, $recoveryroot, $magicked);
    4546my ($replace, $release);
    … …  
    4950           'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
    5051           'camera=s'       => \$camera,     # camera for evaluating file rules
     52           'streaks_path_base=s'      => \$streaks_path_base,    # path_base for streaks data
     53           'inv_streaks_path_base=s'  => \$inv_streaks_path_base, #path_base for streaks from inverse diff
     54           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
    5155           'streaks=s'      => \$streaks,    # file containing the list of streaks
    5256           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
    … …  
    7882    defined $camera and
    7983    defined $streaks and
     84    defined $streaks_path_base and
    8085    defined $stage and
    8186    defined $stage_id and
    … …  
    108113    &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
    109114}
     115$inv_streaks_path_base = undef if defined($inv_streaks_path_base) and ($inv_streaks_path_base eq "NULL");
    110116$inv_streaks = undef if defined($inv_streaks) and ($inv_streaks eq "NULL");
    111117
    … …  
    240246    my ($allstreaks_fh, $allstreaks_name);
    241247
     248    # Set name of streaks files from their path_base. Note. ne 'NULL' test is for backward compatability
     249    # with runs that don't have path_base set yet
     250    if ($streaks_path_base ne 'NULL') {
     251        $streaks = "$streaks_path_base.streaks";
     252    }
     253    if ($inv_streaks_path_base and ($inv_streaks_path_base ne 'NULL')) {
     254        $inv_streaks = "$inv_streaks_path_base.streaks";
     255    }
     256
     257    my $streaks_resolved = $ipprc->file_resolve($streaks) or &my_die("failed to resolve streaks file $streaks", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR);
     258    my $inv_streaks_resolved;
     259    if ($inv_streaks) {
     260        $inv_streaks_resolved = $ipprc->file_resolve($inv_streaks) or &my_die("failed to resolve inverse streaks file $inv_streaks", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR);
     261    }
     262       
     263
    242264    if ($stage eq "raw") {
    243265        $image = $uri;
    … …  
    301323        $sources    = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $path_base);
    302324
    303         if ($inv_streaks) {
     325        if ($inv_streaks_resolved) {
    304326            # create a temporary file containing the contents of the
    305327            # two streaks files
    … …  
    307329                    UNLINK => !$save_temps);
    308330
    309             combine_streaks($allstreaks_fh, $streaks, $inv_streaks);
     331            combine_streaks($allstreaks_fh, $streaks_resolved, $inv_streaks_resolved);
    310332
    311333            # apply the combined streaks to both the forward and inverse diffs
    312             $streaks = $allstreaks_name;
     334            $streaks_resolved = $allstreaks_name;
    313335        }
    314336    }
    315337
    316338    {
    317         my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
     339        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks_resolved -image $image";
    318340
    319341        $command .= " -stats $statsFile";
    … …  
    340362        }
    341363    }
    342     if (($stage eq "diff") and $inv_streaks) {
     364    if (($stage eq "diff") and $inv_streaks_resolved) {
    343365        $image   = $ipprc->filename("PPSUB.INVERSE", $path_base);
    344366        $mask    = $ipprc->filename("PPSUB.INVERSE.MASK", $path_base);
    … …  
    349371        my $invStatsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.inv.stats";
    350372
    351         my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
     373        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks_resolved -image $image";
    352374        $command .= " -stats $invStatsFile";
    353375

  • trunk/ippScripts/scripts/magic_process.pl

    r29493 r29495  
    325325}
    326326
     327### Input result into database
     328{
     329    my $command = "$magictool -addresult";
     330    $command   .= " -magic_id $magic_id";
     331    $command   .= " -node $node";
     332    $command   .= " -path_base $outroot";
     333    $command   .= " -dbname $dbname" if defined $dbname;
     334
     335    # Add the processed file to the database
     336    unless ($no_update) {
     337        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     338            run(command => $command, verbose => $verbose);
     339        unless ($success) {
     340            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     341            &my_die("Unable to perform magictool -addresult: $error_code", $magic_id, $node, $error_code);
     342        }
     343    } else {
     344        print "Skipping command: $command\n";
     345    }
     346}
    327347
    328348if ($node eq "root") {
     349    # XXXX: Since we just added the result above, all of these my_dies are going to fail
     350    # with a duplicate row error.
     351    # see more comments below
    329352    my $streaks_file = "$outroot.streaks";
    330353    my $resolved = $ipprc->file_resolve($streaks_file);
    … …  
    389412    }
    390413
    391 ### Input result into database
    392 # XXXX: if this succeeds but addmask fails the magicRun is in a goofy state XXXX
    393 {
    394     my $command = "$magictool -addresult";
    395     $command   .= " -magic_id $magic_id";
    396     $command   .= " -node $node";
    397     $command   .= " -path_base $outroot";
    398     $command   .= " -dbname $dbname" if defined $dbname;
    399 
    400     # Add the processed file to the database
    401     unless ($no_update) {
    402         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    403             run(command => $command, verbose => $verbose);
    404         unless ($success) {
    405             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    406             &my_die("Unable to perform magictool -addresult: $error_code", $magic_id, $node, $error_code);
    407         }
    408     } else {
    409         print "Skipping command: $command\n";
    410     }
    411 }
    412 
    413 
    414414    {
    415415        my $command = "$magictool -addmask";
    … …  
    425425            unless ($success) {
    426426                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    427                 # This isn't going to work because our result was already entered.
     427                # XXX: This my_die isn't going to work because the result for the root node was already
     428                # added to the database up above.
     429                # There is a magicMask has a fault column. Maybe we should
     430                # use that and add a new revert mode that deletes the magicMask and the magicNodeResult
     431                # for the root node.
    428432                &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code);
    429433            }
    … …  
    433437    }
    434438}
     439
    435440
    436441### Pau.

  • trunk/ippTools/share/magicdstool_todestreak_camera.sql

    r29358 r29495  
    77    camera,
    88    magicMask.uri AS streaks_uri,
     9    magicMask.path_base AS streaks_path_base,
    910    CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
     11    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
    1012    stage,
    1113    stage_id,

  • trunk/ippTools/share/magicdstool_todestreak_chip.sql

    r29358 r29495  
    77    camera,
    88    magicMask.uri AS streaks_uri,
     9    magicMask.path_base AS streaks_path_base,
    910    CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
     11    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
    1012    stage,
    1113    stage_id,

  • trunk/ippTools/share/magicdstool_todestreak_diff.sql

    r29358 r29495  
    77    rawExp.camera,
    88    magicMask.uri AS streaks_uri,
     9    magicMask.path_base AS streaks_path_base,
    910    CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
     11    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
    1012    stage,
    1113    magicRun.diff_id AS stage_id,
    … …  
    5254    rawExp.camera,
    5355    magicMask.uri AS streaks_uri,
     56    magicMask.path_base AS streaks_path_base,
    5457    (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri,
     58    (SELECT path_base from magicMask where magic_id = inv_magic_id) AS inv_streaks_path_base,
    5559    stage,
    5660    magicRun.diff_id AS stage_id,

  • trunk/ippTools/share/magicdstool_todestreak_raw.sql

    r29358 r29495  
    66    rawExp.camera,
    77    magicMask.uri as streaks_uri,
     8    magicMask.path_base as streaks_path_base,
    89    CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
     10    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
    911    stage,
    1012    stage_id,

  • trunk/ippTools/share/magicdstool_todestreak_warp.sql

    r29358 r29495  
    66    camera,
    77    magicMask.uri as streaks_uri,
     8    magicMask.path_base as streaks_path_base,
    89    CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
     10    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
    911    stage,
    1012    stage_id,

  • trunk/ippTools/share/pxadmin_create_tables.sql

    r29328 r29495  
    12211221CREATE TABLE magicInputSkyfile (
    12221222        magic_id BIGINT,
    1223         diff_id BIGINT,
    12241223        node VARCHAR(64),
    1225         PRIMARY KEY(magic_id, diff_id, node),
    1226         FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id),
    1227         FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id)
     1224        PRIMARY KEY(magic_id, node),
     1225        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id)
    12281226) ENGINE=innodb DEFAULT CHARSET=latin1;
    12291227
    … …  
    12531251        magic_id BIGINT,
    12541252        uri VARCHAR(255),
     1253        path_base VARCHAR(255),
    12551254        streaks INT,
    12561255        fault SMALLINT,

  • trunk/ippTools/src/magictool.c

    r29493 r29495  
    11821182
    11831183    // optional
    1184     PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
     1184//    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
     1185    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
    11851186    PXOPT_LOOKUP_S32(streaks, config->args, "-streaks", false, false);
    11861187
    … …  
    11951196    if (!magicMaskInsert(config->dbh,
    11961197                         magic_id,
    1197                          uri,
     1198                         NULL,
     1199                         path_base,
    11981200                         streaks,
    11991201                         fault

  • trunk/ippTools/src/magictoolConfig.c

    r29493 r29495  
    152152    psMetadata *addmaskArgs = psMetadataAlloc();
    153153    psMetadataAddS64(addmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
    154     psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL);
     154    psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL);
    155155    psMetadataAddS32(addmaskArgs, PS_LIST_TAIL, "-streaks", 0, "define number of streaks", 0);
    156156    psMetadataAddS16(addmaskArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
Note: See TracChangeset for help on using the changeset viewer.