IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29493


Ignore:
Timestamp:
Oct 20, 2010, 9:39:26 AM (16 years ago)
Author:
bills
Message:

changes to implment saving magic streaks file and verify outputs in nebulous and to add data_state to Background tables to support cleanup

Location:
trunk
Files:
9 edited

Legend:

Unmodified
Added
Removed

  • trunk/dbconfig/background.md

    r28486 r29493  
    2121    class_id            STR     64
    2222    path_base           STR     255
     23    data_state          STR     64
    2324    magicked            S64     0
    2425    dtime_script        F32     0.0
    … …  
    5758    skycell_id          STR     64
    5859    path_base           STR     255
     60    data_state          STR     64
    5961    magicked            S64     0
    6062    dtime_script        F32     0.0

  • trunk/dbconfig/changes.txt

    r28873 r29493  
    19451945ALTER TABLE diffSummary DROP PRIMARY KEY, ADD PRIMARY KEY (diff_id, projection_cell);
    19461946ALTER TABLE stackSummary DROP PRIMARY KEY, ADD PRIMARY KEY (sass_id, projection_cell);
     1947
     1948-- changes to support cleanup of the background preserved images
     1949ALTER TABLE chipBackgroundImfile ADD COLUMN data_state VARCHAR(64) AFTER path_base;
     1950ALTER TABLE chipBackgroundImfile ADD KEY(data_state);
     1951ALTER TABLE warpBackgroundSkyfile ADD COLUMN data_state VARCHAR(64) AFTER path_base;
     1952ALTER TABLE warpBackgroundSkyfile  ADD KEY(data_state);
     1953
     1954ALTER TABLE stackSumSkyfile ADD COLUMN mjd_obs double AFTER good_frac;

  • trunk/ippScripts/scripts/magic_process.pl

    r28137 r29493  
    1717use IPC::Cmd 0.36 qw( can_run run );
    1818use File::Temp qw( tempfile );
     19use File::Copy;
    1920use PS::IPP::Metadata::Config;
    2021use PS::IPP::Metadata::List qw( parse_md_list );
    … …  
    4142
    4243# Parse the command-line arguments
    43 my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
     44my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile, $final_outroot);
    4445
    4546GetOptions(
    … …  
    4950           'dbname=s'        => \$dbname,     # Database name
    5051           'baseroot=s'      => \$baseroot,   # Output root name
     52           'final-outroot=s' => \$final_outroot,   # location for final outputs
    5153           'save-temps'      => \$save_temps, # Save temporary files?
    5254           'verbose'         => \$verbose,    # Print stuff?
    … …  
    8688
    8789my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
     90
     91# list of VerifyStreaks input and output files to copy to nebulous
     92my @verify_outputs = qw(
     93clusterPos.txt
     94duplicate.png
     95mask.png
     96original.png
     97original.fits
     98residual.png
     99residual.fits
     100clusters.list
     101);
    88102
    89103### Get a list of inputs
    … …  
    312326
    313327
    314 ### Input result into database
    315 {
    316     my $command = "$magictool -addresult";
    317     $command   .= " -magic_id $magic_id";
    318     $command   .= " -node $node";
    319     $command   .= " -path_base $outroot";
    320     $command   .= " -dbname $dbname" if defined $dbname;
    321 
    322     # Add the processed file to the database
    323     unless ($no_update) {
    324         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    325             run(command => $command, verbose => $verbose);
    326         unless ($success) {
    327             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    328             &my_die("Unable to perform magictool -addresult: $error_code", $magic_id, $node, $error_code);
    329         }
    330     } else {
    331         print "Skipping command: $command\n";
    332     }
    333 }
    334 
    335328if ($node eq "root") {
    336329    my $streaks_file = "$outroot.streaks";
    … …  
    349342    }
    350343
    351     &run_verifystreaks($baseroot);
    352344
    353345    my $exp_id;                 # Exposure identifier
    … …  
    366358    }
    367359
     360    &run_verifystreaks($baseroot, $exp_id);
     361
    368362    {
    369363        my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path); # Astrometry file
    … …  
    383377    }
    384378
     379    my $output_streaks = $final_outroot . ".streaks";
     380    if ($output_streaks and ($output_streaks ne $streaks_file)) {
     381        copy_to_nebulous($ipprc, $streaks_file, $output_streaks, 1);
     382        foreach my $f (@verify_outputs) {
     383            my $src = "$baseroot.verify/${exp_id}_$f";
     384            my $dest = "$final_outroot.${f}";
     385            copy_to_nebulous($ipprc, $src, $dest, 1);
     386        }
     387    } else {
     388        $output_streaks = $streaks_file;
     389    }
     390
     391### Input result into database
     392# XXXX: if this succeeds but addmask fails the magicRun is in a goofy state XXXX
     393{
     394    my $command = "$magictool -addresult";
     395    $command   .= " -magic_id $magic_id";
     396    $command   .= " -node $node";
     397    $command   .= " -path_base $outroot";
     398    $command   .= " -dbname $dbname" if defined $dbname;
     399
     400    # Add the processed file to the database
     401    unless ($no_update) {
     402        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     403            run(command => $command, verbose => $verbose);
     404        unless ($success) {
     405            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     406            &my_die("Unable to perform magictool -addresult: $error_code", $magic_id, $node, $error_code);
     407        }
     408    } else {
     409        print "Skipping command: $command\n";
     410    }
     411}
     412
    385413
    386414    {
    387415        my $command = "$magictool -addmask";
    388416        $command   .= " -magic_id $magic_id";
    389         $command   .= " -uri $streaks_file";
     417        $command   .= " -uri $final_outroot";
    390418        $command   .= " -streaks $num_streaks";
    391419        $command   .= " -dbname $dbname" if defined $dbname;
    … …  
    411439
    412440    my $baseroot = shift;
     441    my $exp_id = shift;
    413442
    414443    unless ($VerifyStreaks) {
    … …  
    431460    my @files = <$baseroot.*.clusters>;
    432461
    433     unless (open ($FILE, ">$outdir/clusters.list")) {
    434         print "failed to create cluster file $outdir/clusters.list\n";
     462    my $clusters_list = "$outdir/${exp_id}_clusters.list";
     463    unless (open ($FILE, ">$clusters_list")) {
     464        print "failed to create cluster file $clusters_list\n";
    435465        return 1;
    436466    }
    … …  
    441471    close ($FILE);
    442472    if ($status) {
    443         print "failed to create cluster file $outdir/clusters.list\n";
     473        print "failed to create cluster file $clusters_list\n";
    444474        return 1;
    445475    }
    446476
    447     my $command = "$VerifyStreaks --out $outdir --clusters $outdir/clusters.list $baseroot.root.streakMap";
     477    my $command = "$VerifyStreaks --out $outdir --clusters $clusters_list $baseroot.root.streakMap";
    448478
    449479    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    513543}
    514544
     545# Copy a file to nebulous and optionally replicate it
     546# We should consider making this an ipprc function. For now try it here so we can print
     547# the right error messages
     548sub copy_to_nebulous {
     549    my $ipprc = shift;
     550    my $src = shift;
     551    my $dest = shift;
     552    my $replicate = shift;
     553
     554    print "copying $src to $dest\n";
     555
     556    $ipprc->file_exists($src) or
     557        &my_die("expected output file does not exist: $src", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
     558
     559    # copy the file to it's final destination - which is presumably in nebulous
     560    # we delete it so that all instances are deleted in case it has been replicated
     561    if ($ipprc->file_exists($dest)) {
     562        $ipprc->file_delete($dest) or
     563                &my_die("failed to delete existing file: $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
     564    }
     565    my $dest_resolved = $ipprc->file_resolve($dest, 'create');
     566    &my_die("failed to resolve $dest", $PS_EXIT_UNKNOWN_ERROR) if !$dest_resolved;
     567
     568    copy ($src, $dest_resolved) or
     569        &my_die("failed to copy $src to $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
     570
     571    if ($replicate and (file_scheme($dest) eq 'neb')) {
     572        my $neb = $ipprc->nebulous();
     573        $neb->setxattr($dest, 'user.copies', 2, 'create') or
     574            &my_die("failed to set user.copies for $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
     575
     576        $neb->replicate($dest) or
     577            &my_die("failed to replicate $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
     578    }
     579}
     580
    515581sub my_die
    516582{

  • trunk/ippTools/share/magictool_toprocess_inputs.sql

    r28768 r29493  
    33    magicRun.workdir,
    44    rawExp.exp_id,
     5    rawExp.exp_name,
     6    rawExp.workdir AS raw_workdir,
    57    rawExp.camera,
    68    -- convert magic_id into a boolean value (1 or 0)

  • trunk/ippTools/share/magictool_toprocess_tree.sql

    r28768 r29493  
    33    magicRun.workdir,
    44    rawExp.exp_id,
     5    rawExp.exp_name,
     6    rawExp.workdir AS raw_workdir,
    57    rawExp.camera,
    68    -- convert magic_id into a boolean value (1 or 0)
    … …  
    1618WHERE
    1719    magicRun.state = 'new'
     20    AND (Label.active OR Label.active IS NULL)
    1821-- WHERE hook %s
    1922ORDER BY

  • trunk/ippTools/share/magictool_totree.sql

    r28768 r29493  
    1818    AND magicTree.node IS NULL
    1919    AND magicRun.fault = 0
     20    AND (Label.active OR Label.active IS NULL)

  • trunk/ippTools/src/bgtool.c

    r28941 r29493  
    658658    }
    659659
    660     if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, magicked, dtime_script,
     660    if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, "full", magicked, dtime_script,
    661661                                    hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
    662662                                    maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {
    … …  
    14971497    }
    14981498
    1499     if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, magicked, dtime_script,
     1499    if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, "full", magicked, dtime_script,
    15001500                                    hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
    15011501                                    maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {

  • trunk/ippTools/src/magictool.c

    r28768 r29493  
    5050static bool exposureMode(pxConfig *config);
    5151
    52 static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state);
     52static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString);
    5353static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename);
    5454
    … …  
    131131
    132132    psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else
    133     PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "==");
     133    PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "rawExp.exp_id", "==");
    134134    PXOPT_COPY_STR(config->args, queryWhere, "-select_filter", "rawExp.filter", "==");
    135135
    … …  
    394394    // required
    395395    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
    396     PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
     396    PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false);
     397    PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false);
     398    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
     399    PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false);
     400
     401    psString setString = NULL;
     402    if (fault || clearfault) {
     403        psStringAppend(&setString, ", fault = %d", fault);
     404    }
     405    if (note) {
     406        psStringAppend(&setString, ", note = '%s'", note);
     407    }
    397408
    398409    if (state) {
    399410        // set detRun.state to state
    400         return setmagicRunState(config, magic_id, state);
     411        return setmagicRunState(config, magic_id, state, setString);
    401412    }
    402413
    … …  
    612623    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
    613624
    614     psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0");
     625    psString query = psStringCopy("UPDATE magicRun SET fault = 0 WHERE state = 'new' AND fault != 0");
    615626
    616627    if (psListLength(where->list)) {
    … …  
    13641375}
    13651376
    1366 static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state)
     1377static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString)
    13671378{
    13681379    PS_ASSERT_PTR_NON_NULL(state, false);
    … …  
    13801391        return false;
    13811392    }
    1382 
    1383     char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64;
    1384     if (!p_psDBRunQueryF(config->dbh, query, state, magic_id)) {
     1393    psString query = NULL;
     1394    psStringAppend(&query, "UPDATE magicRun SET state = '%s'", state);
     1395    if (setString) {
     1396        psStringAppend(&query, setString);
     1397    }
     1398    psStringAppend(&query, " WHERE magic_id = %" PRId64, magic_id);;
     1399
     1400//    char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64;
     1401    if (!p_psDBRunQuery(config->dbh, query)) {
    13851402        psError(PS_ERR_UNKNOWN, false,
    13861403                "failed to change state for magic_id %" PRId64, magic_id);

  • trunk/ippTools/src/magictoolConfig.c

    r28039 r29493  
    7979    psMetadata *updaterunArgs = psMetadataAlloc();
    8080    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
    81     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL);
     81    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL);
     82    psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0);
     83    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note",  0, "define note", NULL);
     84    psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-clearfault",  0, "set fault to zero", NULL);
    8285
    8386    // -addinputskyfile
Note: See TracChangeset for help on using the changeset viewer.