IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 30446


Ignore:
Timestamp:
Jan 31, 2011, 2:51:48 PM (15 years ago)
Author:
eugene
Message:

merging changes from trunk

Location:
branches/eam_branches/ipp-20101205
Files:
82 edited
37 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20101205/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

    r29424 r30446  
    1717                    resolve_project
    1818                    getCamRunByCamID
     19                    runToolAndParse
    1920                    parse_md_fast
    2021                    );

  • branches/eam_branches/ipp-20101205/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

    r28048 r30446  
    2929                    $PSTAMP_SELECT_INVERSE
    3030                    $PSTAMP_SELECT_UNCONV
     31                    $PSTAMP_REQUEST_UNCENSORED
     32                    $PSTAMP_REQUIRE_UNCENSORED
    3133                    $PSTAMP_USE_IMFILE_ID
    3234                    $PSTAMP_NO_WAIT_FOR_UPDATE
    … …  
    4446                    $PSTAMP_NO_JOBS_QUEUED
    4547                    $PSTAMP_NO_OVERLAP
     48                    $PSTAMP_NOT_AUTHORIZED
    4649                    );
    4750our %EXPORT_TAGS = (standard => [@EXPORT_OK]);
    … …  
    6467our $PSTAMP_USE_IMFILE_ID    = 16384;
    6568
    66 our $PSTAMP_NO_WAIT_FOR_UPDATE  = 32768;
     69our $PSTAMP_NO_WAIT_FOR_UPDATE = 32768;
     70our $PSTAMP_REQUEST_UNCENSORED = 1 << 16;
     71our $PSTAMP_REQUIRE_UNCENSORED = 1 << 17;
    6772
    6873# job and result codes
    … …  
    8287our $PSTAMP_NO_JOBS_QUEUED   = 27;
    8388our $PSTAMP_NO_OVERLAP       = 28;
     89our $PSTAMP_NOT_AUTHORIZED   = 29;
    8490
    8591
    … …  
    119125PSTAMP_NO_JOBS_QUEUED
    120126PSTAMP_NO_OVERLAP
     127PSTAMP_NOT_AUTHORIZED
    121128);
    122129

  • branches/eam_branches/ipp-20101205/ippMonitor/Makefile.in

    r29651 r30446  
    5555$(DESTWWW)/czartool_labels.php \
    5656$(DESTWWW)/czartool_getplot.php \
     57$(DESTWWW)/czartool_exposures.php \
    5758$(DESTWWW)/histogram.php \
    5859$(DESTWWW)/show_and_delete_image.php \

  • branches/eam_branches/ipp-20101205/ippMonitor/def/mopsStatus.d

    r29483 r30446  
    1 TABLE rawExp LEFT JOIN chipRun using(exp_id) LEFT JOIN camRun using(chip_id) LEFT JOIN fakeRun using(cam_id) LEFT JOIN warpRun using(fake_id) LEFT JOIN diffInputSkyfile AS diffInput ON diffInput.warp1 = warp_id LEFT JOIN diffRun AS diffInputRun ON diffInput.diff_id = diffInputRun.diff_id LEFT JOIN diffInputSkyfile AS diffTemplate ON diffTemplate.warp2 = warp_id LEFT JOIN diffRun AS diffTemplateRun ON diffTemplate.diff_id = diffTemplateRun.diff_id LEFT JOIN publishRun ON stage_id=diffInput.diff_id
     1TABLE rawExp LEFT JOIN chipRun using(exp_id) LEFT JOIN camRun using(chip_id) LEFT JOIN fakeRun using(cam_id) LEFT JOIN warpRun using(fake_id) LEFT JOIN diffInputSkyfile AS diffInput ON diffInput.warp1 = warp_id LEFT JOIN diffRun AS diffInputRun ON diffInput.diff_id = diffInputRun.diff_id LEFT JOIN diffInputSkyfile AS diffTemplate ON diffTemplate.warp2 = warp_id LEFT JOIN diffRun AS diffTemplateRun ON diffTemplate.diff_id = diffTemplateRun.diff_id LEFT JOIN publishRun ON stage_id=diffInput.diff_id AND (client_id=1 OR client_id=5) LEFT JOIN publishRun AS pubRun2 ON pubRun2.stage_id=diffTemplate.diff_id AND (pubRun2.client_id=1 OR pubRun2.client_id=5) LEFT JOIN magicDSRun ON magicDSRun.magic_ds_id=camRun.magicked
    22TITLE MOPS Status
    33FILE mopsStatus.php
    … …  
    1010OP OP1 ($row[8]>$row[9]?$row[8]. "(1)":($row[9]!=0?$row[9]. "(2)":"0"))
    1111# $diffInput.diff_id+$diffTemplate.diff_id
    12 OP OP2 $row[8]+$row[9]
    13 #  "[".$diffInput.diff_id."]:[".$tdiff_id."]"
     12# PUB ID
     13OP OP2 ($row[12]>$row[13]?$row[12]. "(I)":($row[13]!=0?$row[13]. "(T)":"0"))
     14#($row[13]>$row[14])
    1415
    1516#     field                    width format  name          show         link to                   extras
    16 FIELD dateobs,                 15,   %T,     Date Obs
     17FIELD dateobs,                 13,   %T,     Date Obs
    1718FIELD exp_name,                5,    %s,     Exp Name
    1819FIELD rawExp.exp_id,           5,    %d,     Exp ID
    1920FIELD chipRun.data_group,      5,    %s,     Data grp
    2021FIELD chip_id,                 5,    %d,     Chip ID (Stage 1)
    21 FIELD cam_id,                  5,    %d,     Cam ID (Stage 2)
     22FIELD camRun.cam_id AS camID,                  5,    %d,     Cam ID (Stage 2)
    2223FIELD fake_id,                 5,    %d,     Fake ID (Stage 3)
    2324FIELD warp_id,                 5,    %d,     Warp ID (Stage 4)
    … …  
    2526FIELD diffTemplate.diff_id AS tdiff_id,    5,    %d,     tdiff_id, none
    2627FIELD *,                       5,    %s,     Diff ID (Stage 5),         op=OP1
    27 #FIELD *,                      5,    %d,     Diff ID (Stage 5),         op=OP2
    28 FIELD camRun.magicked,         5,    %d,     Cam Magicked (Stage 6)
    29 FIELD publishRun.pub_id,       5,    %d,     Pub ID (Stage 7)
     28FIELD camRun.magicked,         5,    %d,     Magick ID (Stage 6)
     29FIELD magicDSRun.magic_ds_id,       5,    %d, DestreakID (Stage 7)
     30FIELD publishRun.pub_id AS pubId1,       5,    %d,     pubId1, none
     31FIELD pubRun2.pub_id AS pubId2,       5,    %d,     pubId2, none
     32FIELD *,                        10, %s, Pub ID, op=OP2
    3033FIELD filter,                  5,    %s,     Filter
    3134FIELD object,                  15,    %s,     Object
     35FIELD obs_mode,                5,    %s,     Obs Mode
    3236FIELD comment,                 40,    %s,     Comment
    3337
    34 FIELD diffInput.diff_id AS ddiffId,       5,    %s,     ddiffId,        none
    35 FIELD diffTemplate.diff_id AS tdiffId,    5,    %s,     tdiffId,        none
     38#FIELD diffInput.diff_id AS ddiffId,       5,    %s,     ddiffId,       none
     39#FIELD diffTemplate.diff_id AS tdiffId,    5,    %s,     tdiffId,       none
    3640
    3741MODE summary

  • branches/eam_branches/ipp-20101205/ippMonitor/raw/czartool_labels.php

    r30103 r30446  
    6464include 'version.php';
    6565echo "</p>";
    66 echo "<p  align=\"center\"> Current status of the IPP as of $lastUpdateTime (faults are shown in parentheses). <br>Documentation can be found <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\">here</a> and cluster load monitored <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&c=IPP%2520Production\">here</a><br>Current nightly science status: $nsStatus<br>Use <a href=\"$link\"> $plotTypeLink</a> plots <br/><a href=\"http://ipp004.ifa.hawaii.edu/clusterMonitor/top.html\">Who uses the cluster?</a><br><a href=\"http://ipp004.ifa.hawaii.edu/ippMetrics\">IPP Metrics</a><br/><a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/PS1_IPP_CzarLogs\">Czar log pages</a></p>";
    67 
     66echo "<p  align=\"center\"> Current status of the IPP as of $lastUpdateTime (faults are shown in parentheses). <br>Documentation can be found <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\">here</a> and cluster load monitored <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&c=IPP%2520Production\">here</a><br>Current nightly science status: $nsStatus<br>Use <a href=\"$link\"> $plotTypeLink</a> plots <br/><a href=\"http://ipp004.ifa.hawaii.edu/clusterMonitor/top.html\">Who uses the cluster?</a><br><a href=\"http://ipp004.ifa.hawaii.edu/ippMetrics\">IPP Metrics</a><br/><a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/PS1_IPP_CzarLogs\">Czar log pages</a><br><a href=\"czartool_exposures.php?pass=$pass&proj=$proj\">Exposure summary</a></p>";
    6867
    6968// deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later
    … …  
    182181            echo "<td> \n";
    183182              $today = date("Y-m-d");
    184               showSummitData($gpc1db, $today);
     183              createSummitDataTable($gpc1db, $today);
    185184            echo "</td>\n";
    186185          echo "</tr>\n";
    … …  
    189188    echo "</tr>\n";
    190189
    191 
    192190    # 4th row
     191    echo "<tr>\n";
     192      echo "<td> \n";
     193        echo "<table>\n";
     194          echo "<tr valign=top>\n";
     195            echo "<td> \n";
     196              # storage plot
     197              echo "<img src=\"czartool_getplot.php?type=s\"><br>";
     198            echo "</td>\n";
     199            echo "<td>\n";
     200              createDatesTable($czardb);
     201            echo "</td>\n";
     202          echo "</tr>\n";
     203        echo "</table>\n";
     204      echo "</td> \n";
     205    echo "</tr>\n";
     206
     207    # 5th row
    193208    echo "<tr>\n";
    194209      echo "<td>\n";
    … …  
    196211          echo "<tr valign=top>\n";
    197212            echo "<td> \n";
    198               # storage plot
    199               echo "<img src=\"czartool_getplot.php?type=s\"><br>";
     213              if ($selectedServer && !$serverCmd) showServerStatus($selectedServer);
    200214            echo "</td>\n";
    201215            echo "<td>\n";
    … …  
    206220                showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1);
    207221                showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS);
     222
    208223              echo "</table>";
    209224            echo "</td>\n";
    … …  
    213228    echo "</tr>\n";
    214229
    215     # 5th row
    216     echo "<tr>\n";
    217       echo "<td> \n";
    218         echo "<table>\n";
    219           echo "<tr valign=top>\n";
    220             echo "<td> \n";
    221               # server output
    222               if ($selectedServer && !$serverCmd) showServerStatus($selectedServer);
    223             echo "</td>\n";
    224             echo "<td>\n";
    225             echo "</td>\n";
    226           echo "</tr>\n";
    227         echo "</table>\n";
    228       echo "</td> \n";
    229     echo "</tr>\n";
    230 
    231230  echo "</table>\n";
    232231
    … …  
    236235###########################################################################
    237236#
    238 # Checks summitExp table agains rawExp table
    239 #
    240 ###########################################################################
    241 function showSummitData($gpc1db, $date) {
     237# Creates a table of summitExp against rawExp
     238#
     239###########################################################################
     240function createSummitDataTable($gpc1db, $date) {
    242241
    243242    $sql = "SELECT DISTINCT exp_type FROM summitExp WHERE dateobs > '$date'";
    … …  
    590589###########################################################################
    591590#
     591# Populates a tables with the current dates listed in the server_dates table
     592#
     593###########################################################################
     594function createDatesTable($db) {
     595
     596    $class = "list";
     597    echo "<table class=$class>\n";
     598    echo "<tr><td></td>\n";
     599    write_header_cell($class, "Server");
     600    write_header_cell($class, "Dates");
     601    echo "</tr>\n";
     602
     603    createServerDateRow($db, "stdscience");
     604    createServerDateRow($db, "registration");
     605
     606    echo "</table>\n";
     607}
     608
     609
     610###########################################################################
     611#
     612# Checks the status of all the pantasks servers
     613#
     614###########################################################################
     615function createServerDateRow($db, $server) {
     616
     617    $sql = "SELECT date FROM server_dates WHERE server LIKE '$server' ORDER BY date";
     618    if ($debug) {echo "$sql<br>";}
     619
     620    $qry = $db->query($sql);
     621    if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
     622    $dates = "";
     623    $firstIn = true;
     624    while ($qry->fetchInto($row)) {
     625
     626        if ($firstIn) {
     627
     628            $dates = $row[0];
     629            $firstIn = false;
     630        }
     631        else $dates = $dates . ", " . $row[0];
     632    }
     633
     634    $class = "list";
     635    echo "<tr><td></td>\n";
     636    write_table_cell($class, '%s', "", $server);
     637    write_table_cell($class, '%s', "", $dates);
     638    echo "</tr>\n";
     639
     640}
     641
     642###########################################################################
     643#
    592644# Checks the status of all the pantasks servers
    593645#
    … …  
    633685            ."&plottype=".$plotType;
    634686
    635         $link = ""; # TODO removed links temporarily
    636             write_table_cell($class, '%s', $link, $server);
     687        #$link = ""; # TODO removed links temporarily
     688        write_table_cell($class, '%s', $link, $server);
    637689        write_table_cell($class, '%s', "", $alive ? "yes" : "NO");
    638690

  • branches/eam_branches/ipp-20101205/ippScripts/Build.PL

    r30102 r30446  
    5858        scripts/magic_tree.pl
    5959        scripts/magic_process.pl
     60        scripts/magic_cleanup.pl
    6061        scripts/magic_destreak.pl
    6162        scripts/magic_destreak_revert.pl

  • branches/eam_branches/ipp-20101205/ippScripts/scripts/addstar_run.pl

    r29006 r30446  
    7171    defined $camera;
    7272if ($minidvodb && !defined($minidvodb_group)) {
    73                 my_die( "missing minidvodb_group", $add_id, 3 );
    74             }
     73                my_die( "missing minidvodb_group", $add_id, 3 );
     74            }
    7575my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    7676
    … …  
    113113
    114114if (defined $dvodbReal) {
    115         if ($minidvodb) {
    116            
    117             if (!defined($minidvodb_name)) {
    118                 #take the active one, if it's not defined on the command line
    119                 #reverts would have this already set, for example.
    120                 my $command = "addtool -listminidvodbrun ";
    121                 $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group;
    122                 $command .= " -state 'active' -limit 1";
    123                 $command .= " -dbname $dbname" if defined $dbname;
    124                 print $command;
    125                 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    126                     run(command => $command, verbose => $verbose);
    127                 &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success;
    128                 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    129                     &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR);
    130 
    131                 my $components = parse_md_list($metadata) or
    132                     &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
    133                 my $comp = $$components[0];
    134                 $minidvodb_path = $comp->{minidvodb_path};
    135                 $minidvodb_name = $comp->{minidvodb_name};
    136        
    137                 if (!defined($minidvodb_path)) {
    138                     &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR);
    139                 }
    140                 if (!defined($minidvodb_name)) {
    141                     &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR);
    142                 }
    143             }
    144             # tack on the minidvodb part to the db.
    145 #           $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/';
    146 #we don't need this now that I fixed the paths     
    147            
    148         }
     115        if ($minidvodb) {
     116
     117            #if (!defined($minidvodb_name)) {
     118                #take the active one, if it's not defined on the command line
     119                #reverts would have this already set, for example.
     120                my $command = "addtool -listminidvodbrun ";
     121                $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group;
     122                $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
     123                $command .= " -state 'active' -limit 1";
     124                $command .= " -dbname $dbname" if defined $dbname;
     125                print $command;
     126                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     127                    run(command => $command, verbose => $verbose);
     128                &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success;
     129                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     130                    &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR);
     131
     132                my $components = parse_md_list($metadata) or
     133                    &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR);
     134                my $comp = $$components[0];
     135                $minidvodb_path = $comp->{minidvodb_path};
     136                $minidvodb_name = $comp->{minidvodb_name};
     137
     138                if (!defined($minidvodb_path)) {
     139                    &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR);
     140                }
     141                if (!defined($minidvodb_name)) {
     142                    &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR);
     143                }
     144            #}
     145            # tack on the minidvodb part to the db.
     146#           $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/';
     147#we don't need this now that I fixed the paths
     148
     149        }
    149150        else {
    150151
    … …  
    153154
    154155unless ($no_op) {
    155             print $dvodbReal;
     156            print $dvodbReal;
    156157
    157158        ## addstar can either save the full set of detections, or just
    … …  
    222223        $command .= " -addprocessedexp";
    223224        $command .= " -fault $exit_code";
    224         $command .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path;
     225        $command .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path;
    225226        $command .= " -path_base $outroot" if defined $outroot;
    226227        $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400));
    227         $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
     228        $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name;
    228229        $command .= " -dbname $dbname" if defined $dbname;
    229230        system ($command);

  • branches/eam_branches/ipp-20101205/ippScripts/scripts/ipp_apply_burntool_single.pl

    r30102 r30446  
    7474unless ($success) {
    7575    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    76     &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
     76    &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
    7777}
    7878
    7979my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
    80     &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR);
     80    &my_die("Unable to parse metadata config doc", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
    8181
    8282my $burntoolStateGood = 999;
    … …  
    8787}
    8888if ($burntoolStateGood == 999) {
    89     &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR);
     89    &my_die("Failed to determine BURNTOOL.STATE.GOOD", $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
    9090}
    9191my $outState = -1 * abs($burntoolStateGood);
    … …  
    178178        my $exp_id = shift;
    179179        my $class_id = shift;
    180         vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -3",1);
     180        vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state 0 -set_state pending_burntool",1);
    181181    }
    182182    printf STDERR "$message\n";

  • branches/eam_branches/ipp-20101205/ippScripts/scripts/ipp_cleanup.pl

    r29892 r30446  
    167167#            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
    168168#            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
    169             addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id);
    170             addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id);
    171             addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
     169            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1);
     170            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1);
     171            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1);
    172172            if ($mode eq "goto_purged") {
    173173                # additional files to remove for 'purge' mode
    … …  
    429429
    430430                # delete the temporary image datafiles
    431                 addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id );
    432                 addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id);
    433                 addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id);
     431                addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id, 1);
     432                addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1);
     433                addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
    434434#            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
    435435                if ($mode eq "goto_purged") {
    … …  
    706706            my @files = ();
    707707            # delete the temporary image datafiles
    708             addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id);
    709             addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id);
    710             addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id);
    711 
    712             addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id);
    713             addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id);
    714             addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id);
     708            addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id, 1);
     709            addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id, 1);
     710            addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
     711
     712            addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id, 1);
     713            addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id, 1);
     714            addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id, 1);
    715715
    716716            addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id);
    … …  
    20482048    my $path_base  = shift; # base filename
    20492049    my $class_id   = shift; # class_id, if needed
     2050    my $recovery   = shift; # is there is a recovery file to clean?
    20502051
    20512052    my $file = $ipprc->filename($filerule, $path_base, $class_id);
    20522053
    20532054    push @$files, $file;
     2055   
     2056    if ($recovery) {
     2057        # need to clean up the recovery file (the pixels censored by streaksremove)
     2058        $file = $ipprc->recovery_filename($file);
     2059        push @$files, $file;
     2060    }
    20542061    return 1;
    20552062}

  • branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak.pl

    r30102 r30446  
    6565           'cam_reduction=s'=> \$cam_reduction,  # reduction class from camera stage (for chip and raw)
    6666           'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
    67            'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
     67           'recoveryroot=s' => \$recoveryroot,# "prefix" for saving the images of excised pixels
    6868           'replace=s'      => \$replace,    # replace the input images with the results.
    6969           'magicked=s'     => \$magicked,   # magicked state of the run
    … …  
    190190my $recovery_path_base;
    191191if ($recoveryroot) {
    192     if (!inNebulous($recoveryroot)) {
    193         if (! -e $recoveryroot ) {
    194             my $code = system "mkdir -p $recoveryroot";
    195             &my_die("cannot create output directory $recoveryroot", $magic_ds_id, $component,
    196                     $code >> 8) if $code;
    197         }
    198     }
    199     # note: trailing / is necessary
    200     $recovery_path_base = "$recoveryroot/$basename/";
     192    # recoveryroot is a path to prepend to the basenames of the input files
     193    if (inNebulous($recoveryroot)) {
     194        # if recoveryroot is a nebulous path we ignore the actual path and put the files in the
     195        # the same "directory" as the input files
     196        $recoveryroot = "$dirname/REC_";
     197        $recovery_path_base = $recoveryroot . $basename;
     198    } else {
     199        # otherwise we put the files in recoveryroot.
     200        # Regardless, we prefix the basename with 'REC_'
     201        $ipprc->outroot_prepare($recoveryroot);
     202        $recovery_path_base = "$recoveryroot/REC_$basename";
     203    }
    201204}
    202205
    … …  
    601604    $command   .= " -magic_ds_id $magic_ds_id";
    602605    $command   .= " -component $component";
     606    $command .= " -backup_path_base $backup_path_base" if $backup_path_base;
     607    $command .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
    603608    $command   .= " -fault $exit_code";
    604609    $command   .= " -dbname $dbname" if defined $dbname;

  • branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_cleanup.pl

    r29829 r30446  
    233233                $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
    234234                $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
    235                 $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
     235                # bills 2011-01-24
     236                # don't clean up the uncensored sources file
     237                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
    236238            }
    237239            if ($recovery_path_base) {
    … …  
    251253                $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
    252254                $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
    253                 $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
     255                # bills 2011-01-24
     256                # don't clean up the uncensored sources file
     257                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
    254258            }
    255259            if ($recovery_path_base) {

  • branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_defineruns.pl

    r30102 r30446  
    4242
    4343# Parse the command-line arguments
    44 my ($stage, $stage_limit, $workdir);
     44my ($stage, $stage_limit, $workdir, $recoveryroot);
    4545my ($dbname, $save_temps, $verbose, $no_update, $logfile);
    4646my @labels;
    … …  
    5151           'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
    5252           'workdir=s'      => \$workdir,    # output destination
     53           'recoveryroot=s' => \$recoveryroot, # recovery pixels destination
    5354           'dbname=s'       => \$dbname,     # Database name
    5455           'verbose'        => \$verbose,    # Print stuff?
    … …  
    7879    foreach my $label (@labels) {
    7980        my $command = "$magicdstool -definebyquery -stage $stage -workdir $workdir -label $label";
     81        $command .= " -recoveryroot $recoveryroot" if $recoveryroot;
    8082        $command .= " -pretend" if $no_update;
    8183        $command .= " -limit $stage_limit" if $stage_limit;

  • branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_revert.pl

    r30102 r30446  
    3939# Parse the command-line arguments
    4040my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction, $magicked, $run_state);
    41 my ($outroot, $recoveryroot, $replace, $release, $bytes, $md5sum);
     41my ($outroot, $recovery_path_base, $replace, $release, $bytes, $md5sum);
    4242my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
    4343
    … …  
    5353           'cam_reduction=s'=> \$cam_reduction,  # reduction class of the associated camera run
    5454           'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
    55            'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
     55           'recovery_path_base=s' => \$recovery_path_base,# "directory" for saving the images of excised pixels
    5656           'replace=s'      => \$replace,    # replace the input images with the results.
    5757           'bothways=s'     => \$bothways,   # run has inverse files (bothways diff)
    … …  
    130130
    131131# default value is "NULL" do not use it
    132 if (defined($recoveryroot) and ($recoveryroot eq "NULL")) {
    133     $recoveryroot = undef;
     132if (defined($recovery_path_base) and ($recovery_path_base eq "NULL")) {
     133    $recovery_path_base = undef;
    134134}
    135135
    … …  
    149149}
    150150
    151 my $recovery_path_base;
    152 if ($recoveryroot) {
    153     # note: trailing / is necessary
    154     $recovery_path_base = "$recoveryroot/$basename/";
    155 }
    156 
    157151
    158152my ($image, $mask, $ch_mask, $weight, $sources, $astrom);
    159153my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom);
     154my ($rimage, $rmask, $rch_mask, $rweight);
    160155
    161156if ($stage eq "raw") {
    … …  
    214209    $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id);
    215210    $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id);
     211
     212    $rimage  = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $class_id);
     213    # This is somewhat kludgey but it works whether the mask is camera mask or chip mask
     214    $rmask   = dirname($recovery_path_base) . "/REC_" . basename($mask);
     215    $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $class_id);
     216    $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $class_id);
    216217} elsif ($stage eq "camera") {
    217218    $astrom =  $ipprc->filename("PSASTRO.OUTPUT", $path_base);
    … …  
    226227    $bweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $backup_path_base);
    227228    $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base);
     229    $rimage  = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base);
     230    $rmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base);
     231    $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base);
    228232} elsif ($stage eq "diff") {
    229233    my $name = "PPSUB.OUTPUT";
    … …  
    236240    $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
    237241    $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
     242    if ($recovery_path_base) {
     243        $rimage  = $ipprc->filename($name, $recovery_path_base);
     244        $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
     245        $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
     246    }
    238247}
    239248
    240249revert_files($replace, $image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
     250if ($recovery_path_base) {
     251    delete_recovery_files($rimage, $rmask, $rweight, $rch_mask);
     252}
    241253
    242254if ($stage eq "diff" and $bothways) {
    … …  
    251263    $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
    252264    revert_files($replace, $image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef);
     265    if ($recovery_path_base) {
     266        $rimage  = $ipprc->filename($name, $recovery_path_base);
     267        $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
     268        $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
     269        delete_recovery_files($rimage, $rmask, $rweight);
     270    }
    253271}
    254272
    … …  
    426444
    427445
    428 sub file_check
    429 {
    430     my $file = shift;           # Name of file
    431     &my_die("Unable to find output file: $file", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless
    432         $ipprc->file_exists($file);
     446sub delete_recovery_files
     447{
     448    foreach my $file (@_) {
     449        if ($ipprc->file_exists($file)) {
     450            if (!$ipprc->file_delete($file)) {
     451                &my_die("failed to delete recovery file $file", $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR);
     452            }
     453        }
     454    }
    433455}
    434456

  • branches/eam_branches/ipp-20101205/ippScripts/scripts/nightly_science.pl

    r30102 r30446  
    4444my ( $date, $datetime, $camera, $dbname, $logfile, $verbose, $manual);
    4545my ( $help, $isburning, $force_stack_count, $force_diff_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
    46 my ( $registration_status, $burntool_status, $observing_status);
     46my ( $registration_status, $burntool_status, $observing_status, $old_date);
    4747my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
    4848my ( $check_stacks, $queue_stacks, $check_sweetspot, $queue_sweetspot, $check_diffs, $queue_diffs, $clean_old);
    … …  
    6666    'registraion_status'   => \$registration_status,
    6767    'burntool_status'      => \$burntool_status,
     68    'old_date=s'           => \$old_date,
    6869    'check_stacks'         => \$check_stacks,
    6970    'queue_stacks'         => \$queue_stacks,
    … …  
    134135unless ($success) {
    135136    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    136     &my_die("Unable to perform ppConfigDump: $error_code", $date, $PS_EXIT_SYS_ERROR);
     137    &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $date, $PS_EXIT_SYS_ERROR);
    137138}
    138139
    … …  
    278279
    279280$metadata_out{nsObservingState} = &get_observing_state($date);
     281$metadata_out{nsRegistrationState} = &get_registration_state($date);
    280282#
    281283# Mode selection
    … …  
    396398    exit(0);
    397399}
    398 exit(10);
     400
    399401if (defined($check_confirm_stacks) || defined($test_mode)) {
    400402    $metadata_out{nsStackState} = 'CONFIRM_STACKING';
    … …  
    409411    exit(0);
    410412}
    411 
     413exit(10);
    412414if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) {
    413415    $metadata_out{nsSSState} = 'CHECKSWEETSPOT';
    … …  
    10971099    my $pretend = shift;
    10981100
     1101    my $Npotential = 0;
     1102    my $Nqueued = 0;
     1103    my $is_processing = 0;
     1104    my $is_registering;
     1105    if ($metadata_out{nsRegistrationState} eq 'REGISTERED') {
     1106        $is_registering = 0;
     1107    }
     1108    else {
     1109        $is_registering = 1;
     1110    }
     1111
    10991112    foreach my $target (sort (keys %science_config)) {
    11001113        if ($science_config{$target}{STACKABLE} == 1) {
    … …  
    11051118                        print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n";
    11061119                    }
     1120                    $is_processing = 1;
    11071121                    $metadata_out{nsStackState} = 'FORCETOWARP';
    11081122                    next;
    … …  
    11201134                    next;
    11211135                }
     1136                $Npotential++;
    11221137                if ($Nalready != 0) {
     1138                    $Nqueued++;
    11231139                    if ($debug == 1) {
    11241140                        print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n";
    … …  
    11371153                    }
    11381154                    stack_queue($date,$target,$filter);
     1155                    $Nqueued ++;
    11391156                }
    11401157                if (defined($pretend)) {
    … …  
    11521169        }
    11531170    }
     1171    $metadata_out{nsStackPotential} = $Npotential;
     1172    $metadata_out{nsStackQueued}    = $Nqueued;
     1173    if (($Npotential == $Nqueued)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)&&($is_registering == 0)) {
     1174        $metadata_out{nsStackState} = 'FINISHED_STACKS';
     1175    }   
    11541176}
    11551177
    … …  
    12311253    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
    12321254
    1233     my $sth = "SELECT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
     1255    my $count = 0;
     1256   
     1257    my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
    12341258    my $data_ref = $db->selectall_arrayref( $sth );
    1235 
    1236     return($#{ $data_ref } + 1);
     1259    $count += $#{ $data_ref } + 1;
     1260
     1261    $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where data_group = '$data_group' AND warp1 = $warp_id_2 AND warp2 = $warp_id_1";
     1262    $data_ref = $db->selectall_arrayref( $sth );
     1263    $count += $#{ $data_ref } + 1;
     1264
     1265    return($count);
    12371266}
    12381267
    … …  
    12721301    my $pretend = shift;
    12731302
     1303    my $Npotential = 0;
     1304    my $Nnoexp     = 0;
     1305    my $is_processing = 0;
     1306    my $is_registering;
     1307    if ($metadata_out{nsRegistrationState} eq 'REGISTERED') {
     1308        $is_registering = 0;
     1309    }
     1310    else {
     1311        $is_registering = 1;
     1312    }
     1313
    12741314    foreach my $target (sort (keys %science_config)) {
    12751315        if ($science_config{$target}{DIFFABLE} == 1) {
    12761316            foreach my $filter (@filter_list) {
     1317                $Npotential ++;
    12771318                my ($NprocChips,$NprocWarps) = pre_diff_queue($date,$observing_state,$target,$filter);
    12781319                if ((!defined($force_diff_count))&&($NprocChips != $NprocWarps)) {
    … …  
    12801321                        print STDERR "execute_diffs: Target $target in filter $filter on $date is not fully processed. ($NprocChips $NprocWarps)\n";
    12811322                    }
    1282                     $metadata_out{nsDiffState} = 'FORCETOWARP';
    1283                     next;
    1284                 }
     1323                    $is_processing = 1;
     1324                }
    12851325                if ($NprocChips == 0) {
     1326                    $Nnoexp ++;
    12861327                    if ($debug == 1) {
    12871328                        print STDERR "execute_diffs: Target $target in filter $filter on $date has no exposures.\n";
    … …  
    13111352        }
    13121353    }
     1354    if ($debug == 1) {
     1355        print "$metadata_out{nsObservingState} $metadata_out{nsDiffPotential} $metadata_out{nsDiffQueued} $is_processing $is_registering\n";
     1356    }
     1357
     1358    if ($metadata_out{nsObservingState} eq 'END_OF_NIGHT') {
     1359        if ($is_processing == 1) {
     1360            $metadata_out{nsDiffState} = 'DIFFING';
     1361        }
     1362        elsif ($is_registering == 0) {
     1363            if ($Npotential == $Nnoexp) {
     1364                $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
     1365            }
     1366            elsif ($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued}) {
     1367                $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
     1368            }
     1369        }
     1370        else {
     1371            $metadata_out{nsDiffState} = 'DIFFING';
     1372        }
     1373    }
     1374    else {
     1375        $metadata_out{nsDiffState} = 'DIFFING';
     1376    }
     1377
     1378#     if (($Npotential == $Nnoexp)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)) {
     1379#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
     1380#     }
     1381
     1382#     if ($is_processing == 1) {
     1383#       $metadata_out{nsDiffState} = 'DIFFING';
     1384#     }
    13131385}
    13141386
    … …  
    13261398
    13271399    my $object_ref = $db->selectall_arrayref( $obj_sth );
     1400
     1401    my $Npotential = 0;
     1402    my $Nqueued = 0;
    13281403   
    13291404    foreach my $object_row (@{ $object_ref }) {
    13301405        my $this_object = shift @{ $object_row };
    1331         my $input_sth = "select exp_id,warp_id,dateobs from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
    1332         $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
    1333         $input_sth .= " ORDER BY dateobs ";
     1406#       my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
     1407#       $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
     1408#       $input_sth .= " ORDER BY dateobs ";
    13341409       
     1410        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM ";
     1411        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
     1412        $input_sth .=   " WHERE chipRun.label = '$label' AND chipRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
     1413        $input_sth .=   " ORDER BY dateobs ";
     1414
    13351415        my $warps = $db->selectall_arrayref( $input_sth );
     1416
     1417        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
     1418        my %comment_hash = ();
     1419        foreach my $this_warp (@{ $warps }) {
     1420            my $this_comment = ${ $this_warp }[3];
     1421            my $this_exp_id  = ${ $this_warp }[0];
     1422            $comment_hash{$this_comment} = $this_exp_id;
     1423        }
    13361424       
    13371425        if (($#{ $warps } + 1) % 2 != 0) {
    1338             print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps }\n";
    1339             next;
     1426            print STDERR "diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
     1427            if ($#{ $warps} + 1 == 1) {
     1428                print STDERR ": I can do no diffs with only one exposure.\n";
     1429                next;
     1430            }
     1431            else {
     1432                print STDERR ": I should declare an exposure to be faulty.\n";
     1433                my @keep_warps = ();
     1434#               print "@{ $warps }\n";
     1435                foreach my $this_warp (@{ $warps }) {
     1436                    my $this_comment = ${ $this_warp }[3];
     1437                    my $this_exp_id  = ${ $this_warp }[0];
     1438                    if ($comment_hash{$this_comment} == $this_exp_id) {
     1439                        push @keep_warps, $this_warp;
     1440                    }
     1441                    else {
     1442                        print STDERR "diff_queue: excluding $this_exp_id for $this_object\n";
     1443                    }
     1444                }
     1445                @{ $warps } = @keep_warps;
     1446#               print "@{ $warps }\n";
     1447            }
    13401448        }
    13411449       
    13421450        while ($#{ $warps } > -1) {
    13431451            my $input_warp = shift @{ $warps };
     1452            my $input_exp_id = ${ $input_warp }[0];
     1453            my $input_comment = ${ $input_warp }[3];
     1454
     1455           
    13441456            my $template_warp = shift @{ $warps };
    1345             my $input_exp_id = ${ $input_warp }[0];
     1457
    13461458            my $template_exp_id = ${ $template_warp }[0];
    13471459           
    13481460            my $input_warp_id = ${ $input_warp }[1];
    13491461            my $template_warp_id = ${ $template_warp }[1];
     1462
     1463            my $input_warp_state = ${ $input_warp }[4];
     1464            my $template_warp_state = ${ $template_warp }[4];
    13501465           
     1466            $Npotential++;
     1467
     1468            unless (defined($input_warp_id) && defined($template_warp_id) &&
     1469                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
     1470                print STDERR "Diff for this $date $target $input_exp_id $template_exp_id not fully processed\n";
     1471                next;
     1472            }
     1473
    13511474            if (verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
    1352                 print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id\n";
     1475                $Nqueued++;
     1476                print STDERR "Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
    13531477                next;
    13541478            }
    … …  
    13661490                $cmd .= ' -pretend ';
    13671491                print STDERR "diff_queue: $cmd\n";
     1492                print STDERR " $input_warp_id $template_warp_id\n";
    13681493            }
    13691494           
    … …  
    13751500                    &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
    13761501                }
    1377             }
    1378         }
    1379     }
     1502                $Nqueued++;
     1503            }
     1504        }
     1505    }
     1506    $metadata_out{nsDiffPotential} += $Npotential;
     1507    $metadata_out{nsDiffQueued}    += $Nqueued;
     1508#      if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
     1509#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
     1510#      }       
     1511
    13801512}
    13811513           
    … …  
    14931625        }
    14941626    }   
    1495 
     1627#    print "$datetime $eon_dt\n";
    14961628    if (DateTime->compare($datetime,$eon_dt) < 1) {
    14971629        return("OBSERVING");
    … …  
    15011633    }
    15021634}
     1635
     1636# This basically does the end of night check, but does it "the hard way," to prevent the time from fooling us.
     1637sub get_registration_state {
     1638    my $date = shift;
     1639
     1640    foreach my $eon (keys %eon_config) {
     1641        my $command = "$regtool -processedexp -simple ";
     1642        $command .= " -dbname $dbname ";
     1643        $command .= " -dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 ";
     1644        $command .= " -object $eon_config{$eon}{OBJECT} " if defined($eon_config{$eon}{OBJECT});
     1645        $command .= " -obs_mode $eon_config{$eon}{OBSMODE} " if defined($eon_config{$eon}{OBSMODE});
     1646        $command .= " -exp_type $eon_config{$eon}{EXPTYPE} " if defined($eon_config{$eon}{EXPTYPE});
     1647        $command .= " -comment $eon_config{$eon}{COMMENT} " if defined($eon_config{$eon}{COMMENT});
     1648
     1649        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     1650            run ( command => $command, verbose => $verbose );
     1651        unless ($success) {
     1652            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     1653            &my_die("Unable to perform regtool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
     1654        }
     1655        my @eon_exposures = split /\n/, (join '', @$stdout_buf);
     1656        if ($#eon_exposures >= 0) {
     1657            return("REGISTERED");
     1658        }
     1659    }   
     1660    return("NOT_REGISTERED");
     1661}
     1662
     1663   
    15031664
    15041665sub get_tool_parameters {
    … …  
    16201781    my $stage = shift; # stage name
    16211782    my $stage_id = shift; #  identifier
     1783    my $date = shift;
    16221784    my $exit_code = shift; # Exit code
    1623     # outputImage and path_base are globals
    16241785
    16251786    carp($msg);

  • branches/eam_branches/ipp-20101205/ippScripts/scripts/publish_file.pl

    r30102 r30446  
    4242
    4343# Parse the command-line arguments
    44 my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir );
     44my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir, $need_magic );
    4545my ( $dbname, $verbose, $no_update, $no_op, $save_temps, $redirect );
    4646my ( $output_format );
    … …  
    5454    'fileset=s'         => \$fileset,     # Fileset name
    5555    'workdir=s'         => \$workdir,     # Working directory
     56    'need-magic'        => \$need_magic,  # do we require censored detections?
    5657    'dbname=s'          => \$dbname,    # Database name
    5758    'verbose'           => \$verbose,   # Print to stdout
    … …  
    166167    foreach my $comp ( @$components ) {
    167168        my $path_base = $comp->{path_base}; # Base name for file
     169        if (!$need_magic and $comp->{magicked}) {
     170            # This client is authorized to receive uncensored detections
     171            # Get the uri for the "backup" files
     172            print "Using uncensored input from $path_base\n";
     173            $path_base = $ipprc->destreaked_filename($path_base);
     174        }
    168175        next if defined $comp->{quality} and $comp->{quality} > 0;
    169176        print "Warning: mis-matched comments\n" if defined $comment and $comment ne $comp->{comment};
    … …  
    181188            $filename = $ipprc->filename( "PSPHOT.OUT.CMF.MEF", "$path_base.pos" );
    182189        }
    183         $filename = $ipprc->file_resolve($filename);
     190        &my_die("input file does not exist: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$ipprc->file_exists($filename);
     191        my $resolved = $ipprc->file_resolve($filename);
     192       
     193        &my_die("unable to resolve input file: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$resolved;
     194
     195        $filename = $resolved;
    184196
    185197        my $data = { zp => $zp,

  • branches/eam_branches/ipp-20101205/ippScripts/scripts/register_exp.pl

    r30102 r30446  
    348348    if (defined $exp_id and not $no_update) {
    349349        my $command = "$regtool -addprocessedexp -exp_id $exp_id -fault $exit_code";
     350        $command .= " -exp_tag $exp_tag";
     351        $command .= " -exp_name UNKNOWN";
     352        $command .= " -inst UNKNOWN";
     353        $command .= " -telescope UNKNOWN";
     354        $command .= " -telescope UNKNOWN";
     355        $command .= " -filelevel UNKNOWN";
    350356        $command .= " -hostname $host" if defined $host;
    351357        $command .= " -dbname $dbname" if defined $dbname;

  • branches/eam_branches/ipp-20101205/ippScripts/scripts/register_imfile.pl

    r30102 r30446  
    1010my $date = `date`;
    1111print "\n\n";
    12 print "Starting script $0 on $host at $date\n\n";
     12my $cmd_line = join ' ', @ARGV;
     13print "Starting script $0 $cmd_line on $host at $date\n\n";
    1314
    1415use vars qw( $VERSION );
    … …  
    3940
    4041my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $bytes, $md5sum, $dbname, $verbose, $no_update, $no_op, $logfile);
     42my ($sunrise, $sunset);
    4143GetOptions(
    4244    'caches'           => \$cache,
    … …  
    4749    'bytes=s'          => \$bytes,
    4850    'md5sum=s'         => \$md5sum,
     51    'sunrise=s'        => \$sunrise,
     52    'sunset=s'         => \$sunset,
    4953    'dbname|d=s'       => \$dbname,    # Database name
    5054    'verbose'          => \$verbose,   # Print to stdout
    … …  
    5458) or pod2usage( 2 );
    5559
    56 my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    57 $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
     60my $ipprc = PS::IPP::Config->new() or my_die_for_add( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
     61$ipprc->redirect_output($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
    5862
    5963pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    … …  
    6569    defined $uri;
    6670
     71unless (defined($sunset)) {
     72    $sunset = '03:30:00';
     73}
     74unless (defined($sunrise)) {
     75    $sunrise = '17:30:00';
     76}
    6777
    6878my $RECIPE = "REGISTER"; # Recipe to use for ppStats
    … …  
    8494
    8595my $cmdflags;
    86 my $burntoolStateGood;
     96my $burntoolStateTarget;
     97my $burntoolStateCurrent;
    8798# Run ppStats on the input file
    8899{
    … …  
    101112    print "STDOUT:\n$out1";
    102113    print "STDERR:\n$err1";
    103     &my_die("Unable to perform ppStats on exposure id $exp_id: " . $h1->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h1->result() or $PS_EXIT_PROG_ERROR) ) unless $result1;
     114    &my_die_for_add("Unable to perform ppStats on exposure id $exp_id: " . $h1->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h1->result() or $PS_EXIT_PROG_ERROR) ) unless $result1;
    104115
    105116    print "[Running " . join(' ', @command2) . "]\n";
    … …  
    110121    print "STDOUT:\n$out2";
    111122    print "STDERR:\n$err2";
    112     &my_die("Unable to perform ppStatsFromMetadata on exposure id $exp_id: " . $h2->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h2->result() or $PS_EXIT_PROG_ERROR) ) unless $result2;
     123    &my_die_for_add("Unable to perform ppStatsFromMetadata on exposure id $exp_id: " . $h2->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h2->result() or $PS_EXIT_PROG_ERROR) ) unless $result2;
    113124    chomp $out2;
    114125    $cmdflags = $out2;
    115126
    116127    # Manually parse the burntool_state entry.
    117     my $burntoolState = 0;
     128    $burntoolStateCurrent = 0;
    118129    my $isGPC1 = 0;
    119     $burntoolStateGood = 0;
     130    $burntoolStateTarget = 0;
    120131    foreach my $line (split /\n/, $out1) {
    121132        if ($line =~ /FPA.BURNTOOL.APPLIED/) {
    122133            $line =~ s/^\s+//;
    123             $burntoolState = (split /\s+/, $line)[2];
     134            $burntoolStateCurrent = (split /\s+/, $line)[2];
    124135        }
    125136        if ($line =~ /FPA.CAMERA/) {
    … …  
    131142    }
    132143    if ($isGPC1 != 1) {
    133         $burntoolState = 0; # If it's not GPC1, you shouldn't have run burntool.
    134     }
    135     elsif (($isGPC1 == 1) && ($burntoolState == 1)) {
    136 #       print STDERR "In the good region: >>$burntoolState<<\n";
     144        $burntoolStateCurrent = 0; # If it's not GPC1, you shouldn't have run burntool.
     145    }
     146    elsif (($isGPC1 == 1) && ($burntoolStateCurrent == 1)) {
     147#       print STDERR "In the good region: >>$burntoolStateCurrent<<\n";
    137148        my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -";
    138149        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    148159            if ($line =~ /BURNTOOL.STATE.GOOD/) {
    149160                $line =~ s/^\s+//;
    150                 $burntoolStateGood = (split /\s+/, $line)[2];
     161                $burntoolStateTarget = (split /\s+/, $line)[2];
    151162                last;
    152163            }
    153164        }
    154         $burntoolState = $burntoolStateGood; # Positive because this has the header table.
    155 
    156     }
    157     $cmdflags .= " -burntool_state $burntoolState ";
     165        # XXX why was this being equated??
     166        # $burntoolState = $burntoolStateGood; # Positive because this has the header table.
     167
     168    }
     169    $cmdflags .= " -burntool_state $burntoolStateTarget ";
    158170}
    159171
    160172$now_time = localtime();
    161173printf STDERR "\ndone with ppStats: %s\n", $now_time if $verbose;
     174printf STDERR "\nburntool state current: %d target: %d\n", $burntoolStateCurrent, $burntoolStateTarget;
    162175
    163176# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type
    164 if (uc(&value_for_flag ($cmdflags, "NULL", "-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
    165 if (uc(&value_for_flag ($cmdflags, "NULL", "-inst"))      eq "NULL") { &my_die ("inst      not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
    166 if (uc(&value_for_flag ($cmdflags, "NULL", "-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
    167 if (uc(&value_for_flag ($cmdflags, "NULL", "-class_id"))  eq "NULL") { &my_die ("class_id  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
    168 if (uc(&value_for_flag ($cmdflags, "NULL", "-exp_type"))  eq "NULL") { &my_die ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
     177if (uc(&value_for_flag ($cmdflags, "NULL", "-telescope")) eq "NULL") { &my_die_for_add ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
     178if (uc(&value_for_flag ($cmdflags, "NULL", "-inst"))      eq "NULL") { &my_die_for_add ("inst      not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
     179if (uc(&value_for_flag ($cmdflags, "NULL", "-filelevel")) eq "NULL") { &my_die_for_add ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
     180if (uc(&value_for_flag ($cmdflags, "NULL", "-class_id"))  eq "NULL") { &my_die_for_add ("class_id  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
     181if (uc(&value_for_flag ($cmdflags, "NULL", "-exp_type"))  eq "NULL") { &my_die_for_add ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
     182my $dateobs   = &value_for_flag($cmdflags, 0.0, "-dateobs");
    169183
    170184my $command = "$regtool -addprocessedimfile";
    … …  
    177191$command .= " -hostname $host" if defined $host;
    178192$command .= " -dbname $dbname" if defined $dbname;
    179 $command .= " -data_state check_burntool";
     193if (abs($burntoolStateCurrent) == $burntoolStateTarget) {
     194    printf STDERR "This has already been burned.\n";
     195    $command .= " -data_state full";
     196}
     197elsif (is_daytime($dateobs,$sunrise,$sunset)) {
     198    printf STDERR "This is a daytime exposure.\n";
     199    $command .= " -data_state full";
     200}
     201else {
     202    printf STDERR "Need to check burntool.\n";
     203    $command .= " -data_state check_burntool";
     204}
    180205$command .= " $cmdflags";
    181206
    … …  
    187212my $ra        = &value_for_flag($cmdflags, 0.0, "-ra");
    188213my $dec       = &value_for_flag($cmdflags, 0.0, "-decl");
    189 my $dateobs   = &value_for_flag($cmdflags, 0.0, "-dateobs");
     214
    190215
    191216# if the needed data is available, pass it to sunmoon:
    … …  
    216241# lock file?
    217242# if exp_type = DARK and date > MIDNIGHT HST { wait }
    218 # system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log"); 
     243# system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log");
    219244
    220245$now_time = localtime();
    … …  
    236261# We now have an imfile in the database, check if we can burntool it.  If not, continue on.
    237262
    238 {
     263if ((abs($burntoolStateCurrent) != $burntoolStateTarget)&&
     264    (!is_daytime($dateobs,$sunrise,$sunset))) {
    239265    my $mdcParser  = PS::IPP::Metadata::Config->new;
    240266
    … …  
    243269    $check_date =~ s/T.*$//;
    244270    my $exp_name   = $tmp_exp_name;
    245    
     271
    246272
    247273    my $bt_check_command = "$regtool -checkburntoolimfile ";
    248274    $bt_check_command .= " -class_id $class_id ";
    249     $bt_check_command .= " -date $check_date ";
    250     $bt_check_command .= " -valid_burntool $burntoolStateGood ";
     275    $bt_check_command .= " -dateobs_begin ${check_date}T${sunset} ";
     276    $bt_check_command .= " -dateobs_end ${check_date}T${sunrise} ";
     277    $bt_check_command .= " -valid_burntool $burntoolStateTarget ";
    251278    $bt_check_command .= " -exp_name $exp_name ";
    252279    $bt_check_command .= " -dbname $dbname" if defined $dbname;
    253280
    254281    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    255         run ( command => $bt_check_command, verbose => $verbose);
     282        run ( command => $bt_check_command, verbose => $verbose);
    256283    unless ($success) {
    257         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    258         &my_die("Unable to perform regtool: $error_code",
    259                 $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR);
     284        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     285        &my_die_for_update("Unable to perform regtool: $error_code",
     286                           $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
    260287    }
    261288
    … …  
    264291    my $exposures;
    265292    while ( scalar @whole > 0 ) {
    266         my $value = shift @whole;
    267         push @single, $value;
    268         if ($value =~ /^\s*END\s*$/) {
    269             push @single, "\n";
    270            
    271             my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
    272             &my_die("Unable to parse output from regtool",
    273                     $class_id, $exp_name, $date, $PS_EXIT_SYS_ERROR) unless
    274                         defined $list;
    275             push @{ $exposures }, @$list;
    276             @single = ();
    277         }
    278     }
    279    
     293        my $value = shift @whole;
     294        push @single, $value;
     295        if ($value =~ /^\s*END\s*$/) {
     296            push @single, "\n";
     297
     298            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
     299            &my_die_for_update("Unable to parse output from regtool",
     300                               $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless
     301                                   defined $list;
     302            push @{ $exposures }, @$list;
     303            @single = ();
     304        }
     305    }
     306
    280307    # We only care about the final entry, as that contains the exposure we are.
    281    
     308
    282309    my $regtool_update = "$regtool -updateprocessedimfile ";
    283310    $regtool_update .= "-dbname $dbname " if defined $dbname;
    … …  
    286313    my $burntool_data = pop(@{ $exposures });
    287314    if ($burntool_data->{burnable} == 0) {
    288         $regtool_update .= " -burntool_state 0 -set_state pending_burntool ";
     315        $regtool_update .= " -burntool_state 0 -set_state pending_burntool ";
     316        unless ($no_update) {
     317            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     318                IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
     319            unless ($success) {
     320                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     321                warn ("Unable to perform regtool -addprocessedimfile: $error_code");
     322                exit($error_code);
     323            }
     324        } else {
     325            print "skipping command: $command\n";
     326        }
    289327    }
    290328    else {
    291         my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname ";
    292         $apply_command .= " --class_id $class_id --exp_id $exp_id ";
    293         $apply_command .= " --this_uri $burntool_data->{uri} ";
    294         $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri};
    295         $apply_command .= " --imfile_state $burntool_data->{imfile_state} ";
    296         $apply_command .= " --verbose " if $verbose;
    297         print "$apply_command\n";
    298         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    299             IPC::Cmd::run(command => $apply_command, verbose => $verbose);
    300         unless ($success) {
    301             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    302             warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code");
    303             exit($error_code);
    304         }
    305     }   
    306 
    307     unless ($no_update) {
    308         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    309             IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
    310         unless ($success) {
    311             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    312             warn ("Unable to perform regtool -addprocessedimfile: $error_code");
    313             exit($error_code);
    314         }
    315     } else {
    316         print "skipping command: $command\n";
    317     }
     329        my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname ";
     330        $apply_command .= " --class_id $class_id --exp_id $exp_id ";
     331        $apply_command .= " --this_uri $burntool_data->{uri} ";
     332        $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri};
     333        $apply_command .= " --imfile_state $burntool_data->{imfile_state} ";
     334        $apply_command .= " --verbose " if $verbose;
     335        print "$apply_command\n";
     336        unless ($no_update) {
     337            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     338                IPC::Cmd::run(command => $apply_command, verbose => $verbose);
     339            unless ($success) {
     340                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     341                &my_die_for_update("Unable to perform ipp_apply_burntool_single.pl: $error_code",
     342                               $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
     343            }
     344        }
     345    }
     346
    318347
    319348}
    … …  
    351380}
    352381
    353 sub my_die
     382sub is_daytime
     383{
     384    my $dateobs = shift;
     385    my $sunset  = shift;
     386    my $sunrise = shift;
     387
     388    my ($date,$time);
     389
     390    if ($dateobs =~ /T/) {
     391        ($date,$time) = split /T/, $dateobs;
     392    }
     393    else {
     394        ($date,$time) = split / /, $dateobs;
     395    }
     396    my ($hour,$minute,$second) = split /\:/, $time; # /;
     397    my ($ss_hour,$ss_minute,$ss_second) = split /\:/, $sunset; # /;
     398    my ($sr_hour,$sr_minute,$sr_second) = split /\:/, $sunrise; # /;
     399    if ($second =~ /Z/) {
     400        $second =~ s/Z//;
     401    }
     402#     print "this exposure: $hour $minute $second\n";
     403#     print "sunset:        $ss_hour $ss_minute $ss_second\n";
     404#     print "sunrise:       $sr_hour $sr_minute $sr_second\n";
     405#     printf "Hss: %d Mss: %d Sss: %d\n",($hour >= $ss_hour),($minute >= $ss_minute),($second >= $ss_second);
     406#     printf "Hsr: %d Msr: %d Ssr: %d\n",($hour <= $sr_hour),($minute <= $sr_minute),($second <= $sr_second);
     407
     408    if (($hour > $ss_hour)&&($hour <= 24)) {
     409        return(1); # After sunset by more than an hour, before midnight
     410    }
     411    elsif ($hour == $ss_hour) {
     412        if ($minute > $ss_minute) {
     413            return(1); # After sunset by more than a minute
     414        }
     415        elsif ($minute == $ss_minute) {
     416            if ($second >= $ss_second) {
     417                return(1); # After sunset by more than a second
     418            }
     419            else {
     420                return(0);
     421            }
     422        }
     423        else {
     424            return(0);
     425        }
     426    }
     427    elsif (($hour < $sr_hour)&&($hour >= 0)) {
     428        return(1); # Before sunrise by more than an hour, but after midnight
     429    }
     430    elsif ($hour == $sr_hour) {
     431        if ($minute < $sr_minute) {
     432            return(1); # Before sunrise by more than a minute
     433        }
     434        elsif ($minute == $sr_minute) {
     435            if ($second <= $sr_second) {
     436                return(1); # Before sunrise by more than a second
     437            }
     438            else {
     439                return(0);
     440            }
     441        }
     442        else {
     443            return(0);
     444        }
     445    }
     446    else {
     447        return(0); # We should never get here.
     448    }
     449}
     450
     451sub my_die_for_add
    354452{
    355453    my $msg = shift; # Warning message on die
    … …  
    382480    exit $exit_code;
    383481}
     482sub my_die_for_update
     483{
     484    my $msg = shift; # Warning message on die
     485    my $exp_id = shift;
     486    my $class_id = shift;
     487    my $exit_code = shift;
     488
     489    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     490
     491    carp($msg);
     492    if (defined $exp_id && defined $tmp_class_id and not $no_update) {
     493        my $command = "$regtool -updateprocessedimfile";
     494        $command .= " -exp_id $exp_id";
     495        $command .= " -class_id $class_id";
     496        $command .= " -fault $exit_code";
     497        $command .= " -hostname $host" if defined $host;
     498        $command .= " -dbname $dbname" if defined $dbname;
     499        print "Running: $command\n";
     500        system($command);
     501    }
     502    exit $exit_code;
     503}
    384504
    385505# Pau.

  • branches/eam_branches/ipp-20101205/ippTasks/Makefile.am

    r30102 r30446  
    1717        warp.pro \
    1818        magic.pro \
     19        magic.cleanup.pro \
    1920        destreak.pro \
    2021        destreak.cleanup.pro \

  • branches/eam_branches/ipp-20101205/ippTasks/addstar.pro

    r28211 r30446  
    173173    $run = $run --minidvodb_group $MINIDVODB_GROUP
    174174        if (("$MINIDVODB_NAME" != "NULL") && ("$MINIDVODB_NAME" != "(null)"))
    175            $run = $run --minidvodb_name $MINIDVODB_NAME #have addstar_run.pl grab the 'active' name if it is NULL
     175           $run = $run --minidvodb_name $MINIDVODB_NAME
     176    #have addstar_run.pl grab the 'active' name if it is NULL
    176177        end
    177178    end

  • branches/eam_branches/ipp-20101205/ippTasks/diff.pro

    r28375 r30446  
    4545  end
    4646  task diff.revert
    47     active false
     47    active true
    4848  end
    4949  task diff.summary.load
    … …  
    448448  periods      -timeout 120.0
    449449  npending     1
    450   active false
    451450
    452451  stdout NULL
    … …  
    457456    # Only revert failures with fault=2 (SYS_ERROR), which tend to be
    458457    # temporary filesystem problems.  Every other fault type is
    459     # interesting and should be kept for debugging (and so it doesn't
     458    # interesting and should be kept for debugging (and so it does not
    460459    # continue to occur).
    461     $run = difftool -revertdiffskyfile -fault 2
     460    # XXX: actually we are getting enough jobs with fault 4 that work after reverting
     461    # that we are going to remove that cut for now
     462    #
     463    #$run = difftool -revertdiffskyfile -fault 2
     464    $run = difftool -revertdiffskyfile
    462465    if ($DB:n == 0)
    463466      option DEFAULT

  • branches/eam_branches/ipp-20101205/ippTasks/dist.pro

    r29386 r30446  
    6161  end
    6262  task dist.revert
    63     active false
     63    active true
    6464  end
    6565end

  • branches/eam_branches/ipp-20101205/ippTasks/ipphosts.mhpcc.config

    r28500 r30446  
    2222  sky09 STR  ipp014
    2323  sky10 STR  ipp015
    24   sky11 STR  ipp016
     24  sky11 STR  ipp006
    2525  sky12 STR  ipp017
    2626  sky13 STR  ipp018
    … …  
    125125  XY26  STR  ipp027
    126126  XY27  STR  ipp028
    127   XY30  STR  ipp016
     127  XY30  STR  ipp006
    128128  XY31  STR  ipp044
    129129
    … …  
    204204  ota26  STR  ipp027
    205205  ota27  STR  ipp028
    206   ota30  STR  ipp016
     206  ota30  STR  ipp006
    207207  ota31  STR  ipp044
    208208                   
    … …  
    298298  XY11  STR  ipp039
    299299                   
    300   XY12  STR  ipp009
     300  XY12  STR  ipp005
    301301  XY13  STR  ipp024
    302302  XY14  STR  ipp010
    … …  
    305305  XY16  STR  ipp011
    306306  XY17  STR  ipp026
    307   XY20  STR  ipp012
     307  XY20  STR  ipp006
    308308  XY21  STR  ipp041
    309309                   
    310310  XY22  STR  ipp013
    311311  XY23  STR  ipp042
    312   XY24  STR  ipp014
     312  XY24  STR  ipp007
    313313  XY25  STR  ipp043
    314314                   
    315315  XY26  STR  ipp027
    316316  XY27  STR  ipp028
    317   XY30  STR  ipp016
     317  XY30  STR  ipp006
    318318  XY31  STR  ipp044
    319319                   
    … …  
    358358  XY72  STR  ipp052
    359359                   
    360   XY73  STR  ipp053
     360  XY73  STR  ipp007
    361361  XY74  STR  ipp015
    362362  XY75  STR  ipp025
    … …  
    371371  sky02 STR  ipp007
    372372  sky03 STR  ipp008
    373   sky04 STR  ipp009
     373  sky04 STR  ipp005
    374374  sky05 STR  ipp010
    375375  sky06 STR  ipp011
    376   sky07 STR  ipp012
     376  sky07 STR  ipp006
    377377  sky08 STR  ipp013
    378   sky09 STR  ipp014
     378  sky09 STR  ipp007
    379379  sky10 STR  ipp015
    380   sky11 STR  ipp016
     380  sky11 STR  ipp006
    381381  sky12 STR  ipp017
    382382  sky13 STR  ipp018
    … …  
    414414  sky45 STR  ipp051
    415415  sky46 STR  ipp052
    416   sky47 STR  ipp053
     416  sky47 STR  ipp007
    417417  sky48 STR  ipp023
    418418  sky49 STR  ipp024
    … …  
    445445  sky76 STR  ipp051
    446446  sky77 STR  ipp052
    447   sky78 STR  ipp053
    448 END
     447  sky78 STR  ipp006
     448END

  • branches/eam_branches/ipp-20101205/ippTasks/nightly_science.pro

    r30102 r30446  
    77    ns.detrends.off
    88    ns.dqstats.on
    9 #    ns.sweetspot.on
    10 #    ns.registration.on
    11 #    ns.burntool.on
    12 #    ns.chips.on
    139    ns.stacks.on
    1410    ns.diffs.on
    … …  
    1915    ns.detrends.off
    2016    ns.dqstats.off
    21 #    ns.sweetspot.off
    22 #    ns.registration.off
    23 #    ns.burntool.off
    24 #    ns.chips.off
    2517    ns.stacks.off
    2618    ns.diffs.off
    … …  
    5345end
    5446
    55 # macro ns.sweetspot.on
    56 #   task ns.sweetspot.load
    57 #     active true
    58 #   end
    59 # end
    60 
    61 # macro ns.registration.on
    62 #   task ns.registration.load
    63 #     active true
    64 #   end
    65 # end
    66 
    67 # macro ns.burntool.on
    68 #   task ns.burntool.load
    69 #     active true
    70 #   end
    71 #   task ns.burntool.run
    72 #     active true
    73 #   end
    74 # end
    75 
    76 # macro ns.chips.on
    77 #   task ns.chips.load
    78 #     active true
    79 #   end
    80 #   task ns.chips.run
    81 #     active true
    82 #   end
    83 # end
    84 
    8547macro ns.stacks.on
    8648  task ns.stacks.load
    … …  
    9153  end
    9254  task ns.stacks.confirm
    93     active true
     55    active false
    9456  end
    9557end
    … …  
    12284end
    12385
    124 # macro ns.sweetspot.off
    125 #   task ns.sweetspot.load
    126 #     active false
    127 #   end
    128 # end
    129 
    130 # macro ns.registration.off
    131 #   task ns.registration.load
    132 #     active false
    133 #   end
    134 # end
    135 
    136 # macro ns.burntool.off
    137 #   task ns.burntool.load
    138 #     active false
    139 #   end
    140 #   task ns.burntool.run
    141 #     active false
    142 #   end
    143 # end
    144 
    145 # macro ns.chips.off
    146 #   task ns.chips.load
    147 #     active false
    148 #   end
    149 #   task ns.chips.run
    150 #     active false
    151 #   end
    152 # end
    153 
    15486macro ns.stacks.off
    15587  task ns.stacks.load
    … …  
    174106end
    175107
    176 # $ns_regPAGE = 0
    177 # $ns_burnPAGE = 0
    178 # $ns_RburnPAGE = 0
    179 # $ns_RburnCELL = 0
    180 # $ns_chipPAGE = 0
    181 # $ns_RchipPAGE = 0
     108macro ns.stacks.confirm.on
     109  task ns.stacks.confirm
     110    active true
     111  end
     112end
     113
     114macro ns.stacks.confirm.off
     115  task ns.stacks.confirm
     116    active false
     117  end
     118end
     119
     120
    182121$ns_stackPAGE = 0
    183122$ns_RstackPAGE = 0
    … …  
    187126
    188127book init nsData
     128book init nsDiffs
     129book init nsStacks
    189130#
    190131# Macros to control the book.
    191132#
    192133macro ns.add.date
     134  if ($0 != 2)
     135    echo "USAGE: ns.add.date (YYYY-MM-DD)"
     136    break
     137  end
     138
    193139   book newpage nsData $1
    194140   book setword nsData $1 nsState NEW
    195    book setword nsData $1 nsStackState TOWARP
    196    book setword nsData $1 nsDiffState TOWARP
     141   book setword nsData $1 dbname $DB:0
     142
     143   book newpage nsDiffs $1
     144   book setword nsDiffs $1 nsDiffState TOWARP
     145   book setword nsDiffs $1 nsObservingState UNKNOWN
     146
     147   book newpage nsStacks $1
     148   book setword nsStacks $1 nsStackState TOWARP
     149
    197150end
    198151
    … …  
    202155       book getpage nsData $i -var date
    203156       book getword nsData $date nsState -var state
    204        book getword nsData $date nsStackState -var Sstate
    205        book getword nsData $date nsDiffState -var Dstate
    206        book getword nsData $date nsObservingState -var Ostate
     157       book getword nsStacks $date nsStackState -var Sstate
     158       book getword nsDiffs $date nsDiffState -var Dstate
     159       book getword nsDiffs $date nsObservingState -var Ostate
    207160       echo $date $state $Sstate $Dstate $Ostate
    208161    end
    … …  
    216169    book setword nsData $1 nsState $2
    217170end
    218 
    219171
    220172
    … …  
    236188    book newpage nsData $today
    237189    book setword nsData $today nsState NEW
    238     book setword nsData $today nsStackState TOWARP
    239     book setword nsData $today nsDiffState TOWARP
     190    book newpage nsStacks $today
     191    book setword nsStacks $today nsStackState TOWARP
     192    book newpage nsDiffs $today
     193    book setword nsDiffs $today nsDiffState TOWARP
    240194    book setword nsData $today dbname $DB:0
    241195
    242196#    command echo $today
    243197   $run = nightly_science.pl --clean_old --date $today --dbname $DB:0
    244     add_standard_args $run
     198#    add_standard_args run
    245199    command $run
    246200  end
    … …  
    279233
    280234    $run = nightly_science.pl --queue_detrends --date $today --dbname $DB:0
    281      add_standard_args $run
     235#     add_standard_args $run
    282236     command $run
    283237  end
    … …  
    317271
    318272    $run = nightly_science.pl --queue_dqstats --date $today --dbname $DB:0
    319      add_standard_args $run
     273#     add_standard_args $run
    320274     command $run
    321275
    … …  
    342296#
    343297task              ns.stacks.load
     298  active          false
    344299  host            local
    345300  periods         -poll $LOADPOLL
    … …  
    353308
    354309     book getpage nsData $ns_stackPAGE -var date
    355      book getword nsData $date nsStackState -var ns_STATE
    356      book getword nsData $date dbname -var $DBNAME
     310     book getword nsStacks $date nsStackState -var ns_STATE
     311     book getword nsDiffs $date nsDiffState -var ns_diff_STATE
     312     book getword nsData $date dbname -var DBNAME
    357313     book npages nsData -var Npages
    358314
    … …  
    367323     if ("$ns_STATE" != "TOWARP") break
    368324
    369      $run = nightly_science.pl --check_stacks --date $date
    370      add_standard_args $run
     325     $run = nightly_science.pl --check_stacks --date $date --dbname $DBNAME
     326#     add_standard_args $run
    371327     command $run
    372328
    … …  
    376332  task.exit   0
    377333#    convert 'stdout' to book format
    378     book delpage nsData $options:0
    379     ipptool2book stdout nsData -uniq -key date
    380 
    381     book getword nsData $options:0 nsNmacros -var ns_Nmacros
     334    book delpage nsStacks $options:0
     335    ipptool2book stdout nsStacks -uniq -key date
     336
     337    book getword nsStacks $options:0 nsNmacros -var ns_Nmacros
    382338    if ("$ns_Nmacros" != "NULL")
    383339        for i 0 $ns_Nmacros
    384340            sprintf macroName "ns%dMacro" $i
    385             book getword nsData $options:0 $macroName -var macroCmd
     341            book getword nsStacks $options:0 $macroName -var macroCmd
    386342            $macroCmd
    387343        end
    … …  
    422378
    423379     book getpage nsData $ns_RstackPAGE -var date
    424      book getword nsData $date nsStackState -var ns_STATE
    425      book getword nsData $date dbname -var $DBNAME
     380     book getword nsStacks $date nsStackState -var ns_STATE
     381     book getword nsDiffs $date nsDiffState -var ns_diff_STATE
     382     book getword nsData $date dbname -var DBNAME
    426383     book npages nsData -var Npages
    427384
    … …  
    434391     option $date
    435392
    436      if (("$ns_STATE" != "QUEUESTACKS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break
    437      $run = nightly_science.pl --queue_stacks --date $date
    438      add_standard_args $run
     393     if (("$ns_STATE" != "STACKING")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break
     394     $run = nightly_science.pl --queue_stacks --date $date --dbname $DBNAME
     395#     add_standard_args $run
    439396     command $run
    440397   end
    … …  
    442399  task.exit   0
    443400#    convert 'stdout' to book format
    444     book delpage nsData $options:0
     401    book delpage nsStacks $options:0
    445402# We've queued up stacking. We're done with this date, so don't reload the page.
    446     ipptool2book stdout nsData -uniq -key date
    447 
    448     book getword nsData $options:0 nsNmacros -var ns_Nmacros
     403    ipptool2book stdout nsStacks -uniq -key date
     404
     405    book getword nsStacks $options:0 nsNmacros -var ns_Nmacros
    449406    if ("$ns_Nmacros" != "NULL")
    450407        for i 0 $ns_Nmacros
    451408            sprintf macroName "ns%dMacro" $i
    452             book getword nsData $options:0 $macroName -var macroCmd
     409            book getword nsStacks $options:0 $macroName -var macroCmd
    453410            $macroCmd
    454411        end
    … …  
    477434#
    478435task              ns.stacks.confirm
     436  active          false
    479437  host            local
    480438  periods         -poll $LOADPOLL
    … …  
    488446
    489447     book getpage nsData $ns_CstackPAGE -var date
    490      book getword nsData $date nsStackState -var ns_STATE
    491      book getword nsData $date dbname -var $DBNAME
     448     book getword nsStacks $date nsStackState -var ns_STATE
     449     book getword nsData $date dbname -var DBNAME
    492450     book npages nsData -var Npages
    493451
    … …  
    501459
    502460     if ("$ns_STATE" != "STACKING") break
    503      $run = nightly_science.pl --confirm_stacks --date $date
    504      add_standard_args $run
     461     $run = nightly_science.pl --confirm_stacks --date $date --dbname $DBNAME
     462#     add_standard_args $run
    505463     command $run
    506464   end
    … …  
    508466  task.exit   0
    509467#    convert 'stdout' to book format
    510     book delpage nsData $options:0
    511     ipptool2book stdout nsData -uniq -key date
    512 
    513     book getword nsData $options:0 nsNmacros -var ns_Nmacros
     468    book delpage nsStacks $options:0
     469    ipptool2book stdout nsStacks -uniq -key date
     470
     471    book getword nsStacks $options:0 nsNmacros -var ns_Nmacros
    514472    if ("$ns_Nmacros" != "NULL")
    515473        for i 0 $ns_Nmacros
    516474            sprintf macroName "ns%dMacro" $i
    517             book getword nsData $options:0 $macroName -var macroCmd
     475            book getword nsStacks $options:0 $macroName -var macroCmd
    518476            $macroCmd
    519477        end
    … …  
    543501#
    544502task              ns.diffs.load
     503  active          false
    545504  host            local
    546505  periods         -poll $LOADPOLL
    … …  
    554513
    555514     book getpage nsData $ns_diffPAGE -var date
    556      book getword nsData $date nsDiffState -var ns_STATE
    557      book getword nsData $date dbname -var $DBNAME
     515     book getword nsDiffs $date nsDiffState -var ns_STATE
     516     book getword nsStacks $date nsStackState -var ns_stack_STATE
     517     book getword nsData $date dbname -var DBNAME
    558518     book npages nsData -var Npages
    559519
    … …  
    567527
    568528     if ("$ns_STATE" != "TOWARP") break
    569      $run = nightly_science.pl --check_diffs --date $date
    570      add_standard_args $run
     529     $run = nightly_science.pl --check_diffs --date $date --dbname $DBNAME
     530#     add_standard_args $run
    571531     command $run
    572532   end
    … …  
    575535  task.exit   0
    576536#    convert 'stdout' to book format
    577     book delpage nsData $options:0
    578     ipptool2book stdout nsData -uniq -key date
    579 
    580     book getword nsData $options:0 nsNmacros -var ns_Nmacros
     537    book delpage nsDiffs $options:0
     538    ipptool2book stdout nsDiffs -uniq -key date
     539
     540    book getword nsDiffs $options:0 nsNmacros -var ns_Nmacros
    581541    if ("$ns_Nmacros" != "NULL")
    582542        for i 0 $ns_Nmacros
    583543            sprintf macroName "ns%dMacro" $i
    584             book getword nsData $options:0 $macroName -var macroCmd
     544            book getword nsDiffs $options:0 $macroName -var macroCmd
    585545            $macroCmd
    586546        end
    … …  
    613573  periods         -poll $LOADPOLL
    614574  periods         -exec $LOADEXEC
    615   periods         -timeout 120
     575  periods         -timeout 900
    616576  npending        1
    617577
    … …  
    621581
    622582     book getpage nsData $ns_RdiffPAGE -var date
    623      book getword nsData $date nsDiffState -var ns_STATE
    624      book getword nsData $date dbname -var $DBNAME
     583     book getword nsDiffs $date nsDiffState -var ns_STATE
     584     book getword nsStacks $date nsStackState -var ns_stack_STATE
     585     book getword nsData $date dbname -var DBNAME
    625586     book npages nsData -var Npages
    626587
    … …  
    633594     option $date
    634595
    635      if (("$ns_STATE" != "QUEUEDIFFS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break
    636      $run = nightly_science.pl --queue_diffs --date $date
    637      add_standard_args $run
     596     if (("$ns_STATE" != "DIFFING")&&("$ns_STATE" != "QUEUEDIFFS")&&("$ns_STATE" != "TOWARP")&&("$ns_STATE" != "FORCETOWARP")) break
     597     $run = nightly_science.pl --queue_diffs --date $date --dbname $DBNAME
     598#     add_standard_args $run
    638599     command $run
    639600   end
    … …  
    641602  task.exit   0
    642603#    convert 'stdout' to book format
    643     book delpage nsData $options:0
     604    book delpage nsDiffs $options:0
    644605# We've queued up diffing. We're done with this date, so don't reload the page.
    645     ipptool2book stdout nsData -uniq -key date
    646 
    647     book getword nsData $options:0 nsNmacros -var ns_Nmacros
     606    ipptool2book stdout nsDiffs -uniq -key date
     607
     608    book getword nsDiffs $options:0 nsNmacros -var ns_Nmacros
    648609    if ("$ns_Nmacros" != "NULL")
    649610        for i 0 $ns_Nmacros
    650611            sprintf macroName "ns%dMacro" $i
    651             book getword nsData $options:0 $macroName -var macroCmd
     612            book getword nsDiffs $options:0 $macroName -var macroCmd
    652613            $macroCmd
    653614        end

  • branches/eam_branches/ipp-20101205/ippTasks/pstamp.pro

    r29719 r30446  
    258258        book getword pstampRequest $pageName ds_outProduct -var PRODUCT
    259259        book getword pstampRequest $pageName outdir -var OUTDIR
     260        book getword pstampRequest $pageName need_magic -var NEED_MAGIC
    260261        book getword pstampRequest $pageName label -var LABEL
    261262
    … …  
    263264
    264265        $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --outdir $OUTDIR --label $LABEL --redirect-output
     266
     267        if ($NEED_MAGIC != 0)
     268            $run = $run --need_magic
     269        end
    265270
    266271        add_standard_args run

  • branches/eam_branches/ipp-20101205/ippTasks/publish.pro

    r30102 r30446  
    7979    end
    8080    add_poll_args run
     81    add_poll_labels run
    8182    command $run
    8283  end
    … …  
    131132    book getword publishRun $pageName dbname -var DBNAME
    132133    book getword publishRun $pageName output_format -var OUTPUT_FORMAT
     134    book getword publishRun $pageName need_magic -var NEED_MAGIC
    133135
    134136    stdout $LOGDIR/publish.run.log
    … …  
    139141
    140142    $run = publish_file.pl --pub_id $PUB_ID --camera $CAMERA --workdir $WORKDIR --product $PRODUCT --stage $STAGE --stage_id $STAGE_ID --output_format $OUTPUT_FORMAT --redirect-output
     143
     144    if ("$NEED_MAGIC" == "T")
     145        $run = $run --need-magic
     146    end
     147
    141148    add_standard_args run
    142149

  • branches/eam_branches/ipp-20101205/ippTasks/register.pro

    r30102 r30446  
    99book init regPendingExp
    1010book init regPendingBurntoolImfile
     11book init regDates
    1112
    1213macro register.reset
    … …  
    1415  book init regPendingExp
    1516  book init regPendingBurntoolImfile
     17  book init regDates
    1618end
    1719
    … …  
    2123  book listbook regPendingExp
    2224  book listbook regPendingBurntoolImfile
     25  book listbook regDates
    2326end
    2427
    … …  
    3033    active true
    3134  end
     35  task register.imfile.revert
     36    active false
     37  end
    3238  task register.exp.load
    3339    active true
    … …  
    5157    active false
    5258  end
     59  task register.imfile.revert
     60    active false
     61  end
    5362  task register.exp.load
    5463    active false
    … …  
    6574end
    6675
     76macro register.revert.on
     77  task register.imfile.revert
     78    active true
     79  end
     80end
     81
     82macro register.revert.off
     83  task register.imfile.revert
     84    active false
     85  end
     86end
     87
     88macro burntool.on
     89  task register.burntool.load
     90    active true
     91  end
     92  task register.burntool.run
     93    active true
     94  end
     95end
     96
     97macro burntool.off
     98  task register.burntool.load
     99    active false
     100  end
     101  task register.burntool.run
     102    active false
     103  end
     104end
     105
     106macro register.add.date
     107  if ($0 != 2)
     108    echo "USAGE: register.add.date (YYYY-MM-DD)"
     109    break
     110  end
     111
     112   book newpage regDates $1
     113   book setword regDates $1 nsState NEW
     114   book setword regDates $1 dbname $DB:0
     115end
     116
     117
    67118$valid_burntool_value = 14
     119$sunrise = 17:30:00
     120$sunset  = 03:30:00
    68121
    69122macro replace.survey.values
    … …  
    147200$regPendingImfile_DB = 0
    148201$regPendingBurntoolImfile_DB = 0
     202$regRevertImfile_DB = 0
    149203$regPendingExp_DB = 0
    150204
    … …  
    265319    # XXX register_imfile.pl differs from the standard script : it does not have an 'outroot' argument, and it does not take '--redirect'
    266320    $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI --logfile $logfile --bytes $BYTES --md5sum $MD5SUM
     321    $run = $run --sunset $sunset --sunrise $sunrise
    267322    add_standard_args run
    268323
    … …  
    295350  end
    296351end
     352
     353task register.imfile.revert
     354  host         local
     355  periods      -poll 60.0
     356  periods      -exec 1800.0
     357  periods      -timeout 120.0
     358  npending     1
     359
     360  stdout       NULL
     361  stderr       $LOGDIR/register.imfile.revert.log
     362
     363  task.exec
     364    $run = regtool -revertprocessedimfile
     365    if ($DB:n == 0)
     366      option DEFAULT
     367    else
     368      option $DB:$regRevertImfile_DB
     369      $run = $run -dbname $DB:$regRevertImfile_DB
     370      $regRevertImfile_DB ++
     371      if ($regRevertImfile_DB >= $DB:n) set regRevertImfile_DB = 0
     372    end
     373    add_poll_args run
     374    command $run
     375  end
     376
     377  # success
     378  task.exit    0
     379  end
     380
     381  # locked list
     382  task.exit    default
     383    showcommand failure
     384  end
     385
     386  task.exit    crash
     387    showcommand crash
     388  end
     389
     390  # operation times out
     391  task.exit    timeout
     392    showcommand timeout
     393  end
     394end
     395
    297396
    298397# select exposures ready for register_exp.pl
    … …  
    463562  # select entried from the current DB; cycle to the next DB, if it exists
    464563  task.exec
     564     book npages regDates -var Npages
     565     if ($Npages == 0)
     566       $today = `date -u +%Y-%m-%d`
     567     else
     568       book getpage regDates $reg_datePAGE -var today
     569       $reg_datePAGE ++
     570       if ($reg_datePAGE >= $Npages) set reg_datePAGE = 0
     571     end
     572
    465573    $run = regtool -pendingburntoolimfile
    466     $today = `date -u +%Y-%m-%d`
     574
    467575# debugging purposes
    468 #    $today = "2010-12-06"
    469     $run = $run -date $today -valid_burntool $valid_burntool_value
     576#   $today = "2010-12-25"
     577    $dateobs_begin = $today\T$sunset
     578    $dateobs_end   = $today\T$sunrise
     579    $run = $run -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end -valid_burntool $valid_burntool_value
    470580    if ($DB:n == 0)
    471581      option DEFAULT
    … …  
    574684end
    575685
    576    
     686task   register.initday.load
     687  host            local
     688  periods         -poll $LOADPOLL
     689  periods         -exec $LOADEXEC
     690  periods         -timeout 30
     691  trange          0:00:00 1:00:00 -nmax 1
     692  npending        1
     693
     694  task.exec
     695    $today = `date -u +%Y-%m-%d`
     696    book newpage regDates $today
     697    book setword regDates $today nsState NEW
     698    book setword regDates $today dbname $DB:0
     699
     700    command true
     701  end
     702
     703  task.exit       0
     704    # nothing to do here?
     705  end
     706  # locked list
     707  task.exit    default
     708    showcommand failure
     709  end
     710  task.exit    crash
     711    showcommand crash
     712  end
     713  #operation times out?
     714  task.exit    timeout
     715    showcommand timeout
     716  end
     717end
     718
    577719   
    578720 

  • branches/eam_branches/ipp-20101205/ippTasks/survey.pro

    r29768 r30446  
    259259# user functions to manipulate destreak labels
    260260macro survey.add.destreak
    261   if ($0 != 3)
    262     echo "USAGE: survey.add.destreak (label) (workdir base)"
     261  if ($0 != 4)
     262    echo "USAGE: survey.add.destreak (label) (workdir base) (recovery root)"
    263263    break
    264264  end
    265265  book newpage SURVEY_DESTREAK $1
    266266  book setword SURVEY_DESTREAK $1 WORKDIR $2
     267  book setword SURVEY_DESTREAK $1 RECOVERYROOT $3
    267268  book setword SURVEY_DESTREAK $1 STATE PENDING
    268269end
    … …  
    366367# user functions to manipulate publish labels
    367368macro survey.add.publish
    368   if ($0 != 3)
    369     echo "USAGE: survey.add.publish (label) (client_id)"
     369  if ($0 != 5)
     370    echo "USAGE: survey.add.publish (tag) (label) (client_id) (comment)"
    370371    break
    371372  end
    372373  book newpage SURVEY_PUBLISH $1
    373   book setword SURVEY_PUBLISH $1 CLIENT_ID $2
     374  book setword SURVEY_PUBLISH $1 LABEL $2
     375  book setword SURVEY_PUBLISH $1 CLIENT_ID $3
     376  book setword SURVEY_PUBLISH $1 COMMENT $4
    374377  book setword SURVEY_PUBLISH $1 STATE PENDING
    375378end
    … …  
    821824    book setword SURVEY_DESTREAK $label STATE DONE
    822825    book getword SURVEY_DESTREAK $label WORKDIR -var workdir
     826    book getword SURVEY_DESTREAK $label RECOVERYROOT -var recoveryroot
    823827 
    824828    $run = magic_destreak_defineruns.pl --label $label --workdir $workdir/$label
     829    if ("$recoveryroot" != "NULL")
     830        $run = $run --recoveryroot $recoveryroot/$label
     831    end
    825832
    826833    if ($DB:n == 0)
    … …  
    10841091    endif
    10851092
    1086     book getpage SURVEY_PUBLISH 0 -var label -key STATE NEW
    1087     if ("$label" == "NULL")
    1088       # All labels have been done --- reset
    1089       # echo "Resetting labels"
     1093    # survey.publish allows multiple entries per label.
     1094    # The key is called tag and must be unique
     1095    book getpage SURVEY_PUBLISH 0 -var tag -key STATE NEW
     1096    if ("$tag" == "NULL")
     1097      # All tags have been done --- reset
     1098      # echo "Resetting tags"
    10901099      for i 0 $N
    1091         book getpage SURVEY_PUBLISH $i -var label
    1092         book setword SURVEY_PUBLISH $label STATE NEW
     1100        book getpage SURVEY_PUBLISH $i -var tag
     1101        book setword SURVEY_PUBLISH $tag STATE NEW
    10931102      end
    1094       book getpage SURVEY_PUBLISH 0 -var label -key STATE NEW
     1103      book getpage SURVEY_PUBLISH 0 -var tag -key STATE NEW
    10951104
    10961105      # Select different database
    … …  
    10991108    end
    11001109
    1101     book setword SURVEY_PUBLISH $label STATE DONE
    1102     book getword SURVEY_PUBLISH $label CLIENT_ID -var client_id
     1110    book setword SURVEY_PUBLISH $tag STATE DONE
     1111    book getword SURVEY_PUBLISH $tag LABEL -var label
     1112    book getword SURVEY_PUBLISH $tag CLIENT_ID -var client_id
     1113    book getword SURVEY_PUBLISH $tag COMMENT -var comment
    11031114 
    11041115    $run = pubtool -definerun -label $label -client_id $client_id
     1116
     1117    if ("$comment" != "NULL")
     1118        $run = $run -comment $comment
     1119    end
    11051120
    11061121    if ($DB:n == 0)
    … …  
    11111126    end
    11121127   
    1113     #echo $run
     1128    echo $run
     1129
    11141130    command $run
     1131   
    11151132  end
    11161133

  • branches/eam_branches/ipp-20101205/ippTools/share/Makefile.am

    r30102 r30446  
    246246        magictool_exposure.sql \
    247247        magictool_setfull.sql \
     248        magictool_setgotocleaned.sql \
     249        magictool_tocleanup.sql \
    248250        magicdstool_clearstatefaults.sql \
    249251        magicdstool_change_file_data_state.sql \
    … …  
    333335        regtool_updatebyqueryimfile.sql \
    334336        regtool_pendingburntoolimfile.sql \
     337        regtool_checkstatus.sql \
    335338        stacktool_associationdefine_select.sql \
    336339        stacktool_definebyquery_insert.sql \

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_pending_SSdiff.sql

    r27027 r30446  
    1717    diffSkyfile.quality,
    1818    distRun.no_magic,
    19     0 AS magicked
     19    0 AS magicked,
     20    IFNULL(Label.priority, 10000) AS priority
    2021FROM distRun
    2122JOIN distTarget USING(target_id, stage, clean)
    … …  
    4243    ON distRun.dist_id = distComponent.dist_id
    4344    AND diffSkyfile.skycell_id = distComponent.component
     45LEFT JOIN Label ON distRun.label = Label.label
    4446WHERE
    4547    distRun.state = 'new'
    … …  
    4850    AND ((diffRun.magicked > 0) OR distRun.no_magic)
    4951    AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned'))
     52    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_pending_camera.sql

    r29509 r30446  
    1515    camProcessedExp.quality,
    1616    distRun.no_magic,
    17     chipRun.magicked
     17    chipRun.magicked,
     18    IFNULL(Label.priority, 10000) AS priority
    1819FROM distRun
    1920JOIN distTarget USING(target_id, stage, clean)
    … …  
    2122JOIN camProcessedExp USING(cam_id)
    2223JOIN chipRun USING(chip_id)
    23 -- JOIN chipProcessedImfile USING(exp_id, chip_id)
    2424JOIN rawExp using(exp_id)
    2525LEFT JOIN distComponent
    2626    ON distRun.dist_id = distComponent.dist_id
    27 --    AND chipProcessedImfile.class_id = distComponent.component
     27LEFT JOIN Label
     28    ON distRun.label = Label.label
    2829WHERE
    2930    distRun.state = 'new'
    3031    AND distRun.stage = 'camera'
    3132    AND distComponent.dist_id IS NULL
    32     AND (((clean OR (chipRun.magicked > 0)) AND (camRun.magicked > 0)) OR distRun.no_magic)
     33    AND (((clean OR (chipRun.magicked != 0)) AND (camRun.magicked > 0)) OR distRun.no_magic)
    3334    AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned'))
     35    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_pending_chip.sql

    r26567 r30446  
    1515    chipProcessedImfile.quality,
    1616    distRun.no_magic,
    17     chipProcessedImfile.magicked
     17    chipProcessedImfile.magicked,
     18    IFNULL(Label.priority, 10000) AS priority
    1819FROM distRun
    1920JOIN distTarget USING(target_id, stage, clean)
    … …  
    2425    ON distRun.dist_id = distComponent.dist_id
    2526    AND chipProcessedImfile.class_id = distComponent.component
     27LEFT JOIN Label
     28    ON distRun.label = Label.label
    2629WHERE
    2730    distRun.state = 'new'
    … …  
    3033    AND ((chipRun.magicked > 0) OR distRun.no_magic)
    3134    AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned'))
     35    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_pending_chip_bg.sql

    r28730 r30446  
    1515    0 AS quality,
    1616    distRun.no_magic,
    17     chipBackgroundImfile.magicked
     17    chipBackgroundImfile.magicked,
     18    IFNULL(Label.priority, 10000) AS priority
    1819FROM distRun
    1920JOIN distTarget USING(target_id, stage, clean)
    … …  
    2526    ON distRun.dist_id = distComponent.dist_id
    2627    AND chipBackgroundImfile.class_id = distComponent.component
     28LEFT JOIN Label ON distRun.label = Label.label
    2729WHERE
    2830    distRun.state = 'new'
    … …  
    3133    AND ((chipBackgroundRun.magicked > 0) OR distRun.no_magic)
    3234    AND (chipBackgroundRun.state = 'full')
     35    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_pending_diff.sql

    r26567 r30446  
    1717    diffSkyfile.quality,
    1818    distRun.no_magic,
    19     diffSkyfile.magicked
     19    diffSkyfile.magicked,
     20    IFNULL(Label.priority, 10000) AS priority
    2021FROM distRun
    2122JOIN distTarget USING(target_id, stage, clean)
    … …  
    3233    ON distRun.dist_id = distComponent.dist_id
    3334    AND diffSkyfile.skycell_id = distComponent.component
     35LEFT JOIN Label ON distRun.label = Label.label
    3436WHERE
    3537    distRun.state = 'new'
    … …  
    3840    AND ((diffRun.magicked > 0) OR distRun.no_magic)
    3941    AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned'))
     42    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_pending_fake.sql

    r27458 r30446  
    1515    0 as quality,
    1616    distRun.no_magic,
    17     0 as magicked
     17    0 as magicked,
     18    IFNULL(Label.priority, 10000) AS priority
    1819FROM distRun
    1920JOIN distTarget USING(target_id, stage, clean)
    … …  
    2627    ON distRun.dist_id = distComponent.dist_id
    2728    AND fakeProcessedImfile.class_id = distComponent.component
     29LEFT JOIN Label ON distRun.label = Label.label
    2830WHERE
    2931    distRun.state = 'new'
    3032    AND distRun.stage = 'fake'
    3133    AND distComponent.dist_id IS NULL
     34    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_pending_raw.sql

    r27457 r30446  
    1717    0 as quality,
    1818    distRun.no_magic,
    19     rawImfile.magicked
     19    rawImfile.magicked,
     20    IFNULL(Label.priority, 10000) AS priority
    2021FROM distRun
    2122JOIN distTarget USING(target_id, stage, clean)
    … …  
    2829    ON distRun.dist_id = distComponent.dist_id
    2930    AND rawImfile.class_id = distComponent.component
     31LEFT JOIN Label ON distRun.label = Label.label
    3032WHERE
    3133    distRun.state = 'new'
    … …  
    3436    AND distComponent.dist_id IS NULL
    3537    AND (rawExp.magicked OR distRun.no_magic)
     38    AND (Label.active OR Label.active IS NULL)
    3639UNION
    3740    -- raw stage alternate inputs
    … …  
    5457    0 as quality,
    5558    distRun.no_magic,
    56     rawImfile.magicked
     59    rawImfile.magicked,
     60    IFNULL(Label.priority, 10000) AS priority
    5761FROM distRun
    5862JOIN distTarget USING(target_id, stage, clean)
    … …  
    6872    ON distRun.dist_id = distComponent.dist_id
    6973    AND rawImfile.class_id = distComponent.component
     74LEFT JOIN Label ON distRun.label = Label.label
    7075WHERE
    7176    distRun.state = 'new'
    … …  
    7479    AND chipProcessedImfile.data_state = 'full'
    7580    AND chipProcessedImfile.magicked > 0
     81    AND (Label.active OR Label.active IS NULL)
    7682UNION
    7783    -- raw stage clean (dbinfo only)
    … …  
    9399    0 as quality,
    94100    distRun.no_magic,
    95     rawExp.magicked
     101    rawExp.magicked,
     102    IFNULL(Label.priority, 10000) AS priority
    96103FROM distRun
    97104JOIN distTarget USING(target_id, stage, clean)
    … …  
    99106LEFT JOIN distComponent
    100107    ON distRun.dist_id = distComponent.dist_id
     108LEFT JOIN Label ON distRun.label = Label.label
    101109WHERE
    102110    distRun.state = 'new'
    … …  
    104112    AND distRun.clean
    105113    AND distComponent.dist_id IS NULL
     114    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_pending_sky.sql

    r28959 r30446  
    1515    staticskyResult.quality,
    1616    1 AS no_magic,
    17     0 AS magicked
     17    0 AS magicked,
     18    IFNULL(Label.priority, 10000) AS priority
    1819FROM distRun
    1920JOIN distTarget USING(target_id, stage, clean)
    … …  
    4344    ON distRun.dist_id = distComponent.dist_id
    4445    AND stackRun.skycell_id = distComponent.component
     46LEFT JOIN Label ON distRun.label = Label.label
    4547WHERE
    4648    distRun.state = 'new'
    … …  
    4951    AND (staticskyRun.state = 'full' OR (distRun.clean AND staticskyRun.state = 'cleaned'))
    5052    AND (staticskyResult.fault = 0 AND staticskyResult.quality = 0)
     53    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_pending_stack.sql

    r25838 r30446  
    1717    stackSumSkyfile.quality,
    1818    1 AS no_magic,
    19     0 AS magicked
     19    0 AS magicked,
     20    IFNULL(Label.priority, 10000) AS priority
    2021FROM distRun
    2122JOIN distTarget USING(target_id, stage, clean)
    … …  
    4344    ON distRun.dist_id = distComponent.dist_id
    4445    AND stackRun.skycell_id = distComponent.component
     46LEFT JOIN Label ON distRun.label = Label.label
    4547WHERE
    4648    distRun.state = 'new'
    … …  
    4951    AND (stackRun.state = 'full' OR (distRun.clean AND stackRun.state = 'cleaned'))
    5052    AND (stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0)
     53    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_pending_warp.sql

    r26567 r30446  
    1515    warpSkyfile.quality,
    1616    distRun.no_magic,
    17     warpSkyfile.magicked
     17    warpSkyfile.magicked,
     18    IFNULL(Label.priority, 10000) AS priority
    1819FROM distRun
    1920JOIN distTarget USING(target_id, stage, clean)
    … …  
    2728    ON distRun.dist_id = distComponent.dist_id
    2829    AND warpSkyfile.skycell_id = distComponent.component
     30LEFT JOIN Label ON distRun.label = Label.label
    2931WHERE
    3032    distRun.state = 'new'
    … …  
    3335    AND ((warpRun.magicked > 0) OR distRun.no_magic)
    3436    AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned'))
     37    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_pending_warp_bg.sql

    r28742 r30446  
    1515    0 as quality,
    1616    distRun.no_magic,
    17     warpBackgroundSkyfile.magicked
     17    warpBackgroundSkyfile.magicked,
     18    IFNULL(Label.priority, 10000) AS priority
    1819FROM distRun
    1920JOIN distTarget USING(target_id, stage, clean)
    … …  
    2829    ON distRun.dist_id = distComponent.dist_id
    2930    AND warpBackgroundSkyfile.skycell_id = distComponent.component
     31LEFT JOIN Label ON distRun.label = Label.label
    3032WHERE
    3133    distRun.state = 'new'
    … …  
    3436    AND ((warpBackgroundRun.magicked > 0) OR distRun.no_magic)
    3537    AND (warpBackgroundRun.state = 'full')
     38    AND (Label.active OR Label.active IS NULL)

  • branches/eam_branches/ipp-20101205/ippTools/share/disttool_revertrun.sql

    r26266 r30446  
    11UPDATE distRun
    22SET distRun.fault = 0
    3 WHERE distRun.state = 'new'
     3WHERE (distRun.state = 'new' OR distRun.state = 'goto_cleaned')
    44    AND distRun.fault != 0

  • branches/eam_branches/ipp-20101205/ippTools/share/magicdstool_todestreak_camera.sql

    r29561 r30446  
    3838    AND magicDSRun.stage = 'camera'
    3939    AND camRun.state = 'full'
    40     AND chipRun.state = 'full'
    41     AND chipRun.magicked
     40    AND ((chipRun.state = 'full' AND chipRun.magicked > 0)
     41         OR (chipRun.state = 'cleaned' AND chipRun.magicked < 0))
    4242    AND camProcessedExp.fault = 0
    4343    AND camProcessedExp.quality = 0

  • branches/eam_branches/ipp-20101205/ippTools/share/magicdstool_torevert_diff.sql

    r27945 r30446  
    1010    outroot,
    1111    diffSkyfile.path_base,
     12    magicDSFile.backup_path_base,
     13    magicDSFile.recovery_path_base,
    1214    "NULL" AS cam_path_base,
    1315    CAST(diffRun.bothways AS SIGNED) AS bothways,

  • branches/eam_branches/ipp-20101205/ippTools/share/pstamptool_pendingreq.sql

    r27874 r30446  
    33    pstampRequest.*,
    44    pstampDataStore.outProduct AS ds_outProduct,
     5    IFNULL(pstampDataStore.need_magic, 1) AS need_magic,
    56    IFNULL(Label.priority, 10000) AS priority
    67FROM pstampRequest

  • branches/eam_branches/ipp-20101205/ippTools/share/pubtool_pending.sql

    r30102 r30446  
    1010        publishClient.workdir,
    1111        publishClient.output_format,
     12        publishClient.magicked AS need_magic,
    1213        diffRun.diff_id AS stage_id,
    1314        rawExp.camera,
    … …  
    3738        publishClient.workdir,
    3839        publishClient.output_format,
     40        publishClient.magicked AS need_magic,
    3941        camRun.cam_id AS stage_id,
    4042        rawExp.camera,
    … …  
    5961        publishClient.workdir,
    6062        publishClient.output_format,
     63        publishClient.magicked AS need_magic,
    6164        diffPhotRun.diff_phot_id AS stage_id,
    6265        rawExp.camera,

  • branches/eam_branches/ipp-20101205/ippTools/share/pxadmin_create_tables.sql

    r30102 r30446  
    13911391        uri VARCHAR(255),
    13921392        pollInterval INTEGER DEFAULT 60,
     1393        need_magic TINYINT,
    13931394        PRIMARY KEY(ds_id),
    13941395        KEY(ds_id)

  • branches/eam_branches/ipp-20101205/ippTools/share/regtool_checkburntoolimfile.sql

    r30102 r30446  
    11SELECT DISTINCT
    2        exp_name,
     2       summitExp.exp_name,
    33       rawImfile.uri,
    44       summitExp.dateobs AS registered,
    … …  
    1111       newExp.exp_id,
    1212       summitExp.exp_type,
    13        rawExp.dateobs,
    14        rawExp.obs_mode AS obs_mode,
    15        rawExp.object AS object,
     13       rawImfile.dateobs,
     14       rawImfile.obs_mode AS obs_mode,
     15       rawImfile.object AS object,
    1616       rawImfile.burntool_state AS burntool_state,
    1717       rawImfile.class_id,
     18       summitImfile.class_id AS summit_class_id,
    1819       (pzDownloadExp.state = 'stop') AS is_downloaded,
    1920       (rawImfile.burntool_state IS NOT NULL) AS is_registered
    2021       FROM
    21             summitExp
    22             LEFT JOIN pzDownloadExp USING(exp_name)
    23             LEFT JOIN newExp ON exp_name = tmp_exp_name
    24             LEFT JOIN rawExp USING(exp_id, exp_name)
    25             LEFT JOIN rawImfile USING(exp_name)
     22       summitExp
     23       JOIN summitImfile USING(exp_name)
     24       LEFT JOIN pzDownloadExp USING(exp_name)
     25       LEFT JOIN newExp ON summitExp.exp_name = tmp_exp_name
     26       LEFT JOIN newImfile ON (summitImfile.class_id = newImfile.tmp_class_id AND newExp.exp_id = newImfile.exp_id)
     27       LEFT JOIN rawExp ON rawExp.exp_id = newExp.exp_id
     28       LEFT JOIN rawImfile ON (rawImfile.exp_id = newExp.exp_id AND rawImfile.tmp_class_id = newImfile.tmp_class_id)
    2629       WHERE
    27              summitExp.dateobs >= '@DATE@T00:00:00'
    28              AND summitExp.dateobs <= '@DATE@T23:59:59'
    29              AND (rawImfile.class_id = '@CLASS_ID@' OR rawImfile.class_id IS NULL)
    30              AND (exp_name <= '@EXP_NAME@')                                                                     
     30             summitExp.dateobs >= '@DATEOBS_BEGIN@'
     31             AND summitExp.dateobs <= '@DATEOBS_END@'
     32             AND summitImfile.class_id = '@CLASS_ID@'
     33             AND (summitExp.exp_name <= '@EXP_NAME@')                                                                     
    3134ORDER BY summitExp.dateobs

  • branches/eam_branches/ipp-20101205/ippTools/share/regtool_pendingburntoolimfile.sql

    r30102 r30446  
    1212       newImfile.tmp_class_id,
    1313       summitExp.exp_type,
    14        rawExp.dateobs,
    15        rawExp.obs_mode AS obs_mode,
    16        rawExp.object AS object,
     14       rawImfile.dateobs,
     15       rawImfile.obs_mode AS obs_mode,
     16       rawImfile.object AS object,
    1717       rawImfile.burntool_state AS burntool_state,
    1818       rawImfile.class_id,
     19       summitImfile.class_id AS summit_class_id,
    1920       (pzDownloadExp.state = 'stop') AS is_downloaded,
    2021       (rawImfile.burntool_state IS NOT NULL) AS is_registered
    2122       FROM
    22             summitExp
    23             LEFT JOIN pzDownloadExp USING(exp_name)
    24             LEFT JOIN newExp ON exp_name = tmp_exp_name
    25             LEFT JOIN newImfile USING(exp_id)
    26             LEFT JOIN rawExp USING(exp_id, exp_name)
    27             LEFT JOIN rawImfile ON (rawImfile.exp_name = newExp.tmp_exp_name AND rawImfile.tmp_class_id = newImfile.tmp_class_id)
     23       summitExp
     24       JOIN summitImfile USING(exp_name)
     25       LEFT JOIN pzDownloadExp USING(exp_name)
     26       LEFT JOIN newExp ON summitExp.exp_name = tmp_exp_name
     27       LEFT JOIN newImfile ON (summitImfile.class_id = newImfile.tmp_class_id AND newExp.exp_id = newImfile.exp_id)
     28       LEFT JOIN rawExp ON rawExp.exp_id = newExp.exp_id
     29       LEFT JOIN rawImfile ON (rawImfile.exp_id = newExp.exp_id AND rawImfile.tmp_class_id = newImfile.tmp_class_id)
    2830       WHERE
    29              summitExp.dateobs >= '@DATE@T00:00:00'
    30              AND summitExp.dateobs <= '@DATE@T23:59:59'
    31 ORDER BY newImfile.tmp_class_id,summitExp.dateobs
     31             summitExp.dateobs >= '@DATEOBS_BEGIN@'
     32             AND summitExp.dateobs <= '@DATEOBS_END@'
     33ORDER BY summitImfile.class_id,summitExp.dateobs

  • branches/eam_branches/ipp-20101205/ippTools/src/disttool.c

    r29067 r30446  
    857857    psFree(where);
    858858
     859    psStringAppend(&query, "\nORDER BY priority DESC, dist_id");
     860
    859861    // treat limit == 0 as "no limit"
    860862    if (limit) {

  • branches/eam_branches/ipp-20101205/ippTools/src/magicdstool.c

    r30102 r30446  
    11151115    psString queryFile = NULL;
    11161116    if (state) {
    1117         if (! strcmp(state, "new") ) {
     1117        if (! strcmp(state, "new") || !strcmp(state, "goto_restored")) {
    11181118            queryFile = "magicdstool_revertdestreakedfile.sql";
    11191119        } else if (!strcmp(state, "update")) {
    … …  
    17241724    PXOPT_LOOKUP_STR(component, config->args, "-component", false, false);
    17251725    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
     1726    PXOPT_LOOKUP_STR(recoveryroot, config->args, "-set_recoveryroot", false, false);
    17261727
    17271728    psString query = pxDataGet("magicdstool_setfiletoupdate.sql");
    … …  
    17341735    if (label) {
    17351736        psStringAppend(&setHook, "\n , magicDSRun.label = '%s'", label);
     1737    }
     1738    if (recoveryroot) {
     1739        psStringAppend(&setHook, "\n , magicDSRun.recoveryroot = '%s'", recoveryroot);
    17361740    }
    17371741

  • branches/eam_branches/ipp-20101205/ippTools/src/magicdstoolConfig.c

    r30102 r30446  
    230230    psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL);
    231231    psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "set new label", NULL);
     232    psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-set_recoveryroot", 0, "define new recovery directory", NULL);
    232233
    233234    // -destreakedfile

  • branches/eam_branches/ipp-20101205/ippTools/src/magictool.c

    r29625 r30446  
    4949static bool censorrunMode(pxConfig *config);
    5050static bool exposureMode(pxConfig *config);
     51static bool setgotocleanedMode(pxConfig *config);
     52static bool tocleanupMode(pxConfig *config);
     53static bool setworkdirstateMode(pxConfig *config);
    5154
    5255static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString);
    … …  
    8992        MODECASE(MAGICTOOL_MODE_CENSORRUN,           censorrunMode);
    9093        MODECASE(MAGICTOOL_MODE_EXPOSURE,            exposureMode);
     94        MODECASE(MAGICTOOL_MODE_SETGOTOCLEANED,      setgotocleanedMode);
     95        MODECASE(MAGICTOOL_MODE_TOCLEANUP,           tocleanupMode);
     96        MODECASE(MAGICTOOL_MODE_SETWORKDIRSTATE,     setworkdirstateMode);
    9197        default:
    9298            psAbort("invalid option (this should not happen)");
    … …  
    829835    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    830836
    831     // look for "inputs" that need to processed
    832     psString query = pxDataGet("magictool_toprocess_inputs.sql");
    833     if (!query) {
    834         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
    835         return false;
    836     }
    837 
    838837    psString whereString = NULL;
    839838    if (psListLength(where->list)) {
    … …  
    844843    psFree(where);
    845844
    846     psStringAppend(&query, "\nORDER BY priority DESC, magic_id");
    847 
    848     // treat limit == 0 as "no limit"
    849     if (limit) {
    850         // cut limit in half
    851         // hack to prevent pending leaf nodes from blocking branch nodes
    852         psString limitString = psDBGenerateLimitSQL((limit + 1) / 2);
    853         psStringAppend(&query, " %s", limitString);
    854         psFree(limitString);
    855     }
    856 
    857     if (!p_psDBRunQueryF(config->dbh, query, whereString, whereString)) {
    858         psError(PS_ERR_UNKNOWN, false, "database error");
    859         psFree(whereString);
    860         psFree(query);
    861         return false;
    862     }
    863     psFree(query);
    864 
    865     psArray *output = p_psDBFetchResult(config->dbh);
    866     if (!output) {
    867         psErrorCode err = psErrorCodeLast();
    868         switch (err) {
    869             case PS_ERR_DB_CLIENT:
    870                 psError(PXTOOLS_ERR_SYS, false, "database error");
    871             case PS_ERR_DB_SERVER:
    872                 psError(PXTOOLS_ERR_PROG, false, "database error");
    873             default:
    874                 psError(PXTOOLS_ERR_PROG, false, "unknown error");
    875         }
    876 
    877         return false;
    878     }
    879     if (!psArrayLength(output)) {
    880         psTrace("magictool", PS_LOG_INFO, "no rows found");
    881     }
    882 
    883     if (limit) {
    884         if (psArrayLength(output) >= limit) {
    885             // we've found enough (note that limit was applied to the query so '> limit' won't happen)
    886             // negative simple so the default is true
    887             if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
    888                 psError(PS_ERR_UNKNOWN, false, "failed to print array");
    889                 psFree(output);
    890                 return false;
    891             }
    892             psFree(output);
    893             return true;
    894         }
    895     }
    896 
    897     // look for tree nodes that need to be processed
     845    // First look for branch nodes that need to be processed.
     846    // These get priority over skycells because they are from runs
     847    // that are already in progress and there are fewer of them.
     848    // When we looked for skycells first we got starved.
    898849
    899850    // first find incomplete magicRuns
    900     query = pxDataGet("magictool_toprocess_runs.sql");
     851    psString query = pxDataGet("magictool_toprocess_runs.sql");
    901852    if (!query) {
    902853        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
    … …  
    904855    }
    905856
    906     // we limit the query even though it is cheap (only magic_id is selected)
    907 
    908     // XXX: if the first 1000 unfinished magicRuns have no ready nodes
    909     // that haven't faulted, later runs won't get returned even though
    910     // they have work to do. When we used a limit of 100 we actually ran
    911     // into this problem. Since we're using labels a limit of 1000 will
    912     // probably be ok.
     857    // Find outstanding magicRuns in new state.
     858    // We limit the query, but this is problematic. In practice do
     859    // we need to?
     860    // XXX: If the first 1000 magicRuns have no branch nodes ready
     861    // but higher runs do we they won't be noticed.
     862    // Perhaps have this limit be an argument.
    913863    {
    914864        psString limitString = psDBGenerateLimitSQL( 1000 );
    … …  
    940890        return false;
    941891    }
     892
    942893    if (!psArrayLength(magicRuns)) {
     894        // no pending magicRuns nothing to do
    943895        psTrace("magictool", PS_LOG_INFO, "no rows found");
    944         psFree(magicRuns);
    945896        return true;
    946897    }
    947 
    948     query = pxDataGet("magictool_toprocess_tree.sql");
     898    psArray *output = psArrayAllocEmpty(100);
     899
     900    {
     901        query = pxDataGet("magictool_toprocess_tree.sql");
     902        if (!query) {
     903            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
     904            return false;
     905        }
     906
     907        for (psS64 index = 0; index < psArrayLength(magicRuns); index++) {
     908            if (limit && (psArrayLength(output) >= limit)) {
     909                break;
     910            }
     911            bool status;
     912            psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id");
     913            if (!status) {
     914                psAbort("failed to lookup value for magic_id column");
     915            }
     916
     917            whereString = NULL;
     918            psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id);
     919            if (!p_psDBRunQueryF(config->dbh, query, whereString )) {
     920                psError(PS_ERR_UNKNOWN, false, "database error");
     921                psFree(whereString);
     922                psFree(query);
     923                return false;
     924            }
     925            psFree(whereString);
     926            psArray *magicTree = p_psDBFetchResult(config->dbh);
     927            if (!magicTree) {
     928                psErrorCode err = psErrorCodeLast();
     929                switch (err) {
     930                    case PS_ERR_DB_CLIENT:
     931                        psError(PXTOOLS_ERR_SYS, false, "database error");
     932                    case PS_ERR_DB_SERVER:
     933                        psError(PXTOOLS_ERR_PROG, false, "database error");
     934                    default:
     935                        psError(PXTOOLS_ERR_PROG, false, "unknown error");
     936                }
     937
     938                return false;
     939            }
     940            psS64 length = psArrayLength(magicTree);
     941            if (!length) {
     942                psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id);
     943                psFree(magicTree);
     944                continue;
     945            }
     946
     947            psHash *forest = psHashAlloc(length);
     948
     949            // convert the array of metadata into a pxTree structure
     950            for (long i = 0; i < length; i++) {
     951                bool status;
     952                psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
     953                if (!status) {
     954                    psAbort("failed to lookup value for node column");
     955                }
     956
     957                psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep");
     958                if (!status) {
     959                    psAbort("failed to lookup value for dep column");
     960                }
     961
     962                pxTreeBuilder(forest, node, dep, magicTree->data[i]);
     963
     964            }
     965
     966            // find the root of the tree
     967            pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root"));
     968            psFree(forest);
     969            //    pxTreePrint(stdout, root);
     970
     971            // crawl through the tree and looking for nodes with children that are all
     972            // "done"
     973            pxTreeCrawl(root, findReadyNodes, output);
     974            psFree(root);
     975            psFree(magicTree);
     976        }
     977
     978        int len = psArrayLength(output);
     979        if (len) {
     980            if (limit) {
     981                if (limit < psArrayLength(output)) {
     982                    // truncate the array
     983                    long arrayLength = psArrayLength(output);
     984                    for (long i = arrayLength - 1; i >= limit; i--) {
     985                        psArrayRemoveIndex(output, i);
     986                    }
     987                    // negative simple so the default is true
     988                    if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
     989                        psError(PS_ERR_UNKNOWN, false, "failed to print array");
     990                        psFree(output);
     991                        return false;
     992                    }
     993                    psFree(output);
     994                    return true;
     995                } else {
     996                    limit -= len;
     997                }
     998            }
     999        }
     1000    }
     1001
     1002    // look for "inputs" (skycells) that need to processed
     1003    query = pxDataGet("magictool_toprocess_inputs.sql");
    9491004    if (!query) {
    9501005        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
    … …  
    9521007    }
    9531008
    954     for (psS64 index = 0; index < psArrayLength(magicRuns); index++) {
    955         if (limit && (psArrayLength(output) >= limit)) {
    956             break;
    957         }
    958         bool status;
    959         psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id");
    960         if (!status) {
    961             psAbort("failed to lookup value for magic_id column");
    962         }
    963 
    964         whereString = NULL;
    965         psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id);
    966         if (!p_psDBRunQueryF(config->dbh, query, whereString )) {
    967             psError(PS_ERR_UNKNOWN, false, "database error");
    968             psFree(whereString);
    969             psFree(query);
    970             return false;
    971         }
     1009    psStringAppend(&query, "\nORDER BY priority DESC, magic_id");
     1010
     1011    // treat limit == 0 as "no limit"
     1012    if (limit) {
     1013        psString limitString = psDBGenerateLimitSQL(limit);
     1014        psStringAppend(&query, " %s", limitString);
     1015        psFree(limitString);
     1016    }
     1017
     1018    if (!p_psDBRunQueryF(config->dbh, query, whereString, whereString)) {
     1019        psError(PS_ERR_UNKNOWN, false, "database error");
    9721020        psFree(whereString);
    973         psArray *magicTree = p_psDBFetchResult(config->dbh);
    974         if (!magicTree) {
    975             psErrorCode err = psErrorCodeLast();
    976             switch (err) {
    977                 case PS_ERR_DB_CLIENT:
    978                     psError(PXTOOLS_ERR_SYS, false, "database error");
    979                 case PS_ERR_DB_SERVER:
    980                     psError(PXTOOLS_ERR_PROG, false, "database error");
    981                 default:
    982                     psError(PXTOOLS_ERR_PROG, false, "unknown error");
    983             }
    984 
    985             return false;
    986         }
    987         psS64 length = psArrayLength(magicTree);
    988         if (!length) {
    989             psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id);
    990             psFree(magicTree);
    991             continue;
    992         }
    993 
    994         psHash *forest = psHashAlloc(length);
    995 
    996         // convert the array of metadata into a pxTree structure
    997         for (long i = 0; i < length; i++) {
    998             bool status;
    999             psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
    1000             if (!status) {
    1001                 psAbort("failed to lookup value for node column");
    1002             }
    1003 
    1004             psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep");
    1005             if (!status) {
    1006                 psAbort("failed to lookup value for dep column");
    1007             }
    1008 
    1009             pxTreeBuilder(forest, node, dep, magicTree->data[i]);
    1010 
    1011         }
    1012 
    1013         // find the root of the tree
    1014         pxNode *root = psMemIncrRefCounter(psHashLookup(forest, "root"));
    1015         psFree(forest);
    1016         //    pxTreePrint(stdout, root);
    1017 
    1018         // crawl through the tree and looking for nodes with children that are all
    1019         // "done"
    1020         pxTreeCrawl(root, findReadyNodes, output);
    1021         psFree(root);
    1022         psFree(magicTree);
    1023     }
    1024 
     1021        psFree(query);
     1022        return false;
     1023    }
     1024    psFree(query);
     1025
     1026    psArray *skycellOutput = p_psDBFetchResult(config->dbh);
     1027    if (!skycellOutput) {
     1028        psErrorCode err = psErrorCodeLast();
     1029        switch (err) {
     1030            case PS_ERR_DB_CLIENT:
     1031                psError(PXTOOLS_ERR_SYS, false, "database error");
     1032            case PS_ERR_DB_SERVER:
     1033                psError(PXTOOLS_ERR_PROG, false, "database error");
     1034            default:
     1035                psError(PXTOOLS_ERR_PROG, false, "unknown error");
     1036        }
     1037
     1038        return false;
     1039    }
     1040    // merge the arrays so that we can print them all at once
     1041    if (psArrayLength(skycellOutput)) {
     1042        int len = psArrayLength(skycellOutput);
     1043        for (int i=0; i < len; i++) {
     1044            psArrayAdd(output, 0, skycellOutput->data[i]);
     1045        }
     1046    }
     1047    psFree(skycellOutput);
    10251048    if (psArrayLength(output)) {
    1026         if (limit && (limit < psArrayLength(output))) {
    1027             // truncate the array
    1028             long arrayLength = psArrayLength(output);
    1029             for (long i = arrayLength - 1; i >= limit; i--) {
    1030                 psArrayRemoveIndex(output, i);
    1031             }
    1032         }
    1033         // negative simple so the default is true
    10341049        if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
    10351050            psError(PS_ERR_UNKNOWN, false, "failed to print array");
    … …  
    10371052            return false;
    10381053        }
    1039     }
    1040 
     1054    } else {
     1055        psTrace("magictool", PS_LOG_INFO, "no rows found");
     1056    }
    10411057    psFree(output);
    1042 
    10431058    return true;
    10441059}
    … …  
    16221637    return true;
    16231638}
     1639static bool setgotocleanedMode(pxConfig *config)
     1640{
     1641    PS_ASSERT_PTR_NON_NULL(config, false);
     1642
     1643    psMetadata *where = psMetadataAlloc();
     1644    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
     1645    PXOPT_COPY_S64(config->args, where, "-exp_id", "magicRun.exp_id", "==");
     1646    PXOPT_COPY_STR(config->args, where, "-label", "magicRun.label", "LIKE");
     1647    PXOPT_COPY_STR(config->args, where, "-data_group", "magicRun.data_group", "LIKE");
     1648
     1649    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
     1650
     1651    psString query = psStringCopy("UPDATE magicRun SET workdir_state = 'goto_cleaned'\n");
     1652    if (set_label) {
     1653        psStringAppend(&query, ", label = '%s'", set_label);
     1654    }
     1655    // This mode doubles as a revert function for cleanup errors
     1656    PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false);
     1657    if (!clearfault) {
     1658        psStringAppend(&query, "WHERE workdir_state = 'dirty'");
     1659    } else {
     1660        psStringAppend(&query, "WHERE workdir_state = 'error_cleaned'");
     1661    }
     1662    psStringAppend(&query, "\nAND (magicRun.state = 'full' OR magicRun.state = 'drop')");
     1663
     1664    // Require search parameters unless we're just clearing faults
     1665    if (psListLength(where->list)) {
     1666        psString clause = psDBGenerateWhereConditionSQL(where, NULL);
     1667        psStringAppend(&query, "\nAND %s", clause);
     1668        psFree(clause);
     1669        psFree(where);
     1670    } else if ( !clearfault) {
     1671        psError(PS_ERR_UNKNOWN, false, "search parameters are required");
     1672        psFree(where);
     1673        return false;
     1674    }
     1675
     1676    if (!p_psDBRunQuery(config->dbh, query)) {
     1677        psError(PS_ERR_UNKNOWN, false, "database error");
     1678        psFree(query);
     1679        return false;
     1680    }
     1681    psFree(query);
     1682
     1683    return true;
     1684}
     1685
     1686static bool setworkdirstateMode(pxConfig *config)
     1687{
     1688    PS_ASSERT_PTR_NON_NULL(config, false);
     1689
     1690    // required
     1691    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
     1692    PXOPT_LOOKUP_STR(workdir_state, config->args, "-set_workdir_state", true, false);
     1693
     1694    if (strcmp(workdir_state, "cleaned") && strcmp(workdir_state, "error_cleaned")) {
     1695        psError(PS_ERR_UNKNOWN, true, "%s is not a valid value for workdir_state", workdir_state);
     1696        return false;
     1697    }
     1698   
     1699    psString query = NULL;
     1700    psStringAppend(&query, "UPDATE magicRun SET workdir_state = '%s' WHERE magic_id = %" PRId64, workdir_state, magic_id);
     1701
     1702    if (!p_psDBRunQuery(config->dbh, query)) {
     1703        psError(PS_ERR_UNKNOWN, false, "database error");
     1704        psFree(query);
     1705        return false;
     1706    }
     1707    psFree(query);
     1708
     1709    return true;
     1710}
     1711static bool tocleanupMode(pxConfig *config)
     1712{
     1713    PS_ASSERT_PTR_NON_NULL(config, false);
     1714
     1715    psMetadata *where = psMetadataAlloc();
     1716    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
     1717    pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
     1718    pxAddLabelSearchArgs (config, where, "-data_group", "magicRun.data_group", "==");
     1719
     1720    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     1721    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     1722
     1723    psString query = pxDataGet("magictool_tocleanup.sql");
     1724    if (!query) {
     1725        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
     1726        return false;
     1727    }
     1728
     1729    if (psListLength(where->list)) {
     1730        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     1731        psStringAppend(&query, " AND %s", whereClause);
     1732        psFree(whereClause);
     1733    }
     1734    psFree(where);
     1735
     1736    psStringAppend(&query, "\nORDER BY priority DESC, magic_id");
     1737
     1738    // treat limit == 0 as "no limit"
     1739    if (limit) {
     1740        psString limitString = psDBGenerateLimitSQL(limit);
     1741        psStringAppend(&query, " %s", limitString);
     1742        psFree(limitString);
     1743    }
     1744
     1745    if (!p_psDBRunQuery(config->dbh, query)) {
     1746        psError(PS_ERR_UNKNOWN, false, "database error");
     1747        psFree(query);
     1748        return false;
     1749    }
     1750    psFree(query);
     1751
     1752    psArray *output = p_psDBFetchResult(config->dbh);
     1753    if (!output) {
     1754        psErrorCode err = psErrorCodeLast();
     1755        switch (err) {
     1756            case PS_ERR_DB_CLIENT:
     1757                psError(PXTOOLS_ERR_SYS, false, "database error");
     1758            case PS_ERR_DB_SERVER:
     1759                psError(PXTOOLS_ERR_PROG, false, "database error");
     1760            default:
     1761                psError(PXTOOLS_ERR_PROG, false, "unknown error");
     1762        }
     1763
     1764        return false;
     1765    }
     1766    if (!psArrayLength(output)) {
     1767        psTrace("magictool", PS_LOG_INFO, "no rows found");
     1768        psFree(output);
     1769        return true;
     1770    }
     1771
     1772    if (psArrayLength(output)) {
     1773        // negative simple so the default is true
     1774        if (!ippdbPrintMetadatas(stdout, output, "tocleanup", !simple)) {
     1775            psError(PS_ERR_UNKNOWN, false, "failed to print array");
     1776            psFree(output);
     1777            return false;
     1778        }
     1779    }
     1780
     1781    psFree(output);
     1782
     1783    return true;
     1784}

  • branches/eam_branches/ipp-20101205/ippTools/src/magictool.h

    r27982 r30446  
    4343    MAGICTOOL_MODE_CENSORRUN,
    4444    MAGICTOOL_MODE_EXPOSURE,
     45    MAGICTOOL_MODE_SETGOTOCLEANED,
     46    MAGICTOOL_MODE_TOCLEANUP,
     47    MAGICTOOL_MODE_SETWORKDIRSTATE,
    4548} MAGICtoolMode;
    4649

  • branches/eam_branches/ipp-20101205/ippTools/src/magictoolConfig.c

    r29495 r30446  
    178178    psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-inverse", 0, "select the inverse subtraction?", false);
    179179    psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     180
     181    // -setgotocleaned
     182    psMetadata *setgotocleanedArgs = psMetadataAlloc();
     183    psMetadataAddS64(setgotocleanedArgs, PS_LIST_TAIL, "-magic_id", 0, "select by magictool ID", 0);
     184    psMetadataAddS64(setgotocleanedArgs, PS_LIST_TAIL, "-exp_id", 0, "select by exposure ID)", 0);
     185    psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-label",  0, "select by label", NULL);
     186    psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-data_group",  0, "select by data_group", NULL);
     187    psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-set_label",  0, "set new label", NULL);
     188    psMetadataAddBool(setgotocleanedArgs, PS_LIST_TAIL, "-clearfault",  0, "clear cleanup errors", NULL);
     189
     190    // -tocleanup
     191    psMetadata *tocleanupArgs = psMetadataAlloc();
     192    psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
     193    psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-label",    PS_META_DUPLICATE_OK, "search by label", NULL);
     194    psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-data_group",    PS_META_DUPLICATE_OK, "search by label", NULL);
     195    psMetadataAddU64(tocleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
     196    psMetadataAddBool(tocleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     197
     198    // -setworkdirstate
     199    psMetadata *setworkdirstateArgs = psMetadataAlloc();
     200    psMetadataAddS64(setworkdirstateArgs, PS_LIST_TAIL, "-magic_id", 0, "select by magictool ID (required)", 0);
     201    psMetadataAddStr(setworkdirstateArgs, PS_LIST_TAIL, "-set_workdir_state", 0, "new workdir_state (required)", 0);
    180202
    181203    psFree(now);
    … …  
    202224    PXOPT_ADD_MODE("-censorrun",           "", MAGICTOOL_MODE_CENSORRUN,           censorrunArgs);
    203225    PXOPT_ADD_MODE("-exposure",            "", MAGICTOOL_MODE_EXPOSURE,            exposureArgs);
     226    PXOPT_ADD_MODE("-setgotocleaned",      "", MAGICTOOL_MODE_SETGOTOCLEANED,      setgotocleanedArgs);
     227    PXOPT_ADD_MODE("-tocleanup",           "", MAGICTOOL_MODE_TOCLEANUP,           tocleanupArgs);
     228    PXOPT_ADD_MODE("-setworkdirstate",     "", MAGICTOOL_MODE_SETWORKDIRSTATE,     setworkdirstateArgs);
    204229
    205230    if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

  • branches/eam_branches/ipp-20101205/ippTools/src/pstamptool.c

    r30102 r30446  
    134134    PXOPT_LOOKUP_STR(label,       config->args, "-set_label",         false, false);
    135135    PXOPT_LOOKUP_S32(pollInterval, config->args, "-set_poll_interval",false, false);
     136    PXOPT_LOOKUP_BOOL(no_magic, config->args, "-set_no_magic", false);
     137
     138    bool need_magic = ! no_magic;
    136139
    137140    if (!pstampDataStoreInsert(config->dbh,
    … …  
    143146            outProduct,
    144147            uri,
    145             pollInterval
     148            pollInterval,
     149            need_magic
    146150        )) {
    147151        psError(PS_ERR_UNKNOWN, false, "database error");

  • branches/eam_branches/ipp-20101205/ippTools/src/pstamptoolConfig.c

    r30102 r30446  
    5151    psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-set_label",        0, "define datastore label", NULL);
    5252    psMetadataAddS32(adddatastoreArgs, PS_LIST_TAIL, "-set_poll_interval", 0, "define datastore poll interval (seconds)", 60);
     53    psMetadataAddBool(adddatastoreArgs,PS_LIST_TAIL, "-set_no_magic",   0, "set magic not required", true);
    5354
    5455    // -datastore

  • branches/eam_branches/ipp-20101205/ippTools/src/pubtool.c

    r30102 r30446  
    167167    PXOPT_COPY_STR(config->args, diffWhere, "-filter", "rawExp.filter", "LIKE");
    168168    PXOPT_COPY_STR(config->args, diffWhere, "-obs_mode", "rawExp.obs_mode", "LIKE");
     169    PXOPT_COPY_STR(config->args, diffWhere, "-comment", "rawExp.comment", "LIKE");
    169170
    170171    PXOPT_COPY_S64(config->args, camWhere, "-client_id", "client_id", "==");

  • branches/eam_branches/ipp-20101205/ippTools/src/pubtoolConfig.c

    r30102 r30446  
    6666    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "set and search by label", NULL);
    6767    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", NULL);
     68    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL);
    6869    psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL);
    6970    psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL);

  • branches/eam_branches/ipp-20101205/ippTools/src/pztool.c

    r30102 r30446  
    4747static bool advanceMode(pxConfig *config);
    4848
     49static bool updatepzexpMode(pxConfig *config);
     50static bool updatenewexpMode(pxConfig *config);
     51
    4952// static bool copydoneCompleteExp(pxConfig *config);
    5053static psArray *pzGetPendingCameras(pxConfig *config);
    … …  
    8386        MODECASE(PZTOOL_MODE_TOADVANCE, toadvanceMode);
    8487        MODECASE(PZTOOL_MODE_ADVANCE, advanceMode);
     88        MODECASE(PZTOOL_MODE_UPDATEPZEXP, updatepzexpMode);
     89        MODECASE(PZTOOL_MODE_UPDATENEWEXP, updatenewexpMode);
    8590        default:
    8691            psAbort("invalid option (this should not happen)");
    … …  
    294299
    295300    psMetadata *where = psMetadataAlloc();
    296     PXOPT_COPY_STR(config->args, where,  "-exp_name",     "exp_name", "==");
    297     PXOPT_COPY_STR(config->args, where,  "-inst",         "camera", "==");
    298     PXOPT_COPY_STR(config->args, where,  "-telescope",    "telescope", "==");
    299     PXOPT_COPY_STR(config->args, where,  "-exp_type",     "exp_type", "==");
     301    PXOPT_COPY_STR(config->args, where,  "-exp_name",      "exp_name",  "==");
     302    PXOPT_COPY_STR(config->args, where,  "-inst",          "camera",    "==");
     303    PXOPT_COPY_STR(config->args, where,  "-telescope",     "telescope", "==");
     304    PXOPT_COPY_STR(config->args, where,  "-exp_type",      "exp_type",  "==");
     305    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs",   ">=");
     306    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "dateobs",   "<=");
    300307
    301308    PXOPT_LOOKUP_BOOL(desc, config->args, "-desc", false);
    … …  
    325332
    326333        if (psListLength(where->list)) {
    327             psString whereClause = psDBGenerateWhereConditionSQL(where, "pzDownloadImfile");
     334            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
    328335            psStringAppend(&query, " AND %s", whereClause);
    329336            psFree(whereClause);
    … …  
    889896            || (strncmp(state, "stop", 5) == 0)
    890897            || (strncmp(state, "reg", 4) == 0)
     898            || (strncmp(state, "drop", 5) == 0)
    891899        )
    892900    ) {
    … …  
    905913}
    906914
     915static bool updatepzexpMode(pxConfig *config)
     916{
     917  PS_ASSERT_PTR_NON_NULL(config, false);
     918  PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false);
     919  PXOPT_LOOKUP_STR(camera,   config->args, "-inst",   true, false);
     920  PXOPT_LOOKUP_STR(telescope,config->args, "-telescope",true, false);
     921  PXOPT_LOOKUP_STR(state,    config->args, "-set_state",true, false);
     922
     923  if (!pzDownloadExpSetState(config,exp_name,camera,telescope,state)) {
     924    psError(PS_ERR_UNKNOWN, false, "failed to change state for %s:%s:%s", exp_name, camera, telescope);
     925    return false;
     926  }
     927  return true;
     928}
     929 
     930 
     931static bool updatenewexpMode(pxConfig *config)
     932{
     933    PS_ASSERT_PTR_NON_NULL(config, false);
     934
     935    psMetadata *where = psMetadataAlloc();
     936    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
     937    PXOPT_COPY_STR(config->args, where,  "-exp_name", "tmp_exp_name", "==");
     938
     939    PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", true, false);
     940
     941    if (strcmp(new_state, "drop") && strcmp(new_state, "run") && strcmp(new_state, "wait")) {
     942        psError(PXTOOLS_ERR_ARGUMENTS, true, "%s is not a valid value for -set_state", new_state);
     943        psFree(where);
     944        return false;
     945    }
     946
     947    if (psListLength(where->list) < 1) {
     948        psFree(where);
     949        psError(PXTOOLS_ERR_CONFIG, false, "-exp_name or -exp_id is required");
     950        return false;
     951    }
     952
     953    psString query = NULL;
     954    psStringAppend(&query, "UPDATE newExp SET state = '%s'", new_state);
     955
     956    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     957    psStringAppend(&query, "\nWHERE %s", whereClause);
     958
     959    psFree(whereClause);
     960    psFree(where);
     961
     962    if (!p_psDBRunQuery(config->dbh, query)) {
     963        psError(PS_ERR_UNKNOWN, false, "database error");
     964        psFree(query);
     965        return false;
     966    }
     967    psFree(query);
     968
     969    return true;
     970}
     971
     972
     973
    907974
    908975#if 0

  • branches/eam_branches/ipp-20101205/ippTools/src/pztool.h

    r23497 r30446  
    3636    PZTOOL_MODE_CLEARCOMMONFAULTS,
    3737    PZTOOL_MODE_TOADVANCE,
    38     PZTOOL_MODE_ADVANCE
     38    PZTOOL_MODE_ADVANCE,
     39    PZTOOL_MODE_UPDATEPZEXP,
     40    PZTOOL_MODE_UPDATENEWEXP,
    3941} pztoolMode;
    4042

  • branches/eam_branches/ipp-20101205/ippTools/src/pztoolConfig.c

    r27082 r30446  
    7676    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL);
    7777    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL);
     78    psMetadataAddTime(pendingimfileArgs, PS_LIST_TAIL, "-dateobs_begin", 0,  "search for exposures by time (>=)", NULL);
     79    psMetadataAddTime(pendingimfileArgs, PS_LIST_TAIL, "-dateobs_end", 0,  "search for exposures by time (<=)", NULL);
    7880    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_type", 0,            "define exposure type", NULL);
    7981    psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-desc", 0,            "sort ouput in descending format", false);
    … …  
    154156    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label",  0,     "define the label for the chip stage", NULL);
    155157
     158    // -updatepzexp
     159    psMetadata *updatepzexpArgs = psMetadataAlloc();
     160    psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-exp_name",   0,            "search by exposure name (required)", NULL);
     161    psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-inst",       0,            "search by camera (required)", NULL);
     162    psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-telescope",  0,            "search by telescope (required)", NULL);
     163    psMetadataAddStr(updatepzexpArgs, PS_LIST_TAIL, "-set_state",  0,            "define new state (required)", NULL);
     164   
     165    // -updatenewexp
     166    psMetadata *updatenewexpArgs = psMetadataAlloc();
     167    psMetadataAddS64(updatenewexpArgs, PS_LIST_TAIL, "-exp_id",   0,            "search by exposure ID", 0);
     168
     169    psMetadataAddStr(updatenewexpArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID", NULL);
     170    psMetadataAddStr(updatenewexpArgs, PS_LIST_TAIL, "-set_state", 0,            "define new state (required)", NULL);
     171
    156172    psMetadata *argSets = psMetadataAlloc();
    157173    psMetadata *modes = psMetadataAlloc();
    … …  
    169185    PXOPT_ADD_MODE("-toadvance",          "", PZTOOL_MODE_TOADVANCE,    toadvanceArgs);
    170186    PXOPT_ADD_MODE("-advance",          "", PZTOOL_MODE_ADVANCE,    advanceArgs);
     187    PXOPT_ADD_MODE("-updatepzexp",     "", PZTOOL_MODE_UPDATEPZEXP, updatepzexpArgs);
     188    PXOPT_ADD_MODE("-updatenewexp",    "", PZTOOL_MODE_UPDATENEWEXP,updatenewexpArgs);
    171189
    172190    if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

  • branches/eam_branches/ipp-20101205/ippTools/src/regtool.c

    r30102 r30446  
    5151static bool updatebyqueryMode(pxConfig *config);
    5252
     53static bool checkstatusMode(pxConfig *config);
     54
    5355static bool exportrunMode(pxConfig *config);
    5456static bool importrunMode(pxConfig *config);
    … …  
    7476        // imfile
    7577        MODECASE(REGTOOL_MODE_PENDINGIMFILE,         pendingimfileMode);
    76         MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE,   checkburntoolimfileMode);
    77         MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode);
     78        MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE,   checkburntoolimfileMode);
     79        MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode);
    7880        MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE,    addprocessedimfileMode);
    7981        MODECASE(REGTOOL_MODE_PROCESSEDIMFILE,       processedimfileMode);
    … …  
    9092        MODECASE(REGTOOL_MODE_FINISHCOMPRESSEXP,     finishcompressexpMode);
    9193        MODECASE(REGTOOL_MODE_CLEARDUPEXP,           cleardupexpMode);
     94        MODECASE(REGTOOL_MODE_CHECKSTATUS,           checkstatusMode);
    9295        MODECASE(REGTOOL_MODE_EXPORTRUN,             exportrunMode);
    9396        MODECASE(REGTOOL_MODE_IMPORTRUN,             importrunMode);
    … …  
    9598            psAbort("invalid option (this should not happen)");
    9699    }
    97 
     100    psTrace("regtool",9,"Attempting to free config\n");
    98101    psFree(config);
    99102    pmConfigDone();
    … …  
    180183}
    181184
     185 
     186 
     187
    182188static bool checkburntoolimfileMode(pxConfig *config)
    183189{
    … …  
    187193  PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false);
    188194  PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
    189   PXOPT_LOOKUP_STR(date,     config->args, "-date", true, false);
     195  PXOPT_LOOKUP_STR(dateobs_begin,     config->args, "-dateobs_begin", true, false);
     196  PXOPT_LOOKUP_STR(dateobs_end,     config->args, "-dateobs_end", true, false);
    190197  PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false);
    191198  // optional
    … …  
    201208  psFree(query);
    202209  query = rep;
     210
     211  // convert regular class_id format to summitImfile.class_id format
     212  rep = psStringCopy(class_id);
     213  psFree(class_id);
     214  class_id = rep;
     215 
     216  psStringSubstitute(&class_id,"ota","XY");
    203217 
    204218  psStringSubstitute(&query,exp_name,"@EXP_NAME@");
    205219  psStringSubstitute(&query,class_id,"@CLASS_ID@");
    206   psStringSubstitute(&query,date,"@DATE@");
    207 
    208   fprintf(stderr,"%s",query);
     220  psStringSubstitute(&query,dateobs_begin,"@DATEOBS_BEGIN@");
     221  psStringSubstitute(&query,dateobs_end,"@DATEOBS_END@");
     222
     223  // fprintf(stderr,"%s",query);
    209224
    210225  if (!p_psDBRunQuery(config->dbh, query)) {
    … …  
    215230  }
    216231  psFree(query);
    217  
     232
    218233  psArray *output = p_psDBFetchResult(config->dbh);
    219234  if (!output) {
    … …  
    227242      psError(PXTOOLS_ERR_PROG, false, "unknown error");
    228243    }
    229    
     244
    230245    return false;
    231246  }
    232247  if (!psArrayLength(output)) {
    233248    psTrace("regtool", PS_LOG_INFO, "no rows found");
    234     psFree(output);
    235249    return true;
    236250  }
    … …  
    242256    psMetadata *row = output->data[i];
    243257
     258    if (strcasecmp(psMetadataLookupStr(NULL,row,"download_state"),"drop") == 0) {
     259      continue;
     260    }
     261   
    244262    if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
    245         (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
     263        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
    246264      ok_to_burn = false;
    247265    }
    … …  
    255273      already_burned = false;
    256274    }
    257    
     275
    258276    if (previous_uri) {
    259277      psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri);
    … …  
    264282    previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri"));
    265283  }
    266  
     284
    267285  // negate simple so the default is true
    268286  if (!ippdbPrintMetadatas(stdout, output, "regBurntoolImfile", !simple)) {
    … …  
    271289    return false;
    272290  }
    273  
     291
    274292  psFree(output);
    275  
     293
    276294
    277295  return(true);
    278  
     296
    279297}
    280298
    … …  
    284302
    285303  // required
    286   PXOPT_LOOKUP_STR(date,     config->args, "-date", true, false);
     304  PXOPT_LOOKUP_STR(dateobs_begin,     config->args, "-dateobs_begin", true, false);
     305  PXOPT_LOOKUP_STR(dateobs_end,     config->args, "-dateobs_end", true, false);
    287306  PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false);
    288307  // optional
    … …  
    297316  psFree(query);
    298317  query = rep;
    299  
    300   psStringSubstitute(&query,date,"@DATE@");
    301 
    302   //  fprintf(stderr,"%s",query);
     318
     319  psStringSubstitute(&query,dateobs_begin,"@DATEOBS_BEGIN@");
     320  psStringSubstitute(&query,dateobs_end,"@DATEOBS_END@");
     321
     322  // fprintf(stderr,"%s",query);
    303323
    304324  if (!p_psDBRunQuery(config->dbh, query)) {
    … …  
    309329  }
    310330  psFree(query);
    311  
     331
    312332  psArray *output = p_psDBFetchResult(config->dbh);
    313333  if (!output) {
    … …  
    321341      psError(PXTOOLS_ERR_PROG, false, "unknown error");
    322342    }
    323    
     343
    324344    return false;
    325345  }
     346
     347  // fprintf (stderr, "found %ld rows\n", output->n);
    326348  if (!psArrayLength(output)) {
    327349    psTrace("regtool", PS_LOG_INFO, "no rows found");
    … …  
    337359  psString this_class_id = NULL;
    338360  psArray *results = psArrayAllocEmpty(60); // List of suggested imfiles to burntool.
    339  
     361
    340362  for (long i = 0; i < output->n; i++) {
    341363    psMetadata *row = output->data[i];
    342364
     365    if (strcasecmp(psMetadataLookupStr(NULL,row,"download_state"),"drop") == 0) {
     366      continue;
     367    }
     368
     369    bool status = false;
     370    char *tmp_id = psMetadataLookupStr(&status,row,"summit_class_id");
     371    if (!status) {
     372      //      fprintf (stderr, "incomplete on %s\n", psMetadataLookupStr(NULL,row,"exp_name"));
     373        continue;
     374    }
     375
    343376    // Add the information about this row and the previous, if it exists.
    344377    // Write the class_id stuff for debugging.
    345     this_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
    346     psStringSubstitute(&this_class_id,"XY","ota");
     378    // this_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
     379    this_class_id = psStringCopy(tmp_id);
     380
    347381    psMetadataAddStr(row,PS_LIST_TAIL,"this_class_id",PS_META_REPLACE,"",this_class_id);
     382
    348383    if (previous_class_id) {
    349384      psMetadataAddStr(row,PS_LIST_TAIL,"previous_class_id",PS_META_REPLACE,"",previous_class_id);
    … …  
    353388      continue;
    354389    }
     390
     391    if (0 && !strcmp(this_class_id, "ota44")) {
     392        printf("STAT 1: %s (%d %d) %d %d %d\n",
     393               this_class_id,
     394               ok_to_burn, already_burned,
     395               psMetadataLookupS32(NULL,row,"burntool_state"),
     396               psMetadataLookupS32(NULL,row,"is_registered"),
     397               psMetadataLookupS32(NULL,row,"is_downloaded"));
     398    }
     399
    355400    // Determine if we've crossed a class_id boundary, as this resets the bits.
    356401    if (previous_class_id) {
    357402      if (strcmp(this_class_id,previous_class_id) != 0) {
    358         ok_to_burn = true;
    359         already_burned = true;
    360         previous_uri = NULL;
     403        ok_to_burn = true;
     404        already_burned = true;
     405        previous_uri = NULL;
     406        //      fprintf (stderr, "crossed boundary: %s : %s to %s\n", psMetadataLookupStr(NULL,row,"exp_name"), previous_class_id, this_class_id);
    361407      }
    362408    }
    … …  
    364410    // Write the URIs as well.
    365411    this_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Duplicate, but helpful for my debugging.
    366     psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri); 
     412    psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri);
    367413    if (previous_uri) {
    368414      psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri);
    … …  
    372418    // Convert bits of the SQL query into booleans describing the data state
    373419    if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
    374         (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
    375       //      printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id);
    376       ok_to_burn = false;
     420        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
     421      // printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id);
     422        ok_to_burn = false;
    377423    }
    378424    if (already_burned == false) {
    379       //      printf("already_burned looks false %s %s\n",this_uri,this_class_id);
    380       ok_to_burn = false;
     425        // printf("already_burned looks false %s %s\n",this_uri,this_class_id);
     426        ok_to_burn = false;
    381427    }
    382428    if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) {
    … …  
    391437    // Check the uri for this exposure
    392438    if (!this_uri) {
    393         ok_to_burn = false;
    394         already_burned = false;
    395 
    396         // Save this round for next round.
    397         psFree(previous_class_id);
    398         psFree(previous_uri);
    399         previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
    400         psStringSubstitute(&previous_class_id,"XY","ota");
    401         previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
    402         continue;
    403     }     
    404 
    405     //    printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded"));
    406 
    407     // If the state of this imfile is not "pending_burntool" then we can't burn it.
     439        ok_to_burn = false;
     440        already_burned = false;
     441
     442        //      fprintf (stderr, "missing uri: %s %s\n", psMetadataLookupStr(NULL,row,"exp_name"), this_class_id);
     443
     444        // Save this round for next round.
     445        psFree(previous_class_id);
     446        psFree(previous_uri);
     447        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"summit_class_id"));
     448        psStringSubstitute(&previous_class_id,"ota","XY");
     449        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.
     450        continue;
     451    }
     452
     453    if (0 && !strcmp(this_class_id, "ota44")) {
     454        printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded"));
     455    }
     456    // If the state of this imfile is not "pending_burntool" then we can't burn it.
    408457    if (!ignore_state) {
    409458      psString imfile_state = psMetadataLookupStr(NULL,row,"imfile_state");
    410459
    411460      if (!imfile_state) { // imfile state is NULL, so we probably aren't registered.
    412         ok_to_burn = false;
    413 
    414         // Save this round for next round.
    415         psFree(previous_class_id);
    416         psFree(previous_uri);
    417         previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
    418         psStringSubstitute(&previous_class_id,"XY","ota");
    419         previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
    420         continue;
     461        ok_to_burn = false;
     462
     463        // fprintf (stderr, "missing imfile_state: %s %s\n", psMetadataLookupStr(NULL,row,"exp_name"), this_class_id);
     464
     465        // Save this round for next round.
     466        psFree(previous_class_id);
     467        psFree(previous_uri);
     468        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"summit_class_id"));
     469        psStringSubstitute(&previous_class_id,"ota","XY");
     470        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.
     471        continue;
    421472      }
    422       if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states 
    423         // Save this round for next round.
    424         psFree(previous_class_id);
    425         psFree(previous_uri);
    426         previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
    427         psStringSubstitute(&previous_class_id,"XY","ota");
    428         previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
    429         continue;
     473      if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states
     474
     475          if (0 && strcmp(imfile_state, "full")) {
     476              fprintf (stderr, "not pending nor full: %s %s %s\n", psMetadataLookupStr(NULL,row,"exp_name"), this_class_id, imfile_state);
     477          }
     478
     479        // Save this round for next round.
     480        psFree(previous_class_id);
     481        psFree(previous_uri);
     482        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"summit_class_id"));
     483        psStringSubstitute(&previous_class_id,"ota","XY");
     484        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.
     485        continue;
    430486      }
    431487    }
    432    
     488
    433489    // Determine if we've already suggested an entry for this ota, and if not, copy this
    434490    // suggestion into our output result list.
    … …  
    437493      psFree(previous_class_id);
    438494      psFree(previous_uri);
    439       previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
    440       psStringSubstitute(&previous_class_id,"XY","ota");
    441       previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
     495      previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"summit_class_id"));
     496      psStringSubstitute(&previous_class_id,"ota","XY");
     497      previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.
    442498      continue;
    443499    }
    … …  
    448504    psFree(previous_class_id);
    449505    psFree(previous_uri);
    450     previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
    451     psStringSubstitute(&previous_class_id,"XY","ota");
    452     previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.         
    453   }
    454  
     506    previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"summit_class_id"));
     507    psStringSubstitute(&previous_class_id,"ota","XY");
     508    previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.
     509  }
     510
    455511  // negate simple so the default is true
    456512  if (!ippdbPrintMetadatas(stdout, results, "regPendingBurntoolImfile", !simple)) {
    … …  
    460516    return false;
    461517  }
    462  
     518
    463519  psFree(output);
    464520  psFree(results);
    465  
     521
    466522
    467523  return(true);
    468  
    469 }
    470 
    471      
     524
     525}
     526
     527
    472528
    473529static bool addprocessedimfileMode(pxConfig *config)
    … …  
    545601    PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false);
    546602    PXOPT_LOOKUP_BOOL(video_cells, config->args, "-video_cells", false);
    547    
     603
    548604    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
    549605    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
    … …  
    618674        bytes,
    619675        md5sum,
    620         video_cells,
    621         0   // burntool_state
     676        0,   // burntool_state
     677        video_cells
    622678    )) {
    623679        psError(PS_ERR_UNKNOWN, false, "database error");
    … …  
    771827        psStringAppend(&query, " AND %s", whereClause);
    772828        psFree(whereClause);
    773     } else {
    774         psFree(where);
    775         psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
    776         return false;
    777829    }
    778830    psFree(where);
    … …  
    806858    PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false);
    807859
    808     if ((fault == INT16_MAX) && !isfinite(burntool_state) && !(set_state)) {
     860    if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state) {
    809861        psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected");
    810862        return false;
    811863    }
    812     if ((fault != INT16_MAX) && isfinite(burntool_state)) {
     864    if ((fault != INT16_MAX) && (burntool_state != INT16_MAX)) {
    813865        psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected");
    814866        return false;
    … …  
    13121364    if (!pxchipQueueByExpTag(config,
    13131365                exp_id,
    1314                              chip_workdir,
     1366                             chip_workdir,
    13151367                label,
    1316                              data_group,
    1317                              dist_group,
     1368                             data_group,
     1369                             dist_group,
    13181370                reduction,
    13191371                NULL,       // expgroup
    … …  
    18311883}
    18321884
     1885static bool checkstatusMode(pxConfig *config)
     1886{
     1887  PS_ASSERT_PTR_NON_NULL(config,false);
     1888
     1889  // required
     1890  PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
     1891  // Conditionally required
     1892  PXOPT_LOOKUP_STR(dateobs_begin, config->args, "-dateobs_begin", false, false);
     1893  PXOPT_LOOKUP_STR(dateobs_end,   config->args, "-dateobs_end",   false, false);
     1894  PXOPT_LOOKUP_STR(date,          config->args, "-date",          false, false);
     1895  PXOPT_LOOKUP_BOOL(simple,       config->args, "-simple",        false);
     1896
     1897  psString query = pxDataGet("regtool_checkstatus.sql");
     1898  if (!query) {
     1899    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
     1900    return false;
     1901  }
     1902  psString rep = psStringCopy(query);
     1903  psFree(query);
     1904  query = rep;
     1905  psStringSubstitute(&class_id,"ota","XY");
     1906  psStringSubstitute(&query,class_id,"@CLASS_ID@");
     1907   
     1908  if (!date) {
     1909    if (!dateobs_begin || !dateobs_end) {
     1910      psError(PXTOOLS_ERR_CONFIG, false, "Either -date or -dateobs_begin -dateobs_end is required");
     1911      psFree(query);
     1912      return false;
     1913    }
     1914    psStringAppend(&query," AND summitExp.dateobs >= '%s' AND summitExp.dateobs <= '%s' ",
     1915                   dateobs_begin,dateobs_end);
     1916  }
     1917  else {
     1918    psStringAppend(&query," AND summitExp.dateobs >= '%sT00:00:00' AND summitExp.dateobs <= '%sT23:59:59' ",
     1919                   date,date);
     1920  }
     1921
     1922  psStringAppend(&query," ORDER BY summitExp.dateobs ");
     1923 
     1924  if (!p_psDBRunQuery(config->dbh, query)) {
     1925    // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
     1926    psError(PXTOOLS_ERR_PROG, false, "database error");
     1927    psFree(query);
     1928    return false;
     1929  }
     1930  psFree(query);
     1931
     1932  psArray *output = p_psDBFetchResult(config->dbh);
     1933  if (!output) {
     1934    psErrorCode err = psErrorCodeLast();
     1935    switch (err) {
     1936    case PS_ERR_DB_CLIENT:
     1937      psError(PXTOOLS_ERR_SYS, false, "database error");
     1938    case PS_ERR_DB_SERVER:
     1939      psError(PXTOOLS_ERR_PROG, false, "database error");
     1940    default:
     1941      psError(PXTOOLS_ERR_PROG, false, "unknown error");
     1942    }
     1943
     1944    return false;
     1945  }
     1946  if (!psArrayLength(output)) {
     1947    psTrace("regtool", PS_LOG_INFO, "no rows found");
     1948    return true;
     1949  }
     1950
     1951  if (!ippdbPrintMetadatas(stdout, output, "regCheckStatus", !simple)) {
     1952    psError(PXTOOLS_ERR_PROG, false, "failed to print array");
     1953    psFree(output);
     1954    return false;
     1955  }
     1956
     1957  psFree(output);
     1958
     1959  return(true);
     1960}
     1961
    18331962bool exportrunMode(pxConfig *config)
    18341963{

  • branches/eam_branches/ipp-20101205/ippTools/src/regtool.h

    r30102 r30446  
    4141    REGTOOL_MODE_PENDINGCOMPRESSIMFILE,
    4242    REGTOOL_MODE_FINISHCOMPRESSEXP,
     43    REGTOOL_MODE_CHECKSTATUS,
    4344    REGTOOL_MODE_EXPORTRUN,
    4445    REGTOOL_MODE_IMPORTRUN

  • branches/eam_branches/ipp-20101205/ippTools/src/regtoolConfig.c

    r30102 r30446  
    5757    ADD_OPT(Str,  checkburntoolimfileArgs, "-exp_name",       "define the exp_name (required)",         NULL);
    5858    ADD_OPT(Str,  checkburntoolimfileArgs, "-class_id",       "define class ID (required)",         NULL);
    59     ADD_OPT(Str,  checkburntoolimfileArgs, "-date",           "select the date to process (required)", NULL);
     59    ADD_OPT(Str,  checkburntoolimfileArgs, "-dateobs_begin",  "set the earliest summit dateobs to consider (required)", NULL);
     60    ADD_OPT(Str,  checkburntoolimfileArgs, "-dateobs_end",    "set the latest summit dateobs to consider (required)", NULL);
    6061    ADD_OPT(S32,  checkburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0);
    6162    ADD_OPT(Str,  checkburntoolimfileArgs, "-inst",           "define the camera name",     NULL);
    … …  
    6566    // -pendingburntoolimfile
    6667    psMetadata *pendingburntoolimfileArgs = psMetadataAlloc();
    67     ADD_OPT(Str,  pendingburntoolimfileArgs, "-date",           "select the date to process (required)", NULL);
     68    ADD_OPT(Str,  pendingburntoolimfileArgs, "-dateobs_begin",  "set the earliest summit dateobs to consider (required)", NULL);
     69    ADD_OPT(Str,  pendingburntoolimfileArgs, "-dateobs_end",    "set the latest summit dateobs to consider (required)", NULL);
    6870    ADD_OPT(S32,  pendingburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0);
    6971    ADD_OPT(Bool, pendingburntoolimfileArgs, "-simple",    "use the simple output format",          false);
    7072    ADD_OPT(Bool, pendingburntoolimfileArgs, "-ignore_state",   "ignore the data_state when deciding what to work on",  false);
    7173    ADD_OPT(U64,  pendingburntoolimfileArgs, "-limit",     "limit result set to N items",  0);
    72    
     74
    7375    // -addprocessedimfile
    7476    psMetadata *addprocessedimfileArgs = psMetadataAlloc();
    … …  
    178180    ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id_begin",  "search by exposure ID", 0);
    179181    ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id_end",    "search by exposure ID", 0);
     182    // This argument is not used but is needed because the task uses add.poll.args
     183    ADD_OPT(U64, revertprocessedimfileArgs, "-limit",     "for compatability not used", 0);
    180184
    181185    // -updateprocessedimfile
    … …  
    415419    psMetadataAddBool(finishcompressexpArgs, PS_LIST_TAIL, "-simple", 0,   "use the simple output format", false);
    416420    psMetadataAddU64(finishcompressexpArgs, PS_LIST_TAIL, "-limit",   0,   "limit result set to N items", 0);
     421
     422    // -checkstatus
     423    psMetadata *checkstatusArgs = psMetadataAlloc();
     424    ADD_OPT(Str, checkstatusArgs, "-dateobs_begin", "set the earlist summit dateobs to consider", NULL);
     425    ADD_OPT(Str, checkstatusArgs, "-dateobs_end", "set the latest summit dateobs to consider", NULL);
     426    ADD_OPT(Str, checkstatusArgs, "-date", "use default observing extent and look over entire night", NULL);
     427    ADD_OPT(Str, checkstatusArgs, "-class_id",    "define class ID (required)", NULL);
     428    ADD_OPT(Bool, checkstatusArgs, "-simple",    "use the simple output format", false);
    417429   
    418430    psMetadata *argSets = psMetadataAlloc();
    … …  
    435447    PXOPT_ADD_MODE("-pendingcompressimfile",   "", REGTOOL_MODE_PENDINGCOMPRESSIMFILE, pendingcompressimfileArgs);
    436448    PXOPT_ADD_MODE("-finishcompressexp",       "", REGTOOL_MODE_FINISHCOMPRESSEXP, finishcompressexpArgs);
     449    PXOPT_ADD_MODE("-checkstatus",             "", REGTOOL_MODE_CHECKSTATUS, checkstatusArgs);
    437450    PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", REGTOOL_MODE_EXPORTRUN, exportrunArgs);
    438451    PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           REGTOOL_MODE_IMPORTRUN, importrunArgs);

  • branches/eam_branches/ipp-20101205/ippTools/src/stacktool.c

    r29418 r30446  
    497497            psError(PS_ERR_UNKNOWN, false, "database error");
    498498            psFree(output);
    499             psFree(run);
    500499            psFree(insert);
    501500            psFree(list);
    … …  
    517516          psError(PS_ERR_UNKNOWN, false, "database error");
    518517          psFree(output);
    519           psFree(run);
    520518          psFree(insert);
    521519          psFree(list);

  • branches/eam_branches/ipp-20101205/pstamp/scripts/pstamp_checkdependent.pl

    r29624 r30446  
    690690                if ($dsfile->{data_state} eq 'cleaned') {
    691691                    $command = "$magicdstool -setfiletoupdate -magic_ds_id $magic_ds_id -component $c";
     692                    # XXX: get the recoveryroot from a config file (it isn't actually used except to check whether it is in nebulous)
     693                    $command .= " -set_recoveryroot neb://any/gpc1/destreak/recover";
    692694                    $command .= " -set_label $rlabel" if $rlabel;
    693695                    if (!$no_update) {

  • branches/eam_branches/ipp-20101205/pstamp/scripts/pstamp_job_run.pl

    r30103 r30446  
    1414use File::Basename;
    1515use File::Copy;
     16use File::Temp qw(tempfile);
    1617use Digest::MD5::File qw( file_md5_hex );
    1718use PS::IPP::PStamp::RequestFile qw( :standard );
     19use PS::IPP::PStamp::Job qw( :standard );
    1820use IPC::Cmd 0.36 qw( can_run run );
    1921use POSIX;
    … …  
    2628
    2729my ($job_id, $redirect_output, $outputBase, $rownum, $jobType, $options);
    28 my ($verbose, $dbname, $dbserver, $no_update);
     30my ($verbose, $dbname, $dbserver, $no_update, $save_temps);
    2931
    3032GetOptions(
    … …  
    3941    'verbose'           =>  \$verbose,
    4042    'no-update'         =>  \$no_update,
     43    'save-temps'        =>  \$save_temps,
    4144);
    4245
    … …  
    7578my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
    7679my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1);
     80my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
     81my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
    7782
    7883if ($missing_tools) {
    … …  
    8893    my $argString;
    8994    $argString = $params->{job_args};
     95    my $stage = $params->{stage};
    9096
    9197    # XXX: should we do any other sanity checking?
    9298    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
    9399
    94     $argString .= " -file $params->{image}";
     100    my $nan_masked = 1;
     101    my $muggle = 0;
     102    if (!$params->{magicked}) {
     103        $nan_masked = 0;
     104    } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) {
     105        # Attempt to find or create a muggle image
     106        $nan_masked = 0;
     107        $muggle = 1;
     108    }
     109
     110    my $image = $params->{image};
     111    my $mask;
     112    my $variance;
     113    my $fileArgs = " -file $params->{image}";
    95114    my @file_list = ($params->{image});
    96115   
    97     my $nan_masked = 1;
    98     if (!$params->{magicked}) {
    99         $nan_masked = 0;
    100     }
    101116    if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) {
    102         $argString .= " -mask $params->{mask}";
    103         push @file_list, $params->{mask};
     117        $mask = $params->{mask};
     118        $fileArgs .= " -mask $mask";
     119        push @file_list, $mask;
    104120    }
    105121    if ($options & $PSTAMP_SELECT_VARIANCE) {
    106         $argString .= " -variance $params->{weight}";
    107         push @file_list, $params->{weight};
     122        $variance = $params->{weight};
     123        $fileArgs .= " -variance $variance";
     124        push @file_list, $variance;
    108125    }
    109126
    … …  
    113130    }
    114131
    115     check_files(@file_list);
    116 
    117     my $command = "$ppstamp $outputBase $argString";
     132    # check that actual input files exist
     133    check_files($PSTAMP_GONE, @file_list);
     134
     135    # find our output directory
     136    my $outdir = dirname($outputBase);
     137    my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
     138   
     139    if ($muggle) {
     140        # first see if the original uncensored images are around
     141        if (check_for_backups($params, \$fileArgs)) {
     142            # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images
     143            print "Making stamps from backup images\n";
     144        } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) {
     145            # user required uncensored but since stage isn't chip we can't rebuild them
     146            my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE);
     147        } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($stage ne 'chip')) {
     148            # make stamps from uncensored images
     149            $muggle = 0;
     150        } else {
     151            # Try and replace the streaks from the recovery images
     152
     153            @file_list = ();
     154            $tmproot = "$outdir/$job_id";
     155            mkdir $tmproot or
     156                my_die( "failed to create temporary directory $tmproot", $job_id, $PS_EXIT_UNKNOWN_ERROR);
     157            my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot";
     158            # find the "directory" of the input path_base
     159            my $inputdir = dirname($image);
     160            my $base = basename($image);
     161            $tmpImage = "$tmproot/$base";
     162
     163            # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and
     164            # stage work. Import the rule here.
     165            my $recImage = "$inputdir/REC_$base";
     166            my $newFileArgs = " -file $tmpImage";
     167            $muggle_command .= " -image $image -recimage $recImage";
     168            push @file_list, $recImage;
     169
     170            if ($mask) {
     171                $base = basename($mask);
     172                $tmpMask = "$tmproot/$base";
     173                $newFileArgs .= " -mask $tmpMask";
     174                my $recMask = "$inputdir/REC_$base";
     175                $muggle_command .= " -mask $mask -recmask $recMask";
     176                push @file_list, $recMask;
     177            }
     178
     179            if ($variance) {
     180                $base = basename($variance);
     181                $tmpVariance = "$tmproot/$base";
     182                $newFileArgs .= " -weight $tmpVariance";
     183                my $recVariance = "$inputdir/REC_$base";
     184                $muggle_command .= " -weight $variance -recweight $recVariance";
     185                push @file_list, $recVariance;
     186            }
     187            if (check_files(0, @file_list)) {
     188                # recovery files exist and are accessible restore the excised pixels
     189
     190                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     191                    run(command => $muggle_command, verbose => $verbose);
     192                unless ($success) {
     193                    my $exitStatus = WEXITSTATUS($error_code);
     194                    my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus);
     195                }
     196
     197                # set the ppstamp file arguments
     198                $fileArgs = $newFileArgs;
     199            } else {
     200                if ($options & $PSTAMP_REQUIRE_UNCENSORED) {
     201                    my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE);
     202                }
     203                # just make stamps from the censored images
     204                print "Unable to restore uncensored images, will extract stamps from censored images\n";
     205                # these files won't be used so zap them
     206                ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef);
     207            }
     208        }
     209    }
     210
     211    my $command = "$ppstamp $outputBase $argString $fileArgs";
    118212    $command .= " -dbname $dbname" if $dbname;
    119213    $command .= " -dbserver $dbserver" if $dbserver;
    120     $command .= " -stage $params->{stage}" if $params->{stage};
     214    $command .= " -stage $stage";
    121215    $command .= " -no_censor_masked" unless $nan_masked;
    122216    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    131225    }
    132226
     227    if (!$save_temps) {
     228        unlink $tmpImage if $tmpImage;
     229        unlink $tmpMask if $tmpMask;
     230        unlink $tmpVariance if $tmpVariance;
     231        rmdir $tmproot if $tmproot;
     232    }
     233
    133234    if ($exitStatus == 0) {
    134         my $dir = dirname($outputBase);
    135 
    136         my $reglist = "$dir/reglist$job_id";
     235        my $reglist = "$outdir/reglist$job_id";
    137236
    138237        my $F;
    … …  
    363462
    364463sub check_files {
     464    my $error_code = shift;
     465    my $return_code = 1;
    365466    foreach my $f (@_) {
    366467        if (!$ipprc->file_exists($f)) {
    367             my_die( "file $f does not exist:", $job_id, $PSTAMP_GONE, 'stop');
    368         }
    369     }
     468            if ($error_code) {
     469                my_die( "file $f does not exist:", $job_id, $error_code, 'stop');
     470            } else {
     471                print STDERR "file $f does not exist\n";
     472                $return_code = 0;
     473            }
     474        }
     475    }
     476    return $return_code;
     477}
     478
     479sub check_for_backups {
     480    my $params = shift;
     481    my $r_fileArgs = shift;
     482
     483    my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}";
     484    my $results = runToolAndParse($command, $verbose);
     485    my $dsComponent  = $results->[0];
     486
     487    if ($dsComponent and ($dsComponent->{data_state} eq 'full')) {
     488
     489        print "magicDSFile state is full. Backup images should exist\n";
     490
     491        # replace the file names with the backup paths
     492        # fileArgs has the form:  -file imagename [-mask maskname] [-weight weightname]
     493        my @args = split " ", $$r_fileArgs;
     494
     495        my $newFileArgs;
     496        while (@args) {
     497            my $a = shift @args;
     498            my $f = shift @args;
     499            my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f;
     500
     501            my $backup_image_name = $ipprc->destreaked_filename($f);
     502            my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name;
     503            $newFileArgs .= " $a $backup_image_name";
     504        }
     505        $$r_fileArgs = $newFileArgs;
     506
     507        return 1;
     508    }
     509
     510    return 0;
    370511}
    371512

  • branches/eam_branches/ipp-20101205/pstamp/scripts/pstamp_parser_run.pl

    r29576 r30446  
    3131my $outdir;
    3232my $label;
     33my $need_magic;
    3334my $verbose;
    3435my $dbname;
    … …  
    4142    'outdir=s'          =>  \$outdir,
    4243    'label=s'           =>  \$label,
     44    'need_magic'        =>  \$need_magic,
    4345    'redirect-output'   =>  \$redirect_output,
    4446    'verbose'           =>  \$verbose,
    … …  
    160162            $parse_cmd = "$pstampparse";
    161163            $parse_cmd .= " --label $label" if $label;
     164            $parse_cmd .= " --need_magic" if $need_magic;
    162165            $request_fault = 0;
    163166        } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") {

  • branches/eam_branches/ipp-20101205/pstamp/scripts/pstamp_server_status

    r29572 r30446  
    4444chdir $rundir or die "failed to cd to $rundir";
    4545
     46
     47my $down = 0;
     48if ($down) {
     49    print "Postage Stamp Server will be down for maintenance it will back at approximately 00:00 2010-12-11 UTC\n";
     50    exit 0;
     51}
    4652
    4753my ($pts, $pantasks_script) = tempfile ('/tmp/pts.XXXX', UNLINK => !$save_temps);

  • branches/eam_branches/ipp-20101205/pstamp/scripts/pstampparse.pl

    r30103 r30446  
    3030my $save_temps;
    3131my $no_update;
     32my $dest_requires_magic;
    3233
    3334GetOptions(
    … …  
    3839    'label=s'   =>  \$label,
    3940    'mode=s'    =>  \$mode,
     41    'need_magic'=>  \$dest_requires_magic,
    4042    'dbname=s'  =>  \$dbname,
    4143    'dbserver=s'=>  \$dbserver,
    … …  
    126128}
    127129
    128 # Adjust the label for requests coming in over the web interaface
     130
     131# Adjust the label for requests coming in over the web interface
    129132
    130133my $label_changed = 0;
    … …  
    384387    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
    385388    if (!$proj_hash) {
    386         foreach $row (@$rowList) {
    387             insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT);
     389        foreach my $r (@$rowList) {
     390            insertFakeJobForRow($r, 1, $PSTAMP_UNKNOWN_PRODUCT);
    388391            $num_jobs++;
    389392        }
    … …  
    409412    $need_magic    = $proj_hash->{need_magic};
    410413
    411     # Temporary hack so that IFA can get at non-magicked data
    412     my $allow_mops_unmagicked = 1;
    413     if ($allow_mops_unmagicked) {
    414         if ($product and (($product eq "mops-pstamp-results") or
    415                           ($product eq "mops-pstamp-results2") or
    416                           ($product eq "ifa-pstamp-results"))) {
    417             $need_magic = 0;
    418         }
    419     }
    420 
    421414    $need_magic = 0 if $stage eq 'stack';
     415
     416    if ($need_magic) {
     417
     418        # this project requires that postage stamps be extracted from destreaked images
     419
     420        if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) {
     421            # The user has requested uncensored stamps
     422
     423            if (!$dest_requires_magic) {
     424                # and this user's data store destination is allowed uncensored stamps, so accept the request
     425                $need_magic = 0;
     426            } else {
     427                print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
     428                if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) {
     429                    # user required uncensored stamps. Can't do it so fail.
     430                    foreach my $r (@$rowList) {
     431                        insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED);
     432                        $num_jobs++;
     433                    }
     434                    return $num_jobs;
     435                }
     436
     437                # user will accept censored stamps. alter OPTION_MASK and continue
     438
     439                print STDERR "    Will attempt to make destreaked stamps\n";
     440                # zap the offending bit in the option mask
     441                $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED);
     442                foreach my $r (@$rowList) {
     443                    $r->{OPTION_MASK} = $option_mask;
     444                }
     445            }
     446        }
     447    }
    422448   
    423449    my $numRows = scalar @$rowList;

  • branches/eam_branches/ipp-20101205/tools/czarclean.pl

    r29278 r30446  
    66
    77# local classes
     8use czartool::Config;
    89use czartool::CzarDb;
    910
    1011my $czarDbName = undef;
    11 my $from = undef;
    12 my $to = undef;
     12my $begin = undef;
     13my $end = undef;
    1314my $interval = undef;
    1415my $verbose = undef;
    1516my $optimize = undef;
     17my $config = new czartool::Config();
    1618
    1719GetOptions (
    1820        "dbname|d=s" => \$czarDbName,
    1921        "interval|i=s" => \$interval,
    20         "from|f=s" => \$from,
    21         "to|t=s" => \$to,
     22        "begin|b=s" => \$begin,
     23        "end|e=s" => \$end,
    2224        "optimize|o" => \$optimize,
    2325        "verbose|v" => \$verbose,
    … …  
    3234    print "* UNKNKOWN: option                          @ARGV\n";
    3335}
    34 if (!$from) {
     36if (!$begin) {
    3537    $quit=1;
    36     print "* REQUIRED: choose a from date                 -b <datetime>\n";
     38    print "* REQUIRED: choose a begin date                -b <datetime>\n";
    3739}
    38 if (!$to) {
    39     if($from) {$to=$from;} else {$to="NULL";}
    40     print "* OPTIONAL: choose a to date                   -e <datetime>                   (default=$to)\n";
     40if (!$end) {
     41    if($begin) {$end=$begin;} else {$end="NULL";}
     42    print "* OPTIONAL: choose an end date date            -e <datetime>                   (default=$end)\n";
    4143}
    4244if (!$czarDbName) {
    … …  
    4547}
    4648if (!$interval) {
    47     $interval = "30 MINUTE";
    48     print "* OPTIONAL: choose time interval               -i <'30 MINUTE'|'1 hour'|etc>   (default=$interval)\n";
     49    print "* OPTIONAL: choose time interval               -i <'30 MINUTE'|'1 hour'|etc>   (default=".$config->getCzarCleanupInterval().")\n";
    4950}
    5051if (!$optimize) {
    5152    $optimize = 0;
    52     print "* OPTIONAL: optimize database after cleanup    -o <'30 MINUTE'|'1 hour'|etc>   (default=$interval)\n";
     53    print "* OPTIONAL: optimize database after cleanup    -o                              (default=$optimize)\n";
    5354}
    5455print "*\n*******************************************************************************\n";
    … …  
    5657if ($quit) { exit; }
    5758
    58 
    59 my $save_temps = 0;
    60 
    61 
    62 my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg
     59my $czarDb = $config->getCzarDbInstance();
    6360
    6461my $labels = undef;
    6562
    6663
    67 $czarDb->cleanupDateRange($from, $to, $interval);
     64$czarDb->cleanupDateRange($begin, $end, $config->getCzarCleanupInterval());
    6865if ($optimize) {$czarDb->optimize();}

  • branches/eam_branches/ipp-20101205/tools/czarmetrics.pl

    r29868 r30446  
    88use CGI::Pretty qw[:standard];
    99
     10use czartool::Config;
    1011use czartool::DayMetrics;
    1112use czartool::MultiDayMetrics;
    1213use czartool::MetricsIndex;
    13 use czartool::CzarDb;
    14 use czartool::Gpc1Db;
    1514
    1615my $czarDbName = "czardb";
    … …  
    6059if ($quit) {exit;}
    6160
    62 my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
    63 my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", $verbose, $save_temps);
    64 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
     61my $config = new czartool::Config();
    6562
    6663if ($index) {
    6764
    68     my $metricsIndex = new czartool::MetricsIndex($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0);
     65    my $metricsIndex = new czartool::MetricsIndex($config, 1, 0);
    6966    $metricsIndex->writeHTML();
    7067    exit;
    … …  
    8279if ($cumulative) {
    8380
    84         my $multiDayMetrics = new czartool::MultiDayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $begin, $end);
     81        my $multiDayMetrics = new czartool::MultiDayMetrics($config, 1, 0, $begin, $end);
    8582        $multiDayMetrics->writeHTML();
    8683}
    8784else {
    8885
     86    my $czarDb = $config->getCzarDbInstance();
     87
    8988    while (1) {
    9089
    91         my $dayMetrics = new czartool::DayMetrics($gpc1Db, $czarDb, "/data/ipp004.0/ipp/ippMetrics/", 1, 0, $thisDay);
     90        my $dayMetrics = new czartool::DayMetrics($config, 1, 0, $thisDay);
    9291        $dayMetrics->writeHTML();
    9392

  • branches/eam_branches/ipp-20101205/tools/czarplot.pl

    r29952 r30446  
    77use POSIX qw/strftime/;
    88
    9 use czartool::CzarDb;
    10 use czartool::Gpc1Db;
     9use czartool::Config;
    1110use czartool::Plotter;
    1211use czartool::StageMetrics;
    1312
    14 my $czarDbName = "czardb";
    1513my $label = undef;
    1614my $stage = undef;
    … …  
    3634
    3735GetOptions (
    38         "dbname=s" => \$czarDbName,
    3936        "label|l=s" => \$label,
    4037        "stage|s=s" => \$stage,
    … …  
    114111if ($quit) { exit; }
    115112
     113my $config = new czartool::Config();
     114
    116115# default values
    117116if (!$rate && !$magicMask && !$nebulous && !$histogram && !$analysis && !$timeSeries) {$timeSeries = 1;}
    … …  
    121120else {$savingToFile = 1;}
    122121
    123 my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
    124 my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", $verbose, $save_temps);
    125 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
     122my $czarDb = $config->getCzarDbInstance();
    126123
     124my $newDayTime =  $config->getMetricsStartTime();
    127125
    128126my $plotter = undef;
    129127if ($savingToFile) {
    130128
    131     $plotter = czartool::Plotter->new_file(
    132             $gpc1Db,
    133             $czarDb,
    134             $path,
    135             $save_temps);
     129    $plotter = czartool::Plotter->new_file($config, $path, $save_temps);
    136130}
    137131else {
    138132
    139     $plotter = czartool::Plotter->new_display(
    140             $gpc1Db,
    141             $czarDb,
    142             $save_temps);
     133    $plotter = czartool::Plotter->new_display($config, $save_temps);
    143134}
    144135
    145 # sort out times
     136# if a single day has been chosen
    146137if($day) {
    147138
    148139    if ($magicMask) {
    149140
    150         $begin =  $day;
     141        $begin = $day;
    151142        $end = $day;
    152143    }
    153144    else {
    154145
    155         # day plots should run from about 6:30am until midnight
    156         $begin = "$day 06:35";
    157         $end = "$day 23:59";
     146        # plots should span 24 housr from previous day
     147        my $dayBefore = $czarDb->subtractInterval($day, "1 DAY");
     148               
     149        $begin = "$dayBefore $newDayTime";
     150        $end = "$day $newDayTime";
    158151    }
    159152}
    … …  
    163156    if (!$begin) {
    164157
     158        my $today = strftime('%Y-%m-%d', localtime);
     159        my $yesterday = $czarDb->subtractInterval($today, "1 DAY");
     160
     161       
    165162        if ($timeinpast) {$begin = $czarDb->subtractInterval($end, $timeinpast);}
    166         else {$begin =  strftime('%Y-%m-%d 06:35',localtime);}
     163        elsif ($czarDb->isBefore($end, "$today $newDayTime")) {$begin = "$yesterday $newDayTime";}
     164        else {$begin = "$today $newDayTime";}
    167165    }
    168166}
     167
    169168if ($rate) {
    170169

  • branches/eam_branches/ipp-20101205/tools/czarpoll.pl

    r30102 r30446  
    11#!/usr/bin/perl -w
     2
     3###########################################################################
     4#
     5# Main polling program for czartool. Grabs stuff from various sources every few minutes.
     6#
     7###########################################################################
    28
    39use warnings;
    … …  
    713
    814# local classes
     15use czartool::Config;
    916use czartool::CzarDb;
    1017use czartool::Gpc1Db;
    … …  
    1724
    1825my $period = 60;
    19 my $czarDbName = "czardb"; # TODO variables for other Db stuff, host etc
    2026my $save_temps = 0;
    2127
    2228GetOptions (
    2329        "period|p=s" => \$period, # TODO more Db args
    24         "dbname|d=s" => \$czarDbName,
    2530        );
    2631
    27 my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg
    28 my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
     32my $config = new czartool::Config();
     33my $czarDb = $config->getCzarDbInstance();
     34my $gpc1Db = $config->getGpc1Instance;
    2935my $nebulous = new czartool::Nebulous($czarDb);
    3036my $pantasks = new czartool::Pantasks();
    31 my $plotter = czartool::Plotter->new_file($gpc1Db, $czarDb, "/tmp", $save_temps);
     37my $plotter = czartool::Plotter->new_file($config, "/tmp", $save_temps);
    3238my $burntool = new czartool::Burntool();
    3339
    3440
    35 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
    36 
    3741my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
    3842
    3943
    4044timePoll($period);
     45
     46###########################################################################
     47#
     48# Updates the dates from pantasks for all interested servers
     49#
     50###########################################################################
     51sub updateDates {
     52
     53    print "* Updating dates\n";
     54    my @servers = ("stdscience", "registration");
     55
     56    my $server = undef;
     57    $czarDb->emptyServerDates();
     58    foreach $server (@servers) {
     59
     60        my @dates = @{$pantasks->getDates($server)};
     61        if (@dates) {
     62
     63            $czarDb->updateServerDates($server, \@dates);
     64        }
     65        else {
     66
     67            print "WARNING: No dates to update for '$server'\n";
     68        }
     69    }
     70}
    4171
    4272###########################################################################
    … …  
    5585        my @labels = @{$pantasks->getLabels($server)};
    5686        if (@labels) {
    57        
     87
    5888            $czarDb->updateCurrentLabels($server, \@labels);
    5989        }
    6090        else {
    61        
    62              print "WARNING: No labels to update for '$server'\n";
     91
     92            print "WARNING: No labels to update for '$server'\n";
    6393        }
    6494    }
    … …  
    110140    my $today = undef;
    111141    my $yesterday = undef;
    112     my $newDayTime = "18:00";
     142    my $newDayTime = $config->getMetricsStartTime();
    113143    my $lastDayDailyTasks = "2010-01-01";
    114144
    … …  
    125155        # if after 18:00 today, then perform some daily tasks and start plots from 18:00 today
    126156        else {
    127            
     157
    128158            $begin = "$today $newDayTime";
    129159
    130160            # check whether yesterday was cleaned. if not, cleanup tables and optimize
    131161            if ($lastDayDailyTasks ne $yesterday) {
    132    
     162
    133163                print "* performing daily tasks (clean-up, metrics)";
    134164
    135165                # create metrics for last 24 hours
    136166                print "* Creating metrics for last 24 hours\n";
    137                 # TODO hardcoded path needs to be in config
    138                 my $dayMetrics = new czartool::DayMetrics($gpc1Db,
    139                         $czarDb,
    140                         "/data/ipp004.0/ipp/ippMetrics/",
    141                         1, 0, $today);
     167                my $dayMetrics = new czartool::DayMetrics($config, 1, 0, $today);
    142168                $dayMetrics->writeHTML();
    143169
    144170                # now update metrics index page
    145                 my $metricsIndex = new czartool::MetricsIndex($gpc1Db,
    146                         $czarDb,
    147                         "/data/ipp004.0/ipp/ippMetrics/",
    148                         1, 0);
     171                my $metricsIndex = new czartool::MetricsIndex($config, 1, 0);
    149172                $metricsIndex->writeHTML();
    150173
    151174                # now cleanup tables from yesterday and optimize
    152175                print "* New day - performing cleanup\n";
    153                 $czarDb->cleanupDateRange($yesterday, $yesterday, "30 MINUTE");
     176                $czarDb->cleanupDateRange($yesterday, $yesterday, $config->getCzarCleanupInterval());
    154177                $czarDb->optimize();
    155178                $lastDayDailyTasks = $yesterday;
    … …  
    167190        $plotter->plotDiskUsageHistogram();
    168191        updateServerStatus();
     192
     193        # check pantasks dates
     194        updateDates();
    169195
    170196        # check labels

  • branches/eam_branches/ipp-20101205/tools/czartool.pl

    r28925 r30446  
    66
    77# local classes
     8use czartool::Config;
    89use czartool::CzarDb;
    910use czartool::Gpc1Db;
    … …  
    2223
    2324my @states = ("full", "new", "drop", "wait");
    24 my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp");
    25 my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
     25my $config = new czartool::Config();
     26my $czarDb = $config->getCzarDbInstance();
     27my $gpc1Db = $config->getGpc1Instance();
    2628my $pantasks = new czartool::Pantasks();
    27 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
    2829
    2930my @stdscienceLabels = undef;

  • branches/eam_branches/ipp-20101205/tools/czartool/CzarDb.pm

    r29865 r30446  
    152152
    153153    $query->execute;
     154}
     155
     156###########################################################################
     157#
     158# Cleans out server_dates table
     159#
     160###########################################################################
     161sub emptyServerDates {
     162    my ($self, $server, $dates) = @_;
     163
     164    my $query = $self->{_db}->prepare(<<SQL);
     165    DELETE FROM server_dates;
     166SQL
     167
     168    $query->execute;
     169}
     170
     171###########################################################################
     172#
     173# Updates current_labels table
     174#
     175###########################################################################
     176sub updateServerDates {
     177    my ($self, $server, $dates) = @_;
     178
     179    my $size = scalar @{$dates};
     180    if ($size < 1) { return; }
     181
     182    my $date = undef;
     183
     184    foreach $date (@{$dates}) {
     185
     186        my $query = $self->{_db}->prepare(<<SQL);
     187        INSERT INTO server_dates
     188            (server, date)
     189            VALUES
     190            ('$server', '$date');
     191SQL
     192
     193       $query->execute;
     194    }
    154195}
    155196
    … …  
    12431284        elsif ($currentRevision == 10) {$self->createRevision_11();}
    12441285        elsif ($currentRevision == 11) {$self->createRevision_12();}
     1286        elsif ($currentRevision == 12) {$self->createRevision_13();}
    12451287        else {last;}
    12461288    }
    … …  
    16021644# Create revision 12 of the database
    16031645#
     1646# - adding nightlyscience table
     1647#
    16041648#######################################################################################
    16051649sub createRevision_12 {
    … …  
    16081652    print "* Creating revision 12 of '$self->{_dbName}'\n";
    16091653
    1610     # same shape as other stage tables to enable easy update
    16111654    my $query = $self->{_db}->prepare(<<SQL);
    16121655    CREATE TABLE nightlyscience (
    … …  
    16191662    $self->setRevision(12);
    16201663}
     1664#######################################################################################
     1665#
     1666# Create revision 13 of the database
     1667#
     1668# adding server_dates table
     1669#
     1670#######################################################################################
     1671sub createRevision_13 {
     1672    my ($self) = @_;
     1673
     1674    print "* Creating revision 13 of '$self->{_dbName}'\n";
     1675
     1676    my $query = $self->{_db}->prepare(<<SQL);
     1677    CREATE TABLE server_dates (
     1678            timestamp TIMESTAMP DEFAULT NOW(),
     1679            server VARCHAR(64) DEFAULT "NONE",
     1680            date VARCHAR(64) DEFAULT "NONE");
     1681SQL
     1682
     1683        $query->execute;
     1684
     1685    $self->setRevision(13);
     1686}
    162116871;
    16221688

  • branches/eam_branches/ipp-20101205/tools/czartool/DayMetrics.pm

    r30102 r30446  
    2828    # Call the constructor of the parent class
    2929    my $self = $class->SUPER::new(
    30             $_[1],  # gpc1Db
    31             $_[2],  # czarDb
    32             $_[3],  # baseDir
    33             $_[4],  # verbose
    34             $_[5]   # save_temps
     30            $_[1],  # config object
     31            $_[2],  # verbose
     32            $_[3]   # save_temps
    3533            );
    3634
    37     $self->{day} = $_[6];
     35    $self->{day} = $_[4];
    3836
    3937    my $yesterday =  $self->{czarDb}->subtractInterval($self->{day}, "1 DAY");
    4038
    4139    # sort out times
    42     $self->{begin} =  "$yesterday 18:00";
     40    $self->{begin} =  "$yesterday " . $self->{config}->getMetricsStartTime();
    4341    $self->{end} = $self->{czarDb}->addInterval($self->{begin}, "1 DAY");
    4442    $self->{burntoolEnd} = $self->{czarDb}->addInterval($self->{begin}, "12 HOUR");

  • branches/eam_branches/ipp-20101205/tools/czartool/Metrics.pm

    r30102 r30446  
    2424    my $class = shift;
    2525    my $self = {
    26         gpc1Db => shift,
    27         czarDb => shift,
    28         baseDir => shift,
     26        config => shift,
    2927        verbose => shift,
    3028        save_temps => shift,
    3129    };
     30
     31    $self->{gpc1Db} = $self->{config}->getGpc1Instance();
     32    $self->{czarDb} = $self->{config}->getCzarDbInstance();
     33    $self->{baseDir} = $self->{config}->getMetricsSaveLocation();
    3234
    3335    # create path, dir
    … …  
    3638    # instantiate a plotter object
    3739    $self->{plotter} = czartool::Plotter->new_file(
    38             $self->{gpc1Db},
    39             $self->{czarDb},
     40            $self->{config},
    4041            ".",
    4142            $self->{save_temps});

  • branches/eam_branches/ipp-20101205/tools/czartool/MetricsIndex.pm

    r29859 r30446  
    2828    # Call the constructor of the parent class
    2929    my $self = $class->SUPER::new(
    30             $_[1],  # gpc1Db
    31             $_[2],  # czarDb
    32             $_[3],  # baseDir
    33             $_[4],  # verbose
    34             $_[5]   # save_temps
     30            $_[1],  # config object
     31            $_[2],  # verbose
     32            $_[3]   # save_temps
    3533            );
    3634

  • branches/eam_branches/ipp-20101205/tools/czartool/MultiDayMetrics.pm

    r29860 r30446  
    2828    # Call the constructor of the parent class
    2929    my $self = $class->SUPER::new(
    30             $_[1],  # gpc1Db
    31             $_[2],  # czarDb
    32             $_[3],  # baseDir
    33             $_[4],  # verbose
    34             $_[5]   # save_temps
     30            $_[1],  # config object
     31            $_[2],  # verbose
     32            $_[3]   # save_temps
    3533            );
    3634
    37     $self->{startDay} = $_[6];
    38     $self->{endDay} = $_[7];
     35    $self->{startDay} = $_[4];
     36    $self->{endDay} = $_[5];
    3937
    4038    # sort out times
    41     $self->{begin} =  "$self->{startDay} 06:00";
    42     $self->{end} =  "$self->{endDay} 06:00";
     39    $self->{begin} =  "$self->{startDay} " . $self->{config}->getMetricsStartTime();
     40    $self->{end} =  "$self->{endDay} " . $self->{config}->getMetricsStartTime();
    4341
    4442    # create path, dir and html file

  • branches/eam_branches/ipp-20101205/tools/czartool/Pantasks.pm

    r29076 r30446  
    120120        chomp($line);
    121121        if (!$self->outputOk($line)) {return \@labels;}
    122         if ($line =~ m/pantasks:\s+/) {$passedHeader=1; next;}
    123         if ($passedHeader) {push(@labels, $line);}
     122
     123        # sometimes a label can be on the same line as the panstasks prompt
     124        if ($line =~ m/pantasks: (.*)/) {
     125            $line = $1;
     126            $passedHeader=1;
     127        }
     128
     129        if ($passedHeader && length($line) > 0){push(@labels, $line);}
    124130    }
    125131
    … …  
    191197    }
    192198}
     199
     200###########################################################################
     201#
     202# Gets current dates for this server
     203#
     204###########################################################################
     205sub getDates {
     206    my ($self, $server) = @_;
     207
     208    my @dates;
     209
     210    my $prefix;
     211
     212    if ($server eq "stdscience") {$prefix = "ns";}
     213    elsif ($server eq "registration") {$prefix = "register";}
     214
     215    my @cmdOut = `echo "$prefix.show.dates;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
     216    my $line;
     217    my $passedHeader=0;
     218    foreach $line (@cmdOut) {
     219
     220        chomp($line);
     221        if (!$self->outputOk($line)) {return \@dates;}
     222
     223        # sometimes content is on same line as pantasks prompt
     224        if ($line =~ m/pantasks: (.*)/) {
     225            $line =~ s/pantasks: //;
     226                $passedHeader=1;
     227        }
     228
     229        if ($passedHeader && length($line) > 0){
     230
     231            # strip out only the date part
     232            if ($line =~ m/([0-9]{4}-[0-9]{2}-[0-9]{2}).*/) {
     233
     234                push(@dates, $1);
     235            }
     236        }
     237    }
     238
     239    return \@dates;
     240}
     241
     242
    1932431;

  • branches/eam_branches/ipp-20101205/tools/czartool/Plotter.pm

    r29951 r30446  
    77
    88use File::Temp;
     9
     10use czartool::Config;
    911
    1012my @allStages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
    … …  
    1820    my $class = shift;
    1921    my $self = {
    20         _gpc1Db => shift,
    21         _czarDb => shift,
     22        _config => shift,
    2223        _dateFormat => shift,
    2324        _outputFormat => shift,
    … …  
    2728
    2829    bless $self, $class;
     30    $self->init();
    2931    return $self;
    3032}
    … …  
    3840    my $class = shift;
    3941    my $self = {
    40         _gpc1Db => shift,
    41         _czarDb => shift,
     42        _config => shift,
    4243        _outputPath => shift,
    4344        _save_temps => shift,
    4445                                                    };
    45 
    4646    $self->{_dateFormat} = "%Y%m%d-%H%M%S";
    47     $self->{_outputFormat} = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8"; # TODO path needs to be in config
    48 
    49         bless $self, $class;
     47    $self->{_outputFormat} = "png font \"".$self->{_config}->getGnuplotFont()."\" ".$self->{_config}->getGnuplotFontSize();
     48
     49    bless $self, $class;
     50    $self->init();
    5051    return $self;
    5152}
    … …  
    5960    my $class = shift;
    6061    my $self = {
    61         _gpc1Db => shift,
    62         _czarDb => shift,
     62        _config => shift,
    6363        _save_temps => shift,
    6464    };
    … …  
    6969
    7070    bless $self, $class;
     71    $self->init();
    7172    return $self;
    7273}
    7374
    74 
     75###########################################################################
     76#
     77# Some initialisation code common to all constructors
     78#
     79###########################################################################
     80sub init {
     81    my ($self) = @_;
     82
     83    $self->{_gpc1Db} = $self->{_config}->getGpc1Instance();
     84    $self->{_czarDb} = $self->{_config}->getCzarDbInstance();
     85}
    7586
    7687###########################################################################
    … …  
    508519    GP->autoflush(1);
    509520
     521    my $datFile = $tmpFile->filename;
    510522    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
    511     print GP
    512         "set term $self->{_outputFormat};" .
    513         "set title \"Total available cluster space over time\\nFrom $fromTime to $toTime HST\";" .
    514         "set key left top;" .
    515         "set xdata time;" .
    516         "set timefmt \"$self->{_dateFormat}\";" .
    517         "set format x \"$timeFormat\";" .
    518         "set xtics \"$minX\", $divX, \"$maxX\";" .
    519         "set grid xtics;" .
    520         "set xlabel \"Time\";" .
    521         "set ylabel \"Available (TB)\";" .
    522         "plot " .
    523         "'".$tmpFile->filename."' using 1:2 title \"Available\" with lines lt 2 lw 2\n";
     523    print GP <<PLOT;
     524        set term $self->{_outputFormat}
     525        set title "Total available cluster space over time\\nFrom $fromTime to $toTime HST"
     526        set key left top
     527        set xdata time
     528        set timefmt "$self->{_dateFormat}"
     529        set format x "$timeFormat"
     530        set xtics "$minX", $divX, "$maxX"
     531        set grid xtics
     532        set xlabel "Time"
     533        set ylabel "Available (TB)"
     534        plot "$datFile" using 1:2 title "Available" with lines lt 2 lw 2
     535PLOT
    524536
    525537    print GP "\n";

  • branches/eam_branches/ipp-20101205/tools/roboczar.pl

    r29804 r30446  
    77
    88# local classes
     9use czartool::Config;
    910use czartool::CzarDb;
    1011use czartool::Gpc1Db;
    … …  
    1516use czartool::StageMetrics;
    1617
    17 my $czarDbName = "czardb"; # TODO variables for other Db stuff, host etc
    1818my $save_temps = 0;
    1919
    20 GetOptions (
    21         "dbname|d=s" => \$czarDbName,
    22         );
    23 
    24 my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg
    25 
    26 $czarDb->setDateFormat("%Y%m%d-%H%i%s");
     20my $config = new czartool::Config();
     21my $czarDb = $config->getCzarDbInstance();
    2722
    2823my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO get from Pantasks
    … …  
    4136    $message = "";
    4237
    43     if (anyStoppedServers("20 MINUTE", \$serversMessage)) {
     38    if (anyStoppedServers($config->getRoboczarServerInterval(), \$serversMessage)) {
    4439
    4540        $message .= "\n\n" . $serversMessage;
    4641        $anythingToReport = 1;
    4742    }
    48     if (anyStuckStages("2 HOUR", \$stuckMessage)) {
     43#    if (anyStuckStages("2 HOUR", \$stuckMessage)) {
    4944
    50         $message .= "\n\n" . $stuckMessage;
    51         $anythingToReport = 1;
    52     }
     45 #       $message .= "\n\n" . $stuckMessage;
     46  #      $anythingToReport = 1;
     47   # }
    5348
    5449    if ($anythingToReport && $message ne $lastMessage) {
    … …  
    5651print "\n\n$message\n\n";
    5752
    58         sendEmail(
    59                 "ps-ipp-ops\@ifa.hawaii.edu",
     53        sendEmail($config->getRoboczarEmail(),
    6054                "Roboczar warnings",
    6155                "$message\n");
    … …  
    8276    my $anyStuckStages = 0;
    8377
    84     # exception - we don't care if burntool is stalled before 6:30am
     78    # exception - we don't care if burntool is stalled before 6:30am TODO we do now
    8579    my $burntime = strftime('%Y-%m-%d 06:35', localtime);
    8680    my $worryAboutBurntool = $czarDb->isBefore($burntime, $end);
Note: See TracChangeset for help on using the changeset viewer.