IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 27315


Ignore:
Timestamp:
Mar 17, 2010, 4:15:53 PM (16 years ago)
Author:
watersc1
Message:

Updated "split" version of lossy compression, with tasks to do compression and cleaning in parallel. This should help us recover disk space quicker.

Location:
trunk
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/lossy_compress.pro

    r27193 r27315  
    11## lossy_compress.pro : tasks to lossy compress old raw images that have negligible science value : -*- sh -*-
    2 ## use the books compPendingImfile and compPendingExp
     2## use the books compPendingCompress, compPendingClean and compPendingExp
    33
    44# test for required global variables
    55check.globals
    66
    7 book init compPendingImfile
     7book init compPendingCompress
     8book init compPendingClean
    89book init compPendingExp
    910
    1011macro lossycomp.reset
    11     book init compPendingImfile
     12    book init compPendingCompress
     13    book init compPendingClean
    1214    book init compPendingExp
    1315end
    … …  
    1517macro lossycomp.status
    1618    book list
    17     book listbook compPendingImfile
     19    book listbook compPendingCompress
     20    book listbook compPendingClean
    1821    book listbook compPendingExp
    1922end
    2023
    2124macro lossycomp.on
    22     task lossycomp.imfile.load
    23         active true
    24     end
    25     task lossycomp.imfile.run
     25    task lossycomp.compress.load
     26        active true
     27    end
     28    task lossycomp.compress.run
     29        active true
     30    end
     31    task lossycomp.clean.load
     32        active true
     33    end
     34    task lossycomp.clean.run
    2635        active true
    2736    end
    … …  
    3241
    3342macro lossycomp.off
    34     task lossycomp.imfile.load
    35         active false
    36     end
    37     task lossycomp.imfile.run
     43    task lossycomp.compress.load
     44        active false
     45    end
     46    task lossycomp.compress.run
     47        active false
     48    end
     49    task lossycomp.clean.load
     50        active false
     51    end
     52    task lossycomp.clean.run
    3853        active false
    3954    end
    … …  
    4459
    4560# these variables will cycle through the known database names
    46 $compPendingImfile_DB = 0
     61$compPendingCompress_DB = 0
     62$compPendingClean_DB = 0
    4763$comp_finish_DB = 0
    4864
    4965# select images ready to be compressed
    50 task           lossycomp.imfile.load
     66task           lossycomp.compress.load
    5167    host       local
    5268
    … …  
    5773
    5874    stdout     NULL
    59     stderr     $LOGDIR/lossycomp.imfile.load.log
     75    stderr     $LOGDIR/lossycomp.compress.load.log
    6076
    6177    #select entries from the current DB, cycle to the next DB if possible
    6278    task.exec
    63         $run = regtool -pendingcompressimfile
     79        $run = regtool -pendingcompressimfile -compress
    6480        if ($DB:n == 0)
    6581            option DEFAULT
    6682        else
    6783            # save the DB name for the exit tasks
    68             option $DB:$compPendingImfile_DB
    69             $run = $run -dbname $DB:$compPendingImfile_DB
    70             $compPendingImfile_DB ++
    71             if ($compPendingImfile_DB >= $DB:n) set compPendingImfile_DB = 0
     84            option $DB:$compPendingCompress_DB
     85            $run = $run -dbname $DB:$compPendingCompress_DB
     86            $compPendingCompress_DB ++
     87            if ($compPendingCompress_DB >= $DB:n) set compPendingCompress_DB = 0
    7288        end
    7389        add_poll_args run
    … …  
    7894    task.exit $EXIT_SUCCESS
    7995        # convert 'stdout' to book format
    80         ipptool2book stdout compPendingImfile -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
    81         book shuffle compPendingImfile
     96        ipptool2book stdout compPendingCompress -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     97        book shuffle compPendingCompress
    8298        if ($VERBOSE > 2)
    83             book listbook compPendingImfile
     99            book listbook compPendingCompress
    84100        end
    85101       
    86102        # delete existing entries in the appropriate pantaskStates
    87         process_cleanup compPendingImfile
     103        process_cleanup compPendingCompress
    88104    end
    89105
    … …  
    100116
    101117# run the lossy_compress_imfile.pl script on the pending images
    102 task           lossycomp.imfile.run
     118task           lossycomp.compress.run
    103119    periods    -poll $RUNPOLL
    104120    periods    -exec $RUNEXEC
    … …  
    107123    task.exec
    108124        if ($NETWORK == 0) break
    109         book npages compPendingImfile -var N
     125        book npages compPendingCompress -var N
    110126        if ($N == 0)
    111127            periods -exec $RUNEXEC
    … …  
    115131
    116132        # look for new images
    117         book getpage compPendingImfile 0 -var pageName -key pantaskState INIT
     133        book getpage compPendingCompress 0 -var pageName -key pantaskState INIT
    118134        if ("$pageName" == "NULL") break
    119135
    120         book setword compPendingImfile $pageName pantaskState RUN
    121         book getword compPendingImfile $pageName exp_id        -var EXP_ID
    122         book getword compPendingImfile $pageName exp_name      -var EXP_NAME
    123         book getword compPendingImfile $pageName exp_tag       -var EXP_TAG
    124         book getword compPendingImfile $pageName tmp_class_id  -var TMP_CLASS_ID
    125         book getword compPendingImfile $pageName class_id      -var CLASS_ID
    126         book getword compPendingImfile $pageName camera        -var CAMERA
    127         book getword compPendingImfile $pageName uri           -var URI
    128         book getword compPendingImfile $pageName bytes         -var BYTES
    129         book getword compPendingImfile $pageName md5sum        -var MD5SUM
    130         book getword compPendingImfile $pageName workdir       -var WORKDIR
    131         book getword compPendingImfile $pageName data_state    -var STATE
    132         book getword compPendingImfile $pageName dbname        -var DBNAME
     136        book setword compPendingCompress $pageName pantaskState RUN
     137        book getword compPendingCompress $pageName exp_id        -var EXP_ID
     138        book getword compPendingCompress $pageName exp_name      -var EXP_NAME
     139        book getword compPendingCompress $pageName exp_tag       -var EXP_TAG
     140        book getword compPendingCompress $pageName tmp_class_id  -var TMP_CLASS_ID
     141        book getword compPendingCompress $pageName class_id      -var CLASS_ID
     142        book getword compPendingCompress $pageName camera        -var CAMERA
     143        book getword compPendingCompress $pageName uri           -var URI
     144        book getword compPendingCompress $pageName bytes         -var BYTES
     145        book getword compPendingCompress $pageName md5sum        -var MD5SUM
     146        book getword compPendingCompress $pageName workdir       -var WORKDIR
     147        book getword compPendingCompress $pageName data_state    -var STATE
     148        book getword compPendingCompress $pageName dbname        -var DBNAME
    133149
    134150        # specify choice of remote host
    … …  
    142158        end
    143159
    144         stderr $LOGDIR/lossycomp.imfile.run.log
    145         stdout $LOGDIR/lossycomp.imfile.run.log
     160        stderr $LOGDIR/lossycomp.compress.run.log
     161        stdout $LOGDIR/lossycomp.compress.run.log
    146162
    147163        $run = lossy_compress_imfile.pl --exp_id $EXP_ID --class_id $CLASS_ID --exp_name $EXP_NAME --uri $URI --camera $CAMERA --state $STATE  --logfile $logfile --bytes $BYTES --md5sum $MD5SUM
    … …  
    160176    # default exit status
    161177    task.exit default
    162         process_exit compPendingImfile $options:0 $JOB_STATUS
     178        process_exit compPendingCompress $options:0 $JOB_STATUS
    163179    end
    164180
    … …  
    166182        showcommand crash
    167183        echo "hostname: $JOB_HOSTNAME"
    168         book setword compPendingImfile $options:0 pantaskState CRASH
     184        book setword compPendingCompress $options:0 pantaskState CRASH
    169185    end
    170186    task.exit timeout
    171187        showcommand timeout
    172         book setword compPendingImfile $options:0 pantaskState TIMEOUT
     188        book setword compPendingCompress $options:0 pantaskState TIMEOUT
     189    end
     190end
     191# select images ready to be compressed
     192task           lossycomp.clean.load
     193    host       local
     194
     195    periods    -poll $LOADPOLL
     196    periods    -exec $LOADEXEC
     197    periods    -timeout 30
     198    npending   1
     199
     200    stdout     NULL
     201    stderr     $LOGDIR/lossycomp.clean.load.log
     202
     203    #select entries from the current DB, cycle to the next DB if possible
     204    task.exec
     205        $run = regtool -pendingcompressimfile -clean
     206        if ($DB:n == 0)
     207            option DEFAULT
     208        else
     209            # save the DB name for the exit tasks
     210            option $DB:$compPendingClean_DB
     211            $run = $run -dbname $DB:$compPendingClean_DB
     212            $compPendingClean_DB ++
     213            if ($compPendingClean_DB >= $DB:n) set compPendingClean_DB = 0
     214        end
     215        add_poll_args run
     216        command $run
     217    end
     218
     219    # success
     220    task.exit $EXIT_SUCCESS
     221        # convert 'stdout' to book format
     222        ipptool2book stdout compPendingClean -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     223        book shuffle compPendingClean
     224        if ($VERBOSE > 2)
     225            book listbook compPendingClean
     226        end
     227       
     228        # delete existing entries in the appropriate pantaskStates
     229        process_cleanup compPendingClean
     230    end
     231
     232    task.exit  default
     233        showcommand failure
     234    end
     235    task.exit  crash
     236        showcommand crash
     237    end
     238    task.exit  timeout
     239        showcommand timeout
     240    end
     241end
     242
     243# run the lossy_compress_imfile.pl script on the pending images
     244task           lossycomp.clean.run
     245    periods    -poll $RUNPOLL
     246    periods    -exec $RUNEXEC
     247    periods    -timeout 30
     248
     249    task.exec
     250        if ($NETWORK == 0) break
     251        book npages compPendingClean -var N
     252        if ($N == 0)
     253            periods -exec $RUNEXEC
     254            break
     255        end
     256        periods -exec 0.05
     257
     258        # look for new images
     259        book getpage compPendingClean 0 -var pageName -key pantaskState INIT
     260        if ("$pageName" == "NULL") break
     261
     262        book setword compPendingClean $pageName pantaskState RUN
     263        book getword compPendingClean $pageName exp_id        -var EXP_ID
     264        book getword compPendingClean $pageName exp_name      -var EXP_NAME
     265        book getword compPendingClean $pageName exp_tag       -var EXP_TAG
     266        book getword compPendingClean $pageName tmp_class_id  -var TMP_CLASS_ID
     267        book getword compPendingClean $pageName class_id      -var CLASS_ID
     268        book getword compPendingClean $pageName camera        -var CAMERA
     269        book getword compPendingClean $pageName uri           -var URI
     270        book getword compPendingClean $pageName bytes         -var BYTES
     271        book getword compPendingClean $pageName md5sum        -var MD5SUM
     272        book getword compPendingClean $pageName workdir       -var WORKDIR
     273        book getword compPendingClean $pageName data_state    -var STATE
     274        book getword compPendingClean $pageName dbname        -var DBNAME
     275
     276        # specify choice of remote host
     277        set.host.for.camera $CAMERA $TMP_CLASS_ID
     278
     279        # set logfile name
     280        if ("$WORKDIR" == "NULL")
     281            sprintf logfile "compress_log/%s.%d.lossycomp.%s.log" $EXP_NAME $EXP_ID $CLASS_ID
     282        else
     283            sprintf logfile "%s/%s/%s.lossycomp.%s.log" $WORKDIR $EXP_TAG $EXP_TAG $TMP_CLASS_ID
     284        end
     285
     286        stderr $LOGDIR/lossycomp.clean.run.log
     287        stdout $LOGDIR/lossycomp.clean.run.log
     288
     289        $run = lossy_compress_imfile.pl --exp_id $EXP_ID --class_id $CLASS_ID --exp_name $EXP_NAME --uri $URI --camera $CAMERA --state $STATE  --logfile $logfile --bytes $BYTES --md5sum $MD5SUM
     290        add_standard_args run
     291
     292        # save the pageName for future reference below
     293        options $pageName
     294
     295        # create the command line
     296        if ($VERBOSE > 1)
     297            echo command $run
     298        end
     299        command $run
     300    end
     301   
     302    # default exit status
     303    task.exit default
     304        process_exit compPendingClean $options:0 $JOB_STATUS
     305    end
     306
     307    task.exit crash
     308        showcommand crash
     309        echo "hostname: $JOB_HOSTNAME"
     310        book setword compPendingClean $options:0 pantaskState CRASH
     311    end
     312    task.exit timeout
     313        showcommand timeout
     314        book setword compPendingClean $options:0 pantaskState TIMEOUT
    173315    end
    174316end

  • trunk/ippTools/share/regtool_pendingcompressimfile.sql

    r27168 r27315  
    11SELECT rawImfile.*,rawExp.workdir,rawExp.exp_tag from rawImfile
    22       JOIN rawExp USING(exp_id)
    3        WHERE ((data_state = 'goto_compressed' AND state = 'goto_compressed')
    4            OR (data_state = 'goto_lossy' AND state = 'goto_lossy'))
     3       WHERE  1
     4-- ((data_state = 'goto_compressed' AND state = 'goto_compressed')
     5--                 OR (data_state = 'goto_lossy' AND state = 'goto_lossy'))
    56-- where hook %s
    67-- limit hook %s       

  • trunk/ippTools/src/regtool.c

    r27210 r27315  
    584584    psStringPrepend(&whereClause, "\n AND ");
    585585  }
     586  PXOPT_LOOKUP_BOOL(compress, config->args, "-compress", false);
     587  PXOPT_LOOKUP_BOOL(clean,    config->args, "-clean", false);
     588
     589  if ((compress && clean)||(!compress & !clean)) {
     590    psStringAppend(&whereClause, "\n AND ((data_state = 'goto_compressed' AND state = 'goto_compressed')\n  OR (data_state = 'goto_lossy' AND state = 'goto_lossy')) ");
     591  }
     592  else if (compress) {
     593    psStringAppend(&whereClause, "\n AND ((data_state = 'goto_compressed' AND state = 'goto_compressed')) ");
     594  }
     595  else if (clean) {
     596    psStringAppend(&whereClause, "\n AND ((data_state = 'goto_lossy' AND state = 'goto_lossy')) ");
     597  }
     598  else {
     599    psStringAppend(&whereClause, "\n AND ((data_state = 'goto_compressed' AND state = 'goto_compressed')\n  OR (data_state = 'goto_lossy' AND state = 'goto_lossy')) ");
     600  }   
    586601 
    587602  PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    588603  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    589 
     604 
    590605  psString query = pxDataGet("regtool_pendingcompressimfile.sql");
    591606  if (!query) {
    592607    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
    593608    return(false);
    594   }
     609  } 
    595610  psString limitString = NULL;
    596611  if (limit) {

  • trunk/ippTools/src/regtoolConfig.c

    r27176 r27315  
    381381    psMetadataAddBool(pendingcompressimfileArgs, PS_LIST_TAIL, "-simple", 0,     "use the simple output format", false);
    382382    psMetadataAddU64(pendingcompressimfileArgs, PS_LIST_TAIL, "-limit",   0,     "limit result set to N items", 0);
    383 
     383    psMetadataAddBool(pendingcompressimfileArgs, PS_LIST_TAIL, "-compress", 0,   "search only for files to compress", 0);
     384    psMetadataAddBool(pendingcompressimfileArgs, PS_LIST_TAIL, "-clean",  0,     "search only for files to clean originals", 0);
    384385   
    385386    // -finishcompressexp
Note: See TracChangeset for help on using the changeset viewer.