IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 16195


Ignore:
Timestamp:
Jan 22, 2008, 5:35:42 PM (18 years ago)
Author:
eugene
Message:

updates from eam_branch_20080121

Location:
trunk/ippTasks
Files:
11 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippTasks/camera.pro

    r16173 r16195  
    122122    book getword camPendingExp $pageName exp_tag -var EXP_TAG
    123123    book getword camPendingExp $pageName cam_id -var CAM_ID
    124     book getword camPendingExp $pageName workdir -var WORKDIR
     124    book getword camPendingExp $pageName workdir -var WORKDIR_TEMPLATE
    125125    book getword camPendingExp $pageName dvodb  -var DVODB
    126126    book getword camPendingExp $pageName dbname -var DBNAME
    … …  
    130130    set.host.for.camera $camera FPA
    131131
    132     # output log from filerule
    133     if ("$WORKDIR" == "NULL")
    134       $outpath = `pwd`
    135     else
    136       $outpath = `ipp_datapath.pl $WORKDIR`
    137     end
    138     sprintf outpath "%s/%s" $outpath $EXP_TAG
    139     sprintf outroot "%s/%s.cm.%s" $outpath $EXP_TAG $CAM_ID
     132    # raw workdir examples:
     133    # file://data/@HOST@.0/gpc1/20080130
     134    # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?)
     135    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
     136    # out workdir examples:
     137    # file://data/ipp004.0/gpc1/20080130
     138    # neb:///ipp004-vol0/gpc1/20080130
     139
     140    ## generate outroot specific to this exposure (& chip)
     141    sprintf outroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID
     142
     143    ## generate output log based on filerule (convert the URI to a PATH)
    140144    $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot`
    141145    if ("$logfile" == "")
    142       $logfile = $outroot.log
     146      echo "WARNING: logfile not defined in camera.pro:146"
     147      break
    143148    end
    144149
    145150    stdout $logfile
    146151    stderr $logfile
    147     exec mkdir -p $outpath
    148152
    149     $run = camera_exp.pl --exp_tag $EXP_TAG --cam_id $CAM_ID --camera $CAMERA
    150     if ("$WORKDIR" != "NULL")
    151       $run = $run --workdir $WORKDIR
    152     end
     153    $run = camera_exp.pl --exp_tag $EXP_TAG --cam_id $CAM_ID --camera $CAMERA --outroot $outroot
    153154    if ("$REDUCTION" != "NULL")
    154155      $run = $run --reduction $REDUCTION

  • trunk/ippTasks/chip.pro

    r16173 r16195  
    121121    book getword chipPendingImfile $pageName exp_tag -var EXP_TAG
    122122    book getword chipPendingImfile $pageName chip_id -var CHIP_ID
    123     book getword chipPendingImfile $pageName workdir -var WORKDIR
     123    book getword chipPendingImfile $pageName workdir -var WORKDIR_TEMPLATE
    124124    book getword chipPendingImfile $pageName class_id -var CLASS_ID
    125125    book getword chipPendingImfile $pageName uri -var URI
    … …  
    130130    set.host.for.camera $camera $class_id
    131131
    132     ## generate output log based on filerule
    133     if ("$WORKDIR" == "NULL")
    134       $outpath = `pwd`
    135     else
    136       $outpath = `ipp_datapath.pl $WORKDIR`
    137     end
    138     sprintf outpath "%s/%s" $outpath $EXP_TAG
    139     sprintf outroot "%s/%s.ch.%s" $outpath $EXP_TAG $CHIP_ID
     132    # raw workdir examples:
     133    # file://data/@HOST@.0/gpc1/20080130
     134    # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?)
     135    set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     136    # out workdir examples:
     137    # file://data/ipp004.0/gpc1/20080130
     138    # neb:///ipp004-vol0/gpc1/20080130
     139
     140    ## generate outroot specific to this exposure (& chip)
     141    sprintf outroot "%s/%s/%s.ch.%s" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID
     142
     143    ## generate output log based on filerule (convert the URI to a PATH)
    140144    $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot`
    141145    if ("$logfile" == "")
    142       sprintf logfile "%s.%s.log" $outroot $CLASS_ID
     146      echo "WARNING: logfile not defined in chip.pro:152"
     147      break
    143148    end
    144149
    145150    stdout $logfile
    146151    stderr $logfile
    147     exec mkdir -p $outpath
    148152
    149     $run = chip_imfile.pl --exp_id $EXP_ID --chip_id $CHIP_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --uri $URI --camera $CAMERA
    150     if ("$WORKDIR" != "NULL")
    151       $run = $run --workdir $WORKDIR
    152     end
     153    $run = chip_imfile.pl --exp_id $EXP_ID --chip_id $CHIP_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --uri $URI --camera $CAMERA --outroot $outroot
    153154    if ("$REDUCTION" != "NULL")
    154155      $run = $run --reduction $REDUCTION

  • trunk/ippTasks/detrend.norm.pro

    r16173 r16195  
    153153    book getword detPendingNormStatImfile $pageName iteration -var ITERATION
    154154    book getword detPendingNormStatImfile $pageName camera    -var CAMERA 
    155     book getword detPendingNormStatImfile $pageName workdir   -var WORKDIR
     155    book getword detPendingNormStatImfile $pageName workdir   -var WORKDIR_TEMPLATE
    156156    book getword detPendingNormStatImfile $pageName dbname    -var DBNAME
    157157
    … …  
    159159    set.host.for.camera $camera FPA
    160160
    161     if ("$WORKDIR" == "NULL")
    162       $outpath = `pwd`
    163     else
    164       $outpath = `ipp_datapath.pl $WORKDIR`
    165     end
    166     $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID
    167     $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.log
     161    # set workdir (interpolate host; see camera.pro for examples)
     162    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
     163
     164    ## generate outroot specific to this exposure (& chip)
     165    sprintf outroot "%s/%s.%s.%s/%s.%s.norm.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION
     166
     167    ## generate output log (NOT based on filerule?)
     168    $logfile = "$outroot.log"
     169    $logfile = `ipp_datapath.pl $logfile`
     170
    168171    stdout $logfile
    169172    stderr $logfile
    170     exec mkdir -p $outpath
    171 
    172     $run = detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE
    173     if ("$WORKDIR" != "NULL")
    174       $run = $run --workdir $WORKDIR
    175     end
     173
     174    $run = detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --outroot $outroot
    176175    add_standard_args run
    177176
    … …  
    271270    book getword detPendingNormImfile $pageName class_id  -var CLASS_ID
    272271    book getword detPendingNormImfile $pageName norm      -var NORM     
    273     book getword detPendingNormImfile $pageName workdir   -var WORKDIR
     272    book getword detPendingNormImfile $pageName workdir   -var WORKDIR_TEMPLATE
    274273    book getword detPendingNormImfile $pageName dbname    -var DBNAME
    275274
    … …  
    277276    set.host.for.camera $camera $class_id
    278277
    279     # XXX use ipp_filename.pl to lookup output file names
    280     if ("$WORKDIR" == "NULL")
    281       $outpath = `pwd`
    282     else
    283       $outpath = `ipp_datapath.pl $WORKDIR`
    284     end
    285     $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID
    286     $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.$CLASS_ID.log
     278    # set workdir (interpolate host; see camera.pro for examples)
     279    set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     280
     281    ## generate outroot specific to this stack (& chip)
     282    sprintf outroot "%s/%s.%s.%s/%s.%s.norm.%s.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION $CLASS_ID
     283
     284    ## generate output log (NOT based on filerule?)
     285    $logfile = "$outroot.log"
     286    $logfile = `ipp_datapath.pl $logfile`
     287
    287288    stdout $logfile
    288289    stderr $logfile
    289     exec mkdir -p $outpath
    290 
    291     $run = detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE
    292     if ("$WORKDIR" != "NULL")
    293       $run = $run --workdir $WORKDIR
    294     end
     290
     291    $run = detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE --outroot $outroot
    295292    add_standard_args run
    296293   
    … …  
    387384    book getword detPendingNormExp $pageName det_type  -var DET_TYPE
    388385    book getword detPendingNormExp $pageName camera    -var CAMERA 
    389     book getword detPendingNormExp $pageName workdir   -var WORKDIR
     386    book getword detPendingNormExp $pageName workdir   -var WORKDIR_TEMPLATE
    390387    book getword detPendingNormExp $pageName dbname    -var DBNAME
    391388
    … …  
    393390    set.host.for.camera $camera FPA
    394391
    395     # XXX add $WORKDIR/$LOG_DIR
    396     if ("$WORKDIR" == "NULL")
    397       $outpath = `pwd`
    398     else
    399       $outpath = `ipp_datapath.pl $WORKDIR`
    400     end
    401     $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID
    402     $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.log
     392    # set workdir (interpolate host; see camera.pro for examples)
     393    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
     394
     395    ## generate outroot specific to this exposure (& chip)
     396    sprintf outroot "%s/%s.%s.%s/%s.%s.norm.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION
     397
     398    ## generate output log (NOT based on filerule?)
     399    $logfile = "$outroot.log"
     400    $logfile = `ipp_datapath.pl $logfile`
     401
    403402    stdout $logfile
    404403    stderr $logfile
    405     exec mkdir -p $outpath
    406 
    407     $run = detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE
    408     if ("$WORKDIR" != "NULL")
    409       $run = $run --workdir $WORKDIR
    410     end
     404
     405    $run = detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE  --outroot $outroot
    411406    add_standard_args run
    412407

  • trunk/ippTasks/detrend.process.pro

    r16173 r16195  
    143143    book getword detPendingProcessedImfile $pageName uri      -var URI     
    144144    book getword detPendingProcessedImfile $pageName camera   -var CAMERA   
    145     book getword detPendingProcessedImfile $pageName workdir  -var WORKDIR
     145    book getword detPendingProcessedImfile $pageName workdir  -var WORKDIR_TEMPLATE
    146146    book getword detPendingProcessedImfile $pageName dbname   -var DBNAME
    147147    book getword detPendingProcessedImfile $pageName reduction -var REDUCTION
    … …  
    150150    set.host.for.camera $camera $class_id
    151151
    152     ## output log filename
    153     if ("$WORKDIR" == "NULL")
    154       $outpath = `pwd`
    155     else
    156       $outpath = `ipp_datapath.pl $WORKDIR`
    157     end
    158     sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG
    159     sprintf outroot "%s/%s.detproc.%s" $outpath $EXP_TAG $DET_ID
     152    # see chip.pro for examples
     153    set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     154
     155    ## generate outroot specific to this exposure (& chip)
     156    sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID
     157
     158    ## generate output log based on filerule (convert the URI to a PATH)
    160159    $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot`
    161160    if ("$logfile" == "")
    162       sprintf logfile "%s.%s.log" $outroot $CLASS_ID
     161      echo "WARNING: logfile not defined in detrend.process.pro:161"
     162      break
    163163    end
    164164
    165165    stdout $logfile
    166166    stderr $logfile
    167     exec mkdir -p $outpath
    168 
    169     $run = detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --det_type $DET_TYPE --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --camera $CAMERA
    170     if ("$WORKDIR" != "NULL")
    171       $run = $run --workdir $WORKDIR
    172     end
     167
     168    $run = detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --det_type $DET_TYPE --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --camera $CAMERA --outroot $outroot
    173169    if ("$REDUCTION" != "NULL")
    174170      $run = $run --reduction $REDUCTION
    … …  
    269265    book getword detPendingProcessedExp $pageName camera    -var CAMERA 
    270266    book getword detPendingProcessedExp $pageName exp_tag   -var EXP_TAG
    271     book getword detPendingProcessedExp $pageName workdir   -var WORKDIR
     267    book getword detPendingProcessedExp $pageName workdir   -var WORKDIR_TEMPLATE
    272268    book getword detPendingProcessedExp $pageName dbname    -var DBNAME
     269    book getword detPendingProcessedExp $pageName reduction -var REDUCTION
    273270
    274271    # specify choice of local or remote host based on camera and chip (class_id)
    275272    set.host.for.camera $camera FPA
    276273
    277     # XXX push this into the filerules?
    278     if ("$WORKDIR" == "NULL")
    279       $outpath = `pwd`
    280     else
    281       $outpath = `ipp_datapath.pl $WORKDIR`
    282     end
    283     sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG
    284     sprintf outroot "%s/%s.detproc.%s" $outpath $EXP_TAG $DET_ID
     274    # set workdir (interpolate host; see camera.pro for examples)
     275    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
     276
     277    ## generate outroot specific to this exposure (& chip)
     278    sprintf outroot "%s/%s.%s.%s/%s/%s.detproc.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID
     279
     280    ## generate output log based on filerule (convert the URI to a PATH)
    285281    $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot`
    286282    if ("$logfile" == "")
    287       $logfile = $outroot.log
     283      echo "WARNING: logfile not defined in detrend.process.pro:282"
     284      break
    288285    end
    289286
    290287    stdout $logfile
    291288    stderr $logfile
    292     exec mkdir -p $outpath
    293 
    294     $run = detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
    295     if ("$WORKDIR" != "NULL")
    296       $run = $run --workdir $WORKDIR
     289
     290    $run = detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot
     291    if ("$REDUCTION" != "NULL")
     292      $run = $run --reduction $REDUCTION
    297293    end
    298294    add_standard_args run

  • trunk/ippTasks/detrend.reject.pro

    r16173 r16195  
    122122    book getword detRejectExp $pageName mode      -var MODE     
    123123    book getword detRejectExp $pageName camera    -var CAMERA   
    124     book getword detRejectExp $pageName workdir   -var WORKDIR
     124    book getword detRejectExp $pageName workdir   -var WORKDIR_TEMPLATE
    125125    book getword detRejectExp $pageName dbname    -var DBNAME
    126126
    … …  
    128128    set.host.for.camera $camera FPA
    129129
    130     if ("$WORKDIR" == "NULL")
    131       $outroot = `pwd`
    132     else
    133       $outroot = `ipp_datapath.pl $WORKDIR`
    134     end
    135     sprintf outroot "%s/%s.%s.%s" $outroot $CAMERA $DET_TYPE $DET_ID
    136     sprintf logfile "%s/%s.%s.%s.%s.detreject.log" $outroot $CAMERA $DET_TYPE $DET_ID $ITERATION
     130    # set workdir (interpolate host; see camera.pro for examples)
     131    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
     132
     133    ## generate outroot specific to this exposure (& chip)
     134    sprintf outroot "%s/%s.%s.%s/%s/%s.detreject.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION
     135
     136    ## generate output log based on filerule (convert the URI to a PATH)
     137    $logfile = $outroot.log
     138    $logfile = `ipp_datapath.pl $logfile`
     139
    137140    stdout $logfile
    138141    stderr $logfile
    139     exec mkdir -p $outroot
    140142
    141     $run = detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA
    142     if ("$WORKDIR" != "NULL")
    143       $run = $run --workdir $WORKDIR
    144     end
     143    $run = detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA --outroot $outroot
    145144    add_standard_args run
    146145

  • trunk/ippTasks/detrend.resid.pro

    r16173 r16195  
    143143    book getword detPendingResidImfile $pageName det_uri   -var DET_URI 
    144144    book getword detPendingResidImfile $pageName camera    -var CAMERA   
    145     book getword detPendingResidImfile $pageName workdir   -var WORKDIR
     145    book getword detPendingResidImfile $pageName workdir   -var WORKDIR_TEMPLATE
    146146    book getword detPendingResidImfile $pageName dbname    -var DBNAME
    147147    book getword detPendingResidImfile $pageName reduction -var REDUCTION
    … …  
    150150    set.host.for.camera $camera $class_id
    151151
    152     # XXX use ipp_filename.pl to lookup output file names
    153     if ("$WORKDIR" == "NULL")
    154       $outpath = `pwd`
    155     else
    156       $outpath = `ipp_datapath.pl $WORKDIR`
    157     end
    158     sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG
    159     sprintf outroot "%s/%s.detresid.%s.%s" $outpath $EXP_TAG $DET_ID $ITERATION
     152    # set workdir (interpolate host; see camera.pro for examples)
     153    set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     154
     155    ## generate outroot specific to this exposure (& chip)
     156    sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION
     157
     158    ## generate output log based on filerule (convert the URI to a PATH)
    160159    $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot`
    161160    if ("$logfile" == "")
    162       sprintf logfile "%s.%s.log" $outroot $CLASS_ID
     161      echo "WARNING: logfile not defined in detrend.resid.pro:161"
     162      break
    163163    end
    164164
    165165    stdout $logfile
    166166    stderr $logfile
    167     exec mkdir -p $outpath
    168 
    169     $run = detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE
    170     if ("$WORKDIR" != "NULL")
    171       $run = $run --workdir $WORKDIR
    172     end
     167
     168    $run = detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE --outroot $outroot
    173169    if ("$REDUCTION" != "NULL")
    174170      $run = $run --reduction $REDUCTION
    … …  
    272268    book getword detPendingResidExp $pageName include   -var INCLUDE
    273269    book getword detPendingResidExp $pageName camera    -var CAMERA 
    274     book getword detPendingResidExp $pageName workdir   -var WORKDIR
     270    book getword detPendingResidExp $pageName workdir   -var WORKDIR_TEMPLATE
    275271    book getword detPendingResidExp $pageName dbname    -var DBNAME
    276272
    … …  
    278274    set.host.for.camera $camera FPA
    279275
    280     # XXX use ipp_filename.pl to lookup output file names
    281     if ("$WORKDIR" == "NULL")
    282       $outpath = `pwd`
    283     else
    284       $outpath = `ipp_datapath.pl $WORKDIR`
    285     end
    286     sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG
    287     sprintf outroot "%s/%s.detresid.%s.%s" $outpath $EXP_TAG $DET_ID $ITERATION
     276    # set workdir (interpolate host; see camera.pro for examples)
     277    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
     278
     279    ## generate outroot specific to this exposure (& chip)
     280    sprintf outroot "%s/%s.%s.%s/%s/%s.detresid.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $EXP_TAG $EXP_TAG $DET_ID $ITERATION
     281
     282    ## generate output log based on filerule (convert the URI to a PATH)
    288283    $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id NONE --basename $outroot`
     284    if ("$logfile" == "")
     285      echo "WARNING: logfile not defined in detrend.process.pro:285"
     286      break
     287    end
    289288
    290289    stdout $logfile
    291290    stderr $logfile
    292     exec mkdir -p $outpath
    293 
    294     $run = detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA
    295     if ("$WORKDIR" != "NULL")
    296       $run = $run --workdir $WORKDIR
    297     end
     291
     292    $run = detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot
    298293    add_standard_args run
    299294

  • trunk/ippTasks/detrend.stack.pro

    r16173 r16195  
    123123    book getword detPendingStackedImfile $pageName class_id  -var CLASS_ID
    124124    book getword detPendingStackedImfile $pageName camera    -var CAMERA 
    125     book getword detPendingStackedImfile $pageName workdir   -var WORKDIR
     125    book getword detPendingStackedImfile $pageName workdir   -var WORKDIR_TEMPLATE
    126126    book getword detPendingStackedImfile $pageName dbname    -var DBNAME
    127127    book getword detPendingStackedImfile $pageName reduction -var REDUCTION
    … …  
    130130    set.host.for.camera $camera $class_id
    131131
    132     if ("$WORKDIR" == "NULL")
    133       $outpath = `pwd`
    134     else
    135       $outpath = `ipp_datapath.pl $WORKDIR`
    136     end
    137     sprintf outpath "%s/%s.%s.%s" $outpath $CAMERA $DET_TYPE $DET_ID
    138     sprintf outroot "%s/%s.%s.%s.%s" $outpath $CAMERA $DET_TYPE $DET_ID $ITERATION
     132    # set workdir (interpolate host; see camera.pro for examples)
     133    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
     134
     135    ## generate outroot specific to this exposure (& chip)
     136    sprintf outroot "%s/%s.%s.%s/%s.%s.%s.%s" $WORKDIR $CAMERA $DET_TYPE $DET_ID $CAMERA $DET_TYPE $DET_ID $ITERATION
     137
     138    ## generate output log based on filerule (convert the URI to a PATH)
    139139    $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot`
    140140    if ("$logfile" == "")
    141       sprintf logfile "%s.%s.log" $outroot $CLASS_ID
     141      echo "WARNING: logfile not defined in detrend.stack.pro:141"
     142      break
    142143    end
    143144
    144145    stdout $logfile
    145146    stderr $logfile
    146     exec mkdir -p $outpath
    147147
    148     # save the pageName for future reference below
    149     options $pageName
    150 
    151     $run = detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA
    152     if ("$WORKDIR" != "NULL")
    153       $run = $run --workdir $WORKDIR
    154     end
     148    $run = detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA --outroot $outroot
    155149    if ("$REDUCTION" != "NULL")
    156150      $run = $run --reduction $REDUCTION
    157151    end
    158152    add_standard_args run
     153
     154    # save the pageName for future reference below
     155    options $pageName
    159156
    160157    # create the command line

  • trunk/ippTasks/notes.txt

    r16173 r16195  
    11
    2 2008.01.18
     2  proposal to clean scripts/tasks up somewhat wrt names:
     3
     4  * scripts do NOT use workdir to change the workdir in the db
     5  * thus, what is supplied is actually the outputRoot
     6  * require the output root for the scripts
     7  * have the calling tasks construct the outputRoot
     8  ** replace -workdir with -outroot
     9
     10  some filename rules :
     11
     12  * the db entry WORKDIR is the most abstract version:
     13    path://GPC-@HOST@/20080130
     14    file://data/@HOST@.0/gpc1/20080130
     15    neb:///@HOST@/gpc1/20080130
     16    neb://gpc1/20080130
     17    XXX have nebulous distinguish between neb:///host/path and
     18    neb://path?
     19
     20  * the db WORKDIR entry is carried from step to step unless
     21    over-ridden with a -queue command
     22
     23  * psLib-based binaries (ppImage, etc) may be passed URIs, but not
     24    files with the @HOST@ abstraction
     25
     26  * IPP perl scripts may be passed URIs, but not files with the @HOST@
     27    abstraction
     28
     29  * nebulous : convert neb:///@HOST@/path/name
     30
     31  *** examples:
     32
     33  * this tells ppImage to use the ipp004 version for both input and output:
     34  ppImage -file neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01.fits neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01
     35 
     36  * we already know the hostname at this point; include it in the call
     37  chip_imfile.pl --uri neb:///ipp004-v0/gpc1/20080130/o4414g0001o/o4414g0001o.XY01.fits --workdir neb:///ipp004-v0/gpc1/20080130/
     38
     39
     40
     41
     42
     432008.01.21
    344
    445  I'm working on the summit.copy.pro interaction with nebulous (and
    … …  
    2566    host. 
    2667
     68  * specific output filenames
     69
     70    In both ippTasks and ippScripts, there are references to files
     71    which must correctly located on the host.
     72
     73  * workdir choice of output directory.   
     74
     75    the
     76
    2777  * dsget output target:
    2878
    2979    dsget is called by pantasks in the task 'dsget'.  currently, dsget
    3080    accepts a --filename argument which specifies the output filename
    31     to use.  we have to tell t
     81    to use.  in the nebulous context, this is of the form
     82    neb://top/next/stuff, without any host-specific information.  To
     83    target the file instance to a specific host, we pass dsget the
     84    --volume (name) argument.  The volumes refer to the specific
     85    devices (potentially more than one per host), and the names can
     86    then be chosen to include the host. 
    3287 
    33 
    34   * choice of output directory.   
    35 
    36     the
    37 
    38 
    39882007.10.11
    4089

  • trunk/ippTasks/pantasks.pro

    r16173 r16195  
    342342
    343343macro set.workdir.by.camera
     344  if ($0 != 6)
     345    echo "USAGE: set.workdir.by.camera (camera) (class_id) (template) (default) (varname)"
     346    echo " uses global variable NEBULOUS to determine if nebulous is active"
     347    break
     348  end
     349
     350  local host default template camera classID varname
     351  $camera = $1
     352  $classID = $2
     353  $template = $3
     354  $default = $4
     355  $varname = $5
     356
     357  # missing camera and/or chiphosts table results in host = NULL
     358  book getword chiphosts $camera $classID -var host
     359
     360  if ("$host" == "NULL")
     361    strsub $template @HOST@ $default -var $$varname
     362  else
     363    strsub $template @HOST@ $host -var $$varname
     364  end
     365end
     366
     367macro set.volume.by.camera
    344368  if ($0 != 3)
    345     echo "USAGE: set.workdir.by.camera (camera) (class_id)"
    346     break
    347   end
     369    echo "USAGE: set.volume.by.camera (camera) (class_id) (template) (default) (varname)"
     370    echo " uses global variable NEBULOUS to determine if nebulous is active"
     371    break
     372  end
     373
     374  local host default template camera classID varname
     375  $camera = $1
     376  $classID = $2
     377  $template = $3
     378  $default = $4
     379  $varname = $5
    348380
    349381  # missing camera and/or chiphosts table results in host = NULL
    350   book getword chiphosts $1 $2 -var $host
    351 
    352   if ($NEBULOUS)
    353     if ("$host" == "NULL")
    354       $workdir = $workdir_default
    355       $volume  = "NULL"
    356       return
    357     end
    358     strsub $workdir_template @HOST@ $host -var workdir
    359     strsub $volume_template @HOST@ $host -var volume
    360     return
    361   end
    362 
    363   if ("$host" == "NULL")
    364     $workdir = $workdir_default
    365     return
    366   end
    367   strsub $workdir_template @HOST@ $host -var workdir
     382  book getword chiphosts $camera $classID -var host
     383
     384  if (not($NEBULOUS))
     385    $$varname = "NULL"
     386    return
     387  end
     388
     389  if ("$host" == "NULL")
     390    $$varname = $default
     391    return
     392  end
     393
     394  strsub $template @HOST@ $host -var $$varname
    368395end
    369396

  • trunk/ippTasks/register.pro

    r16174 r16195  
    137137    book setword regPendingImfile $pageName pantaskState RUN
    138138
    139     book getword regPendingImfile $pageName exp_id -var EXP_ID
     139    book getword regPendingImfile $pageName exp_id       -var EXP_ID
    140140    book getword regPendingImfile $pageName tmp_exp_name -var TMP_EXP_NAME
     141    book getword regPendingImfile $pageName tmp_camera   -var TMP_CAMERA
    141142    book getword regPendingImfile $pageName tmp_class_id -var TMP_CLASS_ID
    142     book getword regPendingImfile $pageName uri -var URI
    143     book getword regPendingImfile $pageName workdir -var WORKDIR_TEMPLATE
    144     book getword regPendingImfile $pageName dbname -var DBNAME
     143    book getword regPendingImfile $pageName uri          -var URI
     144    book getword regPendingImfile $pageName workdir      -var WORKDIR_TEMPLATE
     145    book getword regPendingImfile $pageName dbname       -var DBNAME
    145146
    146147    # EXP_TAG is used to generate the unique, but human-readable, filenames
    … …  
    148149
    149150    # specify choice of remote host
    150     set.host.for.camera $camera $class_id
    151 
    152     # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship
    153     # this function uses workdir_template, workdir_default, volume_template, volume_default,
    154     # it sets workdir and volume
    155     ### this function needs to be improved: it is using fixed input and output var names
    156     ### do we want these types of small outputs to go to a common filesystem?
    157     set.workdir.by.camera $CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE
    158 
    159     ## generate output log based on filerule XXX put this in a function?
    160     $outroot = `ipp_datapath.pl $WORKDIR`
    161     sprintf outroot "%s/%s" $outroot $EXP_TAG
    162     sprintf logfile "%s/%s.reg.log" $outroot $TMP_CLASS_ID
     151    set.host.for.camera $TMP_CAMERA $TMP_CLASS_ID
     152
     153    # raw workdir examples:
     154    # file://data/@HOST@.0/gpc1/20080130
     155    # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?)
     156    set.workdir.by.camera $TMP_CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     157    # out workdir examples:
     158    # file://data/ipp004.0/gpc1/20080130
     159    # neb:///ipp004-vol0/gpc1/20080130
     160
     161    ## generate output log based on filerule
     162    sprintf logfile "%s/%s/%s.%s.reg.log" $WORKDIR $EXP_TAG $EXP_TAG $TMP_CLASS_ID
     163    $logfile = `ipp_datapath.pl $logfile`
     164
    163165    stdout $logfile
    164166    stderr $logfile
    165     exec mkdir -p $outroot
     167
     168    $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI
     169    add_standard_args run
    166170
    167171    # save the pageName for future reference below
    168172    options $pageName
    169 
    170     $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI
    171     add_standard_args run
    172173
    173174    # create the command line
    … …  
    254255
    255256    book setword regPendingExp $pageName pantaskState RUN
    256     book getword regPendingExp $pageName exp_id -var EXP_ID
     257    book getword regPendingExp $pageName exp_id       -var EXP_ID
    257258    book getword regPendingExp $pageName tmp_exp_name -var TMP_EXP_NAME
    258     book getword regPendingExp $pageName workdir -var WORKDIR
    259     book getword regPendingExp $pageName dbname -var DBNAME
     259    book getword regPendingExp $pageName tmp_camera   -var TMP_CAMERA
     260    book getword regPendingExp $pageName workdir      -var WORKDIR_TEMPLATE
     261    book getword regPendingExp $pageName dbname       -var DBNAME
    260262
    261263    # EXP_TAG is used to generate the unique, but human-readable, filenames
    262264    sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID
    263265
    264     # specify choice of remote host:(need to choose based on chips)
    265     if ($parallel)
    266       host anyhost
    267     else
    268       host local
    269     end
     266    # specify choice of remote host
     267    set.host.for.camera $TMP_CAMERA $TMP_CLASS_ID
     268
     269    # raw workdir examples:
     270    # file://data/@HOST@.0/gpc1/20080130
     271    # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?)
     272    set.workdir.by.camera $TMP_CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     273    # out workdir examples:
     274    # file://data/ipp004.0/gpc1/20080130
     275    # neb:///ipp004-vol0/gpc1/20080130
    270276
    271277    ## generate output log based on filerule
    272     $outroot = `ipp_datapath.pl $WORKDIR`
    273     sprintf outroot "%s/%s" $outroot $EXP_TAG
    274     sprintf logfile "%s/%s.reg.log" $outroot $EXP_TAG
     278    sprintf logfile "%s/%s/%s.reg.log" $WORKDIR $EXP_TAG $EXP_TAG
     279    $logfile = `ipp_datapath.pl $logfile`
     280
    275281    stdout $logfile
    276282    stderr $logfile
    277     exec mkdir -p $outroot
     283
     284    $run = register_exp.pl --exp_id $EXP_ID --exp_tag $EXP_TAG
     285    add_standard_args run
    278286
    279287    # save the pageName for future reference below
    280288    options $pageName
    281 
    282     $run = register_exp.pl --exp_id $EXP_ID --exp_tag $EXP_TAG
    283     add_standard_args run
    284289
    285290    # create the command line

  • trunk/ippTasks/summit.copy.pro

    r16174 r16195  
    1818# the templates are used if we have a class_id/host relationship; if none is found, the default values are used
    1919# XXX not sure how to handle the .N value if we need to use more than one
     20# XXX this stuff should all be placed in a 'site' config file
    2021if ($NEBULOUS)
    21  $workdir_default  = neb://
    22  $workdir_template = neb://
    23  $volume_default  = ipp004-v0
    24  $volume_template = @HOST@-v0
     22 $default_host     = ipp004
     23 $workdir_template = neb:///@HOST@-v0
     24 $default_volume   = ipp004-v0
     25 $volume_template  =
    2526else
    26  $workdir_default  = /data/ipp004.0/
     27 $default_host     = ipp004
    2728 $workdir_template = /data/@HOST@.0/
    2829end
    … …  
    278279        book getword pzPendingImfile $pageName camera -var $CAMERA
    279280
     281        set.host.by.camera $CAMERA $CLASS_ID
     282        host        anyhost
     283
    280284        # 2007-08-30T05:09:59Z
    281285        substr $DATEOBS 0 4 YEAR
    … …  
    284288
    285289        # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship
    286         # this function uses workdir_template, workdir_default, volume_template, volume_default,
     290        # this function uses workdir_template, default_host, volume_template, volume_default,
    287291        # it sets workdir and volume
    288         set.workdir.by.camera $CAMERA $CLASS_ID
     292        set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base
     293        set.volume.by.camera $CAMERA $CLASS_ID $volume_template $default_volume volume
    289294
    290295        # figure out filename
    291         # XXXX this filename needs to use nebulous names
    292         $FILENAME = $workdir/$CAMERA/$YEAR/$MONTH/$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID.fits
    293         $FILEROOT = $workdir_template/$CAMERA/$YEAR/$MONTH/$DAY
     296        # XXX may need to use sprintf here
     297        $FILENAME = $workdir_base/$CAMERA/$YEAR$MONTH$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID.fits
     298        $workdir = $workdir_template/$CAMERA/$YEAR$MONTH$DAY
     299        # workdir examples:
     300        # file://data/@HOST@.0/gpc1/20080130
     301        # neb://gpc1/20080130 (need to supply volname?, or are we re-defining this each time?)
    294302
    295303        book setword pzPendingImfile $pageName filename $FILENAME
    296 
    297         set.host.by.camera $CAMERA $CLASS_ID
    298         host        anyhost
    299304
    300305        # store the pageName with this job
    301306        options $pageName
    302         options $FILEROOT
     307        options $workdir
    303308
    304309        $run = dsget --uri $URI --filename $FILENAME --bytes $BYTES
    … …  
    330335        # these are functions of (at least) EXP_TYPE and SURVEY_ID, both of which should come from the data store
    331336
     337        # we saved the workdir so we can pass it to pztool
     338        $workdir = $options:1
     339
    332340        # update the database with the success
    333         $run = pztool -copydone -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -class $CLASS -class_id $CLASS_ID -uri $FILENAME -workdir $options:1
     341        # for the moment, let's not propagate these
     342        $run = pztool -copydone -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -class $CLASS -class_id $CLASS_ID -uri $FILENAME -workdir $workdir -end_stage reg
    334343        # tess_id, dvodb, workdir, end_stage
    335344
Note: See TracChangeset for help on using the changeset viewer.