Changeset 20360

Timestamp:

Oct 24, 2008, 2:30:41 PM (18 years ago)

Author:

bills

Message:

Change tmp file and recovery destinations to be directories rather than
path_bases. Get name of file from input to avoid proliferation of
file rules.

Require -replace fo -remove.

If tmp file already exists in nebulous delete it since there may
be multiple instances

File:

• trunk/magic/remove/src/streaksremove.c (modified) (25 diffs)

trunk/magic/remove/src/streaksremove.c

@@ -1 +1 @@
 #include "streaks.h"
 #include "streaksextern.h"
+#include "libgen.h"
 
 extern bool  sFileLock(sFile * sfile);
 extern bool  sFileUnlock(sFile * sfile);
 extern bool  sFileReplace(sFile *dest, sFile *src);
+
+static nebServer *ourNebServer = NULL;
 
 // for clarity in this program we don't propagate errors up the stack
@@ -13 +16 @@
 }
 
-static void
-sFileFree(sFile *sfile)
+static void sFileFree(sFile *sfile)
 {
     psFree(sfile->resolved_name);
@@ -21 +23 @@
 }
 
+// getNebServer()
+//
+// it gets created the first time a nebulous file is accessed.
+// if config is passed in we are to create it and exit if not able.
+// if config is null we return NULL indicating that there are no nebulous files in use
+// The purpose of this is to avoid passing extra arguments between all of the functions.
+// but it is probably a bad idea.
+//
+static nebServer *getNebServer(pmConfig *config)
+{
+    if (ourNebServer) {
+        return ourNebServer;
+    }
+
+    if (!config) {
+        return NULL;
+    }
+
+    psString serverURI = getenv("NEB_SERVER");
+
+    // XXX: Note: all of the flags to psError should be true, but I don't think nebclient
+    // uses psError
+    if (!serverURI) {
+        bool status;
+        serverURI = psMetadataLookupStr(&status, config->site, "NEB_SERVER");
+        if (!status) {
+            psError(PM_ERR_CONFIG, true, "failed to lookup config value for NEB_SERVER.");
+            streaksremoveExit("", PS_EXIT_CONFIG_ERROR);
+        }
+    }
+
+    if (!serverURI) {
+        psError(PM_ERR_CONFIG, true, "Could not determine nebulous server URI.");
+
+        streaksremoveExit("", PS_EXIT_CONFIG_ERROR);
+    }
+
+    ourNebServer = nebServerAlloc(serverURI);
+    if (!ourNebServer) {
+        psError(PM_ERR_SYS, true, "failed to create a nebServer object from %s.", serverURI);
+        streaksremoveExit("", PS_EXIT_CONFIG_ERROR);
+    }
+
+    return ourNebServer;
+}
+
 psString
 resolveFilename(pmConfig *config, sFile *sfile, bool create)
 {
     sfile->inNebulous = IN_NEBULOUS(sfile->name);
+
+    if (sfile->inNebulous) {
+        // make sure we have our neb server connection
+        nebServer *server = getNebServer(config);
+        if (create) {
+            // delete the existing file, since there may be more than one
+            // instance. It will get created below in pmConfigConvertFilename
+            if (nebFind(server, sfile->name)) {
+                nebDelete(server, sfile->name);
+            }
+        }
+    }
 
     return pmConfigConvertFilename(sfile->name, config, create, create);
@@ -37 +97 @@
 
     // We use psFits directly to read the image file unless the stage is warp
-    // or diff. In that case we use the pmFPAfile functions to read the image file
-    // to make managing the astromety easy
+    // or diff. In those cases we use the pmFPAfile functions to read the image file
+    // to make managing the astrometry easy
 
     if (!CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) {
@@ -67 +127 @@
         }
 
-        sfile->resolved_name = psStringCopy(sfile->pmfile->filename);
         pmFPAview *view = pmFPAviewAlloc(0);
         if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
@@ -74 +133 @@
         }
         psFree(view);
+        sfile->resolved_name = psStringCopy(sfile->pmfile->filename);
         
         // copy header from fpu
@@ -80 +140 @@
         return sfile;
     }
+
+    // using psFits for i/o
 
     sfile->name = psMetadataLookupStr(&status, config->arguments, fileSelect);
@@ -211 +273 @@
     sf->inImage = sFileOpen(config, stage, "INPUT", NULL, true);
     sf->nHDU = sf->inImage->nHDU;
-    sf->outImage = sFileOpen(config, stage, "OUTPUT", ".fits", true);
-    sf->recImage = sFileOpen(config, stage, "RECOVERY", ".fits", true);
+
+    // don't need to free this see basename(3)
+    char *inputBasename = basename(sf->inImage->name);
+    sf->outImage = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
+    sf->recImage = sFileOpen(config, stage, "RECOVERY", inputBasename, true);
 
     sf->inMask = sFileOpen(config, stage, "INPUT.MASK", NULL, false);
     if (sf->inMask) {
-        sf->outMask = sFileOpen(config, stage, "OUTPUT", ".mk.fits", true);
-        sf->recMask = sFileOpen(config, stage, "RECOVERY", "wt.fits", true);
+        inputBasename = basename(sf->inMask->name);
+        sf->outMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
+        sf->recMask = sFileOpen(config, stage, "RECOVERY", inputBasename, true);
     }
     sf->inWeight = sFileOpen(config, stage, "INPUT.WEIGHT", NULL, false);
     if (sf->inWeight) {
-        sf->outWeight = sFileOpen(config, stage, "OUTPUT", ".wt.fits", true);
-        sf->recWeight = sFileOpen(config, stage, "RECOVERY", ".wt.fits", true);
-    }
-
-    // load astrometry if supplied
+        inputBasename = basename(sf->inWeight->name);
+        sf->outWeight = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
+        sf->recWeight = sFileOpen(config, stage, "RECOVERY", inputBasename, true);
+    }
+
+    // load astrometry file
     if (USE_SUPPLIED_ASTROM(sf->stage)) {
         sf->inAstrom = pmFPAfileDefineFromArgs(&status, config, "PSWARP.ASTROM", "ASTROM");
     } else {
-        sf->inAstrom = sf->inImage->pmfile;
+        // otherwise get astrometry from pmfile
         if (!sf->inImage->pmfile) {
             streaksremoveExit("unexpected null pmFPAfile", PS_EXIT_CONFIG_ERROR);
         }
+        sf->inAstrom = sf->inImage->pmfile;
     }
     setupAstromFromFPA(sf);
@@ -316 +384 @@
 }
 
+psString checkdir(char *path)
+{
+    // insure that path ends in /
+    int len = strlen(path);
+    psString dir = NULL;
+    if (strrchr(path, '/') != (path + len - 1)) {
+        psStringAppend(&dir, "%s/", path);
+    } else {
+        dir = psStringCopy(path);
+    }
+    return dir;
+}
 
 ippStage
@@ -378 +458 @@
     }
 
+
+    // If input image is in nebulous we require mask, weight, and tmproot to all be
+    // nebulous paths
+    bool nebulousImage = false;
     if ((argnum = psArgumentGet(argc, argv, "-image"))) {
         // for raw and chip level images we use psFits for I/O and ...
+        nebulousImage = IN_NEBULOUS(argv[argnum+1]);
         if (CHIP_LEVEL_INPUT(stage)) {
             psArgumentRemove(argnum, &argc, argv);
@@ -406 +491 @@
     if ((argnum = psArgumentGet(argc, argv, "-mask"))) {
         psArgumentRemove(argnum, &argc, argv);
+        bool nebulousMask = IN_NEBULOUS(argv[argnum]);
+        if (nebulousMask != nebulousImage) {
+            psError(PS_ERR_UNKNOWN, true, "mask image must be %snebulous path with %s image path\n",
+                nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
+            return NULL;
+        }
         psMetadataAddStr(config->arguments, PS_LIST_TAIL, "MASK", 0,
                 "name of input mask image", argv[argnum]);
@@ -412 +503 @@
     if ((argnum = psArgumentGet(argc, argv, "-weight"))) {
         psArgumentRemove(argnum, &argc, argv);
+        bool nebulousWeight = IN_NEBULOUS(argv[argnum]);
+        if (nebulousWeight != nebulousImage) {
+            psError(PS_ERR_UNKNOWN, true, "weight image must have %snebulous path with %s image path\n",
+                nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
+            return NULL;
+        }
         psMetadataAddStr(config->arguments, PS_LIST_TAIL, "WEIGHT", 0,
                 "name of input weight image", argv[argnum]);
@@ -418 +515 @@
     if ((argnum = psArgumentGet(argc, argv, "-tmproot"))) {
         psArgumentRemove(argnum, &argc, argv);
-        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "root name of temporary files",
-            argv[argnum]);
-        psArgumentRemove(argnum, &argc, argv);
-    } else {
-        psError(PS_ERR_UNKNOWN, true, "-outroot is required\n");
+        bool nebulousTmp = IN_NEBULOUS(argv[argnum]);
+        if (nebulousTmp != nebulousImage) {
+            psError(PS_ERR_UNKNOWN, true, "tmproot must have %snebulous path with %s image path\n",
+                nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
+            return NULL;
+        }
+        psString dir = checkdir(argv[argnum]);
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "directory for temporary files",
+            dir);
+        psArgumentRemove(argnum, &argc, argv);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "-tmproot is required\n");
         return NULL;
     }
     if ((argnum = psArgumentGet(argc, argv, "-recovery"))) {
         psArgumentRemove(argnum, &argc, argv);
-        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "Name of recovery root",
-            argv[argnum]);
+        psString dir = checkdir(argv[argnum]);
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "directory for recovery files",
+            dir);
         psArgumentRemove(argnum, &argc, argv);
     } else if (stage == IPP_STAGE_RAW) {
@@ -434 +539 @@
         return NULL;
     }
+    bool gotReplace = false;
     if ((argnum = psArgumentGet(argc, argv, "-replace"))) {
+        gotReplace = true;
         psArgumentRemove(argnum, &argc, argv);
         psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REPLACE", 0, "replace input files",
@@ -440 +547 @@
     }
     if ((argnum = psArgumentGet(argc, argv, "-remove"))) {
-        psArgumentRemove(argnum, &argc, argv);
-        psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REMOVE", 0, "remove input files",
+        if (!gotReplace) {
+            psError(PS_ERR_UNKNOWN, true, "-replace is required with -remove\n");
+            return NULL;
+        }
+        psArgumentRemove(argnum, &argc, argv);
+        psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REMOVE", 0, "remove original files",
             true);
+    }
+
+    if (argc != 1) {
+        psString unexpectedArguments = NULL;
+        for (int i=1; i<argc; i++) {
+            psStringAppend(&unexpectedArguments, "%s ", argv[i]);
+        }
+        psError(PS_ERR_UNKNOWN, true, "unexpected arguments: %s", unexpectedArguments);
+        psFree(unexpectedArguments);
+        return NULL;
     }
 
@@ -709 +830 @@
 createRecoveryImage(sFile *rec, sFile *in, int extnum)
 {
-    // PFS: may not be necessary but replaced in->numRows with in->image->numRows
     rec->image = psImageRecycle(rec->image, in->image->numCols, in->image->numRows, in->image->type.type);
     if (!rec->image) {
@@ -879 +999 @@
 
 static bool
-replicate(nebServer *server, sFile *sfile, void *xattr)
+replicate(sFile *sfile, void *xattr)
 {
     if (!sfile->inNebulous) {
         return true;
     }
+    nebServer *server = getNebServer(NULL);
+
     // for now just set "user.copies" to 2
     if (!nebSetXattr(server, sfile->name, "user.copies", "2",  NEB_REPLACE)) {
@@ -903 +1025 @@
 {
     bool status = false;
-    psString neb_server = getenv("NEB_SERVER");
-
-    // XXX: Note: all of the flags to psError should be true, but I don't think nebclient
-    // uses psError
-    if (!neb_server) {
-        neb_server = psMetadataLookupStr(&status, sfiles->config->site, "NEB_SERVER");
-        if (!status) {
-            psError(PM_ERR_CONFIG, true, "failed to lookup config value for NEB_SERVER.");
-            return false;
-        }
-    }
-
-    if (!neb_server) {
-        psError(PM_ERR_CONFIG, true, "Could not determine nebulous server URI.");
-        return false;
-    }
-
-    sfiles->nebServer = nebServerAlloc(neb_server);
-    if (!sfiles->nebServer) {
-        psError(PM_ERR_SYS, true, "failed to create a nebServer object.");
-        return false;
-    }
 
     // XXX: TODO: need a nebGetXatrr function, but there isn't one
+    // another option would be to take the number of copies to be
+    // created as an option. That way the system could decide
+    // whether to replicate anything other than raw Image files
     void *xattr = NULL;
 
-    if (!replicate(sfiles->nebServer, sfiles->outImage, xattr)) {
+    if (!replicate(sfiles->outImage, xattr)) {
         psError(PM_ERR_SYS, false, "failed to replicate outImage.");
         return false;
@@ -935 +1038 @@
 
 #ifdef notyet
-    // don't replicate mask and weight images until we can look up
+    // XXX: don't replicate mask and weight images until we can look up
     // the input's xattr. There may be a perl program that can getXattr
     if (sfiles->outMask) {
         // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->nebServer, sfiles->outMask, xattr)) {
+        if (!replicate(sfiles->outMask, xattr)) {
             psError(PM_ERR_SYS, false, "failed to replicate outImage.");
             return false;
@@ -946 +1049 @@
     if (sfiles->outWeight) {
         // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->nebServer, sfiles->outWeight, xattr)) {
+        if (!replicate(sfiles->outWeight, xattr)) {
             psError(PM_ERR_SYS, false, "failed to replicate outImage.");
             return false;
@@ -959 +1062 @@
     return true;
 }
+
+static bool
+swapOutputToInput(sFile *in, sFile *out)
+{
+
+    if (in->inNebulous) {
+        nebServer *server = getNebServer(NULL);
+
+        if (!nebSwap(server, in->name, out->name)) {
+            psError(PM_ERR_SYS, true, "failed to swap files for: %s.",
+                in->name, nebErr(server));
+            return false;
+        }
+    } else {
+        psString command = NULL;
+
+        // regular files. use mv creating a backup
+        // NOTE: -b is a gnu option and may not always be available
+        psStringAppend(&command, "mv -b --suffix=.bak %s %s", out->resolved_name, in->resolved_name);
+        int status = system(command);
+        if (status != 0) {
+            psError(PM_ERR_SYS, true, "%s failed with status %d.",
+                command, status);
+            psFree(command);
+            return false;
+        }
+        psFree(command);
+    }
+
+    return true;
+}
 static bool
 swapOutputsToInputs(streakFiles *sfiles)
 {
-    bool status = false;
-
-    if (!sfiles->nebServer) {
-        // no output files are in nebulous
-        return true;
-    }
-
-    // XXX: we should check for consistency before here
-    // We need to insure that the database is consistent
-    // If input is in nebulous, output must be
-    // if input is not in nebulous output must not be Catch in parseArguments
-    // if not in nebulous create a backup of input before overwriting
-
-    if (sfiles->inImage->inNebulous && sfiles->outImage->inNebulous) {
-        if (!nebSwap(sfiles->nebServer, sfiles->outImage->name, sfiles->inImage->name)) {
-            psError(PM_ERR_SYS, false, "failed to swap instances for Image\n%s.",
-                nebErr(sfiles->nebServer));
+    if (sfiles->inMask) {
+        if (!swapOutputToInput(sfiles->inMask, sfiles->outMask)) {
+            psError(PM_ERR_SYS, false, "failed to swap instances for Mask.");
             return false;
         }
     }
 
-    // TODO: handle weight and mask images
+    if (sfiles->inWeight) {
+        if (!swapOutputToInput(sfiles->inWeight, sfiles->outWeight)) {
+            psError(PM_ERR_SYS, false, "failed to swap instances for Weight.");
+            return false;
+        }
+    }
+
+    if (!swapOutputToInput(sfiles->inImage, sfiles->outImage)) {
+        psError(PM_ERR_SYS, false, "failed to swap instances for Image.");
+        return false;
+    }
 
     return true;
 }
+
 static bool
 deleteTemps(streakFiles *sfiles)
@@ -992 +1123 @@
     bool status = false;
 
-    if (!sfiles->nebServer) {
-        // no output files are in nebulous
+    nebServer *server = getNebServer(NULL);
+
+    if (!server) {
+        // TODO: handle this
+        //
+        // no files are in nebulous
         return true;
     }
-
-    // XXX: we should check for consistency before here
-    // We need to insure that the database is consistent
-    // If input is in nebulous, output must be
-    // if input is not in nebulous output must not be Catch in parseArguments
-    // if not in nebulous create a backup of input before overwriting
-
     if (sfiles->outImage->inNebulous) {
-        if (!nebDelete(sfiles->nebServer, sfiles->outImage->name)) {
+        if (!nebDelete(server, sfiles->outImage->name)) {
             psError(PM_ERR_SYS, false, "failed to delete %s\n%s.", sfiles->outImage->name,
-                nebErr(sfiles->nebServer));
+                nebErr(getNebServer(NULL)));
             return false;
         }
@@ -1031 +1159 @@
     psString streaksFileName = psMetadataLookupStr(NULL, config->arguments, "STREAKS");
     Streaks *streaks = readStreaksFile(streaksFileName);
+
+    if (!streaks) {
+        psErrorStackPrint(stderr, "failed to read streaks file: %s", streaksFileName);
+        exit (PS_EXIT_PROG_ERROR);
+    }
 
     streakFiles *sfiles = openFiles(config);
@@ -1085 +1218 @@
 #endif
 
+        // XXX: This loop following will SEGV for raw video Cells becuase the image is null
         for (i = 0; i < psArrayLength (pixels); ++i) {
             pixelPos = psArrayGet (pixels, i);
@@ -1127 +1261 @@
         //     Note this is a database operation. No file I/O is performed
         if (!swapOutputsToInputs(sfiles)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to replicate output files");
+            psError(PS_ERR_UNKNOWN, false, "failed to swap files");
+
             // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that
             // it has done and give a detailed report of what happened
+
             psErrorStackPrint(stderr, "");
             exit(PS_EXIT_UNKNOWN_ERROR);
         }
-    }
-
-    if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) {
-        //      delete the temporary storage objects (which now points to the original image(s)
-        if (!deleteTemps(sfiles)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files");
-            // XXX: Now what? At this point the output files have been swapped, so we can't
-            // repeat the operation.
-            // Returning error status is problematic. Maybe just print an error message and
-            // let other system tools clean up
-            psErrorStackPrint(stderr, "");
-            exit(PS_EXIT_UNKNOWN_ERROR);
+
+        if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) {
+            //      delete the temporary storage objects (which now points to the original image(s)
+            if (!deleteTemps(sfiles)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files");
+                // XXX: Now what? At this point the output files have been swapped, so we can't
+                // repeat the operation.
+
+                // Returning error status here is problematic. The inputs have been streak removed
+                // but they're still lying around
+                // Maybe just print an error message and
+                // let other system tools clean up
+                psErrorStackPrint(stderr, "");
+                exit(PS_EXIT_UNKNOWN_ERROR);
+            }
         }
     }