Changeset 24556

Timestamp:

Jun 25, 2009, 1:53:26 PM (17 years ago)

Author:

bills

Message:

various changes to insure that we don't delete non-destreked files by mistake

Location:

trunk/magic/remove/src

Files:

• Makefile.simple (modified) (4 diffs)
• isdestreaked.c (added)
• streaksio.c (modified) (9 diffs)
• streaksio.h (modified) (1 diff)
• streaksremove.c (modified) (7 diffs)
• streaksremove.h (modified) (2 diffs)
• streaksreplace.c (modified) (1 diff)
• streaksutil.c (modified) (1 diff)

trunk/magic/remove/src/Makefile.simple

@@ -28 +28 @@
     streaksrelease.o
 
+ISDESTREAKED_OBJECTS=      \
+    ${COMMON_OBJECTS} \
+    isdestreaked.o
+
+
+HEADERS= Line.h  streaksastrom.h  streaksextern.h  streaksio.h  streaksremove.h
+
 STREAKSFLAGS=-DSTREAKS_COMPRESS_OUTPUT=1
 OPTFLAGS= -g -O2
@@ -35 +42 @@
 LDFLAGS=`psmodules-config --libs`
 
-PROGRAMS= streaksremove streaksreplace streakscompare streaksrelease
+PROGRAMS= streaksremove streaksreplace streakscompare streaksrelease isdestreaked
+HEADERS=Line.h streaksastrom.h streaksextern.h streaksio.h streaksremove.h
 
 all:    ${PROGRAMS}
 
+${REMOVE_OBJECTS}:      ${HEADERS}
 streaksremove:  ${REMOVE_OBJECTS}
 
@@ -47 +56 @@
 streaksrelease:  ${RELEASE_OBJECTS}
 
+isdestreaked:   ${ISDESTREAKED_OBJECTS}
+
+
 install:        ${PROGRAMS}
         install -t  $(PSCONFDIR)/$(PSCONFIG)/bin streaksremove
@@ -52 +64 @@
         install -t  $(PSCONFDIR)/$(PSCONFIG)/bin streakscompare
         install -t  $(PSCONFDIR)/$(PSCONFIG)/bin streaksrelease
+        install -t  $(PSCONFDIR)/$(PSCONFIG)/bin isdestreaked
 
 clean:

trunk/magic/remove/src/streaksio.c

@@ -19 +19 @@
     psMemSetDeallocator(sf, (psFreeFunc) streakFilesFree);
     memset(sf, 0, sizeof(*sf));
+
+    if (remove) {
+        // remember pointer so that streaksExit can delete temps
+        setStreakFiles(sf);
+    }
 
     sf->config = config;
@@ -160 +165 @@
 }
 
+// figure out if a nebulous instance is a non-destreaked file
+static bool
+nebFileIsDestreaked(sFile *sfile)
+{
+    if (!sfile->resolved_name) {
+        psError(PS_ERR_PROGRAMMING, true, "resolved name is null");
+        return false;
+    }
+    psFits *fits = psFitsOpen(sfile->resolved_name, "r");
+    if (!fits) {
+        psError(PS_ERR_IO, true, "failed open %s", sfile->name);
+        // can't tell if it is a destreaked file
+        return false;
+    }
+    psMetadata *header = psFitsReadHeader(NULL, fits);
+    if (!header) {
+        psError(PS_ERR_IO, true, "failed to read header for: %s", sfile->name);
+        return false;
+    }
+    bool mdok;
+    bool isDestreaked = psMetadataLookupBool(&mdok, header, "PSDESTRK");
+    if (mdok && isDestreaked) {
+        return true;
+    } else {
+        psError(PS_ERR_IO, false, "output file already exists and may not be de-streaked: %s", sfile->name);
+        return false;
+    }
+}
+
 static psString
 resolveFilename(pmConfig *config, sFile *sfile, bool create)
@@ -171 +205 @@
             // delete the existing file, since there may be more than one
             // instance. It will get created below in pmConfigConvertFilename
-            if (nebFind(server, sfile->name)) {
+            if ((sfile->resolved_name = nebFind(server, sfile->name)) != NULL) {
+                if (!nebFileIsDestreaked(sfile)) {
+                    psError(PS_ERR_IO, false, "attempting to delete file that has not been destreaked %s", sfile->name);
+                    return NULL;
+                }
+                nebFree(sfile->resolved_name);
+                sfile->resolved_name = NULL;
                 nebDelete(server, sfile->name);
             }
@@ -193 +233 @@
     // all of the keywords in the raw image files written to the output destreaked files
 
-    if (!CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) {
+    if (!outputFilename && !CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) {
         // stage is warp or diff AND fileSelect eq "INPUT"
         // get data from pmFPAfile.
@@ -250 +290 @@
     }
 
-    // if outputFilename is not null name it contains the "directory"
+    // if outputFilename is not null name it contains the "directory" (perhaps with a prefix like SR_)
     // and outputFilename is the basename name of the file (or nebulous key)
     // and the file is to be opened for writing
@@ -834 +874 @@
 
 bool
-replicate(sFile *sfile, void *xattr)
-{
-    if (!sfile->inNebulous) {
+replicate(sFile *outFile, sFile *inFile)
+{
+    if (!outFile->inNebulous) {
         return true;
     }
     nebServer *server = getNebServer(NULL);
 
-    // for now just set "user.copies" to 2
-    if (!nebSetXattr(server, sfile->name, "user.copies", "2",  NEB_REPLACE)) {
-        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server));
+    char *user_copies = nebGetXattr(server, inFile->name, "user.copies");
+    bool free_user_copies = true;
+    if (user_copies == NULL) {
+        user_copies = "2";
+        free_user_copies = false;
+    }
+    if (!nebSetXattr(server, outFile->name, "user.copies", user_copies,  NEB_REPLACE)) {
+        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", outFile->name, nebErr(server));
         return false;
     }
-    if (!nebReplicate(server, sfile->name, NULL, NULL)) {
-        psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server));
+    if (!nebReplicate(server, outFile->name, "any", NULL)) {
+        psError(PM_ERR_UNKNOWN, true, "nebReplicate failed for %s\n%s", outFile->name, nebErr(server));
         return false;
+    }
+    if (free_user_copies) {
+        nebFree(user_copies);
     }
     return true;
@@ -861 +909 @@
     bool status = false;
 
-    // XXX: TODO: need a nebGetXatrr function, but there isn't one
-    // another option would be to take the number of copies to be
-    // created as an option. That way the system could decide
-    // whether to replicate anything other than raw Image files
-    void *xattr = NULL;
-
-    if (!replicate(sfiles->outImage, xattr)) {
+    if (!replicate(sfiles->outImage, sfiles->inImage)) {
         psError(PM_ERR_SYS, false, "failed to replicate outImage.");
         return false;
     }
 
-#ifdef notyet
-    // XXX: don't replicate mask and weight images until we can look up
-    // the input's xattr. There may be a perl program that can getXattr
     if (sfiles->outMask) {
-        // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->outMask, xattr)) {
+        if (!replicate(sfiles->outMask, sfiles->inMask)) {
             psError(PM_ERR_SYS, false, "failed to replicate outImage.");
             return false;
         }
     }
-    if (sfiles->outWeight) {
-        // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->outWeight, xattr)) {
+    if (sfiles->outChMask) {
+        if (!replicate(sfiles->outChMask, sfiles->inChMask)) {
             psError(PM_ERR_SYS, false, "failed to replicate outImage.");
             return false;
         }
     }
-#endif
-
-//      replicate the recovery images (if in nebulous)
+    if (sfiles->outWeight) {
+        if (!replicate(sfiles->outWeight, sfiles->inWeight)) {
+            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
+            return false;
+        }
+    }
+
+//      XXX: replicate the recovery images (if in nebulous)
 //      perhaps whether we do that or not should be configurable.
 //      Sounds like we need a recipe
@@ -954 +996 @@
         }
     }
+
     if (!swapOutputToInput(sfiles->inImage, sfiles->outImage)) {
         psError(PM_ERR_SYS, false, "failed to swap instances for Image.");
@@ -986 +1029 @@
 {
     if (sfiles->outMask) {
-        if (!deleteFile(sfiles->outMask)) {
-            psError(PM_ERR_SYS, false, "failed to delete Mask.");
-            return false;
-        }
+        deleteFile(sfiles->outMask);
+    }
+
+    if (sfiles->outChMask) {
+        deleteFile(sfiles->outChMask);
     }
 
     if (sfiles->outWeight) {
-        if (!deleteFile(sfiles->outWeight)) {
-            psError(PM_ERR_SYS, false, "failed to delete Weight.");
-            return false;
-        }
-    }
-
-    if (!deleteFile(sfiles->outImage)) {
-        psError(PM_ERR_SYS, false, "failed to delete Image.");
-        return false;
+        deleteFile(sfiles->outWeight);
+    }
+
+    if (sfiles->outImage) {
+        deleteFile(sfiles->outImage);
     }
 

trunk/magic/remove/src/streaksio.h

@@ -20 +20 @@
 void writeImage(sFile *sfile, psString extname, int extnum);
 void writeImageCube(sFile *sfile, psArray *imagecube, psString extname, int extnum);
-bool replicate(sFile *sfile, void *xattr);
+bool replicate(sFile *outFile, sFile *inFile);
 void readImageFrom_pmFile(streakFiles *sf);
 

trunk/magic/remove/src/streaksremove.c

@@ -41 +41 @@
     Streaks *streaks = readStreaksFile(streaksFileName);
     if (!streaks) {
-        psErrorStackPrint(stderr, "failed to read streaks file: %s", streaksFileName);
+        psError(PS_ERR_UNKNOWN, "failed to read streaks file: %s", streaksFileName);
         streaksExit("", PS_EXIT_PROG_ERROR);
     }
@@ -209 +209 @@
     if (!replicateOutputs(sfiles)) {
         psError(PS_ERR_UNKNOWN, false, "failed to replicate output files");
+        deleteTemps(sfiles);
         psErrorStackPrint(stderr, "");
         exit(PS_EXIT_UNKNOWN_ERROR);
@@ -215 +216 @@
     // NOTE: from here on we can't just quit if something goes wrong.
     // especially if we're working at the raw stage
+    // turn off automatic deletion of output files by streaksExit
+    setStreakFiles(NULL);
 
     if (psMetadataLookupBool(&status, config->arguments, "REPLACE")) {
@@ -220 +223 @@
         //     Note this is a nebulous database operation. No file I/O is performed
         if (!swapOutputsToInputs(sfiles)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to swap files");
-
-            // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that
-            // it has done and give a detailed report of what happened
-
-            psErrorStackPrint(stderr, "");
+            // XXX: Now what?
+            // It is up to the program that reverts failed destreak runs to insure that
+            // any input files that have been swapped are restored
+
+            psErrorStackPrint(stderr, "failed to swap files");
 
             // XXX: pick a specific error code for this failure
             exit(PS_EXIT_UNKNOWN_ERROR);
         }
-
-#ifdef notdef
-        // XXX: we've decided to not do the remove step here
-        // Instead we leave the backup images (the original images) in place and remove them
-        // later on
-        if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) {
-            //      delete the temporary storage objects (which now points to the original image(s)
-            if (!deleteTemps(sfiles)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files");
-                // XXX: Now what? At this point the output files have been swapped, so we can't
-                // repeat the operation.
-
-                // Returning error status here is problematic. The inputs have been streak removed
-                // but they're still lying around
-                // Maybe just print an error message and
-                // let other system tools clean up
-                psErrorStackPrint(stderr, "");
-                exit(PS_EXIT_UNKNOWN_ERROR);
-            }
-        }
-#endif
-
     }
     // all done. Clean up to look for memory leaks.
@@ -446 +426 @@
             usage();
         }
-        psString dir = pathToDirectory(argv[argnum]);
-        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "directory for temporary files",
-            dir);
-        psFree(dir);
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "path for (temporary if replae) output files",
+            argv[argnum]);
         psArgumentRemove(argnum, &argc, argv);
     } else {
@@ -458 +436 @@
     if ((argnum = psArgumentGet(argc, argv, "-recovery"))) {
         psArgumentRemove(argnum, &argc, argv);
-        psString dir = pathToDirectory(argv[argnum]);
-        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "directory for recovery files",
-            dir);
-        psFree(dir);
+        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "path for recovery files",
+            argv[argnum]);
         psArgumentRemove(argnum, &argc, argv);
     } else if ((stage == IPP_STAGE_RAW) && gotReplace) {
@@ -467 +443 @@
         usage();
     }
-
-#ifdef notdef
-    // This parameter is no longer supported
-    if ((argnum = psArgumentGet(argc, argv, "-remove"))) {
-        if (!gotReplace) {
-            psError(PS_ERR_UNKNOWN, true, "-replace is required with -remove\n");
-            usage();
-        }
-        psArgumentRemove(argnum, &argc, argv);
-        psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REMOVE", 0, "remove original files",
-            true);
-    }
-#endif
 
     if (argc != 1) {

trunk/magic/remove/src/streaksremove.h

@@ -88 +88 @@
 extern ippStage parseStage(psString);
 extern psString pathToDirectory(char *path);
+extern void setStreakFiles( streakFiles *);
 
 #define CHIP_LEVEL_INPUT(_stage) ((_stage == IPP_STAGE_RAW) || (_stage == IPP_STAGE_CHIP))
@@ -95 +96 @@
 #define IN_NEBULOUS(_filename) (!strncasecmp(_filename, "neb://", strlen("neb://")))
 
+
 #endif // STREAKS_H

trunk/magic/remove/src/streaksreplace.c

@@ -97 +97 @@
     }
 
-#ifdef NOTYET
-    if (psMetadataLookupBool(&status, config->arguments, "REPLACE")) {
-        //     swap the instances for the input and output
-        //     Note this is a database operation. No file I/O is performed
-        if (!swapOutputsToInputs(sfiles)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to swap files");
-
-            // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that
-            // it has done and give a detailed report of what happened
-
-            psErrorStackPrint(stderr, "");
-            exit(PS_EXIT_UNKNOWN_ERROR);
-        }
-
-        if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) {
-            //      delete the temporary storage objects (which now points to the original image(s)
-            if (!deleteTemps(sfiles)) {
-                psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files");
-                // XXX: Now what? At this point the output files have been swapped, so we can't
-                // repeat the operation.
-
-                // Returning error status here is problematic. The inputs have been streak removed
-                // but they're still lying around
-                // Maybe just print an error message and
-                // let other system tools clean up
-                psErrorStackPrint(stderr, "");
-                exit(PS_EXIT_UNKNOWN_ERROR);
-            }
-        }
-    }
-#endif  // REPLACE, REMOVE
 //    nebServerFree(ourNebServer);
     psFree(config);

trunk/magic/remove/src/streaksutil.c

@@ -33 +33 @@
 }
 
+streakFiles *ourStreakFiles = NULL;
+
+void
+setStreakFiles(streakFiles *sfiles)
+{
+    ourStreakFiles = sfiles;
+}
+
 // to enhance clarity in these programs we don't propagate errors up the stack
 // we just bail out
 void streaksExit(psString str, int exitCode) {
     psErrorStackPrint(stderr, str);
+    if (ourStreakFiles) {
+        deleteTemps(ourStreakFiles);
+    }
     exit(exitCode);
 }