Multiple files decompression refactoring

2 files changed, 99 insertions, 101 deletions

diff --git a/programs/Makefile b/programs/Makefile
index 7bbe060..fa6b04d 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -191,7 +191,19 @@ test-lz4-frame-concatenation: lz4 datagen
 	@rm *.test
 	@echo frame concatenation test completed
 
-test-lz4: lz4 datagen test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
+test-lz4-multiple: lz4 datagen
+	@echo ---- test multiple files ----
+	@./datagen -s1        > tmp1
+	@./datagen -s2 -g100K > tmp2
+	@./datagen -s3 -g1M   > tmp3
+	./lz4 -f -m tmp*
+	ls -ls tmp*
+	rm tmp1 tmp2 tmp3
+	./lz4 -df -m *.lz4
+	ls -ls tmp*
+	@rm tmp*
+
+test-lz4: lz4 datagen test-lz4-multiple test-lz4-sparse test-lz4-contentSize test-lz4-frame-concatenation
 	@echo ---- test lz4 basic compression/decompression ----
 	./datagen -g0     | ./lz4 -v     | ./lz4 -t
 	./datagen -g16KB  | ./lz4 -9     | ./lz4 -t
@@ -202,13 +214,6 @@ test-lz4: lz4 datagen test-lz4-sparse test-lz4-contentSize test-lz4-frame-concat
 	./datagen -g256MB | ./lz4 -vqB4D | ./lz4 -t
 	./datagen -g6GB   | ./lz4 -vqB5D | ./lz4 -qt
 	./datagen -g6GB   | ./lz4 -vq9BD | ./lz4 -qt
-	@echo ---- test multiple input files ----
-	@./datagen -s1 > file1
-	@./datagen -s2 > file2
-	@./datagen -s3 > file3
-	./lz4 -f -m file1 file2 file3
-	ls -l file*
-	@rm file1 file2 file3 file1.lz4 file2.lz4 file3.lz4
 	@echo ---- test pass-through ----
 	./datagen | ./lz4 -tf
 
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 6ca7ee7..130a1de 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -159,7 +159,7 @@ static const int maxBlockSizeID = 7;
 #define EXTENDED_ARGUMENTS
 #define EXTENDED_HELP
 #define EXTENDED_FORMAT
-#define DEFAULT_DECOMPRESSOR LZ4IO_decompressFile
+#define DEFAULT_DECOMPRESSOR LZ4IO_decompressLZ4F
 
 
 /* ************************************************** */
@@ -751,16 +751,44 @@ static unsigned long long LZ4IO_decodeLegacyStream(FILE* finput, FILE* foutput)
 
 
 typedef struct {
-    FILE*  sFile;
-    FILE*  dFile;
     void*  srcBuffer;
     size_t srcBufferSize;
     void*  dstBuffer;
     size_t dstBufferSize;
     LZ4F_decompressionContext_t dCtx;
-} decompressionJob_t;
+} dRess_t;
 
-static unsigned long long LZ4IO_decompressJob(decompressionJob_t ress)
+static const size_t LZ4IO_dBufferSize = 64 KB;
+
+static dRess_t LZ4IO_createDResources(void)
+{
+    dRess_t ress;
+    LZ4F_errorCode_t errorCode;
+
+    /* init */
+    errorCode = LZ4F_createDecompressionContext(&ress.dCtx, LZ4F_VERSION);
+    if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create LZ4F context : %s", LZ4F_getErrorName(errorCode));
+
+    /* Allocate Memory */
+    ress.srcBufferSize = LZ4IO_dBufferSize;
+    ress.srcBuffer = malloc(ress.srcBufferSize);
+    ress.dstBufferSize = LZ4IO_dBufferSize;
+    ress.dstBuffer = malloc(ress.dstBufferSize);
+    if (!ress.srcBuffer || !ress.dstBuffer) EXM_THROW(61, "Allocation error : not enough memory");
+
+    return ress;
+}
+
+static void LZ4IO_freeDResources(dRess_t ress)
+{
+    LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(ress.dCtx);
+    if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
+    free(ress.srcBuffer);
+    free(ress.dstBuffer);
+}
+
+
+static unsigned long long LZ4IO_decompressLZ4F(dRess_t ress, FILE* srcFile, FILE* dstFile)
 {
     unsigned long long filesize = 0;
     LZ4F_errorCode_t errorCode;
@@ -782,7 +810,7 @@ static unsigned long long LZ4IO_decompressJob(decompressionJob_t ress)
         size_t pos = 0;
 
         /* Read input */
-        readSize = fread(ress.srcBuffer, 1, ress.srcBufferSize, ress.sFile);
+        readSize = fread(ress.srcBuffer, 1, ress.srcBufferSize, srcFile);
         if (!readSize) break;   /* empty file or stream */
 
         while (pos < readSize)
@@ -799,69 +827,26 @@ static unsigned long long LZ4IO_decompressJob(decompressionJob_t ress)
                 /* Write Block */
                 filesize += decodedBytes;
                 DISPLAYUPDATE(2, "\rDecompressed : %u MB  ", (unsigned)(filesize>>20));
-                storedSkips = LZ4IO_fwriteSparse(ress.dFile, ress.dstBuffer, decodedBytes, storedSkips);
+                storedSkips = LZ4IO_fwriteSparse(dstFile, ress.dstBuffer, decodedBytes, storedSkips);
             }
         }
     }
 
-    LZ4IO_fwriteSparseEnd(ress.dFile, storedSkips);
-
-    return filesize;
-}
-
-
-static const size_t LZ4IO_dBufferSize = 64 KB;
-
-static unsigned long long LZ4IO_decompressFile(FILE* finput, FILE* foutput)
-{
-    unsigned long long filesize = 0;
-    void* inBuff;
-    void* outBuff;
-    const size_t inBuffSize = LZ4IO_dBufferSize;
-    const size_t outBuffSize = LZ4IO_dBufferSize;
-    LZ4F_decompressionContext_t dCtx;
-    LZ4F_errorCode_t errorCode;
-    decompressionJob_t ress;
-
-    /* init */
-    errorCode = LZ4F_createDecompressionContext(&dCtx, LZ4F_VERSION);
-    if (LZ4F_isError(errorCode)) EXM_THROW(60, "Can't create context : %s", LZ4F_getErrorName(errorCode));
-
-    /* Allocate Memory */
-    inBuff = malloc(inBuffSize);
-    outBuff = malloc(outBuffSize);
-    if (!inBuff || !outBuff) EXM_THROW(61, "Allocation error : not enough memory");
-
-    /* Decompression Job */
-    ress.dCtx = dCtx;
-    ress.dFile = foutput;
-    ress.dstBuffer = outBuff;
-    ress.dstBufferSize = outBuffSize;
-    ress.sFile = finput;
-    ress.srcBuffer = inBuff;
-    ress.srcBufferSize = inBuffSize;
-
-    filesize = LZ4IO_decompressJob(ress);
-
-    /* Free */
-    free(inBuff);
-    free(outBuff);
-    errorCode = LZ4F_freeDecompressionContext(dCtx);
-    if (LZ4F_isError(errorCode)) EXM_THROW(69, "Error : can't free LZ4F context resource : %s", LZ4F_getErrorName(errorCode));
+    LZ4IO_fwriteSparseEnd(dstFile, storedSkips);
 
     return filesize;
 }
 
 
-static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char U32store[MAGICNUMBER_SIZE])
+static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigned char MNstore[MAGICNUMBER_SIZE])
 {
     void* buffer = malloc(64 KB);
     size_t read = 1, sizeCheck;
     unsigned long long total = MAGICNUMBER_SIZE;
     unsigned storedSkips = 0;
 
-    sizeCheck = fwrite(U32store, 1, MAGICNUMBER_SIZE, foutput);
-    if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through error at start");
+    sizeCheck = fwrite(MNstore, 1, MAGICNUMBER_SIZE, foutput);
+    if (sizeCheck != MAGICNUMBER_SIZE) EXM_THROW(50, "Pass-through write error");
 
     while (read)
     {
@@ -877,9 +862,9 @@ static unsigned long long LZ4IO_passThrough(FILE* finput, FILE* foutput, unsigne
 
 
 #define ENDOFSTREAM ((unsigned long long)-1)
-static unsigned long long selectDecoder( FILE* finput,  FILE* foutput)
+static unsigned long long selectDecoder(dRess_t ress, FILE* finput, FILE* foutput)
 {
-    unsigned char U32store[MAGICNUMBER_SIZE];
+    unsigned char MNstore[MAGICNUMBER_SIZE];
     unsigned magicNumber, size;
     int errorNb;
     size_t nbReadBytes;
@@ -896,34 +881,34 @@ static unsigned long long selectDecoder( FILE* finput,  FILE* foutput)
     }
     else
     {
-      nbReadBytes = fread(U32store, 1, MAGICNUMBER_SIZE, finput);
+      nbReadBytes = fread(MNstore, 1, MAGICNUMBER_SIZE, finput);
       if (nbReadBytes==0) return ENDOFSTREAM;                  /* EOF */
       if (nbReadBytes != MAGICNUMBER_SIZE) EXM_THROW(40, "Unrecognized header : Magic Number unreadable");
-      magicNumber = LZ4IO_readLE32(U32store);   /* Little Endian format */
+      magicNumber = LZ4IO_readLE32(MNstore);   /* Little Endian format */
     }
     if (LZ4IO_isSkippableMagicNumber(magicNumber)) magicNumber = LZ4IO_SKIPPABLE0;  /* fold skippable magic numbers */
 
     switch(magicNumber)
     {
     case LZ4IO_MAGICNUMBER:
-        return DEFAULT_DECOMPRESSOR(finput, foutput);
+        return LZ4IO_decompressLZ4F(ress, finput, foutput);
     case LEGACY_MAGICNUMBER:
         DISPLAYLEVEL(4, "Detected : Legacy format \n");
         return LZ4IO_decodeLegacyStream(finput, foutput);
     case LZ4IO_SKIPPABLE0:
         DISPLAYLEVEL(4, "Skipping detected skippable area \n");
-        nbReadBytes = fread(U32store, 1, 4, finput);
+        nbReadBytes = fread(MNstore, 1, 4, finput);
         if (nbReadBytes != 4) EXM_THROW(42, "Stream error : skippable size unreadable");
-        size = LZ4IO_readLE32(U32store);     /* Little Endian format */
+        size = LZ4IO_readLE32(MNstore);     /* Little Endian format */
         errorNb = fseek(finput, size, SEEK_CUR);
         if (errorNb != 0) EXM_THROW(43, "Stream error : cannot skip skippable area");
-        return selectDecoder(finput, foutput);
+        return selectDecoder(ress, finput, foutput);
     EXTENDED_FORMAT;
     default:
         if (nbCalls == 1)   /* just started */
         {
             if (g_overwrite)
-                return LZ4IO_passThrough(finput, foutput, U32store);
+                return LZ4IO_passThrough(finput, foutput, MNstore);
             EXM_THROW(44,"Unrecognized header : file cannot be decoded");   /* Wrong magic number at the beginning of 1st stream */
         }
         DISPLAYLEVEL(2, "Stream followed by unrecognized data\n");
@@ -932,18 +917,16 @@ static unsigned long long selectDecoder( FILE* finput,  FILE* foutput)
 }
 
 
-int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename)
+static int LZ4IO_decompressFile_extRess(dRess_t ress, const char* input_filename, const char* output_filename)
 {
     unsigned long long filesize = 0, decodedSize=0;
     FILE* finput;
     FILE* foutput;
-    clock_t start, end;
 
 
     /* Init */
-    start = clock();
     if (LZ4IO_getFiles(input_filename, output_filename, &finput, &foutput))
-        EXM_THROW(50, "File error");
+        return 1;
 
     /* sparse file */
     if (g_sparseFileSupport) { SET_SPARSE_FILE_MODE(foutput); }
@@ -951,67 +934,77 @@ int LZ4IO_decompressFilename(const char* input_filename, const char* output_file
     /* Loop over multiple streams */
     do
     {
-        decodedSize = selectDecoder(finput, foutput);
+        decodedSize = selectDecoder(ress, finput, foutput);
         if (decodedSize != ENDOFSTREAM)
             filesize += decodedSize;
     } while (decodedSize != ENDOFSTREAM);
 
     /* Final Status */
-    end = clock();
     DISPLAYLEVEL(2, "\r%79s\r", "");
     DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
-    if (end==start) end=start+1;
-    {
-        double seconds = (double)(end - start)/CLOCKS_PER_SEC;
-        DISPLAYLEVEL(4, "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024);
-    }
 
     /* Close */
     fclose(finput);
     fclose(foutput);
 
-    /*  Error status = OK */
     return 0;
 }
 
 
+int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename)
+{
+    dRess_t ress;
+    clock_t start, end;
+    int missingFiles = 0;
+
+    start = clock();
+
+    ress = LZ4IO_createDResources();
+    missingFiles += LZ4IO_decompressFile_extRess(ress, input_filename, output_filename);
+    LZ4IO_freeDResources(ress);
+
+    end = clock();
+    if (end==start) end=start+1;
+    {
+        double seconds = (double)(end - start)/CLOCKS_PER_SEC;
+        DISPLAYLEVEL(4, "Done in %.2f sec  \n", seconds);
+    }
+
+    return missingFiles;
+}
+
+
 int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix)
 {
     int i;
-    int skipped_files = 0;
-    int missing_files = 0;
+    int skippedFiles = 0;
+    int missingFiles = 0;
     char* outFileName = (char*)malloc(FNSPACE);
     size_t ofnSize = FNSPACE;
     const size_t suffixSize = strlen(suffix);
     char* ifnSuffix = (char*)malloc(suffixSize + 1);
+    dRess_t ress;
+
+    ress = LZ4IO_createDResources();
 
     for (i=0; i<ifntSize; i++)
     {
-        size_t ifnSize;
-        FILE* ifp = fopen(inFileNamesTable[i], "r");
-        if (ifp == NULL)
-        {
-            DISPLAYLEVEL(2, "Unable to access file for processing: %s\n", inFileNamesTable[i]);
-            missing_files++;
-            continue;
-        }
-        fclose(ifp);
-        ifnSize = strlen(inFileNamesTable[i]);
+        size_t ifnSize = strlen(inFileNamesTable[i]);
         strcpy(ifnSuffix, inFileNamesTable[i] + ifnSize - suffixSize);
         if (ofnSize <= ifnSize-suffixSize+1) { free(outFileName); ofnSize = ifnSize + 20; outFileName = (char*)malloc(ofnSize); }
         if (ifnSize <= suffixSize  ||  strcmp(ifnSuffix, suffix) != 0)
         {
             DISPLAYLEVEL(2, "File extension doesn't match expected LZ4_EXTENSION (%4s); will not process file: %s\n", suffix, inFileNamesTable[i]);
-            skipped_files++;
+            skippedFiles++;
             continue;
         }
         memcpy(outFileName, inFileNamesTable[i], ifnSize - suffixSize);
         outFileName[ifnSize-suffixSize] = '\0';
-        LZ4IO_decompressFilename(inFileNamesTable[i], outFileName);
+
+        missingFiles += LZ4IO_decompressFile_extRess(ress, inFileNamesTable[i], outFileName);
     }
+
+    LZ4IO_freeDResources(ress);
     free(outFileName);
-    free(ifnSuffix);
-    if (skipped_files > 0) return 1;
-    if (missing_files > 0) return 1;
-    return 0;
+    return missingFiles + skippedFiles;
 }