Merge pull request #1119 from lz4/skipFrames

support skippable frames within pipe

6 files changed, 37 insertions, 8 deletions

diff --git a/lib/lz4frame.c b/lib/lz4frame.c
index 5373083..0516f1f 100644
--- a/lib/lz4frame.c
+++ b/lib/lz4frame.c
@@ -206,8 +206,6 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
 #define _4BITS 0x0F
 #define _8BITS 0xFF
 
-#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
-#define LZ4F_MAGICNUMBER 0x184D2204U
 #define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U
 #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
 
diff --git a/lib/lz4frame.h b/lib/lz4frame.h
index 1de09d8..7ebbfec 100644
--- a/lib/lz4frame.h
+++ b/lib/lz4frame.h
@@ -376,6 +376,8 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx);
 *  Streaming decompression functions
 *************************************/
 
+#define LZ4F_MAGICNUMBER 0x184D2204U
+#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
 #define LZ4F_MIN_SIZE_TO_KNOW_HEADER_LENGTH 5
 
 /*! LZ4F_headerSize() : v1.9.0+
diff --git a/programs/lz4io.c b/programs/lz4io.c
index 36736c2..5644775 100644
--- a/programs/lz4io.c
+++ b/programs/lz4io.c
@@ -1152,6 +1152,23 @@ LZ4IO_passThrough(FILE* finput, FILE* foutput,
     return total;
 }
 
+/* when fseek() doesn't work (pipe scenario),
+ * read and forget from input.
+**/
+#define SKIP_BUFF_SIZE (16 KB)
+#define MIN(a,b)   ( ((a)<(b)) ? (a) : (b) )
+static int skipStream(FILE* f, unsigned offset)
+{
+    char buf[SKIP_BUFF_SIZE];
+    while (offset > 0) {
+        size_t const tr = MIN(offset, sizeof(buf));
+        size_t const r = fread(buf, 1, tr, f);
+        if (r != tr) return 1; /* error reading f */
+        offset -= (unsigned)tr;
+    }
+    assert(offset == 0);
+    return 0;
+}
 
 /** Safely handle cases when (unsigned)offset > LONG_MAX */
 static int fseek_u32(FILE *fp, unsigned offset, int where)
@@ -1163,13 +1180,15 @@ static int fseek_u32(FILE *fp, unsigned offset, int where)
     while (offset > 0) {
         unsigned s = offset;
         if (s > stepMax) s = stepMax;
-        errorNb = UTIL_fseek(fp, (long) s, SEEK_CUR);
-        if (errorNb != 0) break;
-        offset -= s;
+        errorNb = UTIL_fseek(fp, (long)s, SEEK_CUR);
+        if (errorNb==0) { offset -= s; continue; }
+        errorNb = skipStream(fp, offset);
+        offset = 0;
     }
     return errorNb;
 }
 
+
 #define ENDOFSTREAM ((unsigned long long)-1)
 #define DECODING_ERROR ((unsigned long long)-2)
 static unsigned long long
diff --git a/tests/Makefile b/tests/Makefile
index 6f063a1..002fb40 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -342,6 +342,17 @@ test-lz4-multiple-legacy: lz4 datagen
 	! $(LZ4) -f -l -m tmp-tlm-concat1 notHere-legacy tmp-tlm-concat2  # must fail : notHere-legacy not present
 	@$(RM) tmp-tlm*
 
+SKIPFILE = goldenSamples/skip.bin
+test-lz4-skippable: lz4 datagen
+	@echo "\n ---- test lz4 with skippable frames ----"
+	$(LZ4) -dc $(SKIPFILE)
+	$(LZ4) -dc < $(SKIPFILE)
+	cat $(SKIPFILE) | $(LZ4) -dc
+	echo "Hello from Valid Frame!\n" > tmplsk
+	$(LZ4) tmplsk -c > tmplsk.lz4
+	cat $(SKIPFILE) tmplsk.lz4 $(SKIPFILE) | $(LZ4) -dc
+	$(RM) tmplsk*
+
 test-lz4-basic: lz4 datagen unlz4 lz4cat
 	@echo "\n ---- test lz4 basic compression/decompression ----"
 	$(DATAGEN) -g0       | $(LZ4) -v     | $(LZ4) -t
@@ -484,7 +495,8 @@ test-lz4-essentials : lz4 datagen test-lz4-basic test-lz4-multiple test-lz4-mult
 	@$(RM) tmp*
 
 test-lz4: lz4 datagen test-lz4-essentials test-lz4-opt-parser \
-          test-lz4-sparse test-lz4-hugefile test-lz4-dict
+          test-lz4-sparse test-lz4-hugefile test-lz4-dict \
+          test-lz4-skippable
 	@$(RM) tmp*
 
 test-lz4c: lz4c datagen
diff --git a/tests/frametest.c b/tests/frametest.c
index 58eac38..2d355bb 100644
--- a/tests/frametest.c
+++ b/tests/frametest.c
@@ -65,8 +65,6 @@ static void FUZ_writeLE32 (void* dstVoidPtr, U32 value32)
 /*-************************************
 *  Constants
 **************************************/
-#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U
-
 #define KB *(1U<<10)
 #define MB *(1U<<20)
 #define GB *(1U<<30)
diff --git a/tests/goldenSamples/skip.bin b/tests/goldenSamples/skip.bin
new file mode 100644
index 0000000..1a8b9d5
--- /dev/null
+++ b/tests/goldenSamples/skip.bin