/* LZ4frame API example : compress a file * Modified from an example code by Zbigniew Jędrzejewski-Szmek * * This example streams an input file into an output file * using a bounded memory budget. * Input is read in chunks of IN_CHUNK_SIZE */ #include #include #include #include #include #include #include #include #define IN_CHUNK_SIZE (16*1024) static const LZ4F_preferences_t kPrefs = { { LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0 /* unknown content size */, 0 /* no dictID */ , LZ4F_noBlockChecksum }, 0, /* compression level; 0 == default */ 0, /* autoflush */ 0, /* favor decompression speed */ { 0, 0, 0 }, /* reserved, must be set to 0 */ }; /* safe_fwrite() : * performs fwrite(), ensure operation success, or immediately exit() */ static void safe_fwrite(void* buf, size_t eltSize, size_t nbElt, FILE* f) { size_t const writtenSize = fwrite(buf, eltSize, nbElt, f); size_t const expectedSize = eltSize * nbElt; if (nbElt>0) assert(expectedSize / nbElt == eltSize); /* check overflow */ if (writtenSize < expectedSize) { if (ferror(f)) /* note : ferror() must follow fwrite */ fprintf(stderr, "Write failed \n"); else fprintf(stderr, "Write too short \n"); exit(1); } } /* ================================================= */ /* Streaming Compression example */ /* ================================================= */ typedef struct { int error; unsigned long long size_in; unsigned long long size_out; } compressResult_t; static compressResult_t compress_file_internal(FILE* f_in, FILE* f_out, LZ4F_compressionContext_t ctx, void* inBuff, size_t inChunkSize, void* outBuff, size_t outCapacity, FILE* f_unc, long uncOffset) { compressResult_t result = { 1, 0, 0 }; /* result for an error */ long long count_in = 0, count_out, bytesToOffset = -1; assert(f_in != NULL); assert(f_out != NULL); assert(ctx != NULL); assert(outCapacity >= LZ4F_HEADER_SIZE_MAX); assert(outCapacity >= LZ4F_compressBound(inChunkSize, &kPrefs)); /* write frame header */ { size_t const headerSize = LZ4F_compressBegin(ctx, outBuff, outCapacity, &kPrefs); if (LZ4F_isError(headerSize)) { printf("Failed to start compression: error %u \n", (unsigned)headerSize); return result; } count_out = headerSize; printf("Buffer size is %u bytes, header size %u bytes \n", (unsigned)outCapacity, (unsigned)headerSize); safe_fwrite(outBuff, 1, headerSize, f_out); } /* stream file */ for (;;) { size_t compressedSize; long long inSize = IN_CHUNK_SIZE; if (uncOffset >= 0) { bytesToOffset = uncOffset - count_in; /* read only remaining bytes to offset position */ if (bytesToOffset < IN_CHUNK_SIZE && bytesToOffset > 0) { inSize = bytesToOffset; } } /* input data is at uncompressed data offset */ if (bytesToOffset <= 0 && uncOffset >= 0 && f_unc) { size_t const readSize = fread(inBuff, 1, inSize, f_unc); if (readSize == 0) { uncOffset = -1; continue; } count_in += readSize; compressedSize = LZ4F_uncompressedUpdate(ctx, outBuff, outCapacity, inBuff, readSize, NULL); } else { size_t const readSize = fread(inBuff, 1, inSize, f_in); if (readSize == 0) break; /* nothing left to read from input file */ count_in += readSize; compressedSize = LZ4F_compressUpdate(ctx, outBuff, outCapacity, inBuff, readSize, NULL); } if (LZ4F_isError(compressedSize)) { printf("Compression failed: error %u \n", (unsigned)compressedSize); return result; } printf("Writing %u bytes\n", (unsigned)compressedSize); safe_fwrite(outBuff, 1, compressedSize, f_out); count_out += compressedSize; } /* flush whatever remains within internal buffers */ { size_t const compressedSize = LZ4F_compressEnd(ctx, outBuff, outCapacity, NULL); if (LZ4F_isError(compressedSize)) { printf("Failed to end compression: error %u \n", (unsigned)compressedSize); return result; } printf("Writing %u bytes \n", (unsigned)compressedSize); safe_fwrite(outBuff, 1, compressedSize, f_out); count_out += compressedSize; } result.size_in = count_in; result.size_out = count_out; result.error = 0; return result; } static compressResult_t compress_file(FILE* f_in, FILE* f_out, FILE* f_unc, int uncOffset) { assert(f_in != NULL); assert(f_out != NULL); /* resource allocation */ LZ4F_compressionContext_t ctx; size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); void* const src = malloc(IN_CHUNK_SIZE); size_t const outbufCapacity = LZ4F_compressBound(IN_CHUNK_SIZE, &kPrefs); /* large enough for any input <= IN_CHUNK_SIZE */ void* const outbuff = malloc(outbufCapacity); compressResult_t result = { 1, 0, 0 }; /* == error (default) */ if (!LZ4F_isError(ctxCreation) && src && outbuff) { result = compress_file_internal(f_in, f_out, ctx, src, IN_CHUNK_SIZE, outbuff, outbufCapacity, f_unc, uncOffset); } else { printf("error : resource allocation failed \n"); } LZ4F_freeCompressionContext(ctx); /* supports free on NULL */ free(src); free(outbuff); return result; } /* ================================================= */ /* Streaming decompression example */ /* ================================================= */ static size_t get_block_size(const LZ4F_frameInfo_t* info) { switch (info->blockSizeID) { case LZ4F_default: case LZ4F_max64KB: return 1 << 16; case LZ4F_max256KB: return 1 << 18; case LZ4F_max1MB: return 1 << 20; case LZ4F_max4MB: return 1 << 22; default: printf("Impossible with expected frame specification (<=v1.6.1)\n"); exit(1); } } /* @return : 1==error, 0==success */ static int decompress_file_internal(FILE* f_in, FILE* f_out, LZ4F_dctx* dctx, void* src, size_t srcCapacity, size_t filled, size_t alreadyConsumed, void* dst, size_t dstCapacity) { int firstChunk = 1; size_t ret = 1; assert(f_in != NULL); assert(f_out != NULL); assert(dctx != NULL); assert(src != NULL); assert(srcCapacity > 0); assert(filled <= srcCapacity); assert(alreadyConsumed <= filled); assert(dst != NULL); assert(dstCapacity > 0); /* Decompression */ while (ret != 0) { /* Load more input */ size_t readSize = firstChunk ? filled : fread(src, 1, srcCapacity, f_in); firstChunk=0; const void* srcPtr = (const char*)src + alreadyConsumed; alreadyConsumed=0; const void* const srcEnd = (const char*)srcPtr + readSize; if (readSize == 0 || ferror(f_in)) { printf("Decompress: not enough input or error reading file\n"); return 1; } /* Decompress: * Continue while there is more input to read (srcPtr != srcEnd) * and the frame isn't over (ret != 0) */ while (srcPtr < srcEnd && ret != 0) { /* Any data within dst has been flushed at this stage */ size_t dstSize = dstCapacity; size_t srcSize = (const char*)srcEnd - (const char*)srcPtr; ret = LZ4F_decompress(dctx, dst, &dstSize, srcPtr, &srcSize, /* LZ4F_decompressOptions_t */ NULL); if (LZ4F_isError(ret)) { printf("Decompression error: %s\n", LZ4F_getErrorName(ret)); return 1; } /* Flush output */ if (dstSize != 0) safe_fwrite(dst, 1, dstSize, f_out); /* Update input */ srcPtr = (const char*)srcPtr + srcSize; } assert(srcPtr <= srcEnd); /* Ensure all input data has been consumed. * It is valid to have multiple frames in the same file, * but this example only supports one frame. */ if (srcPtr < srcEnd) { printf("Decompress: Trailing data left in file after frame\n"); return 1; } } /* Check that there isn't trailing data in the file after the frame. * It is valid to have multiple frames in the same file, * but this example only supports one frame. */ { size_t const readSize = fread(src, 1, 1, f_in); if (readSize != 0 || !feof(f_in)) { printf("Decompress: Trailing data left in file after frame\n"); return 1; } } return 0; } /* @return : 1==error, 0==completed */ static int decompress_file_allocDst(FILE* f_in, FILE* f_out, LZ4F_dctx* dctx, void* src, size_t srcCapacity) { assert(f_in != NULL); assert(f_out != NULL); assert(dctx != NULL); assert(src != NULL); assert(srcCapacity >= LZ4F_HEADER_SIZE_MAX); /* ensure LZ4F_getFrameInfo() can read enough data */ /* Read Frame header */ size_t const readSize = fread(src, 1, srcCapacity, f_in); if (readSize == 0 || ferror(f_in)) { printf("Decompress: not enough input or error reading file\n"); return 1; } LZ4F_frameInfo_t info; size_t consumedSize = readSize; { size_t const fires = LZ4F_getFrameInfo(dctx, &info, src, &consumedSize); if (LZ4F_isError(fires)) { printf("LZ4F_getFrameInfo error: %s\n", LZ4F_getErrorName(fires)); return 1; } } /* Allocating enough space for an entire block isn't necessary for * correctness, but it allows some memcpy's to be elided. */ size_t const dstCapacity = get_block_size(&info); void* const dst = malloc(dstCapacity); if (!dst) { perror("decompress_file(dst)"); return 1; } int const decompressionResult = decompress_file_internal( f_in, f_out, dctx, src, srcCapacity, readSize-consumedSize, consumedSize, dst, dstCapacity); free(dst); return decompressionResult; } /* @result : 1==error, 0==success */ static int decompress_file(FILE* f_in, FILE* f_out) { assert(f_in != NULL); assert(f_out != NULL); /* Resource allocation */ void* const src = malloc(IN_CHUNK_SIZE); if (!src) { perror("decompress_file(src)"); return 1; } LZ4F_dctx* dctx; { size_t const dctxStatus = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); if (LZ4F_isError(dctxStatus)) { printf("LZ4F_dctx creation error: %s\n", LZ4F_getErrorName(dctxStatus)); } } int const result = !dctx ? 1 /* error */ : decompress_file_allocDst(f_in, f_out, dctx, src, IN_CHUNK_SIZE); free(src); LZ4F_freeDecompressionContext(dctx); /* note : free works on NULL */ return result; } int compareFiles(FILE* fp0, FILE* fp1, FILE* fpUnc, long uncOffset) { int result = 0; long bytesRead = 0; long bytesToOffset = -1; long b1Size = 1024; while (result==0) { char b1[b1Size]; size_t r1; size_t bytesToRead = sizeof b1; if (uncOffset >= 0) { bytesToOffset = uncOffset - bytesRead; /* read remainder to offset */ if (bytesToOffset < b1Size) { bytesToRead = bytesToOffset; } } char b0[1024]; size_t r0; if (bytesToOffset <= 0 && fpUnc) { bytesToRead = sizeof b1; r0 = fread(b0, 1,bytesToRead, fpUnc); } else { r0 = fread(b0, 1, bytesToRead, fp0); } r1 = fread(b1, 1, r0, fp1); result = (r0 != r1); if (!r0 || !r1) break; if (!result) result = memcmp(b0, b1, r0); bytesRead += r1; } return result; } int main(int argc, char **argv) { char inpFilename[256] = { 0 }; char lz4Filename[256] = { 0 }; char decFilename[256] = { 0 }; int uncOffset = -1; char uncFilename[256] = { 0 }; int opt; if (argc < 2) { printf("Please specify input filename\n"); return EXIT_FAILURE; } snprintf(inpFilename, 256, "%s", argv[1]); snprintf(lz4Filename, 256, "%s.lz4", argv[1]); snprintf(decFilename, 256, "%s.lz4.dec", argv[1]); while ((opt = getopt(argc, argv, "o:d:")) != -1) { switch (opt) { case 'd': snprintf(uncFilename, 256, "%s", optarg); break; case 'o': uncOffset = atoi(optarg); break; default: printf("usage: %s [-o -d ]\n", argv[0]); printf("-o uncompressed data offset\n"); printf(" inject uncompressed data at this offset into the lz4 file\n"); printf("-d uncompressed file\n"); printf(" file to inject without compression into the lz4 file\n"); return EXIT_FAILURE; } } printf("inp = [%s]\n", inpFilename); printf("lz4 = [%s]\n", lz4Filename); printf("dec = [%s]\n", decFilename); if (uncOffset > 0) { printf("unc = [%s]\n", uncFilename); printf("ofs = [%i]\n", uncOffset); } /* compress */ { FILE* const inpFp = fopen(inpFilename, "rb"); FILE* const outFp = fopen(lz4Filename, "wb"); FILE* const uncFp = fopen(uncFilename, "rb"); printf("compress : %s -> %s\n", inpFilename, lz4Filename); compressResult_t const ret = compress_file( inpFp, outFp, uncFp, uncOffset); fclose(outFp); fclose(inpFp); if (uncFp) fclose(uncFp); if (ret.error) { printf("compress : failed with code %i\n", ret.error); return ret.error; } printf("%s: %zu → %zu bytes, %.1f%%\n", inpFilename, (size_t)ret.size_in, (size_t)ret.size_out, /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */ (double)ret.size_out / ret.size_in * 100); printf("compress : done\n"); } /* decompress */ { FILE* const inpFp = fopen(lz4Filename, "rb"); FILE* const outFp = fopen(decFilename, "wb"); printf("decompress : %s -> %s\n", lz4Filename, decFilename); int const ret = decompress_file(inpFp, outFp); fclose(outFp); fclose(inpFp); if (ret) { printf("decompress : failed with code %i\n", ret); return ret; } printf("decompress : done\n"); } /* verify */ { FILE* const inpFp = fopen(inpFilename, "rb"); FILE* const decFp = fopen(decFilename, "rb"); FILE* const uncFp = fopen(uncFilename, "rb"); printf("verify : %s <-> %s\n", inpFilename, decFilename); int const cmp = compareFiles(inpFp, decFp, uncFp, uncOffset); fclose(decFp); fclose(inpFp); if (uncFp) fclose(uncFp); if (cmp) { printf("corruption detected : decompressed file differs from original\n"); return cmp; } printf("verify : OK\n"); } return 0; }