Add support for bzip2 compression and decompression. More correctly set "version needed".
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 92e21eb..18c0884 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt
@@ -57,6 +57,7 @@ zip_add.c zip_add_dir.c zip_add_entry.c + zip_algorithm_bzip2.c zip_algorithm_deflate.c zip_buffer.c zip_close.c
diff --git a/lib/Makefile.am b/lib/Makefile.am index 0389e08..524f418 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am
@@ -33,6 +33,7 @@ zip_add.c \ zip_add_dir.c \ zip_add_entry.c \ + zip_algorithm_bzip2.c \ zip_algorithm_deflate.c \ zip_buffer.c \ zip_close.c \
diff --git a/lib/zip.h b/lib/zip.h index dd8afe8..e1965cb 100644 --- a/lib/zip.h +++ b/lib/zip.h
@@ -102,37 +102,38 @@ /* libzip error codes */ -#define ZIP_ER_OK 0 /* N No error */ -#define ZIP_ER_MULTIDISK 1 /* N Multi-disk zip archives not supported */ -#define ZIP_ER_RENAME 2 /* S Renaming temporary file failed */ -#define ZIP_ER_CLOSE 3 /* S Closing zip archive failed */ -#define ZIP_ER_SEEK 4 /* S Seek error */ -#define ZIP_ER_READ 5 /* S Read error */ -#define ZIP_ER_WRITE 6 /* S Write error */ -#define ZIP_ER_CRC 7 /* N CRC error */ -#define ZIP_ER_ZIPCLOSED 8 /* N Containing zip archive was closed */ -#define ZIP_ER_NOENT 9 /* N No such file */ -#define ZIP_ER_EXISTS 10 /* N File already exists */ -#define ZIP_ER_OPEN 11 /* S Can't open file */ -#define ZIP_ER_TMPOPEN 12 /* S Failure to create temporary file */ -#define ZIP_ER_ZLIB 13 /* Z Zlib error */ -#define ZIP_ER_MEMORY 14 /* N Malloc failure */ -#define ZIP_ER_CHANGED 15 /* N Entry has been changed */ -#define ZIP_ER_COMPNOTSUPP 16 /* N Compression method not supported */ -#define ZIP_ER_EOF 17 /* N Premature end of file */ -#define ZIP_ER_INVAL 18 /* N Invalid argument */ -#define ZIP_ER_NOZIP 19 /* N Not a zip archive */ -#define ZIP_ER_INTERNAL 20 /* N Internal error */ -#define ZIP_ER_INCONS 21 /* N Zip archive inconsistent */ -#define ZIP_ER_REMOVE 22 /* S Can't remove file */ -#define ZIP_ER_DELETED 23 /* N Entry has been deleted */ -#define ZIP_ER_ENCRNOTSUPP 24 /* N Encryption method not supported */ -#define ZIP_ER_RDONLY 25 /* N Read-only archive */ -#define ZIP_ER_NOPASSWD 26 /* N No password provided */ -#define ZIP_ER_WRONGPASSWD 27 /* N Wrong password provided */ -#define ZIP_ER_OPNOTSUPP 28 /* N Operation not supported */ -#define ZIP_ER_INUSE 29 /* N Resource still in use */ -#define ZIP_ER_TELL 30 /* S Tell error */ +#define ZIP_ER_OK 0 /* N No error */ +#define ZIP_ER_MULTIDISK 1 /* N Multi-disk zip archives not supported */ +#define ZIP_ER_RENAME 2 /* S Renaming temporary file failed */ +#define ZIP_ER_CLOSE 3 /* S Closing zip archive failed */ +#define ZIP_ER_SEEK 4 /* S Seek error */ +#define ZIP_ER_READ 5 /* S Read error */ +#define ZIP_ER_WRITE 6 /* S Write error */ +#define ZIP_ER_CRC 7 /* N CRC error */ +#define ZIP_ER_ZIPCLOSED 8 /* N Containing zip archive was closed */ +#define ZIP_ER_NOENT 9 /* N No such file */ +#define ZIP_ER_EXISTS 10 /* N File already exists */ +#define ZIP_ER_OPEN 11 /* S Can't open file */ +#define ZIP_ER_TMPOPEN 12 /* S Failure to create temporary file */ +#define ZIP_ER_ZLIB 13 /* Z Zlib error */ +#define ZIP_ER_MEMORY 14 /* N Malloc failure */ +#define ZIP_ER_CHANGED 15 /* N Entry has been changed */ +#define ZIP_ER_COMPNOTSUPP 16 /* N Compression method not supported */ +#define ZIP_ER_EOF 17 /* N Premature end of file */ +#define ZIP_ER_INVAL 18 /* N Invalid argument */ +#define ZIP_ER_NOZIP 19 /* N Not a zip archive */ +#define ZIP_ER_INTERNAL 20 /* N Internal error */ +#define ZIP_ER_INCONS 21 /* N Zip archive inconsistent */ +#define ZIP_ER_REMOVE 22 /* S Can't remove file */ +#define ZIP_ER_DELETED 23 /* N Entry has been deleted */ +#define ZIP_ER_ENCRNOTSUPP 24 /* N Encryption method not supported */ +#define ZIP_ER_RDONLY 25 /* N Read-only archive */ +#define ZIP_ER_NOPASSWD 26 /* N No password provided */ +#define ZIP_ER_WRONGPASSWD 27 /* N Wrong password provided */ +#define ZIP_ER_OPNOTSUPP 28 /* N Operation not supported */ +#define ZIP_ER_INUSE 29 /* N Resource still in use */ +#define ZIP_ER_TELL 30 /* S Tell error */ +#define ZIP_ER_COMPRESSED_DATA 31 /* N Compressed data invalid */ /* type of system error value */
diff --git a/lib/zip_algorithm_bzip2.c b/lib/zip_algorithm_bzip2.c new file mode 100644 index 0000000..96bab75 --- /dev/null +++ b/lib/zip_algorithm_bzip2.c
@@ -0,0 +1,268 @@ +/* + zip_algorithm_bzip2.c -- bzip2 (de)compression routines + Copyright (C) 2017 Dieter Baron and Thomas Klausner + + This file is part of libzip, a library to manipulate ZIP archives. + The authors can be contacted at <libzip@nih.at> + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. The names of the authors may not be used to endorse or promote + products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS + OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "zipint.h" + +#if defined(HAVE_LIBBZ2) + +#include <bzlib.h> +#include <stdlib.h> + +struct ctx { + zip_error_t *error; + bool compress; + int compression_flags; + bool end_of_input; + bz_stream zstr; +}; + + +static void * +allocate(bool compress, int compression_flags, zip_error_t *error) { + struct ctx *ctx; + + if ((ctx = (struct ctx *)malloc(sizeof(*ctx))) == NULL) { + return NULL; + } + + ctx->error = error; + ctx->compress = compress; + ctx->compression_flags = compression_flags; + ctx->end_of_input = false; + + ctx->zstr.bzalloc = NULL; + ctx->zstr.bzfree = NULL; + ctx->zstr.opaque = NULL; + + return ctx; +} + + +static void * +compress_allocate(zip_uint16_t method, int compression_flags, zip_error_t *error) { + return allocate(true, compression_flags, error); +} + + +static void * +decompress_allocate(zip_uint16_t method, int compression_flags, zip_error_t *error) { + return allocate(false, compression_flags, error); +} + + +static void +deallocate(void *ud) { + struct ctx *ctx = (struct ctx *)ud; + + free(ctx); +} + + +static int +compression_flags(void *ud) { + return 0; +} + + +static int +map_error(int ret) { + switch (ret) { + case BZ_FINISH_OK: + case BZ_FLUSH_OK: + case BZ_OK: + case BZ_RUN_OK: + case BZ_STREAM_END: + return ZIP_ER_OK; + + case BZ_DATA_ERROR: + case BZ_DATA_ERROR_MAGIC: + case BZ_UNEXPECTED_EOF: + return ZIP_ER_COMPRESSED_DATA; + + case BZ_MEM_ERROR: + return ZIP_ER_MEMORY; + + case BZ_PARAM_ERROR: + return ZIP_ER_INVAL; + + case BZ_CONFIG_ERROR: /* actually, bzip2 miscompiled */ + case BZ_IO_ERROR: + case BZ_OUTBUFF_FULL: + case BZ_SEQUENCE_ERROR: + return ZIP_ER_INTERNAL; + + default: + return ZIP_ER_INTERNAL; + } + +} + +static bool +start(void *ud) { + struct ctx *ctx = (struct ctx *)ud; + int ret; + + ctx->zstr.avail_in = 0; + ctx->zstr.next_in = NULL; + ctx->zstr.avail_out = 0; + ctx->zstr.next_out = NULL; + + if (ctx->compress) { + /* TODO: use ctx->compression_flags */ + ret = BZ2_bzCompressInit(&ctx->zstr, 9, 0, 30); + + } + else { + ret = BZ2_bzDecompressInit(&ctx->zstr, 0, 0); + } + + if (ret != BZ_OK) { + zip_error_set(ctx->error, map_error(ret), 0); + return false; + } + + + return true; +} + + +static bool +end(void *ud) { + struct ctx *ctx = (struct ctx *)ud; + + /* TODO: can this fail? */ + if (ctx->compress) { + BZ2_bzCompressEnd(&ctx->zstr); + } + else { + BZ2_bzDecompressEnd(&ctx->zstr); + } + + return true; +} + + +static bool input(void *ud, zip_uint8_t *data, zip_uint64_t length) { + struct ctx *ctx = (struct ctx *)ud; + + if (length > UINT_MAX || ctx->zstr.avail_in > 0) { + zip_error_set(ctx->error, ZIP_ER_INVAL, 0); + return false; + } + + ctx->zstr.avail_in = (unsigned int)length; + ctx->zstr.next_in = (char *)data; + + return true; +} + + +static void end_of_input(void *ud) { + struct ctx *ctx = (struct ctx *)ud; + + ctx->end_of_input = true; +} + + +static zip_compression_status_t +process(void *ud, zip_uint8_t *data, zip_uint64_t *length) { + struct ctx *ctx = (struct ctx *)ud; + + int ret; + + if (ctx->zstr.avail_in == 0 && !ctx->end_of_input) { + *length = 0; + return ZIP_COMPRESSION_NEED_DATA; + } + + ctx->zstr.avail_out = (unsigned int)ZIP_MIN(UINT_MAX, *length); + ctx->zstr.next_out = (char *)data; + + if (ctx->compress) { + ret = BZ2_bzCompress(&ctx->zstr, ctx->end_of_input ? BZ_FINISH : BZ_RUN); + } + else { + ret = BZ2_bzDecompress(&ctx->zstr); + } + + *length = *length - ctx->zstr.avail_out; + + switch (ret) { + case BZ_FINISH_OK: /* compression */ + return ZIP_COMPRESSION_OK; + + case BZ_OK: /* decompression */ + case BZ_RUN_OK: /* compression */ + if (ctx->zstr.avail_in == 0) { + return ZIP_COMPRESSION_NEED_DATA; + } + return ZIP_COMPRESSION_OK; + + case BZ_STREAM_END: + return ZIP_COMPRESSION_END; + + default: + zip_error_set(ctx->error, map_error(ret), 0); + return ZIP_COMPRESSION_ERROR; + } +} + + +zip_compression_algorithm_t zip_algorithm_bzip2_compress = { + compress_allocate, + deallocate, + compression_flags, + start, + end, + input, + end_of_input, + process +}; + + +zip_compression_algorithm_t zip_algorithm_bzip2_decompress = { + decompress_allocate, + deallocate, + compression_flags, + start, + end, + input, + end_of_input, + process +}; + +#else + +static int dummy; + +#endif /* HAVE_LIBBZ2 */
diff --git a/lib/zip_algorithm_deflate.c b/lib/zip_algorithm_deflate.c index 8abced4..c40a32a 100644 --- a/lib/zip_algorithm_deflate.c +++ b/lib/zip_algorithm_deflate.c
@@ -34,6 +34,7 @@ #include "zipint.h" #include <stdlib.h> +#include <zlib.h> struct ctx { zip_error_t *error; @@ -206,40 +207,6 @@ } -static zip_compression_status_t -decompress_process(void *ud, zip_uint8_t *data, zip_uint64_t *length) { - struct ctx *ctx = (struct ctx *)ud; - - int ret; - - ctx->zstr.avail_out = (uInt)ZIP_MIN(UINT_MAX, *length); - ctx->zstr.next_out = (Bytef *)data; - - ret = deflate(&ctx->zstr, ctx->end_of_input ? Z_FINISH : 0); - - *length = *length - ctx->zstr.avail_out; - - switch (ret) { - case Z_OK: - return ZIP_COMPRESSION_OK; - - case Z_STREAM_END: - return ZIP_COMPRESSION_END; - - case Z_BUF_ERROR: - if (ctx->zstr.avail_in == 0) { - return ZIP_COMPRESSION_NEED_DATA; - } - - /* fallthrough */ - - default: - zip_error_set(ctx->error, ZIP_ER_ZLIB, ret); - return ZIP_COMPRESSION_ERROR; - } -} - - zip_compression_algorithm_t zip_algorithm_deflate_compress = { compress_allocate, deallocate,
diff --git a/lib/zip_close.c b/lib/zip_close.c index c3a4b39..f20b019 100644 --- a/lib/zip_close.c +++ b/lib/zip_close.c
@@ -51,9 +51,6 @@ #endif -/* max deflate size increase: size + ceil(size/16k)*5+6 */ -#define MAX_DEFLATE_SIZE_32 4293656963u - typedef struct { double last_update; /* last value callback function was called with */ @@ -300,9 +297,31 @@ data_length = (zip_int64_t)st.size; if ((st.valid & ZIP_STAT_COMP_SIZE) == 0) { - if (( ((de->comp_method == ZIP_CM_DEFLATE || ZIP_CM_IS_DEFAULT(de->comp_method)) && st.size > MAX_DEFLATE_SIZE_32) - || (de->comp_method != ZIP_CM_STORE && de->comp_method != ZIP_CM_DEFLATE && !ZIP_CM_IS_DEFAULT(de->comp_method)))) + zip_uint64_t max_size; + + switch (ZIP_CM_ACTUAL(de->comp_method)) { + case ZIP_CM_BZIP2: + /* computed by looking at increase of 10 random files of size 1MB when + * compressed with bzip2, rounded up: 1.006 */ + max_size = 4269351188u; + break; + + case ZIP_CM_DEFLATE: + /* max deflate size increase: size + ceil(size/16k)*5+6 */ + max_size = 4293656963u; + break; + + case ZIP_CM_STORE: + max_size = 0xffffffffu; + break; + + default: + max_size = 0; + } + + if (st.size > max_size) { flags |= ZIP_FL_FORCE_ZIP64; + } } else de->comp_size = st.comp_size; @@ -319,7 +338,7 @@ return -1; } - needs_recompress = !((st.comp_method == de->comp_method) || (ZIP_CM_IS_DEFAULT(de->comp_method) && st.comp_method == ZIP_CM_DEFLATE)); + needs_recompress = st.comp_method != ZIP_CM_ACTUAL(de->comp_method); needs_decompress = needs_recompress && (st.comp_method != ZIP_CM_STORE); needs_crc = (st.comp_method == ZIP_CM_STORE) || needs_decompress; needs_compress = needs_recompress && (de->comp_method != ZIP_CM_STORE); @@ -455,9 +474,9 @@ de->crc = st.crc; de->uncomp_size = st.size; de->comp_size = (zip_uint64_t)(offend - offdata); - /* TODO: version needed */ de->bitflags = (zip_uint16_t)((de->bitflags & (zip_uint16_t)~6) | ((zip_uint8_t)compression_flags << 1)); - + _zip_dirent_set_version_needed(de, (flags & ZIP_FL_FORCE_ZIP64) != 0); + if ((ret=_zip_dirent_write(za, de, flags)) < 0) return -1; @@ -466,7 +485,6 @@ zip_error_set(&za->error, ZIP_ER_INTERNAL, 0); return -1; } - if (zip_source_seek_write(za->src, offend, SEEK_SET) < 0) { _zip_error_set_from_source(&za->error, za->src); @@ -540,7 +558,6 @@ return ret; } - static int write_cdir(zip_t *za, const zip_filelist_t *filelist, zip_uint64_t survivors) {
diff --git a/lib/zip_dirent.c b/lib/zip_dirent.c index bfef9fe..197bb3b 100644 --- a/lib/zip_dirent.c +++ b/lib/zip_dirent.c
@@ -281,8 +281,8 @@ de->cloned = 0; de->crc_valid = true; - de->version_madeby = 20 | (ZIP_OPSYS_DEFAULT << 8); - de->version_needed = 20; /* 2.0 */ + de->version_madeby = 63 | (ZIP_OPSYS_DEFAULT << 8); + de->version_needed = 10; /* 1.0 */ de->bitflags = 0; de->comp_method = ZIP_CM_DEFAULT; de->last_mod = 0; @@ -1103,3 +1103,39 @@ return; } + + +void +_zip_dirent_set_version_needed(zip_dirent_t *de, bool force_zip64) { + zip_uint16_t length; + + if (de->comp_method == ZIP_CM_LZMA) { + de->version_needed = 63; + return; + } + + if (de->comp_method == ZIP_CM_BZIP2) { + de->version_needed = 46; + return; + } + + if (force_zip64 || _zip_dirent_needs_zip64(de, 0)) { + de->version_needed = 45; + return; + } + + if (de->comp_method == ZIP_CM_DEFLATE || de->encryption_method == ZIP_EM_TRAD_PKWARE) { + de->version_needed = 20; + return; + } + + /* directory */ + if ((length = _zip_string_length(de->filename)) > 0) { + if (de->filename->raw[length-1] == '/') { + de->version_needed = 20; + return; + } + } + + de->version_needed = 10; +}
diff --git a/lib/zip_err_str.c b/lib/zip_err_str.c index 9c9adb5..6560bba 100644 --- a/lib/zip_err_str.c +++ b/lib/zip_err_str.c
@@ -37,6 +37,7 @@ "Operation not supported", "Resource still in use", "Tell error", + "Compressed data invalid", }; const int _zip_nerr_str = sizeof(_zip_err_str)/sizeof(_zip_err_str[0]); @@ -77,4 +78,5 @@ N, N, S, + N, };
diff --git a/lib/zip_random_unix.c b/lib/zip_random_unix.c index 7a932ef..26706f0 100644 --- a/lib/zip_random_unix.c +++ b/lib/zip_random_unix.c
@@ -34,6 +34,7 @@ #include "zipint.h" #include <fcntl.h> +#include <unistd.h> bool zip_random(zip_uint8_t *buffer, zip_uint16_t length)
diff --git a/lib/zip_set_file_compression.c b/lib/zip_set_file_compression.c index 7bb0bf9..5321687 100644 --- a/lib/zip_set_file_compression.c +++ b/lib/zip_set_file_compression.c
@@ -51,7 +51,7 @@ return -1; } - if (method != ZIP_CM_DEFAULT && method != ZIP_CM_STORE && method != ZIP_CM_DEFLATE) { + if (!zip_compression_method_supported(method, true)) { zip_error_set(&za->error, ZIP_ER_COMPNOTSUPP, 0); return -1; }
diff --git a/lib/zip_source_compress.c b/lib/zip_source_compress.c index 03f6280..37e0318 100644 --- a/lib/zip_source_compress.c +++ b/lib/zip_source_compress.c
@@ -64,6 +64,9 @@ static struct implementation implementations[] = { { ZIP_CM_DEFLATE, &zip_algorithm_deflate_compress, &zip_algorithm_deflate_decompress }, +#if defined(HAVE_LIBBZ2) + { ZIP_CM_BZIP2, &zip_algorithm_bzip2_compress, &zip_algorithm_bzip2_decompress }, +#endif }; static size_t implementations_size = sizeof(implementations) / sizeof(implementations[0]); @@ -74,6 +77,33 @@ static struct context *context_new(zip_int32_t method, bool compress, int compression_flags, zip_compression_algorithm_t *algorithm); static zip_int64_t compress_read(zip_source_t *, struct context *, void *, zip_uint64_t); +static zip_compression_algorithm_t * +get_algorithm(zip_int32_t method, bool compress) { + size_t i; + zip_uint16_t real_method = ZIP_CM_ACTUAL(method); + + for (i = 0; i < implementations_size; i++) { + if (implementations[i].method == real_method) { + if (compress) { + return implementations[i].compress; + } + else { + return implementations[i].decompress; + } + } + } + + return NULL; +} + +bool +zip_compression_method_supported(zip_int32_t method, bool compress) { + if (method == ZIP_CM_STORE) { + return true; + } + return get_algorithm(method, compress) != NULL; +} + zip_source_t * zip_source_compress(zip_t *za, zip_source_t *src, zip_int32_t method, int compression_flags) { return compression_source_new(za, src, method, true, compression_flags); @@ -90,8 +120,6 @@ { struct context *ctx; zip_source_t *s2; - zip_uint16_t real_method; - size_t i; zip_compression_algorithm_t *algorithm = NULL; if (src == NULL) { @@ -99,21 +127,7 @@ return NULL; } - real_method = ZIP_CM_ACTUAL(method); - - for (i = 0; i < implementations_size; i++) { - if (implementations[i].method == real_method) { - if (compress) { - algorithm = implementations[i].compress; - } - else { - algorithm = implementations[i].decompress; - } - break; - } - } - - if (algorithm == NULL) { + if ((algorithm = get_algorithm(method, compress)) == NULL) { zip_error_set(&za->error, ZIP_ER_COMPNOTSUPP, 0); return NULL; }
diff --git a/lib/zipint.h b/lib/zipint.h index 7198889..0e725b2 100644 --- a/lib/zipint.h +++ b/lib/zipint.h
@@ -133,10 +133,13 @@ }; typedef struct zip_compression_algorithm zip_compression_algorithm_t; - +extern zip_compression_algorithm_t zip_algorithm_bzip2_compress; +extern zip_compression_algorithm_t zip_algorithm_bzip2_decompress; extern zip_compression_algorithm_t zip_algorithm_deflate_compress; extern zip_compression_algorithm_t zip_algorithm_deflate_decompress; +bool zip_compression_method_supported(zip_int32_t method, bool compress); + /* This API is not final yet, but we need it internally, so it's private for now. */ const zip_uint8_t *zip_get_extra_field_by_id(zip_t *, int, int, zip_uint16_t, int, zip_uint16_t *); @@ -441,6 +444,7 @@ bool _zip_dirent_needs_zip64(const zip_dirent_t *, zip_flags_t); zip_dirent_t *_zip_dirent_new(void); zip_int64_t _zip_dirent_read(zip_dirent_t *zde, zip_source_t *src, zip_buffer_t *buffer, bool local, zip_error_t *error); +void _zip_dirent_set_version_needed(zip_dirent_t *de, bool force_zip64); zip_int32_t _zip_dirent_size(zip_source_t *src, zip_uint16_t, zip_error_t *); int _zip_dirent_write(zip_t *za, zip_dirent_t *dirent, zip_flags_t flags);