Merge branch 'zstd-support'
diff --git a/CMakeLists.txt b/CMakeLists.txt
index d42303a..64e6676 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -17,6 +17,7 @@
option(ENABLE_BZIP2 "Enable use of BZip2" ON)
option(ENABLE_LZMA "Enable use of LZMA" ON)
+option(ENABLE_ZSTD "Enable use of Zstandard" ON)
option(BUILD_TOOLS "Build tools in the src directory (zipcmp, zipmerge, ziptool)" ON)
option(BUILD_REGRESS "Build regression tests" ON)
@@ -203,6 +204,14 @@
endif(LIBLZMA_FOUND)
endif(ENABLE_LZMA)
+if(ENABLE_ZSTD)
+ find_package(Zstd)
+ if(Zstd_FOUND)
+ set(HAVE_LIBZSTD 1)
+ else()
+ message(WARNING "-- zstd library not found; zstandard support disabled")
+ endif(Zstd_FOUND)
+endif(ENABLE_ZSTD)
if (COMMONCRYPTO_FOUND)
set(HAVE_CRYPTO 1)
diff --git a/THANKS b/THANKS
index 3661cf3..fcb469d 100644
--- a/THANKS
+++ b/THANKS
@@ -32,6 +32,7 @@
Erwin Haid <erwin.haid@gmx.de>
Eun-cheol Joo
Florian Delizy <florian.delizy@gmail.com>
+Force Charlie <charlieio@outlook.com>
François Simon <AT.GFI.Francois.SIMON@sesam-vitale.fr>
Frederik Ramm <frederik@remote.org>
gk7huki <gk7huki@gmail.com>
diff --git a/cmake-config.h.in b/cmake-config.h.in
index 2df6f82..544adbd 100644
--- a/cmake-config.h.in
+++ b/cmake-config.h.in
@@ -29,6 +29,7 @@
#cmakedefine HAVE_GNUTLS
#cmakedefine HAVE_LIBBZ2
#cmakedefine HAVE_LIBLZMA
+#cmakedefine HAVE_LIBZSTD
#cmakedefine HAVE_LOCALTIME_R
#cmakedefine HAVE_MBEDTLS
#cmakedefine HAVE_MKSTEMP
diff --git a/cmake/FindZstd.cmake b/cmake/FindZstd.cmake
new file mode 100644
index 0000000..a0da503
--- /dev/null
+++ b/cmake/FindZstd.cmake
@@ -0,0 +1,135 @@
+# Copyright (C) 2020 Dieter Baron and Thomas Klausner
+#
+# The authors can be contacted at <libzip@nih.at>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+#
+# 3. The names of the authors may not be used to endorse or promote
+# products derived from this software without specific prior
+# written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#[=======================================================================[.rst:
+FindZstd
+-------
+
+Finds the Zstandard (zstd) library.
+
+Imported Targets
+^^^^^^^^^^^^^^^^
+
+This module provides the following imported targets, if found:
+
+``Zstd::Zstd``
+ The Zstandard library
+
+Result Variables
+^^^^^^^^^^^^^^^^
+
+This will define the following variables:
+
+``Zstd_FOUND``
+ True if the system has the Zstandard library.
+``Zstd_VERSION``
+ The version of the Zstandard library which was found.
+``Zstd_INCLUDE_DIRS``
+ Include directories needed to use Zstandard.
+``Zstd_LIBRARIES``
+ Libraries needed to link to Zstandard.
+
+Cache Variables
+^^^^^^^^^^^^^^^
+
+The following cache variables may also be set:
+
+``Zstd_INCLUDE_DIR``
+ The directory containing ``zstd.h``.
+``Zstd_LIBRARY``
+ The path to the Zstandard library.
+
+#]=======================================================================]
+
+find_package(PkgConfig)
+pkg_check_modules(PC_Zstd QUIET zstd)
+
+find_path(Zstd_INCLUDE_DIR
+ NAMES zstd.h
+ PATHS ${PC_Zstd_INCLUDE_DIRS}
+)
+find_library(Zstd_LIBRARY
+ NAMES zstd
+ PATHS ${PC_Zstd_LIBRARY_DIRS}
+)
+
+# Extract version information from the header file
+if(Zstd_INCLUDE_DIR)
+ file(STRINGS ${Zstd_INCLUDE_DIR}/zstd.h _ver_major_line
+ REGEX "^#define ZSTD_VERSION_MAJOR *[0-9]+"
+ LIMIT_COUNT 1)
+ string(REGEX MATCH "[0-9]+"
+ Zstd_MAJOR_VERSION "${_ver_major_line}")
+ file(STRINGS ${Zstd_INCLUDE_DIR}/zstd.h _ver_minor_line
+ REGEX "^#define ZSTD_VERSION_MINOR *[0-9]+"
+ LIMIT_COUNT 1)
+ string(REGEX MATCH "[0-9]+"
+ Zstd_MINOR_VERSION "${_ver_minor_line}")
+ file(STRINGS ${Zstd_INCLUDE_DIR}/zstd.h _ver_release_line
+ REGEX "^#define ZSTD_VERSION_RELEASE *[0-9]+"
+ LIMIT_COUNT 1)
+ string(REGEX MATCH "[0-9]+"
+ Zstd_RELEASE_VERSION "${_ver_release_line}")
+ set(Zstd_VERSION "${Zstd_MAJOR_VERSION}.${Zstd_MINOR_VERSION}.${Zstd_RELEASE_VERSION}")
+ unset(_ver_major_line)
+ unset(_ver_minor_line)
+ unset(_ver_release_line)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Zstd
+ FOUND_VAR Zstd_FOUND
+ REQUIRED_VARS
+ Zstd_LIBRARY
+ Zstd_INCLUDE_DIR
+ VERSION_VAR Zstd_VERSION
+)
+
+if(Zstd_FOUND)
+ set(Zstd_LIBRARIES ${Zstd_LIBRARY})
+ set(Zstd_INCLUDE_DIRS ${Zstd_INCLUDE_DIR})
+ set(Zstd_DEFINITIONS ${PC_Zstd_CFLAGS_OTHER})
+endif()
+
+if(Zstd_FOUND AND NOT TARGET Zstd::Zstd)
+ add_library(Zstd::Zstd UNKNOWN IMPORTED)
+ set_target_properties(Zstd::Zstd PROPERTIES
+ IMPORTED_LOCATION "${Zstd_LIBRARY}"
+ INTERFACE_COMPILE_OPTIONS "${PC_Zstd_CFLAGS_OTHER}"
+ INTERFACE_INCLUDE_DIRECTORIES "${Zstd_INCLUDE_DIR}"
+ )
+endif()
+
+mark_as_advanced(
+ Zstd_INCLUDE_DIR
+ Zstd_LIBRARY
+)
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
index 082203d..a81367e 100644
--- a/lib/CMakeLists.txt
+++ b/lib/CMakeLists.txt
@@ -146,6 +146,11 @@
target_link_libraries(zip PRIVATE LibLZMA::LibLZMA)
endif()
+if(HAVE_LIBZSTD)
+ target_sources(zip PRIVATE zip_algorithm_zstd.c)
+ target_link_libraries(zip PRIVATE Zstd::Zstd)
+endif()
+
if(HAVE_COMMONCRYPTO)
target_sources(zip PRIVATE zip_crypto_commoncrypto.c)
elseif(HAVE_WINDOWS_CRYPTO)
diff --git a/lib/zip.h b/lib/zip.h
index a90776d..4a1bf89 100644
--- a/lib/zip.h
+++ b/lib/zip.h
@@ -163,7 +163,9 @@
/* 15-17 - Reserved by PKWARE */
#define ZIP_CM_TERSE 18 /* compressed using IBM TERSE (new) */
#define ZIP_CM_LZ77 19 /* IBM LZ77 z Architecture (PFS) */
+/* 20 - old value for Zstandard */
#define ZIP_CM_LZMA2 33
+#define ZIP_CM_ZSTD 93 /* Zstandard compressed data */
#define ZIP_CM_XZ 95 /* XZ compressed data */
#define ZIP_CM_JPEG 96 /* Compressed Jpeg data */
#define ZIP_CM_WAVPACK 97 /* WavPack compressed data */
diff --git a/lib/zip_algorithm_zstd.c b/lib/zip_algorithm_zstd.c
new file mode 100644
index 0000000..025a46d
--- /dev/null
+++ b/lib/zip_algorithm_zstd.c
@@ -0,0 +1,287 @@
+/*
+ zip_algorithm_zstd.c -- zstd (de)compression routines
+ Copyright (C) 2020 Dieter Baron and Thomas Klausner
+
+ This file is part of libzip, a library to manipulate ZIP archives.
+ The authors can be contacted at <libzip@nih.at>
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+ 3. The names of the authors may not be used to endorse or promote
+ products derived from this software without specific prior
+ written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
+ OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY
+ DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+ GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#include "zipint.h"
+
+#include <limits.h>
+#include <stdlib.h>
+#include <zstd.h>
+#include <zstd_errors.h>
+
+struct ctx {
+ zip_error_t *error;
+ bool compress;
+ int compression_flags;
+ bool end_of_input;
+ ZSTD_DStream *zdstream;
+ ZSTD_CStream *zcstream;
+ ZSTD_outBuffer out;
+ ZSTD_inBuffer in;
+};
+
+
+static void *
+allocate(bool compress, int compression_flags, zip_error_t *error) {
+ struct ctx *ctx;
+
+ /* 0: let zstd choose */
+ if (compression_flags < 0 || compression_flags > 9) {
+ compression_flags = 0;
+ }
+
+ if ((ctx = (struct ctx *)malloc(sizeof(*ctx))) == NULL) {
+ return NULL;
+ }
+
+ ctx->error = error;
+ ctx->compress = compress;
+ ctx->compression_flags = compression_flags;
+ ctx->end_of_input = false;
+
+ ctx->zdstream = NULL;
+ ctx->zcstream = NULL;
+ ctx->in.src = NULL;
+ ctx->in.pos = 0;
+ ctx->in.size = 0;
+ ctx->out.dst = NULL;
+ ctx->out.pos = 0;
+ ctx->out.size = 0;
+
+ return ctx;
+}
+
+
+static void *
+compress_allocate(zip_uint16_t method, int compression_flags, zip_error_t *error) {
+ return allocate(true, compression_flags, error);
+}
+
+
+static void *
+decompress_allocate(zip_uint16_t method, int compression_flags, zip_error_t *error) {
+ return allocate(false, compression_flags, error);
+}
+
+
+static void
+deallocate(void *ud) {
+ struct ctx *ctx = (struct ctx *)ud;
+ free(ctx);
+}
+
+
+static zip_uint16_t
+general_purpose_bit_flags(void *ud) {
+ /* struct ctx *ctx = (struct ctx *)ud; */
+ return 0;
+}
+
+static int
+map_error(size_t ret) {
+ switch (ret) {
+ case ZSTD_error_no_error:
+ return ZIP_ER_OK;
+
+ case ZSTD_error_corruption_detected:
+ case ZSTD_error_checksum_wrong:
+ case ZSTD_error_dictionary_corrupted:
+ case ZSTD_error_dictionary_wrong:
+ return ZIP_ER_COMPRESSED_DATA;
+
+ case ZSTD_error_memory_allocation:
+ return ZIP_ER_MEMORY;
+
+ case ZSTD_error_parameter_unsupported:
+ case ZSTD_error_parameter_outOfBound:
+ return ZIP_ER_INVAL;
+
+ default:
+ return ZIP_ER_INTERNAL;
+ }
+}
+
+
+static bool
+start(void *ud) {
+ struct ctx *ctx = (struct ctx *)ud;
+ ctx->in.src = NULL;
+ ctx->in.pos = 0;
+ ctx->in.size = 0;
+ ctx->out.dst = NULL;
+ ctx->out.pos = 0;
+ ctx->out.size = 0;
+ if (ctx->compress) {
+ size_t ret;
+ ctx->zcstream = ZSTD_createCStream();
+ if (ctx->zcstream == NULL) {
+ zip_error_set(ctx->error, ZIP_ER_MEMORY, 0);
+ return false;
+ }
+ ret = ZSTD_initCStream(ctx->zcstream, ctx->compression_flags);
+ if (ZSTD_isError(ret)) {
+ zip_error_set(ctx->error, ZIP_ER_ZLIB, map_error(ret));
+ return false;
+ }
+ }
+ else {
+ ctx->zdstream = ZSTD_createDStream();
+ if (ctx->zdstream == NULL) {
+ zip_error_set(ctx->error, ZIP_ER_MEMORY, 0);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+
+static bool
+end(void *ud) {
+ struct ctx *ctx = (struct ctx *)ud;
+ size_t ret;
+
+ if (ctx->compress) {
+ ret = ZSTD_freeCStream(ctx->zcstream);
+ ctx->zcstream = NULL;
+ }
+ else {
+ ret = ZSTD_freeDStream(ctx->zdstream);
+ ctx->zdstream = NULL;
+ }
+
+ if (ZSTD_isError(ret)) {
+ zip_error_set(ctx->error, map_error(ret), 0);
+ return false;
+ }
+
+ return true;
+}
+
+
+static bool
+input(void *ud, zip_uint8_t *data, zip_uint64_t length) {
+ struct ctx *ctx = (struct ctx *)ud;
+ if (length > SIZE_MAX || ctx->in.pos != ctx->in.size) {
+ zip_error_set(ctx->error, ZIP_ER_INVAL, 0);
+ return false;
+ }
+ ctx->in.src = (const void *)data;
+ ctx->in.size = (size_t)length;
+ ctx->in.pos = 0;
+ return true;
+}
+
+
+static void
+end_of_input(void *ud) {
+ struct ctx *ctx = (struct ctx *)ud;
+
+ ctx->end_of_input = true;
+}
+
+
+static zip_compression_status_t
+process(void *ud, zip_uint8_t *data, zip_uint64_t *length) {
+ struct ctx *ctx = (struct ctx *)ud;
+
+ size_t ret;
+
+ if (ctx->in.pos == ctx->in.size && !ctx->end_of_input) {
+ *length = 0;
+ return ZIP_COMPRESSION_NEED_DATA;
+ }
+
+ ctx->out.dst = data;
+ ctx->out.pos = 0;
+ ctx->out.size = ZIP_MIN(SIZE_MAX, *length);
+
+ if (ctx->compress) {
+ if (ctx->in.pos == ctx->in.size && ctx->end_of_input) {
+ ret = ZSTD_endStream(ctx->zcstream, &ctx->out);
+ if (ret == 0) {
+ *length = ctx->out.pos;
+ return ZIP_COMPRESSION_END;
+ }
+ }
+ else {
+ ret = ZSTD_compressStream(ctx->zcstream, &ctx->out, &ctx->in);
+ }
+ }
+ else {
+ ret = ZSTD_decompressStream(ctx->zdstream, &ctx->out, &ctx->in);
+ }
+ if (ZSTD_isError(ret)) {
+ zip_error_set(ctx->error, map_error(ret), 0);
+ return ZIP_COMPRESSION_ERROR;
+ }
+
+ *length = ctx->out.pos;
+ if (ctx->in.pos == ctx->in.size) {
+ return ZIP_COMPRESSION_NEED_DATA;
+ }
+
+ return ZIP_COMPRESSION_OK;
+}
+
+/* clang-format off */
+
+/* Version Required should be set to 63 (6.3) because this compression
+ method was only defined in appnote.txt version 6.3.7, but Winzip
+ does not unpack it if the value is not 20. */
+
+zip_compression_algorithm_t zip_algorithm_zstd_compress = {
+ compress_allocate,
+ deallocate,
+ general_purpose_bit_flags,
+ 20,
+ start,
+ end,
+ input,
+ end_of_input,
+ process
+};
+
+
+zip_compression_algorithm_t zip_algorithm_zstd_decompress = {
+ decompress_allocate,
+ deallocate,
+ general_purpose_bit_flags,
+ 20,
+ start,
+ end,
+ input,
+ end_of_input,
+ process
+};
+
+/* clang-format on */
diff --git a/lib/zip_close.c b/lib/zip_close.c
index 816fcef..25ff4d5 100644
--- a/lib/zip_close.c
+++ b/lib/zip_close.c
@@ -359,8 +359,10 @@
flags |= ZIP_FL_FORCE_ZIP64;
}
}
- else
+ else {
de->comp_size = st.comp_size;
+ data_length = (zip_int64_t)st.comp_size;
+ }
}
if ((offstart = zip_source_tell_write(za->src)) < 0) {
diff --git a/lib/zip_source_compress.c b/lib/zip_source_compress.c
index e49929f..1c4c8bd 100644
--- a/lib/zip_source_compress.c
+++ b/lib/zip_source_compress.c
@@ -75,6 +75,9 @@
*/
{ZIP_CM_XZ, &zip_algorithm_xz_compress, &zip_algorithm_xz_decompress},
#endif
+#if defined(HAVE_LIBZSTD)
+ {ZIP_CM_ZSTD, &zip_algorithm_zstd_compress, &zip_algorithm_zstd_decompress},
+#endif
};
diff --git a/lib/zipint.h b/lib/zipint.h
index 99c1806..139eb98 100644
--- a/lib/zipint.h
+++ b/lib/zipint.h
@@ -153,6 +153,8 @@
extern zip_compression_algorithm_t zip_algorithm_deflate_decompress;
extern zip_compression_algorithm_t zip_algorithm_xz_compress;
extern zip_compression_algorithm_t zip_algorithm_xz_decompress;
+extern zip_compression_algorithm_t zip_algorithm_zstd_compress;
+extern zip_compression_algorithm_t zip_algorithm_zstd_decompress;
/* This API is not final yet, but we need it internally, so it's private for now. */
diff --git a/regress/CMakeLists.txt b/regress/CMakeLists.txt
index e7ec59d..00b30b8 100644
--- a/regress/CMakeLists.txt
+++ b/regress/CMakeLists.txt
@@ -193,8 +193,10 @@
set_compression_store_to_deflate.test
set_compression_store_to_store.test
set_compression_store_to_xz.test
+ set_compression_store_to_zstd.test
set_compression_unknown.test
set_compression_xz_to_store.test
+ set_compression_zstd_to_store.test
set_file_dostime.test
stat_index_cp437_guess.test
stat_index_cp437_raw.test
diff --git a/regress/set_compression_store_to_zstd.test b/regress/set_compression_store_to_zstd.test
new file mode 100644
index 0000000..898a3d8
--- /dev/null
+++ b/regress/set_compression_store_to_zstd.test
@@ -0,0 +1,5 @@
+# change method from stored to zstd-compressed
+features LIBZSTD
+return 0
+args test.zip set_file_compression 0 zstd 0
+file test.zip testfile-stored-dos.zip testfile-zstd.zip
diff --git a/regress/set_compression_zstd_to_store.test b/regress/set_compression_zstd_to_store.test
new file mode 100644
index 0000000..6f8cd7e
--- /dev/null
+++ b/regress/set_compression_zstd_to_store.test
@@ -0,0 +1,5 @@
+# change method from zstd-compressed to stored
+features LIBZSTD
+return 0
+args test.zip set_file_compression 0 store 0
+file test.zip testfile-zstd.zip testfile-stored-dos.zip
diff --git a/regress/testfile-zstd.zip b/regress/testfile-zstd.zip
new file mode 100755
index 0000000..bf42d3e
--- /dev/null
+++ b/regress/testfile-zstd.zip
Binary files differ
diff --git a/src/ziptool.c b/src/ziptool.c
index cdcecc7..125c1a7 100644
--- a/src/ziptool.c
+++ b/src/ziptool.c
@@ -653,6 +653,11 @@
return ZIP_CM_XZ;
#endif
+#if defined(HAVE_LIBZSTD)
+ else if (strcasecmp(arg, "zstd") == 0)
+ return ZIP_CM_ZSTD;
+
+#endif
else if (strcasecmp(arg, "unknown") == 0)
return 100;
return 0; /* TODO: error handling */
@@ -836,6 +841,9 @@
if (zip_compression_method_supported(ZIP_CM_XZ, 1)) {
fprintf(out, "\txz\n");
}
+ if (zip_compression_method_supported(ZIP_CM_ZSTD, 1)) {
+ fprintf(out, "\tzstd\n");
+ }
fprintf(out, "\nSupported encryption methods are:\n"
"\tnone\n");
if (zip_encryption_method_supported(ZIP_EM_AES_128, 1)) {
