Start adding UTF-8 support: first step Add an internal function that guesses the encoding of a string: ASCII, UTF-8, or CP437. Add a test program and 3 test cases, one for each encoding. --HG-- branch : HEAD
diff --git a/.hgignore b/.hgignore index d9a7c49..47d2182 100644 --- a/.hgignore +++ b/.hgignore
@@ -22,11 +22,11 @@ ^regress/add_from_file$ ^regress/add_from_filep$ ^regress/add_from_zip$ +^regress/encoding$ ^regress/encrypt$ ^regress/fread$ ^regress/get_comment$ ^regress/name_locate$ -^regress/open$ ^regress/set_comment_all$ ^regress/set_comment_localonly$ ^regress/set_comment_removeglobal$
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index af09a39..4780b71 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt
@@ -124,6 +124,7 @@ zip_unchange_all.c zip_unchange_archive.c zip_unchange_data.c + zip_utf-8.c ) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}
diff --git a/lib/Makefile.am b/lib/Makefile.am index 9f7be38..31bd17a 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am
@@ -81,7 +81,8 @@ zip_unchange.c \ zip_unchange_all.c \ zip_unchange_archive.c \ - zip_unchange_data.c + zip_unchange_data.c \ + zip_utf-8.c BUILT_SOURCES=zip_err_str.c zipconf.h CLEANFILES= ${BUILT_SOURCES}
diff --git a/lib/zip.h b/lib/zip.h index f8b55ac..5b3e2ce 100644 --- a/lib/zip.h +++ b/lib/zip.h
@@ -162,6 +162,15 @@ +/* encoding type */ +enum zip_encoding_type { + ZIP_ENCODING_ASCII, /* plain ASCII */ + ZIP_ENCODING_UTF8, /* possibly UTF-8 */ + ZIP_ENCODING_CP437 /* Code Page 437 */ +}; + + + enum zip_source_cmd { ZIP_SOURCE_OPEN, /* prepare for reading */ ZIP_SOURCE_READ, /* read data */
diff --git a/lib/zip_utf-8.c b/lib/zip_utf-8.c new file mode 100644 index 0000000..4e19e31 --- /dev/null +++ b/lib/zip_utf-8.c
@@ -0,0 +1,151 @@ +/* + zip_utf-8.c -- UTF-8 support functions for libzip + Copyright (C) 2011 Dieter Baron and Thomas Klausner + + This file is part of libzip, a library to manipulate ZIP archives. + The authors can be contacted at <libzip@nih.at> + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. The names of the authors may not be used to endorse or promote + products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS + OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + + + +#include "zip.h" +#include "zipint.h" + + + +static const zip_uint16_t _cp437_to_utf8[256] = { + /* 0x00 - 0x0F */ 0x2007, 0x263A, 0x263B, 0x2665, 0x2666, 0x2663, 0x2660, 0x2022, + 0x25D8, 0x25CB, 0x25D9, 0x2642, 0x2640, 0x266A, 0x266B, 0x263C, + + /* 0x10 - 0x1F */ 0x25BA, 0x25C4, 0x2195, 0x203C, 0x00B6, 0x00A7, 0x25AC, 0x21A8, + 0x2191, 0x2193, 0x2192, 0x2190, 0x221F, 0x2194, 0x25B2, 0x25BC, + + /* 0x20 - 0x2F */ 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F, + + /* 0x30 - 0x3F */ 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F, + + /* 0x40 - 0x4F */ 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, + + /* 0x50 - 0x5F */ 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, + + /* 0x60 - 0x6F */ 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, + + /* 0x70 - 0x7F */ 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x2302, + + /* 0x80 - 0x8F */ 0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7, + 0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5, + + /* 0x90 - 0x9F */ 0x00C9, 0x00E6, 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9, + 0x00FF, 0x00D6, 0x00DC, 0x00A2, 0x00A3, 0x00A5, 0x20A7, 0x0192, + + /* 0xA0 - 0xAF */ 0x00E1, 0x00ED, 0x00F3, 0x00FA, 0x00F1, 0x00D1, 0x00AA, 0x00BA, + 0x00BF, 0x2310, 0x00AC, 0x00BD, 0x00BC, 0x00A1, 0x00AB, 0x00BB, + + /* 0xB0 - 0xBF */ 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556, + 0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510, + + /* 0xC0 - 0xCF */ 0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F, + 0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567, + + /* 0xD0 - 0xDF */ 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B, + 0x256A, 0x2518, 0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580, + + /* 0xE0 - 0xEF */ 0x03B1, 0x00DF, 0x0393, 0x03C0, 0x03A3, 0x03C3, 0x00B5, 0x03C4, + 0x03A6, 0x0398, 0x03A9, 0x03B4, 0x221E, 0x03C6, 0x03B5, 0x2229, + + /* 0xF0 - 0xFF */ 0x2261, 0x00B1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00F7, 0x2248, + 0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0 +}; + +#define UTF_8_LEN_2_MASK 0xe0 +#define UTF_8_LEN_2_MATCH 0xc0 +#define UTF_8_LEN_3_MASK 0xf0 +#define UTF_8_LEN_3_MATCH 0xe0 +#define UTF_8_LEN_4_MASK 0xf8 +#define UTF_8_LEN_4_MATCH 0xf0 +#define UTF_8_CONTINUE_MASK 0xc0 +#define UTF_8_CONTINUE_MATCH 0x80 + + + +enum zip_encoding_type +_zip_guess_encoding(const zip_uint8_t * const name, zip_uint32_t len) +{ + zip_uint32_t i; + int ret; + int j, ulen; + + ret = ZIP_ENCODING_ASCII; + for (i=0; i<len; i++) { + if (name[i] < 128) + continue; + + ret = ZIP_ENCODING_UTF8; + if ((name[i] & UTF_8_LEN_2_MASK) == UTF_8_LEN_2_MATCH) { + ulen = 1; + } else if ((name[i] & UTF_8_LEN_3_MASK) == UTF_8_LEN_3_MATCH) { + ulen = 2; + } else if ((name[i] & UTF_8_LEN_4_MASK) == UTF_8_LEN_4_MATCH) { + ulen = 3; + } else + return ZIP_ENCODING_CP437; + + if (i + ulen >= len) + return ZIP_ENCODING_CP437; + for (j=1; j<=ulen; j++) { + if ((name[i+j] & UTF_8_CONTINUE_MASK) != UTF_8_CONTINUE_MATCH) + return ZIP_ENCODING_CP437; + } + i += ulen; + } + + return ret; +} + + + +#if 0 + + +zip_uint8_t * +_zip_cp437_to_utf8(const zip_uint8_t * const in, zip_uint32_t len) +{ + zip_uint8_t *out; + + /* maximum of three UTF-8 characters per CP-437 character */ + out = malloc(3*len); + + +} +#endif
diff --git a/lib/zipint.h b/lib/zipint.h index 2f748bd..c651447 100644 --- a/lib/zipint.h +++ b/lib/zipint.h
@@ -333,6 +333,8 @@ int _zip_filerange_crc(FILE *, off_t, off_t, uLong *, struct zip_error *); +enum zip_encoding_type _zip_guess_encoding(const zip_uint8_t * const, zip_uint32_t); + struct zip *_zip_open(const char *, FILE *, int, int, int *); struct zip_source *_zip_source_file_or_p(struct zip *, const char *, FILE *,
diff --git a/regress/CMakeLists.txt b/regress/CMakeLists.txt index c93a0e5..82bc870 100644 --- a/regress/CMakeLists.txt +++ b/regress/CMakeLists.txt
@@ -5,6 +5,7 @@ add_from_file add_from_filep add_from_zip + encoding encrypt fread get_comment
diff --git a/regress/Makefile.am b/regress/Makefile.am index 9cf0cd6..6912213 100644 --- a/regress/Makefile.am +++ b/regress/Makefile.am
@@ -4,6 +4,7 @@ add_from_file \ add_from_filep \ add_from_zip \ + encoding \ encrypt \ fread \ get_comment \ @@ -45,6 +46,9 @@ add_from_file.test \ add_from_filep.test \ add_from_zip.test \ + encoding-ascii.test \ + encoding-cp437.test \ + encoding-utf-8.test \ encrypt.test \ fread.test \ get_comment.test \
diff --git a/regress/encoding-ascii.test b/regress/encoding-ascii.test new file mode 100644 index 0000000..5d086f8 --- /dev/null +++ b/regress/encoding-ascii.test
@@ -0,0 +1,5 @@ +# recognize ASCII +program encoding +return 0 +args ABC%^&&*!@#_.as./- +stdout guessing ABC%^&&*!@#_.as./-: ASCII
diff --git a/regress/encoding-cp437.test b/regress/encoding-cp437.test new file mode 100644 index 0000000..fca9109 --- /dev/null +++ b/regress/encoding-cp437.test
@@ -0,0 +1,5 @@ +# recognize CP437 +program encoding +return 0 +args TestAá^$ +stdout guessing TestAá^$: CP437
diff --git a/regress/encoding-utf-8.test b/regress/encoding-utf-8.test new file mode 100644 index 0000000..74e431b --- /dev/null +++ b/regress/encoding-utf-8.test
@@ -0,0 +1,5 @@ +# recognize UTF-8 +program encoding +return 0 +args TestÄÖAÜß^$ +stdout guessing TestÄÖAÜß^$: UTF-8
diff --git a/regress/encoding.c b/regress/encoding.c new file mode 100644 index 0000000..2bf6d65 --- /dev/null +++ b/regress/encoding.c
@@ -0,0 +1,56 @@ +/* + encoding.c -- tool for encoding tests + Copyright (C) 2011 Dieter Baron and Thomas Klausner + + This file is part of libzip, a library to manipulate ZIP archives. + The authors can be contacted at <libzip@nih.at> + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. The names of the authors may not be used to endorse or promote + products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS + OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "zip.h" +#include "zipint.h" + +const char *result[] = { + "ASCII", + "UTF-8", + "CP437" +}; + +int +main(int argc, char *argv[]) +{ + if (argc != 2) { + fprintf(stderr, "usage: %s string_to_guess\n", argv[0]); + exit(1); + } + printf("guessing %s: %s\n", argv[1], result[_zip_guess_encoding(argv[1], strlen(argv[1]))]); + exit(0); +}