Merge pull request #18 from SpringMT/update-zstd-for-v1.3.8

Update zstd to v1.3.8

Author: SpringMT

README.md

@@ -10,7 +10,7 @@ See https://github.com/facebook/zstd
 Fork from https://github.com/jarredholman/ruby-zstd.
 
 ## Zstd version
-v1.3.7 (https://github.com/facebook/zstd/tree/v1.3.7)
+v1.3.8 (https://github.com/facebook/zstd/tree/v1.3.8)
 
 ## Installation
 

ext/zstdruby/libzstd/BUCK

@@ -1,6 +1,7 @@
 cxx_library(
     name='zstd',
     header_namespace='',
+    exported_headers=['zstd.h'],
     visibility=['PUBLIC'],
     deps=[
         ':common',
@@ -17,7 +18,7 @@ cxx_library(
     exported_headers=subdir_glob([
         ('compress', 'zstd*.h'),
     ]),
-    srcs=glob(['compress/zstd*.c']),
+    srcs=glob(['compress/zstd*.c', 'compress/hist.c']),
     deps=[':common'],
 )
 
@@ -40,7 +41,7 @@ cxx_library(
     header_namespace='',
     visibility=['PUBLIC'],
     exported_headers=subdir_glob([
-        ('decprecated', '*.h'),
+        ('deprecated', '*.h'),
     ]),
     srcs=glob(['deprecated/*.c']),
     deps=[':common'],
@@ -118,6 +119,7 @@ cxx_library(
         'decompress/huf_decompress.c',
     ],
     deps=[
+        ':debug',
         ':bitstream',
         ':compiler',
         ':errors',
@@ -204,9 +206,20 @@ cxx_library(
     ],
 )
 
+cxx_library(
+    name='debug',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'debug.h'),
+    ]),
+    srcs=['common/debug.c'],
+)
+
 cxx_library(
     name='common',
     deps=[
+        ':debug',
         ':bitstream',
         ':compiler',
         ':cpu',

ext/zstdruby/libzstd/Makefile

@@ -27,11 +27,16 @@ DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
             -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security \
             -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
-            -Wredundant-decls -Wmissing-prototypes
+            -Wredundant-decls -Wmissing-prototypes -Wc++-compat
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
-GREP = grep --color=never
+HAVE_COLORNEVER = $(shell echo a | grep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
+GREP_OPTIONS ?=
+ifeq ($HAVE_COLORNEVER, 1)
+GREP_OPTIONS += --color=never
+endif
+GREP = grep $(GREP_OPTIONS)
 
 ZSTDCOMMON_FILES := $(sort $(wildcard common/*.c))
 ZSTDCOMP_FILES := $(sort $(wildcard compress/*.c))
@@ -45,6 +50,12 @@ ZSTD_LIB_COMPRESSION ?= 1
 ZSTD_LIB_DECOMPRESSION ?= 1
 ZSTD_LIB_DICTBUILDER ?= 1
 ZSTD_LIB_DEPRECATED ?= 1
+HUF_FORCE_DECOMPRESS_X1 ?= 0
+HUF_FORCE_DECOMPRESS_X2 ?= 0
+ZSTD_FORCE_DECOMPRESS_SHORT ?= 0
+ZSTD_FORCE_DECOMPRESS_LONG ?= 0
+ZSTD_NO_INLINE ?= 0
+ZSTD_STRIP_ERROR_STRINGS ?= 0
 
 ifeq ($(ZSTD_LIB_COMPRESSION), 0)
 	ZSTD_LIB_DICTBUILDER = 0
@@ -72,6 +83,30 @@ ifneq ($(ZSTD_LIB_DICTBUILDER), 0)
 	ZSTD_FILES += $(ZDICT_FILES)
 endif
 
+ifneq ($(HUF_FORCE_DECOMPRESS_X1), 0)
+	CFLAGS += -DHUF_FORCE_DECOMPRESS_X1
+endif
+
+ifneq ($(HUF_FORCE_DECOMPRESS_X2), 0)
+	CFLAGS += -DHUF_FORCE_DECOMPRESS_X2
+endif
+
+ifneq ($(ZSTD_FORCE_DECOMPRESS_SHORT), 0)
+	CFLAGS += -DZSTD_FORCE_DECOMPRESS_SHORT
+endif
+
+ifneq ($(ZSTD_FORCE_DECOMPRESS_LONG), 0)
+	CFLAGS += -DZSTD_FORCE_DECOMPRESS_LONG
+endif
+
+ifneq ($(ZSTD_NO_INLINE), 0)
+	CFLAGS += -DZSTD_NO_INLINE
+endif
+
+ifneq ($(ZSTD_STRIP_ERROR_STRINGS), 0)
+	CFLAGS += -DZSTD_STRIP_ERROR_STRINGS
+endif
+
 ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
 ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
 	ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')

ext/zstdruby/libzstd/README.md

@@ -7,13 +7,32 @@ in order to make it easier to select or exclude features.
 
 #### Building
 
-`Makefile` script is provided, supporting all standard [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
+`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
 including commands variables, staged install, directory variables and standard targets.
 - `make` : generates both static and dynamic libraries
-- `make install` : install libraries in default system directories
+- `make install` : install libraries and headers in target system directories
 
-`libzstd` default scope includes compression, decompression, dictionary building,
-and decoding support for legacy formats >= v0.5.0.
+`libzstd` default scope is pretty large, including compression, decompression, dictionary builder,
+and support for decoding legacy formats >= v0.5.0.
+The scope can be reduced on demand (see paragraph _modular build_).
+
+
+#### Multithreading support
+
+Multithreading is disabled by default when building with `make`.
+Enabling multithreading requires 2 conditions :
+- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`)
+- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
+
+Both conditions are automatically applied when invoking `make lib-mt` target.
+
+When linking a POSIX program with a multithreaded version of `libzstd`,
+note that it's necessary to request the `-pthread` flag during link stage.
+
+Multithreading capabilities are exposed
+via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.3.8/lib/zstd.h#L592).
+This API is still labelled experimental,
+but is expected to become "stable" in the near future.
 
 
 #### API
@@ -26,63 +45,70 @@ Zstandard's stable API is exposed within [lib/zstd.h](zstd.h).
 Optional advanced features are exposed via :
 
 - `lib/common/zstd_errors.h` : translates `size_t` function results
-                              into an `ZSTD_ErrorCode`, for accurate error handling.
+                               into a `ZSTD_ErrorCode`, for accurate error handling.
+
 - `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
-                          it unlocks access to advanced experimental API,
-                          exposed in second part of `zstd.h`.
-                          These APIs are not "stable", their definition may change in the future.
-                          As a consequence, it shall ___never be used with dynamic library___ !
+                          it unlocks access to the experimental API,
+                          exposed in the second part of `zstd.h`.
+                          All definitions in the experimental APIs are unstable,
+                          they may still change in the future, or even be removed.
+                          As a consequence, experimental definitions shall ___never be used with dynamic library___ !
                           Only static linking is allowed.
 
 
 #### Modular build
 
-It's possible to compile only a limited set of features.
+It's possible to compile only a limited set of features within `libzstd`.
+The file structure is designed to make this selection manually achievable for any build system :
 
 - Directory `lib/common` is always required, for all variants.
+
 - Compression source code lies in `lib/compress`
+
 - Decompression source code lies in `lib/decompress`
+
 - It's possible to include only `compress` or only `decompress`, they don't depend on each other.
+
 - `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples.
         The API is exposed in `lib/dictBuilder/zdict.h`.
         This module depends on both `lib/common` and `lib/compress` .
-- `lib/legacy` : source code to decompress legacy zstd formats, starting from `v0.1.0`.
+
+- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`.
         This module depends on `lib/common` and `lib/decompress`.
         To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation.
         Specifying a number limits versions supported to that version onward.
         For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
-        `ZSTD_LEGACY_SUPPORT=3` means : "support legacy formats >= v0.3.0", and so on.
-        Currently, the default library setting is `ZST_LEGACY_SUPPORT=5`.
-        It can be changed at build by any other value.
-        Note that any number >= 8 translates into "do __not__ support legacy formats",
-        since all versions of `zstd` >= v0.8 are compatible with v1+ specification.
-        `ZSTD_LEGACY_SUPPORT=0` also means "do __not__ support legacy formats".
-        Once enabled, this capability is transparently triggered within decompression functions.
-        It's also possible to invoke directly legacy API, as exposed in `lib/legacy/zstd_legacy.h`.
-        Each version also provides an additional dedicated set of advanced API.
+        Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
+        By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`.
+        Decoding supported legacy format is a transparent capability triggered within decompression functions.
+        It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`.
+        Each version does also provide its own set of advanced API.
         For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
-        Note : `lib/legacy` only supports _decoding_ legacy formats.
-- Similarly, you can define `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
-        and `ZSTD_LIB_DEPRECATED` as 0 to forgo compilation of the corresponding features. This will
-        also disable compilation of all dependencies (eg. `ZSTD_LIB_COMPRESSION=0` will also disable
-        dictBuilder).
 
+- While invoking `make libzstd`, it's possible to define build macros
+        `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
+        and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the corresponding features.
+        This will also disable compilation of all dependencies
+        (eg. `ZSTD_LIB_COMPRESSION=0` will also disable dictBuilder).
 
-#### Multithreading support
+- There are some additional build macros that can be used to minify the decoder.
 
-Multithreading is disabled by default when building with `make`.
-Enabling multithreading requires 2 conditions :
-- set macro `ZSTD_MULTITHREAD`
-- on POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
-
-Both conditions are automatically triggered by invoking `make lib-mt` target.
-Note that, when linking a POSIX program with a multithreaded version of `libzstd`,
-it's necessary to trigger `-pthread` flag during link stage.
+  Zstandard often has more than one implementation of a piece of functionality,
+  where each implementation optimizes for different scenarios. For example, the
+  Huffman decoder has complementary implementations that decode the stream one
+  symbol at a time or two symbols at a time. Zstd normally includes both (and
+  dispatches between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1`
+  or `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding
+  compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`
+  and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of
+  only one or the other of two decompression implementations. The smallest
+  binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and
+  `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`.
 
-Multithreading capabilities are exposed
-via [advanced API `ZSTD_compress_generic()` defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/dev/lib/zstd.h#L919).
-This API is still considered experimental,
-but is expected to become "stable" at some point in the future.
+  For squeezing the last ounce of size out, you can also define
+  `ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`,
+  which removes the error messages that are otherwise returned by
+  `ZSTD_getErrorName`.
 
 
 #### Windows : using MinGW+MSYS to create DLL
@@ -113,8 +139,8 @@ Consider migrating code towards supported streaming API exposed in `zstd.h`.
 
 The other files are not source code. There are :
 
- - `LICENSE` : contains the BSD license text
- - `Makefile` : `make` script to build and install zstd library (static and dynamic)
  - `BUCK` : support for `buck` build system (https://buckbuild.com/)
- - `libzstd.pc.in` : for `pkg-config` (used in `make install`)
+ - `Makefile` : `make` script to build and install zstd library (static and dynamic)
  - `README.md` : this file
+ - `dll/` : resources directory for Windows compilation
+ - `libzstd.pc.in` : script for `pkg-config` (used in `make install`)

ext/zstdruby/libzstd/common/bitstream.h

@@ -389,7 +389,7 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
  *  Read (consume) next n bits from local register and update.
  *  Pay attention to not read more than nbBits contained into local register.
  * @return : extracted value. */
-MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
+MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
 {
     size_t const value = BIT_lookBits(bitD, nbBits);
     BIT_skipBits(bitD, nbBits);
@@ -398,7 +398,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
 
 /*! BIT_readBitsFast() :
  *  unsafe version; only works only if nbBits >= 1 */
-MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
+MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
 {
     size_t const value = BIT_lookBitsFast(bitD, nbBits);
     assert(nbBits >= 1);

ext/zstdruby/libzstd/common/compiler.h

@@ -15,6 +15,8 @@
 *  Compiler specifics
 *********************************************************/
 /* force inlining */
+
+#if !defined(ZSTD_NO_INLINE)
 #if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
 #  define INLINE_KEYWORD inline
 #else
@@ -29,6 +31,13 @@
 #  define FORCE_INLINE_ATTR
 #endif
 
+#else
+
+#define INLINE_KEYWORD
+#define FORCE_INLINE_ATTR
+
+#endif
+
 /**
  * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
  * parameters. They must be inlined for the compiler to elimininate the constant
@@ -89,23 +98,21 @@
 #endif
 
 /* prefetch
- * can be disabled, by declaring NO_PREFETCH macro
- * All prefetch invocations use a single default locality 2,
- * generating instruction prefetcht1,
- * which, according to Intel, means "load data into L2 cache".
- * This is a good enough "middle ground" for the time being,
- * though in theory, it would be better to specialize locality depending on data being prefetched.
- * Tests could not determine any sensible difference based on locality value. */
+ * can be disabled, by declaring NO_PREFETCH build macro */
 #if defined(NO_PREFETCH)
-#  define PREFETCH(ptr)     (void)(ptr)  /* disabled */
+#  define PREFETCH_L1(ptr)  (void)(ptr)  /* disabled */
+#  define PREFETCH_L2(ptr)  (void)(ptr)  /* disabled */
 #else
 #  if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))  /* _mm_prefetch() is not defined outside of x86/x64 */
 #    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
-#    define PREFETCH(ptr)   _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
+#    define PREFETCH_L1(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+#    define PREFETCH_L2(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
 #  elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
-#    define PREFETCH(ptr)   __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
+#    define PREFETCH_L1(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
+#    define PREFETCH_L2(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
 #  else
-#    define PREFETCH(ptr)   (void)(ptr)  /* disabled */
+#    define PREFETCH_L1(ptr) (void)(ptr)  /* disabled */
+#    define PREFETCH_L2(ptr) (void)(ptr)  /* disabled */
 #  endif
 #endif  /* NO_PREFETCH */
 
@@ -116,7 +123,7 @@
     size_t const _size = (size_t)(s);     \
     size_t _pos;                          \
     for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) {  \
-        PREFETCH(_ptr + _pos);            \
+        PREFETCH_L2(_ptr + _pos);         \
     }                                     \
 }
 

ext/zstdruby/libzstd/common/cpu.h

@@ -78,7 +78,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
       __asm__(
           "pushl %%ebx\n\t"
           "cpuid\n\t"
-          "movl %%ebx, %%eax\n\r"
+          "movl %%ebx, %%eax\n\t"
           "popl %%ebx"
           : "=a"(f7b), "=c"(f7c)
           : "a"(7), "c"(0)