Update zstd to v1.3.4

Author: haruyama-makotpo

ext/zstdruby/libzstd/BUCK

@@ -25,6 +25,9 @@ cxx_library(
     name='decompress',
     header_namespace='',
     visibility=['PUBLIC'],
+    headers=subdir_glob([
+        ('decompress', '*_impl.h'),
+    ]),
     srcs=glob(['decompress/zstd*.c']),
     deps=[
         ':common',
@@ -80,6 +83,15 @@ cxx_library(
     ]),
 )
 
+cxx_library(
+    name='cpu',
+    header_namespace='',
+    visibility=['PUBLIC'],
+    exported_headers=subdir_glob([
+        ('common', 'cpu.h'),
+    ]),
+)
+
 cxx_library(
     name='bitstream',
     header_namespace='',
@@ -196,6 +208,7 @@ cxx_library(
     deps=[
         ':bitstream',
         ':compiler',
+        ':cpu',
         ':entropy',
         ':errors',
         ':mem',

ext/zstdruby/libzstd/README.md

@@ -2,16 +2,19 @@ Zstandard library files
 ================================
 
 The __lib__ directory is split into several sub-directories,
-in order to make it easier to select or exclude specific features.
+in order to make it easier to select or exclude features.
 
 
 #### Building
 
-`Makefile` script is provided, supporting the standard set of commands,
-directories, and variables (see https://www.gnu.org/prep/standards/html_node/Command-Variables.html).
+`Makefile` script is provided, supporting all standard [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
+including commands variables, staged install, directory variables and standard targets.
 - `make` : generates both static and dynamic libraries
 - `make install` : install libraries in default system directories
 
+`libzstd` default scope includes compression, decompression, dictionary building,
+and decoding support for legacy formats >= v0.4.0.
+
 
 #### API
 
@@ -27,49 +30,54 @@ Optional advanced features are exposed via :
 - `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
                           it unlocks access to advanced experimental API,
                           exposed in second part of `zstd.h`.
-                          These APIs shall ___never be used with dynamic library___ !
-                          They are not "stable", their definition may change in the future.
+                          These APIs are not "stable", their definition may change in the future.
+                          As a consequence, it shall ___never be used with dynamic library___ !
                           Only static linking is allowed.
 
 
 #### Modular build
 
+It's possible to compile only a limited set of features.
+
 - Directory `lib/common` is always required, for all variants.
 - Compression source code lies in `lib/compress`
 - Decompression source code lies in `lib/decompress`
 - It's possible to include only `compress` or only `decompress`, they don't depend on each other.
 - `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples.
-    The API is exposed in `lib/dictBuilder/zdict.h`.
-    This module depends on both `lib/common` and `lib/compress` .
-- `lib/legacy` : source code to decompress older zstd formats, starting from `v0.1`.
-              This module depends on `lib/common` and `lib/decompress`.
-              To enable this feature, it's necessary to define `ZSTD_LEGACY_SUPPORT = 1` during compilation.
-              Typically, with `gcc`, add argument `-DZSTD_LEGACY_SUPPORT=1`.
-              Using higher number limits the number of version supported.
-              For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats starting from v0.2+".
-              The API is exposed in `lib/legacy/zstd_legacy.h`.
-              Each version also provides a (dedicated) set of advanced API.
-              For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
+        The API is exposed in `lib/dictBuilder/zdict.h`.
+        This module depends on both `lib/common` and `lib/compress` .
+- `lib/legacy` : source code to decompress legacy zstd formats, starting from `v0.1.0`.
+        This module depends on `lib/common` and `lib/decompress`.
+        To enable this feature, it's required to define `ZSTD_LEGACY_SUPPORT` during compilation.
+        Typically, with `gcc`, add argument `-DZSTD_LEGACY_SUPPORT=1`.
+        Using higher number limits versions supported.
+        For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
+        `ZSTD_LEGACY_SUPPORT=3` means : "support legacy formats >= v0.3.0", and so on.
+        Starting v0.8.0, all versions of `zstd` produce frames compliant with specification.
+        As a consequence, `ZSTD_LEGACY_SUPPORT=8` (or more) doesn't trigger legacy support.
+        Also, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
+        Once enabled, this capability is transparently triggered within decompression functions.
+        It's also possible to invoke directly legacy API, as exposed in `lib/legacy/zstd_legacy.h`.
+        Each version also provides an additional dedicated set of advanced API.
+        For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
+        Note : `lib/legacy` only supports _decoding_ legacy formats.
 
 
 #### Multithreading support
 
 Multithreading is disabled by default when building with `make`.
 Enabling multithreading requires 2 conditions :
 - set macro `ZSTD_MULTITHREAD`
-- on POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc` for example)
+- on POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
 
 Both conditions are automatically triggered by invoking `make lib-mt` target.
 Note that, when linking a POSIX program with a multithreaded version of `libzstd`,
 it's necessary to trigger `-pthread` flag during link stage.
 
-Multithreading capabilities are exposed via :
-- private API `lib/compress/zstdmt_compress.h`.
-  Symbols defined in this header are currently exposed in `libzstd`, hence usable.
-  Note however that this API is planned to be locked and remain strictly internal in the future.
-- advanced API `ZSTD_compress_generic()`, defined in `lib/zstd.h`, experimental section.
-  This API is still considered experimental, but is designed to be labelled "stable" at some point in the future.
-  It's the recommended entry point for multi-threading operations.
+Multithreading capabilities are exposed
+via [advanced API `ZSTD_compress_generic()` defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/dev/lib/zstd.h#L919).
+This API is still considered experimental,
+but is expected to become "stable" at some point in the future.
 
 
 #### Windows : using MinGW+MSYS to create DLL
@@ -92,7 +100,6 @@ The compiled executable will require ZSTD DLL which is available at `dll\libzstd
 
 Obsolete API on their way out are stored in directory `lib/deprecated`.
 At this stage, it contains older streaming prototypes, in `lib/deprecated/zbuff.h`.
-Presence in this directory is temporary.
 These prototypes will be removed in some future version.
 Consider migrating code towards supported streaming API exposed in `zstd.h`.
 

ext/zstdruby/libzstd/common/bitstream.h

@@ -426,7 +426,7 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
  *  Refill `bitD` from buffer previously set in BIT_initDStream() .
  *  This function is safe, it guarantees it will not read beyond src buffer.
  * @return : status of `BIT_DStream_t` internal register.
- *           when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
+ *           when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
 MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
 {
     if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* overflow detected, like end of stream */

ext/zstdruby/libzstd/common/compiler.h

@@ -63,6 +63,31 @@
 #  endif
 #endif
 
+/* target attribute */
+#ifndef __has_attribute
+  #define __has_attribute(x) 0  /* Compatibility with non-clang compilers. */
+#endif
+#if defined(__GNUC__)
+#  define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
+#else
+#  define TARGET_ATTRIBUTE(target)
+#endif
+
+/* Enable runtime BMI2 dispatch based on the CPU.
+ * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default.
+ */
+#ifndef DYNAMIC_BMI2
+  #if (defined(__clang__) && __has_attribute(__target__)) \
+      || (defined(__GNUC__) \
+          && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) \
+      && (defined(__x86_64__) || defined(_M_X86)) \
+      && !defined(__BMI2__)
+  #  define DYNAMIC_BMI2 1
+  #else
+  #  define DYNAMIC_BMI2 0
+  #endif
+#endif
+
 /* prefetch */
 #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))  /* _mm_prefetch() is not defined outside of x86/x64 */
 #  include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */

ext/zstdruby/libzstd/common/cpu.h

@@ -0,0 +1,216 @@
+/*
+ * Copyright (c) 2018-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_COMMON_CPU_H
+#define ZSTD_COMMON_CPU_H
+
+/**
+ * Implementation taken from folly/CpuId.h
+ * https://github.com/facebook/folly/blob/master/folly/CpuId.h
+ */
+
+#include <string.h>
+
+#include "mem.h"
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+
+typedef struct {
+    U32 f1c;
+    U32 f1d;
+    U32 f7b;
+    U32 f7c;
+} ZSTD_cpuid_t;
+
+MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
+    U32 f1c = 0;
+    U32 f1d = 0;
+    U32 f7b = 0;
+    U32 f7c = 0;
+#ifdef _MSC_VER
+    int reg[4];
+    __cpuid((int*)reg, 0);
+    {
+        int const n = reg[0];
+        if (n >= 1) {
+            __cpuid((int*)reg, 1);
+            f1c = (U32)reg[2];
+            f1d = (U32)reg[3];
+        }
+        if (n >= 7) {
+            __cpuidex((int*)reg, 7, 0);
+            f7b = (U32)reg[1];
+            f7c = (U32)reg[2];
+        }
+    }
+#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__)
+    /* The following block like the normal cpuid branch below, but gcc
+     * reserves ebx for use of its pic register so we must specially
+     * handle the save and restore to avoid clobbering the register
+     */
+    U32 n;
+    __asm__(
+        "pushl %%ebx\n\t"
+        "cpuid\n\t"
+        "popl %%ebx\n\t"
+        : "=a"(n)
+        : "a"(0)
+        : "ecx", "edx");
+    if (n >= 1) {
+      U32 f1a;
+      __asm__(
+          "pushl %%ebx\n\t"
+          "cpuid\n\t"
+          "popl %%ebx\n\t"
+          : "=a"(f1a), "=c"(f1c), "=d"(f1d)
+          : "a"(1)
+          :);
+    }
+    if (n >= 7) {
+      __asm__(
+          "pushl %%ebx\n\t"
+          "cpuid\n\t"
+          "movl %%ebx, %%eax\n\r"
+          "popl %%ebx"
+          : "=a"(f7b), "=c"(f7c)
+          : "a"(7), "c"(0)
+          : "edx");
+    }
+#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
+    U32 n;
+    __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx");
+    if (n >= 1) {
+      U32 f1a;
+      __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx");
+    }
+    if (n >= 7) {
+      U32 f7a;
+      __asm__("cpuid"
+              : "=a"(f7a), "=b"(f7b), "=c"(f7c)
+              : "a"(7), "c"(0)
+              : "edx");
+    }
+#endif
+    {
+        ZSTD_cpuid_t cpuid;
+        cpuid.f1c = f1c;
+        cpuid.f1d = f1d;
+        cpuid.f7b = f7b;
+        cpuid.f7c = f7c;
+        return cpuid;
+    }
+}
+
+#define X(name, r, bit)                                                        \
+  MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) {                 \
+    return ((cpuid.r) & (1U << bit)) != 0;                                     \
+  }
+
+/* cpuid(1): Processor Info and Feature Bits. */
+#define C(name, bit) X(name, f1c, bit)
+  C(sse3, 0)
+  C(pclmuldq, 1)
+  C(dtes64, 2)
+  C(monitor, 3)
+  C(dscpl, 4)
+  C(vmx, 5)
+  C(smx, 6)
+  C(eist, 7)
+  C(tm2, 8)
+  C(ssse3, 9)
+  C(cnxtid, 10)
+  C(fma, 12)
+  C(cx16, 13)
+  C(xtpr, 14)
+  C(pdcm, 15)
+  C(pcid, 17)
+  C(dca, 18)
+  C(sse41, 19)
+  C(sse42, 20)
+  C(x2apic, 21)
+  C(movbe, 22)
+  C(popcnt, 23)
+  C(tscdeadline, 24)
+  C(aes, 25)
+  C(xsave, 26)
+  C(osxsave, 27)
+  C(avx, 28)
+  C(f16c, 29)
+  C(rdrand, 30)
+#undef C
+#define D(name, bit) X(name, f1d, bit)
+  D(fpu, 0)
+  D(vme, 1)
+  D(de, 2)
+  D(pse, 3)
+  D(tsc, 4)
+  D(msr, 5)
+  D(pae, 6)
+  D(mce, 7)
+  D(cx8, 8)
+  D(apic, 9)
+  D(sep, 11)
+  D(mtrr, 12)
+  D(pge, 13)
+  D(mca, 14)
+  D(cmov, 15)
+  D(pat, 16)
+  D(pse36, 17)
+  D(psn, 18)
+  D(clfsh, 19)
+  D(ds, 21)
+  D(acpi, 22)
+  D(mmx, 23)
+  D(fxsr, 24)
+  D(sse, 25)
+  D(sse2, 26)
+  D(ss, 27)
+  D(htt, 28)
+  D(tm, 29)
+  D(pbe, 31)
+#undef D
+
+/* cpuid(7): Extended Features. */
+#define B(name, bit) X(name, f7b, bit)
+  B(bmi1, 3)
+  B(hle, 4)
+  B(avx2, 5)
+  B(smep, 7)
+  B(bmi2, 8)
+  B(erms, 9)
+  B(invpcid, 10)
+  B(rtm, 11)
+  B(mpx, 14)
+  B(avx512f, 16)
+  B(avx512dq, 17)
+  B(rdseed, 18)
+  B(adx, 19)
+  B(smap, 20)
+  B(avx512ifma, 21)
+  B(pcommit, 22)
+  B(clflushopt, 23)
+  B(clwb, 24)
+  B(avx512pf, 26)
+  B(avx512er, 27)
+  B(avx512cd, 28)
+  B(sha, 29)
+  B(avx512bw, 30)
+  B(avx512vl, 31)
+#undef B
+#define C(name, bit) X(name, f7c, bit)
+  C(prefetchwt1, 0)
+  C(avx512vbmi, 1)
+#undef C
+
+#undef X
+
+#endif /* ZSTD_COMMON_CPU_H */