Skip to content

Commit 946b5cb

Browse files
SpringMTSpringMT
authored
Merge pull request #10 from SpringMT/update-zstd-version-to-1_3_2
Update zstd to v1.3.2
2 parents 99ff61a + a9193c9 commit 946b5cb

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+5799
-3733
lines changed

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ See https://github.com/facebook/zstd
99
Fork from https://github.com/jarredholman/ruby-zstd.
1010

1111
## Zstd version
12-
v1.3.1 (https://github.com/facebook/zstd/tree/v1.3.1)
12+
v1.3.2 (https://github.com/facebook/zstd/tree/v1.3.2)
1313

1414
## Installation
1515

‎ext/zstdruby/libzstd/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
# make install artefact
22
libzstd.pc
3+
libzstd-nomt

‎ext/zstdruby/libzstd/Makefile

Lines changed: 40 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
1-
# ##########################################################################
2-
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
1+
# ################################################################
2+
# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
33
# All rights reserved.
44
#
5-
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
6-
#
7-
# This source code is licensed under the BSD-style license found in the
8-
# LICENSE file in the root directory of this source tree. An additional grant
9-
# of patent rights can be found in the PATENTS file in the same directory.
10-
# ##########################################################################
5+
# This source code is licensed under both the BSD-style license (found in the
6+
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
7+
# in the COPYING file in the root directory of this source tree).
8+
# ################################################################
119

1210
# Version numbers
1311
LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
@@ -31,7 +29,7 @@ CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
3129
FLAGS = $(CPPFLAGS) $(CFLAGS)
3230

3331

34-
ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c)
32+
ZSTD_FILES := $(sort $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c))
3533

3634
ZSTD_LEGACY_SUPPORT ?= 4
3735

@@ -103,10 +101,19 @@ lib-release lib-release-mt: DEBUGFLAGS :=
103101
lib-release: lib
104102
lib-release-mt: lib-mt
105103

104+
# Special case : building library in single-thread mode _and_ without zstdmt_compress.c
105+
ZSTDMT_FILES = compress/zstdmt_compress.c
106+
ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES))
107+
libzstd-nomt: LDFLAGS += -shared -fPIC -fvisibility=hidden
108+
libzstd-nomt: $(ZSTD_NOMT_FILES)
109+
@echo compiling single-thread dynamic library $(LIBVER)
110+
@echo files : $(ZSTD_NOMT_FILES)
111+
@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
112+
106113
clean:
107114
@$(RM) -r *.dSYM # Mac OS-X specific
108115
@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
109-
@$(RM) dll/libzstd.dll dll/libzstd.lib
116+
@$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
110117
@$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
111118
@echo Cleaning library completed
112119

@@ -115,25 +122,32 @@ clean:
115122
#-----------------------------------------------------------------------------
116123
ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
117124

118-
ifneq (,$(filter $(shell uname),SunOS))
119-
INSTALL ?= ginstall
120-
else
121-
INSTALL ?= install
122-
endif
123-
124-
PREFIX ?= /usr/local
125-
DESTDIR ?=
126-
LIBDIR ?= $(PREFIX)/lib
127-
INCLUDEDIR ?= $(PREFIX)/include
125+
DESTDIR ?=
126+
# directory variables : GNU conventions prefer lowercase
127+
# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
128+
# support both lower and uppercase (BSD), use uppercase in script
129+
prefix ?= /usr/local
130+
PREFIX ?= $(prefix)
131+
exec_prefix ?= $(PREFIX)
132+
libdir ?= $(exec_prefix)/lib
133+
LIBDIR ?= $(libdir)
134+
includedir ?= $(PREFIX)/include
135+
INCLUDEDIR ?= $(includedir)
128136

129137
ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
130138
PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
131139
else
132140
PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
133141
endif
134142

135-
INSTALL_LIB ?= $(INSTALL) -m 755
136-
INSTALL_DATA ?= $(INSTALL) -m 644
143+
ifneq (,$(filter $(shell uname),SunOS))
144+
INSTALL ?= ginstall
145+
else
146+
INSTALL ?= install
147+
endif
148+
149+
INSTALL_PROGRAM ?= $(INSTALL)
150+
INSTALL_DATA ?= $(INSTALL) -m 644
137151

138152

139153
libzstd.pc:
@@ -150,9 +164,9 @@ install: libzstd.a libzstd libzstd.pc
150164
@$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
151165
@echo Installing libraries
152166
@$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
153-
@$(INSTALL_LIB) libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
154-
@ln -sf libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
155-
@ln -sf libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
167+
@$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
168+
@ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
169+
@ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
156170
@echo Installing includes
157171
@$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
158172
@$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
@@ -164,7 +178,7 @@ uninstall:
164178
@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
165179
@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
166180
@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
167-
@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
181+
@$(RM) $(DESTDIR)$(LIBDIR)/$(LIBZSTD)
168182
@$(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc
169183
@$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
170184
@$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h

‎ext/zstdruby/libzstd/README.md

Lines changed: 68 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,78 @@
11
Zstandard library files
22
================================
33

4-
The __lib__ directory contains several directories.
5-
Depending on target use case, it's enough to include only files from relevant directories.
4+
The __lib__ directory is split into several sub-directories,
5+
in order to make it easier to select or exclude specific features.
6+
7+
8+
#### Building
9+
10+
`Makefile` script is provided, supporting the standard set of commands,
11+
directories, and variables (see https://www.gnu.org/prep/standards/html_node/Command-Variables.html).
12+
- `make` : generates both static and dynamic libraries
13+
- `make install` : install libraries in default system directories
614

715

816
#### API
917

10-
Zstandard's stable API is exposed within [zstd.h](zstd.h),
11-
at the root of `lib` directory.
18+
Zstandard's stable API is exposed within [lib/zstd.h](zstd.h).
1219

1320

1421
#### Advanced API
1522

16-
Some additional API may be useful if you're looking into advanced features :
17-
- common/error_public.h : transforms `size_t` function results into an `enum`,
18-
for precise error handling.
19-
- ZSTD_STATIC_LINKING_ONLY : if you define this macro _before_ including `zstd.h`,
20-
it will give access to advanced and experimental API.
23+
Optional advanced features are exposed via :
24+
25+
- `lib/common/zstd_errors.h` : translates `size_t` function results
26+
into an `ZSTD_ErrorCode`, for accurate error handling.
27+
- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
28+
it unlocks access to advanced experimental API,
29+
exposed in second part of `zstd.h`.
2130
These APIs shall ___never be used with dynamic library___ !
2231
They are not "stable", their definition may change in the future.
2332
Only static linking is allowed.
2433

25-
#### ZSTDMT API
26-
27-
To enable multithreaded compression within the library, invoke `make lib-mt` target.
28-
Prototypes are defined in header file `compress/zstdmt_compress.h`.
29-
When linking a program that uses ZSTDMT API against libzstd.a on a POSIX system,
30-
`-pthread` flag must be provided to the compiler and linker.
31-
Note : ZSTDMT prototypes can still be used with a library built without multithread support,
32-
but in this case, they will be single threaded only.
3334

3435
#### Modular build
3536

36-
Directory `common/` is required in all circumstances.
37-
You can select to support compression only, by just adding files from the `compress/` directory,
38-
In a similar way, you can build a decompressor-only library with the `decompress/` directory.
39-
40-
Other optional functionalities provided are :
41-
42-
- `dictBuilder/` : source files to create dictionaries.
43-
The API can be consulted in `dictBuilder/zdict.h`.
44-
This module also depends on `common/` and `compress/` .
45-
46-
- `legacy/` : source code to decompress previous versions of zstd, starting from `v0.1`.
47-
This module also depends on `common/` and `decompress/` .
48-
Library compilation must include directive `ZSTD_LEGACY_SUPPORT = 1` .
49-
The main API can be consulted in `legacy/zstd_legacy.h`.
50-
Advanced API from each version can be found in their relevant header file.
51-
For example, advanced API for version `v0.4` is in `legacy/zstd_v04.h` .
52-
53-
54-
#### Using MinGW+MSYS to create DLL
37+
- Directory `lib/common` is always required, for all variants.
38+
- Compression source code lies in `lib/compress`
39+
- Decompression source code lies in `lib/decompress`
40+
- It's possible to include only `compress` or only `decompress`, they don't depend on each other.
41+
- `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples.
42+
The API is exposed in `lib/dictBuilder/zdict.h`.
43+
This module depends on both `lib/common` and `lib/compress` .
44+
- `lib/legacy` : source code to decompress older zstd formats, starting from `v0.1`.
45+
This module depends on `lib/common` and `lib/decompress`.
46+
To enable this feature, it's necessary to define `ZSTD_LEGACY_SUPPORT = 1` during compilation.
47+
Typically, with `gcc`, add argument `-DZSTD_LEGACY_SUPPORT=1`.
48+
Using higher number limits the number of version supported.
49+
For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats starting from v0.2+".
50+
The API is exposed in `lib/legacy/zstd_legacy.h`.
51+
Each version also provides a (dedicated) set of advanced API.
52+
For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
53+
54+
55+
#### Multithreading support
56+
57+
Multithreading is disabled by default when building with `make`.
58+
Enabling multithreading requires 2 conditions :
59+
- set macro `ZSTD_MULTITHREAD`
60+
- on POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc` for example)
61+
62+
Both conditions are automatically triggered by invoking `make lib-mt` target.
63+
Note that, when linking a POSIX program with a multithreaded version of `libzstd`,
64+
it's necessary to trigger `-pthread` flag during link stage.
65+
66+
Multithreading capabilities are exposed via :
67+
- private API `lib/compress/zstdmt_compress.h`.
68+
Symbols defined in this header are currently exposed in `libzstd`, hence usable.
69+
Note however that this API is planned to be locked and remain strictly internal in the future.
70+
- advanced API `ZSTD_compress_generic()`, defined in `lib/zstd.h`, experimental section.
71+
This API is still considered experimental, but is designed to be labelled "stable" at some point in the future.
72+
It's the recommended entry point for multi-threading operations.
73+
74+
75+
#### Windows : using MinGW+MSYS to create DLL
5576

5677
DLL can be created using MinGW+MSYS with the `make libzstd` command.
5778
This command creates `dll\libzstd.dll` and the import library `dll\libzstd.lib`.
@@ -67,19 +88,21 @@ file it should be linked with `dll\libzstd.dll`. For example:
6788
The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
6889

6990

70-
#### Obsolete streaming API
91+
#### Deprecated API
7192

72-
Streaming is now provided within `zstd.h`.
73-
Older streaming API is still available within `deprecated/zbuff.h`.
74-
It will be removed in a future version.
75-
Consider migrating code towards newer streaming API in `zstd.h`.
93+
Obsolete API on their way out are stored in directory `lib/deprecated`.
94+
At this stage, it contains older streaming prototypes, in `lib/deprecated/zbuff.h`.
95+
Presence in this directory is temporary.
96+
These prototypes will be removed in some future version.
97+
Consider migrating code towards supported streaming API exposed in `zstd.h`.
7698

7799

78100
#### Miscellaneous
79101

80102
The other files are not source code. There are :
81103

82-
- LICENSE : contains the BSD license text
83-
- Makefile : script to compile or install zstd library (static and dynamic)
84-
- libzstd.pc.in : for pkg-config (`make install`)
85-
- README.md : this file
104+
- `LICENSE` : contains the BSD license text
105+
- `Makefile` : `make` script to build and install zstd library (static and dynamic)
106+
- `BUCK` : support for `buck` build system (https://buckbuild.com/)
107+
- `libzstd.pc.in` : for `pkg-config` (used in `make install`)
108+
- `README.md` : this file

‎ext/zstdruby/libzstd/common/bitstream.h

Lines changed: 35 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -169,33 +169,39 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
169169
****************************************************************/
170170
MEM_STATIC unsigned BIT_highbit32 (register U32 val)
171171
{
172+
assert(val != 0);
173+
{
172174
# if defined(_MSC_VER) /* Visual */
173-
unsigned long r=0;
174-
_BitScanReverse ( &r, val );
175-
return (unsigned) r;
175+
unsigned long r=0;
176+
_BitScanReverse ( &r, val );
177+
return (unsigned) r;
176178
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
177-
return 31 - __builtin_clz (val);
179+
return 31 - __builtin_clz (val);
178180
# else /* Software version */
179-
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29,
180-
11, 14, 16, 18, 22, 25, 3, 30,
181-
8, 12, 20, 28, 15, 17, 24, 7,
182-
19, 27, 23, 6, 26, 5, 4, 31 };
183-
U32 v = val;
184-
v |= v >> 1;
185-
v |= v >> 2;
186-
v |= v >> 4;
187-
v |= v >> 8;
188-
v |= v >> 16;
189-
return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
181+
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29,
182+
11, 14, 16, 18, 22, 25, 3, 30,
183+
8, 12, 20, 28, 15, 17, 24, 7,
184+
19, 27, 23, 6, 26, 5, 4, 31 };
185+
U32 v = val;
186+
v |= v >> 1;
187+
v |= v >> 2;
188+
v |= v >> 4;
189+
v |= v >> 8;
190+
v |= v >> 16;
191+
return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
190192
# endif
193+
}
191194
}
192195

193196
/*===== Local Constants =====*/
194-
static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F,
195-
0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF,
196-
0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
197-
0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
198-
197+
static const unsigned BIT_mask[] = {
198+
0, 1, 3, 7, 0xF, 0x1F,
199+
0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF,
200+
0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF,
201+
0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
202+
0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
203+
0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */
204+
#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))
199205

200206
/*-**************************************************************
201207
* bitStream encoding
@@ -217,11 +223,14 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
217223
}
218224

219225
/*! BIT_addBits() :
220-
* can add up to 26 bits into `bitC`.
226+
* can add up to 31 bits into `bitC`.
221227
* Note : does not check for register overflow ! */
222228
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
223229
size_t value, unsigned nbBits)
224230
{
231+
MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32);
232+
assert(nbBits < BIT_MASK_SIZE);
233+
assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
225234
bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
226235
bitC->bitPos += nbBits;
227236
}
@@ -232,6 +241,7 @@ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
232241
size_t value, unsigned nbBits)
233242
{
234243
assert((value>>nbBits) == 0);
244+
assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
235245
bitC->bitContainer |= value << bitC->bitPos;
236246
bitC->bitPos += nbBits;
237247
}
@@ -242,7 +252,7 @@ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
242252
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
243253
{
244254
size_t const nbBytes = bitC->bitPos >> 3;
245-
assert( bitC->bitPos <= (sizeof(bitC->bitContainer)*8) );
255+
assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
246256
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
247257
bitC->ptr += nbBytes;
248258
assert(bitC->ptr <= bitC->endPtr);
@@ -258,7 +268,7 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
258268
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
259269
{
260270
size_t const nbBytes = bitC->bitPos >> 3;
261-
assert( bitC->bitPos <= (sizeof(bitC->bitContainer)*8) );
271+
assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
262272
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
263273
bitC->ptr += nbBytes;
264274
if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
@@ -350,12 +360,14 @@ MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 co
350360
# endif
351361
return _bextr_u32(bitContainer, start, nbBits);
352362
#else
363+
assert(nbBits < BIT_MASK_SIZE);
353364
return (bitContainer >> start) & BIT_mask[nbBits];
354365
#endif
355366
}
356367

357368
MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
358369
{
370+
assert(nbBits < BIT_MASK_SIZE);
359371
return bitContainer & BIT_mask[nbBits];
360372
}
361373

0 commit comments

Comments
 (0)