Skip to content

Commit 2850724

Browse files
GenwaldGenwald
committed
update zstd to 1.4.4
1 parent ec97ed4 commit 2850724

File tree

6 files changed

+897
-59
lines changed

6 files changed

+897
-59
lines changed

‎libs/include/cover.h‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
#include <stdio.h> /* fprintf */
2+
#include <stdlib.h> /* malloc, free, qsort */
3+
#include <string.h> /* memset */
4+
#include <time.h> /* clock */
5+
#include "mem.h" /* read */
6+
#include "pool.h"
7+
#include "threading.h"
8+
#include "zstd_internal.h" /* includes zstd.h */
9+
#ifndef ZDICT_STATIC_LINKING_ONLY
10+
#define ZDICT_STATIC_LINKING_ONLY
11+
#endif
12+
#include "zdict.h"
13+
14+
/**
15+
* COVER_best_t is used for two purposes:
16+
* 1. Synchronizing threads.
17+
* 2. Saving the best parameters and dictionary.
18+
*
19+
* All of the methods except COVER_best_init() are thread safe if zstd is
20+
* compiled with multithreaded support.
21+
*/
22+
typedef struct COVER_best_s {
23+
ZSTD_pthread_mutex_t mutex;
24+
ZSTD_pthread_cond_t cond;
25+
size_t liveJobs;
26+
void *dict;
27+
size_t dictSize;
28+
ZDICT_cover_params_t parameters;
29+
size_t compressedSize;
30+
} COVER_best_t;
31+
32+
/**
33+
* A segment is a range in the source as well as the score of the segment.
34+
*/
35+
typedef struct {
36+
U32 begin;
37+
U32 end;
38+
U32 score;
39+
} COVER_segment_t;
40+
41+
/**
42+
*Number of epochs and size of each epoch.
43+
*/
44+
typedef struct {
45+
U32 num;
46+
U32 size;
47+
} COVER_epoch_info_t;
48+
49+
/**
50+
* Struct used for the dictionary selection function.
51+
*/
52+
typedef struct COVER_dictSelection {
53+
BYTE* dictContent;
54+
size_t dictSize;
55+
size_t totalCompressedSize;
56+
} COVER_dictSelection_t;
57+
58+
/**
59+
* Computes the number of epochs and the size of each epoch.
60+
* We will make sure that each epoch gets at least 10 * k bytes.
61+
*
62+
* The COVER algorithms divide the data up into epochs of equal size and
63+
* select one segment from each epoch.
64+
*
65+
* @param maxDictSize The maximum allowed dictionary size.
66+
* @param nbDmers The number of dmers we are training on.
67+
* @param k The parameter k (segment size).
68+
* @param passes The target number of passes over the dmer corpus.
69+
* More passes means a better dictionary.
70+
*/
71+
COVER_epoch_info_t COVER_computeEpochs(U32 maxDictSize, U32 nbDmers,
72+
U32 k, U32 passes);
73+
74+
/**
75+
* Warns the user when their corpus is too small.
76+
*/
77+
void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel);
78+
79+
/**
80+
* Checks total compressed size of a dictionary
81+
*/
82+
size_t COVER_checkTotalCompressedSize(const ZDICT_cover_params_t parameters,
83+
const size_t *samplesSizes, const BYTE *samples,
84+
size_t *offsets,
85+
size_t nbTrainSamples, size_t nbSamples,
86+
BYTE *const dict, size_t dictBufferCapacity);
87+
88+
/**
89+
* Returns the sum of the sample sizes.
90+
*/
91+
size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) ;
92+
93+
/**
94+
* Initialize the `COVER_best_t`.
95+
*/
96+
void COVER_best_init(COVER_best_t *best);
97+
98+
/**
99+
* Wait until liveJobs == 0.
100+
*/
101+
void COVER_best_wait(COVER_best_t *best);
102+
103+
/**
104+
* Call COVER_best_wait() and then destroy the COVER_best_t.
105+
*/
106+
void COVER_best_destroy(COVER_best_t *best);
107+
108+
/**
109+
* Called when a thread is about to be launched.
110+
* Increments liveJobs.
111+
*/
112+
void COVER_best_start(COVER_best_t *best);
113+
114+
/**
115+
* Called when a thread finishes executing, both on error or success.
116+
* Decrements liveJobs and signals any waiting threads if liveJobs == 0.
117+
* If this dictionary is the best so far save it and its parameters.
118+
*/
119+
void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters,
120+
COVER_dictSelection_t selection);
121+
/**
122+
* Error function for COVER_selectDict function. Checks if the return
123+
* value is an error.
124+
*/
125+
unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection);
126+
127+
/**
128+
* Error function for COVER_selectDict function. Returns a struct where
129+
* return.totalCompressedSize is a ZSTD error.
130+
*/
131+
COVER_dictSelection_t COVER_dictSelectionError(size_t error);
132+
133+
/**
134+
* Always call after selectDict is called to free up used memory from
135+
* newly created dictionary.
136+
*/
137+
void COVER_dictSelectionFree(COVER_dictSelection_t selection);
138+
139+
/**
140+
* Called to finalize the dictionary and select one based on whether or not
141+
* the shrink-dict flag was enabled. If enabled the dictionary used is the
142+
* smallest dictionary within a specified regression of the compressed size
143+
* from the largest dictionary.
144+
*/
145+
COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
146+
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
147+
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);

‎libs/include/zbuff.h‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
/*
2+
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under both the BSD-style license (found in the
6+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
7+
* in the COPYING file in the root directory of this source tree).
8+
* You may select, at your option, one of the above-listed licenses.
9+
*/
10+
11+
/* ***************************************************************
12+
* NOTES/WARNINGS
13+
******************************************************************/
14+
/* The streaming API defined here is deprecated.
15+
* Consider migrating towards ZSTD_compressStream() API in `zstd.h`
16+
* See 'lib/README.md'.
17+
*****************************************************************/
18+
19+
20+
#if defined (__cplusplus)
21+
extern "C" {
22+
#endif
23+
24+
#ifndef ZSTD_BUFFERED_H_23987
25+
#define ZSTD_BUFFERED_H_23987
26+
27+
/* *************************************
28+
* Dependencies
29+
***************************************/
30+
#include <stddef.h> /* size_t */
31+
#include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
32+
33+
34+
/* ***************************************************************
35+
* Compiler specifics
36+
*****************************************************************/
37+
/* Deprecation warnings */
38+
/* Should these warnings be a problem,
39+
* it is generally possible to disable them,
40+
* typically with -Wno-deprecated-declarations for gcc
41+
* or _CRT_SECURE_NO_WARNINGS in Visual.
42+
* Otherwise, it's also possible to define ZBUFF_DISABLE_DEPRECATE_WARNINGS
43+
*/
44+
#ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS
45+
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */
46+
#else
47+
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
48+
# define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API
49+
# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
50+
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message)))
51+
# elif defined(__GNUC__) && (__GNUC__ >= 3)
52+
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated))
53+
# elif defined(_MSC_VER)
54+
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API __declspec(deprecated(message))
55+
# else
56+
# pragma message("WARNING: You need to implement ZBUFF_DEPRECATED for this compiler")
57+
# define ZBUFF_DEPRECATED(message) ZSTDLIB_API
58+
# endif
59+
#endif /* ZBUFF_DISABLE_DEPRECATE_WARNINGS */
60+
61+
62+
/* *************************************
63+
* Streaming functions
64+
***************************************/
65+
/* This is the easier "buffered" streaming API,
66+
* using an internal buffer to lift all restrictions on user-provided buffers
67+
* which can be any size, any place, for both input and output.
68+
* ZBUFF and ZSTD are 100% interoperable,
69+
* frames created by one can be decoded by the other one */
70+
71+
typedef ZSTD_CStream ZBUFF_CCtx;
72+
ZBUFF_DEPRECATED("use ZSTD_createCStream") ZBUFF_CCtx* ZBUFF_createCCtx(void);
73+
ZBUFF_DEPRECATED("use ZSTD_freeCStream") size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
74+
75+
ZBUFF_DEPRECATED("use ZSTD_initCStream") size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
76+
ZBUFF_DEPRECATED("use ZSTD_initCStream_usingDict") size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
77+
78+
ZBUFF_DEPRECATED("use ZSTD_compressStream") size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
79+
ZBUFF_DEPRECATED("use ZSTD_flushStream") size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
80+
ZBUFF_DEPRECATED("use ZSTD_endStream") size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
81+
82+
/*-*************************************************
83+
* Streaming compression - howto
84+
*
85+
* A ZBUFF_CCtx object is required to track streaming operation.
86+
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
87+
* ZBUFF_CCtx objects can be reused multiple times.
88+
*
89+
* Start by initializing ZBUF_CCtx.
90+
* Use ZBUFF_compressInit() to start a new compression operation.
91+
* Use ZBUFF_compressInitDictionary() for a compression which requires a dictionary.
92+
*
93+
* Use ZBUFF_compressContinue() repetitively to consume input stream.
94+
* *srcSizePtr and *dstCapacityPtr can be any size.
95+
* The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr.
96+
* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
97+
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst .
98+
* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
99+
* or an error code, which can be tested using ZBUFF_isError().
100+
*
101+
* At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush().
102+
* The nb of bytes written into `dst` will be reported into *dstCapacityPtr.
103+
* Note that the function cannot output more than *dstCapacityPtr,
104+
* therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small.
105+
* @return : nb of bytes still present into internal buffer (0 if it's empty)
106+
* or an error code, which can be tested using ZBUFF_isError().
107+
*
108+
* ZBUFF_compressEnd() instructs to finish a frame.
109+
* It will perform a flush and write frame epilogue.
110+
* The epilogue is required for decoders to consider a frame completed.
111+
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
112+
* In which case, call again ZBUFF_compressFlush() to complete the flush.
113+
* @return : nb of bytes still present into internal buffer (0 if it's empty)
114+
* or an error code, which can be tested using ZBUFF_isError().
115+
*
116+
* Hint : _recommended buffer_ sizes (not compulsory) : ZBUFF_recommendedCInSize() / ZBUFF_recommendedCOutSize()
117+
* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, use this value to reduce intermediate stages (better latency)
118+
* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering.
119+
* By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering.
120+
* **************************************************/
121+
122+
123+
typedef ZSTD_DStream ZBUFF_DCtx;
124+
ZBUFF_DEPRECATED("use ZSTD_createDStream") ZBUFF_DCtx* ZBUFF_createDCtx(void);
125+
ZBUFF_DEPRECATED("use ZSTD_freeDStream") size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
126+
127+
ZBUFF_DEPRECATED("use ZSTD_initDStream") size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
128+
ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
129+
130+
ZBUFF_DEPRECATED("use ZSTD_decompressStream") size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
131+
void* dst, size_t* dstCapacityPtr,
132+
const void* src, size_t* srcSizePtr);
133+
134+
/*-***************************************************************************
135+
* Streaming decompression howto
136+
*
137+
* A ZBUFF_DCtx object is required to track streaming operations.
138+
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
139+
* Use ZBUFF_decompressInit() to start a new decompression operation,
140+
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
141+
* Note that ZBUFF_DCtx objects can be re-init multiple times.
142+
*
143+
* Use ZBUFF_decompressContinue() repetitively to consume your input.
144+
* *srcSizePtr and *dstCapacityPtr can be any size.
145+
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
146+
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
147+
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
148+
* @return : 0 when a frame is completely decoded and fully flushed,
149+
* 1 when there is still some data left within internal buffer to flush,
150+
* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency),
151+
* or an error code, which can be tested using ZBUFF_isError().
152+
*
153+
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
154+
* output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
155+
* input : ZBUFF_recommendedDInSize == 128KB + 3;
156+
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
157+
* *******************************************************************************/
158+
159+
160+
/* *************************************
161+
* Tool functions
162+
***************************************/
163+
ZBUFF_DEPRECATED("use ZSTD_isError") unsigned ZBUFF_isError(size_t errorCode);
164+
ZBUFF_DEPRECATED("use ZSTD_getErrorName") const char* ZBUFF_getErrorName(size_t errorCode);
165+
166+
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
167+
* These sizes are just hints, they tend to offer better latency */
168+
ZBUFF_DEPRECATED("use ZSTD_CStreamInSize") size_t ZBUFF_recommendedCInSize(void);
169+
ZBUFF_DEPRECATED("use ZSTD_CStreamOutSize") size_t ZBUFF_recommendedCOutSize(void);
170+
ZBUFF_DEPRECATED("use ZSTD_DStreamInSize") size_t ZBUFF_recommendedDInSize(void);
171+
ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(void);
172+
173+
#endif /* ZSTD_BUFFERED_H_23987 */
174+
175+
176+
#ifdef ZBUFF_STATIC_LINKING_ONLY
177+
#ifndef ZBUFF_STATIC_H_30298098432
178+
#define ZBUFF_STATIC_H_30298098432
179+
180+
/* ====================================================================================
181+
* The definitions in this section are considered experimental.
182+
* They should never be used in association with a dynamic library, as they may change in the future.
183+
* They are provided for advanced usages.
184+
* Use them only in association with static linking.
185+
* ==================================================================================== */
186+
187+
/*--- Dependency ---*/
188+
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */
189+
#include "zstd.h"
190+
191+
192+
/*--- Custom memory allocator ---*/
193+
/*! ZBUFF_createCCtx_advanced() :
194+
* Create a ZBUFF compression context using external alloc and free functions */
195+
ZBUFF_DEPRECATED("use ZSTD_createCStream_advanced") ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
196+
197+
/*! ZBUFF_createDCtx_advanced() :
198+
* Create a ZBUFF decompression context using external alloc and free functions */
199+
ZBUFF_DEPRECATED("use ZSTD_createDStream_advanced") ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
200+
201+
202+
/*--- Advanced Streaming Initialization ---*/
203+
ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
204+
const void* dict, size_t dictSize,
205+
ZSTD_parameters params, unsigned long long pledgedSrcSize);
206+
207+
208+
#endif /* ZBUFF_STATIC_H_30298098432 */
209+
#endif /* ZBUFF_STATIC_LINKING_ONLY */
210+
211+
212+
#if defined (__cplusplus)
213+
}
214+
#endif

0 commit comments

Comments
 (0)