@@ -90,19 +90,20 @@ MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
9090/*-********************************************
9191* bitStream decoding API (read backward)
9292**********************************************/
93+ typedef size_t BitContainerType ;
9394typedef struct {
94- size_t bitContainer ;
95+ BitContainerType bitContainer ;
9596 unsigned bitsConsumed ;
9697 const char * ptr ;
9798 const char * start ;
9899 const char * limitPtr ;
99100} BIT_DStream_t ;
100101
101- typedef enum { BIT_DStream_unfinished = 0 ,
102- BIT_DStream_endOfBuffer = 1 ,
103- BIT_DStream_completed = 2 ,
104- BIT_DStream_overflow = 3 } BIT_DStream_status ; /* result of BIT_reloadDStream() */
105- /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
102+ typedef enum { BIT_DStream_unfinished = 0 , /* fully refilled */
103+ BIT_DStream_endOfBuffer = 1 , /* still some bits left in bitstream */
104+ BIT_DStream_completed = 2 , /* bitstream entirely consumed, bit-exact */
105+ BIT_DStream_overflow = 3 /* user requested more bits than present in bitstream */
106+ } BIT_DStream_status ; /* result of BIT_reloadDStream() */
106107
107108MEM_STATIC size_t BIT_initDStream (BIT_DStream_t * bitD , const void * srcBuffer , size_t srcSize );
108109MEM_STATIC size_t BIT_readBits (BIT_DStream_t * bitD , unsigned nbBits );
@@ -112,7 +113,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
112113
113114/* Start by invoking BIT_initDStream().
114115* A chunk of the bitStream is then stored into a local register.
115- * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t ).
116+ * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType ).
116117* You can then retrieve bitFields stored into the local register, **in reverse order**.
117118* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
118119* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
@@ -162,7 +163,7 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
162163 return 0 ;
163164}
164165
165- MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits (size_t bitContainer , U32 const nbBits )
166+ FORCE_INLINE_TEMPLATE size_t BIT_getLowerBits (size_t bitContainer , U32 const nbBits )
166167{
167168#if defined(STATIC_BMI2 ) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS )
168169 return _bzhi_u64 (bitContainer , nbBits );
@@ -267,22 +268,22 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
267268 bitD -> bitContainer = * (const BYTE * )(bitD -> start );
268269 switch (srcSize )
269270 {
270- case 7 : bitD -> bitContainer += (size_t )(((const BYTE * )(srcBuffer ))[6 ]) << (sizeof (bitD -> bitContainer )* 8 - 16 );
271+ case 7 : bitD -> bitContainer += (BitContainerType )(((const BYTE * )(srcBuffer ))[6 ]) << (sizeof (bitD -> bitContainer )* 8 - 16 );
271272 ZSTD_FALLTHROUGH ;
272273
273- case 6 : bitD -> bitContainer += (size_t )(((const BYTE * )(srcBuffer ))[5 ]) << (sizeof (bitD -> bitContainer )* 8 - 24 );
274+ case 6 : bitD -> bitContainer += (BitContainerType )(((const BYTE * )(srcBuffer ))[5 ]) << (sizeof (bitD -> bitContainer )* 8 - 24 );
274275 ZSTD_FALLTHROUGH ;
275276
276- case 5 : bitD -> bitContainer += (size_t )(((const BYTE * )(srcBuffer ))[4 ]) << (sizeof (bitD -> bitContainer )* 8 - 32 );
277+ case 5 : bitD -> bitContainer += (BitContainerType )(((const BYTE * )(srcBuffer ))[4 ]) << (sizeof (bitD -> bitContainer )* 8 - 32 );
277278 ZSTD_FALLTHROUGH ;
278279
279- case 4 : bitD -> bitContainer += (size_t )(((const BYTE * )(srcBuffer ))[3 ]) << 24 ;
280+ case 4 : bitD -> bitContainer += (BitContainerType )(((const BYTE * )(srcBuffer ))[3 ]) << 24 ;
280281 ZSTD_FALLTHROUGH ;
281282
282- case 3 : bitD -> bitContainer += (size_t )(((const BYTE * )(srcBuffer ))[2 ]) << 16 ;
283+ case 3 : bitD -> bitContainer += (BitContainerType )(((const BYTE * )(srcBuffer ))[2 ]) << 16 ;
283284 ZSTD_FALLTHROUGH ;
284285
285- case 2 : bitD -> bitContainer += (size_t )(((const BYTE * )(srcBuffer ))[1 ]) << 8 ;
286+ case 2 : bitD -> bitContainer += (BitContainerType )(((const BYTE * )(srcBuffer ))[1 ]) << 8 ;
286287 ZSTD_FALLTHROUGH ;
287288
288289 default : break ;
@@ -297,12 +298,12 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
297298 return srcSize ;
298299}
299300
300- MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits (size_t bitContainer , U32 const start )
301+ FORCE_INLINE_TEMPLATE size_t BIT_getUpperBits (BitContainerType bitContainer , U32 const start )
301302{
302303 return bitContainer >> start ;
303304}
304305
305- MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits (size_t bitContainer , U32 const start , U32 const nbBits )
306+ FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits (BitContainerType bitContainer , U32 const start , U32 const nbBits )
306307{
307308 U32 const regMask = sizeof (bitContainer )* 8 - 1 ;
308309 /* if start > regMask, bitstream is corrupted, and result is undefined */
@@ -325,7 +326,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 c
325326 * On 32-bits, maxNbBits==24.
326327 * On 64-bits, maxNbBits==56.
327328 * @return : value extracted */
328- MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits (const BIT_DStream_t * bitD , U32 nbBits )
329+ FORCE_INLINE_TEMPLATE size_t BIT_lookBits (const BIT_DStream_t * bitD , U32 nbBits )
329330{
330331 /* arbitrate between double-shift and shift+mask */
331332#if 1
@@ -348,7 +349,7 @@ MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
348349 return (bitD -> bitContainer << (bitD -> bitsConsumed & regMask )) >> (((regMask + 1 )- nbBits ) & regMask );
349350}
350351
351- MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits (BIT_DStream_t * bitD , U32 nbBits )
352+ FORCE_INLINE_TEMPLATE void BIT_skipBits (BIT_DStream_t * bitD , U32 nbBits )
352353{
353354 bitD -> bitsConsumed += nbBits ;
354355}
@@ -357,7 +358,7 @@ MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
357358 * Read (consume) next n bits from local register and update.
358359 * Pay attention to not read more than nbBits contained into local register.
359360 * @return : extracted value. */
360- MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits (BIT_DStream_t * bitD , unsigned nbBits )
361+ FORCE_INLINE_TEMPLATE size_t BIT_readBits (BIT_DStream_t * bitD , unsigned nbBits )
361362{
362363 size_t const value = BIT_lookBits (bitD , nbBits );
363364 BIT_skipBits (bitD , nbBits );
@@ -374,6 +375,21 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
374375 return value ;
375376}
376377
378+ /*! BIT_reloadDStream_internal() :
379+ * Simple variant of BIT_reloadDStream(), with two conditions:
380+ * 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8
381+ * 2. look window is valid after shifted down : bitD->ptr >= bitD->start
382+ */
383+ MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal (BIT_DStream_t * bitD )
384+ {
385+ assert (bitD -> bitsConsumed <= sizeof (bitD -> bitContainer )* 8 );
386+ bitD -> ptr -= bitD -> bitsConsumed >> 3 ;
387+ assert (bitD -> ptr >= bitD -> start );
388+ bitD -> bitsConsumed &= 7 ;
389+ bitD -> bitContainer = MEM_readLEST (bitD -> ptr );
390+ return BIT_DStream_unfinished ;
391+ }
392+
377393/*! BIT_reloadDStreamFast() :
378394 * Similar to BIT_reloadDStream(), but with two differences:
379395 * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold!
@@ -384,31 +400,35 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
384400{
385401 if (UNLIKELY (bitD -> ptr < bitD -> limitPtr ))
386402 return BIT_DStream_overflow ;
387- assert (bitD -> bitsConsumed <= sizeof (bitD -> bitContainer )* 8 );
388- bitD -> ptr -= bitD -> bitsConsumed >> 3 ;
389- bitD -> bitsConsumed &= 7 ;
390- bitD -> bitContainer = MEM_readLEST (bitD -> ptr );
391- return BIT_DStream_unfinished ;
403+ return BIT_reloadDStream_internal (bitD );
392404}
393405
394406/*! BIT_reloadDStream() :
395407 * Refill `bitD` from buffer previously set in BIT_initDStream() .
396- * This function is safe, it guarantees it will not read beyond src buffer.
408+ * This function is safe, it guarantees it will not never beyond src buffer.
397409 * @return : status of `BIT_DStream_t` internal register.
398410 * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
399- MEM_STATIC FORCE_INLINE_ATTR BIT_DStream_status BIT_reloadDStream (BIT_DStream_t * bitD )
411+ FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream (BIT_DStream_t * bitD )
400412{
401- if (bitD -> bitsConsumed > (sizeof (bitD -> bitContainer )* 8 )) /* overflow detected, like end of stream */
413+ /* note : once in overflow mode, a bitstream remains in this mode until it's reset */
414+ if (UNLIKELY (bitD -> bitsConsumed > (sizeof (bitD -> bitContainer )* 8 ))) {
415+ static const BitContainerType zeroFilled = 0 ;
416+ bitD -> ptr = (const char * )& zeroFilled ; /* aliasing is allowed for char */
417+ /* overflow detected, erroneous scenario or end of stream: no update */
402418 return BIT_DStream_overflow ;
419+ }
420+
421+ assert (bitD -> ptr >= bitD -> start );
403422
404423 if (bitD -> ptr >= bitD -> limitPtr ) {
405- return BIT_reloadDStreamFast (bitD );
424+ return BIT_reloadDStream_internal (bitD );
406425 }
407426 if (bitD -> ptr == bitD -> start ) {
427+ /* reached end of bitStream => no update */
408428 if (bitD -> bitsConsumed < sizeof (bitD -> bitContainer )* 8 ) return BIT_DStream_endOfBuffer ;
409429 return BIT_DStream_completed ;
410430 }
411- /* start < ptr < limitPtr */
431+ /* start < ptr < limitPtr => cautious update */
412432 { U32 nbBytes = bitD -> bitsConsumed >> 3 ;
413433 BIT_DStream_status result = BIT_DStream_unfinished ;
414434 if (bitD -> ptr - nbBytes < bitD -> start ) {
0 commit comments