Update unrar to 7.0b3

Author: Dutchman101

vendor/unrar/arccmt.cpp

@@ -174,7 +174,8 @@ void Archive::ViewComment()
     auto EndPos=CmtBuf.find(0x1A);
     if (EndPos!=std::wstring::npos)
       CmtSize=EndPos;
-    mprintf(L"\n");
+    mprintf(St(MArcComment));
+    mprintf(L":\n");
     OutComment(CmtBuf);
   }
 }

vendor/unrar/archive.hpp

@@ -85,9 +85,6 @@ class Archive:public File
     HEADER_TYPE GetHeaderType() {return CurHeaderType;}
     CommandData* GetCommandData() {return Cmd;}
     void SetSilentOpen(bool Mode) {SilentOpen=Mode;}
-#if 0
-    void GetRecoveryInfo(bool Required,int64 *Size,int *Percent);
-#endif
 #ifdef USE_QOPEN
     bool Open(const std::wstring &Name,uint Mode=FMF_READ) override;
     int Read(void *Data,size_t Size) override;

vendor/unrar/arcread.cpp

@@ -424,7 +424,7 @@ size_t Archive::ReadHeader15()
         NextBlockPos=SafeAdd(NextBlockPos,hd->PackSize,0);
 
         bool CRCProcessedOnly=hd->CommentInHeader;
-        ushort HeaderCRC=Raw.GetCRC15(CRCProcessedOnly);
+        uint HeaderCRC=Raw.GetCRC15(CRCProcessedOnly);
         if (hd->HeadCRC!=HeaderCRC)
         {
           BrokenHeader=true;
@@ -507,7 +507,7 @@ size_t Archive::ReadHeader15()
       break;
   }
 
-  ushort HeaderCRC=Raw.GetCRC15(false);
+  uint HeaderCRC=Raw.GetCRC15(false);
 
   // Old AV header does not have header CRC properly set.
   // Old Unix owners header didn't include string fields into header size,
@@ -649,7 +649,7 @@ size_t Archive::ReadHeader50()
   }
 
   int SizeToRead=int(BlockSize);
-  SizeToRead-=FirstReadSize-SizeBytes-4; // Adjust overread size bytes if any.
+  SizeToRead-=int(FirstReadSize-SizeBytes-4); // Adjust overread size bytes if any.
   uint HeaderSize=4+SizeBytes+(uint)BlockSize;
 
   if (SizeToRead<0 || HeaderSize<SIZEOF_SHORTBLOCKHEAD5)

vendor/unrar/blake2sp.cpp

@@ -20,7 +20,7 @@ void blake2sp_init( blake2sp_state *S )
 
   blake2s_init_param( &S->R, 0, 1 ); // Init root.
 
-  for( uint i = 0; i < PARALLELISM_DEGREE; ++i )
+  for( uint32 i = 0; i < PARALLELISM_DEGREE; ++i )
     blake2s_init_param( &S->S[i], i, 0 ); // Init leaf.
 
   S->R.last_node = 1;
@@ -49,6 +49,8 @@ void Blake2ThreadData::Update()
     if (_SSE_Version>=SSE_SSE && inlen__ >= 2 * PARALLELISM_DEGREE * BLAKE2S_BLOCKBYTES)
       _mm_prefetch((char*)(in__ +  PARALLELISM_DEGREE * BLAKE2S_BLOCKBYTES), _MM_HINT_T0);
 #endif
+    // We tried to _forceinline blake2s_update and blake2s_compress_sse,
+    // but it didn't improve performance.
     blake2s_update( S, in__, BLAKE2S_BLOCKBYTES );
     in__ += PARALLELISM_DEGREE * BLAKE2S_BLOCKBYTES;
     inlen__ -= PARALLELISM_DEGREE * BLAKE2S_BLOCKBYTES;

vendor/unrar/cmdfilter.cpp

@@ -35,20 +35,28 @@ bool CommandData::CheckArgs(StringList *Args,bool Dir,const std::wstring &CheckN
       }
       else
       {
+        // This code doesn't allow to apply -n and -x wildcard masks without
+        // trailing slash to folders unless these masks are * and *.*.
+        // See the changes history below.
         // 2023.03.26: Previously we removed this code completely to let
-        // -npath\* include empty folders in 'path' too. But then we received
-        // an email from user not willing -x*.avi to exclude folders like
-        // dir.avi with non-avi files. Also rar.txt mentions that masks like
-        // *.avi exclude only files. Initially we wanted masks like -npath\*
-        // or -xpath\* to match the entire contents of path including empty
-        // folders and added the special check for "*" and "*.*".
-        // But this is not very straightforward, when *.* and *.avi are
-        // processed differently, especially taking into account that
-        // we can specify the exact folder name without wildcards to process
-        // it and masks like 'dir*\' can be used to exclude folders.
-        // So we decided to skip all usual wildcard masks for folders.
+        // 'rar a arc dir -ndir\path\*' include empty folders in 'path' too.
+        // But then we received an email from user not willing -x*.avi to
+        // exclude folders like dir.avi with non-avi files. Also rar.txt
+        // mentions that masks like *.avi exclude only files. Initially
+        // we wanted masks like -npath\* or -xpath\* to match the entire
+        // contents of path including empty folders and added the special
+        // check for "*" and "*.*". But this is not very straightforward,
+        // when *.* and *.avi are processed differently, especially taking
+        // into account that we can specify the exact folder name without
+        // wildcards to process it and masks like 'dir*\' can be used to
+        // exclude folders. So we decided to skip all usual wildcard masks
+        // for folders.
+        // 2023.11.22: We returned the special check for "*" and "*.*",
+        // because users expected 'rar a arc dir -xdir\*' to exclude
+        // everything including subfolders in 'dir'. For now we returned it
+        // both for -n and -x, but we can limit it to -x only if needed.
         std::wstring Name=PointToName(CurMask);
-        if (IsWildcard(Name)/* && wcscmp(Name,L"*")!=0 && wcscmp(Name,L"*.*")!=0*/)
+        if (IsWildcard(Name) && Name!=L"*" && Name!=L"*.*")
          continue;
       }
     }

vendor/unrar/coder.cpp

@@ -1,8 +1,8 @@
 
 
-inline unsigned int RangeCoder::GetChar()
+inline byte RangeCoder::GetChar()
 {
-  return(UnpackRead->GetChar());
+  return UnpackRead->GetChar();
 }
 
 
@@ -11,8 +11,8 @@ void RangeCoder::InitDecoder(Unpack *UnpackRead)
   RangeCoder::UnpackRead=UnpackRead;
 
   low=code=0;
-  range=uint(-1);
-  for (int i=0;i < 4;i++)
+  range=0xffffffff;
+  for (uint i = 0; i < 4; i++)
     code=(code << 8) | GetChar();
 }
 

vendor/unrar/coder.hpp

@@ -11,7 +11,7 @@ class RangeCoder
     inline uint GetCurrentShiftCount(uint SHIFT);
     inline void Decode();
     inline void PutChar(unsigned int c);
-    inline unsigned int GetChar();
+    inline byte GetChar();
 
     uint low, code, range;
     struct SUBRANGE 

vendor/unrar/crc.cpp

@@ -20,7 +20,11 @@
 #define USE_SLICING
 #endif
 
-static uint crc_tables[8][256]; // Tables for Slicing-by-8.
+static uint crc_tables[16][256]; // Tables for Slicing-by-16.
+
+#ifdef USE_NEON_CRC32
+static bool CRC_Neon;
+#endif
 
 
 // Build the classic CRC32 lookup table.
@@ -36,6 +40,19 @@ void InitCRC32(uint *CRCTab)
       C=(C & 1) ? (C>>1)^0xEDB88320 : (C>>1);
     CRCTab[I]=C;
   }
+
+#ifdef USE_NEON_CRC32
+  #ifdef _APPLE
+    // getauxval isn't available in OS X
+    uint Value=0;
+    size_t Size=sizeof(Value);
+    int RetCode=sysctlbyname("hw.optional.armv8_crc32",&Value,&Size,NULL,0);
+    CRC_Neon=RetCode==0 && Value!=0;
+  #else
+    CRC_Neon=(getauxval(AT_HWCAP) & HWCAP_CRC32)!=0;
+  #endif
+#endif
+
 }
 
 
@@ -47,7 +64,7 @@ static void InitTables()
   for (uint I=0;I<256;I++) // Build additional lookup tables.
   {
     uint C=crc_tables[0][I];
-    for (uint J=1;J<8;J++)
+    for (uint J=1;J<16;J++)
     {
       C=crc_tables[0][(byte)C]^(C>>8);
       crc_tables[J][I]=C;
@@ -63,29 +80,66 @@ uint CRC32(uint StartCRC,const void *Addr,size_t Size)
 {
   byte *Data=(byte *)Addr;
 
+#ifdef USE_NEON_CRC32
+  if (CRC_Neon)
+  {
+    for (;Size>=8;Size-=8,Data+=8)
+#ifdef __clang__
+      StartCRC = __builtin_arm_crc32d(StartCRC, RawGet8(Data));
+#else
+      StartCRC = __builtin_aarch64_crc32x(StartCRC, RawGet8(Data));
+#endif
+    for (;Size>0;Size--,Data++) // Process left data.
+#ifdef __clang__
+      StartCRC = __builtin_arm_crc32b(StartCRC, *Data);
+#else
+      StartCRC = __builtin_aarch64_crc32b(StartCRC, *Data);
+#endif
+    return StartCRC;
+  }
+#endif
+
 #ifdef USE_SLICING
-  // Align Data to 8 for better performance and to avoid ALLOW_MISALIGNED
+  // Align Data to 16 for better performance and to avoid ALLOW_MISALIGNED
   // check below.
-  for (;Size>0 && ((size_t)Data & 7);Size--,Data++)
+  for (;Size>0 && ((size_t)Data & 15)!=0;Size--,Data++)
     StartCRC=crc_tables[0][(byte)(StartCRC^Data[0])]^(StartCRC>>8);
 
-  for (;Size>=8;Size-=8,Data+=8)
+  // 2023.12.06: We switched to slicing-by-16, which seems to be faster than
+  // slicing-by-8 on modern CPUs. Slicing-by-32 would require 32 KB for tables
+  // and could be limited by L1 cache size on some CPUs.
+  for (;Size>=16;Size-=16,Data+=16)
   {
 #ifdef BIG_ENDIAN
-    StartCRC ^= Data[0]|(Data[1] << 8)|(Data[2] << 16)|(Data[3] << 24);
-    uint NextData = Data[4]|(Data[5] << 8)|(Data[6] << 16)|(Data[7] << 24);
+    StartCRC ^= RawGet4(Data);
+    uint D1 = RawGet4(Data+4);
+    uint D2 = RawGet4(Data+8);
+    uint D3 = RawGet4(Data+12);
 #else
+    // We avoid RawGet4 here for performance reason, to access uint32
+    // directly even if ALLOW_MISALIGNED isn't defined. We can do it,
+    // because we aligned 'Data' above.
     StartCRC ^= *(uint32 *) Data;
-    uint NextData = *(uint32 *) (Data+4);
+    uint D1 = *(uint32 *) (Data+4);
+    uint D2 = *(uint32 *) (Data+8);
+    uint D3 = *(uint32 *) (Data+12);
 #endif
-    StartCRC = crc_tables[7][(byte) StartCRC       ] ^
-               crc_tables[6][(byte)(StartCRC >> 8) ] ^
-               crc_tables[5][(byte)(StartCRC >> 16)] ^
-               crc_tables[4][(byte)(StartCRC >> 24)] ^
-               crc_tables[3][(byte) NextData       ] ^
-               crc_tables[2][(byte)(NextData >> 8) ] ^
-               crc_tables[1][(byte)(NextData >> 16)] ^
-               crc_tables[0][(byte)(NextData >> 24)];
+    StartCRC = crc_tables[15][(byte) StartCRC       ] ^
+               crc_tables[14][(byte)(StartCRC >> 8) ] ^
+               crc_tables[13][(byte)(StartCRC >> 16)] ^
+               crc_tables[12][(byte)(StartCRC >> 24)] ^
+               crc_tables[11][(byte) D1             ] ^
+               crc_tables[10][(byte)(D1       >> 8) ] ^
+               crc_tables[ 9][(byte)(D1       >> 16)] ^
+               crc_tables[ 8][(byte)(D1       >> 24)] ^
+               crc_tables[ 7][(byte) D2             ] ^
+               crc_tables[ 6][(byte)(D2       >>  8)] ^
+               crc_tables[ 5][(byte)(D2       >> 16)] ^
+               crc_tables[ 4][(byte)(D2       >> 24)] ^
+               crc_tables[ 3][(byte) D3             ] ^
+               crc_tables[ 2][(byte)(D3       >>  8)] ^
+               crc_tables[ 1][(byte)(D3       >> 16)] ^
+               crc_tables[ 0][(byte)(D3       >> 24)];
   }
 #endif
 
@@ -111,74 +165,6 @@ ushort Checksum14(ushort StartCRC,const void *Addr,size_t Size)
 #endif
 
 
-#if 0
-static uint64 crc64_tables[8][256]; // Tables for Slicing-by-8 for CRC64.
-
-void InitCRC64(uint64 *CRCTab)
-{
-  const uint64 poly=INT32TO64(0xC96C5795, 0xD7870F42); // 0xC96C5795D7870F42;
-  for (uint I=0;I<256;I++)
-  {
-    uint64 C=I;
-    for (uint J=0;J<8;J++)
-      C=(C & 1) ? (C>>1)^poly: (C>>1);
-    CRCTab[I]=C;
-  }
-}
-
-
-static void InitTables64()
-{
-  InitCRC64(crc64_tables[0]);
-
-  for (uint I=0;I<256;I++) // Build additional lookup tables.
-  {
-    uint64 C=crc64_tables[0][I];
-    for (uint J=1;J<8;J++)
-    {
-      C=crc64_tables[0][(byte)C]^(C>>8);
-      crc64_tables[J][I]=C;
-    }
-  }
-}
-
-
-// We cannot place the intialization to CRC64(), because we use this function
-// in multithreaded mode and it conflicts with multithreading.
-struct CallInitCRC64 {CallInitCRC64() {InitTables64();}} static CallInit64;
-
-uint64 CRC64(uint64 StartCRC,const void *Addr,size_t Size)
-{
-  byte *Data=(byte *)Addr;
-
-  // Align Data to 8 for better performance.
-  for (;Size>0 && ((size_t)Data & 7)!=0;Size--,Data++)
-    StartCRC=crc64_tables[0][(byte)(StartCRC^Data[0])]^(StartCRC>>8);
-
-  for (byte *DataEnd=Data+Size/8*8; Data<DataEnd; Data+=8 )
-  {
-    uint64 Index=StartCRC;
-#ifdef BIG_ENDIAN
-    Index ^= (uint64(Data[0])|(uint64(Data[1])<<8)|(uint64(Data[2])<<16)|(uint64(Data[3])<<24))|
-             (uint64(Data[4])<<32)|(uint64(Data[5])<<40)|(uint64(Data[6])<<48)|(uint64(Data[7])<<56);
-#else
-    Index ^= *(uint64 *)Data;
-#endif
-    StartCRC = crc64_tables[ 7 ] [ ( byte ) (Index       ) ] ^
-               crc64_tables[ 6 ] [ ( byte ) (Index >>  8 ) ] ^
-               crc64_tables[ 5 ] [ ( byte ) (Index >> 16 ) ] ^
-               crc64_tables[ 4 ] [ ( byte ) (Index >> 24 ) ] ^
-               crc64_tables[ 3 ] [ ( byte ) (Index >> 32 ) ] ^
-               crc64_tables[ 2 ] [ ( byte ) (Index >> 40 ) ] ^
-               crc64_tables[ 1 ] [ ( byte ) (Index >> 48 ) ] ^
-               crc64_tables[ 0 ] [ ( byte ) (Index >> 56 ) ] ;
-  }
-
-  for (Size%=8;Size>0;Size--,Data++) // Process left data.
-    StartCRC=crc64_tables[0][(byte)(StartCRC^Data[0])]^(StartCRC>>8);
-
-  return StartCRC;
-}
 
 
 #if 0
@@ -187,6 +173,11 @@ struct TestCRCStruct {TestCRCStruct() {TestCRC();exit(0);}} GlobalTesCRC;
 
 void TestCRC()
 {
+  // This function is invoked from global object and _SSE_Version is global
+  // and can be initialized after this function. So we explicitly initialize
+  // it here to enable SSE support in Blake2sp.
+  _SSE_Version=GetSSEVersion();
+
   const uint FirstSize=300;
   byte b[FirstSize];
 
@@ -252,23 +243,38 @@ void TestCRC()
 
   const size_t BufSize=0x100000;
   byte *Buf=new byte[BufSize];
-  memset(Buf,0,BufSize);
+  GetRnd(Buf,BufSize);
 
   clock_t StartTime=clock();
   r32=0xffffffff;
-  const uint BufCount=5000;
+  const uint64 BufCount=5000;
   for (uint I=0;I<BufCount;I++)
     r32=CRC32(r32,Buf,BufSize);
   if (r32!=0) // Otherwise compiler optimizer removes CRC calculation.
-    mprintf(L"\nCRC32 speed: %d MB/s",BufCount*1000/(clock()-StartTime));
+    mprintf(L"\nCRC32 speed: %llu MB/s",BufCount*CLOCKS_PER_SEC/(clock()-StartTime));
+
+  StartTime=clock();
+  DataHash Hash;
+  Hash.Init(HASH_CRC32,MaxPoolThreads);
+  const uint64 BufCountMT=20000;
+  for (uint I=0;I<BufCountMT;I++)
+    Hash.Update(Buf,BufSize);
+  HashValue Result;
+  Hash.Result(&Result);
+  mprintf(L"\nCRC32 MT speed: %llu MB/s",BufCountMT*CLOCKS_PER_SEC/(clock()-StartTime));
+
+  StartTime=clock();
+  Hash.Init(HASH_BLAKE2,MaxPoolThreads);
+  for (uint I=0;I<BufCount;I++)
+    Hash.Update(Buf,BufSize);
+  Hash.Result(&Result);
+  mprintf(L"\nBlake2sp speed: %llu MB/s",BufCount*CLOCKS_PER_SEC/(clock()-StartTime));
 
   StartTime=clock();
   r64=0xffffffffffffffff;
   for (uint I=0;I<BufCount;I++)
     r64=CRC64(r64,Buf,BufSize);
   if (r64!=0) // Otherwise compiler optimizer removes CRC calculation.
-    mprintf(L"\nCRC64 speed: %d MB/s",BufCount*1000/(clock()-StartTime));
+    mprintf(L"\nCRC64 speed: %llu MB/s",BufCount*CLOCKS_PER_SEC/(clock()-StartTime));
 }
 #endif
-
-#endif