Put brotli on the FetchContent plan (Try 2) (dotnet#109707)

* Reapply "Put brotli on the FetchContent plan (dotnet#107166)" (dotnet#109108)

This reverts commit b780069.

* Don't double-install the brotli libs

* Don't try looking up locations for system brotli. Rely on the distro maintainer to set the paths correctly.

Fixes dotnet#109105

* Install brotli where we install the compression native lib so they're always side-by-side (and we don't need to add it to the Mono build)

* Brotli should get pulled from the same place as Compression.Native

* Fix looking for libbrotlicommon.a on disk to determine if we should use system brotli

Author: jkoritzinsky

eng/DotNetBuild.props

@@ -98,7 +98,7 @@
 
       <!-- Handle system libraries -->
       <UseSystemLibs Condition="'$(UseSystemLibs)' != ''">+$(UseSystemLibs)+</UseSystemLibs>
-      <InnerBuildArgs Condition="'$(PortableBuild)' != 'true'">$(InnerBuildArgs) --cmakeargs -DCLR_CMAKE_USE_SYSTEM_BROTLI=true</InnerBuildArgs>
+      <InnerBuildArgs Condition="$(UseSystemLibs.Contains('+brotli+'))">$(InnerBuildArgs) --cmakeargs -DCLR_CMAKE_USE_SYSTEM_BROTLI=true</InnerBuildArgs>
       <InnerBuildArgs Condition="$(UseSystemLibs.Contains('+libunwind+'))">$(InnerBuildArgs) --cmakeargs -DCLR_CMAKE_USE_SYSTEM_LIBUNWIND=true</InnerBuildArgs>
       <!-- TODO: llvm-libunwind -->
       <!-- TODO: LinuxTracepoints -->

src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets

@@ -30,6 +30,9 @@ The .NET Foundation licenses this file to you under the MIT license.
 
     <PropertyGroup>
       <UseSystemZlib Condition="'$(UseSystemZlib)' == '' and !Exists('$(IlcFrameworkNativePath)libz.a')">true</UseSystemZlib>
+      <!-- Use libbrotlicommon.a as the sentinel for the three brotli libs. -->
+      <UseSystemBrotli Condition="'$(UseSystemBrotli)' == '' and !Exists('$(IlcFrameworkNativePath)libbrotlicommon.a')">true</UseSystemBrotli>
+
       <FullRuntimeName>libRuntime.WorkstationGC</FullRuntimeName>
       <FullRuntimeName Condition="'$(ServerGarbageCollection)' == 'true' or '$(IlcLinkServerGC)' == 'true'">libRuntime.ServerGC</FullRuntimeName>
 
@@ -169,6 +172,14 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeLibrary Condition="'$(UseSystemZlib)' != 'true'" Include="$(IlcFrameworkNativePath)libz.a" />
     </ItemGroup>
 
+    <!-- brotli must be added after System.IO.Compression.Native and brotlicommon must be added last, order matters. -->
+    <ItemGroup Condition="'$(UseSystemBrotli)' != 'true'">
+      <NativeLibrary Include="$(IlcFrameworkNativePath)libbrotlienc.a" />
+      <NativeLibrary Include="$(IlcFrameworkNativePath)libbrotlidec.a" />
+      <NativeLibrary Include="$(IlcFrameworkNativePath)libbrotlicommon.a" />
+    </ItemGroup>
+
+
     <ItemGroup Condition="'$(StaticICULinking)' == 'true' and '$(NativeLib)' != 'Static' and '$(InvariantGlobalization)' != 'true'">
       <NativeLibrary Include="$(IntermediateOutputPath)libs/System.Globalization.Native/build/libSystem.Globalization.Native.a" />
       <DirectPInvoke Include="libSystem.Globalization.Native" />
@@ -182,11 +193,6 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeSystemLibrary Include="crypto" />
     </ItemGroup>
 
-    <ItemGroup Condition="'$(UseSystemBrotli)' != 'false' and Exists('$(IlcSdkPath)nonportable.txt')">
-      <NativeSystemLibrary Include="brotlienc" />
-      <NativeSystemLibrary Include="brotlidec" />
-    </ItemGroup>
-
     <ItemGroup Condition="'$(StaticOpenSslLinking)' == 'true' and '$(NativeLib)' != 'Static'">
       <NativeLibrary Include="$(IntermediateOutputPath)libs/System.Security.Cryptography.Native/build/libSystem.Security.Cryptography.Native.OpenSsl.a" />
       <DirectPInvoke Include="libSystem.Security.Cryptography.Native.OpenSsl" />
@@ -211,6 +217,7 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeSystemLibrary Include="swiftCore" Condition="'$(_IsApplePlatform)' == 'true'" />
       <NativeSystemLibrary Include="swiftFoundation" Condition="'$(_IsApplePlatform)' == 'true'" />
       <NativeSystemLibrary Include="z" Condition="'$(UseSystemZlib)' == 'true'" />
+      <NativeSystemLibrary Include="brotlienc;brotlidec;brotlicommon" Condition="'$(UseSystemBrotli)' == 'true'" />
       <NativeSystemLibrary Include="rt" Condition="'$(_IsApplePlatform)' != 'true' and '$(_linuxLibcFlavor)' != 'bionic'" />
       <NativeSystemLibrary Include="log" Condition="'$(_linuxLibcFlavor)' == 'bionic'" />
       <NativeSystemLibrary Include="icucore" Condition="'$(_IsApplePlatform)' == 'true'" />

src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Windows.targets

@@ -49,6 +49,9 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeLibrary Include="$(IlcSdkPath)$(StandaloneGCSupportName)$(LibrarySuffix)" />
       <NativeLibrary Include="$(IlcSdkPath)aotminipal$(LibFileExt)" />
       <NativeLibrary Include="$(IlcSdkPath)zlibstatic$(LibFileExt)" />
+      <NativeLibrary Include="$(IlcSdkPath)brotlicommon$(LibFileExt)" />
+      <NativeLibrary Include="$(IlcSdkPath)brotlienc$(LibFileExt)" />
+      <NativeLibrary Include="$(IlcSdkPath)brotlidec$(LibFileExt)" />
     </ItemGroup>
 
     <ItemGroup>

src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props

@@ -282,6 +282,13 @@
     <!-- zlib-specific files -->
     <PlatformManifestFileEntry Include="libz.a" IsNative="true" />
     <PlatformManifestFileEntry Include="zlibstatic.lib" IsNative="true" />
+    <!-- brotli-specific files -->
+    <PlatformManifestFileEntry Include="libbrotlienc.a" IsNative="true" />
+    <PlatformManifestFileEntry Include="libbrotlidec.a" IsNative="true" />
+    <PlatformManifestFileEntry Include="libbrotlicommon.a" IsNative="true" />
+    <PlatformManifestFileEntry Include="brotlienc.lib" IsNative="true" />
+    <PlatformManifestFileEntry Include="brotlidec.lib" IsNative="true" />
+    <PlatformManifestFileEntry Include="brotlicommon.lib" IsNative="true" />
   </ItemGroup>
 
  <ItemGroup>

src/mono/msbuild/android/build/AndroidBuild.targets

@@ -203,14 +203,15 @@
       <ProfiledAOTProfilePaths Include="$(MibcFilePath)" />
     </ItemGroup>
 
-    <!-- 
-      In order for the runtime to work when static linking, we must supply 
+    <!--
+      In order for the runtime to work when static linking, we must supply
       a list of direct pinvokes otherwise the runtime will crash
     -->
     <ItemGroup Condition="'$(_IsLibraryMode)' == 'true'">
       <DirectPInvokes Include="libSystem.Native" />
       <DirectPInvokes Include="libSystem.IO.Compression.Native" />
       <DirectPInvokes Include="libSystem.Security.Cryptography.Native.Android" />
+      <DirectPInvokes Include="libbrotlienc;libbrotlidec" />
     </ItemGroup>
 
     <PropertyGroup>

src/mono/msbuild/apple/build/AppleBuild.targets

@@ -10,20 +10,20 @@
   </PropertyGroup>
 
   <UsingTask Condition="'$(AppleGenerateAppBundle)' == 'true'"
-             TaskName="AppleAppBuilderTask" 
+             TaskName="AppleAppBuilderTask"
              AssemblyFile="$(AppleAppBuilderTasksAssemblyPath)" />
   <UsingTask Condition="'$(RunAOTCompilation)' == 'true'"
              TaskName="ILStrip"
              AssemblyFile="$(MonoTargetsTasksAssemblyPath)" />
-  <UsingTask TaskName="MonoTargetsTasks.MarshalingPInvokeScanner" 
+  <UsingTask TaskName="MonoTargetsTasks.MarshalingPInvokeScanner"
              AssemblyFile="$(MonoTargetsTasksAssemblyPath)" />
 
   <Import Condition="Exists('$(ILCompilerTargetsPath)') and '$(UseNativeAOTRuntime)' == 'true'"
           Project="$(ILCompilerTargetsPath)" />
   <Import Condition="Exists('$(ILLinkTargetsPath)') and '$(UseNativeAOTRuntime)' == 'true'"
           Project="$(ILLinkTargetsPath)" />
 
-  <Target Name="_CleanPublish" 
+  <Target Name="_CleanPublish"
           BeforeTargets="Build">
     <RemoveDir Directories="$(PublishDir)" />
   </Target>
@@ -196,7 +196,7 @@
         <_IsNative>false</_IsNative>
       </_AssembliesToBundleInternal>
 
-      <_AotInputAssemblies Include="@(_AssembliesToBundleInternal)" 
+      <_AotInputAssemblies Include="@(_AssembliesToBundleInternal)"
                            Condition="'%(_AssembliesToBundleInternal._InternalForceInterpret)' != 'true'">
         <AotArguments>$(AotArguments)</AotArguments>
         <ProcessArguments>$(ProcessArguments)</ProcessArguments>
@@ -205,7 +205,7 @@
       <_AOT_InternalForceInterpretAssemblies Include="@(_AssembliesToBundleInternal->WithMetadataValue('_InternalForceInterpret', 'true'))" />
       <_AssembliesToBundleInternal Remove="@(_AssembliesToBundleInternal)" />
     </ItemGroup>
-    
+
     <MakeDir Directories="$(_MobileIntermediateOutputPath)" />
 
     <PropertyGroup Condition="'$(iOSLikeDedup)' == 'true'">
@@ -228,15 +228,16 @@
       <_ExcludeFromAppDir Include="$(_iOSLikeDedupAssembly)" />
     </ItemGroup>
 
-    <!-- 
-      In order for the runtime to work when static linking, we must supply 
+    <!--
+      In order for the runtime to work when static linking, we must supply
       a list of direct pinvokes otherwise the runtime will crash
     -->
     <ItemGroup Condition="'$(_IsLibraryMode)' == 'true'">
       <DirectPInvokes Include="libSystem.Native" />
       <DirectPInvokes Include="libSystem.IO.Compression.Native" />
       <DirectPInvokes Include="libSystem.Net.Security.Native" />
       <DirectPInvokes Include="libSystem.Security.Cryptography.Native.Apple" />
+      <DirectPInvokes Include="libbrotlienc;libbrotlidec" />
     </ItemGroup>
 
     <PropertyGroup>
@@ -274,7 +275,7 @@
   <Target Name="_AppleNativeAotCompile"
           DependsOnTargets="SetupProperties;ComputeIlcCompileInputs;IlcCompile;$(_IlcLibraryBuildDependsOn)" />
 
-  <Target Name="_AppleGenerateAppBundle" 
+  <Target Name="_AppleGenerateAppBundle"
           Condition="'$(AppleGenerateAppBundle)' == 'true'"
           DependsOnTargets="_AppleGenerateRuntimeConfig">
     <!-- Run App bundler, it uses AOT libs (if needed), link all native bits, compile simple UI (written in ObjC)
@@ -336,7 +337,7 @@
   </Target>
 
   <Target Name="_AfterAppleBuild">
-    
+
   </Target>
 
   <Target Name="_AppleGenerateRuntimeConfig"
@@ -355,4 +356,4 @@
         RuntimeConfigReservedProperties="@(_RuntimeConfigReservedProperties)">
     </RuntimeConfigParserTask>
   </Target>
-</Project>
+</Project>

src/native/corehost/apphost/static/CMakeLists.txt

@@ -153,6 +153,10 @@ if(CLR_CMAKE_TARGET_WIN32)
         delayimp.lib
     )
 
+    # additional requirements for System.IO.Compression.Native
+    include(${CLR_SRC_NATIVE_DIR}/libs/System.IO.Compression.Native/extra_libs.cmake)
+    append_extra_compression_libs(NATIVE_LIBS)
+
     set(RUNTIMEINFO_LIB runtimeinfo)
 
 else()

src/native/external/brotli-version.txt

@@ -1,6 +1,5 @@
 1.1.0
 https://github.com/google/brotli/releases/tag/v1.1.0
 
-Copy the c/dec, c/enc, c/common, and c/include folders into the root and remove all other files.
-
-Apply https://github.com/google/brotli/commit/85d88cbfc2b742e0742286ec277b73bdbf7be433.patch
+Deleted tests, python, docs folders
+Apply https://github.com/google/brotli/commit/85d88cbfc2b742e0742286ec277b73bdbf7be433.patch

src/native/external/brotli.cmake

@@ -1,39 +1,32 @@
 # IMPORTANT: do not use add_compile_options(), add_definitions() or similar functions here since it will leak to the including projects
 
-include_directories(BEFORE "${CMAKE_CURRENT_LIST_DIR}/brotli/include")
+include(FetchContent)
 
-set (BROTLI_SOURCES_BASE
-    common/constants.c
-    common/context.c
-    common/dictionary.c
-    common/platform.c
-    common/shared_dictionary.c
-    common/transform.c
-    dec/bit_reader.c
-    dec/decode.c
-    dec/huffman.c
-    dec/state.c
-    enc/backward_references.c
-    enc/backward_references_hq.c
-    enc/bit_cost.c
-    enc/block_splitter.c
-    enc/brotli_bit_stream.c
-    enc/cluster.c
-    enc/command.c
-    enc/compound_dictionary.c
-    enc/compress_fragment.c
-    enc/compress_fragment_two_pass.c
-    enc/dictionary_hash.c
-    enc/encode.c
-    enc/encoder_dict.c
-    enc/entropy_encode.c
-    enc/fast_log.c
-    enc/histogram.c
-    enc/literal_cost.c
-    enc/memory.c
-    enc/metablock.c
-    enc/static_dict.c
-    enc/utf8_util.c
+FetchContent_Declare(
+    brotli
+    SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/brotli
 )
 
-addprefix(BROTLI_SOURCES "${CMAKE_CURRENT_LIST_DIR}/brotli" "${BROTLI_SOURCES_BASE}")
+set(BROTLI_DISABLE_TESTS ON)
+set(__CURRENT_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+set(BUILD_SHARED_LIBS OFF)
+FetchContent_MakeAvailable(brotli)
+set(BUILD_SHARED_LIBS ${__CURRENT_BUILD_SHARED_LIBS})
+
+target_compile_options(brotlicommon PRIVATE $<$<COMPILE_LANG_AND_ID:C,MSVC>:/guard:cf>)
+target_compile_options(brotlienc PRIVATE $<$<COMPILE_LANG_AND_ID:C,MSVC>:/guard:cf>)
+target_compile_options(brotlidec PRIVATE $<$<COMPILE_LANG_AND_ID:C,MSVC>:/guard:cf>)
+target_compile_options(brotlienc PRIVATE $<$<COMPILE_LANG_AND_ID:C,Clang,AppleClang,GNU>:-Wno-implicit-fallthrough>)
+target_compile_options(brotlidec PRIVATE $<$<COMPILE_LANG_AND_ID:C,Clang,AppleClang,GNU>:-Wno-implicit-fallthrough>)
+
+# Even though we aren't building brotli as shared libraries, we still need to be able to export the symbols
+# from the brotli libraries so that they can be used by System.IO.Compression.
+# Since we link all of the static libraries into a single shared library, we need to define BROTLICOMMON_SHARED_COMPILATION
+# for all targets so brotlienc and brotlidec don't expect to link against a separate brotlicommon shared library.
+target_compile_definitions(brotlienc PRIVATE BROTLI_SHARED_COMPILATION BROTLIENC_SHARED_COMPILATION BROTLICOMMON_SHARED_COMPILATION)
+target_compile_definitions(brotlidec PRIVATE BROTLI_SHARED_COMPILATION BROTLIDEC_SHARED_COMPILATION BROTLICOMMON_SHARED_COMPILATION)
+target_compile_definitions(brotlicommon PRIVATE BROTLI_SHARED_COMPILATION BROTLICOMMON_SHARED_COMPILATION)
+
+# Don't build the brotli command line tool unless explicitly requested
+# (which we never do)
+set_target_properties(brotli PROPERTIES EXCLUDE_FROM_ALL ON)