Start to bazel rules for curl

- mostly generated with configure2bazel
- Fix IDN configure flags and update overlay
- Add macos and linux arm overlays.
- generates working curl library of the right size for macos
  cc toolchain is injectined a link arg that doesn't work.
```
/root/.cache/bazel/_bazel_root/81a15fa9a2846e82038a778136785275/external/gcc_toolchain++gcc_toolchains+gcc_toolchain_aarch64/bin/aarch64-unknown-linux-gnu-ld.bfd: unrecognized option '--start-lib'
```

There is some real ugly stuff going on to try to use the output of configure_make
as shared libarries.
- copying the ssl headers to a new place so we can make a cc_library
  This is needed because the `includes` attribute on cc_import doesn't work.
  We need to create a real cc_library to depend on to get that right.
- the cc_library / cc_shared_library pair for everything
  Again, cc_import doesn't seem to work

Look at /opt/datadog-agent on local laptop
```
-rwxr-xr-x  1 root  wheel  4815040 Dec  9 14:57 /opt/datadog-agent/embedded/lib/libcrypto.3.dylib
-rwxr-xr-x  1 root  wheel  4857760 Dec  9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcrypto.3.dylib
-rwxr-xr-x  1 root  wheel  4857760 Dec  9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/psycopg_c/.dylibs/libcrypto.3.dylib
-rwxr-xr-x  1 root  wheel  965408 Dec  9 14:57 /opt/datadog-agent/embedded/lib/libssl.3.dylib
-rwxr-xr-x  1 root  wheel  946896 Dec  9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libssl.3.dylib
-rwxr-xr-x  1 root  wheel  946896 Dec  9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/psycopg_c/.dylibs/libssl.3.dylib
-rwxr-xr-x  1 root  wheel  731904 Dec  9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcurl.4.dylib
```

```
 ls -l bazel-bin/external/+_repo_rules+curl/libcurl.dylib bazel-bin/external/+_repo_rules+openssl/libimported_*dylib
-r-xr-xr-x  1 tony.aiuto  wheel   803968 Jan 13 23:02 bazel-bin/external/+_repo_rules+curl/libcurl.dylib
-r-xr-xr-x  1 tony.aiuto  wheel  4839472 Jan 13 16:54 bazel-bin/external/+_repo_rules+openssl/libimported_crypto_shared.dylib
-r-xr-xr-x  1 tony.aiuto  wheel   949008 Jan 13 22:06 bazel-bin/external/+_repo_rules+openssl/libimported_ssl_shared.dylib

Note the similar sizes. Drift is probably because of nghttp2.

Looking at the linking
```
$ otool -L /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcurl.4.dylib
/opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcurl.4.dylib:
	/DLC/confluent_kafka/.dylibs/libcurl.4.dylib (compatibility version 13.0.0, current version 13.0.0)
	@loader_path/libssl.3.dylib (compatibility version 3.0.0, current version 3.0.0)
	@loader_path/libcrypto.3.dylib (compatibility version 3.0.0, current version 3.0.0)
	@loader_path/libz.1.3.1.dylib (compatibility version 1.0.0, current version 1.3.1)
	/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation ...
	/System/Library/Frameworks/CoreServices.framework/Versions/A/CoreServices ...
	/System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration ...
	/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1345.120.2)

$ otool -L bazel-bin/external/+_repo_rules+curl/libcurl.dylib
bazel-bin/external/+_repo_rules+curl/libcurl.dylib:
	@rpath/libcurl.dylib (compatibility version 0.0.0, current version 0.0.0)
	@rpath/nghttp2.so.14 (compatibility version 0.0.0, current version 0.0.0)
	@rpath/libimported_crypto_shared.dylib (compatibility version 0.0.0, current version 0.0.0)
	@rpath/libimported_ssl_shared.dylib (compatibility version 0.0.0, current version 0.0.0)
	@rpath/libz.dylib (compatibility version 0.0.0, current version 0.0.0)
	/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation ...
	/System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration ...
	/usr/lib/libc++.1.dylib ...
	/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1356.0.0)
```

The extra CoreServices in the other version is unexpected, but may be a fault in the way Kafka builds it.

checkpoint curl

got it back to linking

no time for jokes

maybe

readme

readme

tmp

finally

delete some apis we do not need

comments

comments

allow haxx.se in adms config

log the generated curl_config.h

split curl / curlu - does not help

Author: aiuto

.adms/bazel/adms.mirror.cfg

@@ -104,6 +104,7 @@ rewrite (mirrors.edge.kernel.org)/(.*) depot-read-api-bzl.us1.ddbuild.io/$1/$2
 rewrite (pyyaml.org)/(.*) depot-read-api-bzl.us1.ddbuild.io/$1/$2
 rewrite (python.org)/(.*) depot-read-api-bzl.us1.ddbuild.io/$1/$2
 rewrite (mirrors.kernel.org)/(.*) depot-read-api-bzl.us1.ddbuild.io/$1/$2
+rewrite (curl.haxx.se)/(.*) depot-read-api-bzl.us1.ddbuild.io/$1/$2
 
 # Rewrite Rules for External Hosts - Specific Language Ecosystems
 #######################################################################

deps/curl/overlay/curl.BUILD.bazel

@@ -16,6 +16,7 @@ load(
     "LIB_CURLTOOL_SRCS_LINUX_ARM64",
     "LIB_CURLU_SRCS_DARWIN_ARM64",
     "LIB_CURLU_SRCS_LINUX_ARM64",
+    "LIB_CURL_SRCS_LINUX_ARM64",
 )
 
 package(
@@ -53,27 +54,25 @@ PUBLIC_HEADERS = [
 ]
 
 LOCAL_DEFINES = [
-    "-DHAVE_CONFIG_H",
-]
+    "HAVE_CONFIG_H",
+    "CURL_DISABLE_DIGEST_AUTH",
+    "BSAES_ASM",
+    "ECP_NISTZ256_ASM",
+    "ECP_SM2P256_ASM",
+    "KECCAK1600_ASM",
+    "L_ENDIAN",
+    "OPENSSL_CPUID_OBJ",
+    "OPENSSL_PIC",
+    "SHA1_ASM",
+    "SHA256_ASM",
+    "SHA512_ASM",
+    "SM4_ASM",
+    "VPAES_ASM",
+    "VPSM4_ASM",
 
-# TODO: Bring this into LOCAL_DEFINES
-XLOCAL_DEFINES = [
-    "-DOPENSSLDIR='\"/opt/datadog/embedded/ssl/cacerts\"'",
-    "-DENGINESDIR='\"_/opt/datadog/embedded/lib/engines-3\"'",
-    "-DMODULESDIR='\"/opt/datadog/embedded/lib/ossl-modules\"'",
-    "-DBSAES_ASM",
-    "-DECP_NISTZ256_ASM",
-    "-DECP_SM2P256_ASM",
-    "-DKECCAK1600_ASM",
-    "-DOPENSSL_CPUID_OBJ",
-    "-DSHA1_ASM",
-    "-DSHA256_ASM",
-    "-DSHA512_ASM",
-    "-DSM4_ASM",
-    "-DVPAES_ASM",
-    "-DVPSM4_ASM",
-    "-DL_ENDIAN",
-    "-DOPENSSL_PIC",
+    #"OPENSSLDIR='\"/opt/datadog/embedded/ssl/cacerts\"'",
+    #"ENGINESDIR='\"_/opt/datadog/embedded/lib/engines-3\"'",
+    #"MODULESDIR='\"/opt/datadog/embedded/lib/ossl-modules\"'",
 ]
 
 ARCH_HDRS_COND = {
@@ -87,13 +86,13 @@ cc_binary(
         "@platforms//os:linux": LIB_CURLTOOL_SRCS_LINUX_ARM64,
         "@platforms//os:macos": LIB_CURLTOOL_SRCS_DARWIN_ARM64,
     }),
-    defines = ["HAVE_CONFIG_H"],
+    defines = LOCAL_DEFINES,
     includes = [
         ".",
         "include",
         "lib",
     ],
-    copts = LOCAL_DEFINES + [
+    copts = [
         "-I.",
     ],
     dynamic_deps = [
@@ -121,6 +120,57 @@ cc_binary(
 
 cc_library(
     name = "curl_static",
+    srcs = ["lib/curl_config.h"] + HDRS + select(ARCH_HDRS_COND) + select({
+        "@platforms//os:linux": LIB_CURL_SRCS_LINUX_ARM64,
+        "@platforms//os:macos": LIB_CURLU_SRCS_DARWIN_ARM64,
+    }) + ["@openssl//:headers"],
+    hdrs = PUBLIC_HEADERS,
+    # Using includes because -I just does not work with bzlmod.
+    # But yuch. What we need is to  port local_incudes concept from Google out
+    # to rules_cc.
+    includes = [
+        "include",
+    ],
+    local_defines = LOCAL_DEFINES + [
+        "BUILDING_LIBCURL",
+        "NDEBUG",
+        "CURL_STATICLIB",
+    ] + select({
+        "@platforms//os:macos": [
+            "CURL_MACOS_CALL_COPYPROXIES",
+        ],
+        "//conditions:default": [],
+    }),
+    copts = [
+        "-Wno-implicit-function-declaration",
+        "-Wno-int-conversion",
+        "-Wno-system-headers",
+        "-I.",
+        "-O2",
+        # "-fvisibility=hidden",
+    ],
+    linkopts = select({
+        "@platforms//os:macos": [
+            "-Wl,-framework,CoreFoundation",
+            "-Wl,-framework,SystemConfiguration",
+        ],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+    deps = [
+        "@nghttp2//:nghttp2",
+        "@openssl//:openssl",
+        "@zlib//:zlib",
+    ],
+    # Do not build in ... expansion. It triggers an incompatiblity between
+    # the toolchain decls, rules_cc, and the compiler we use.  rules_cc
+    # ends up using --start-lib/--end-lib options, which are not recognize
+    # by the compiler we use as of 2026-02-03.
+    tags = ["manual"],
+)
+
+cc_library(
+    name = "curlu_static",
     srcs = ["lib/curl_config.h"] + HDRS + select(ARCH_HDRS_COND) + select({
         "@platforms//os:linux": LIB_CURLU_SRCS_LINUX_ARM64,
         "@platforms//os:macos": LIB_CURLU_SRCS_DARWIN_ARM64,
@@ -132,10 +182,10 @@ cc_library(
     includes = [
         "include",
     ],
-    local_defines = [
+    local_defines = LOCAL_DEFINES + [
         "BUILDING_LIBCURL",
-        "HAVE_CONFIG_H",
         "NDEBUG",
+        # "CURL_STATICLIB",
     ] + select({
         "@platforms//os:macos": [
             "CURL_MACOS_CALL_COPYPROXIES",
@@ -145,6 +195,7 @@ cc_library(
     copts = [
         "-Wno-implicit-function-declaration",
         "-Wno-int-conversion",
+        "-Wno-system-headers",
         "-I.",
         "-O2",
     ],
@@ -157,6 +208,7 @@ cc_library(
     }),
     visibility = ["//visibility:public"],
     deps = [
+        ":curl_static",
         "@nghttp2//:nghttp2",
         "@openssl//:openssl",
         "@zlib//:zlib",
@@ -172,6 +224,7 @@ cc_shared_library(
     name = "curl",
     deps = [
         ":curl_static",
+        ":curlu_static",
     ],
     dynamic_deps = [
         "@nghttp2//:nghttp2_shared",
@@ -180,6 +233,7 @@ cc_shared_library(
     ],
     user_link_flags = select({
         "@platforms//os:linux": [
+            "-fvisibility=hidden",
             "-Wl,-L/opt/datadog-agent/embedded/lib",
             "-Wl,-rpath=/opt/datadog-agent/embedded/lib",
         ],

deps/curl/overlay/lib_contents.bzl

@@ -695,6 +695,11 @@ LIB_CURLU_SRCS_LINUX_ARM64 = [
     "lib/uint-table.c",
     "lib/url.c",
     "lib/urlapi.c",
+    "lib/version.c",
+    "lib/ws.c",
+]
+
+LIB_CURL_SRCS_LINUX_ARM64 = [
     "lib/vauth/cleartext.c",
     "lib/vauth/cram.c",
     "lib/vauth/digest_sspi.c",
@@ -708,7 +713,6 @@ LIB_CURLU_SRCS_LINUX_ARM64 = [
     "lib/vauth/spnego_gssapi.c",
     "lib/vauth/spnego_sspi.c",
     "lib/vauth/vauth.c",
-    "lib/version.c",
     "lib/vquic/curl_ngtcp2.c",
     "lib/vquic/curl_osslq.c",
     "lib/vquic/curl_quiche.c",
@@ -732,7 +736,6 @@ LIB_CURLU_SRCS_LINUX_ARM64 = [
     "lib/vtls/vtls.c",
     "lib/vtls/wolfssl.c",
     "lib/vtls/x509asn1.c",
-    "lib/ws.c",
 ]
 
 # Textual headers (included .c files and other non-.h files)

deps/curl/overlay/linux_arm64/lib/curl_config.h

@@ -44,7 +44,6 @@
 
 /* to disable digest authentication */
 /* #undef CURL_DISABLE_DIGEST_AUTH */
-#define CURL_DISABLE_DIGEST_AUTH 1
 
 /* disable DoH */
 /* #undef CURL_DISABLE_DOH */
@@ -54,7 +53,6 @@
 
 /* disable form API */
 /* #undef CURL_DISABLE_FORM_API */
-#define CURL_DISABLE_FORM_API 1
 
 /* to disable FTP */
 /* #undef CURL_DISABLE_FTP */
@@ -64,7 +62,6 @@
 
 /* to disable Gopher */
 /* #undef CURL_DISABLE_GOPHER */
-#define CURL_DISABLE_GOPHER 1
 
 /* disable headers-api */
 /* #undef CURL_DISABLE_HEADERS_API */
@@ -80,7 +77,6 @@
 
 /* to disable IMAP */
 /* #undef CURL_DISABLE_IMAP */
-#define CURL_DISABLE_IMAP 1
 
 /* to disable IPFS */
 /* #undef CURL_DISABLE_IPFS */
@@ -111,7 +107,6 @@
 
 /* to disable NTLM support */
 /* #undef CURL_DISABLE_NTLM */
-#define CURL_DISABLE_NTLM 1
 
 /* if the OpenSSL configuration will not be loaded automatically */
 /* #undef CURL_DISABLE_OPENSSL_AUTO_LOAD_CONFIG */
@@ -643,6 +638,7 @@
 #define HAVE_STRING_H 1
 
 /* Define to 1 if you have the <stropts.h> header file. */
+// #define HAVE_STROPTS_H 1
 /* #undef HAVE_STROPTS_H */
 
 /* if struct sockaddr_storage is defined */

omnibus/config/software/curl.rb

@@ -62,6 +62,7 @@
   ]
   configure(*configure_options, env: env)
 
-  command "make -j #{workers}", env: env
-  command "make install"
+  command "make -j #{workers}", env: env,  :live_stream => Omnibus.logger.live_stream(:info)
+  command "echo === OMNI ENV; printenv | sort ; echo === OMNI CONFIG ; cat lib/curl_config.h", env: env, :live_stream => Omnibus.logger.live_stream(:info)
+  command "make install",  :live_stream => Omnibus.logger.live_stream(:info)
 end