Skip to content

Commit 12cc469

Browse files
aiutoaiuto
committed
Start to bazel rules for curl
- mostly generated with configure2bazel - Fix IDN configure flags and update overlay - Add macos and linux arm overlays. - generates working curl library of the right size for macos cc toolchain is injectined a link arg that doesn't work. ``` /root/.cache/bazel/_bazel_root/81a15fa9a2846e82038a778136785275/external/gcc_toolchain++gcc_toolchains+gcc_toolchain_aarch64/bin/aarch64-unknown-linux-gnu-ld.bfd: unrecognized option '--start-lib' ``` There is some real ugly stuff going on to try to use the output of configure_make as shared libarries. - copying the ssl headers to a new place so we can make a cc_library This is needed because the `includes` attribute on cc_import doesn't work. We need to create a real cc_library to depend on to get that right. - the cc_library / cc_shared_library pair for everything Again, cc_import doesn't seem to work Look at /opt/datadog-agent on local laptop ``` -rwxr-xr-x 1 root wheel 4815040 Dec 9 14:57 /opt/datadog-agent/embedded/lib/libcrypto.3.dylib -rwxr-xr-x 1 root wheel 4857760 Dec 9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcrypto.3.dylib -rwxr-xr-x 1 root wheel 4857760 Dec 9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/psycopg_c/.dylibs/libcrypto.3.dylib -rwxr-xr-x 1 root wheel 965408 Dec 9 14:57 /opt/datadog-agent/embedded/lib/libssl.3.dylib -rwxr-xr-x 1 root wheel 946896 Dec 9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libssl.3.dylib -rwxr-xr-x 1 root wheel 946896 Dec 9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/psycopg_c/.dylibs/libssl.3.dylib -rwxr-xr-x 1 root wheel 731904 Dec 9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcurl.4.dylib ``` ``` ls -l bazel-bin/external/+_repo_rules+curl/libcurl.dylib bazel-bin/external/+_repo_rules+openssl/libimported_*dylib -r-xr-xr-x 1 tony.aiuto wheel 803968 Jan 13 23:02 bazel-bin/external/+_repo_rules+curl/libcurl.dylib -r-xr-xr-x 1 tony.aiuto wheel 4839472 Jan 13 16:54 bazel-bin/external/+_repo_rules+openssl/libimported_crypto_shared.dylib -r-xr-xr-x 1 tony.aiuto wheel 949008 Jan 13 22:06 bazel-bin/external/+_repo_rules+openssl/libimported_ssl_shared.dylib Note the similar sizes. Drift is probably because of nghttp2. Looking at the linking ``` $ otool -L /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcurl.4.dylib /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcurl.4.dylib: /DLC/confluent_kafka/.dylibs/libcurl.4.dylib (compatibility version 13.0.0, current version 13.0.0) @loader_path/libssl.3.dylib (compatibility version 3.0.0, current version 3.0.0) @loader_path/libcrypto.3.dylib (compatibility version 3.0.0, current version 3.0.0) @loader_path/libz.1.3.1.dylib (compatibility version 1.0.0, current version 1.3.1) /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation ... /System/Library/Frameworks/CoreServices.framework/Versions/A/CoreServices ... /System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration ... /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1345.120.2) $ otool -L bazel-bin/external/+_repo_rules+curl/libcurl.dylib bazel-bin/external/+_repo_rules+curl/libcurl.dylib: @rpath/libcurl.dylib (compatibility version 0.0.0, current version 0.0.0) @rpath/nghttp2.so.14 (compatibility version 0.0.0, current version 0.0.0) @rpath/libimported_crypto_shared.dylib (compatibility version 0.0.0, current version 0.0.0) @rpath/libimported_ssl_shared.dylib (compatibility version 0.0.0, current version 0.0.0) @rpath/libz.dylib (compatibility version 0.0.0, current version 0.0.0) /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation ... /System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration ... /usr/lib/libc++.1.dylib ... /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1356.0.0) ``` The extra CoreServices in the other version is unexpected, but may be a fault in the way Kafka builds it. checkpoint curl got it back to linking no time for jokes maybe readme readme tmp finally
1 parent 79bc07c commit 12cc469

File tree

11 files changed

+3017
-1
lines changed

11 files changed

+3017
-1
lines changed

‎MODULE.bazel‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,9 @@ include("//deps/compile_policy:compile_policy.MODULE.bazel")
225225
# buildifier: leave-alone
226226
include("//deps/cpython:cpython.MODULE.bazel")
227227

228+
# buildifier: leave-alone
229+
include("//deps/curl:curl.MODULE.bazel")
230+
228231
# buildifier: leave-alone
229232
include("//deps/freetds:freetds.MODULE.bazel")
230233

‎deps/curl/BUILD.bazel‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
2+
load("//bazel/tools:generate_module_bazel.bzl", "generate_module_bazel")
3+
4+
VERSION = "8.16.0"
5+
6+
generate_module_bazel(
7+
name = "curl_module_bazel_new",
8+
out = "curl.MODULE.bazel.new",
9+
module = "curl",
10+
sha256 = "a21e20476e39eca5a4fc5cfb00acf84bbc1f5d8443ec3853ad14c26b3c85b970",
11+
strip_prefix = "curl-%s" % VERSION,
12+
tags = ["manual"],
13+
target_compatible_with = ["@platforms//os:linux"],
14+
url = "https://curl.haxx.se/download/curl-%s.tar.gz" % VERSION,
15+
)
16+
17+
diff_test(
18+
name = "module_file_up_to_date_test",
19+
file1 = ":curl.MODULE.bazel.new",
20+
file2 = "curl.MODULE.bazel",
21+
target_compatible_with = ["@platforms//os:linux"],
22+
)

‎deps/curl/Makefile‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Hints for how to update curl from first principles
2+
#
3+
# This is not part of any build process. Think of it as a readme with
4+
# executable parts.
5+
6+
# A tool to look at configure output and create parts to support bazel targets
7+
C2B=$(HOME)/ws/experimental/teams/agent-supply-chain/configure2bazel
8+
9+
PRISTINE=curl-8.18.0
10+
11+
next:
12+
echo Do both steps on each platform.
13+
echo Adjust overlay/overlay.BUILD.bazel
14+
echo Adjust overlay/lib/curl_config.h
15+
16+
# Run configure and grab the generated files to the overlay tree
17+
# Run make to create the library outputs
18+
macos_step1:
19+
python3 $(C2B)/add_configuration.py \
20+
--pristine_dir=$(PRISTINE) \
21+
--configure_options=config_opts.txt
22+
echo Now do: '(cd linux_aarch64 ; make)'
23+
echo Now do: '(cd darwin_arm64 ; make)'
24+
25+
26+
# Step 3
27+
# Do some reasoning about the generated config.h. For almost every project,
28+
# the file should be identical for x86 and arm within each OS.
29+
# Then look at config.h differences per OS. They should almost always be
30+
# very simliar. What might differ is the inclusion of packages which are
31+
# os specific. Basic stuff should be in both.
32+
# Look for HAVE_* settings which are wrong because configure can not find
33+
# the bazel built dependency. This happens often. Edit config.h as needed
34+
# to match the goal you want, rather than what configure finds.
35+
36+
# Analyze the built libraries to map library content back to the source files
37+
# in them. Save lists by library name and arch back to lib_contents.bzl
38+
macos_step3:
39+
python3 $(C2B)/analyze.py \
40+
--pristine_dir=$(PRISTINE) \
41+
--configured_dir=darwin_arm64 \
42+
--configured_name=darwin_arm64 \
43+
--overlay=overlay
44+
git diff overlay/lib_contents.bzl
45+
46+
# For linux, do the same as macos, but with different config names.
47+
48+
# Step 4
49+
# Create your cc_library() target. Use any of the other projects using
50+
# configure2bazel as an sample starting point.

‎deps/curl/config_opts.txt‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
--disable-manual
2+
--disable-debug
3+
--enable-optimize
4+
--disable-static
5+
--disable-ldap
6+
--disable-ldaps
7+
--disable-rtsp
8+
--enable-proxy
9+
--disable-dependency-tracking
10+
--enable-ipv6
11+
--without-applic-idn
12+
--without-libidn2
13+
--without-gnutls
14+
--without-librtmp
15+
--without-libssh2
16+
--without-libpsl
17+
--without-libuv
18+
--without-winidn
19+
--without-zsh-functions-dir
20+
--without-fish-functions-dir
21+
--with-ssl
22+
--with-zlib
23+
--with-nghttp2
24+
--disable-docs
25+
--disable-libcurl-option
26+
--disable-versioned-symbols
27+
--disable-libuv
28+
--disable-verbose
29+
--disable-progress-meter

‎deps/curl/curl.MODULE.bazel‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# This file is generated. Do not hand edit.
2+
3+
http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
4+
5+
http_archive(
6+
name = "curl",
7+
files = {
8+
"BUILD.bazel": "//deps/curl:overlay/curl.BUILD.bazel",
9+
"darwin_arm64/lib/curl_config.h": "//deps/curl:overlay/darwin_arm64/lib/curl_config.h",
10+
"lib/curl_config.h": "//deps/curl:overlay/lib/curl_config.h",
11+
"lib_contents.bzl": "//deps/curl:overlay/lib_contents.bzl",
12+
"linux_arm64/lib/curl_config.h": "//deps/curl:overlay/linux_arm64/lib/curl_config.h",
13+
},
14+
sha256 = "e9274a5f8ab5271c0e0e6762d2fce194d5f98acc568e4ce816845b2dcc0cf88f",
15+
strip_prefix = "curl-8.18.0",
16+
url = "https://curl.haxx.se/download/curl-8.18.0.tar.gz",
17+
)

‎deps/curl/overlay/curl.BUILD.bazel‎

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
""" Agent specific curl build."""
2+
3+
load("@@//bazel/rules:so_symlink.bzl", "so_symlink")
4+
load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
5+
load("@rules_cc//cc:cc_library.bzl", "cc_library")
6+
load("@rules_cc//cc:cc_shared_library.bzl", "cc_shared_library")
7+
load("@rules_license//rules:license.bzl", "license")
8+
load("@rules_pkg//pkg:install.bzl", "pkg_install")
9+
load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
10+
load(
11+
"lib_contents.bzl",
12+
"HDRS",
13+
"HDRS_DARWIN_ARM64",
14+
"HDRS_LINUX_ARM64",
15+
"LIB_CURLTOOL_SRCS_DARWIN_ARM64",
16+
"LIB_CURLTOOL_SRCS_LINUX_ARM64",
17+
"LIB_CURLU_SRCS_DARWIN_ARM64",
18+
"LIB_CURLU_SRCS_LINUX_ARM64",
19+
)
20+
21+
package(
22+
default_package_metadata = [":license"],
23+
default_visibility = ["@@//packages:__subpackages__"],
24+
)
25+
26+
VERSION = "8.16.0"
27+
28+
SO_VERSION = "4.8.0"
29+
30+
license(
31+
name = "license",
32+
# The "curl" license is a specially designated type.
33+
license_kinds = ["@rules_license//licenses/spdx:curl"],
34+
license_text = "COPYING",
35+
visibility = ["//visibility:public"],
36+
)
37+
38+
SONAME = "libcurl.so.%s" % VERSION
39+
40+
PUBLIC_HEADERS = [
41+
"include/curl/curl.h",
42+
"include/curl/curlver.h",
43+
"include/curl/easy.h",
44+
"include/curl/header.h",
45+
"include/curl/mprintf.h",
46+
"include/curl/multi.h",
47+
"include/curl/options.h",
48+
"include/curl/stdcheaders.h",
49+
"include/curl/system.h",
50+
"include/curl/typecheck-gcc.h",
51+
"include/curl/urlapi.h",
52+
"include/curl/websockets.h",
53+
]
54+
55+
LOCAL_DEFINES = [
56+
"-DHAVE_CONFIG_H",
57+
]
58+
59+
# TODO: Bring this into LOCAL_DEFINES
60+
XLOCAL_DEFINES = [
61+
"-DOPENSSLDIR='\"/opt/datadog/embedded/ssl/cacerts\"'",
62+
"-DENGINESDIR='\"_/opt/datadog/embedded/lib/engines-3\"'",
63+
"-DMODULESDIR='\"/opt/datadog/embedded/lib/ossl-modules\"'",
64+
"-DBSAES_ASM",
65+
"-DECP_NISTZ256_ASM",
66+
"-DECP_SM2P256_ASM",
67+
"-DKECCAK1600_ASM",
68+
"-DOPENSSL_CPUID_OBJ",
69+
"-DSHA1_ASM",
70+
"-DSHA256_ASM",
71+
"-DSHA512_ASM",
72+
"-DSM4_ASM",
73+
"-DVPAES_ASM",
74+
"-DVPSM4_ASM",
75+
"-DL_ENDIAN",
76+
"-DOPENSSL_PIC",
77+
]
78+
79+
ARCH_HDRS_COND = {
80+
"@platforms//os:linux": HDRS_LINUX_ARM64,
81+
"@platforms//os:macos": HDRS_DARWIN_ARM64,
82+
}
83+
84+
cc_binary(
85+
name = "curl_bin",
86+
srcs = ["lib/curl_config.h"] + HDRS + PUBLIC_HEADERS + select(ARCH_HDRS_COND) + select({
87+
"@platforms//os:linux": LIB_CURLTOOL_SRCS_LINUX_ARM64,
88+
"@platforms//os:macos": LIB_CURLTOOL_SRCS_DARWIN_ARM64,
89+
}),
90+
defines = ["HAVE_CONFIG_H"],
91+
includes = [
92+
".",
93+
"include",
94+
"lib",
95+
],
96+
copts = LOCAL_DEFINES + [
97+
"-I.",
98+
],
99+
dynamic_deps = [
100+
":curl",
101+
"@nghttp2//:nghttp2_shared",
102+
"@openssl//:openssl_shared",
103+
"@zlib//:z",
104+
],
105+
linkstatic = False,
106+
linkopts = select({
107+
"@platforms//os:linux": [
108+
"-Wl,-L/opt/datadog-agent/embedded/lib",
109+
"-Wl,-rpath=/opt/datadog-agent/embedded/lib",
110+
],
111+
"@platforms//os:macos": [
112+
"-Wl,-framework,CoreFoundation",
113+
"-Wl,-framework,SystemConfiguration",
114+
"-Wl,-L/opt/datadog-agent/embedded/lib",
115+
"-Wl,-rpath,/opt/datadog-agent/embedded/lib",
116+
],
117+
"//conditions:default": [],
118+
}),
119+
visibility = ["//visibility:public"],
120+
)
121+
122+
cc_library(
123+
name = "curl_static",
124+
srcs = ["lib/curl_config.h"] + HDRS + select(ARCH_HDRS_COND) + select({
125+
"@platforms//os:linux": LIB_CURLU_SRCS_LINUX_ARM64,
126+
"@platforms//os:macos": LIB_CURLU_SRCS_DARWIN_ARM64,
127+
}) + ["@openssl//:headers"],
128+
hdrs = PUBLIC_HEADERS,
129+
# Using includes because -I just does not work with bzlmod.
130+
# But yuch. What we need is to port local_incudes concept from Google out
131+
# to rules_cc.
132+
includes = [
133+
"include",
134+
],
135+
local_defines = [
136+
"BUILDING_LIBCURL",
137+
"HAVE_CONFIG_H",
138+
"NDEBUG",
139+
] + select({
140+
"@platforms//os:macos": [
141+
"CURL_MACOS_CALL_COPYPROXIES",
142+
],
143+
"//conditions:default": [],
144+
}),
145+
copts = [
146+
"-Wno-implicit-function-declaration",
147+
"-Wno-int-conversion",
148+
"-I.",
149+
"-O2",
150+
],
151+
linkopts = select({
152+
"@platforms//os:macos": [
153+
"-Wl,-framework,CoreFoundation",
154+
"-Wl,-framework,SystemConfiguration",
155+
],
156+
"//conditions:default": [],
157+
}),
158+
visibility = ["//visibility:public"],
159+
deps = [
160+
"@nghttp2//:nghttp2",
161+
"@openssl//:openssl",
162+
"@zlib//:zlib",
163+
],
164+
# Do not build in ... expansion. It triggers an incompatiblity between
165+
# the toolchain decls, rules_cc, and the compiler we use. rules_cc
166+
# ends up using --start-lib/--end-lib options, which are not recognize
167+
# by the compiler we use as of 2026-02-03.
168+
tags = ["manual"],
169+
)
170+
171+
cc_shared_library(
172+
name = "curl",
173+
deps = [
174+
":curl_static",
175+
],
176+
dynamic_deps = [
177+
"@nghttp2//:nghttp2_shared",
178+
"@openssl//:openssl_shared",
179+
"@zlib//:z",
180+
],
181+
user_link_flags = select({
182+
"@platforms//os:linux": [
183+
"-Wl,-L/opt/datadog-agent/embedded/lib",
184+
"-Wl,-rpath=/opt/datadog-agent/embedded/lib",
185+
],
186+
"@platforms//os:macos": [
187+
"-Wl,-L/opt/datadog-agent/embedded/lib",
188+
"-Wl,-rpath,/opt/datadog-agent/embedded/lib",
189+
],
190+
"//conditions:default": [],
191+
}),
192+
visibility = ["//visibility:public"],
193+
)
194+
195+
pkg_files(
196+
name = "bin_files",
197+
srcs = [
198+
":curl_bin",
199+
],
200+
attributes = pkg_attributes(mode = "0755"),
201+
renames = {
202+
"curl_bin": "curl",
203+
},
204+
prefix = "embedded/bin",
205+
)
206+
207+
so_symlink(
208+
name = "lib_files",
209+
src = ":curl",
210+
libname = "libcurl",
211+
version = SO_VERSION,
212+
prefix = "embedded",
213+
)
214+
215+
pkg_files(
216+
name = "hdr_files",
217+
srcs = PUBLIC_HEADERS,
218+
prefix = "embedded/include/curl",
219+
)
220+
221+
pkg_filegroup(
222+
name = "all_files",
223+
srcs = [
224+
":bin_files",
225+
":hdr_files",
226+
":lib_files",
227+
],
228+
)
229+
230+
pkg_install(
231+
name = "install",
232+
srcs = [":all_files"],
233+
)

0 commit comments

Comments
 (0)