Skip to content

Commit f7e2573

Browse files
committed
Start to bazel rules for curl
- mostly generated with configure2bazel - Fix IDN configure flags and update overlay - Add macos and linux arm overlays. XXX - splice it in to the smoke test - generates working curl library of the right size for macos - not working for linux yet. There is a problem creating curl_static, where the cc toolchain is injectined a link arg that doesn't work. ``` /root/.cache/bazel/_bazel_root/81a15fa9a2846e82038a778136785275/external/gcc_toolchain++gcc_toolchains+gcc_toolchain_aarch64/bin/aarch64-unknown-linux-gnu-ld.bfd: unrecognized option '--start-lib' ``` There is some real ugly stuff going on to try to use the output of configure_make as shared libarries. - copying the ssl headers to a new place so we can make a cc_library This is needed because the `includes` attribute on cc_import doesn't work. We need to create a real cc_library to depend on to get that right. - the cc_library / cc_shared_library pair for everything Again, cc_import doesn't seem to work Look at /opt/datadog-agent on local laptop ``` -rwxr-xr-x 1 root wheel 4815040 Dec 9 14:57 /opt/datadog-agent/embedded/lib/libcrypto.3.dylib -rwxr-xr-x 1 root wheel 4857760 Dec 9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcrypto.3.dylib -rwxr-xr-x 1 root wheel 4857760 Dec 9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/psycopg_c/.dylibs/libcrypto.3.dylib -rwxr-xr-x 1 root wheel 965408 Dec 9 14:57 /opt/datadog-agent/embedded/lib/libssl.3.dylib -rwxr-xr-x 1 root wheel 946896 Dec 9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libssl.3.dylib -rwxr-xr-x 1 root wheel 946896 Dec 9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/psycopg_c/.dylibs/libssl.3.dylib -rwxr-xr-x 1 root wheel 731904 Dec 9 14:57 /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcurl.4.dylib ``` ``` ls -l bazel-bin/external/+_repo_rules+curl/libcurl.dylib bazel-bin/external/+_repo_rules+openssl/libimported_*dylib -r-xr-xr-x 1 tony.aiuto wheel 803968 Jan 13 23:02 bazel-bin/external/+_repo_rules+curl/libcurl.dylib -r-xr-xr-x 1 tony.aiuto wheel 4839472 Jan 13 16:54 bazel-bin/external/+_repo_rules+openssl/libimported_crypto_shared.dylib -r-xr-xr-x 1 tony.aiuto wheel 949008 Jan 13 22:06 bazel-bin/external/+_repo_rules+openssl/libimported_ssl_shared.dylib Note the similar sizes. Drift is probably because of nghttp2. Looking at the linking ``` $ otool -L /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcurl.4.dylib /opt/datadog-agent/embedded/lib/python3.13/site-packages/confluent_kafka/.dylibs/libcurl.4.dylib: /DLC/confluent_kafka/.dylibs/libcurl.4.dylib (compatibility version 13.0.0, current version 13.0.0) @loader_path/libssl.3.dylib (compatibility version 3.0.0, current version 3.0.0) @loader_path/libcrypto.3.dylib (compatibility version 3.0.0, current version 3.0.0) @loader_path/libz.1.3.1.dylib (compatibility version 1.0.0, current version 1.3.1) /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation ... /System/Library/Frameworks/CoreServices.framework/Versions/A/CoreServices ... /System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration ... /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1345.120.2) $ otool -L bazel-bin/external/+_repo_rules+curl/libcurl.dylib bazel-bin/external/+_repo_rules+curl/libcurl.dylib: @rpath/libcurl.dylib (compatibility version 0.0.0, current version 0.0.0) @rpath/nghttp2.so.14 (compatibility version 0.0.0, current version 0.0.0) @rpath/libimported_crypto_shared.dylib (compatibility version 0.0.0, current version 0.0.0) @rpath/libimported_ssl_shared.dylib (compatibility version 0.0.0, current version 0.0.0) @rpath/libz.dylib (compatibility version 0.0.0, current version 0.0.0) /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation ... /System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration ... /usr/lib/libc++.1.dylib ... /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1356.0.0) ``` The extra CoreServices in the other version is unexpected, but may be a fault in the way Kafka builds it.
1 parent 8039d49 commit f7e2573

File tree

11 files changed

+2998
-0
lines changed

11 files changed

+2998
-0
lines changed

MODULE.bazel

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,9 @@ include("//deps/compile_policy:compile_policy.MODULE.bazel")
226226
# buildifier: leave-alone
227227
include("//deps/cpython:cpython.MODULE.bazel")
228228

229+
# buildifier: leave-alone
230+
include("//deps/curl:curl.MODULE.bazel")
231+
229232
# buildifier: leave-alone
230233
include("//deps/freetds:freetds.MODULE.bazel")
231234

deps/BUILD.bazel

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ filegroup(
2727
"@lua//:liblua",
2828
"@nghttp2",
2929
],
30+
"//:macos_arm64": [
31+
"@curl",
32+
"@nghttp2",
33+
"@openssl3//:libcrypto",
34+
],
3035
"@platforms//os:windows": [
3136
"@cpython//:python_win",
3237
],

deps/curl/BUILD.bazel

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
2+
load("//bazel/tools:generate_module_bazel.bzl", "generate_module_bazel")
3+
4+
VERSION = "8.16.0"
5+
6+
generate_module_bazel(
7+
name = "curl_module_bazel_new",
8+
out = "curl.MODULE.bazel.new",
9+
module = "curl",
10+
sha256 = "a21e20476e39eca5a4fc5cfb00acf84bbc1f5d8443ec3853ad14c26b3c85b970",
11+
strip_prefix = "curl-%s" % VERSION,
12+
tags = ["manual"],
13+
target_compatible_with = ["@platforms//os:linux"],
14+
url = "https://curl.haxx.se/download/curl-%s.tar.gz" % VERSION,
15+
)
16+
17+
diff_test(
18+
name = "module_file_up_to_date_test",
19+
file1 = ":curl.MODULE.bazel.new",
20+
file2 = "curl.MODULE.bazel",
21+
target_compatible_with = ["@platforms//os:linux"],
22+
)

deps/curl/Makefile

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
C2B=$(HOME)/ws/experimental/teams/agent-supply-chain/configure2bazel
2+
PRISTINE=curl-8.18.0
3+
4+
next:
5+
echo Do both steps on each platform.
6+
echo Adjust overlay/overlay.BUILD.bazel
7+
echo Adjust overlay/lib/curl_config.h
8+
9+
macos1:
10+
python3 add_configuration.py \
11+
--pristine_dir=$(PRISTINE) \
12+
--configure_options=config_opts.txt
13+
echo Maybe: cd linux_aarch64 ; make
14+
echo Maybe: cd darwin_arm64 ; make
15+
16+
macos2:
17+
python3 $(C2B)/analyze.py \
18+
--pristine_dir=$(PRISTINE) \
19+
--configured_dir=darwin_arm64 \
20+
--configured_name=darwin_arm64 \
21+
--overlay=overlay
22+
git diff overlay/lib_contents.bzl
23+
24+
linux1:
25+
python3 add_configuration.py \
26+
--pristine_dir=$(PRISTINE) \
27+
--configured_name=linux_arm64 \
28+
--configure_options=config_opts.txt
29+
30+
linux2:
31+
python3 /tmp/analyze.py \
32+
--pristine_dir=$(PRISTINE) \
33+
--configured_dir=linux_arm64 \
34+
--configured_name=linux_arm64 \
35+
--overlay=overlay
36+
git diff overlay/lib_contents.bzl

deps/curl/config_opts.txt

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
--disable-manual
2+
--disable-debug
3+
--enable-optimize
4+
--disable-static
5+
--disable-ldap
6+
--disable-ldaps
7+
--disable-rtsp
8+
--enable-proxy
9+
--disable-dependency-tracking
10+
--enable-ipv6
11+
--without-applic-idn
12+
--without-libidn2
13+
--without-gnutls
14+
--without-librtmp
15+
--without-libssh2
16+
--without-libpsl
17+
--without-libuv
18+
--without-winidn
19+
--without-zsh-functions-dir
20+
--without-fish-functions-dir
21+
--with-ssl
22+
--with-zlib
23+
--with-nghttp2
24+
--disable-docs
25+
--disable-libcurl-option
26+
--disable-versioned-symbols
27+
--disable-libuv
28+
--disable-verbose
29+
--disable-progress-meter

deps/curl/curl.MODULE.bazel

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# This file is generated. Do not hand edit.
2+
3+
http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
4+
5+
http_archive(
6+
name = "curl",
7+
files = {
8+
"BUILD.bazel": "//deps/curl:overlay/overlay.BUILD.bazel",
9+
"darwin_arm64/lib/curl_config.h": "//deps/curl:overlay/darwin_arm64/lib/curl_config.h",
10+
"lib/curl_config.h": "//deps/curl:overlay/lib/curl_config.h",
11+
"lib_contents.bzl": "//deps/curl:overlay/lib_contents.bzl",
12+
"linux_arm64/lib/curl_config.h": "//deps/curl:overlay/linux_arm64/lib/curl_config.h",
13+
},
14+
sha256 = "e9274a5f8ab5271c0e0e6762d2fce194d5f98acc568e4ce816845b2dcc0cf88f",
15+
strip_prefix = "curl-8.18.0",
16+
url = "https://curl.haxx.se/download/curl-8.18.0.tar.gz",
17+
)

deps/curl/overlay/curl.BUILD.bazel

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
""" Agent specific curl build."""
2+
3+
load("@@//bazel/rules:so_symlink.bzl", "so_symlink")
4+
5+
# load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
6+
load("@rules_cc//cc:cc_binary.bzl", "cc_binary")
7+
load("@rules_cc//cc:cc_library.bzl", "cc_library")
8+
load("@rules_cc//cc:cc_shared_library.bzl", "cc_shared_library")
9+
load("@rules_license//rules:license.bzl", "license")
10+
load("@rules_pkg//pkg:install.bzl", "pkg_install")
11+
load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files")
12+
load(
13+
"lib_contents.bzl",
14+
"HDRS",
15+
"HDRS_DARWIN_ARM64",
16+
"HDRS_LINUX_ARM64",
17+
"LIB_CURLTOOL_SRCS_DARWIN_ARM64",
18+
"LIB_CURLTOOL_SRCS_LINUX_ARM64",
19+
"LIB_CURLU_SRCS_DARWIN_ARM64",
20+
"LIB_CURLU_SRCS_LINUX_ARM64",
21+
)
22+
23+
SO_VERSION = "4.8.0"
24+
25+
package(
26+
default_package_metadata = [":license"],
27+
default_visibility = ["@@//packages:__subpackages__"],
28+
)
29+
30+
license(
31+
name = "license",
32+
# The "curl" license is a specially designated type.
33+
license_kinds = ["@rules_license//licenses/spdx:curl"],
34+
license_text = "COPYING",
35+
visibility = ["//visibility:public"],
36+
)
37+
38+
VERSION = "8.16.0"
39+
SONAME = "libcurl.so.%s" % VERSION
40+
41+
PUBLIC_HEADERS = [
42+
"include/curl/curl.h",
43+
"include/curl/curlver.h",
44+
"include/curl/easy.h",
45+
"include/curl/header.h",
46+
"include/curl/mprintf.h",
47+
"include/curl/multi.h",
48+
"include/curl/options.h",
49+
"include/curl/stdcheaders.h",
50+
"include/curl/system.h",
51+
"include/curl/typecheck-gcc.h",
52+
"include/curl/urlapi.h",
53+
"include/curl/websockets.h",
54+
]
55+
56+
LOCAL_DEFINES = [
57+
"-DHAVE_CONFIG_H",
58+
]
59+
60+
# TODO: Bring this into LOCAL_DEFINES
61+
XLOCAL_DEFINES = [
62+
"-DOPENSSLDIR='\"/opt/datadog/embedded/ssl/cacerts\"'",
63+
"-DENGINESDIR='\"_/opt/datadog/embedded/lib/engines-3\"'",
64+
"-DMODULESDIR='\"/opt/datadog/embedded/lib/ossl-modules\"'",
65+
"-DBSAES_ASM",
66+
"-DECP_NISTZ256_ASM",
67+
"-DECP_SM2P256_ASM",
68+
"-DKECCAK1600_ASM",
69+
"-DOPENSSL_CPUID_OBJ",
70+
"-DSHA1_ASM",
71+
"-DSHA256_ASM",
72+
"-DSHA512_ASM",
73+
"-DSM4_ASM",
74+
"-DVPAES_ASM",
75+
"-DVPSM4_ASM",
76+
"-DL_ENDIAN",
77+
"-DOPENSSL_PIC",
78+
]
79+
80+
ARCH_HDRS_COND = {
81+
"@platforms//os:linux": HDRS_LINUX_ARM64,
82+
"@platforms//os:macos": HDRS_DARWIN_ARM64,
83+
}
84+
85+
cc_binary(
86+
name = "curl_bin",
87+
srcs = ["lib/curl_config.h"] + HDRS + PUBLIC_HEADERS + select(ARCH_HDRS_COND) + select({
88+
"@platforms//os:linux": LIB_CURLTOOL_SRCS_LINUX_ARM64,
89+
"@platforms//os:macos": LIB_CURLTOOL_SRCS_DARWIN_ARM64,
90+
}),
91+
defines = ["HAVE_CONFIG_H"],
92+
includes = [
93+
".",
94+
"include",
95+
"lib",
96+
],
97+
copts = LOCAL_DEFINES + [
98+
"-I.",
99+
],
100+
dynamic_deps = ["curl"],
101+
linkstatic = False,
102+
linkopts = select({
103+
"@platforms//os:linux": [
104+
"-Wl,-L/opt/datadog-agent/embedded/lib",
105+
"-Wl,-rpath=/opt/datadog-agent/embedded/lib",
106+
# "-Wl,-lnghttp2",
107+
#"-Wl,-lz",
108+
],
109+
"@platforms//os:macos": [
110+
"-Wl,-framework,CoreFoundation",
111+
"-Wl,-framework,SystemConfiguration",
112+
# "-Wl,-rpath=/opt/datadog-agent/embedded/lib",
113+
],
114+
"//conditions:default": [],
115+
}),
116+
visibility = ["//visibility:public"],
117+
)
118+
119+
cc_library(
120+
name = "curl_static",
121+
srcs = ["lib/curl_config.h"] + HDRS + select(ARCH_HDRS_COND) + select({
122+
"@platforms//os:linux": LIB_CURLU_SRCS_LINUX_ARM64,
123+
"@platforms//os:macos": LIB_CURLU_SRCS_DARWIN_ARM64,
124+
}) + ["@openssl//:headers"],
125+
hdrs = PUBLIC_HEADERS,
126+
# Using includes because -I just does not work with bzlmod.
127+
# But yuch. What we need is to port local_incudes concept from Google out
128+
# to rules_cc.
129+
includes = [
130+
"include",
131+
],
132+
local_defines = [
133+
"BUILDING_LIBCURL",
134+
"HAVE_CONFIG_H",
135+
"NDEBUG",
136+
] + select({
137+
"@platforms//os:macos": [
138+
"CURL_MACOS_CALL_COPYPROXIES",
139+
],
140+
"//conditions:default": [],
141+
}),
142+
copts = [
143+
"-Wno-implicit-function-declaration",
144+
"-Wno-int-conversion",
145+
"-I.",
146+
] + select({
147+
"@platforms//os:linux": [
148+
"-fvisibility=hidden",
149+
"-O2",
150+
],
151+
"@platforms//os:macos": [
152+
"-O2",
153+
],
154+
}),
155+
linkopts = select({
156+
"@platforms//os:macos": [
157+
"-Wl,-framework,CoreFoundation",
158+
"-Wl,-framework,SystemConfiguration",
159+
],
160+
"//conditions:default": [],
161+
}),
162+
visibility = ["//visibility:public"],
163+
deps = [
164+
"@nghttp2//:nghttp2",
165+
"@openssl//:crypto",
166+
"@openssl//:ssl",
167+
"@openssl//:imported_ssl_headers",
168+
"@zlib//:zlib",
169+
],
170+
)
171+
172+
cc_shared_library(
173+
name = "curl",
174+
deps = [
175+
":curl_static",
176+
],
177+
dynamic_deps = [
178+
"@nghttp2//:nghttp2_shared",
179+
"@openssl//:ssl_shared",
180+
"@openssl//:crypto_shared",
181+
"@zlib//:z",
182+
],
183+
visibility = ["//visibility:public"],
184+
)
185+
186+
pkg_files(
187+
name = "bin_files",
188+
srcs = [
189+
":curl_bin",
190+
],
191+
attributes = pkg_attributes(mode = "0755"),
192+
renames = {
193+
"curl_bin": "curl",
194+
},
195+
prefix = "embedded/bin",
196+
)
197+
198+
so_symlink(
199+
name = "lib_files",
200+
src = ":curl",
201+
libname = "libcurl",
202+
version = SO_VERSION,
203+
prefix = "embedded",
204+
)
205+
206+
pkg_files(
207+
name = "hdr_files",
208+
srcs = PUBLIC_HEADERS,
209+
prefix = "embedded/include/curl",
210+
)
211+
212+
pkg_filegroup(
213+
name = "all_files",
214+
srcs = [
215+
":bin_files",
216+
":hdr_files",
217+
":lib_files",
218+
],
219+
)
220+
221+
pkg_install(
222+
name = "install",
223+
srcs = [":all_files"],
224+
)

0 commit comments

Comments
 (0)