Skip to content

Commit d870a3e

Browse files
committed
feat: compile cuda plugin statically
The patch modifies how plugin loading works by allowing plugins to register themselves if their object file is statically linked against the criu binary. For now only the cuda plugin uses that since having it working is the goal of this patch series but the system works for multiple plugins. If we contribute the patch upstream we have to think about adding tests to it so that the criu test suite can validate the approach.
1 parent 15a4970 commit d870a3e

File tree

2 files changed

+264
-5
lines changed

2 files changed

+264
-5
lines changed

CMakeLists.txt

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,10 @@ get_target_property(LIBINTL_LDFLAGS libintl::static CONSUMER_LDFLAGS)
6969
set(CRIU_FEATURE_DEFINES "-DCONFIG_HAS_NFTABLES_LIB_API_1")
7070
set(CRIU_CFLAGS "${PROTOBUF_C_CFLAGS} ${CRIU_FEATURE_DEFINES} ${PROTOBUF_CFLAGS} ${LIBNET_CFLAGS} ${LIBNL_CFLAGS} ${LIBCAP_CFLAGS} ${LIBAIO_CFLAGS} ${ZLIB_CFLAGS} ${LIBNFTABLES_CFLAGS} ${LIBNFTNL_CFLAGS} ${LIBMNL_CFLAGS} ${LIBUUID_CFLAGS} ${LIBINTL_CFLAGS}")
7171
set(CRIU_LDFLAGS "-static ${PROTOBUF_C_LDFLAGS} ${PROTOBUF_LDFLAGS} ${LIBNET_LDFLAGS} ${LIBNL_LDFLAGS} ${LIBCAP_LDFLAGS} ${LIBAIO_LDFLAGS} ${ZLIB_LDFLAGS} ${LIBNFTABLES_LDFLAGS} ${LIBNFTNL_LDFLAGS} ${LIBMNL_LDFLAGS} ${LIBUUID_LDFLAGS} ${LIBINTL_LDFLAGS}")
72-
72+
set(CRIU_MAKE_PATH_ENV "${PROTOC_BIN_PATH}:${PROTOC_C_BIN_PATH}:$ENV{PATH}")
7373
message("-- criu CFLAGS ${CRIU_CFLAGS}")
7474
message("-- criu LDFLAGS ${CRIU_LDFLAGS}")
75+
message("-- criu PATH ${CRIU_MAKE_PATH_ENV}")
7576

7677
# CRIU build directories
7778
set(CRIU_BUILD_DIR ${CMAKE_BINARY_DIR}/criu-build)
@@ -93,7 +94,7 @@ ExternalProject_Add(criu
9394
URL ${DEP_criu_URL}
9495
URL_HASH SHA256=${DEP_criu_SHA256}
9596
UPDATE_DISCONNECTED 1
96-
PATCH_COMMAND patch -p1 -i ${CMAKE_SOURCE_DIR}/patch/criu-build.patch
97+
PATCH_COMMAND patch -p1 -i ${CMAKE_SOURCE_DIR}/patch/criu-build.patch && patch -p1 -i ${CMAKE_SOURCE_DIR}/patch/criu-static-plugin.patch
9798
CONFIGURE_COMMAND ""
9899
DOWNLOAD_DIR ${SOURCE_DOWNLOADS_DIR}
99100
DOWNLOAD_NAME ${DEP_criu_FILENAME}
@@ -105,11 +106,12 @@ ExternalProject_Add(criu
105106
"CFLAGS=${CRIU_CFLAGS}"
106107
"LDFLAGS=${CRIU_LDFLAGS}"
107108
"CC=${CMAKE_C_COMPILER}"
108-
"PATH=${PROTOC_BIN_PATH}:${PROTOC_C_BIN_PATH}:$ENV{PATH}"
109+
"PATH=${CRIU_MAKE_PATH_ENV}"
109110
"CUDA_PLUGIN_LIBCAP_CFLAGS=${LIBCAP_CFLAGS}"
110111
"SKIP_PIP_INSTALL=1"
111112
${CMAKE_MAKE_PROGRAM} -C <SOURCE_DIR> ${CRIU_MAKE_VARS} -j${CMAKE_BUILD_PARALLEL_LEVEL}
112113
CONFIG_AMDGPU=n
114+
STATIC_PLUGINS=y
113115
lib/c/built-in.o
114116
criu
115117
cuda_plugin
@@ -121,15 +123,15 @@ ExternalProject_Add(criu
121123
COMMAND ${CMAKE_COMMAND} -E copy <SOURCE_DIR>/criu/include/version.h ${CRIU_INSTALL_DIR}/include/criu/version.h
122124
COMMAND ${CMAKE_COMMAND} -E copy <SOURCE_DIR>/images/rpc.pb-c.h ${CRIU_INSTALL_DIR}/include/criu/rpc.pb-c.h
123125
COMMAND ${CMAKE_COMMAND} -E copy <SOURCE_DIR>/criu/criu ${CRIU_INSTALL_DIR}/bin/criu
124-
COMMAND ${CMAKE_COMMAND} -E copy <SOURCE_DIR>/plugins/cuda/cuda_plugin.so ${CRIU_INSTALL_DIR}/lib/criu-static-plugins/cuda_plugin.so
126+
COMMAND ${CMAKE_COMMAND} -E copy <SOURCE_DIR>/plugins/cuda/cuda_plugin.so ${CRIU_INSTALL_DIR}/lib/criu/cuda_plugin.so
125127
BINARY_DIR ${CRIU_BUILD_DIR}
126128
DEPENDS protobuf::static protobuf-c::static libnet::static libnl::static libcap::static libaio::static zlib::static libmnl::static libnftnl::static libnftables::static uuid::static libintl::static
127129
)
128130

129131
install(FILES ${CRIU_INSTALL_DIR}/lib/libcriu.o DESTINATION ${CMAKE_INSTALL_LIBDIR})
130132
install(PROGRAMS ${CRIU_INSTALL_DIR}/bin/criu DESTINATION ${CMAKE_INSTALL_BINDIR})
131133
install(DIRECTORY ${CRIU_INSTALL_DIR}/include/criu DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
132-
install(FILES ${CRIU_INSTALL_DIR}/lib/criu-static-plugins/cuda_plugin.so DESTINATION ${CMAKE_INSTALL_LIBDIR}/criu-static-plugins)
134+
install(FILES ${CRIU_INSTALL_DIR}/lib/criu/cuda_plugin.so DESTINATION ${CMAKE_INSTALL_LIBDIR}/criu)
133135

134136
install(DIRECTORY ${PROTOBUF_C_HEADER_DIR}/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
135137

patch/criu-static-plugin.patch

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
diff --git a/criu/Makefile b/criu/Makefile
2+
index 4cdb0c7d5..efb86b018 100644
3+
--- a/criu/Makefile
4+
+++ b/criu/Makefile
5+
@@ -19,6 +19,15 @@ CONFIG-DEFINES += -DGLOBAL_CONFIG_DIR='"/etc/criu/"'
6+
CONFIG-DEFINES += -DDEFAULT_CONFIG_FILENAME='"default.conf"'
7+
CONFIG-DEFINES += -DUSER_CONFIG_DIR='".criu/"'
8+
9+
+#
10+
+# Compile plugins statically if requested
11+
+ifeq ($(STATIC_PLUGINS),y)
12+
+ CONFIG-DEFINES += -DCONFIG_STATIC_PLUGINS
13+
+ STATIC_PLUGIN_OBJS += plugins/cuda/cuda_plugin.o
14+
+else
15+
+ STATIC_PLUGIN_OBJS :=
16+
+endif
17+
+
18+
#
19+
# General flags.
20+
CFLAGS += -fno-strict-aliasing
21+
@@ -66,11 +75,18 @@ PROGRAM-BUILTINS += $(obj)/built-in.o
22+
PROGRAM-BUILTINS += $(ARCH-LIB)
23+
PROGRAM-BUILTINS += soccr/libsoccr.a
24+
PROGRAM-BUILTINS += $(COMPEL_LIBS)
25+
+PROGRAM-BUILTINS += $(STATIC_PLUGIN_OBJS)
26+
27+
-$(obj)/built-in.o: pie
28+
+$(obj)/built-in.o: pie $(STATIC_PLUGIN_OBJS)
29+
$(Q) $(MAKE) $(call build-as,Makefile.crtools,criu) all
30+
31+
32+
+ifneq ($(STATIC_PLUGIN_OBJS),)
33+
+plugins/cuda/cuda_plugin.o:
34+
+ $(Q) $(MAKE) -C plugins/cuda cuda_plugin.o
35+
+endif
36+
+
37+
+
38+
$(obj)/Makefile: ;
39+
$(obj)/Makefile.crtools: ;
40+
$(obj)/Makefile.packages: ;
41+
diff --git a/criu/include/criu-plugin.h b/criu/include/criu-plugin.h
42+
index 392ea9f53..a01872a2d 100644
43+
--- a/criu/include/criu-plugin.h
44+
+++ b/criu/include/criu-plugin.h
45+
@@ -99,8 +99,25 @@ typedef struct {
46+
void *hooks[CR_PLUGIN_HOOK__MAX];
47+
} cr_plugin_desc_t;
48+
49+
-extern cr_plugin_desc_t CR_PLUGIN_DESC;
50+
51+
+#ifdef CONFIG_STATIC_PLUGINS
52+
+extern int cr_register_static_plugin_desc(cr_plugin_desc_t *desc);
53+
+#endif
54+
+
55+
+#ifdef CONFIG_STATIC_PLUGINS
56+
+#define CR_PLUGIN_REGISTER(___name, ___init, ___exit) \
57+
+ static cr_plugin_desc_t CR_PLUGIN_DESC = { \
58+
+ .name = ___name, \
59+
+ .init = ___init, \
60+
+ .exit = ___exit, \
61+
+ .version = CRIU_PLUGIN_VERSION, \
62+
+ .max_hooks = CR_PLUGIN_HOOK__MAX, \
63+
+ }; \
64+
+ static void __attribute__((constructor, used)) \
65+
+ register_static_plugin__ ## ___init(void) { \
66+
+ cr_register_static_plugin_desc(&CR_PLUGIN_DESC); \
67+
+ }
68+
+#else
69+
#define CR_PLUGIN_REGISTER(___name, ___init, ___exit) \
70+
cr_plugin_desc_t CR_PLUGIN_DESC = { \
71+
.name = ___name, \
72+
@@ -110,6 +127,9 @@ extern cr_plugin_desc_t CR_PLUGIN_DESC;
73+
.max_hooks = CR_PLUGIN_HOOK__MAX, \
74+
};
75+
76+
+extern cr_plugin_desc_t CR_PLUGIN_DESC;
77+
+#endif
78+
+
79+
static inline int cr_plugin_dummy_init(int stage)
80+
{
81+
return 0;
82+
diff --git a/criu/plugin.c b/criu/plugin.c
83+
index 65e79a069..56ec2bab8 100644
84+
--- a/criu/plugin.c
85+
+++ b/criu/plugin.c
86+
@@ -95,6 +95,77 @@ static int verify_plugin(cr_plugin_desc_t *d)
87+
return 0;
88+
}
89+
90+
+#ifdef CONFIG_STATIC_PLUGINS
91+
+static cr_plugin_desc_t **static_plugins = NULL;
92+
+static int num_static_plugins = 0;
93+
+static int max_static_plugins = 0;
94+
+
95+
+int cr_register_static_plugin_desc(cr_plugin_desc_t *desc)
96+
+{
97+
+ if (num_static_plugins >= max_static_plugins) {
98+
+ cr_plugin_desc_t **new_plugins;
99+
+ int new_max = max_static_plugins ? max_static_plugins * 2 : 4;
100+
+ new_plugins = xrealloc(static_plugins, new_max * sizeof(cr_plugin_desc_t *));
101+
+ if (!new_plugins) {
102+
+ pr_err("Failed to allocate memory for static plugins\n");
103+
+ return -1;
104+
+ }
105+
+ static_plugins = new_plugins;
106+
+ max_static_plugins = new_max;
107+
+ }
108+
+
109+
+ static_plugins[num_static_plugins++] = desc;
110+
+ return 0;
111+
+}
112+
+
113+
+static int cr_register_static_plugin(cr_plugin_desc_t *d, int stage)
114+
+{
115+
+ plugin_desc_t *this;
116+
+ size_t i;
117+
+
118+
+ this = xzalloc(sizeof(*this));
119+
+ if (!this)
120+
+ return -1;
121+
+
122+
+ if (verify_plugin(d)) {
123+
+ pr_err("Corrupted static plugin %s\n", d->name);
124+
+ goto err;
125+
+ }
126+
+
127+
+ this->d = d;
128+
+ this->dlhandle = NULL;
129+
+ INIT_LIST_HEAD(&this->list);
130+
+
131+
+ for (i = 0; i < d->max_hooks; i++)
132+
+ INIT_LIST_HEAD(&this->link[i]);
133+
+
134+
+ list_add_tail(&this->list, &cr_plugin_ctl.head);
135+
+ show_plugin_desc(d);
136+
+
137+
+ if (d->init && d->init(stage)) {
138+
+ pr_err("Failed in init(%d) of static plugin \"%s\"\n", stage, d->name);
139+
+ list_del(&this->list);
140+
+ goto err;
141+
+ }
142+
+
143+
+ /*
144+
+ * Chain hooks into appropriate places for
145+
+ * fast handler access.
146+
+ */
147+
+ for (i = 0; i < d->max_hooks; i++) {
148+
+ if (!d->hooks[i])
149+
+ continue;
150+
+ list_add_tail(&this->link[i], &cr_plugin_ctl.hook_chain[i]);
151+
+ }
152+
+
153+
+ return 0;
154+
+
155+
+err:
156+
+ xfree(this);
157+
+ return -1;
158+
+}
159+
+#endif /* STATIC_PLUGINS */
160+
+
161+
int criu_get_image_dir(void)
162+
{
163+
return get_service_fd(IMG_FD_OFF);
164+
@@ -195,7 +266,11 @@ void cr_plugin_fini(int stage, int ret)
165+
166+
if (this->d->version == CRIU_PLUGIN_VERSION_OLD)
167+
xfree(this->d);
168+
- dlclose(h);
169+
+
170+
+ if (h)
171+
+ dlclose(h);
172+
+
173+
+ xfree(this);
174+
}
175+
}
176+
177+
@@ -210,6 +285,16 @@ int cr_plugin_init(int stage)
178+
for (i = 0; i < ARRAY_SIZE(cr_plugin_ctl.hook_chain); i++)
179+
INIT_LIST_HEAD(&cr_plugin_ctl.hook_chain[i]);
180+
181+
+#ifdef CONFIG_STATIC_PLUGINS
182+
+ for (i = 0; i < num_static_plugins; i++) {
183+
+ if (cr_register_static_plugin(static_plugins[i], stage)) {
184+
+ pr_err("Failed to register static plugin %s\n",
185+
+ static_plugins[i]->name);
186+
+ return -1;
187+
+ }
188+
+ }
189+
+#endif
190+
+
191+
if (opts.libdir == NULL) {
192+
path = getenv("CRIU_LIBS_DIR");
193+
if (path)
194+
diff --git a/plugins/cuda/Makefile b/plugins/cuda/Makefile
195+
index 6ba0defe8..81959ba67 100644
196+
--- a/plugins/cuda/Makefile
197+
+++ b/plugins/cuda/Makefile
198+
@@ -1,7 +1,8 @@
199+
PLUGIN_NAME := cuda_plugin
200+
PLUGIN_SOBJ := cuda_plugin.so
201+
+PLUGIN_OBJ := cuda_plugin.o
202+
203+
-DEPS_CUDA := $(PLUGIN_SOBJ)
204+
+DEPS_CUDA := $(PLUGIN_SOBJ) $(PLUGIN_OBJ)
205+
206+
PLUGIN_INCLUDE := -iquote../../include
207+
PLUGIN_INCLUDE += -iquote../../criu/include
208+
@@ -11,20 +12,24 @@ PLUGIN_INCLUDE += $(CUDA_PLUGIN_LIBCAP_CFLAGS)
209+
210+
COMPEL := ../../compel/compel-host
211+
212+
-PLUGIN_CFLAGS := -g -Wall -Werror -shared -nostartfiles -fPIC
213+
+PLUGIN_CFLAGS := -g -Wall -Werror -fPIC
214+
215+
__nmk_dir ?= ../../scripts/nmk/scripts/
216+
include $(__nmk_dir)msg.mk
217+
218+
all: $(DEPS_CUDA)
219+
220+
+cuda_plugin.o: cuda_plugin.c
221+
+ $(call msg-gen, $@)
222+
+ $(Q) $(CC) $(PLUGIN_CFLAGS) -DCONFIG_STATIC_PLUGINS $(shell $(COMPEL) includes) -c $^ -o $@ $(PLUGIN_INCLUDE)
223+
+
224+
cuda_plugin.so: cuda_plugin.c
225+
$(call msg-gen, $@)
226+
- $(Q) $(CC) $(PLUGIN_CFLAGS) $(shell $(COMPEL) includes) $^ -o $@ $(PLUGIN_INCLUDE) $(PLUGIN_LDFLAGS)
227+
+ $(Q) $(CC) $(PLUGIN_CFLAGS) -shared -nostartfiles $(shell $(COMPEL) includes) $^ -o $@ $(PLUGIN_INCLUDE) $(PLUGIN_LDFLAGS)
228+
229+
clean:
230+
$(call msg-clean, $@)
231+
- $(Q) $(RM) $(PLUGIN_SOBJ)
232+
+ $(Q) $(RM) $(PLUGIN_SOBJ) $(PLUGIN_OBJ)
233+
.PHONY: clean
234+
235+
mrproper: clean
236+
diff --git a/plugins/cuda/cuda_plugin.c b/plugins/cuda/cuda_plugin.c
237+
index 99e4caf74..b27e45852 100644
238+
--- a/plugins/cuda/cuda_plugin.c
239+
+++ b/plugins/cuda/cuda_plugin.c
240+
@@ -47,6 +47,12 @@ bool plugin_disabled = false;
241+
242+
bool plugin_added_to_inventory = false;
243+
244+
+
245+
+int cuda_plugin_init(int stage);
246+
+void cuda_plugin_fini(int stage, int ret);
247+
+
248+
+CR_PLUGIN_REGISTER("cuda_plugin", cuda_plugin_init, cuda_plugin_fini)
249+
+
250+
struct pid_info {
251+
int pid;
252+
char checkpointed;
253+
@@ -632,4 +638,3 @@ void cuda_plugin_fini(int stage, int ret)
254+
dealloc_pid_buffer(&cuda_pids);
255+
}
256+
}
257+
-CR_PLUGIN_REGISTER("cuda_plugin", cuda_plugin_init, cuda_plugin_fini)

0 commit comments

Comments
 (0)