workshop: implement plan option "translate"

Can be used instead of "exec" to let the translation server decide how
to execute the job process.

Author: MaxKellermann

debian/changelog

@@ -2,6 +2,7 @@ cm4all-workshop (7.0) unstable; urgency=low
 
   * workshop: fix use-after-free bug in "${0}" expansion
   * workshop: report spawner errors
+  * workshop: implement plan option "translate"
   * no delay for first PostgreSQL auto-reconnect
 
  --   

doc/index.rst

@@ -287,11 +287,34 @@ text file for each plan.  Example::
 The program :command:`/usr/bin/my-plan` is executed as user `bar` with
 a CPU scheduler priority of 5 (10 is the default if not specified).
 
+Instead of ``exec``, you can use ``translate`` to let a translation
+server decide how to execute the job::
+
+  translate
+
 The following options are available:
 
 * :samp:`exec PROGRAM ARG1 ...`: Command line.  The program path must
   be absolute, because Workshop will not consider the :envvar:`PATH`.
 
+* :samp:`translate`: Can be used instead of ``exec``.  Queries the
+  configured translation server for information on how to execute the
+  job process.
+
+  This sends a translation request with the following packets:
+
+  - ``EXECUTE`` (no payload)
+  - ``SERVICE=workshop``
+  - ``PLAN=<name>``: name of the plan
+  - ``TAG=<tag>``: value of the ``tag`` configuration option (only if
+    one is configured
+  - ``APPEND=<arg>``: one packet for each ``jobs.args`` item
+
+  The plan may not contain any other process execute options, because
+  that will be decided by the translation server.
+
+  The ``control_channel`` option is allowed, but not ``allow_spawn``.
+
 * :samp:`control_channel`: see `Control Channel`_.
 
 .. _allow_spawn:

src/translation/SpawnClient.cxx

@@ -20,7 +20,8 @@ using std::string_view_literals::operator""sv;
 static void
 SendTranslateSpawn(SocketDescriptor s, const char *tag,
 		   const char *plan_name,
-		   const char *execute, const char *param)
+		   const char *execute, const char *param,
+		   const std::forward_list<std::string> &args)
 
 {
 	assert(execute != nullptr);
@@ -45,6 +46,9 @@ SendTranslateSpawn(SocketDescriptor s, const char *tag,
 	if (tag != nullptr)
 		m.Write(TranslationCommand::LISTENER_TAG, tag);
 
+	for (const auto &i : args)
+		m.Write(TranslationCommand::APPEND, i);
+
 	m.Write(TranslationCommand::END);
 
 	SendFull(s, m.Commit());
@@ -54,9 +58,10 @@ Co::Task<TranslateResponse>
 TranslateSpawn(EventLoop &event_loop,
 	       AllocatorPtr alloc, SocketDescriptor s,
 	       const char *tag,
-	       const char *plan_name, const char *execute, const char *param)
+	       const char *plan_name, const char *execute, const char *param,
+	       const std::forward_list<std::string> &args)
 {
-	SendTranslateSpawn(s, tag, plan_name, execute, param);
+	SendTranslateSpawn(s, tag, plan_name, execute, param, args);
 	co_await AwaitableSocketEvent(event_loop, s, SocketEvent::READ);
 	co_return ReceiveTranslateResponse(alloc, s);
 }

src/translation/SpawnClient.hxx

@@ -4,6 +4,9 @@
 
 #pragma once
 
+#include <forward_list>
+#include <string>
+
 namespace Co { template<typename T> class Task; }
 struct TranslateResponse;
 class AllocatorPtr;
@@ -14,4 +17,5 @@ Co::Task<TranslateResponse>
 TranslateSpawn(EventLoop &event_loop,
 	       AllocatorPtr alloc, SocketDescriptor s,
 	       const char *tag,
-	       const char *plan_name, const char *execute, const char *param);
+	       const char *plan_name, const char *execute, const char *param,
+	       const std::forward_list<std::string> &args);

src/workshop/Operator.cxx

@@ -168,35 +168,44 @@ PrepareChildProcess(AllocatorPtr alloc,
 }
 
 static void
-PrepareChildProcess(PreparedChildProcess &p, const char *plan_name,
+PrepareChildProcess(AllocatorPtr alloc, PreparedChildProcess &p,
+		    const char *plan_name,
 		    const Plan &plan,
-		    FileDescriptor stderr_fd, SocketDescriptor control_fd)
+		    const TranslateResponse &translation,
+		    FileDescriptor stderr_fd, SocketDescriptor control_fd,
+		    FdHolder &close_fds)
 {
 	p.hook_info = plan_name;
 	p.stderr_fd = p.stdout_fd = stderr_fd;
 	p.control_fd = control_fd.ToFileDescriptor();
 
-	if (!debug_mode) {
-		p.uid_gid.effective_uid = plan.uid;
-		p.uid_gid.effective_gid = plan.gid;
+	if (plan.translate) {
+		PrepareChildProcess(alloc, p, translation, close_fds);
+	} else {
+		if (!debug_mode) {
+			p.uid_gid.effective_uid = plan.uid;
+			p.uid_gid.effective_gid = plan.gid;
 
-		std::copy(plan.groups.begin(), plan.groups.end(),
-			  p.uid_gid.supplementary_groups.begin());
-	}
+			std::copy(plan.groups.begin(), plan.groups.end(),
+				  p.uid_gid.supplementary_groups.begin());
+		}
 
-	if (!plan.chroot.empty())
-		p.chroot = plan.chroot.c_str();
+		if (!plan.chroot.empty())
+			p.chroot = plan.chroot.c_str();
 
-	p.umask = plan.umask;
-	p.rlimits = plan.rlimits;
-	p.priority = plan.priority;
-	p.ioprio_idle = plan.ioprio_idle;
-	p.ns.enable_network = plan.private_network;
+		p.umask = plan.umask;
+		p.rlimits = plan.rlimits;
+		p.priority = plan.priority;
+		p.sched_idle = plan.sched_idle;
+		p.ioprio_idle = plan.ioprio_idle;
+		p.ns.enable_network = plan.private_network;
 
-	if (plan.private_tmp)
-		p.ns.mount.mount_tmp_tmpfs = "";
+		if (plan.private_tmp)
+			p.ns.mount.mount_tmp_tmpfs = "";
+
+		p.no_new_privs = true;
+	}
 
-	p.no_new_privs = true;
 }
 
 void
@@ -215,6 +224,22 @@ WorkshopOperator::Start2(std::size_t max_log_buffer,
 {
 	assert(!pid);
 
+	Allocator alloc;
+	TranslateResponse translation;
+	if (plan->translate) {
+		const auto translation_socket = workplace.GetTranslationSocket();
+		if (translation_socket == nullptr)
+			throw std::runtime_error{"No 'translation_server' configured"};
+
+		translation = co_await
+			TranslateSpawn(event_loop, alloc,
+				       CreateConnectSocket(translation_socket, SOCK_STREAM),
+				       workplace.GetListenerTag(),
+				       job.plan_name.c_str(),
+				       "", nullptr,
+				       job.args);
+	}
+
 	auto &spawn_service = workplace.GetSpawnService();
 
 	co_await CoEnqueueSpawner{spawn_service};
@@ -227,9 +252,12 @@ WorkshopOperator::Start2(std::size_t max_log_buffer,
 
 	const auto control_child = InitControl();
 
+	FdHolder close_fds;
 	PreparedChildProcess p;
-	PrepareChildProcess(p, job.plan_name.c_str(), *plan,
-			    stderr_w, control_child);
+	PrepareChildProcess(alloc, p, job.plan_name.c_str(), *plan,
+			    translation,
+			    stderr_w, control_child,
+			    close_fds);
 
 	/* use a per-plan cgroup */
 
@@ -253,7 +281,7 @@ WorkshopOperator::Start2(std::size_t max_log_buffer,
 
 	UniqueFileDescriptor stdout_w;
 
-	if (!plan->control_channel) {
+	if (plan->translate || !plan->control_channel) {
 		/* if there is no control channel, read progress from the
 		   stdout pipe */
 		UniqueFileDescriptor stdout_r;
@@ -266,27 +294,31 @@ WorkshopOperator::Start2(std::size_t max_log_buffer,
 	/* build command line */
 
 	std::list<std::string> args;
-	args.insert(args.end(), plan->args.begin(), plan->args.end());
-	args.insert(args.end(), job.args.begin(), job.args.end());
 
-	Expand(args);
+	if (!plan->translate) {
+		args.insert(args.end(), plan->args.begin(), plan->args.end());
+		args.insert(args.end(), job.args.begin(), job.args.end());
 
-	for (const auto &i : args) {
-		if (p.args.size() >= 4096)
-			throw std::runtime_error("Too many command-line arguments");
+		Expand(args);
 
-		p.args.push_back(i.c_str());
-	}
+		for (const auto &i : args) {
+			if (p.args.size() >= 4096)
+				throw std::runtime_error("Too many command-line arguments");
+
+			p.args.push_back(i.c_str());
+		}
 
-	for (const auto &i : job.env) {
-		if (p.env.size() >= 64)
-			throw std::runtime_error("Too many environment variables");
+		// TODO do we want to allow job.env for "translate" plans?
+		for (const auto &i : job.env) {
+			if (p.env.size() >= 64)
+				throw std::runtime_error("Too many environment variables");
 
-		if (StringStartsWith(i.c_str(), "LD_"))
-			/* reject - too dangerous */
-			continue;
+			if (StringStartsWith(i.c_str(), "LD_"))
+				/* reject - too dangerous */
+				continue;
 
-		p.env.push_back(i.c_str());
+			p.env.push_back(i.c_str());
+		}
 	}
 
 	/* fork */
@@ -558,7 +590,8 @@ WorkshopOperator::OnControlSpawn(const char *token, const char *param)
 			       CreateConnectSocket(translation_socket, SOCK_STREAM),
 			       workplace.GetListenerTag(),
 			       job.plan_name.c_str(),
-			       token, param);
+			       token, param,
+			       {});
 
 	auto &spawn_service = workplace.GetSpawnService();
 

src/workshop/Plan.hxx

@@ -20,6 +20,16 @@ class Error;
 
 /** a plan describes how to perform a specific job */
 struct Plan {
+	enum class Type {
+		EXEC,
+		TRANSLATE,
+	};
+
+	/**
+	 * If non-empty, then this plan executes a process with these
+	 * command-line arguments; the first string is the path of the
+	 * executable.
+	 */
 	std::vector<std::string> args;
 
 	std::string timeout, chroot;
@@ -55,6 +65,8 @@ struct Plan {
 
 	bool allow_spawn = false;
 
+	bool translate = false;
+
 	Plan() = default;
 
 	Plan(Plan &&) = default;
@@ -64,6 +76,16 @@ struct Plan {
 	Plan &operator=(Plan &&other) = default;
 	Plan &operator=(const Plan &other) = delete;
 
+	[[gnu::pure]]
+	Type GetType() const noexcept {
+		if (translate)
+			return Type::TRANSLATE;
+
+		assert(!args.empty());
+
+		return Type::EXEC;
+	}
+
 	const std::string &GetExecutablePath() const {
 		assert(!args.empty());
 

src/workshop/PlanLoader.cxx

@@ -23,6 +23,8 @@
 class PlanLoader final : public ConfigParser {
 	Plan plan;
 
+	bool seen_exec_option = false;
+
 public:
 	Plan &&Release() {
 		return std::move(plan);
@@ -65,6 +67,10 @@ PlanLoader::ParseLine(FileLineParser &line)
 			plan.args.emplace_back(value);
 			value = line.NextRelaxedValue();
 		} while (value != nullptr);
+
+		seen_exec_option = true;
+	} else if (StringIsEqual(key, "translate")) {
+		plan.translate = true;
 	} else if (StringIsEqual(key, "control_channel")) {
 		/* previously, the "yes"/"no" parameter was mandatory, but
 		   that's deprecated since 2.0.36 */
@@ -99,6 +105,7 @@ PlanLoader::ParseLine(FileLineParser &line)
 			throw FmtRuntimeError("not a directory: {}", value);
 
 		plan.chroot = value;
+		seen_exec_option = true;
 	} else if (StringIsEqual(key, "user")) {
 		const char *value = line.ExpectValueAndEnd();
 
@@ -118,6 +125,8 @@ PlanLoader::ParseLine(FileLineParser &line)
 		plan.gid = pw->pw_gid;
 
 		plan.groups = get_user_groups(value, plan.gid);
+
+		seen_exec_option = true;
 	} else if (StringIsEqual(key, "umask")) {
 		const char *s = line.ExpectValueAndEnd();
 		if (*s != '0')
@@ -132,26 +141,34 @@ PlanLoader::ParseLine(FileLineParser &line)
 			throw std::runtime_error("umask is too large");
 
 		plan.umask = value;
+		seen_exec_option = true;
 	} else if (StringIsEqual(key, "nice")) {
 		plan.priority = atoi(line.ExpectValueAndEnd());
+		seen_exec_option = true;
 	} else if (StringIsEqual(key, "sched_idle")) {
 		plan.sched_idle = true;
 		line.ExpectEnd();
+		seen_exec_option = true;
 	} else if (StringIsEqual(key, "ioprio_idle")) {
 		plan.ioprio_idle = true;
 		line.ExpectEnd();
+		seen_exec_option = true;
 	} else if (StringIsEqual(key, "idle")) {
 		plan.sched_idle = plan.ioprio_idle = true;
 		line.ExpectEnd();
+		seen_exec_option = true;
 	} else if (StringIsEqual(key, "private_network")) {
 		line.ExpectEnd();
 		plan.private_network = true;
+		seen_exec_option = true;
 	} else if (StringIsEqual(key, "private_tmp")) {
 		line.ExpectEnd();
 		plan.private_tmp = true;
+		seen_exec_option = true;
 	} else if (StringIsEqual(key, "rlimits")) {
 		if (!plan.rlimits.Parse(line.ExpectValueAndEnd()))
 			throw std::runtime_error("Failed to parse rlimits");
+		seen_exec_option = true;
 	} else if (StringIsEqual(key, "concurrency")) {
 		plan.concurrency = line.NextPositiveInteger();
 		line.ExpectEnd();
@@ -164,8 +181,18 @@ PlanLoader::ParseLine(FileLineParser &line)
 void
 PlanLoader::Finish()
 {
-	if (plan.args.empty())
-		throw std::runtime_error("no 'exec'");
+	if (plan.translate) {
+		if (seen_exec_option)
+			throw std::runtime_error("Cannot use 'translate' with execute options");
+
+		if (plan.allow_spawn)
+			/* this is forbidden until we have a secure
+			   implementation */
+			throw std::runtime_error("Cannot use 'translate' with 'allow_spawn'");
+	} else {
+		if (plan.args.empty())
+			throw std::runtime_error("no 'exec'");
+	}
 
 	if (plan.timeout.empty())
 		plan.timeout = "10 minutes";