Skip to content

Commit c506940

Browse files
ecpullenecpullen
authored
Merge pull request #108 from ecpullen/alpha-build
twoliter: `twoliter build` alpha
2 parents 74c77b3 + 66ba51f commit c506940

File tree

9 files changed

+185
-61
lines changed

9 files changed

+185
-61
lines changed

‎twoliter/embedded/Dockerfile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,6 @@ USER builder
144144
RUN --mount=source=.cargo,target=/home/builder/.cargo \
145145
--mount=type=cache,target=/home/builder/.cache,from=cache,source=/cache \
146146
--mount=type=cache,target=/home/builder/rpmbuild/BUILD/sources/models/src/variant,from=variantcache,source=/variantcache \
147-
--mount=type=cache,target=/home/builder/rpmbuild/BUILD/sources/logdog/conf/current,from=variantcache,source=/variantcache \
148147
--mount=source=sources,target=/home/builder/rpmbuild/BUILD/sources \
149148
rpmbuild -ba --clean \
150149
--undefine _auto_set_build_flags \

‎twoliter/embedded/Makefile.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,8 @@ BUILDSYS_JOBS = "8"
9797
CARGO_HOME = "${BUILDSYS_ROOT_DIR}/.cargo"
9898
# This needs to end with pkg/mod so that we can mount the parent of pkg/mod as GOPATH.
9999
GO_MOD_CACHE = "${BUILDSYS_ROOT_DIR}/.gomodcache/pkg/mod"
100-
GO_MODULES = "ecs-gpu-init host-ctr"
100+
# Dynamically load a list of go modules from ${BUILDSYS_SOURCE_DIR}
101+
GO_MODULES = { script = ['find ${BUILDSYS_SOURCES_DIR} -name go.mod -type f -printf "%h\n" | xargs -n1 basename'] }
101102
DOCKER_BUILDKIT = "1"
102103

103104
# This is the filename suffix for operations that write out AMI information to

‎twoliter/embedded/rpm2img‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,6 @@ INVENTORY_DATA="$(jq --slurp 'sort_by(.Name)' <<< "${INVENTORY_DATA}" | jq '{"Co
238238
printf "%s\n" "${INVENTORY_DATA}" > "${ROOT_MOUNT}/usr/share/bottlerocket/application-inventory.json"
239239

240240
# install licenses
241-
install -p -m 0644 /host/{COPYRIGHT,LICENSE-APACHE,LICENSE-MIT} "${ROOT_MOUNT}"/usr/share/licenses/
242241
mksquashfs \
243242
"${ROOT_MOUNT}"/usr/share/licenses \
244243
"${ROOT_MOUNT}"/usr/share/bottlerocket/licenses.squashfs \

‎twoliter/src/cargo_make.rs‎

Lines changed: 1 addition & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -94,41 +94,8 @@ impl CargoMake {
9494
self
9595
}
9696

97-
/// Specify environment variables that should be applied for this comand
98-
pub(crate) fn _envs<S1, S2, V>(mut self, env_vars: V) -> Self
99-
where
100-
S1: Into<String>,
101-
S2: Into<String>,
102-
V: Into<Vec<(S1, S2)>>,
103-
{
104-
for (key, value) in env_vars.into() {
105-
self.args
106-
.push(format!("-e={}={}", key.into(), value.into()));
107-
}
108-
self
109-
}
110-
111-
/// Specify `cargo make` arguments that should be applied for this comand
112-
pub(crate) fn _arg<S>(mut self, arg: S) -> Self
113-
where
114-
S: Into<String>,
115-
{
116-
self.args.push(arg.into());
117-
self
118-
}
119-
120-
/// Specify `cargo make` arguments that should be applied for this comand
121-
pub(crate) fn _args<V, S>(mut self, args: V) -> Self
122-
where
123-
S: Into<String>,
124-
V: Into<Vec<S>>,
125-
{
126-
self.args.extend(args.into().into_iter().map(Into::into));
127-
self
128-
}
129-
13097
/// Execute the `cargo make` task
131-
pub(crate) async fn _exec<S>(&self, task: S) -> Result<()>
98+
pub(crate) async fn exec<S>(&self, task: S) -> Result<()>
13299
where
133100
S: Into<String>,
134101
{

‎twoliter/src/cmd/build.rs‎

Lines changed: 96 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
1-
use anyhow::Result;
1+
use crate::cargo_make::CargoMake;
2+
use crate::docker::DockerContainer;
3+
use crate::project;
4+
use crate::tools::{install_tools, tools_tempdir};
5+
use anyhow::{Context, Result};
26
use clap::Parser;
3-
use std::path::PathBuf;
7+
use log::debug;
8+
use std::fs;
9+
use std::path::{Path, PathBuf};
10+
use tempfile::TempDir;
11+
use tokio::fs::{remove_dir_all, remove_file};
412

513
#[derive(Debug, Parser)]
614
pub(crate) enum BuildCommand {
@@ -25,10 +33,95 @@ pub(crate) struct BuildVariant {
2533
/// The architecture to build for.
2634
#[clap(long = "arch", default_value = "x86_64")]
2735
arch: String,
36+
37+
/// The variant to build.
38+
variant: String,
2839
}
2940

3041
impl BuildVariant {
3142
pub(super) async fn run(&self) -> Result<()> {
32-
Ok(())
43+
let project = project::load_or_find_project(self.project_path.clone()).await?;
44+
let token = project.token();
45+
let tempdir = tools_tempdir()?;
46+
install_tools(&tempdir).await?;
47+
let makefile_path = tempdir.path().join("Makefile.toml");
48+
// A temporary directory in the `build` directory
49+
let build_temp_dir = TempDir::new_in(project.project_dir())
50+
.context("Unable to create a tempdir for Twoliter's build")?;
51+
let packages_dir = build_temp_dir.path().join("sdk_rpms");
52+
fs::create_dir_all(&packages_dir)?;
53+
54+
let sdk_container = DockerContainer::new(
55+
format!("sdk-{}", token),
56+
project
57+
.sdk(&self.arch)
58+
.context(format!(
59+
"No SDK defined in {} for {}",
60+
project.filepath().display(),
61+
&self.arch
62+
))?
63+
.uri(),
64+
)
65+
.await?;
66+
sdk_container
67+
.cp_out(Path::new("twoliter/alpha/build/rpms"), &packages_dir)
68+
.await?;
69+
70+
let rpms_dir = project.project_dir().join("build").join("rpms");
71+
fs::create_dir_all(&rpms_dir)?;
72+
debug!("Moving rpms to build dir");
73+
for maybe_file in fs::read_dir(packages_dir.join("rpms"))? {
74+
let file = maybe_file?;
75+
debug!("Moving '{}'", file.path().display());
76+
fs::rename(file.path(), rpms_dir.join(file.file_name()))?;
77+
}
78+
79+
let mut created_files = Vec::new();
80+
81+
let sbkeys_dir = project.project_dir().join("sbkeys");
82+
if !sbkeys_dir.is_dir() {
83+
// Create a sbkeys directory in the main project
84+
debug!("sbkeys dir not found. Creating a temporary directory");
85+
fs::create_dir_all(&sbkeys_dir)?;
86+
sdk_container
87+
.cp_out(
88+
Path::new("twoliter/alpha/sbkeys/generate-local-sbkeys"),
89+
&sbkeys_dir,
90+
)
91+
.await?;
92+
};
93+
94+
// TODO: Remove once models is no longer conditionally compiled.
95+
// Create the models directory for the sdk to mount
96+
let models_dir = project.project_dir().join("sources/models");
97+
if !models_dir.is_dir() {
98+
debug!("models source dir not found. Creating a temporary directory");
99+
fs::create_dir_all(&models_dir.join("src/variant"))
100+
.context("Unable to create models source directory")?;
101+
created_files.push(models_dir)
102+
}
103+
104+
// Hold the result of the cargo make call so we can clean up the project directory first.
105+
let res = CargoMake::new(&project, &self.arch)?
106+
.env("TWOLITER_TOOLS_DIR", tempdir.path().display().to_string())
107+
.env("BUILDSYS_ARCH", &self.arch)
108+
.env("BUILDSYS_VARIANT", &self.variant)
109+
.env("BUILDSYS_SBKEYS_DIR", sbkeys_dir.display().to_string())
110+
.makefile(makefile_path)
111+
.project_dir(project.project_dir())
112+
.exec("build")
113+
.await;
114+
115+
// Clean up all of the files we created
116+
for file_name in created_files {
117+
let added = Path::new(&file_name);
118+
if added.is_file() {
119+
remove_file(added).await?;
120+
} else if added.is_dir() {
121+
remove_dir_all(added).await?;
122+
}
123+
}
124+
125+
res
33126
}
34127
}

‎twoliter/src/docker/container.rs‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
use crate::common::exec;
2+
use anyhow::Result;
3+
use log::{debug, log, Level};
4+
use std::path::Path;
5+
use tokio::process::Command;
6+
7+
pub(crate) struct DockerContainer {
8+
name: String,
9+
}
10+
11+
impl DockerContainer {
12+
/// Create a docker image with the given name from the image by using `docker create`.
13+
pub(crate) async fn new<S1, S2>(container_name: S1, image: S2) -> Result<Self>
14+
where
15+
S1: Into<String>,
16+
S2: Into<String>,
17+
{
18+
let name = container_name.into();
19+
let image = image.into();
20+
21+
// Make sure previous versions of this container are stopped deleted.
22+
cleanup_container(&name, Level::Trace).await;
23+
24+
debug!("Creating docker container '{name}' from image '{image}'");
25+
26+
// Create the new container.
27+
let args = vec![
28+
"create".to_string(),
29+
"--rm".to_string(),
30+
"--name".to_string(),
31+
name.to_string(),
32+
image.to_string(),
33+
];
34+
35+
exec(Command::new("docker").args(args), true).await?;
36+
Ok(Self { name })
37+
}
38+
39+
/// Copy the data from this container to a local destination.
40+
pub(crate) async fn cp_out<P1, P2>(&self, src: P1, dest: P2) -> Result<()>
41+
where
42+
P1: AsRef<Path>,
43+
P2: AsRef<Path>,
44+
{
45+
debug!(
46+
"Copying '{}' from '{}' to '{}'",
47+
src.as_ref().display(),
48+
self.name,
49+
dest.as_ref().display()
50+
);
51+
let mut args = vec!["cp".to_string()];
52+
args.push(format!("{}:{}", self.name, src.as_ref().display()));
53+
args.push(dest.as_ref().display().to_string());
54+
exec(Command::new("docker").args(args), true).await
55+
}
56+
}
57+
58+
impl Drop for DockerContainer {
59+
fn drop(&mut self) {
60+
let name = self.name.clone();
61+
tokio::task::spawn(async move { cleanup_container(&name, Level::Error).await });
62+
}
63+
}
64+
65+
async fn cleanup_container(name: &str, log_level: Level) {
66+
let args = vec!["stop".to_string(), name.to_string()];
67+
if let Err(e) = exec(Command::new("docker").args(args), true).await {
68+
log!(log_level, "Unable to stop container '{}': {e}", name)
69+
}
70+
let args = vec!["rm".to_string(), name.to_string()];
71+
if let Err(e) = exec(Command::new("docker").args(args), true).await {
72+
log!(log_level, "Unable to remove container '{}': {e}", name)
73+
}
74+
}

‎twoliter/src/docker/mod.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
mod commands;
2+
mod container;
23
mod image;
34

4-
#[allow(unused)]
5+
pub(crate) use self::container::DockerContainer;
6+
#[allow(unused_imports)]
57
pub(crate) use self::image::{ImageArchUri, ImageUri};

‎twoliter/src/project.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ use log::{debug, trace};
55
use non_empty_string::NonEmptyString;
66
use serde::de::Error;
77
use serde::{Deserialize, Deserializer, Serialize, Serializer};
8+
use sha2::{Digest, Sha512};
89
use std::fmt;
910
use std::path::{Path, PathBuf};
1011
use tokio::fs;
@@ -115,6 +116,13 @@ impl Project {
115116
pub(crate) fn toolchain(&self, arch: &str) -> Option<ImageArchUri> {
116117
self.toolchain_name().map(|s| s.uri(arch))
117118
}
119+
120+
pub(crate) fn token(&self) -> String {
121+
let mut d = Sha512::new();
122+
d.update(self.filepath().display().to_string());
123+
let digest = hex::encode(d.finalize());
124+
(digest[..12]).to_string()
125+
}
118126
}
119127

120128
/// A base name for an image that can be suffixed using a naming convention. For example,

‎twoliter/src/test/cargo_make.rs‎

Lines changed: 1 addition & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ async fn test_cargo_make() {
77
let cargo_make = CargoMake::new(&project, "arch")
88
.unwrap()
99
.makefile(data_dir().join("Makefile.toml"));
10-
cargo_make._exec("verify-twoliter-env").await.unwrap();
10+
cargo_make.exec("verify-twoliter-env").await.unwrap();
1111
cargo_make
1212
.clone()
1313
.env("FOO", "bar")
@@ -29,23 +29,4 @@ async fn test_cargo_make() {
2929
)
3030
.await
3131
.unwrap();
32-
cargo_make
33-
.clone()
34-
._arg("--env")
35-
._arg("FOO=bar")
36-
.exec_with_args("verify-env-value-with-arg", ["FOO", "bar"])
37-
.await
38-
.unwrap();
39-
cargo_make
40-
.clone()
41-
._args(["--env", "FOO=bar"])
42-
.exec_with_args("verify-env-value-with-arg", ["FOO", "bar"])
43-
.await
44-
.unwrap();
45-
cargo_make
46-
.clone()
47-
._envs([("FOO", "bar"), ("BAR", "baz")])
48-
.exec_with_args("verify-env-value-with-arg", ["BAR", "baz"])
49-
.await
50-
.unwrap();
5132
}

0 commit comments

Comments
 (0)