Merge pull request #537 from cbgbt/compress-twoliter-tools

twoliter: compress included tool archives

Author: cbgbt

Cargo.lock

@@ -1,11 +1,14 @@
 [workspace]
 resolver = "2"
 
+default-members = ["twoliter"]
 members = [
     "tools/amispec",
     "tools/bottlerocket-variant",
     "tools/buildsys",
     "tools/buildsys-config",
+    "tools/include-env-compressed",
+    "tools/include-env-compressed/include-env-compressed-macro",
     "tools/oci-cli-wrapper",
     "tools/parse-datetime",
     "tools/pipesys",
@@ -18,7 +21,9 @@ members = [
     "tools/testsys-config",
     "tools/unplug",
     "tools/update-metadata",
+
     "twoliter",
+    "twoliter/src/tool-crates/*",
 
     "tests/integration-tests",
 ]
@@ -37,6 +42,7 @@ ci = ["github"]
 installers = []
 # Target platforms to build apps for (Rust target-triple syntax)
 targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"]
+features = ["pubsys", "testsys"]
 
 [workspace.metadata.cross.build]
 pre-build = [
@@ -54,6 +60,8 @@ bottlerocket-types = { version = "0.0.16", git = "https://github.com/bottlerocke
 bottlerocket-variant = { version = "0.1", path = "tools/bottlerocket-variant" }
 buildsys = { version = "0.1", path = "tools/buildsys", lib = true, artifact = [ "bin:buildsys" ] }
 buildsys-config = { version = "0.1", path = "tools/buildsys-config" }
+include-env-compressed = { version = "0.1", path = "tools/include-env-compressed" }
+include-env-compressed-macro = { version = "0.1", path = "tools/include-env-compressed/include-env-compressed-macro" }
 krane-bundle = { version = "0.1", path = "tools/krane" }
 oci-cli-wrapper = { version = "0.1", path = "tools/oci-cli-wrapper" }
 parse-datetime = { version = "0.1", path = "tools/parse-datetime" }
@@ -67,7 +75,17 @@ serde-templated-derive = { version = "0.1", path = "tools/serde-templated-derive
 testsys = { version = "0.1", path = "tools/testsys", artifact = [ "bin:testsys" ] }
 testsys-config = { version = "0.1", path = "tools/testsys-config" }
 testsys-model = { version = "0.0.16", git = "https://github.com/bottlerocket-os/bottlerocket-test-system", tag = "v0.0.16" }
+
 twoliter = { version = "0.10.1", path = "twoliter", artifact = [ "bin:twoliter" ] }
+twoliter-tool-buildsys = { version = "0.1", path = "twoliter/src/tool-crates/buildsys" }
+twoliter-tool-embedded-bundle = { version = "0.1", path = "twoliter/src/tool-crates/embedded-bundle" }
+twoliter-tool-pipesys = { version = "0.1", path = "twoliter/src/tool-crates/pipesys" }
+twoliter-tool-pubsys = { version = "0.1", path = "twoliter/src/tool-crates/pubsys" }
+twoliter-tool-pubsys-setup = { version = "0.1", path = "twoliter/src/tool-crates/pubsys-setup" }
+twoliter-tool-testsys = { version = "0.1", path = "twoliter/src/tool-crates/testsys" }
+twoliter-tool-tuftool = { version = "0.1", path = "twoliter/src/tool-crates/tuftool" }
+twoliter-tool-unplug = { version = "0.1", path = "twoliter/src/tool-crates/unplug" }
+
 unplug = { version = "0.1", path = "tools/unplug", artifact = [ "bin:unplug" ] }
 update-metadata = { version = "0.1", path = "tools/update-metadata" }
 
@@ -156,6 +174,7 @@ url = "2"
 uuid = "1"
 walkdir = "2"
 which = "6"
+zstd = "0.13"
 
 # The profile that 'cargo dist' will build with
 [profile.dist]

Cargo.toml

@@ -74,3 +74,24 @@ license-files = [
     { path = "LICENSE-MIT", hash = 0xb9f15462 },
     { path = "LICENSE", hash = 0xa4618a29 },
 ]
+
+[clarify.zstd-safe]
+expression = "MIT OR Apache-2.0"
+license-files = [
+    { path = "LICENSE.Apache-2.0", hash = 0x7b466be4 },
+    { path = "LICENSE.Mit", hash = 0xa237d234 },
+    { path = "LICENSE", hash = 0x742401ae },
+]
+
+[clarify.zstd-sys]
+# The zstd-sys rust crate is MIT OR Apache-2.0
+# zstd itself is BSD-3-Clause or GPL-2.0 (we choose BSD-3-Clause)
+expression = "(MIT OR Apache-2.0) AND BSD-3-Clause"
+license-files = [
+    { path = "LICENSE", hash = 0x742401ae },
+    { path = "LICENSE.Apache-2.0", hash = 0x7b466be4 },
+    { path = "LICENSE.BSD-3-Clause", hash = 0xc9f5c4f6 },
+    { path = "LICENSE.Mit", hash = 0xa237d234 },
+    { path = "zstd/COPYING", hash = 0x96841aa4 },
+    { path = "zstd/LICENSE", hash = 0x3bfe1fb1 },
+ ]

clarify.toml

@@ -0,0 +1,13 @@
+[package]
+name = "include-env-compressed"
+version = "0.1.0"
+authors = ["Sean P. Kelly <[email protected]>"]
+license = "Apache-2.0 OR MIT"
+edition = "2024"
+
+[dependencies]
+include-env-compressed-macro.workspace = true
+zstd.workspace = true
+
+[lints]
+workspace = true

tools/include-env-compressed/Cargo.toml

@@ -0,0 +1,31 @@
+## `include-env-compressed`
+
+Like `include-bytes!`, but adds `zstd` compression for release builds.
+
+This macro is intended to be used alongside [cargo artifact-dependencies].
+As such, the input must be an environment variable which is set at compile time to a path to the
+target asset.
+
+Use the macro like so:
+
+```rust
+use include_env_compressed: {Archive, include_archive_from_env};
+
+const MY_ARCHIVE: Archive = include_archive_from_env!("ENV_VAR");
+
+// Returns a `Box<dyn std::io::Read + Send + Sync + 'static>`
+let reader = MY_ARCHIVE.reader();
+```
+
+You may optionally specify a zstd compression level as the second argument.
+
+```rust
+const VERY_COMPRESSED_ARCHIVE: Archive = include_archive_from_env!("ENV_VAR", 22);
+```
+
+Uses zstd's default compression level if no level is provided.
+
+Note: Due to limitations, does not compose with the `env!()` macro in the same way that
+`include_bytes!` does.
+
+[cargo artifact-dependencies]: https://doc.rust-lang.org/beta/cargo/reference/unstable.html#artifact-dependencies

tools/include-env-compressed/README.md

@@ -0,0 +1,7 @@
+use std::path::PathBuf;
+
+fn main() {
+    // Set an environment variable to be used during integration tests
+    let manifest = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()).join("build.rs");
+    println!("cargo:rustc-env=MY_BUILD_SCRIPT={}", manifest.display())
+}

tools/include-env-compressed/build.rs

@@ -0,0 +1,19 @@
+[package]
+name = "include-env-compressed-macro"
+version = "0.1.0"
+authors = ["Sean P. Kelly <[email protected]>"]
+license = "Apache-2.0 OR MIT"
+edition = "2024"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2.workspace = true
+quote.workspace = true
+snafu.workspace = true
+syn = { workspace = true, features = ["full"] }
+zstd.workspace = true
+
+[lints]
+workspace = true