[antlir2][debian] snapshot metadata tree

Summary:
Snapshot a cheaply-produced tree of metadata artifacts into Manifold.
This is not yet wired up to anything that can persist the output into something
usable in a Buck target - that will follow.

Test Plan:
```
❯ buck2 bxl fbcode//mode/opt fbcode//antlir/antlir2/package_managers/snapshot/snapshot.bxl:main -- --target fbcode//antlir/antlir2/package_managers/deb:trixie --storage {"type":"manifold", "bucket": "antlir_snapshots", "api_key": "antlir_snapshots-key", "path_prefix": "tree/deb/snap-2026-02-27+16:06:14"}
buck-out/v2/art-bxl/fbcode/86037fb0bf2cbdca/antlir/antlir2/package_managers/snapshot/snapshot.bxl/__main__/34f11e79e91b530d/metadata.json

❯ jq < buck-out/v2/art-bxl/fbcode/86037fb0bf2cbdca/antlir/antlir2/package_managers/snapshot/snapshot.bxl/__main__/34f11e79e91b530d/metadata.json
{
  "files": {
    "archive/dists/trixie/InRelease": "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/archive/dists/trixie/InRelease",
    "archive/dists/trixie/contrib/binary-amd64/Packages": "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/archive/dists/trixie/contrib/binary-amd64/Packages",
    "archive/dists/trixie/main/binary-amd64/Packages": "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/archive/dists/trixie/main/binary-amd64/Packages",
    "components/contrib/packages.json": "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/components/contrib/packages.json",
    "components/main/packages.json": "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/components/main/packages.json",
    "release.json": "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/release.json"
  },
  "checksums": {
    "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/archive/dists/trixie/InRelease": {
      "sha1": "986061afe57bd34842e497fdd7dc654fe3d8941f",
      "sha256": "94f4e8cbf0bd93ebe686aa8fc95410608613bb832536770fe5ab5c8b45527e49"
    },
    "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/archive/dists/trixie/contrib/binary-amd64/Packages": {
      "sha1": "f247cc6a94e2c678b378069aa7df31caad9bb52f",
      "sha256": "4699ac4502079b1dcbb5933d02c3dc31ae2a67dfbfdde4f4adf69d9382682dc4"
    },
    "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/archive/dists/trixie/main/binary-amd64/Packages": {
      "sha1": "5ef3ee32dbc68dce80b6f397b638035c63b871a9",
      "sha256": "056028ca06f66849ee86f378cb8dc5ad41e6322ea2d2877317921d4d2a50a5ba"
    },
    "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/components/contrib/packages.json": {
      "sha1": "59bac48a3ed445a263a07c18087a9d34bc8cb4db",
      "sha256": "fed74781da90720bd07ff5a355339d63ea817cfca305ff764ccbe0ae0c7e2b03"
    },
    "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/components/main/packages.json": {
      "sha1": "f9f98f088fd1427e08c3a45cba209800e1d88e6e",
      "sha256": "e0845069da0671ba734d1b1bcf8e27fed4bac8095ce005b0aa4b9ca52ae4f615"
    },
    "mf://antlir_snapshots-key@antlir_snapshots/tree/tree/deb/snap-2026-02-27+16:06:14/release.json": {
      "sha1": "6a80e5e5720e5b1182dcf781d9f74a876405084f",
      "sha256": "a52749a9b21f1ff161c423e42346f0003ec2aecd19b8c680c5b3799ae8d2fecd"
    }
  }
}
```

Reviewed By: marcinpe

Differential Revision: D93956589

fbshipit-source-id: 16601e1d28b62e1ea9dcbcbba0758c1586c554e7

Author: vmagro

antlir/antlir2/package_managers/deb/suite.bzl

@@ -5,6 +5,7 @@
 
 load("//antlir/antlir2/bzl:platform.bzl", "rule_with_default_target_platform")
 load("//antlir/antlir2/package_managers/snapshot:download.bzl", "download")
+load("//antlir/antlir2/package_managers/snapshot:snapshottable.bzl", "SnapshottableInfo")
 load(":packages_index.bzl", "download_component_package_indexes")
 
 ComponentInfo = provider(fields = {
@@ -37,7 +38,10 @@ def _suite_impl(ctx: AnalysisContext) -> list[Provider]:
         allow_nondeterministic_downloads = True,
     )
 
+    metadata_tree = {}
+
     release_json = ctx.actions.declare_output("release.json", has_content_based_path = True)
+    metadata_tree["release.json"] = release_json
     ctx.actions.run(
         cmd_args(
             ctx.attrs._snapshot_bin[RunInfo],
@@ -68,6 +72,7 @@ def _suite_impl(ctx: AnalysisContext) -> list[Provider]:
     components_subtargets = {}
     for cname, c in components_package_indexes.items():
         components_subtargets[cname] = [DefaultInfo(sub_targets = {"packages.json": [DefaultInfo(c.json)]})]
+        metadata_tree["components/" + cname + "/packages.json"] = c.json
 
     # Generate a custom InRelease that only lists the components we care
     # about, signed with a dummy key so apt can verify it.
@@ -100,6 +105,8 @@ def _suite_impl(ctx: AnalysisContext) -> list[Provider]:
         archive_dir_srcs[dist_prefix + cname + "/binary-" + ctx.attrs._arch + "/Packages"] = c.txt
 
     archive_dir = ctx.actions.symlinked_dir("archive", archive_dir_srcs)
+    metadata_tree["archive"] = archive_dir
+    metadata_tree = ctx.actions.symlinked_dir("snapshottable", metadata_tree)
 
     return [
         DefaultInfo(sub_targets = {
@@ -117,6 +124,9 @@ def _suite_impl(ctx: AnalysisContext) -> list[Provider]:
             components = components,
             suite_baseurl = suite_baseurl,
         ),
+        SnapshottableInfo(
+            metadata_tree = metadata_tree,
+        ),
     ]
 
 _suite = rule(

antlir/antlir2/package_managers/snapshot/BUCK

@@ -5,11 +5,18 @@ oncall("antlir")
 rust_binary(
     name = "snapshot",
     srcs = glob(["src/**/*.rs"]),
+    crate_root = "src/main.rs",
+    fb_deps = [
+        "//manifold/clients/rust:manifold_client",
+    ],
     test_srcs = glob(["testdata/**"]),
     deps = [
         "anyhow",
+        "async-trait",
         "bon",
         "clap",
+        "fbinit",
+        "fbinit-tokio",
         "flate2",
         "hex",
         "liblzma",
@@ -20,6 +27,8 @@ rust_binary(
         "sha2",
         "tracing",
         "tracing-subscriber",
+        "url",
+        "walkdir",
         "//antlir/filesystem/stdio_path:stdio_path",
         "//antlir/util/cli/json_arg:json_arg",
     ],

antlir/antlir2/package_managers/snapshot/snapshot.bxl

@@ -0,0 +1,49 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+load("@prelude//utils:expect.bzl", "expect")
+load("//antlir/antlir2/package_managers/snapshot:snapshottable.bzl", "SnapshottableInfo")
+
+def _main(ctx: bxl.Context):
+    # Get the snapshot binary as an exec dependency
+    snapshot_bin = ctx.unconfigured_sub_targets("fbcode//antlir/antlir2/package_managers/snapshot:snapshot")
+
+    # Set up BXL actions with the snapshot binary
+    bxl_actions = ctx.bxl_actions(exec_deps = [snapshot_bin])
+    actions = bxl_actions.actions
+    snapshot_run_info = bxl_actions.exec_deps[snapshot_bin][RunInfo]
+
+    target_labels = ctx.cli_args.target
+    expect(len(target_labels) == 1, "Only one target is supported at this time")
+    target = ctx.configured_targets(target_labels[0], modifiers = ctx.modifiers)
+    analysis = ctx.analysis(target)
+    snapshottable_info = analysis.providers()[SnapshottableInfo]
+
+    metadata_json = actions.declare_output("metadata.json")
+    logs = actions.declare_output("logs.txt")
+    actions.run(
+        cmd_args(
+            snapshot_run_info,
+            cmd_args(logs.as_output(), format = "--log={}"),
+            "snapshot",
+            cmd_args(metadata_json.as_output(), format = "--out={}"),
+            cmd_args(ctx.cli_args.storage, format = "--storage={}"),
+            "metadata",
+            cmd_args(snapshottable_info.metadata_tree, format = "--tree={}"),
+        ),
+        category = "snapshot",
+        identifier = "metadata",
+        local_only = True,  # needs network access
+        allow_cache_upload = True,  # the result is deterministic
+    )
+    ctx.output.stream("Metadata snapshot:", ctx.output.ensure(metadata_json), wait_on = [ctx.output.ensure(metadata_json)])
+
+main = bxl_main(
+    impl = _main,
+    cli_args = {
+        "storage": cli_args.string(),
+        "target": cli_args.target_expr(),
+    },
+)

antlir/antlir2/package_managers/snapshot/snapshottable.bzl

@@ -0,0 +1,14 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+SnapshottableInfo = provider(fields = {
+    # A directory artifact that should contain all the metadata artifacts that
+    # should be snapshotted into persistent storage. This is considered cheap to
+    # produce and store, as opposed to package blobs (which are belligerent and
+    # numerous https://www.youtube.com/watch?v=HvVJQMnB-N8), so the target that
+    # contains this provider should just produce the entire tree of metadata
+    # artifacts every time.
+    "metadata_tree": Artifact,
+})

antlir/antlir2/package_managers/snapshot/src/checksums.rs

@@ -69,3 +69,23 @@ impl Checksums {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_from_reader() {
+        let input = "Hello world\n".as_bytes();
+        let checksums = Checksums::from_reader(input).expect("failed to compute checksums");
+        assert_eq!(
+            checksums,
+            Checksums {
+                sha1: Some("33ab5639bfd8e7b95eb1d8d0b87781d4ffea4d5d".to_string()),
+                sha256: Some(
+                    "1894a19c85ba153acbf743ac4e43fc004c891604b26f8c69e1e83ea2afc7c48f".to_string()
+                ),
+            }
+        );
+    }
+}

antlir/antlir2/package_managers/snapshot/src/main.rs

@@ -5,7 +5,10 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+use std::fs::File;
+use std::path::PathBuf;
 use std::process::ExitCode;
+use std::sync::Mutex;
 
 use anyhow::Result;
 use clap::Parser;
@@ -16,48 +19,65 @@ mod checksums;
 mod decompress;
 mod generate;
 mod parse;
+mod snapshot;
 
 #[derive(Debug, Parser)]
 struct Args {
     #[clap(short, long, default_value_t=1, action = clap::ArgAction::Count)]
     verbose: u8,
     #[clap(subcommand)]
     sub: Sub,
+    #[clap(long)]
+    log: Option<PathBuf>,
 }
 
 #[derive(Debug, Parser)]
 enum Sub {
     Decompress(decompress::Decompress),
     Generate(generate::Generate),
     Parse(parse::Parse),
+    Snapshot(snapshot::Snapshot),
 }
 
-fn main() -> ExitCode {
+#[fbinit::main]
+async fn main(fb: fbinit::FacebookInit) -> ExitCode {
     // Wrap the real main function with this so that we can print out the full
     // error
-    if let Err(e) = do_main() {
+    if let Err(e) = do_main(fb).await {
         error!("{e:#?}");
         return ExitCode::FAILURE;
     }
     ExitCode::SUCCESS
 }
 
-fn do_main() -> Result<()> {
+async fn do_main(fb: fbinit::FacebookInit) -> Result<()> {
     let args = Args::parse();
-    tracing_subscriber::FmtSubscriber::builder()
-        .with_max_level(match args.verbose {
-            0 => tracing::Level::ERROR,
-            1 => tracing::Level::WARN,
-            2 => tracing::Level::INFO,
-            3 => tracing::Level::DEBUG,
-            _ => tracing::Level::TRACE,
-        })
-        .finish()
+    let stderr_level = match args.verbose {
+        0 => tracing_subscriber::filter::LevelFilter::ERROR,
+        1 => tracing_subscriber::filter::LevelFilter::WARN,
+        2 => tracing_subscriber::filter::LevelFilter::INFO,
+        3 => tracing_subscriber::filter::LevelFilter::DEBUG,
+        _ => tracing_subscriber::filter::LevelFilter::TRACE,
+    };
+    let stderr_layer = tracing_subscriber::fmt::layer().with_filter(stderr_level);
+    let file_layer = if let Some(log) = &args.log {
+        let file = File::create(log)?;
+        Some(
+            tracing_subscriber::fmt::layer()
+                .with_writer(Mutex::new(file))
+                .with_filter(tracing_subscriber::filter::LevelFilter::DEBUG),
+        )
+    } else {
+        None
+    };
+    tracing_subscriber::registry()
+        .with(stderr_layer)
+        .with(file_layer)
         .init();
-
     match args.sub {
         Sub::Decompress(sub) => sub.run(),
         Sub::Generate(sub) => sub.run(),
         Sub::Parse(sub) => sub.run(),
+        Sub::Snapshot(sub) => sub.run(fb).await,
     }
 }

antlir/antlir2/package_managers/snapshot/src/snapshot.rs

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+use std::collections::BTreeMap;
+use std::io::BufWriter;
+use std::path::PathBuf;
+
+use anyhow::Result;
+use clap::Parser;
+use clap::Subcommand;
+use json_arg::Json;
+use serde::Serialize;
+
+use crate::checksums::Checksums;
+
+mod metadata;
+mod storage;
+use storage::StorageConfig;
+
+#[derive(Debug, Parser)]
+pub(crate) struct Snapshot {
+    #[clap(subcommand)]
+    sub: Sub,
+    #[clap(long)]
+    out: PathBuf,
+    #[clap(long)]
+    storage: Json<StorageConfig>,
+}
+
+#[derive(Debug, Serialize)]
+pub(super) struct Out {
+    pub(super) files: BTreeMap<String, String>,
+    pub(super) checksums: BTreeMap<String, Checksums>,
+}
+
+#[derive(Debug, Subcommand)]
+enum Sub {
+    /// Snapshot metadata tree
+    Metadata(metadata::Metadata),
+}
+
+impl Snapshot {
+    pub(crate) async fn run(self, fb: fbinit::FacebookInit) -> Result<()> {
+        let storage = self.storage.into_inner().build(fb)?;
+        let out = match &self.sub {
+            Sub::Metadata(metadata) => metadata.run(storage).await,
+        }?;
+        let outfile = BufWriter::new(stdio_path::create(&self.out)?);
+        serde_json::to_writer(outfile, &out)?;
+        Ok(())
+    }
+}

antlir/antlir2/package_managers/snapshot/src/snapshot/metadata.rs

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+
+use anyhow::Context;
+use anyhow::Result;
+use clap::Parser;
+use tracing::debug;
+use walkdir::WalkDir;
+
+use super::Out;
+use super::storage::Storage;
+
+#[derive(Parser, Debug)]
+pub(crate) struct Metadata {
+    #[clap(long)]
+    tree: PathBuf,
+}
+
+impl Metadata {
+    #[tracing::instrument(skip(self, storage), ret, err)]
+    pub(crate) async fn run(&self, storage: Box<dyn Storage>) -> Result<Out> {
+        let mut files = BTreeMap::new();
+        let mut checksums = BTreeMap::new();
+        debug!("walking {self:?}");
+
+        for entry in WalkDir::new(&self.tree).follow_links(true) {
+            let entry = entry.context("failed to walk metadata tree")?;
+            debug!("processing {entry:?}");
+            if !entry.file_type().is_file() {
+                debug!("skipping because file type is {:?}", entry.file_type());
+                continue;
+            }
+            let relative = entry
+                .path()
+                .strip_prefix(&self.tree)
+                .context("failed to compute relative path")?;
+            let key = relative.to_str().context("non-utf8 path")?;
+            let result = storage.store(entry.path(), key).await?;
+            let url = result.url.to_string();
+            checksums.insert(url.clone(), result.checksums);
+            files.insert(key.to_owned(), url);
+        }
+
+        Ok(Out { files, checksums })
+    }
+}