[antlir2][unprivileged_dir] add option for buck-out safe encoding

Summary:
`buck2` has rules against certain filename patterns, so this adds an optional base64 filename encoding feature behind a boolean flag. We only escape paths that require escaping, and we track those that have been escaped in a JSON map so they can be reconstructed where needed (e.g. during CAF upload).

This required some refactoring as I needed to move unprivileged_dir to a custom impl rather than leaving it in defs.bzl, given I needed to emit a subtarget. I extracted the shared attrs to a separate attrs.bzl file so they can be imported without circular deps.

Test Plan:
```
$ buck2 test mode/opt fbcode//antlir/antlir2/test_images/package/unprivileged_dir/...
```
https://www.internalfb.com/intern/testinfra/testrun/6473924782953215

```
$ buck2 test mode/opt fbcode//antlir/antlir2/test_images/package/unprivileged_dir/...
```
https://www.internalfb.com/intern/testinfra/testrun/2251800148706544

Reviewed By: vmagro

Differential Revision: D88997743

fbshipit-source-id: e1d1266263534452aedaf87c77196e0bd5c321e1

Author: justintrudell

antlir/antlir2/antlir2_packager/BUCK

@@ -17,6 +17,7 @@ rust_binary(
     visibility = ["PUBLIC"],
     deps = [
         "anyhow",
+        "base64",
         "blake3",
         "bytesize",
         "cap-std",

antlir/antlir2/antlir2_packager/src/unprivileged_dir.rs

@@ -5,19 +5,32 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+use std::collections::BTreeMap;
 use std::fs::Permissions;
+use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs::MetadataExt;
 use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
+use std::path::PathBuf;
 
 use anyhow::Context;
 use anyhow::Result;
+use base64::Engine as _;
+use base64::engine::general_purpose::URL_SAFE;
 use serde::Deserialize;
 use walkdir::WalkDir;
 
+/// Check if a filename contains characters that Buck2 doesn't allow.
+/// Buck2 disallows forward slash '/' and backslash '\' in filenames.
+fn needs_escaping(component: &std::ffi::OsStr) -> bool {
+    component.as_bytes().contains(&b'\\')
+}
+
 #[derive(Debug, Clone, Deserialize)]
 #[serde(deny_unknown_fields)]
-pub struct UnprivilegedDir {}
+pub struct UnprivilegedDir {
+    base64_encoded_filenames: Option<PathBuf>,
+}
 
 impl UnprivilegedDir {
     pub(crate) fn build(
@@ -27,7 +40,13 @@ impl UnprivilegedDir {
         root_guard: Option<antlir2_rootless::EscalationGuard>,
     ) -> Result<()> {
         let layer = layer.canonicalize()?;
+
+        // Track escaped paths: escaped_relative_path -> original_relative_path
+        // such that they can be reconstructed by consumers of the dir
+        let mut escaped_paths: BTreeMap<PathBuf, PathBuf> = BTreeMap::new();
+
         std::fs::create_dir(out).context("while creating root")?;
+
         std::os::unix::fs::lchown(
             out,
             root_guard
@@ -46,7 +65,29 @@ impl UnprivilegedDir {
             if relpath == Path::new("") {
                 continue;
             }
-            let dst = out.join(relpath);
+            let dst = if self.base64_encoded_filenames.is_some() {
+                let encoded_path = relpath
+                    .components()
+                    .map(|component| {
+                        let component_os = component.as_os_str();
+                        if needs_escaping(component_os) {
+                            PathBuf::from(URL_SAFE.encode(component_os.as_bytes()))
+                        } else {
+                            PathBuf::from(component_os)
+                        }
+                    })
+                    .collect::<PathBuf>();
+                if encoded_path != relpath {
+                    // Ensure the mapping always contains absolute paths
+                    escaped_paths.insert(
+                        PathBuf::from("/").join(&encoded_path),
+                        PathBuf::from("/").join(relpath),
+                    );
+                }
+                out.join(encoded_path)
+            } else {
+                out.join(relpath)
+            };
             if entry.file_type().is_dir() {
                 std::fs::create_dir(&dst)
                     .with_context(|| format!("while creating directory '{}'", dst.display()))?;
@@ -56,8 +97,13 @@ impl UnprivilegedDir {
                 std::os::unix::fs::symlink(target, &dst)
                     .with_context(|| format!("while creating symlink '{}'", dst.display()))?;
             } else if entry.file_type().is_file() {
-                std::fs::copy(entry.path(), &dst)
-                    .with_context(|| format!("while copying file '{}'", dst.display()))?;
+                std::fs::copy(entry.path(), &dst).with_context(|| {
+                    format!(
+                        "while copying file '{}' -> '{}'",
+                        entry.path().display(),
+                        dst.display()
+                    )
+                })?;
                 let mut mode = entry.metadata()?.mode();
                 // preserve executable bit
                 if (mode & 0o111) != 0 {
@@ -82,6 +128,17 @@ impl UnprivilegedDir {
             )
             .with_context(|| format!("while chowning '{}'", dst.display()))?;
         }
+
+        if let Some(base64_encoded_filenames) = &self.base64_encoded_filenames {
+            std::fs::write(
+                base64_encoded_filenames,
+                serde_json::to_string_pretty(&escaped_paths)
+                    .context("while serializing escaped paths mapping")?
+                    .as_bytes(),
+            )
+            .context("while writing escaped paths mapping")?;
+        }
+
         Ok(())
     }
 }

antlir/antlir2/bzl/package/attrs.bzl

@@ -0,0 +1,32 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+load("//antlir/antlir2/bzl:platform.bzl", "arch_select")
+load("//antlir/antlir2/bzl/image:cfg.bzl", "attrs_selected_by_cfg")
+load("//antlir/antlir2/features:defs.bzl", "FeaturePluginInfo", "FeaturePluginPluginKind")
+load(":cfg.bzl", "layer_attrs")
+
+# Attrs that are required by all packages
+common_attrs = {
+    "labels": attrs.list(attrs.string(), default = []),
+    "out": attrs.option(attrs.string(doc = "Output filename"), default = None),
+    "_plugins": attrs.list(
+        attrs.dep(providers = [FeaturePluginInfo]),
+        default = [],
+        doc = "Used as a way to pass plugins to anon layer targets",
+    ),
+} | layer_attrs
+
+# Attrs that are not expected for users to pass
+default_attrs = {
+    "_analyze_feature": attrs.exec_dep(default = "antlir//antlir/antlir2/antlir2_depgraph_if:analyze"),
+    "_antlir2": attrs.exec_dep(default = "antlir//antlir/antlir2/antlir2:antlir2"),
+    "_antlir2_packager": attrs.default_only(attrs.exec_dep(default = "antlir//antlir/antlir2/antlir2_packager:antlir2-packager")),
+    "_dot_meta_feature": attrs.dep(default = "antlir//antlir/antlir2/bzl/package:dot-meta", pulls_plugins = [FeaturePluginPluginKind]),
+    "_run_container": attrs.exec_dep(default = "antlir//antlir/antlir2/container_subtarget:run"),
+    "_target_arch": attrs.default_only(attrs.string(
+        default = arch_select(aarch64 = "aarch64", x86_64 = "x86_64"),
+    )),
+} | attrs_selected_by_cfg()

antlir/antlir2/bzl/package/defs.bzl

@@ -5,39 +5,17 @@
 
 load("//antlir/antlir2/bzl:platform.bzl", "arch_select", "os_select")
 load("//antlir/antlir2/bzl:types.bzl", "BuildApplianceInfo", "LayerInfo")
-load("//antlir/antlir2/bzl/image:cfg.bzl", "attrs_selected_by_cfg")
-load("//antlir/antlir2/features:defs.bzl", "FeaturePluginInfo", "FeaturePluginPluginKind")
+load("//antlir/antlir2/features:defs.bzl", "FeaturePluginPluginKind")
 load("//antlir/buck2/bzl:ensure_single_output.bzl", "ensure_single_output")
 load("//antlir/bzl:internal_external.bzl", "internal_external")
+load(":attrs.bzl", "common_attrs", "default_attrs")
 load(":btrfs.bzl", "btrfs")
-load(":cfg.bzl", "layer_attrs", "package_cfg")
+load(":cfg.bzl", "package_cfg")
 load(":gpt.bzl", "GptPartitionSource", "gpt")
 load(":macro.bzl", "package_macro")
 load(":sendstream.bzl", "sendstream_v2")
 load(":stamp_buildinfo.bzl", "stamp_buildinfo_rule")
-
-# Attrs that are required by all packages
-common_attrs = {
-    "labels": attrs.list(attrs.string(), default = []),
-    "out": attrs.option(attrs.string(doc = "Output filename"), default = None),
-    "_plugins": attrs.list(
-        attrs.dep(providers = [FeaturePluginInfo]),
-        default = [],
-        doc = "Used as a way to pass plugins to anon layer targets",
-    ),
-} | layer_attrs
-
-# Attrs that are not expected for users to pass
-default_attrs = {
-    "_analyze_feature": attrs.exec_dep(default = "antlir//antlir/antlir2/antlir2_depgraph_if:analyze"),
-    "_antlir2": attrs.exec_dep(default = "antlir//antlir/antlir2/antlir2:antlir2"),
-    "_antlir2_packager": attrs.default_only(attrs.exec_dep(default = "antlir//antlir/antlir2/antlir2_packager:antlir2-packager")),
-    "_dot_meta_feature": attrs.dep(default = "antlir//antlir/antlir2/bzl/package:dot-meta", pulls_plugins = [FeaturePluginPluginKind]),
-    "_run_container": attrs.exec_dep(default = "antlir//antlir/antlir2/container_subtarget:run"),
-    "_target_arch": attrs.default_only(attrs.string(
-        default = arch_select(aarch64 = "aarch64", x86_64 = "x86_64"),
-    )),
-} | attrs_selected_by_cfg()
+load(":unprivileged_dir.bzl", "unprivileged_dir")
 
 def _generic_impl_with_layer(
         layer: [Dependency, ProviderCollection],
@@ -434,13 +412,6 @@ _ext4, _ext4_anon = _new_package_rule(
     uses_build_appliance = True,
 )
 
-# @unused
-_unprivileged_dir, unprivileged_dir_anon = _new_package_rule(
-    format = "unprivileged_dir",
-    is_dir = True,
-    sudo = True,
-)
-
 # @unused
 _erofs, _erofs_anon = _new_package_rule(
     format = "erofs",
@@ -468,6 +439,6 @@ package = struct(
     tar = package_macro(_tar),
     tar_gz = package_macro(_tar_gz),
     tar_zst = package_macro(tar_zst_rule),
-    unprivileged_dir = package_macro(_unprivileged_dir),
+    unprivileged_dir = unprivileged_dir,
     vfat = package_macro(_vfat),
 )

antlir/antlir2/bzl/package/docker_archive.bzl

@@ -5,8 +5,8 @@
 
 load("//antlir/antlir2/bzl:types.bzl", "BuildApplianceInfo", "LayerInfo")
 load("//antlir/buck2/bzl:ensure_single_output.bzl", "ensure_single_output")
+load(":attrs.bzl", "common_attrs", "default_attrs")
 load(":cfg.bzl", "layer_attrs", "package_cfg")
-load(":defs.bzl", "common_attrs", "default_attrs")
 load(":macro.bzl", "package_macro")
 load(":oci.bzl", "oci_attrs", "oci_rule")
 

antlir/antlir2/bzl/package/oci.bzl

@@ -4,8 +4,8 @@
 # LICENSE file in the root directory of this source tree.
 
 load("//antlir/antlir2/bzl:types.bzl", "LayerInfo")
+load(":attrs.bzl", "common_attrs", "default_attrs")
 load(":cfg.bzl", "layer_attrs", "package_cfg")
-load(":defs.bzl", "common_attrs", "default_attrs")
 load(":macro.bzl", "package_macro")
 
 OciLayer = record(

antlir/antlir2/bzl/package/unprivileged_dir.bzl

@@ -0,0 +1,99 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+load("//antlir/antlir2/bzl:types.bzl", "LayerInfo")
+load("//antlir/antlir2/features:defs.bzl", "FeaturePluginPluginKind")
+load("//antlir/buck2/bzl:ensure_single_output.bzl", "ensure_single_output")
+load(":attrs.bzl", "common_attrs", "default_attrs")
+load(":cfg.bzl", "package_cfg")
+load(":macro.bzl", "package_macro")
+load(":stamp_buildinfo.bzl", "stamp_buildinfo_rule")
+
+def _unprivileged_dir_impl_with_layer(
+        layer: [Dependency, ProviderCollection],
+        *,
+        ctx: AnalysisContext) -> list[Provider]:
+    output_name = ctx.attrs.out or ctx.label.name
+    package = ctx.actions.declare_output(output_name, dir = True)
+
+    encoded_path_mapping = ctx.actions.declare_output("encoded_path_mapping.json")
+    if not ctx.attrs.base64_encode_filenames:
+        ctx.actions.write_json(encoded_path_mapping.as_output(), {})
+    spec = ctx.actions.write_json(
+        "spec.json",
+        {"unprivileged_dir": {
+            "base64_encoded_filenames": encoded_path_mapping.as_output(),
+        } if ctx.attrs.base64_encode_filenames else {}},
+        with_inputs = True,
+    )
+    ctx.actions.run(
+        cmd_args(
+            cmd_args("sudo", "--preserve-env=TMPDIR") if not ctx.attrs._rootless else cmd_args(),
+            ctx.attrs._antlir2_packager[RunInfo],
+            cmd_args(spec, format = "--spec={}"),
+            cmd_args(layer[LayerInfo].contents.subvol_symlink, format = "--layer={}"),
+            "--dir",
+            cmd_args(package.as_output(), format = "--out={}"),
+            "--rootless" if ctx.attrs._rootless else cmd_args(),
+        ),
+        local_only = True,
+        category = "antlir2_package",
+        identifier = "unprivileged_dir",
+    )
+
+    return [DefaultInfo(package, sub_targets = {
+        "base64_encoded_path_mapping": [DefaultInfo(encoded_path_mapping)],
+    })]
+
+def _unprivileged_dir_impl(ctx: AnalysisContext):
+    if ctx.attrs.dot_meta:
+        return ctx.actions.anon_target(stamp_buildinfo_rule, {
+            "build_appliance": ctx.attrs.build_appliance,
+            "flavor": ctx.attrs.flavor,
+            "layer": ctx.attrs.layer,
+            "name": str(ctx.label.raw_target()),
+            "_analyze_feature": ctx.attrs._analyze_feature,
+            "_antlir2": ctx.attrs._antlir2,
+            "_dot_meta_feature": ctx.attrs._dot_meta_feature,
+            "_plugins": ctx.attrs._plugins + (ctx.plugins[FeaturePluginPluginKind] if FeaturePluginPluginKind in ctx.plugins else []),
+            "_rootless": ctx.attrs._rootless,
+            "_run_container": ctx.attrs._run_container,
+            "_target_arch": ctx.attrs._target_arch,
+            "_working_format": ctx.attrs._working_format,
+        }).promise.map(partial(
+            _unprivileged_dir_impl_with_layer,
+            ctx = ctx,
+        ))
+    else:
+        return _unprivileged_dir_impl_with_layer(
+            layer = ctx.attrs.layer,
+            ctx = ctx,
+        )
+
+_unprivileged_dir_attrs = {
+    "base64_encode_filenames": attrs.bool(
+        default = False,
+        doc = "Encode filenames that contain invalid characters (/ or \\) in base64 so they are legal in buck-out",
+    ),
+    "dot_meta": attrs.bool(default = True),
+}
+
+_unprivileged_dir = rule(
+    impl = _unprivileged_dir_impl,
+    cfg = package_cfg,
+    uses_plugins = [FeaturePluginPluginKind],
+    attrs = default_attrs | common_attrs | _unprivileged_dir_attrs,
+)
+
+unprivileged_dir_anon = anon_rule(
+    impl = lambda ctx: _unprivileged_dir_impl_with_layer(ctx.attrs.layer, ctx = ctx),
+    artifact_promise_mappings = {
+        "base64_encoded_path_mapping": lambda x: ensure_single_output(x[DefaultInfo].sub_targets["base64_encoded_path_mapping"]),
+        "package": lambda x: ensure_single_output(x),
+    },
+    attrs = default_attrs | common_attrs | _unprivileged_dir_attrs,
+)
+
+unprivileged_dir = package_macro(_unprivileged_dir)

antlir/antlir2/bzl/package/xar.bzl

@@ -4,8 +4,9 @@
 # LICENSE file in the root directory of this source tree.
 
 load("//antlir/antlir2/features:defs.bzl", "FeaturePluginPluginKind")
+load(":attrs.bzl", "common_attrs", "default_attrs")
 load(":cfg.bzl", "layer_attrs", "package_cfg")
-load(":defs.bzl", "common_attrs", "default_attrs", "squashfs_anon")
+load(":defs.bzl", "squashfs_anon")
 load(":macro.bzl", "package_macro")
 
 def _impl(ctx: AnalysisContext) -> list[Provider]: