feat(package): add unstable --message-format flag (#15311)

### What does this PR try to resolve?

#11666

This adds an unstable `--message-format` flag to `cargo package` to help
`--list` output in newline-delimited JSON format.

See
<https://github.com/weihanglo/cargo/blob/package-list-fmt/src/doc/man/cargo-package.md#package-options>
for more on what is provided.

Open questions
- `--list json` or `--message-format json`, see
#15311 (comment)
- a single json blob vs N? What is N? See
#15311 (comment)
- Is the current format `plain` or `human`, see
#15311 (comment)
- snake_case or kebab-case, see
#15311 (comment)

### How should we test and review this PR?

* This currently outputs absolute paths.
  If we don't want to show absolute paths in the JSON output,
  could switch to relative path to either package root or cwd
  (I prefer the latter though).
* The actual schema, format option names, and field names is open to
discuss and change. See also
<#12377> for reference.

Author: epage

crates/cargo-util-schemas/src/lib.rs

@@ -10,6 +10,7 @@
 
 pub mod core;
 pub mod manifest;
+pub mod messages;
 #[cfg(feature = "unstable-schema")]
 pub mod schema;
 

crates/cargo-util-schemas/src/messages.rs

@@ -0,0 +1,32 @@
+//! Schemas for JSON messages emitted by Cargo.
+
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+
+/// File information of a package archive generated by `cargo package --list`.
+#[derive(Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+pub struct PackageList {
+    /// The Package ID Spec of the package.
+    pub id: crate::core::PackageIdSpec,
+    /// A map of relative paths in the archive to their detailed file information.
+    pub files: BTreeMap<PathBuf, PackageFile>,
+}
+
+/// Where the file is from.
+#[derive(Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case", tag = "kind")]
+pub enum PackageFile {
+    /// File being copied from another location.
+    Copy {
+        /// An absolute path to the actual file content
+        path: PathBuf,
+    },
+    /// File being generated during packaging
+    Generate {
+        /// An absolute path to the original file the generated one is based on.
+        /// if any.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        path: Option<PathBuf>,
+    },
+}

src/bin/cargo/commands/package.rs

@@ -1,6 +1,8 @@
 use crate::command_prelude::*;
 
-use cargo::ops::{self, PackageOpts};
+use cargo::ops;
+use cargo::ops::PackageMessageFormat;
+use cargo::ops::PackageOpts;
 
 pub fn cli() -> Command {
     subcommand("package")
@@ -30,6 +32,13 @@ pub fn cli() -> Command {
             "exclude-lockfile",
             "Don't include the lock file when packaging",
         ))
+        .arg(
+            opt("message-format", "Output representation (unstable)")
+                .value_name("FMT")
+                // This currently requires and only works with `--list`.
+                .requires("list")
+                .value_parser(PackageMessageFormat::POSSIBLE_VALUES),
+        )
         .arg_silent_suggestion()
         .arg_package_spec_no_all(
             "Package(s) to assemble",
@@ -75,12 +84,21 @@ pub fn exec(gctx: &mut GlobalContext, args: &ArgMatches) -> CliResult {
     }
     let specs = args.packages_from_flags()?;
 
+    let fmt = if let Some(fmt) = args._value_of("message-format") {
+        gctx.cli_unstable()
+            .fail_if_stable_opt("--message-format", 11666)?;
+        fmt.parse()?
+    } else {
+        PackageMessageFormat::Human
+    };
+
     ops::package(
         &ws,
         &PackageOpts {
             gctx,
             verify: !args.flag("no-verify"),
             list: args.flag("list"),
+            fmt,
             check_metadata: !args.flag("no-metadata"),
             allow_dirty: args.flag("allow-dirty"),
             include_lockfile: !args.flag("exclude-lockfile"),

src/cargo/ops/cargo_package/mod.rs

@@ -1,4 +1,6 @@
-use std::collections::{BTreeSet, HashMap};
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
+use std::collections::HashMap;
 use std::fs::{self, File};
 use std::io::prelude::*;
 use std::io::SeekFrom;
@@ -32,6 +34,7 @@ use crate::util::HumanBytes;
 use crate::{drop_println, ops};
 use anyhow::{bail, Context as _};
 use cargo_util::paths;
+use cargo_util_schemas::messages;
 use flate2::{Compression, GzBuilder};
 use tar::{Builder, EntryType, Header, HeaderMode};
 use tracing::debug;
@@ -40,10 +43,38 @@ use unicase::Ascii as UncasedAscii;
 mod vcs;
 mod verify;
 
+/// Message format for `cargo package`.
+///
+/// Currently only affect the output of the `--list` flag.
+#[derive(Debug, Clone)]
+pub enum PackageMessageFormat {
+    Human,
+    Json,
+}
+
+impl PackageMessageFormat {
+    pub const POSSIBLE_VALUES: [&str; 2] = ["human", "json"];
+
+    pub const DEFAULT: &str = "human";
+}
+
+impl std::str::FromStr for PackageMessageFormat {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<PackageMessageFormat, anyhow::Error> {
+        match s {
+            "human" => Ok(PackageMessageFormat::Human),
+            "json" => Ok(PackageMessageFormat::Json),
+            f => bail!("unknown message format `{f}`"),
+        }
+    }
+}
+
 #[derive(Clone)]
 pub struct PackageOpts<'gctx> {
     pub gctx: &'gctx GlobalContext,
     pub list: bool,
+    pub fmt: PackageMessageFormat,
     pub check_metadata: bool,
     pub allow_dirty: bool,
     pub include_lockfile: bool,
@@ -78,9 +109,13 @@ enum FileContents {
 
 enum GeneratedFile {
     /// Generates `Cargo.toml` by rewriting the original.
-    Manifest,
-    /// Generates `Cargo.lock` in some cases (like if there is a binary).
-    Lockfile,
+    ///
+    /// Associated path is the original manifest path.
+    Manifest(PathBuf),
+    /// Generates `Cargo.lock`.
+    ///
+    /// Associated path is the path to the original lock file, if existing.
+    Lockfile(Option<PathBuf>),
     /// Adds a `.cargo_vcs_info.json` file if in a git repo.
     VcsInfo(vcs::VcsInfo),
 }
@@ -236,8 +271,33 @@ fn do_package<'a>(
         let ar_files = prepare_archive(ws, &pkg, &opts)?;
 
         if opts.list {
-            for ar_file in &ar_files {
-                drop_println!(ws.gctx(), "{}", ar_file.rel_str);
+            match opts.fmt {
+                PackageMessageFormat::Human => {
+                    // While this form is called "human",
+                    // it keeps the old file-per-line format for compatibility.
+                    for ar_file in &ar_files {
+                        drop_println!(ws.gctx(), "{}", ar_file.rel_str);
+                    }
+                }
+                PackageMessageFormat::Json => {
+                    let message = messages::PackageList {
+                        id: pkg.package_id().to_spec(),
+                        files: BTreeMap::from_iter(ar_files.into_iter().map(|f| {
+                            let file = match f.contents {
+                                FileContents::OnDisk(path) => messages::PackageFile::Copy { path },
+                                FileContents::Generated(
+                                    GeneratedFile::Manifest(path)
+                                    | GeneratedFile::Lockfile(Some(path)),
+                                ) => messages::PackageFile::Generate { path: Some(path) },
+                                FileContents::Generated(
+                                    GeneratedFile::VcsInfo(_) | GeneratedFile::Lockfile(None),
+                                ) => messages::PackageFile::Generate { path: None },
+                            };
+                            (f.rel_path, file)
+                        })),
+                    };
+                    let _ = ws.gctx().shell().print_json(&message);
+                }
             }
         } else {
             let tarball = create_package(ws, &pkg, ar_files, local_reg.as_ref())?;
@@ -444,7 +504,9 @@ fn build_ar_list(
             .push(ArchiveFile {
                 rel_path: PathBuf::from("Cargo.toml"),
                 rel_str: "Cargo.toml".to_string(),
-                contents: FileContents::Generated(GeneratedFile::Manifest),
+                contents: FileContents::Generated(GeneratedFile::Manifest(
+                    pkg.manifest_path().to_owned(),
+                )),
             });
     } else {
         ws.gctx().shell().warn(&format!(
@@ -454,14 +516,16 @@ fn build_ar_list(
     }
 
     if include_lockfile {
+        let lockfile_path = ws.lock_root().as_path_unlocked().join(LOCKFILE_NAME);
+        let lockfile_path = lockfile_path.exists().then_some(lockfile_path);
         let rel_str = "Cargo.lock";
         result
             .entry(UncasedAscii::new(rel_str))
             .or_insert_with(Vec::new)
             .push(ArchiveFile {
                 rel_path: PathBuf::from(rel_str),
                 rel_str: rel_str.to_string(),
-                contents: FileContents::Generated(GeneratedFile::Lockfile),
+                contents: FileContents::Generated(GeneratedFile::Lockfile(lockfile_path)),
             });
     }
 
@@ -780,8 +844,10 @@ fn tar(
             }
             FileContents::Generated(generated_kind) => {
                 let contents = match generated_kind {
-                    GeneratedFile::Manifest => publish_pkg.manifest().to_normalized_contents()?,
-                    GeneratedFile::Lockfile => build_lock(ws, &publish_pkg, local_reg)?,
+                    GeneratedFile::Manifest(_) => {
+                        publish_pkg.manifest().to_normalized_contents()?
+                    }
+                    GeneratedFile::Lockfile(_) => build_lock(ws, &publish_pkg, local_reg)?,
                     GeneratedFile::VcsInfo(ref s) => serde_json::to_string_pretty(s)?,
                 };
                 header.set_entry_type(EntryType::file());

src/cargo/ops/mod.rs

@@ -10,7 +10,10 @@ pub use self::cargo_fetch::{fetch, FetchOptions};
 pub use self::cargo_install::{install, install_list};
 pub use self::cargo_new::{init, new, NewOptions, NewProjectKind, VersionControl};
 pub use self::cargo_output_metadata::{output_metadata, ExportInfo, OutputMetadataOptions};
-pub use self::cargo_package::{check_yanked, package, PackageOpts};
+pub use self::cargo_package::check_yanked;
+pub use self::cargo_package::package;
+pub use self::cargo_package::PackageMessageFormat;
+pub use self::cargo_package::PackageOpts;
 pub use self::cargo_pkgid::pkgid;
 pub use self::cargo_read_manifest::read_package;
 pub use self::cargo_run::run;

src/cargo/ops/registry/publish.rs

@@ -144,6 +144,7 @@ pub fn publish(ws: &Workspace<'_>, opts: &PublishOpts<'_>) -> CargoResult<()> {
             gctx: opts.gctx,
             verify: opts.verify,
             list: false,
+            fmt: ops::PackageMessageFormat::Human,
             check_metadata: true,
             allow_dirty: opts.allow_dirty,
             include_lockfile: true,

src/doc/man/cargo-package.md

@@ -116,6 +116,48 @@ lock-files will be generated under the assumption that dependencies will be
 published to this registry.
 {{/option}}
 
+{{#option "`--message-format` _fmt_" }}
+Specifies the output message format.
+Currently, it only works with `--list` and affects the file listing format.
+This is unstable and requires `-Zunstable-options`.
+Valid output formats:
+
+- `human` (default): Display in a file-per-line format.
+- `json`: Emit machine-readable JSON information about each package.
+  One package per JSON line (Newline delimited JSON).
+  ```javascript
+  {
+    /* The Package ID Spec of the package. */
+    "id": "path+file:///home/foo#0.0.0",
+    /* Files of this package */
+    "files" {
+      /* Relative path in the archive file. */
+      "Cargo.toml.orig": {
+        /* Where the file is from.
+           - "generate" for file being generated during packaging
+           - "copy" for file being copied from another location.
+        */
+        "kind": "copy",
+        /* For the "copy" kind,
+           it is an absolute path to the actual file content.
+           For the "generate" kind,
+           it is the original file the generated one is based on.
+        */
+        "path": "/home/foo/Cargo.toml"
+      },
+      "Cargo.toml": {
+        "kind": "generate",
+        "path": "/home/foo/Cargo.toml"
+      },
+      "src/main.rs": {
+        "kind": "copy",
+        "path": "/home/foo/src/main.rs"
+      }
+    }
+  }
+  ```
+{{/option}}
+
 {{/options}}
 
 {{> section-package-selection }}

src/doc/man/generated_txt/cargo-package.txt

@@ -114,6 +114,45 @@ OPTIONS
            multiple inter-dependent crates, lock-files will be generated under
            the assumption that dependencies will be published to this registry.
 
+       --message-format fmt
+           Specifies the output message format. Currently, it only works with
+           --list and affects the file listing format. This is unstable and
+           requires -Zunstable-options. Valid output formats:
+
+           o  human (default): Display in a file-per-line format.
+
+           o  json: Emit machine-readable JSON information about each package.
+              One package per JSON line (Newline delimited JSON).
+                  {
+                    /* The Package ID Spec of the package. */
+                    "id": "path+file:///home/foo#0.0.0",
+                    /* Files of this package */
+                    "files" {
+                      /* Relative path in the archive file. */
+                      "Cargo.toml.orig": {
+                        /* Where the file is from.
+                           - "generate" for file being generated during packaging
+                           - "copy" for file being copied from another location.
+                        */
+                        "kind": "copy",
+                        /* For the "copy" kind,
+                           it is an absolute path to the actual file content.
+                           For the "generate" kind,
+                           it is the original file the generated one is based on.
+                        */
+                        "path": "/home/foo/Cargo.toml"
+                      },
+                      "Cargo.toml": {
+                        "kind": "generate",
+                        "path": "/home/foo/Cargo.toml"
+                      },
+                      "src/main.rs": {
+                        "kind": "copy",
+                        "path": "/home/foo/src/main.rs"
+                      }
+                    }
+                  }
+
    Package Selection
        By default, when no package selection options are given, the packages
        selected depend on the selected manifest file (based on the current