Updates for PREVIEW builds (#4108)

## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->
<!--- If it fixes an open issue, please link to the issue here -->
See internal notes

## Description
<!--- Describe your changes in detail -->

## Testing
<!--- Please describe in detail how you tested your changes -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->
----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

Author: landonxjames

tools/ci-build/publisher/src/subcommand/fix_manifests.rs

@@ -11,13 +11,14 @@
 
 use crate::fs::Fs;
 use crate::package::discover_manifests;
-use crate::SDK_REPO_NAME;
 use anyhow::{bail, Context, Result};
 use clap::Parser;
 use semver::Version;
-use smithy_rs_tool_common::{ci::running_in_ci, package::parse_version};
+use smithy_rs_tool_common::{
+    ci::{is_in_example_dir, is_preview_build, running_in_ci},
+    package::parse_version,
+};
 use std::collections::BTreeMap;
-use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
 use toml::value::Table;
 use toml::Value;
@@ -64,6 +65,7 @@ pub async fn subcommand_fix_manifests(
     Ok(())
 }
 
+#[derive(Debug)]
 struct Manifest {
     path: PathBuf,
     metadata: toml::Value,
@@ -82,6 +84,7 @@ impl Manifest {
     }
 }
 
+#[derive(Debug)]
 struct Versions(BTreeMap<String, VersionWithMetadata>);
 #[derive(Copy, Clone)]
 enum FilterType {
@@ -110,6 +113,7 @@ impl Versions {
     }
 }
 
+#[derive(Debug)]
 struct VersionWithMetadata {
     version: Version,
     publish: bool,
@@ -205,32 +209,19 @@ fn fix_dep_sets(versions: &VersionView, metadata: &mut toml::Value) -> Result<us
     Ok(changed)
 }
 
-fn is_example_manifest(manifest_path: impl AsRef<Path>) -> bool {
-    // Examine parent directories until either `examples/` or `aws-sdk-rust/` is found
-    let mut path = manifest_path.as_ref();
-    while let Some(parent) = path.parent() {
-        path = parent;
-        if path.file_name() == Some(OsStr::new("examples")) {
-            return true;
-        } else if path.file_name() == Some(OsStr::new(SDK_REPO_NAME)) {
-            break;
-        }
-    }
-    false
-}
-
 fn conditionally_disallow_publish(
     manifest_path: &Path,
     metadata: &mut toml::Value,
 ) -> Result<bool> {
-    let is_github_actions = running_in_ci();
-    let is_example = is_example_manifest(manifest_path);
+    let is_gh_action_or_smithy_rs_docker = running_in_ci();
+    let is_example = is_in_example_dir(manifest_path);
+    let is_preview_build = is_preview_build();
 
     // Safe-guard to prevent accidental publish to crates.io. Add some friction
     // to publishing from a local development machine by detecting that the tool
     // is not being run from CI, and disallow publish in that case. Also disallow
-    // publishing of examples.
-    if !is_github_actions || is_example {
+    // publishing of examples and Trebuchet preview builds.
+    if !is_gh_action_or_smithy_rs_docker || is_example || is_preview_build {
         if let Some(value) = set_publish_false(manifest_path, metadata, is_example) {
             return Ok(value);
         }
@@ -293,6 +284,13 @@ async fn fix_manifests(
 }
 
 fn fix_manifest(versions: &Versions, manifest: &mut Manifest) -> Result<usize> {
+    // In the case of a preview build we do not update the examples manifests
+    // since most SDKs will not be generated so the particular crate referred to
+    // by an example is unlikely to exist
+    if is_in_example_dir(&manifest.path) && is_preview_build() {
+        debug!(package = ?&manifest.path, "Skipping example package for preview build");
+        return Ok(0);
+    }
     let mut view = versions.published();
     if !manifest.publish()? {
         debug!(package = ?&manifest.path, "package has publishing disabled, allowing unpublished crates to be used");
@@ -428,21 +426,4 @@ mod tests {
             actual_build_deps.to_string()
         );
     }
-
-    #[test]
-    fn test_is_example_manifest() {
-        assert!(!is_example_manifest("aws-sdk-rust/sdk/s3/Cargo.toml"));
-        assert!(!is_example_manifest(
-            "aws-sdk-rust/sdk/aws-config/Cargo.toml"
-        ));
-        assert!(!is_example_manifest(
-            "/path/to/aws-sdk-rust/sdk/aws-config/Cargo.toml"
-        ));
-        assert!(!is_example_manifest("sdk/aws-config/Cargo.toml"));
-        assert!(is_example_manifest("examples/foo/Cargo.toml"));
-        assert!(is_example_manifest("examples/foo/bar/Cargo.toml"));
-        assert!(is_example_manifest(
-            "aws-sdk-rust/examples/foo/bar/Cargo.toml"
-        ));
-    }
 }

tools/ci-build/publisher/src/subcommand/generate_version_manifest.rs

@@ -9,6 +9,7 @@ use anyhow::{bail, Context, Result};
 use clap::Parser;
 use semver::Version;
 use serde::Deserialize;
+use smithy_rs_tool_common::ci::is_preview_build;
 use smithy_rs_tool_common::git::{find_git_repository_root, Git, GitCLI};
 use smithy_rs_tool_common::package::PackageCategory;
 use smithy_rs_tool_common::shell;
@@ -113,8 +114,10 @@ pub async fn subcommand_generate_version_manifest(
         crates,
         release: None,
     };
+
     versions_manifest.release =
         generate_release_metadata(&versions_manifest, previous_release_versions)?;
+
     let manifest_file_name = output_location.join("versions.toml");
     info!("Writing {:?}...", manifest_file_name);
     versions_manifest.write_to_file(&manifest_file_name)?;
@@ -125,14 +128,21 @@ fn generate_release_metadata(
     versions_manifest: &VersionsManifest,
     maybe_previous_release_versions: &Option<PathBuf>,
 ) -> Result<Option<Release>> {
-    if let Some(previous_release_versions) = maybe_previous_release_versions {
+    // For release builds we generate a real release section
+    if let (Some(previous_release_versions), false) =
+        (maybe_previous_release_versions, is_preview_build())
+    {
         let old_versions = VersionsManifest::from_file(previous_release_versions)?;
         Ok(Some(Release {
             tag: None,
             crates: find_released_versions(&old_versions, versions_manifest)?,
         }))
+    // For preview builds we insert an empty section
     } else {
-        Ok(None)
+        Ok(Some(Release {
+            tag: None,
+            crates: BTreeMap::new(),
+        }))
     }
 }
 

tools/ci-build/publisher/src/subcommand/tag_versions_manifest.rs

@@ -21,6 +21,7 @@ pub struct TagVersionsManifestArgs {
 pub fn subcommand_tag_versions_manifest(
     TagVersionsManifestArgs { manifest_path, tag }: &TagVersionsManifestArgs,
 ) -> Result<()> {
+    println!("Tagging manifest at: {manifest_path:#?}");
     let mut manifest = VersionsManifest::from_file(manifest_path)?;
     if let Some(release) = manifest.release.as_mut() {
         release.tag = Some(tag.to_string());

tools/ci-build/sdk-versioner/src/main.rs

@@ -5,6 +5,7 @@
 
 use anyhow::{bail, Context, Result};
 use clap::Parser;
+use smithy_rs_tool_common::ci::{is_in_example_dir, is_preview_build};
 use smithy_rs_tool_common::package::{PackageCategory, SDK_PREFIX};
 use smithy_rs_tool_common::versions_manifest::VersionsManifest;
 use std::ffi::OsStr;
@@ -114,6 +115,13 @@ fn main() -> Result<()> {
         },
     };
 
+    // In the case of a preview build we avoid updating the examples directories since
+    // we only generate the single preview SDK, so most SDKs referred to in the examples
+    // will be missing
+    if is_preview_build() && is_in_example_dir(&args.crate_paths()[0]) {
+        return Ok(());
+    }
+
     let start_time = Instant::now();
     let mut manifest_paths = Vec::new();
     for crate_path in args.crate_paths() {
@@ -236,7 +244,7 @@ fn updated_dependency_value(
             let dependency_path = sdk_path
                 .join(crate_path_name(dependency_name))
                 .canonicalize()
-                .context("failed to canonicalize dependency path")?;
+                .context(format!("failed to canonicalize sdk_path: {sdk_path:#?} with dependency_name: {dependency_name}"))?;
             if let Some(relative_path) = pathdiff::diff_paths(&dependency_path, &crate_path) {
                 value["path"] = toml_edit::value(
                     relative_path

tools/ci-build/smithy-rs-tool-common/src/ci.rs

@@ -2,9 +2,64 @@
  * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0
  */
+use std::ffi::OsStr;
+use std::path::Path;
 
 /// Returns `true` if this code is being run in CI
 pub fn running_in_ci() -> bool {
     std::env::var("GITHUB_ACTIONS").unwrap_or_default() == "true"
         || std::env::var("SMITHY_RS_DOCKER_BUILD_IMAGE").unwrap_or_default() == "1"
 }
+
+/// The `BUILD_TYPE` env var is only set for Codebuild jobs, will always
+/// return `false` in other CI environments
+pub fn is_preview_build() -> bool {
+    let build_type = std::env::var("BUILD_TYPE");
+
+    if let Ok(build_type) = build_type {
+        if build_type.eq_ignore_ascii_case("PREVIEW") {
+            return true;
+        }
+    }
+
+    false
+}
+
+pub fn is_in_example_dir(manifest_path: impl AsRef<Path>) -> bool {
+    let mut path = manifest_path.as_ref();
+    // Check if current dir is examples
+    if path.ends_with("examples") {
+        return true;
+    }
+    // Examine parent directories until either `examples/` or `aws-sdk-rust/` is found
+    while let Some(parent) = path.parent() {
+        path = parent;
+        if path.file_name() == Some(OsStr::new("examples")) {
+            return true;
+        } else if path.file_name() == Some(OsStr::new("aws-sdk-rust")) {
+            break;
+        }
+    }
+    false
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_is_in_example_dir() {
+        assert!(!is_in_example_dir("aws-sdk-rust/sdk/s3/Cargo.toml"));
+        assert!(!is_in_example_dir("aws-sdk-rust/sdk/aws-config/Cargo.toml"));
+        assert!(!is_in_example_dir(
+            "/path/to/aws-sdk-rust/sdk/aws-config/Cargo.toml"
+        ));
+        assert!(!is_in_example_dir("sdk/aws-config/Cargo.toml"));
+        assert!(is_in_example_dir("examples/foo/Cargo.toml"));
+        assert!(is_in_example_dir("examples/foo/bar/Cargo.toml"));
+        assert!(is_in_example_dir(
+            "aws-sdk-rust/examples/foo/bar/Cargo.toml"
+        ));
+        assert!(is_in_example_dir("aws-sdk-rust/examples/"));
+    }
+}