diff --git a/common/src/api/external/http_pagination.rs b/common/src/api/external/http_pagination.rs index d4d237b2341..7cfef9a09c0 100644 --- a/common/src/api/external/http_pagination.rs +++ b/common/src/api/external/http_pagination.rs @@ -312,6 +312,19 @@ pub type PaginatedByNameOrId = PaginationParams< pub type PageSelectorByNameOrId = PageSelector, NameOrId>; +pub fn id_pagination<'a, Selector>( + pag_params: &'a DataPageParams, + scan_params: &'a ScanById, +) -> Result, HttpError> +where + Selector: + Clone + Debug + DeserializeOwned + JsonSchema + PartialEq + Serialize, +{ + match scan_params.sort_by { + IdSortMode::IdAscending => Ok(PaginatedBy::Id(pag_params.clone())), + } +} + pub fn name_or_id_pagination<'a, Selector>( pag_params: &'a DataPageParams, scan_params: &'a ScanByNameOrId, diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index fd5cbe38042..98832487d5a 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -982,6 +982,10 @@ impl JsonSchema for Hostname { pub enum ResourceType { AddressLot, AddressLotBlock, + AffinityGroup, + AffinityGroupMember, + AntiAffinityGroup, + AntiAffinityGroupMember, AllowList, BackgroundTask, BgpConfig, @@ -1312,6 +1316,56 @@ pub enum InstanceAutoRestartPolicy { BestEffort, } +// AFFINITY GROUPS + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum AffinityPolicy { + /// If the affinity request cannot be satisfied, allow it anyway. + /// + /// This enables a "best-effort" attempt to satisfy the affinity policy. + Allow, + + /// If the affinity request cannot be satisfied, fail explicitly. + Fail, +} + +/// Describes the scope of affinity for the purposes of co-location. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FailureDomain { + /// Instances are considered co-located if they are on the same sled + Sled, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +#[serde(tag = "type", content = "value", rename_all = "snake_case")] +pub enum AffinityGroupMember { + Instance(Uuid), +} + +impl SimpleIdentity for AffinityGroupMember { + fn id(&self) -> Uuid { + match self { + AffinityGroupMember::Instance(id) => *id, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +#[serde(tag = "type", content = "value", rename_all = "snake_case")] +pub enum AntiAffinityGroupMember { + Instance(Uuid), +} + +impl SimpleIdentity for AntiAffinityGroupMember { + fn id(&self) -> Uuid { + match self { + AntiAffinityGroupMember::Instance(id) => *id, + } + } +} + // DISKS /// View of a Disk diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index bfd7c594a3b..c5f8a1f26c5 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -6191,6 +6191,7 @@ async fn cmd_db_vmm_info( rss_ram: ByteCount(rss), reservoir_ram: ByteCount(reservoir), }, + instance_id: _, } = resource; const SLED_ID: &'static str = "sled ID"; const THREADS: &'static str = "hardware threads"; diff --git a/nexus/auth/src/authz/api_resources.rs b/nexus/auth/src/authz/api_resources.rs index 745a699cf2b..4d588036b7e 100644 --- a/nexus/auth/src/authz/api_resources.rs +++ b/nexus/auth/src/authz/api_resources.rs @@ -713,6 +713,22 @@ authz_resource! { polar_snippet = InProject, } +authz_resource! { + name = "AffinityGroup", + parent = "Project", + primary_key = Uuid, + roles_allowed = false, + polar_snippet = InProject, +} + +authz_resource! { + name = "AntiAffinityGroup", + parent = "Project", + primary_key = Uuid, + roles_allowed = false, + polar_snippet = InProject, +} + authz_resource! { name = "InstanceNetworkInterface", parent = "Instance", diff --git a/nexus/auth/src/authz/oso_generic.rs b/nexus/auth/src/authz/oso_generic.rs index 321bb98b1c6..32b3dbd1f80 100644 --- a/nexus/auth/src/authz/oso_generic.rs +++ b/nexus/auth/src/authz/oso_generic.rs @@ -125,6 +125,8 @@ pub fn make_omicron_oso(log: &slog::Logger) -> Result { Disk::init(), Snapshot::init(), ProjectImage::init(), + AffinityGroup::init(), + AntiAffinityGroup::init(), Instance::init(), IpPool::init(), InstanceNetworkInterface::init(), diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs new file mode 100644 index 00000000000..df97e7d1633 --- /dev/null +++ b/nexus/db-model/src/affinity.rs @@ -0,0 +1,260 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/5.0/. + +// Copyright 2024 Oxide Computer Company + +//! Database representation of affinity and anti-affinity groups + +use super::impl_enum_type; +use super::Name; +use crate::schema::affinity_group; +use crate::schema::affinity_group_instance_membership; +use crate::schema::anti_affinity_group; +use crate::schema::anti_affinity_group_instance_membership; +use crate::typed_uuid::DbTypedUuid; +use chrono::{DateTime, Utc}; +use db_macros::Resource; +use nexus_types::external_api::params; +use nexus_types::external_api::views; +use omicron_common::api::external; +use omicron_common::api::external::IdentityMetadata; +use omicron_uuid_kinds::AffinityGroupKind; +use omicron_uuid_kinds::AffinityGroupUuid; +use omicron_uuid_kinds::AntiAffinityGroupKind; +use omicron_uuid_kinds::AntiAffinityGroupUuid; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceKind; +use omicron_uuid_kinds::InstanceUuid; +use uuid::Uuid; + +impl_enum_type!( + #[derive(SqlType, Debug, QueryId)] + #[diesel(postgres_type(name = "affinity_policy", schema = "public"))] + pub struct AffinityPolicyEnum; + + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Ord, PartialOrd)] + #[diesel(sql_type = AffinityPolicyEnum)] + pub enum AffinityPolicy; + + // Enum values + Fail => b"fail" + Allow => b"allow" +); + +impl From for external::AffinityPolicy { + fn from(policy: AffinityPolicy) -> Self { + match policy { + AffinityPolicy::Fail => Self::Fail, + AffinityPolicy::Allow => Self::Allow, + } + } +} + +impl From for AffinityPolicy { + fn from(policy: external::AffinityPolicy) -> Self { + match policy { + external::AffinityPolicy::Fail => Self::Fail, + external::AffinityPolicy::Allow => Self::Allow, + } + } +} + +impl_enum_type!( + #[derive(SqlType, Debug)] + #[diesel(postgres_type(name = "failure_domain", schema = "public"))] + pub struct FailureDomainEnum; + + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq)] + #[diesel(sql_type = FailureDomainEnum)] + pub enum FailureDomain; + + // Enum values + Sled => b"sled" +); + +impl From for external::FailureDomain { + fn from(domain: FailureDomain) -> Self { + match domain { + FailureDomain::Sled => Self::Sled, + } + } +} + +impl From for FailureDomain { + fn from(domain: external::FailureDomain) -> Self { + match domain { + external::FailureDomain::Sled => Self::Sled, + } + } +} + +#[derive( + Queryable, Insertable, Clone, Debug, Resource, Selectable, PartialEq, +)] +#[diesel(table_name = affinity_group)] +pub struct AffinityGroup { + #[diesel(embed)] + pub identity: AffinityGroupIdentity, + pub project_id: Uuid, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +impl AffinityGroup { + pub fn new(project_id: Uuid, params: params::AffinityGroupCreate) -> Self { + Self { + identity: AffinityGroupIdentity::new( + Uuid::new_v4(), + params.identity, + ), + project_id, + policy: params.policy.into(), + failure_domain: params.failure_domain.into(), + } + } +} + +impl From for views::AffinityGroup { + fn from(group: AffinityGroup) -> Self { + let identity = IdentityMetadata { + id: group.identity.id, + name: group.identity.name.into(), + description: group.identity.description, + time_created: group.identity.time_created, + time_modified: group.identity.time_modified, + }; + Self { + identity, + policy: group.policy.into(), + failure_domain: group.failure_domain.into(), + } + } +} + +/// Describes a set of updates for the [`AffinityGroup`] model. +#[derive(AsChangeset)] +#[diesel(table_name = affinity_group)] +pub struct AffinityGroupUpdate { + pub name: Option, + pub description: Option, + pub time_modified: DateTime, +} + +impl From for AffinityGroupUpdate { + fn from(params: params::AffinityGroupUpdate) -> Self { + Self { + name: params.identity.name.map(Name), + description: params.identity.description, + time_modified: Utc::now(), + } + } +} + +#[derive( + Queryable, Insertable, Clone, Debug, Resource, Selectable, PartialEq, +)] +#[diesel(table_name = anti_affinity_group)] +pub struct AntiAffinityGroup { + #[diesel(embed)] + identity: AntiAffinityGroupIdentity, + pub project_id: Uuid, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +impl AntiAffinityGroup { + pub fn new( + project_id: Uuid, + params: params::AntiAffinityGroupCreate, + ) -> Self { + Self { + identity: AntiAffinityGroupIdentity::new( + Uuid::new_v4(), + params.identity, + ), + project_id, + policy: params.policy.into(), + failure_domain: params.failure_domain.into(), + } + } +} + +impl From for views::AntiAffinityGroup { + fn from(group: AntiAffinityGroup) -> Self { + let identity = IdentityMetadata { + id: group.identity.id, + name: group.identity.name.into(), + description: group.identity.description, + time_created: group.identity.time_created, + time_modified: group.identity.time_modified, + }; + Self { + identity, + policy: group.policy.into(), + failure_domain: group.failure_domain.into(), + } + } +} + +/// Describes a set of updates for the [`AntiAffinityGroup`] model. +#[derive(AsChangeset)] +#[diesel(table_name = anti_affinity_group)] +pub struct AntiAffinityGroupUpdate { + pub name: Option, + pub description: Option, + pub time_modified: DateTime, +} + +impl From for AntiAffinityGroupUpdate { + fn from(params: params::AntiAffinityGroupUpdate) -> Self { + Self { + name: params.identity.name.map(Name), + description: params.identity.description, + time_modified: Utc::now(), + } + } +} + +#[derive(Queryable, Insertable, Clone, Debug, Selectable)] +#[diesel(table_name = affinity_group_instance_membership)] +pub struct AffinityGroupInstanceMembership { + pub group_id: DbTypedUuid, + pub instance_id: DbTypedUuid, +} + +impl AffinityGroupInstanceMembership { + pub fn new(group_id: AffinityGroupUuid, instance_id: InstanceUuid) -> Self { + Self { group_id: group_id.into(), instance_id: instance_id.into() } + } +} + +impl From for external::AffinityGroupMember { + fn from(member: AffinityGroupInstanceMembership) -> Self { + Self::Instance(member.instance_id.into_untyped_uuid()) + } +} + +#[derive(Queryable, Insertable, Clone, Debug, Selectable)] +#[diesel(table_name = anti_affinity_group_instance_membership)] +pub struct AntiAffinityGroupInstanceMembership { + pub group_id: DbTypedUuid, + pub instance_id: DbTypedUuid, +} + +impl AntiAffinityGroupInstanceMembership { + pub fn new( + group_id: AntiAffinityGroupUuid, + instance_id: InstanceUuid, + ) -> Self { + Self { group_id: group_id.into(), instance_id: instance_id.into() } + } +} + +impl From + for external::AntiAffinityGroupMember +{ + fn from(member: AntiAffinityGroupInstanceMembership) -> Self { + Self::Instance(member.instance_id.into_untyped_uuid()) + } +} diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index bec35c233c6..939388174e8 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -10,6 +10,7 @@ extern crate diesel; extern crate newtype_derive; mod address_lot; +mod affinity; mod allow_list; mod bfd; mod bgp; @@ -130,6 +131,7 @@ mod db { pub use self::macaddr::*; pub use self::unsigned::*; pub use address_lot::*; +pub use affinity::*; pub use allow_list::*; pub use bfd::*; pub use bgp::*; diff --git a/nexus/db-model/src/project.rs b/nexus/db-model/src/project.rs index 900a13e9c54..2079751a1c4 100644 --- a/nexus/db-model/src/project.rs +++ b/nexus/db-model/src/project.rs @@ -2,9 +2,15 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use super::{Disk, Generation, Instance, Name, Snapshot, Vpc}; +use super::{ + AffinityGroup, AntiAffinityGroup, Disk, Generation, Instance, Name, + Snapshot, Vpc, +}; use crate::collection::DatastoreCollectionConfig; -use crate::schema::{disk, image, instance, project, snapshot, vpc}; +use crate::schema::{ + affinity_group, anti_affinity_group, disk, image, instance, project, + snapshot, vpc, +}; use crate::Image; use chrono::{DateTime, Utc}; use db_macros::Resource; @@ -69,6 +75,20 @@ impl DatastoreCollectionConfig for Project { type CollectionIdColumn = instance::dsl::project_id; } +impl DatastoreCollectionConfig for Project { + type CollectionId = Uuid; + type GenerationNumberColumn = project::dsl::rcgen; + type CollectionTimeDeletedColumn = project::dsl::time_deleted; + type CollectionIdColumn = affinity_group::dsl::project_id; +} + +impl DatastoreCollectionConfig for Project { + type CollectionId = Uuid; + type GenerationNumberColumn = project::dsl::rcgen; + type CollectionTimeDeletedColumn = project::dsl::time_deleted; + type CollectionIdColumn = anti_affinity_group::dsl::project_id; +} + impl DatastoreCollectionConfig for Project { type CollectionId = Uuid; type GenerationNumberColumn = project::dsl::rcgen; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index f734d1e88f3..8881d7dd0ed 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -468,6 +468,48 @@ table! { } } +table! { + affinity_group (id) { + id -> Uuid, + name -> Text, + description -> Text, + time_created -> Timestamptz, + time_modified -> Timestamptz, + time_deleted -> Nullable, + project_id -> Uuid, + policy -> crate::AffinityPolicyEnum, + failure_domain -> crate::FailureDomainEnum, + } +} + +table! { + anti_affinity_group (id) { + id -> Uuid, + name -> Text, + description -> Text, + time_created -> Timestamptz, + time_modified -> Timestamptz, + time_deleted -> Nullable, + project_id -> Uuid, + policy -> crate::AffinityPolicyEnum, + failure_domain -> crate::FailureDomainEnum, + } +} + +table! { + affinity_group_instance_membership (group_id, instance_id) { + group_id -> Uuid, + instance_id -> Uuid, + } +} + +table! { + anti_affinity_group_instance_membership (group_id, instance_id) { + group_id -> Uuid, + instance_id -> Uuid, + } +} + table! { metric_producer (id) { id -> Uuid, @@ -915,10 +957,11 @@ table! { sled_resource (id) { id -> Uuid, sled_id -> Uuid, - kind -> crate::SledResourceKindEnum, hardware_threads -> Int8, rss_ram -> Int8, reservoir_ram -> Int8, + kind -> crate::SledResourceKindEnum, + instance_id -> Nullable, } } @@ -2025,6 +2068,10 @@ allow_tables_to_appear_in_same_query!( allow_tables_to_appear_in_same_query!(hw_baseboard_id, inv_sled_agent,); allow_tables_to_appear_in_same_query!( + anti_affinity_group, + anti_affinity_group_instance_membership, + affinity_group, + affinity_group_instance_membership, bp_omicron_zone, bp_target, rendezvous_debug_dataset, diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 150f4d092df..852de12f4c5 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(122, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(123, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(123, "affinity"), KnownVersion::new(122, "tuf-artifact-replication"), KnownVersion::new(121, "dataset-to-crucible-dataset"), KnownVersion::new(120, "rendezvous-debug-dataset"), diff --git a/nexus/db-model/src/sled_resource.rs b/nexus/db-model/src/sled_resource.rs index 27471f4d7a9..de5833aa41f 100644 --- a/nexus/db-model/src/sled_resource.rs +++ b/nexus/db-model/src/sled_resource.rs @@ -3,9 +3,19 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use crate::schema::sled_resource; +use crate::typed_uuid::DbTypedUuid; use crate::{ByteCount, SledResourceKind, SqlU32}; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceKind; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisUuid; +use omicron_uuid_kinds::SledKind; +use omicron_uuid_kinds::SledUuid; use uuid::Uuid; +type DbInstanceUuid = DbTypedUuid; +type DbSledUuid = DbTypedUuid; + #[derive(Clone, Selectable, Queryable, Insertable, Debug)] #[diesel(table_name = sled_resource)] pub struct Resources { @@ -29,20 +39,28 @@ impl Resources { #[diesel(table_name = sled_resource)] pub struct SledResource { pub id: Uuid, - pub sled_id: Uuid, - pub kind: SledResourceKind, + pub sled_id: DbSledUuid, #[diesel(embed)] pub resources: Resources, + + pub kind: SledResourceKind, + pub instance_id: Option, } impl SledResource { - pub fn new( - id: Uuid, - sled_id: Uuid, - kind: SledResourceKind, + pub fn new_for_vmm( + id: PropolisUuid, + instance_id: InstanceUuid, + sled_id: SledUuid, resources: Resources, ) -> Self { - Self { id, sled_id, kind, resources } + Self { + id: id.into_untyped_uuid(), + instance_id: Some(instance_id.into()), + sled_id: sled_id.into(), + kind: SledResourceKind::Instance, + resources, + } } } diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs new file mode 100644 index 00000000000..e09a0e43406 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -0,0 +1,2701 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! [`DataStore`] methods on Affinity Groups + +use super::DataStore; +use crate::authz; +use crate::authz::ApiResource; +use crate::db; +use crate::db::collection_insert::AsyncInsertError; +use crate::db::collection_insert::DatastoreCollection; +use crate::db::datastore::OpContext; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; +use crate::db::identity::Resource; +use crate::db::model::AffinityGroup; +use crate::db::model::AffinityGroupInstanceMembership; +use crate::db::model::AffinityGroupUpdate; +use crate::db::model::AntiAffinityGroup; +use crate::db::model::AntiAffinityGroupInstanceMembership; +use crate::db::model::AntiAffinityGroupUpdate; +use crate::db::model::InstanceState; +use crate::db::model::Name; +use crate::db::model::Project; +use crate::db::pagination::paginated; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::AsyncRunQueryDsl; +use chrono::Utc; +use diesel::prelude::*; +use omicron_common::api::external; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; +use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; +use omicron_common::api::external::LookupType; +use omicron_common::api::external::ResourceType; +use omicron_common::api::external::UpdateResult; +use omicron_uuid_kinds::AffinityGroupUuid; +use omicron_uuid_kinds::AntiAffinityGroupUuid; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; +use ref_cast::RefCast; + +impl DataStore { + pub async fn affinity_group_list( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + use db::schema::affinity_group::dsl; + + opctx.authorize(authz::Action::ListChildren, authz_project).await?; + + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::affinity_group, dsl::id, &pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::affinity_group, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::project_id.eq(authz_project.id())) + .filter(dsl::time_deleted.is_null()) + .select(AffinityGroup::as_select()) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn anti_affinity_group_list( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + use db::schema::anti_affinity_group::dsl; + + opctx.authorize(authz::Action::ListChildren, authz_project).await?; + + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::anti_affinity_group, dsl::id, &pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::anti_affinity_group, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::project_id.eq(authz_project.id())) + .filter(dsl::time_deleted.is_null()) + .select(AntiAffinityGroup::as_select()) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn affinity_group_create( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + group: AffinityGroup, + ) -> CreateResult { + use db::schema::affinity_group::dsl; + + opctx.authorize(authz::Action::CreateChild, authz_project).await?; + + let conn = self.pool_connection_authorized(opctx).await?; + let name = group.name().as_str().to_string(); + + let affinity_group: AffinityGroup = Project::insert_resource( + authz_project.id(), + diesel::insert_into(dsl::affinity_group).values(group), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| match e { + AsyncInsertError::CollectionNotFound => authz_project.not_found(), + AsyncInsertError::DatabaseError(diesel_error) => { + public_error_from_diesel( + diesel_error, + ErrorHandler::Conflict(ResourceType::AffinityGroup, &name), + ) + } + })?; + Ok(affinity_group) + } + + pub async fn anti_affinity_group_create( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + group: AntiAffinityGroup, + ) -> CreateResult { + use db::schema::anti_affinity_group::dsl; + + opctx.authorize(authz::Action::CreateChild, authz_project).await?; + + let conn = self.pool_connection_authorized(opctx).await?; + let name = group.name().as_str().to_string(); + + let anti_affinity_group: AntiAffinityGroup = Project::insert_resource( + authz_project.id(), + diesel::insert_into(dsl::anti_affinity_group).values(group), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| match e { + AsyncInsertError::CollectionNotFound => authz_project.not_found(), + AsyncInsertError::DatabaseError(diesel_error) => { + public_error_from_diesel( + diesel_error, + ErrorHandler::Conflict( + ResourceType::AntiAffinityGroup, + &name, + ), + ) + } + })?; + Ok(anti_affinity_group) + } + + pub async fn affinity_group_update( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + updates: AffinityGroupUpdate, + ) -> UpdateResult { + opctx.authorize(authz::Action::Modify, authz_affinity_group).await?; + + use db::schema::affinity_group::dsl; + diesel::update(dsl::affinity_group) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_affinity_group.id())) + .set(updates) + .returning(AffinityGroup::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_affinity_group), + ) + }) + } + + pub async fn affinity_group_delete( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_affinity_group).await?; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("affinity_group_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + use db::schema::affinity_group::dsl as group_dsl; + let now = Utc::now(); + + // Delete the Affinity Group + diesel::update(group_dsl::affinity_group) + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_affinity_group.id())) + .set(group_dsl::time_deleted.eq(now)) + .returning(AffinityGroup::as_returning()) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_affinity_group, + ), + ) + }) + })?; + + // Ensure all memberships in the affinity group are deleted + use db::schema::affinity_group_instance_membership::dsl as member_dsl; + diesel::delete(member_dsl::affinity_group_instance_membership) + .filter(member_dsl::group_id.eq(authz_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn anti_affinity_group_update( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + updates: AntiAffinityGroupUpdate, + ) -> UpdateResult { + opctx + .authorize(authz::Action::Modify, authz_anti_affinity_group) + .await?; + + use db::schema::anti_affinity_group::dsl; + diesel::update(dsl::anti_affinity_group) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_anti_affinity_group.id())) + .set(updates) + .returning(AntiAffinityGroup::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_anti_affinity_group), + ) + }) + } + + pub async fn anti_affinity_group_delete( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + ) -> DeleteResult { + opctx + .authorize(authz::Action::Delete, authz_anti_affinity_group) + .await?; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + use db::schema::anti_affinity_group::dsl as group_dsl; + let now = Utc::now(); + + // Delete the Anti Affinity Group + diesel::update(group_dsl::anti_affinity_group) + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) + .set(group_dsl::time_deleted.eq(now)) + .returning(AntiAffinityGroup::as_returning()) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + // Ensure all memberships in the anti affinity group are deleted + use db::schema::anti_affinity_group_instance_membership::dsl as member_dsl; + diesel::delete(member_dsl::anti_affinity_group_instance_membership) + .filter(member_dsl::group_id.eq(authz_anti_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn affinity_group_member_list( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, authz_affinity_group).await?; + + use db::schema::affinity_group_instance_membership::dsl; + match pagparams { + PaginatedBy::Id(pagparams) => paginated( + dsl::affinity_group_instance_membership, + dsl::instance_id, + &pagparams, + ), + PaginatedBy::Name(_) => { + return Err(Error::invalid_request( + "Cannot paginate group members by name", + )); + } + } + .filter(dsl::group_id.eq(authz_affinity_group.id())) + .select(AffinityGroupInstanceMembership::as_select()) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn anti_affinity_group_member_list( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; + + use db::schema::anti_affinity_group_instance_membership::dsl; + match pagparams { + PaginatedBy::Id(pagparams) => paginated( + dsl::anti_affinity_group_instance_membership, + dsl::instance_id, + &pagparams, + ), + PaginatedBy::Name(_) => { + return Err(Error::invalid_request( + "Cannot paginate group members by name", + )); + } + } + .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) + .select(AntiAffinityGroupInstanceMembership::as_select()) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn affinity_group_member_view( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + member: external::AffinityGroupMember, + ) -> Result { + opctx.authorize(authz::Action::Read, authz_affinity_group).await?; + let conn = self.pool_connection_authorized(opctx).await?; + + let instance_id = match member { + external::AffinityGroupMember::Instance(id) => id, + }; + + use db::schema::affinity_group_instance_membership::dsl; + dsl::affinity_group_instance_membership + .filter(dsl::group_id.eq(authz_affinity_group.id())) + .filter(dsl::instance_id.eq(instance_id)) + .select(AffinityGroupInstanceMembership::as_select()) + .get_result_async(&*conn) + .await + .map(|m| m.into()) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AffinityGroupMember, + LookupType::by_id(instance_id), + ), + ) + }) + } + + pub async fn anti_affinity_group_member_view( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + member: external::AntiAffinityGroupMember, + ) -> Result { + opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; + let conn = self.pool_connection_authorized(opctx).await?; + + let instance_id = match member { + external::AntiAffinityGroupMember::Instance(id) => id, + }; + + use db::schema::anti_affinity_group_instance_membership::dsl; + dsl::anti_affinity_group_instance_membership + .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) + .filter(dsl::instance_id.eq(instance_id)) + .select(AntiAffinityGroupInstanceMembership::as_select()) + .get_result_async(&*conn) + .await + .map(|m| m.into()) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AntiAffinityGroupMember, + LookupType::by_id(instance_id), + ), + ) + }) + } + + pub async fn affinity_group_member_add( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + member: external::AffinityGroupMember, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, authz_affinity_group).await?; + + let instance_id = match member { + external::AffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("affinity_group_member_add") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::affinity_group::dsl as group_dsl; + use db::schema::affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists, and has no VMM + // + // NOTE: I'd prefer to use the "LookupPath" infrastructure + // to look up the path, but that API does not give the + // option to use the transaction's database connection. + // + // Looking up the instance on a different database + // connection than the transaction risks several concurrency + // issues, so we do the lookup manually. + let instance_state = instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::state) + .first_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + // NOTE: It may be possible to add non-stopped instances to + // affinity groups, depending on where they have already + // been placed. However, only operating on "stopped" + // instances is much easier to work with, as it does not + // require any understanding of the group policy. + match instance_state { + InstanceState::NoVmm => (), + other => { + return Err(err.bail(Error::invalid_request( + format!( + "Instance cannot be added to affinity group in state: {other}" + ) + ))); + }, + } + + diesel::insert_into(membership_dsl::affinity_group_instance_membership) + .values(AffinityGroupInstanceMembership::new( + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), + InstanceUuid::from_untyped_uuid(instance_id), + )) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::AffinityGroupMember, + &instance_id.to_string(), + ), + ) + }) + })?; + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn anti_affinity_group_member_add( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + member: external::AntiAffinityGroupMember, + ) -> Result<(), Error> { + opctx + .authorize(authz::Action::Modify, authz_anti_affinity_group) + .await?; + + let instance_id = match member { + external::AntiAffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_member_add") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::anti_affinity_group::dsl as group_dsl; + use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::anti_affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists, and has no VMM + let instance_state = instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::state) + .first_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + // NOTE: It may be possible to add non-stopped instances to + // anti affinity groups, depending on where they have already + // been placed. However, only operating on "stopped" + // instances is much easier to work with, as it does not + // require any understanding of the group policy. + match instance_state { + InstanceState::NoVmm => (), + other => { + return Err(err.bail(Error::invalid_request( + format!( + "Instance cannot be added to anti-affinity group in state: {other}" + ) + ))); + }, + } + + diesel::insert_into(membership_dsl::anti_affinity_group_instance_membership) + .values(AntiAffinityGroupInstanceMembership::new( + AntiAffinityGroupUuid::from_untyped_uuid(authz_anti_affinity_group.id()), + InstanceUuid::from_untyped_uuid(instance_id), + )) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::AntiAffinityGroupMember, + &instance_id.to_string(), + ), + ) + }) + })?; + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn instance_affinity_group_memberships_delete( + &self, + opctx: &OpContext, + instance_id: InstanceUuid, + ) -> Result<(), Error> { + use db::schema::affinity_group_instance_membership::dsl; + + diesel::delete(dsl::affinity_group_instance_membership) + .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) + } + + pub async fn instance_anti_affinity_group_memberships_delete( + &self, + opctx: &OpContext, + instance_id: InstanceUuid, + ) -> Result<(), Error> { + use db::schema::anti_affinity_group_instance_membership::dsl; + + diesel::delete(dsl::anti_affinity_group_instance_membership) + .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) + } + + pub async fn affinity_group_member_delete( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + member: external::AffinityGroupMember, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, authz_affinity_group).await?; + + let instance_id = match member { + external::AffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("affinity_group_member_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::affinity_group::dsl as group_dsl; + use db::schema::affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists + instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + let rows = diesel::delete(membership_dsl::affinity_group_instance_membership) + .filter(membership_dsl::group_id.eq(authz_affinity_group.id())) + .filter(membership_dsl::instance_id.eq(instance_id)) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + if rows == 0 { + return Err(err.bail(LookupType::ById(instance_id).into_not_found( + ResourceType::AffinityGroupMember, + ))); + } + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn anti_affinity_group_member_delete( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + member: external::AntiAffinityGroupMember, + ) -> Result<(), Error> { + opctx + .authorize(authz::Action::Modify, authz_anti_affinity_group) + .await?; + + let instance_id = match member { + external::AntiAffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_member_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::anti_affinity_group::dsl as group_dsl; + use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::anti_affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists + instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + let rows = diesel::delete(membership_dsl::anti_affinity_group_instance_membership) + .filter(membership_dsl::group_id.eq(authz_anti_affinity_group.id())) + .filter(membership_dsl::instance_id.eq(instance_id)) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + if rows == 0 { + return Err(err.bail(LookupType::ById(instance_id).into_not_found( + ResourceType::AntiAffinityGroupMember, + ))); + } + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::db::lookup::LookupPath; + use crate::db::pub_test_utils::TestDatabase; + use nexus_db_model::Instance; + use nexus_types::external_api::params; + use omicron_common::api::external::{ + self, ByteCount, DataPageParams, IdentityMetadataCreateParams, + }; + use omicron_test_utils::dev; + use omicron_uuid_kinds::InstanceUuid; + use std::num::NonZeroU32; + + // Helper function for creating a project + async fn create_project( + opctx: &OpContext, + datastore: &DataStore, + name: &str, + ) -> (authz::Project, Project) { + let authz_silo = opctx.authn.silo_required().unwrap(); + let project = Project::new( + authz_silo.id(), + params::ProjectCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + }, + ); + datastore.project_create(&opctx, project).await.unwrap() + } + + // Helper function for creating an affinity group with + // arbitrary configuration. + async fn create_affinity_group( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, + ) -> CreateResult { + let group = AffinityGroup::new( + authz_project.id(), + params::AffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + policy: external::AffinityPolicy::Fail, + failure_domain: external::FailureDomain::Sled, + }, + ); + datastore.affinity_group_create(&opctx, &authz_project, group).await + } + + // Helper function for creating an anti-affinity group with + // arbitrary configuration. + async fn create_anti_affinity_group( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, + ) -> CreateResult { + let group = AntiAffinityGroup::new( + authz_project.id(), + params::AntiAffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + policy: external::AffinityPolicy::Fail, + failure_domain: external::FailureDomain::Sled, + }, + ); + datastore + .anti_affinity_group_create(&opctx, &authz_project, group) + .await + } + + // Helper function for creating an instance without a VMM. + async fn create_instance_record( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, + ) -> Instance { + let instance = Instance::new( + InstanceUuid::new_v4(), + authz_project.id(), + ¶ms::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + ncpus: 2i64.try_into().unwrap(), + memory: ByteCount::from_gibibytes_u32(16), + hostname: "myhostname".try_into().unwrap(), + user_data: Vec::new(), + network_interfaces: + params::InstanceNetworkInterfaceAttachment::None, + external_ips: Vec::new(), + disks: Vec::new(), + boot_disk: None, + ssh_public_keys: None, + start: false, + auto_restart_policy: Default::default(), + }, + ); + + let instance = datastore + .project_create_instance(&opctx, &authz_project, instance) + .await + .unwrap(); + + set_instance_state_stopped(&datastore, instance.id()).await; + + instance + } + + // Helper for explicitly modifying instance state. + // + // The interaction we typically use to create and modify instance state + // is more complex in production, since it's the result of a back-and-forth + // between Nexus and Sled Agent, using carefully crafted rcgen values. + // + // Here, we just set the value of state explicitly. Be warned, there + // are no guardrails! + async fn set_instance_state_stopped( + datastore: &DataStore, + instance: uuid::Uuid, + ) { + use db::schema::instance::dsl; + diesel::update(dsl::instance) + .filter(dsl::id.eq(instance)) + .set(( + dsl::state.eq(db::model::InstanceState::NoVmm), + dsl::active_propolis_id.eq(None::), + )) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + } + + async fn set_instance_state_running( + datastore: &DataStore, + instance: uuid::Uuid, + ) { + use db::schema::instance::dsl; + diesel::update(dsl::instance) + .filter(dsl::id.eq(instance)) + .set(( + dsl::state.eq(db::model::InstanceState::Vmm), + dsl::active_propolis_id.eq(uuid::Uuid::new_v4()), + )) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn affinity_groups_are_project_scoped() { + // Setup + let logctx = dev::test_setup_log("affinity_groups_are_project_scoped"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let (authz_project, _) = + create_project(&opctx, &datastore, "my-project").await; + + let (authz_other_project, _) = + create_project(&opctx, &datastore, "my-other-project").await; + + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + + // To start: No groups exist + let groups = datastore + .affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Create a group + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // Now when we list groups, we'll see the one we created. + let groups = datastore + .affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0], group); + + // This group won't appear in the other project + let groups = datastore + .affinity_group_list(&opctx, &authz_other_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_groups_are_project_scoped() { + // Setup + let logctx = + dev::test_setup_log("anti_affinity_groups_are_project_scoped"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let (authz_project, _) = + create_project(&opctx, &datastore, "my-project").await; + + let (authz_other_project, _) = + create_project(&opctx, &datastore, "my-other-project").await; + + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + + // To start: No groups exist + let groups = datastore + .anti_affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Create a group + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // Now when we list groups, we'll see the one we created. + let groups = datastore + .anti_affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0], group); + + // This group won't appear in the other project + let groups = datastore + .anti_affinity_group_list(&opctx, &authz_other_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_groups_prevent_project_deletion() { + // Setup + let logctx = + dev::test_setup_log("affinity_groups_prevent_project_deletion"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, mut project) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // If we try to delete the project, we'll fail. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string() + .contains("project to be deleted contains an affinity group"), + "{err:?}" + ); + + // Delete the group, then try to delete the project again. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore.affinity_group_delete(&opctx, &authz_group).await.unwrap(); + + // When the group was created, it bumped the rcgen in the project. If we + // have an old view of the project, we expect a "concurrent + // modification" error. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(err.to_string().contains("concurrent modification"), "{err:?}"); + + // If we update rcgen, however, and the group has been deleted, + // we can successfully delete the project. + project.rcgen = project.rcgen.next().into(); + datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap(); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_groups_prevent_project_deletion() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_groups_prevent_project_deletion", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, mut project) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // If we try to delete the project, we'll fail. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string().contains( + "project to be deleted contains an anti affinity group" + ), + "{err:?}" + ); + + // Delete the group, then try to delete the project again. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + + // When the group was created, it bumped the rcgen in the project. If we + // have an old view of the project, we expect a "concurrent + // modification" error. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(err.to_string().contains("concurrent modification"), "{err:?}"); + + // If we update rcgen, however, and the group has been deleted, + // we can successfully delete the project. + project.rcgen = project.rcgen.next().into(); + datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap(); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_names_are_unique_in_project() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_names_are_unique_in_project"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create two projects + let (authz_project1, _) = + create_project(&opctx, &datastore, "my-project-1").await; + let (authz_project2, _) = + create_project(&opctx, &datastore, "my-project-2").await; + + // We can create a group wiht the same name in different projects + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap(); + create_affinity_group(&opctx, &datastore, &authz_project2, "my-group") + .await + .unwrap(); + + // If we try to create a new group with the same name in the same + // project, we'll see an error. + let err = create_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap_err(); + assert!( + matches!(&err, Error::ObjectAlreadyExists { + type_name, + object_name, + } if *type_name == ResourceType::AffinityGroup && + *object_name == "my-group"), + "Unexpected error: {err:?}" + ); + + // If we delete the group from the project, we can re-use the name. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore.affinity_group_delete(&opctx, &authz_group).await.unwrap(); + + create_affinity_group(&opctx, &datastore, &authz_project1, "my-group") + .await + .expect("Should have been able to re-use name after deletion"); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_names_are_unique_in_project() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_names_are_unique_in_project", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create two projects + let (authz_project1, _) = + create_project(&opctx, &datastore, "my-project-1").await; + let (authz_project2, _) = + create_project(&opctx, &datastore, "my-project-2").await; + + // We can create a group wiht the same name in different projects + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap(); + create_anti_affinity_group( + &opctx, + &datastore, + &authz_project2, + "my-group", + ) + .await + .unwrap(); + + // If we try to create a new group with the same name in the same + // project, we'll see an error. + let err = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap_err(); + assert!( + matches!(&err, Error::ObjectAlreadyExists { + type_name, + object_name, + } if *type_name == ResourceType::AntiAffinityGroup && + *object_name == "my-group"), + "Unexpected error: {err:?}" + ); + + // If we delete the group from the project, we can re-use the name. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + + create_anti_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .expect("Should have been able to re-use name after deletion"); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_add_list_remove() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_membership_add_list_remove"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance as a member to the group + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // We should now be able to list the new member + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list + datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_add_list_remove() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_membership_add_list_remove", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance as a member to the group + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // We should now be able to list the new member + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AntiAffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list + datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_add_remove_instance_with_vmm() { + // Setup + let logctx = dev::test_setup_log( + "affinity_group_membership_add_remove_instance_with_vmm", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance with a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + set_instance_state_running(&datastore, instance.id()).await; + + // Cannot add the instance to the group while it's running. + let err = datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err( + "Shouldn't be able to add running instances to affinity groups", + ); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string().contains( + "Instance cannot be added to affinity group in state" + ), + "{err:?}" + ); + + // If we stop the instance, we can add it to the group. + set_instance_state_stopped(&datastore, instance.id()).await; + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Now we can set the instance state to "running" once more. + set_instance_state_running(&datastore, instance.id()).await; + + // We should now be able to list the new member + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list -- even + // though it's running! + datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_add_remove_instance_with_vmm() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_membership_add_remove_instance_with_vmm", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance with a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + set_instance_state_running(&datastore, instance.id()).await; + + // Cannot add the instance to the group while it's running. + let err = datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err( + "Shouldn't be able to add running instances to anti-affinity groups", + ); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string().contains( + "Instance cannot be added to anti-affinity group in state" + ), + "{err:?}" + ); + + // If we stop the instance, we can add it to the group. + set_instance_state_stopped(&datastore, instance.id()).await; + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Now we can set the instance state to "running" once more. + set_instance_state_running(&datastore, instance.id()).await; + + // We should now be able to list the new member + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AntiAffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list -- even + // though it's running! + datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_delete_group_deletes_members() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_delete_group_deletes_members"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the group + datastore.affinity_group_delete(&opctx, &authz_group).await.unwrap(); + + // Confirm that no instance members exist + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_delete_group_deletes_members() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_delete_group_deletes_members", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the group + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + + // Confirm that no instance members exist + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_delete_instance_deletes_membership() { + // Setup + let logctx = dev::test_setup_log( + "affinity_group_delete_instance_deletes_membership", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the instance + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + + // Confirm that no instance members exist + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_delete_instance_deletes_membership() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_delete_instance_deletes_membership", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the instance + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + + // Confirm that no instance members exist + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_for_deleted_objects() { + // Setup + let logctx = dev::test_setup_log( + "affinity_group_membership_for_deleted_objects", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + + struct TestArgs { + // Does the group exist? + group: bool, + // Does the instance exist? + instance: bool, + } + + let args = [ + TestArgs { group: false, instance: false }, + TestArgs { group: true, instance: false }, + TestArgs { group: false, instance: true }, + ]; + + for arg in args { + // Create an affinity group, and maybe delete it. + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + if !arg.group { + datastore + .affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + + // Create an instance, and maybe delete it. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + if !arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + + // Try to add the instance to the group. + // + // Expect to see specific errors, depending on whether or not the + // group/instance exist. + let err = datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Do the same thing, but for group membership removal. + let err = datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Cleanup, if we actually created anything. + if arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + if arg.group { + datastore + .affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + } + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_for_deleted_objects() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_membership_for_deleted_objects", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + + struct TestArgs { + // Does the group exist? + group: bool, + // Does the instance exist? + instance: bool, + } + + let args = [ + TestArgs { group: false, instance: false }, + TestArgs { group: true, instance: false }, + TestArgs { group: false, instance: true }, + ]; + + for arg in args { + // Create an anti-affinity group, and maybe delete it. + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + if !arg.group { + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + + // Create an instance, and maybe delete it. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + if !arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + + // Try to add the instance to the group. + // + // Expect to see specific errors, depending on whether or not the + // group/instance exist. + let err = datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AntiAffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Do the same thing, but for group membership removal. + let err = datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AntiAffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Cleanup, if we actually created anything. + if arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + if arg.group { + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + } + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_idempotency() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_membership_idempotency"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // Create an instance + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance to the group + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Add the instance to the group again + let err = datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectAlreadyExists { + type_name: ResourceType::AffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + // We should still only observe a single member in the group. + // + // Two calls to "affinity_group_member_add" should be the same + // as a single call. + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + + // We should be able to delete the membership idempotently. + datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let err = datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectNotFound { + type_name: ResourceType::AffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_idempotency() { + // Setup + let logctx = + dev::test_setup_log("anti_affinity_group_membership_idempotency"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // Create an instance + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance to the group + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Add the instance to the group again + let err = datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectAlreadyExists { + type_name: ResourceType::AntiAffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + // We should still only observe a single member in the group. + // + // Two calls to "anti_affinity_group_member_add" should be the same + // as a single call. + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + + // We should be able to delete the membership idempotently. + datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let err = datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectNotFound { + type_name: ResourceType::AntiAffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index 65d42fbe7c2..9e4bec81d05 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -1387,6 +1387,13 @@ impl DataStore { })?; let instance_id = InstanceUuid::from_untyped_uuid(authz_instance.id()); + self.instance_affinity_group_memberships_delete(opctx, instance_id) + .await?; + self.instance_anti_affinity_group_memberships_delete( + opctx, + instance_id, + ) + .await?; self.instance_ssh_keys_delete(opctx, instance_id).await?; self.instance_mark_migrations_deleted(opctx, instance_id).await?; diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index b2d9f8f2471..5c241578a50 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -49,6 +49,7 @@ use std::num::NonZeroU32; use std::sync::Arc; mod address_lot; +mod affinity; mod allow_list; mod auth; mod bfd; diff --git a/nexus/db-queries/src/db/datastore/project.rs b/nexus/db-queries/src/db/datastore/project.rs index 58b7b315c1c..367e094b2e5 100644 --- a/nexus/db-queries/src/db/datastore/project.rs +++ b/nexus/db-queries/src/db/datastore/project.rs @@ -225,6 +225,8 @@ impl DataStore { generate_fn_to_ensure_none_in_project!(project_image, name, String); generate_fn_to_ensure_none_in_project!(snapshot, name, String); generate_fn_to_ensure_none_in_project!(vpc, name, String); + generate_fn_to_ensure_none_in_project!(affinity_group, name, String); + generate_fn_to_ensure_none_in_project!(anti_affinity_group, name, String); /// Delete a project pub async fn project_delete( @@ -242,6 +244,9 @@ impl DataStore { self.ensure_no_project_images_in_project(opctx, authz_project).await?; self.ensure_no_snapshots_in_project(opctx, authz_project).await?; self.ensure_no_vpcs_in_project(opctx, authz_project).await?; + self.ensure_no_affinity_groups_in_project(opctx, authz_project).await?; + self.ensure_no_anti_affinity_groups_in_project(opctx, authz_project) + .await?; use db::schema::project::dsl; diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 87d5c324c63..b402cade0fd 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -13,6 +13,7 @@ use crate::db::datastore::ValidateTransition; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::model::to_db_sled_policy; +use crate::db::model::AffinityPolicy; use crate::db::model::Sled; use crate::db::model::SledResource; use crate::db::model::SledState; @@ -20,6 +21,8 @@ use crate::db::model::SledUpdate; use crate::db::pagination::paginated; use crate::db::pagination::Paginator; use crate::db::pool::DbConnection; +use crate::db::queries::affinity::lookup_affinity_sleds_query; +use crate::db::queries::affinity::lookup_anti_affinity_sleds_query; use crate::db::update_and_check::{UpdateAndCheck, UpdateStatus}; use crate::db::TransactionError; use crate::transaction_retry::OptionalError; @@ -40,12 +43,37 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; use omicron_common::bail_unless; use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisUuid; use omicron_uuid_kinds::SledUuid; +use std::collections::HashSet; use std::fmt; use strum::IntoEnumIterator; use thiserror::Error; use uuid::Uuid; +#[derive(Debug, thiserror::Error)] +enum SledReservationError { + #[error("No reservation could be found")] + NotFound, + #[error("More than one sled was found with 'affinity = required'")] + TooManyAffinityConstraints, + #[error("A sled that is required is also considered banned by anti-affinity rules")] + ConflictingAntiAndAffinityConstraints, + #[error("A sled required by an affinity group is not a valid target for other reasons")] + RequiredAffinitySledNotValid, +} + +#[derive(Debug, thiserror::Error)] +enum SledReservationTransactionError { + #[error(transparent)] + Connection(#[from] Error), + #[error(transparent)] + Diesel(#[from] diesel::result::Error), + #[error(transparent)] + Reservation(#[from] SledReservationError), +} + impl DataStore { /// Stores a new sled in the database. /// @@ -185,18 +213,48 @@ impl DataStore { pub async fn sled_reservation_create( &self, opctx: &OpContext, - resource_id: Uuid, - resource_kind: db::model::SledResourceKind, + instance_id: InstanceUuid, + propolis_id: PropolisUuid, resources: db::model::Resources, constraints: db::model::SledReservationConstraints, ) -> CreateResult { - #[derive(Debug)] - enum SledReservationError { - NotFound, - } + self.sled_reservation_create_inner( + opctx, + instance_id, + propolis_id, + resources, + constraints, + ) + .await + .map_err(|e| match e { + SledReservationTransactionError::Connection(e) => e, + SledReservationTransactionError::Diesel(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + SledReservationTransactionError::Reservation(e) => match e { + SledReservationError::NotFound + | SledReservationError::TooManyAffinityConstraints + | SledReservationError::ConflictingAntiAndAffinityConstraints + | SledReservationError::RequiredAffinitySledNotValid => { + return external::Error::insufficient_capacity( + "No sleds can fit the requested instance", + "No sled targets found that had enough \ + capacity to fit the requested instance.", + ); + } + }, + }) + } + async fn sled_reservation_create_inner( + &self, + opctx: &OpContext, + instance_id: InstanceUuid, + propolis_id: PropolisUuid, + resources: db::model::Resources, + constraints: db::model::SledReservationConstraints, + ) -> Result { let err = OptionalError::new(); - let conn = self.pool_connection_authorized(opctx).await?; self.transaction_retry_wrapper("sled_reservation_create") @@ -210,7 +268,7 @@ impl DataStore { use db::schema::sled_resource::dsl as resource_dsl; // Check if resource ID already exists - if so, return it. let old_resource = resource_dsl::sled_resource - .filter(resource_dsl::id.eq(resource_id)) + .filter(resource_dsl::id.eq(*propolis_id.as_untyped_uuid())) .select(SledResource::as_select()) .limit(1) .load_async(&conn) @@ -289,30 +347,197 @@ impl DataStore { define_sql_function!(fn random() -> diesel::sql_types::Float); - // We only actually care about one target here, so this - // query should have a `.limit(1)` attached. We fetch all - // sled targets to leave additional debugging information in - // the logs, for now. + // Fetch all viable sled targets let sled_targets = sled_targets .order(random()) .get_results_async::(&conn) .await?; + info!( opctx.log, - "found {} available sled targets", sled_targets.len(); + "found {} available sled targets before considering affinity", sled_targets.len(); "sled_ids" => ?sled_targets, ); - if sled_targets.is_empty() { - return Err(err.bail(SledReservationError::NotFound)); + let anti_affinity_sleds = lookup_anti_affinity_sleds_query( + instance_id, + ).get_results_async::<(AffinityPolicy, Uuid)>(&conn).await?; + + let affinity_sleds = lookup_affinity_sleds_query( + instance_id, + ).get_results_async::<(AffinityPolicy, Uuid)>(&conn).await?; + + // We use the following logic to calculate a desirable sled, + // given a possible set of "targets", and the information + // from affinity groups. + // + // # Rules vs Preferences + // + // Due to the flavors "affinity policy", it's possible to bucket affinity + // choices into two categories: "rules" and "preferences". "rules" are affinity + // dispositions for or against sled placement that must be followed, and + // "preferences" are affinity dispositions that should be followed for sled + // selection, in order of "most preferential" to "least preferential". + // + // As example of a "rule" is "an anti-affinity group exists, containing a + // target sled, with affinity_policy = 'fail'". + // + // An example of a "preference" is "an anti-affinity group exists, containing a + // target sled, but the policy is 'allow'. We don't want to use it as a target, + // but we will if there are no other choices." + // + // We apply rules before preferences to ensure they are always respected. + // Furthermore, the evaluation of preferences is a target-seeking operation, + // which identifies the distinct sets of targets, and searches them in + // decreasing preference order. + // + // # Logic + // + // ## Background: Notation + // + // We use the following symbols for sets below: + // - ∩: Intersection of two sets (A ∩ B is "everything that exists in A and + // also exists in B"). + // - \: difference of two sets (A \ B is "everything that exists in A that does + // not exist in B). + // + // We also use the following notation for brevity: + // - AA,P=Fail: All sleds sharing an anti-affinity instance within a group with + // policy = 'fail'. + // - AA,P=Allow: Same as above, but with policy = 'allow'. + // - A,P=Fail: All sleds sharing an affinity instance within a group with + // policy = 'fail'. + // - A,P=Allow: Same as above, but with policy = 'allow'. + // + // ## Affinity: Apply Rules + // + // - Targets := All viable sleds for instance placement + // - Banned := AA,P=Fail + // - Required := A,P=Fail + // - if Required.len() > 1: Fail (too many constraints). + // - if Required.len() == 1... + // - ... if the entry exists in the "Banned" set: Fail + // (contradicting constraints 'Banned' + 'Required') + // - ... if the entry does not exist in "Targets": Fail + // ('Required' constraint not satisfiable) + // - ... if the entry does not exist in "Banned": Use it. + // + // If we have not yet picked a target, we can filter the + // set of targets to ignore "banned" sleds, and then apply + // preferences. + // + // - Targets := Targets \ Banned + // + // ## Affinity: Apply Preferences + // + // - Preferred := Targets ∩ A,P=Allow + // - Unpreferred := Targets ∩ AA,P=Allow + // - Neither := Preferred ∩ Unpreferred + // - Preferred := Preferred \ Neither + // - Unpreferred := Unpreferred \ Neither + // - If Preferred isn't empty, pick a target from it. + // - Targets := Targets \ Unpreferred + // - If Targets isn't empty, pick a target from it. + // - If Unpreferred isn't empty, pick a target from it. + // - Fail, no targets are available. + + let mut targets: HashSet<_> = sled_targets.into_iter().collect(); + + let banned = anti_affinity_sleds.iter().filter_map(|(policy, id)| { + if *policy == AffinityPolicy::Fail { + Some(*id) + } else { + None + } + }).collect::>(); + let required = affinity_sleds.iter().filter_map(|(policy, id)| { + if *policy == AffinityPolicy::Fail { + Some(*id) + } else { + None + } + }).collect::>(); + + if !banned.is_empty() { + info!( + opctx.log, + "affinity policy prohibits placement on {} sleds", banned.len(); + "banned" => ?banned, + ); + } + if !required.is_empty() { + info!( + opctx.log, + "affinity policy requires placement on {} sleds", required.len(); + "required" => ?required, + ); } + let sled_target = if required.len() > 1 { + return Err(err.bail(SledReservationError::TooManyAffinityConstraints)); + } else if let Some(required_id) = required.iter().next() { + // If we have a "required" sled, it must be chosen. + + if banned.contains(&required_id) { + return Err(err.bail(SledReservationError::ConflictingAntiAndAffinityConstraints)); + } + if !targets.contains(&required_id) { + return Err(err.bail(SledReservationError::RequiredAffinitySledNotValid)); + } + *required_id + } else { + // We have no "required" sleds, but might have preferences. + + targets = targets.difference(&banned).cloned().collect(); + + let mut preferred = affinity_sleds.iter().filter_map(|(policy, id)| { + if *policy == AffinityPolicy::Allow { + Some(*id) + } else { + None + } + }).collect::>(); + let mut unpreferred = anti_affinity_sleds.iter().filter_map(|(policy, id)| { + if *policy == AffinityPolicy::Allow { + Some(*id) + } else { + None + } + }).collect::>(); + + // Only consider "preferred" sleds that are viable targets + preferred = targets.intersection(&preferred).cloned().collect(); + // Only consider "unpreferred" sleds that are viable targets + unpreferred = targets.intersection(&unpreferred).cloned().collect(); + + // If a target is both preferred and unpreferred, it is removed + // from both sets. + let both = preferred.intersection(&unpreferred).cloned().collect(); + preferred = preferred.difference(&both).cloned().collect(); + unpreferred = unpreferred.difference(&both).cloned().collect(); + + if let Some(target) = preferred.iter().next() { + *target + } else { + targets = targets.difference(&unpreferred).cloned().collect(); + if let Some(target) = targets.iter().next() { + *target + } else { + if let Some(target) = unpreferred.iter().next() { + *target + } else { + return Err(err.bail(SledReservationError::NotFound)); + } + } + } + }; + // Create a SledResource record, associate it with the target // sled. - let resource = SledResource::new( - resource_id, - sled_targets[0], - resource_kind, + let resource = SledResource::new_for_vmm( + propolis_id, + instance_id, + SledUuid::from_untyped_uuid(sled_target), resources, ); @@ -326,17 +551,9 @@ impl DataStore { .await .map_err(|e| { if let Some(err) = err.take() { - match err { - SledReservationError::NotFound => { - return external::Error::insufficient_capacity( - "No sleds can fit the requested instance", - "No sled targets found that had enough \ - capacity to fit the requested instance.", - ); - } - } + return SledReservationTransactionError::Reservation(err); } - public_error_from_diesel(e, ErrorHandler::Server) + SledReservationTransactionError::Diesel(e) }) } @@ -847,7 +1064,10 @@ pub(in crate::db::datastore) mod test { }; use crate::db::lookup::LookupPath; use crate::db::model::to_db_typed_uuid; + use crate::db::model::AffinityGroup; + use crate::db::model::AntiAffinityGroup; use crate::db::model::ByteCount; + use crate::db::model::Project; use crate::db::model::SqlU32; use crate::db::pub_test_utils::TestDatabase; use anyhow::{Context, Result}; @@ -857,13 +1077,16 @@ pub(in crate::db::datastore) mod test { use nexus_db_model::PhysicalDiskKind; use nexus_db_model::PhysicalDiskPolicy; use nexus_db_model::PhysicalDiskState; + use nexus_types::external_api::params; use nexus_types::identity::Asset; + use nexus_types::identity::Resource; use omicron_common::api::external; use omicron_test_utils::dev; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::SledUuid; use predicates::{prelude::*, BoxPredicate}; + use std::collections::HashMap; use std::net::{Ipv6Addr, SocketAddrV6}; fn rack_id() -> Uuid { @@ -1113,8 +1336,8 @@ pub(in crate::db::datastore) mod test { let error = datastore .sled_reservation_create( &opctx, - Uuid::new_v4(), - db::model::SledResourceKind::Instance, + InstanceUuid::new_v4(), + PropolisUuid::new_v4(), resources.clone(), constraints, ) @@ -1134,15 +1357,15 @@ pub(in crate::db::datastore) mod test { let resource = datastore .sled_reservation_create( &opctx, - Uuid::new_v4(), - db::model::SledResourceKind::Instance, + InstanceUuid::new_v4(), + PropolisUuid::new_v4(), resources.clone(), constraints, ) .await .unwrap(); assert_eq!( - resource.sled_id, + resource.sled_id.into_untyped_uuid(), provisionable_sled.id(), "resource is always allocated to the provisionable sled" ); @@ -1157,6 +1380,1067 @@ pub(in crate::db::datastore) mod test { logctx.cleanup_successful(); } + // Utilities to help with Affinity Testing + + // Create a resource request that will probably fit on a sled. + fn small_resource_request() -> db::model::Resources { + db::model::Resources::new( + 1, + // Just require the bare non-zero amount of RAM. + ByteCount::try_from(1024).unwrap(), + ByteCount::try_from(1024).unwrap(), + ) + } + + // Create a resource request that will entirely fill a sled. + fn large_resource_request() -> db::model::Resources { + let sled_resources = sled_system_hardware_for_test(); + let threads = sled_resources.usable_hardware_threads; + let rss_ram = sled_resources.usable_physical_ram; + let reservoir_ram = sled_resources.reservoir_size; + db::model::Resources::new(threads, rss_ram, reservoir_ram) + } + + async fn create_project( + opctx: &OpContext, + datastore: &DataStore, + ) -> (authz::Project, db::model::Project) { + let authz_silo = opctx.authn.silo_required().unwrap(); + + // Create a project + let project = Project::new( + authz_silo.id(), + params::ProjectCreate { + identity: external::IdentityMetadataCreateParams { + name: "project".parse().unwrap(), + description: "desc".to_string(), + }, + }, + ); + datastore.project_create(&opctx, project).await.unwrap() + } + + async fn create_anti_affinity_group( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &'static str, + policy: external::AffinityPolicy, + ) -> AntiAffinityGroup { + datastore + .anti_affinity_group_create( + &opctx, + &authz_project, + AntiAffinityGroup::new( + authz_project.id(), + params::AntiAffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ), + ) + .await + .unwrap() + } + + // This short-circuits some of the logic and checks we normally have when + // creating affinity groups, but makes testing easier. + async fn add_instance_to_anti_affinity_group( + datastore: &DataStore, + group_id: Uuid, + instance_id: Uuid, + ) { + use db::model::AntiAffinityGroupInstanceMembership; + use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; + use omicron_uuid_kinds::AntiAffinityGroupUuid; + + diesel::insert_into( + membership_dsl::anti_affinity_group_instance_membership, + ) + .values(AntiAffinityGroupInstanceMembership::new( + AntiAffinityGroupUuid::from_untyped_uuid(group_id), + InstanceUuid::from_untyped_uuid(instance_id), + )) + .on_conflict((membership_dsl::group_id, membership_dsl::instance_id)) + .do_nothing() + .execute_async(&*datastore.pool_connection_for_tests().await.unwrap()) + .await + .unwrap(); + } + + async fn create_affinity_group( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &'static str, + policy: external::AffinityPolicy, + ) -> AffinityGroup { + datastore + .affinity_group_create( + &opctx, + &authz_project, + AffinityGroup::new( + authz_project.id(), + params::AffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ), + ) + .await + .unwrap() + } + + // This short-circuits some of the logic and checks we normally have when + // creating affinity groups, but makes testing easier. + async fn add_instance_to_affinity_group( + datastore: &DataStore, + group_id: Uuid, + instance_id: Uuid, + ) { + use db::model::AffinityGroupInstanceMembership; + use db::schema::affinity_group_instance_membership::dsl as membership_dsl; + use omicron_uuid_kinds::AffinityGroupUuid; + + diesel::insert_into(membership_dsl::affinity_group_instance_membership) + .values(AffinityGroupInstanceMembership::new( + AffinityGroupUuid::from_untyped_uuid(group_id), + InstanceUuid::from_untyped_uuid(instance_id), + )) + .on_conflict(( + membership_dsl::group_id, + membership_dsl::instance_id, + )) + .do_nothing() + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + } + + async fn create_sleds(datastore: &DataStore, count: usize) -> Vec { + let mut sleds = vec![]; + for _ in 0..count { + let (sled, _) = + datastore.sled_upsert(test_new_sled_update()).await.unwrap(); + sleds.push(sled); + } + sleds + } + + type GroupName = &'static str; + + #[derive(Copy, Clone, Debug)] + enum Affinity { + Positive, + Negative, + } + + struct Group { + affinity: Affinity, + name: GroupName, + policy: external::AffinityPolicy, + } + + impl Group { + async fn create( + &self, + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + ) -> Uuid { + match self.affinity { + Affinity::Positive => create_affinity_group( + &opctx, + &datastore, + &authz_project, + self.name, + self.policy, + ) + .await + .id(), + Affinity::Negative => create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + self.name, + self.policy, + ) + .await + .id(), + } + } + } + + struct AllGroups { + id_by_name: HashMap<&'static str, (Affinity, Uuid)>, + } + + impl AllGroups { + async fn create( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + groups: &[Group], + ) -> AllGroups { + let mut id_by_name = HashMap::new(); + + for group in groups { + id_by_name.insert( + group.name, + ( + group.affinity, + group.create(&opctx, &datastore, &authz_project).await, + ), + ); + } + + Self { id_by_name } + } + } + + struct Instance { + id: InstanceUuid, + groups: Vec, + force_onto_sled: Option, + resources: db::model::Resources, + } + + impl Instance { + fn new() -> Self { + Self { + id: InstanceUuid::new_v4(), + groups: vec![], + force_onto_sled: None, + resources: small_resource_request(), + } + } + + fn use_many_resources(mut self) -> Self { + self.resources = large_resource_request(); + self + } + + // Adds this instance to a group. Can be called multiple times. + fn group(mut self, group: GroupName) -> Self { + self.groups.push(group); + self + } + + // Force this instance to be placed on a specific sled + fn sled(mut self, sled: Uuid) -> Self { + self.force_onto_sled = Some(SledUuid::from_untyped_uuid(sled)); + self + } + + async fn add_to_groups_and_reserve( + &self, + opctx: &OpContext, + datastore: &DataStore, + all_groups: &AllGroups, + ) -> Result + { + self.add_to_groups(&datastore, &all_groups).await; + create_instance_reservation(&datastore, &opctx, self).await + } + + async fn add_to_groups( + &self, + datastore: &DataStore, + all_groups: &AllGroups, + ) { + for our_group_name in &self.groups { + let (affinity, group_id) = all_groups + .id_by_name + .get(our_group_name) + .expect("Group not found: {our_group_name}"); + match *affinity { + Affinity::Positive => { + add_instance_to_affinity_group( + &datastore, + *group_id, + self.id.into_untyped_uuid(), + ) + .await + } + Affinity::Negative => { + add_instance_to_anti_affinity_group( + &datastore, + *group_id, + self.id.into_untyped_uuid(), + ) + .await + } + } + } + } + } + + async fn create_instance_reservation( + db: &DataStore, + opctx: &OpContext, + instance: &Instance, + ) -> Result { + // Pick a specific sled, if requested + let constraints = db::model::SledReservationConstraintBuilder::new(); + let constraints = if let Some(sled_target) = instance.force_onto_sled { + constraints.must_select_from(&[sled_target.into_untyped_uuid()]) + } else { + constraints + }; + + // We're accessing the inner implementation to get a more detailed + // view of the error code. + let result = db + .sled_reservation_create_inner( + &opctx, + instance.id, + PropolisUuid::new_v4(), + instance.resources.clone(), + constraints.build(), + ) + .await?; + + if let Some(sled_target) = instance.force_onto_sled { + assert_eq!(SledUuid::from(result.sled_id), sled_target); + } + + Ok(result) + } + + // Anti-Affinity, Policy = Fail + // No sleds available => Insufficient Capacity Error + #[tokio::test] + async fn anti_affinity_policy_fail_no_capacity() { + let logctx = + dev::test_setup_log("anti_affinity_policy_fail_no_capacity"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [ + Instance::new().group("anti-affinity").sled(sleds[0].id()), + Instance::new().group("anti-affinity").sled(sleds[1].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("anti-affinity"); + let err = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect_err("Should have failed to place instance"); + + let SledReservationTransactionError::Reservation( + SledReservationError::NotFound, + ) = err + else { + panic!("Unexpected error: {err:?}"); + }; + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Anti-Affinity, Policy = Fail + // We should reliably pick a sled not occupied by another instance + #[tokio::test] + async fn anti_affinity_policy_fail() { + let logctx = dev::test_setup_log("anti_affinity_policy_fail"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 3; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [ + Instance::new().group("anti-affinity").sled(sleds[0].id()), + Instance::new().group("anti-affinity").sled(sleds[1].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("anti-affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[2].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Anti-Affinity, Policy = Allow + // + // Create two sleds, put instances on each belonging to an anti-affinity group. + // We can continue to add instances belonging to that anti-affinity group, because + // it has "AffinityPolicy::Allow". + #[tokio::test] + async fn anti_affinity_policy_allow() { + let logctx = dev::test_setup_log("anti_affinity_policy_allow"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Allow, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [ + Instance::new().group("anti-affinity").sled(sleds[0].id()), + Instance::new().group("anti-affinity").sled(sleds[1].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("anti-affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + assert!( + [sleds[0].id(), sleds[1].id()] + .contains(resource.sled_id.as_untyped_uuid()), + "Should have been provisioned to one of the two viable sleds" + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Affinity, Policy = Fail + // + // Placement of instances with positive affinity will share a sled. + #[tokio::test] + async fn affinity_policy_fail() { + let logctx = dev::test_setup_log("affinity_policy_fail"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Fail, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [Instance::new().group("affinity").sled(sleds[0].id())]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have placed instance"); + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[0].id()); + + let another_test_instance = Instance::new().group("affinity"); + let resource = another_test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have placed instance (again)"); + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[0].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Affinity, Policy = Fail, with too many constraints. + #[tokio::test] + async fn affinity_policy_fail_too_many_constraints() { + let logctx = + dev::test_setup_log("affinity_policy_fail_too_many_constraints"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 3; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Positive, + name: "affinity1", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity2", + policy: external::AffinityPolicy::Fail, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + + // We are constrained so that an instance belonging to both groups must be + // placed on both sled 0 and sled 1. This cannot be satisfied, and it returns + // an error. + let instances = [ + Instance::new().group("affinity1").sled(sleds[0].id()), + Instance::new().group("affinity2").sled(sleds[1].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().group("affinity1").group("affinity2"); + let err = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect_err("Should have failed to place instance"); + + let SledReservationTransactionError::Reservation( + SledReservationError::TooManyAffinityConstraints, + ) = err + else { + panic!("Unexpected error: {err:?}"); + }; + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Affinity, Policy = Fail forces "no space" early + // + // Placement of instances with positive affinity with "policy = Fail" + // will always be forced to share a sled, even if there are other options. + #[tokio::test] + async fn affinity_policy_fail_no_capacity() { + let logctx = dev::test_setup_log("affinity_policy_fail_no_capacity"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Fail, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [Instance::new() + .use_many_resources() + .group("affinity") + .sled(sleds[0].id())]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().use_many_resources().group("affinity"); + let err = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect_err("Should have failed to place instance"); + let SledReservationTransactionError::Reservation( + SledReservationError::RequiredAffinitySledNotValid, + ) = err + else { + panic!("Unexpected error: {err:?}"); + }; + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Affinity, Policy = Allow lets us use other sleds + // + // This is similar to "affinity_policy_fail_no_capacity", but by + // using "Policy = Allow" instead of "Policy = Fail", we are able to pick + // a different sled for the reservation. + #[tokio::test] + async fn affinity_policy_allow_picks_different_sled() { + let logctx = + dev::test_setup_log("affinity_policy_allow_picks_different_sled"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Allow, + }]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [Instance::new() + .use_many_resources() + .group("affinity") + .sled(sleds[0].id())]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().use_many_resources().group("affinity"); + let reservation = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have made reservation"); + assert_eq!(reservation.sled_id.into_untyped_uuid(), sleds[1].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Anti-Affinity, Policy = Fail + Affinity, Policy = Fail + // + // These constraints are contradictory - we're asking the allocator + // to colocate and NOT colocate our new instance with existing ones, which + // should not be satisfiable. + #[tokio::test] + async fn affinity_and_anti_affinity_policy_fail() { + let logctx = + dev::test_setup_log("affinity_and_anti_affinity_policy_fail"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Fail, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [Instance::new() + .group("anti-affinity") + .group("affinity") + .sled(sleds[0].id())]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().group("anti-affinity").group("affinity"); + let err = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect_err("Contradictory constraints should not be satisfiable"); + let SledReservationTransactionError::Reservation( + SledReservationError::ConflictingAntiAndAffinityConstraints, + ) = err + else { + panic!("Unexpected error: {err:?}"); + }; + + db.terminate().await; + logctx.cleanup_successful(); + } + + // Anti-Affinity, Policy = Allow + Affinity, Policy = Allow + // + // These constraints are contradictory, but since they encode + // "preferences" rather than "rules", they cancel out. + #[tokio::test] + async fn affinity_and_anti_affinity_policy_allow() { + let logctx = + dev::test_setup_log("affinity_and_anti_affinity_policy_allow"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 2; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + let instances = [ + Instance::new() + .group("anti-affinity") + .group("affinity") + .sled(sleds[0].id()), + Instance::new() + .group("anti-affinity") + .group("affinity") + .sled(sleds[1].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().group("anti-affinity").group("affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + assert!( + [sleds[0].id(), sleds[1].id()] + .contains(resource.sled_id.as_untyped_uuid()), + "Should have been provisioned to one of the two viable sleds" + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_multi_group() { + let logctx = dev::test_setup_log("anti_affinity_multi_group"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 4; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "strict-anti-affinity", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + + // Sleds 0 and 1 contain the "strict-anti-affinity" group instances, + // and won't be used. + // + // Sled 3 has an "anti-affinity" group instance, and also won't be used. + // + // This only leaves sled 2. + let instances = [ + Instance::new() + .group("strict-anti-affinity") + .group("affinity") + .sled(sleds[0].id()), + Instance::new().group("strict-anti-affinity").sled(sleds[1].id()), + Instance::new().group("anti-affinity").sled(sleds[3].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new() + .group("strict-anti-affinity") + .group("anti-affinity") + .group("affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[2].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_multi_group() { + let logctx = dev::test_setup_log("affinity_multi_group"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 4; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Positive, + name: "affinity", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + + // Sled 0 contains an affinity group, but it's large enough to make it + // non-viable for future allocations. + // + // Sled 1 contains an affinity and anti-affinity group, so they cancel out. + // This gives it "no priority". + // + // Sled 2 contains nothing. It's a viable target, neither preferred nor + // unpreferred. + // + // Sled 3 contains an affinity group, which is prioritized. + let instances = [ + Instance::new() + .group("affinity") + .use_many_resources() + .sled(sleds[0].id()), + Instance::new() + .group("affinity") + .group("anti-affinity") + .sled(sleds[1].id()), + Instance::new().group("affinity").sled(sleds[3].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().group("affinity").group("anti-affinity"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[3].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_ignored_from_other_groups() { + let logctx = dev::test_setup_log("affinity_ignored_from_other_groups"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 3; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Positive, + name: "affinity1", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity2", + policy: external::AffinityPolicy::Fail, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + + // Only "sleds[1]" has space. We ignore the affinity policy because + // our new instance won't belong to either group. + let instances = [ + Instance::new() + .group("affinity1") + .use_many_resources() + .sled(sleds[0].id()), + Instance::new() + .group("affinity2") + .use_many_resources() + .sled(sleds[2].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new(); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[1].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_ignored_from_other_groups() { + let logctx = + dev::test_setup_log("anti_affinity_ignored_from_other_groups"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 3; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity1", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Negative, + name: "anti-affinity2", + policy: external::AffinityPolicy::Fail, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + + // Only "sleds[2]" has space, even though it also contains an anti-affinity group. + // However, if we don't belong to this group, we won't care. + + let instances = [ + Instance::new().group("anti-affinity1").sled(sleds[0].id()), + Instance::new().use_many_resources().sled(sleds[1].id()), + Instance::new().group("anti-affinity2").sled(sleds[2].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("anti-affinity1"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!(resource.sled_id.into_untyped_uuid(), sleds[2].id()); + + db.terminate().await; + logctx.cleanup_successful(); + } + async fn lookup_physical_disk( datastore: &DataStore, id: PhysicalDiskUuid, diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index c629dbfc425..d4591024e7a 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -175,6 +175,16 @@ impl<'a> LookupPath<'a> { Instance::PrimaryKey(Root { lookup_root: self }, id) } + /// Select a resource of type AffinityGroup, identified by its id + pub fn affinity_group_id(self, id: Uuid) -> AffinityGroup<'a> { + AffinityGroup::PrimaryKey(Root { lookup_root: self }, id) + } + + /// Select a resource of type AntiAffinityGroup, identified by its id + pub fn anti_affinity_group_id(self, id: Uuid) -> AntiAffinityGroup<'a> { + AntiAffinityGroup::PrimaryKey(Root { lookup_root: self }, id) + } + /// Select a resource of type IpPool, identified by its name pub fn ip_pool_name<'b, 'c>(self, name: &'b Name) -> IpPool<'c> where @@ -645,7 +655,7 @@ lookup_resource! { lookup_resource! { name = "Project", ancestors = [ "Silo" ], - children = [ "Disk", "Instance", "Vpc", "Snapshot", "ProjectImage", "FloatingIp" ], + children = [ "AffinityGroup", "AntiAffinityGroup", "Disk", "Instance", "Vpc", "Snapshot", "ProjectImage", "FloatingIp" ], lookup_by_name = true, soft_deletes = true, primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] @@ -696,6 +706,24 @@ lookup_resource! { primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] } +lookup_resource! { + name = "AffinityGroup", + ancestors = [ "Silo", "Project" ], + children = [], + lookup_by_name = true, + soft_deletes = true, + primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] +} + +lookup_resource! { + name = "AntiAffinityGroup", + ancestors = [ "Silo", "Project" ], + children = [], + lookup_by_name = true, + soft_deletes = true, + primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] +} + lookup_resource! { name = "InstanceNetworkInterface", ancestors = [ "Silo", "Project", "Instance" ], diff --git a/nexus/db-queries/src/db/queries/affinity.rs b/nexus/db-queries/src/db/queries/affinity.rs new file mode 100644 index 00000000000..6da857669da --- /dev/null +++ b/nexus/db-queries/src/db/queries/affinity.rs @@ -0,0 +1,608 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Implementation of queries for affinity groups + +use crate::db::model::AffinityPolicyEnum; +use crate::db::raw_query_builder::{QueryBuilder, TypedSqlQuery}; +use diesel::sql_types; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; + +/// For an instance, look up all anti-affinity groups, and return +/// a list of sleds with other instances in that anti-affinity group +/// already reserved. +pub fn lookup_anti_affinity_sleds_query( + instance_id: InstanceUuid, +) -> TypedSqlQuery<(AffinityPolicyEnum, sql_types::Uuid)> { + QueryBuilder::new().sql( + "WITH our_groups AS ( + SELECT group_id + FROM anti_affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_instances AS ( + SELECT anti_affinity_group_instance_membership.group_id,instance_id + FROM anti_affinity_group_instance_membership + JOIN our_groups + ON anti_affinity_group_instance_membership.group_id = our_groups.group_id + WHERE instance_id != ").param().sql(" + ), + other_instances_by_policy AS ( + SELECT policy,instance_id + FROM other_instances + JOIN anti_affinity_group + ON + anti_affinity_group.id = other_instances.group_id AND + anti_affinity_group.failure_domain = 'sled' + WHERE anti_affinity_group.time_deleted IS NULL + ) + SELECT DISTINCT policy,sled_id + FROM other_instances_by_policy + JOIN sled_resource + ON + sled_resource.instance_id = other_instances_by_policy.instance_id AND + sled_resource.kind = 'instance'") + .bind::(instance_id.into_untyped_uuid()) + .bind::(instance_id.into_untyped_uuid()) + .query() +} + +/// For an instance, look up all affinity groups, and return a list of sleds +/// with other instances in that affinity group already reserved. +pub fn lookup_affinity_sleds_query( + instance_id: InstanceUuid, +) -> TypedSqlQuery<(AffinityPolicyEnum, sql_types::Uuid)> { + QueryBuilder::new() + .sql( + "WITH our_groups AS ( + SELECT group_id + FROM affinity_group_instance_membership + WHERE instance_id = ", + ) + .param() + .sql( + " + ), + other_instances AS ( + SELECT affinity_group_instance_membership.group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_groups + ON affinity_group_instance_membership.group_id = our_groups.group_id + WHERE instance_id != ", + ) + .param() + .sql( + " + ), + other_instances_by_policy AS ( + SELECT policy,instance_id + FROM other_instances + JOIN affinity_group + ON + affinity_group.id = other_instances.group_id AND + affinity_group.failure_domain = 'sled' + WHERE affinity_group.time_deleted IS NULL + ) + SELECT DISTINCT policy,sled_id + FROM other_instances_by_policy + JOIN sled_resource + ON + sled_resource.instance_id = other_instances_by_policy.instance_id AND + sled_resource.kind = 'instance'", + ) + .bind::(instance_id.into_untyped_uuid()) + .bind::(instance_id.into_untyped_uuid()) + .query() +} + +#[cfg(test)] +mod test { + use super::*; + use crate::db::explain::ExplainableAsync; + use crate::db::model; + use crate::db::pub_test_utils::TestDatabase; + use crate::db::raw_query_builder::expectorate_query_contents; + + use anyhow::Context; + use async_bb8_diesel::AsyncRunQueryDsl; + use nexus_types::external_api::params; + use nexus_types::identity::Resource; + use omicron_common::api::external; + use omicron_test_utils::dev; + use omicron_uuid_kinds::AffinityGroupUuid; + use omicron_uuid_kinds::AntiAffinityGroupUuid; + use omicron_uuid_kinds::PropolisUuid; + use omicron_uuid_kinds::SledUuid; + use uuid::Uuid; + + #[tokio::test] + async fn expectorate_lookup_anti_affinity_sleds_query() { + let id = InstanceUuid::nil(); + + let query = lookup_anti_affinity_sleds_query(id); + expectorate_query_contents( + &query, + "tests/output/lookup_anti_affinity_sleds_query.sql", + ) + .await; + } + + #[tokio::test] + async fn expectorate_lookup_affinity_sleds_query() { + let id = InstanceUuid::nil(); + + let query = lookup_affinity_sleds_query(id); + expectorate_query_contents( + &query, + "tests/output/lookup_affinity_sleds_query.sql", + ) + .await; + } + + #[tokio::test] + async fn explain_lookup_anti_affinity_sleds_query() { + let logctx = + dev::test_setup_log("explain_lookup_anti_affinity_sleds_query"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let id = InstanceUuid::nil(); + let query = lookup_anti_affinity_sleds_query(id); + let _ = query + .explain_async(&conn) + .await + .expect("Failed to explain query - is it valid SQL?"); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn explain_lookup_affinity_sleds_query() { + let logctx = dev::test_setup_log("explain_lookup_affinity_sleds_query"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let id = InstanceUuid::nil(); + let query = lookup_affinity_sleds_query(id); + let _ = query + .explain_async(&conn) + .await + .expect("Failed to explain query - is it valid SQL?"); + + db.terminate().await; + logctx.cleanup_successful(); + } + + async fn make_affinity_group( + project_id: Uuid, + name: &'static str, + policy: external::AffinityPolicy, + conn: &async_bb8_diesel::Connection, + ) -> anyhow::Result { + let group = model::AffinityGroup::new( + project_id, + params::AffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ); + use crate::db::schema::affinity_group::dsl; + diesel::insert_into(dsl::affinity_group) + .values(group.clone()) + .execute_async(conn) + .await + .context("Cannot create affinity group")?; + Ok(group) + } + + async fn make_anti_affinity_group( + project_id: Uuid, + name: &'static str, + policy: external::AffinityPolicy, + conn: &async_bb8_diesel::Connection, + ) -> anyhow::Result { + let group = model::AntiAffinityGroup::new( + project_id, + params::AntiAffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "desc".to_string(), + }, + policy, + failure_domain: external::FailureDomain::Sled, + }, + ); + use crate::db::schema::anti_affinity_group::dsl; + diesel::insert_into(dsl::anti_affinity_group) + .values(group.clone()) + .execute_async(conn) + .await + .context("Cannot create anti affinity group")?; + Ok(group) + } + + async fn make_affinity_group_instance_membership( + group_id: AffinityGroupUuid, + instance_id: InstanceUuid, + conn: &async_bb8_diesel::Connection, + ) -> anyhow::Result<()> { + // Let's claim an instance belongs to that group + let membership = + model::AffinityGroupInstanceMembership::new(group_id, instance_id); + use crate::db::schema::affinity_group_instance_membership::dsl; + diesel::insert_into(dsl::affinity_group_instance_membership) + .values(membership) + .execute_async(conn) + .await + .context("Cannot create affinity group instance membership")?; + Ok(()) + } + + async fn make_anti_affinity_group_instance_membership( + group_id: AntiAffinityGroupUuid, + instance_id: InstanceUuid, + conn: &async_bb8_diesel::Connection, + ) -> anyhow::Result<()> { + // Let's claim an instance belongs to that group + let membership = model::AntiAffinityGroupInstanceMembership::new( + group_id, + instance_id, + ); + use crate::db::schema::anti_affinity_group_instance_membership::dsl; + diesel::insert_into(dsl::anti_affinity_group_instance_membership) + .values(membership) + .execute_async(conn) + .await + .context("Cannot create anti affinity group instance membership")?; + Ok(()) + } + + async fn make_instance_sled_resource( + sled_id: SledUuid, + instance_id: InstanceUuid, + conn: &async_bb8_diesel::Connection, + ) -> anyhow::Result<()> { + let resource = model::SledResource::new_for_vmm( + PropolisUuid::new_v4(), + instance_id, + sled_id, + model::Resources::new( + 0, + model::ByteCount::from( + external::ByteCount::from_gibibytes_u32(0), + ), + model::ByteCount::from( + external::ByteCount::from_gibibytes_u32(0), + ), + ), + ); + use crate::db::schema::sled_resource::dsl; + diesel::insert_into(dsl::sled_resource) + .values(resource) + .execute_async(conn) + .await + .context("Cannot create sled resource")?; + Ok(()) + } + + #[tokio::test] + async fn lookup_affinity_sleds() { + let logctx = dev::test_setup_log("lookup_affinity_sleds"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let our_instance_id = InstanceUuid::new_v4(); + let other_instance_id = InstanceUuid::new_v4(); + + // With no groups and no instances, we should see no other instances + // belonging to our affinity group. + assert_eq!( + lookup_affinity_sleds_query(our_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![], + ); + + let sled_id = SledUuid::new_v4(); + let project_id = Uuid::new_v4(); + + // Make a group, add our instance to it + let group = make_affinity_group( + project_id, + "group-allow", + external::AffinityPolicy::Allow, + &conn, + ) + .await + .unwrap(); + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + + // Create an instance which belongs to that group + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + + make_instance_sled_resource(sled_id, other_instance_id, &conn) + .await + .unwrap(); + + // Now if we look, we'll find the "other sled", on which the "other + // instance" was placed. + assert_eq!( + lookup_affinity_sleds_query(our_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![(model::AffinityPolicy::Allow, sled_id.into_untyped_uuid())], + ); + + // If we look from the perspective of the other instance, + // we "ignore ourselves" for placement, so the set of sleds to consider + // is still empty. + assert_eq!( + lookup_affinity_sleds_query(other_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![] + ); + + // If we make another group (note the policy is different this time!) + // it'll also be reflected in the results. + let group = make_affinity_group( + project_id, + "group-fail", + external::AffinityPolicy::Fail, + &conn, + ) + .await + .unwrap(); + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + + // We see the outcome of both groups, with a different policy for each. + let mut results = lookup_affinity_sleds_query(our_instance_id) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(); + results.sort(); + assert_eq!( + results, + vec![ + (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), + (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), + ], + ); + + // Let's add one more group, to see what happens to the query output. + let group = make_affinity_group( + project_id, + "group-fail2", + external::AffinityPolicy::Fail, + &conn, + ) + .await + .unwrap(); + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + make_affinity_group_instance_membership( + AffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + let mut results = lookup_affinity_sleds_query(our_instance_id) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(); + results.sort(); + + // Since we use "SELECT DISTINCT", the results are bounded, and do not + // grow as we keep on finding more sleds that have the same policy. + assert_eq!( + results, + vec![ + (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), + (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), + ], + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn lookup_anti_affinity_sleds() { + let logctx = dev::test_setup_log("lookup_anti_affinity_sleds"); + let db = TestDatabase::new_with_pool(&logctx.log).await; + let pool = db.pool(); + let conn = pool.claim().await.unwrap(); + + let our_instance_id = InstanceUuid::new_v4(); + let other_instance_id = InstanceUuid::new_v4(); + + // With no groups and no instances, we should see no other instances + // belonging to our anti-affinity group. + assert_eq!( + lookup_anti_affinity_sleds_query(our_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![], + ); + + let sled_id = SledUuid::new_v4(); + let project_id = Uuid::new_v4(); + + // Make a group, add our instance to it + let group = make_anti_affinity_group( + project_id, + "group-allow", + external::AffinityPolicy::Allow, + &conn, + ) + .await + .unwrap(); + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + + // Create an instance which belongs to that group + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + + make_instance_sled_resource(sled_id, other_instance_id, &conn) + .await + .unwrap(); + + // Now if we look, we'll find the "other sled", on which the "other + // instance" was placed. + assert_eq!( + lookup_anti_affinity_sleds_query(our_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![(model::AffinityPolicy::Allow, sled_id.into_untyped_uuid())], + ); + + // If we look from the perspective of the other instance, + // we "ignore ourselves" for placement, so the set of sleds to consider + // is still empty. + assert_eq!( + lookup_anti_affinity_sleds_query(other_instance_id,) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(), + vec![] + ); + + // If we make another group (note the policy is different this time!) + // it'll also be reflected in the results. + let group = make_anti_affinity_group( + project_id, + "group-fail", + external::AffinityPolicy::Fail, + &conn, + ) + .await + .unwrap(); + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + + // We see the outcome of both groups, with a different policy for each. + let mut results = lookup_anti_affinity_sleds_query(our_instance_id) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(); + results.sort(); + assert_eq!( + results, + vec![ + (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), + (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), + ], + ); + + // Let's add one more group, to see what happens to the query output. + let group = make_anti_affinity_group( + project_id, + "group-fail2", + external::AffinityPolicy::Fail, + &conn, + ) + .await + .unwrap(); + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + our_instance_id, + &conn, + ) + .await + .unwrap(); + make_anti_affinity_group_instance_membership( + AntiAffinityGroupUuid::from_untyped_uuid(group.id()), + other_instance_id, + &conn, + ) + .await + .unwrap(); + let mut results = lookup_anti_affinity_sleds_query(our_instance_id) + .get_results_async::<(model::AffinityPolicy, Uuid)>(&*conn) + .await + .unwrap(); + results.sort(); + + // Since we use "SELECT DISTINCT", the results are bounded, and do not + // grow as we keep on finding more sleds that have the same policy. + assert_eq!( + results, + vec![ + (model::AffinityPolicy::Fail, sled_id.into_untyped_uuid()), + (model::AffinityPolicy::Allow, sled_id.into_untyped_uuid()), + ], + ); + + db.terminate().await; + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/src/db/queries/mod.rs b/nexus/db-queries/src/db/queries/mod.rs index 5f34c7cfb3d..f2af3aebc99 100644 --- a/nexus/db-queries/src/db/queries/mod.rs +++ b/nexus/db-queries/src/db/queries/mod.rs @@ -5,6 +5,7 @@ //! Specialized queries for inserting database records, usually to maintain //! complex invariants that are most accurately expressed in a single query. +pub mod affinity; pub mod disk; pub mod external_ip; pub mod ip_pool; diff --git a/nexus/db-queries/src/policy_test/resource_builder.rs b/nexus/db-queries/src/policy_test/resource_builder.rs index b6d7d97553e..310c11adf3c 100644 --- a/nexus/db-queries/src/policy_test/resource_builder.rs +++ b/nexus/db-queries/src/policy_test/resource_builder.rs @@ -243,6 +243,8 @@ macro_rules! impl_dyn_authorized_resource_for_resource { } impl_dyn_authorized_resource_for_resource!(authz::AddressLot); +impl_dyn_authorized_resource_for_resource!(authz::AffinityGroup); +impl_dyn_authorized_resource_for_resource!(authz::AntiAffinityGroup); impl_dyn_authorized_resource_for_resource!(authz::Blueprint); impl_dyn_authorized_resource_for_resource!(authz::Certificate); impl_dyn_authorized_resource_for_resource!(authz::DeviceAccessToken); diff --git a/nexus/db-queries/src/policy_test/resources.rs b/nexus/db-queries/src/policy_test/resources.rs index 6ee92e167cf..04655410533 100644 --- a/nexus/db-queries/src/policy_test/resources.rs +++ b/nexus/db-queries/src/policy_test/resources.rs @@ -300,6 +300,21 @@ async fn make_project( LookupType::ByName(vpc1_name.clone()), ); + let affinity_group_name = format!("{}-affinity-group1", project_name); + let affinity_group = authz::AffinityGroup::new( + project.clone(), + Uuid::new_v4(), + LookupType::ByName(affinity_group_name.clone()), + ); + + let anti_affinity_group_name = + format!("{}-anti-affinity-group1", project_name); + let anti_affinity_group = authz::AntiAffinityGroup::new( + project.clone(), + Uuid::new_v4(), + LookupType::ByName(anti_affinity_group_name.clone()), + ); + let instance_name = format!("{}-instance1", project_name); let instance = authz::Instance::new( project.clone(), @@ -313,6 +328,8 @@ async fn make_project( Uuid::new_v4(), LookupType::ByName(disk_name.clone()), )); + builder.new_resource(affinity_group.clone()); + builder.new_resource(anti_affinity_group.clone()); builder.new_resource(instance.clone()); builder.new_resource(authz::InstanceNetworkInterface::new( instance, diff --git a/nexus/db-queries/tests/output/authz-roles.out b/nexus/db-queries/tests/output/authz-roles.out index 4b24e649ccb..e0d43250d13 100644 --- a/nexus/db-queries/tests/output/authz-roles.out +++ b/nexus/db-queries/tests/output/authz-roles.out @@ -306,6 +306,34 @@ resource: Disk "silo1-proj1-disk1" silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo1-proj1-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo1-proj1-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo1-proj1-instance1" USER Q R LC RP M MP CC D @@ -474,6 +502,34 @@ resource: Disk "silo1-proj2-disk1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo1-proj2-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo1-proj2-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo1-proj2-instance1" USER Q R LC RP M MP CC D @@ -810,6 +866,34 @@ resource: Disk "silo2-proj1-disk1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo2-proj1-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo2-proj1-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo2-proj1-instance1" USER Q R LC RP M MP CC D diff --git a/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql b/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql new file mode 100644 index 00000000000..1435a0b8f62 --- /dev/null +++ b/nexus/db-queries/tests/output/lookup_affinity_sleds_query.sql @@ -0,0 +1,30 @@ +WITH + our_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $1), + other_instances + AS ( + SELECT + affinity_group_instance_membership.group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_groups ON affinity_group_instance_membership.group_id = our_groups.group_id + WHERE + instance_id != $2 + ), + other_instances_by_policy + AS ( + SELECT + policy, instance_id + FROM + other_instances + JOIN affinity_group ON + affinity_group.id = other_instances.group_id AND affinity_group.failure_domain = 'sled' + WHERE + affinity_group.time_deleted IS NULL + ) +SELECT + DISTINCT policy, sled_id +FROM + other_instances_by_policy + JOIN sled_resource ON + sled_resource.instance_id = other_instances_by_policy.instance_id + AND sled_resource.kind = 'instance' diff --git a/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql b/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql new file mode 100644 index 00000000000..d5e7cd66d86 --- /dev/null +++ b/nexus/db-queries/tests/output/lookup_anti_affinity_sleds_query.sql @@ -0,0 +1,32 @@ +WITH + our_groups + AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $1), + other_instances + AS ( + SELECT + anti_affinity_group_instance_membership.group_id, instance_id + FROM + anti_affinity_group_instance_membership + JOIN our_groups ON anti_affinity_group_instance_membership.group_id = our_groups.group_id + WHERE + instance_id != $2 + ), + other_instances_by_policy + AS ( + SELECT + policy, instance_id + FROM + other_instances + JOIN anti_affinity_group ON + anti_affinity_group.id = other_instances.group_id + AND anti_affinity_group.failure_domain = 'sled' + WHERE + anti_affinity_group.time_deleted IS NULL + ) +SELECT + DISTINCT policy, sled_id +FROM + other_instances_by_policy + JOIN sled_resource ON + sled_resource.instance_id = other_instances_by_policy.instance_id + AND sled_resource.kind = 'instance' diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 82767e399c0..59ba14e5fb8 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -1,3 +1,24 @@ +API operations found with tag "affinity" +OPERATION ID METHOD URL PATH +affinity_group_create POST /v1/affinity-groups +affinity_group_delete DELETE /v1/affinity-groups/{affinity_group} +affinity_group_list GET /v1/affinity-groups +affinity_group_member_instance_add POST /v1/affinity-groups/{affinity_group}/members/instance/{instance} +affinity_group_member_instance_delete DELETE /v1/affinity-groups/{affinity_group}/members/instance/{instance} +affinity_group_member_instance_view GET /v1/affinity-groups/{affinity_group}/members/instance/{instance} +affinity_group_member_list GET /v1/affinity-groups/{affinity_group}/members +affinity_group_update PUT /v1/affinity-groups/{affinity_group} +affinity_group_view GET /v1/affinity-groups/{affinity_group} +anti_affinity_group_create POST /v1/anti-affinity-groups +anti_affinity_group_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group} +anti_affinity_group_list GET /v1/anti-affinity-groups +anti_affinity_group_member_instance_add POST /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} +anti_affinity_group_member_instance_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} +anti_affinity_group_member_instance_view GET /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} +anti_affinity_group_member_list GET /v1/anti-affinity-groups/{anti_affinity_group}/members +anti_affinity_group_update PUT /v1/anti-affinity-groups/{anti_affinity_group} +anti_affinity_group_view GET /v1/anti-affinity-groups/{anti_affinity_group} + API operations found with tag "disks" OPERATION ID METHOD URL PATH disk_bulk_write_import POST /v1/disks/{disk}/bulk-write diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index a832bde3199..7daa5514619 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -74,6 +74,13 @@ const PUT_UPDATE_REPOSITORY_MAX_BYTES: usize = 4 * GIB; allow_other_tags = false, policy = EndpointTagPolicy::ExactlyOne, tags = { + "affinity" = { + description = "Affinity groups give control over instance placement.", + external_docs = { + url = "http://docs.oxide.computer/api/affinity" + } + + }, "disks" = { description = "Virtual disks are used to store instance-local data which includes the operating system.", external_docs = { @@ -1257,6 +1264,224 @@ pub trait NexusExternalApi { disk_to_detach: TypedBody, ) -> Result, HttpError>; + // Affinity Groups + + /// List affinity groups + #[endpoint { + method = GET, + path = "/v1/affinity-groups", + tags = ["affinity"], + }] + async fn affinity_group_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Fetch an affinity group + #[endpoint { + method = GET, + path = "/v1/affinity-groups/{affinity_group}", + tags = ["affinity"], + }] + async fn affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// List members of an affinity group + #[endpoint { + method = GET, + path = "/v1/affinity-groups/{affinity_group}/members", + tags = ["affinity"], + }] + async fn affinity_group_member_list( + rqctx: RequestContext, + query_params: Query>, + path_params: Path, + ) -> Result>, HttpError>; + + /// Fetch an affinity group member + #[endpoint { + method = GET, + path = "/v1/affinity-groups/{affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn affinity_group_member_instance_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Add a member to an affinity group + #[endpoint { + method = POST, + path = "/v1/affinity-groups/{affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn affinity_group_member_instance_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Remove a member from an affinity group + #[endpoint { + method = DELETE, + path = "/v1/affinity-groups/{affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn affinity_group_member_instance_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + + /// Create an affinity group + #[endpoint { + method = POST, + path = "/v1/affinity-groups", + tags = ["affinity"], + }] + async fn affinity_group_create( + rqctx: RequestContext, + query_params: Query, + new_affinity_group_params: TypedBody, + ) -> Result, HttpError>; + + /// Update an affinity group + #[endpoint { + method = PUT, + path = "/v1/affinity-groups/{affinity_group}", + tags = ["affinity"], + }] + async fn affinity_group_update( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + updated_group: TypedBody, + ) -> Result, HttpError>; + + /// Delete an affinity group + #[endpoint { + method = DELETE, + path = "/v1/affinity-groups/{affinity_group}", + tags = ["affinity"], + }] + async fn affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + + /// List anti-affinity groups + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups", + tags = ["affinity"], + }] + async fn anti_affinity_group_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Fetch an anti-affinity group + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups/{anti_affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// List members of an anti-affinity group + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_list( + rqctx: RequestContext, + query_params: Query>, + path_params: Path, + ) -> Result>, HttpError>; + + /// Fetch an anti-affinity group member + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_instance_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Add a member to an anti-affinity group + #[endpoint { + method = POST, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_instance_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Remove a member from an anti-affinity group + #[endpoint { + method = DELETE, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_instance_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + + /// Create an anti-affinity group + #[endpoint { + method = POST, + path = "/v1/anti-affinity-groups", + tags = ["affinity"], + }] + async fn anti_affinity_group_create( + rqctx: RequestContext, + query_params: Query, + new_affinity_group_params: TypedBody, + ) -> Result, HttpError>; + + /// Update an anti-affinity group + #[endpoint { + method = PUT, + path = "/v1/anti-affinity-groups/{anti_affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_update( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + updated_group: TypedBody, + ) -> Result, HttpError>; + + /// Delete an anti-affinity group + #[endpoint { + method = DELETE, + path = "/v1/anti-affinity-groups/{anti_affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + // Certificates /// List certificates for external endpoints diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs new file mode 100644 index 00000000000..894f29416e7 --- /dev/null +++ b/nexus/src/app/affinity.rs @@ -0,0 +1,399 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Affinity groups + +use nexus_db_model::AffinityGroup; +use nexus_db_model::AntiAffinityGroup; +use nexus_db_queries::authz; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::lookup; +use nexus_db_queries::db::lookup::LookupPath; +use nexus_types::external_api::params; +use nexus_types::external_api::views; +use omicron_common::api::external; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; +use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; +use omicron_common::api::external::LookupResult; +use omicron_common::api::external::NameOrId; +use omicron_common::api::external::UpdateResult; + +impl super::Nexus { + pub fn affinity_group_lookup<'a>( + &'a self, + opctx: &'a OpContext, + affinity_group_selector: params::AffinityGroupSelector, + ) -> LookupResult> { + match affinity_group_selector { + params::AffinityGroupSelector { + affinity_group: NameOrId::Id(id), + project: None + } => { + let affinity_group = + LookupPath::new(opctx, &self.db_datastore).affinity_group_id(id); + Ok(affinity_group) + } + params::AffinityGroupSelector { + affinity_group: NameOrId::Name(name), + project: Some(project) + } => { + let affinity_group = self + .project_lookup(opctx, params::ProjectSelector { project })? + .affinity_group_name_owned(name.into()); + Ok(affinity_group) + } + params::AffinityGroupSelector { + affinity_group: NameOrId::Id(_), + .. + } => { + Err(Error::invalid_request( + "when providing affinity_group as an ID project should not be specified", + )) + } + _ => { + Err(Error::invalid_request( + "affinity_group should either be UUID or project should be specified", + )) + } + } + } + + pub fn anti_affinity_group_lookup<'a>( + &'a self, + opctx: &'a OpContext, + anti_affinity_group_selector: params::AntiAffinityGroupSelector, + ) -> LookupResult> { + match anti_affinity_group_selector { + params::AntiAffinityGroupSelector { + anti_affinity_group: NameOrId::Id(id), + project: None + } => { + let anti_affinity_group = + LookupPath::new(opctx, &self.db_datastore).anti_affinity_group_id(id); + Ok(anti_affinity_group) + } + params::AntiAffinityGroupSelector { + anti_affinity_group: NameOrId::Name(name), + project: Some(project) + } => { + let anti_affinity_group = self + .project_lookup(opctx, params::ProjectSelector { project })? + .anti_affinity_group_name_owned(name.into()); + Ok(anti_affinity_group) + } + params::AntiAffinityGroupSelector { + anti_affinity_group: NameOrId::Id(_), + .. + } => { + Err(Error::invalid_request( + "when providing anti_affinity_group as an ID project should not be specified", + )) + } + _ => { + Err(Error::invalid_request( + "anti_affinity_group should either be UUID or project should be specified", + )) + } + } + } + + pub(crate) async fn affinity_group_list( + &self, + opctx: &OpContext, + project_lookup: &lookup::Project<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_project) = + project_lookup.lookup_for(authz::Action::ListChildren).await?; + + Ok(self + .db_datastore + .affinity_group_list(opctx, &authz_project, pagparams) + .await? + .into_iter() + .map(Into::into) + .collect()) + } + + pub(crate) async fn anti_affinity_group_list( + &self, + opctx: &OpContext, + project_lookup: &lookup::Project<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_project) = + project_lookup.lookup_for(authz::Action::ListChildren).await?; + + Ok(self + .db_datastore + .anti_affinity_group_list(opctx, &authz_project, pagparams) + .await? + .into_iter() + .map(Into::into) + .collect()) + } + + pub(crate) async fn affinity_group_create( + &self, + opctx: &OpContext, + project_lookup: &lookup::Project<'_>, + affinity_group_params: params::AffinityGroupCreate, + ) -> CreateResult { + let (.., authz_project) = + project_lookup.lookup_for(authz::Action::CreateChild).await?; + + let affinity_group = + AffinityGroup::new(authz_project.id(), affinity_group_params); + self.db_datastore + .affinity_group_create(opctx, &authz_project, affinity_group) + .await + .map(Into::into) + } + + pub(crate) async fn anti_affinity_group_create( + &self, + opctx: &OpContext, + project_lookup: &lookup::Project<'_>, + anti_affinity_group_params: params::AntiAffinityGroupCreate, + ) -> CreateResult { + let (.., authz_project) = + project_lookup.lookup_for(authz::Action::CreateChild).await?; + + let anti_affinity_group = AntiAffinityGroup::new( + authz_project.id(), + anti_affinity_group_params, + ); + self.db_datastore + .anti_affinity_group_create( + opctx, + &authz_project, + anti_affinity_group, + ) + .await + .map(Into::into) + } + + pub(crate) async fn affinity_group_update( + &self, + opctx: &OpContext, + group_lookup: &lookup::AffinityGroup<'_>, + updates: ¶ms::AffinityGroupUpdate, + ) -> UpdateResult { + let (.., authz_group) = + group_lookup.lookup_for(authz::Action::Modify).await?; + self.db_datastore + .affinity_group_update(opctx, &authz_group, updates.clone().into()) + .await + .map(|g| g.into()) + } + + pub(crate) async fn anti_affinity_group_update( + &self, + opctx: &OpContext, + group_lookup: &lookup::AntiAffinityGroup<'_>, + updates: ¶ms::AntiAffinityGroupUpdate, + ) -> UpdateResult { + let (.., authz_group) = + group_lookup.lookup_for(authz::Action::Modify).await?; + self.db_datastore + .anti_affinity_group_update( + opctx, + &authz_group, + updates.clone().into(), + ) + .await + .map(|g| g.into()) + } + + pub(crate) async fn affinity_group_delete( + &self, + opctx: &OpContext, + group_lookup: &lookup::AffinityGroup<'_>, + ) -> DeleteResult { + let (.., authz_group) = + group_lookup.lookup_for(authz::Action::Delete).await?; + self.db_datastore.affinity_group_delete(opctx, &authz_group).await + } + + pub(crate) async fn anti_affinity_group_delete( + &self, + opctx: &OpContext, + group_lookup: &lookup::AntiAffinityGroup<'_>, + ) -> DeleteResult { + let (.., authz_group) = + group_lookup.lookup_for(authz::Action::Delete).await?; + self.db_datastore.anti_affinity_group_delete(opctx, &authz_group).await + } + + pub(crate) async fn affinity_group_member_list( + &self, + opctx: &OpContext, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_affinity_group) = affinity_group_lookup + .lookup_for(authz::Action::ListChildren) + .await?; + Ok(self + .db_datastore + .affinity_group_member_list(opctx, &authz_affinity_group, pagparams) + .await? + .into_iter() + .map(Into::into) + .collect()) + } + + pub(crate) async fn anti_affinity_group_member_list( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_anti_affinity_group) = anti_affinity_group_lookup + .lookup_for(authz::Action::ListChildren) + .await?; + Ok(self + .db_datastore + .anti_affinity_group_member_list( + opctx, + &authz_anti_affinity_group, + pagparams, + ) + .await? + .into_iter() + .map(Into::into) + .collect()) + } + + pub(crate) async fn affinity_group_member_view( + &self, + opctx: &OpContext, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result { + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .affinity_group_member_view(opctx, &authz_affinity_group, member) + .await + } + + pub(crate) async fn anti_affinity_group_member_view( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result { + let (.., authz_anti_affinity_group) = + anti_affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AntiAffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .anti_affinity_group_member_view( + opctx, + &authz_anti_affinity_group, + member, + ) + .await + } + + pub(crate) async fn affinity_group_member_add( + &self, + opctx: &OpContext, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result { + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Modify).await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .affinity_group_member_add( + opctx, + &authz_affinity_group, + member.clone(), + ) + .await?; + Ok(member) + } + + pub(crate) async fn anti_affinity_group_member_add( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result { + let (.., authz_anti_affinity_group) = anti_affinity_group_lookup + .lookup_for(authz::Action::Modify) + .await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AntiAffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .anti_affinity_group_member_add( + opctx, + &authz_anti_affinity_group, + member.clone(), + ) + .await?; + Ok(member) + } + + pub(crate) async fn affinity_group_member_delete( + &self, + opctx: &OpContext, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result<(), Error> { + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Modify).await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .affinity_group_member_delete(opctx, &authz_affinity_group, member) + .await + } + + pub(crate) async fn anti_affinity_group_member_delete( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + instance_lookup: &lookup::Instance<'_>, + ) -> Result<(), Error> { + let (.., authz_anti_affinity_group) = anti_affinity_group_lookup + .lookup_for(authz::Action::Modify) + .await?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let member = + external::AntiAffinityGroupMember::Instance(authz_instance.id()); + + self.db_datastore + .anti_affinity_group_member_delete( + opctx, + &authz_anti_affinity_group, + member, + ) + .await + } +} diff --git a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs index ce232a14113..1853a82fb19 100644 --- a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs +++ b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs @@ -201,12 +201,12 @@ mod tests { use nexus_db_model::Generation; use nexus_db_model::Resources; use nexus_db_model::SledResource; - use nexus_db_model::SledResourceKind; use nexus_db_model::Vmm; use nexus_db_model::VmmRuntimeState; use nexus_db_model::VmmState; use nexus_test_utils::resource_helpers; use nexus_test_utils_macros::nexus_test; + use omicron_uuid_kinds::InstanceUuid; use uuid::Uuid; type ControlPlaneTestContext = @@ -267,8 +267,8 @@ mod tests { dbg!(datastore .sled_reservation_create( &opctx, - destroyed_vmm_id.into_untyped_uuid(), - SledResourceKind::Instance, + InstanceUuid::from_untyped_uuid(instance.identity.id), + destroyed_vmm_id, resources.clone(), constraints, ) diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index db87d7db7d7..08d84244d83 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -46,6 +46,7 @@ use uuid::Uuid; // The implementation of Nexus is large, and split into a number of submodules // by resource. mod address_lot; +mod affinity; mod allow_list; pub(crate) mod background; mod bfd; diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs index 9f9fefea629..22134e8f6d1 100644 --- a/nexus/src/app/sagas/instance_common.rs +++ b/nexus/src/app/sagas/instance_common.rs @@ -38,6 +38,7 @@ pub(super) struct VmmAndSledIds { /// `propolis_id`. pub async fn reserve_vmm_resources( nexus: &Nexus, + instance_id: InstanceUuid, propolis_id: PropolisUuid, ncpus: u32, guest_memory: ByteCount, @@ -50,8 +51,8 @@ pub async fn reserve_vmm_resources( // See https://rfd.shared.oxide.computer/rfd/0205 for a more complete // discussion. // - // Right now, allocate an instance to any random sled agent. This has a few - // problems: + // Right now, allocate an instance to any random sled agent, as long as + // "constraints" and affinity rules are respected. This has a few problems: // // - There's no consideration for "health of the sled" here, other than // "time_deleted = Null". If the sled is rebooting, in a known unhealthy @@ -61,10 +62,6 @@ pub async fn reserve_vmm_resources( // - This is selecting a random sled from all sleds in the cluster. For // multi-rack, this is going to fling the sled to an arbitrary system. // Maybe that's okay, but worth knowing about explicitly. - // - // - This doesn't take into account anti-affinity - users will want to - // schedule instances that belong to a cluster on different failure - // domains. See https://github.com/oxidecomputer/omicron/issues/1705. let resources = db::model::Resources::new( ncpus, ByteCount::try_from(0i64).unwrap(), @@ -73,8 +70,8 @@ pub async fn reserve_vmm_resources( let resource = nexus .reserve_on_random_sled( - propolis_id.into_untyped_uuid(), - nexus_db_model::SledResourceKind::Instance, + instance_id, + propolis_id, resources, constraints, ) diff --git a/nexus/src/app/sagas/instance_migrate.rs b/nexus/src/app/sagas/instance_migrate.rs index 0789ef7a484..465105a813f 100644 --- a/nexus/src/app/sagas/instance_migrate.rs +++ b/nexus/src/app/sagas/instance_migrate.rs @@ -186,6 +186,7 @@ async fn sim_reserve_sled_resources( let resource = super::instance_common::reserve_vmm_resources( osagactx.nexus(), + InstanceUuid::from_untyped_uuid(params.instance.id()), propolis_id, u32::from(params.instance.ncpus.0 .0), params.instance.memory, @@ -193,7 +194,7 @@ async fn sim_reserve_sled_resources( ) .await?; - Ok(SledUuid::from_untyped_uuid(resource.sled_id)) + Ok(resource.sled_id.into()) } async fn sim_release_sled_resources( diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index aa09a165c83..e6af6453291 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -163,6 +163,7 @@ async fn sis_alloc_server( let resource = super::instance_common::reserve_vmm_resources( osagactx.nexus(), + InstanceUuid::from_untyped_uuid(params.db_instance.id()), propolis_id, u32::from(hardware_threads.0), reservoir_ram, @@ -170,7 +171,7 @@ async fn sis_alloc_server( ) .await?; - Ok(SledUuid::from_untyped_uuid(resource.sled_id)) + Ok(resource.sled_id.into()) } async fn sis_alloc_server_undo( diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index d58e111b6e6..ffa507ffb8e 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -25,7 +25,9 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PhysicalDiskUuid; +use omicron_uuid_kinds::PropolisUuid; use omicron_uuid_kinds::SledUuid; use sled_agent_client::Client as SledAgentClient; use std::net::SocketAddrV6; @@ -164,16 +166,16 @@ impl super::Nexus { pub(crate) async fn reserve_on_random_sled( &self, - resource_id: Uuid, - resource_kind: db::model::SledResourceKind, + instance_id: InstanceUuid, + propolis_id: PropolisUuid, resources: db::model::Resources, constraints: db::model::SledReservationConstraints, ) -> Result { self.db_datastore .sled_reservation_create( &self.opctx_alloc, - resource_id, - resource_kind, + instance_id, + propolis_id, resources, constraints, ) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 191c304216d..b1cf5b59e5a 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -54,6 +54,7 @@ use nexus_types::{ }, }; use omicron_common::api::external::http_pagination::data_page_params_for; +use omicron_common::api::external::http_pagination::id_pagination; use omicron_common::api::external::http_pagination::marker_for_id; use omicron_common::api::external::http_pagination::marker_for_name; use omicron_common::api::external::http_pagination::marker_for_name_or_id; @@ -69,7 +70,9 @@ use omicron_common::api::external::http_pagination::ScanParams; use omicron_common::api::external::AddressLot; use omicron_common::api::external::AddressLotBlock; use omicron_common::api::external::AddressLotCreateResponse; +use omicron_common::api::external::AffinityGroupMember; use omicron_common::api::external::AggregateBgpMessageHistory; +use omicron_common::api::external::AntiAffinityGroupMember; use omicron_common::api::external::BgpAnnounceSet; use omicron_common::api::external::BgpAnnouncement; use omicron_common::api::external::BgpConfig; @@ -2506,6 +2509,678 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } + // Affinity Groups + + async fn affinity_group_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let project_lookup = + nexus.project_lookup(&opctx, scan_params.selector.clone())?; + let groups = nexus + .affinity_group_list(&opctx, &project_lookup, &paginated_by) + .await?; + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + groups, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + + let group_selector = params::AffinityGroupSelector { + affinity_group: path.affinity_group, + project: query.project.clone(), + }; + + let (.., group) = nexus + .affinity_group_lookup(&opctx, group_selector)? + .fetch() + .await?; + + Ok(HttpResponseOk(group.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_list( + rqctx: RequestContext, + query_params: Query>, + path_params: Path, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanById::from_query(&query)?; + let paginated_by = id_pagination(&pag_params, scan_params)?; + + let group_selector = params::AffinityGroupSelector { + project: scan_params.selector.project.clone(), + affinity_group: path.affinity_group, + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; + let affinity_group_member_instances = nexus + .affinity_group_member_list( + &opctx, + &group_lookup, + &paginated_by, + ) + .await?; + Ok(HttpResponseOk(ScanById::results_page( + &query, + affinity_group_member_instances, + &marker_for_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_instance_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AffinityGroupSelector { + affinity_group: path.affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; + + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + + let group = nexus + .affinity_group_member_view( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + + Ok(HttpResponseOk(group)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_instance_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AffinityGroupSelector { + affinity_group: path.affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; + + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + + let member = nexus + .affinity_group_member_add( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + Ok(HttpResponseCreated(member)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_instance_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AffinityGroupSelector { + affinity_group: path.affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; + + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + nexus + .affinity_group_member_delete( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_create( + rqctx: RequestContext, + query_params: Query, + new_affinity_group_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let project_lookup = nexus.project_lookup(&opctx, query)?; + let new_affinity_group = new_affinity_group_params.into_inner(); + let affinity_group = nexus + .affinity_group_create( + &opctx, + &project_lookup, + new_affinity_group, + ) + .await?; + Ok(HttpResponseCreated(affinity_group)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_update( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + updated_group: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let updates = updated_group.into_inner(); + let query = query_params.into_inner(); + let group_selector = params::AffinityGroupSelector { + project: query.project, + affinity_group: path.affinity_group, + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; + let affinity_group = nexus + .affinity_group_update(&opctx, &group_lookup, &updates) + .await?; + Ok(HttpResponseOk(affinity_group)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let group_selector = params::AffinityGroupSelector { + project: query.project, + affinity_group: path.affinity_group, + }; + let group_lookup = + nexus.affinity_group_lookup(&opctx, group_selector)?; + nexus.affinity_group_delete(&opctx, &group_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let project_lookup = + nexus.project_lookup(&opctx, scan_params.selector.clone())?; + let groups = nexus + .anti_affinity_group_list( + &opctx, + &project_lookup, + &paginated_by, + ) + .await?; + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + groups, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + + let (.., group) = nexus + .anti_affinity_group_lookup(&opctx, group_selector)? + .fetch() + .await?; + + Ok(HttpResponseOk(group.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_list( + rqctx: RequestContext, + query_params: Query>, + path_params: Path, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanById::from_query(&query)?; + let paginated_by = id_pagination(&pag_params, scan_params)?; + + let group_selector = params::AntiAffinityGroupSelector { + project: scan_params.selector.project.clone(), + anti_affinity_group: path.anti_affinity_group, + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + let group_members = nexus + .anti_affinity_group_member_list( + &opctx, + &group_lookup, + &paginated_by, + ) + .await?; + Ok(HttpResponseOk(ScanById::results_page( + &query, + group_members, + &marker_for_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_instance_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + + let group = nexus + .anti_affinity_group_member_view( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + + Ok(HttpResponseOk(group)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_instance_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + + let member = nexus + .anti_affinity_group_member_add( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + Ok(HttpResponseCreated(member)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_instance_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select instance + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + + nexus + .anti_affinity_group_member_delete( + &opctx, + &group_lookup, + &instance_lookup, + ) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_create( + rqctx: RequestContext, + query_params: Query, + new_anti_affinity_group_params: TypedBody< + params::AntiAffinityGroupCreate, + >, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let project_lookup = nexus.project_lookup(&opctx, query)?; + let new_anti_affinity_group = + new_anti_affinity_group_params.into_inner(); + let anti_affinity_group = nexus + .anti_affinity_group_create( + &opctx, + &project_lookup, + new_anti_affinity_group, + ) + .await?; + Ok(HttpResponseCreated(anti_affinity_group)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_update( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + updated_group: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let updates = updated_group.into_inner(); + let query = query_params.into_inner(); + let group_selector = params::AntiAffinityGroupSelector { + project: query.project, + anti_affinity_group: path.anti_affinity_group, + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + let anti_affinity_group = nexus + .anti_affinity_group_update(&opctx, &group_lookup, &updates) + .await?; + Ok(HttpResponseOk(anti_affinity_group)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let group_selector = params::AntiAffinityGroupSelector { + project: query.project, + anti_affinity_group: path.anti_affinity_group, + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + nexus.anti_affinity_group_delete(&opctx, &group_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + // Certificates async fn certificate_list( diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index 9fe2b62a276..3ef1858cfa2 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -21,6 +21,8 @@ use nexus_types::external_api::shared::Baseboard; use nexus_types::external_api::shared::IdentityType; use nexus_types::external_api::shared::IpRange; use nexus_types::external_api::views; +use nexus_types::external_api::views::AffinityGroup; +use nexus_types::external_api::views::AntiAffinityGroup; use nexus_types::external_api::views::Certificate; use nexus_types::external_api::views::FloatingIp; use nexus_types::external_api::views::InternetGateway; @@ -33,9 +35,11 @@ use nexus_types::external_api::views::VpcSubnet; use nexus_types::external_api::views::{Project, Silo, Vpc, VpcRouter}; use nexus_types::identity::Resource; use nexus_types::internal_api::params as internal_params; +use omicron_common::api::external::AffinityPolicy; use omicron_common::api::external::ByteCount; use omicron_common::api::external::Disk; use omicron_common::api::external::Error; +use omicron_common::api::external::FailureDomain; use omicron_common::api::external::Generation; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::Instance; @@ -581,6 +585,46 @@ where object_create_error(client, &url, body, status).await } +pub async fn create_affinity_group( + client: &ClientTestContext, + project_name: &str, + group_name: &str, +) -> AffinityGroup { + object_create( + &client, + format!("/v1/affinity-groups?project={}", &project_name).as_str(), + ¶ms::AffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: String::from("affinity group description"), + }, + policy: AffinityPolicy::Fail, + failure_domain: FailureDomain::Sled, + }, + ) + .await +} + +pub async fn create_anti_affinity_group( + client: &ClientTestContext, + project_name: &str, + group_name: &str, +) -> AntiAffinityGroup { + object_create( + &client, + format!("/v1/anti-affinity-groups?project={}", &project_name).as_str(), + ¶ms::AntiAffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: String::from("affinity group description"), + }, + policy: AffinityPolicy::Fail, + failure_domain: FailureDomain::Sled, + }, + ) + .await +} + pub async fn create_vpc( client: &ClientTestContext, project_name: &str, diff --git a/nexus/tests/integration_tests/affinity.rs b/nexus/tests/integration_tests/affinity.rs new file mode 100644 index 00000000000..65ab96b0a90 --- /dev/null +++ b/nexus/tests/integration_tests/affinity.rs @@ -0,0 +1,895 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tests Affinity (and Anti-Affinity) Groups + +use dropshot::test_util::ClientTestContext; +use dropshot::HttpErrorResponseBody; +use http::StatusCode; +use nexus_test_utils::http_testing::AuthnMode; +use nexus_test_utils::http_testing::NexusRequest; +use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::resource_helpers::create_default_ip_pool; +use nexus_test_utils::resource_helpers::create_instance_with; +use nexus_test_utils::resource_helpers::create_project; +use nexus_test_utils::resource_helpers::object_create; +use nexus_test_utils::resource_helpers::object_create_error; +use nexus_test_utils::resource_helpers::object_delete; +use nexus_test_utils::resource_helpers::object_delete_error; +use nexus_test_utils::resource_helpers::object_get; +use nexus_test_utils::resource_helpers::object_get_error; +use nexus_test_utils::resource_helpers::object_put; +use nexus_test_utils::resource_helpers::object_put_error; +use nexus_test_utils::resource_helpers::objects_list_page_authz; +use nexus_test_utils_macros::nexus_test; +use nexus_types::external_api::params; +use nexus_types::external_api::views::AffinityGroup; +use nexus_types::external_api::views::AntiAffinityGroup; +use nexus_types::external_api::views::Sled; +use nexus_types::external_api::views::SledInstance; +use omicron_common::api::external; +use omicron_common::api::external::AffinityGroupMember; +use omicron_common::api::external::AntiAffinityGroupMember; +use omicron_common::api::external::ObjectIdentity; +use std::collections::BTreeSet; +use std::marker::PhantomData; + +type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + +// Simplifying mechanism for making calls to Nexus' external API +struct ApiHelper<'a> { + client: &'a ClientTestContext, +} + +// This is an extention of the "ApiHelper", with an opinion about: +// +// - What project (if any) is selected +// - Whether or not we're accessing affinity/anti-affinity groups +struct ProjectScopedApiHelper<'a, T> { + client: &'a ClientTestContext, + project: Option<&'a str>, + affinity_type: PhantomData, +} + +impl ProjectScopedApiHelper<'_, T> { + async fn create_stopped_instance( + &self, + instance_name: &str, + ) -> external::Instance { + create_instance_with( + &self.client, + &self.project.as_ref().expect("Need to specify project name"), + instance_name, + ¶ms::InstanceNetworkInterfaceAttachment::None, + // Disks= + Vec::::new(), + // External IPs= + Vec::::new(), + // Start= + false, + // Auto-restart policy= + None, + ) + .await + } + + async fn groups_list(&self) -> Vec { + let url = groups_url(T::URL_COMPONENT, self.project); + objects_list_page_authz(&self.client, &url).await.items + } + + async fn groups_list_expect_error( + &self, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = groups_url(T::URL_COMPONENT, self.project); + object_get_error(&self.client, &url, status).await + } + + async fn group_create(&self, group: &str) -> T::Group { + let url = groups_url(T::URL_COMPONENT, self.project); + let params = T::make_create_params(group); + object_create(&self.client, &url, ¶ms).await + } + + async fn group_create_expect_error( + &self, + group: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = groups_url(T::URL_COMPONENT, self.project); + let params = T::make_create_params(group); + object_create_error(&self.client, &url, ¶ms, status).await + } + + async fn group_get(&self, group: &str) -> T::Group { + let url = group_url(T::URL_COMPONENT, self.project, group); + object_get(&self.client, &url).await + } + + async fn group_get_expect_error( + &self, + group: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_url(T::URL_COMPONENT, self.project, group); + object_get_error(&self.client, &url, status).await + } + + async fn group_update(&self, group: &str) -> T::Group { + let url = group_url(T::URL_COMPONENT, self.project, group); + let params = T::make_update_params(); + object_put(&self.client, &url, ¶ms).await + } + + async fn group_update_expect_error( + &self, + group: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_url(T::URL_COMPONENT, self.project, group); + let params = T::make_update_params(); + object_put_error(&self.client, &url, ¶ms, status).await + } + + async fn group_delete(&self, group: &str) { + let url = group_url(T::URL_COMPONENT, self.project, group); + object_delete(&self.client, &url).await + } + + async fn group_delete_expect_error( + &self, + group: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_url(T::URL_COMPONENT, self.project, group); + object_delete_error(&self.client, &url, status).await + } + + async fn group_members_list(&self, group: &str) -> Vec { + let url = group_members_url(T::URL_COMPONENT, self.project, group); + objects_list_page_authz(&self.client, &url).await.items + } + + async fn group_members_list_expect_error( + &self, + group: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_members_url(T::URL_COMPONENT, self.project, group); + object_get_error(&self.client, &url, status).await + } + + async fn group_member_add(&self, group: &str, instance: &str) -> T::Member { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_create(&self.client, &url, &()).await + } + + async fn group_member_add_expect_error( + &self, + group: &str, + instance: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_create_error(&self.client, &url, &(), status).await + } + + async fn group_member_get(&self, group: &str, instance: &str) -> T::Member { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_get(&self.client, &url).await + } + + async fn group_member_get_expect_error( + &self, + group: &str, + instance: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_get_error(&self.client, &url, status).await + } + + async fn group_member_delete(&self, group: &str, instance: &str) { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_delete(&self.client, &url).await + } + + async fn group_member_delete_expect_error( + &self, + group: &str, + instance: &str, + status: StatusCode, + ) -> HttpErrorResponseBody { + let url = group_member_instance_url( + T::URL_COMPONENT, + self.project, + group, + instance, + ); + object_delete_error(&self.client, &url, status).await + } +} + +impl<'a> ApiHelper<'a> { + fn new(client: &'a ClientTestContext) -> Self { + Self { client } + } + + fn use_project( + &'a self, + project: &'a str, + ) -> ProjectScopedApiHelper<'a, T> { + ProjectScopedApiHelper { + client: self.client, + project: Some(project), + affinity_type: PhantomData, + } + } + + fn no_project(&'a self) -> ProjectScopedApiHelper<'a, T> { + ProjectScopedApiHelper { + client: self.client, + project: None, + affinity_type: PhantomData, + } + } + + async fn create_project(&self, name: &str) { + create_project(&self.client, name).await; + } + + async fn sleds_list(&self) -> Vec { + let url = "/v1/system/hardware/sleds"; + objects_list_page_authz(&self.client, url).await.items + } + + async fn sled_instance_list(&self, sled: &str) -> Vec { + let url = format!("/v1/system/hardware/sleds/{sled}/instances"); + objects_list_page_authz(&self.client, &url).await.items + } + + async fn start_instance( + &self, + instance: &external::Instance, + ) -> external::Instance { + let uri = format!("/v1/instances/{}/start", instance.identity.id); + + NexusRequest::new( + RequestBuilder::new(&self.client, http::Method::POST, &uri) + .expect_status(Some(http::StatusCode::ACCEPTED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap_or_else(|e| { + panic!("failed to make \"POST\" request to {uri}: {e}") + }) + .parsed_body() + .unwrap() + } +} + +fn project_query_param_suffix(project: Option<&str>) -> String { + if let Some(project) = project { + format!("?project={project}") + } else { + String::new() + } +} + +/// Types and traits used by both affinity and anti-affinity groups. +/// +/// Use this trait if you're trying to test something which appilies to both +/// group types. Conversely, if you're trying to test behavior specific to one +/// type or the other, I recommend that you avoid making your tests generic. +trait AffinityGroupish { + /// The struct used to represent this group. + /// + /// Should be the result of GET-ing this group. + type Group: serde::de::DeserializeOwned + ObjectIdentity; + + /// The struct representing a single member within this group. + type Member: serde::de::DeserializeOwned; + + /// Parameters that can be used to construct this group as a part of a POST + /// request. + type CreateParams: serde::Serialize; + + /// Parameters that can be used to update this group as a part of a PUT + /// request. + type UpdateParams: serde::Serialize; + + const URL_COMPONENT: &'static str; + const RESOURCE_NAME: &'static str; + + fn make_create_params(group_name: &str) -> Self::CreateParams; + fn make_update_params() -> Self::UpdateParams; +} + +// Arbitrary text used to validate PUT calls to groups +const NEW_DESCRIPTION: &'static str = "Updated description"; + +struct AffinityType; + +impl AffinityGroupish for AffinityType { + type Group = AffinityGroup; + type Member = AffinityGroupMember; + type CreateParams = params::AffinityGroupCreate; + type UpdateParams = params::AffinityGroupUpdate; + + const URL_COMPONENT: &'static str = "affinity-groups"; + const RESOURCE_NAME: &'static str = "affinity-group"; + + fn make_create_params(group_name: &str) -> Self::CreateParams { + params::AffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: String::from("This is a description"), + }, + policy: external::AffinityPolicy::Fail, + failure_domain: external::FailureDomain::Sled, + } + } + + fn make_update_params() -> Self::UpdateParams { + params::AffinityGroupUpdate { + identity: external::IdentityMetadataUpdateParams { + name: None, + description: Some(NEW_DESCRIPTION.to_string()), + }, + } + } +} + +struct AntiAffinityType; + +impl AffinityGroupish for AntiAffinityType { + type Group = AntiAffinityGroup; + type Member = AntiAffinityGroupMember; + type CreateParams = params::AntiAffinityGroupCreate; + type UpdateParams = params::AntiAffinityGroupUpdate; + + const URL_COMPONENT: &'static str = "anti-affinity-groups"; + const RESOURCE_NAME: &'static str = "anti-affinity-group"; + + fn make_create_params(group_name: &str) -> Self::CreateParams { + params::AntiAffinityGroupCreate { + identity: external::IdentityMetadataCreateParams { + name: group_name.parse().unwrap(), + description: String::from("This is a description"), + }, + policy: external::AffinityPolicy::Fail, + failure_domain: external::FailureDomain::Sled, + } + } + + fn make_update_params() -> Self::UpdateParams { + params::AntiAffinityGroupUpdate { + identity: external::IdentityMetadataUpdateParams { + name: None, + description: Some(NEW_DESCRIPTION.to_string()), + }, + } + } +} + +fn groups_url(ty: &str, project: Option<&str>) -> String { + let query_params = project_query_param_suffix(project); + format!("/v1/{ty}{query_params}") +} + +fn group_url(ty: &str, project: Option<&str>, group: &str) -> String { + let query_params = project_query_param_suffix(project); + format!("/v1/{ty}/{group}{query_params}") +} + +fn group_members_url(ty: &str, project: Option<&str>, group: &str) -> String { + let query_params = project_query_param_suffix(project); + format!("/v1/{ty}/{group}/members{query_params}") +} + +fn group_member_instance_url( + ty: &str, + project: Option<&str>, + group: &str, + instance: &str, +) -> String { + let query_params = project_query_param_suffix(project); + format!("/v1/{ty}/{group}/members/instance/{instance}{query_params}") +} + +#[nexus_test(extra_sled_agents = 2)] +async fn test_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { + let external_client = &cptestctx.external_client; + + const PROJECT_NAME: &'static str = "test-project"; + const GROUP_NAME: &'static str = "group"; + const EXPECTED_SLEDS: usize = 3; + const INSTANCE_COUNT: usize = EXPECTED_SLEDS; + + let api = ApiHelper::new(external_client); + + // Verify the expected sleds to begin with. + let sleds = api.sleds_list().await; + assert_eq!(sleds.len(), EXPECTED_SLEDS); + + // Verify that there are no instances on the sleds. + for sled in &sleds { + let sled_id = sled.identity.id.to_string(); + assert!(api.sled_instance_list(&sled_id).await.is_empty()); + } + + // Create an IP pool and project that we'll use for testing. + create_default_ip_pool(&external_client).await; + api.create_project(PROJECT_NAME).await; + + let project_api = api.use_project::(PROJECT_NAME); + + let mut instances = Vec::new(); + for i in 0..INSTANCE_COUNT { + instances.push( + project_api + .create_stopped_instance(&format!("test-instance-{i}")) + .await, + ); + } + + // When we start, we observe no affinity groups + let groups = project_api.groups_list().await; + assert!(groups.is_empty()); + + // We can now create a group and observe it + let group = project_api.group_create(GROUP_NAME).await; + + // We can list it and also GET the group specifically + let groups = project_api.groups_list().await; + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].identity.id, group.identity.id); + + let observed_group = project_api.group_get(GROUP_NAME).await; + assert_eq!(observed_group.identity.id, group.identity.id); + + // List all members of the affinity group (expect nothing) + let members = project_api.group_members_list(GROUP_NAME).await; + assert!(members.is_empty()); + + // Add these instances to an affinity group + for instance in &instances { + project_api + .group_member_add(GROUP_NAME, &instance.identity.name.to_string()) + .await; + } + + // List members again (expect all instances) + let members = project_api.group_members_list(GROUP_NAME).await; + assert_eq!(members.len(), instances.len()); + + // We can also list each member + for instance in &instances { + project_api + .group_member_get(GROUP_NAME, instance.identity.name.as_str()) + .await; + } + + // Start the instances we created earlier. + // + // We don't actually care that they're "running" from the perspective of the + // simulated sled agent, we just want placement to be triggered from Nexus. + for instance in &instances { + api.start_instance(&instance).await; + } + + // Use a BTreeSet so we can ignore ordering when comparing instance + // placement. + let expected_instances = instances + .iter() + .map(|instance| instance.identity.id) + .collect::>(); + + // We expect that all sleds will be empty, except for one, which will have + // all the instances in our affinity group. + let mut empty_sleds = 0; + let mut populated_sleds = 0; + for sled in &sleds { + let observed_instances = api + .sled_instance_list(&sled.identity.id.to_string()) + .await + .into_iter() + .map(|sled_instance| sled_instance.identity.id) + .collect::>(); + + if !observed_instances.is_empty() { + assert_eq!(observed_instances, expected_instances); + populated_sleds += 1; + } else { + empty_sleds += 1; + } + } + assert_eq!(populated_sleds, 1); + assert_eq!(empty_sleds, 2); +} + +#[nexus_test(extra_sled_agents = 2)] +async fn test_anti_affinity_group_usage(cptestctx: &ControlPlaneTestContext) { + let external_client = &cptestctx.external_client; + + const PROJECT_NAME: &'static str = "test-project"; + const GROUP_NAME: &'static str = "group"; + const EXPECTED_SLEDS: usize = 3; + const INSTANCE_COUNT: usize = EXPECTED_SLEDS; + + let api = ApiHelper::new(external_client); + + // Verify the expected sleds to begin with. + let sleds = api.sleds_list().await; + assert_eq!(sleds.len(), EXPECTED_SLEDS); + + // Verify that there are no instances on the sleds. + for sled in &sleds { + let sled_id = sled.identity.id.to_string(); + assert!(api.sled_instance_list(&sled_id).await.is_empty()); + } + + // Create an IP pool and project that we'll use for testing. + create_default_ip_pool(&external_client).await; + api.create_project(PROJECT_NAME).await; + + let project_api = api.use_project::(PROJECT_NAME); + + let mut instances = Vec::new(); + for i in 0..INSTANCE_COUNT { + instances.push( + project_api + .create_stopped_instance(&format!("test-instance-{i}")) + .await, + ); + } + + // When we start, we observe no anti-affinity groups + let groups = project_api.groups_list().await; + assert!(groups.is_empty()); + + // We can now create a group and observe it + let group = project_api.group_create(GROUP_NAME).await; + + // We can list it and also GET the group specifically + let groups = project_api.groups_list().await; + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].identity.id, group.identity.id); + + let observed_group = project_api.group_get(GROUP_NAME).await; + assert_eq!(observed_group.identity.id, group.identity.id); + + // List all members of the anti-affinity group (expect nothing) + let members = project_api.group_members_list(GROUP_NAME).await; + assert!(members.is_empty()); + + // Add these instances to the anti-affinity group + for instance in &instances { + project_api + .group_member_add(GROUP_NAME, &instance.identity.name.to_string()) + .await; + } + + // List members again (expect all instances) + let members = project_api.group_members_list(GROUP_NAME).await; + assert_eq!(members.len(), instances.len()); + + // We can also list each member + for instance in &instances { + project_api + .group_member_get(GROUP_NAME, instance.identity.name.as_str()) + .await; + } + + // Start the instances we created earlier. + // + // We don't actually care that they're "running" from the perspective of the + // simulated sled agent, we just want placement to be triggered from Nexus. + for instance in &instances { + api.start_instance(&instance).await; + } + + let mut expected_instances = instances + .iter() + .map(|instance| instance.identity.id) + .collect::>(); + + // We expect that each sled will have a since instance, as none of the + // instances will want to be anti-located from each other. + for sled in &sleds { + let observed_instances = api + .sled_instance_list(&sled.identity.id.to_string()) + .await + .into_iter() + .map(|sled_instance| sled_instance.identity.id) + .collect::>(); + + assert_eq!( + observed_instances.len(), + 1, + "All instances should be placed on distinct sleds" + ); + + assert!( + expected_instances.remove(&observed_instances[0]), + "The instance {} was observed on multiple sleds", + observed_instances[0] + ); + } + + assert!( + expected_instances.is_empty(), + "Did not find allocations for some instances: {expected_instances:?}" + ); +} + +#[nexus_test] +async fn test_affinity_group_crud(cptestctx: &ControlPlaneTestContext) { + let external_client = &cptestctx.external_client; + test_group_crud::(external_client).await; +} + +#[nexus_test] +async fn test_anti_affinity_group_crud(cptestctx: &ControlPlaneTestContext) { + let external_client = &cptestctx.external_client; + test_group_crud::(external_client).await; +} + +async fn test_group_crud(client: &ClientTestContext) { + const PROJECT_NAME: &'static str = "test-project"; + const GROUP_NAME: &'static str = "group"; + + let api = ApiHelper::new(client); + + // Create an IP pool and project that we'll use for testing. + create_default_ip_pool(&client).await; + api.create_project(PROJECT_NAME).await; + + let project_api = api.use_project::(PROJECT_NAME); + + let instance = project_api.create_stopped_instance("test-instance").await; + + // When we start, we observe no affinity groups + let groups = project_api.groups_list().await; + assert!(groups.is_empty()); + + // We can now create a group and observe it + project_api.group_create(GROUP_NAME).await; + let response = project_api + .group_create_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + assert_eq!( + response.message, + format!("already exists: {} \"{GROUP_NAME}\"", T::RESOURCE_NAME), + ); + + // We can modify the group itself + let group = project_api.group_update(GROUP_NAME).await; + assert_eq!(group.identity().description, NEW_DESCRIPTION); + + // List all members of the affinity group (expect nothing) + let members = project_api.group_members_list(GROUP_NAME).await; + assert!(members.is_empty()); + + // Add the instance to the affinity group + let instance_name = &instance.identity.name.to_string(); + project_api.group_member_add(GROUP_NAME, &instance_name).await; + let response = project_api + .group_member_add_expect_error( + GROUP_NAME, + &instance_name, + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!( + response.message, + format!( + "already exists: {}-member \"{}\"", + T::RESOURCE_NAME, + instance.identity.id + ), + ); + + // List members again (expect the instance) + let members = project_api.group_members_list(GROUP_NAME).await; + assert_eq!(members.len(), 1); + project_api + .group_member_get(GROUP_NAME, instance.identity.name.as_str()) + .await; + + // Delete the member, observe that it is gone + project_api.group_member_delete(GROUP_NAME, &instance_name).await; + project_api + .group_member_delete_expect_error( + GROUP_NAME, + &instance_name, + StatusCode::NOT_FOUND, + ) + .await; + let members = project_api.group_members_list(GROUP_NAME).await; + assert_eq!(members.len(), 0); + project_api + .group_member_get_expect_error( + GROUP_NAME, + &instance_name, + StatusCode::NOT_FOUND, + ) + .await; + + // Delete the group, observe that it is gone + project_api.group_delete(GROUP_NAME).await; + project_api + .group_delete_expect_error(GROUP_NAME, StatusCode::NOT_FOUND) + .await; + project_api.group_get_expect_error(GROUP_NAME, StatusCode::NOT_FOUND).await; + let groups = project_api.groups_list().await; + assert!(groups.is_empty()); +} + +#[nexus_test] +async fn test_affinity_group_project_selector( + cptestctx: &ControlPlaneTestContext, +) { + let external_client = &cptestctx.external_client; + test_group_project_selector::(external_client).await; +} + +#[nexus_test] +async fn test_anti_affinity_group_project_selector( + cptestctx: &ControlPlaneTestContext, +) { + let external_client = &cptestctx.external_client; + test_group_project_selector::(external_client).await; +} + +async fn test_group_project_selector( + client: &ClientTestContext, +) { + const PROJECT_NAME: &'static str = "test-project"; + const GROUP_NAME: &'static str = "group"; + + let api = ApiHelper::new(client); + + // Create an IP pool and project that we'll use for testing. + create_default_ip_pool(&client).await; + api.create_project(PROJECT_NAME).await; + + // All requests use the "?project={PROJECT_NAME}" query parameter + let project_api = api.use_project::(PROJECT_NAME); + // All requests omit the project query parameter + let no_project_api = api.no_project::(); + + let instance = project_api.create_stopped_instance("test-instance").await; + + // We can only list groups within a project + no_project_api.groups_list_expect_error(StatusCode::BAD_REQUEST).await; + let _groups = project_api.groups_list().await; + + // We can only create a group within a project + no_project_api + .group_create_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + let group = project_api.group_create(GROUP_NAME).await; + + // Once we've created a group, we can access it by: + // + // - Project + Group Name, or + // - No Project + Group ID + // + // Other combinations are considered bad requests. + let group_id = group.identity().id.to_string(); + + project_api.group_get(GROUP_NAME).await; + no_project_api.group_get(&group_id).await; + project_api + .group_get_expect_error(&group_id, StatusCode::BAD_REQUEST) + .await; + no_project_api + .group_get_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + + // Same for listing members + project_api.group_members_list(GROUP_NAME).await; + no_project_api.group_members_list(&group_id).await; + project_api + .group_members_list_expect_error(&group_id, StatusCode::BAD_REQUEST) + .await; + no_project_api + .group_members_list_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + + // Same for updating the group + project_api.group_update(GROUP_NAME).await; + no_project_api.group_update(&group_id).await; + project_api + .group_update_expect_error(&group_id, StatusCode::BAD_REQUEST) + .await; + no_project_api + .group_update_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + + // Group Members can be added by name or UUID + let instance_name = instance.identity.name.as_str(); + let instance_id = instance.identity.id.to_string(); + project_api.group_member_add(GROUP_NAME, instance_name).await; + project_api.group_member_delete(GROUP_NAME, instance_name).await; + no_project_api.group_member_add(&group_id, &instance_id).await; + no_project_api.group_member_delete(&group_id, &instance_id).await; + + // Trying to use UUIDs with the project selector is invalid + project_api + .group_member_add_expect_error( + GROUP_NAME, + &instance_id, + StatusCode::BAD_REQUEST, + ) + .await; + project_api + .group_member_add_expect_error( + &group_id, + instance_name, + StatusCode::BAD_REQUEST, + ) + .await; + + // Using any names without the project selector is invalid + no_project_api + .group_member_add_expect_error( + GROUP_NAME, + &instance_id, + StatusCode::BAD_REQUEST, + ) + .await; + no_project_api + .group_member_add_expect_error( + &group_id, + instance_name, + StatusCode::BAD_REQUEST, + ) + .await; + no_project_api + .group_member_add_expect_error( + GROUP_NAME, + instance_name, + StatusCode::BAD_REQUEST, + ) + .await; + + // Group deletion also prevents mixing {project, ID} and {no-project, name}. + project_api + .group_delete_expect_error(&group_id, StatusCode::BAD_REQUEST) + .await; + no_project_api + .group_delete_expect_error(GROUP_NAME, StatusCode::BAD_REQUEST) + .await; + no_project_api.group_delete(&group_id).await; +} diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index 656b4ba8266..cb145a840d0 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -24,8 +24,10 @@ use nexus_types::external_api::shared::IpRange; use nexus_types::external_api::shared::Ipv4Range; use nexus_types::external_api::views::SledProvisionPolicy; use omicron_common::api::external::AddressLotKind; +use omicron_common::api::external::AffinityPolicy; use omicron_common::api::external::AllowedSourceIps; use omicron_common::api::external::ByteCount; +use omicron_common::api::external::FailureDomain; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::IdentityMetadataUpdateParams; use omicron_common::api::external::InstanceCpuCount; @@ -151,6 +153,12 @@ pub static DEMO_PROJECT_URL_IMAGES: Lazy = Lazy::new(|| format!("/v1/images?project={}", *DEMO_PROJECT_NAME)); pub static DEMO_PROJECT_URL_INSTANCES: Lazy = Lazy::new(|| format!("/v1/instances?project={}", *DEMO_PROJECT_NAME)); +pub static DEMO_PROJECT_URL_AFFINITY_GROUPS: Lazy = + Lazy::new(|| format!("/v1/affinity-groups?project={}", *DEMO_PROJECT_NAME)); +pub static DEMO_PROJECT_URL_ANTI_AFFINITY_GROUPS: Lazy = + Lazy::new(|| { + format!("/v1/anti-affinity-groups?project={}", *DEMO_PROJECT_NAME) + }); pub static DEMO_PROJECT_URL_SNAPSHOTS: Lazy = Lazy::new(|| format!("/v1/snapshots?project={}", *DEMO_PROJECT_NAME)); pub static DEMO_PROJECT_URL_VPCS: Lazy = @@ -415,9 +423,99 @@ pub static DEMO_IMPORT_DISK_FINALIZE_URL: Lazy = Lazy::new(|| { ) }); +// Affinity/Anti- group used for testing + +pub static DEMO_AFFINITY_GROUP_NAME: Lazy = + Lazy::new(|| "demo-affinity-group".parse().unwrap()); +pub static DEMO_AFFINITY_GROUP_URL: Lazy = Lazy::new(|| { + format!( + "/v1/affinity-groups/{}?{}", + *DEMO_AFFINITY_GROUP_NAME, *DEMO_PROJECT_SELECTOR + ) +}); +pub static DEMO_AFFINITY_GROUP_MEMBERS_URL: Lazy = Lazy::new(|| { + format!( + "/v1/affinity-groups/{}/members?{}", + *DEMO_AFFINITY_GROUP_NAME, *DEMO_PROJECT_SELECTOR + ) +}); +pub static DEMO_AFFINITY_GROUP_INSTANCE_MEMBER_URL: Lazy = + Lazy::new(|| { + format!( + "/v1/affinity-groups/{}/members/instance/{}?{}", + *DEMO_AFFINITY_GROUP_NAME, + *DEMO_STOPPED_INSTANCE_NAME, + *DEMO_PROJECT_SELECTOR + ) + }); +pub static DEMO_AFFINITY_GROUP_CREATE: Lazy = + Lazy::new(|| params::AffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: DEMO_AFFINITY_GROUP_NAME.clone(), + description: String::from(""), + }, + policy: AffinityPolicy::Allow, + failure_domain: FailureDomain::Sled, + }); +pub static DEMO_AFFINITY_GROUP_UPDATE: Lazy = + Lazy::new(|| params::AffinityGroupUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: Some(String::from("an updated description")), + }, + }); + +pub static DEMO_ANTI_AFFINITY_GROUP_NAME: Lazy = + Lazy::new(|| "demo-anti-affinity-group".parse().unwrap()); +pub static DEMO_ANTI_AFFINITY_GROUPS_URL: Lazy = Lazy::new(|| { + format!("/v1/anti-affinity-groups?{}", *DEMO_PROJECT_SELECTOR) +}); +pub static DEMO_ANTI_AFFINITY_GROUP_URL: Lazy = Lazy::new(|| { + format!( + "/v1/anti-affinity-groups/{}?{}", + *DEMO_ANTI_AFFINITY_GROUP_NAME, *DEMO_PROJECT_SELECTOR + ) +}); +pub static DEMO_ANTI_AFFINITY_GROUP_MEMBERS_URL: Lazy = + Lazy::new(|| { + format!( + "/v1/anti-affinity-groups/{}/members?{}", + *DEMO_ANTI_AFFINITY_GROUP_NAME, *DEMO_PROJECT_SELECTOR + ) + }); +pub static DEMO_ANTI_AFFINITY_GROUP_INSTANCE_MEMBER_URL: Lazy = + Lazy::new(|| { + format!( + "/v1/anti-affinity-groups/{}/members/instance/{}?{}", + *DEMO_ANTI_AFFINITY_GROUP_NAME, + *DEMO_STOPPED_INSTANCE_NAME, + *DEMO_PROJECT_SELECTOR + ) + }); +pub static DEMO_ANTI_AFFINITY_GROUP_CREATE: Lazy< + params::AntiAffinityGroupCreate, +> = Lazy::new(|| params::AntiAffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: DEMO_ANTI_AFFINITY_GROUP_NAME.clone(), + description: String::from(""), + }, + policy: AffinityPolicy::Allow, + failure_domain: FailureDomain::Sled, +}); +pub static DEMO_ANTI_AFFINITY_GROUP_UPDATE: Lazy< + params::AntiAffinityGroupUpdate, +> = Lazy::new(|| params::AntiAffinityGroupUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: Some(String::from("an updated description")), + }, +}); + // Instance used for testing pub static DEMO_INSTANCE_NAME: Lazy = Lazy::new(|| "demo-instance".parse().unwrap()); +pub static DEMO_STOPPED_INSTANCE_NAME: Lazy = + Lazy::new(|| "demo-stopped-instance".parse().unwrap()); pub static DEMO_INSTANCE_URL: Lazy = Lazy::new(|| { format!("/v1/instances/{}?{}", *DEMO_INSTANCE_NAME, *DEMO_PROJECT_SELECTOR) }); @@ -513,6 +611,26 @@ pub static DEMO_INSTANCE_CREATE: Lazy = start: true, auto_restart_policy: Default::default(), }); +pub static DEMO_STOPPED_INSTANCE_CREATE: Lazy = + Lazy::new(|| params::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: DEMO_STOPPED_INSTANCE_NAME.clone(), + description: String::from(""), + }, + ncpus: InstanceCpuCount(1), + memory: ByteCount::from_gibibytes_u32(16), + hostname: "demo-instance".parse().unwrap(), + user_data: vec![], + ssh_public_keys: Some(Vec::new()), + network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, + external_ips: vec![params::ExternalIpCreate::Ephemeral { + pool: Some(DEMO_IP_POOL_NAME.clone().into()), + }], + disks: vec![], + boot_disk: None, + start: true, + auto_restart_policy: Default::default(), + }); pub static DEMO_INSTANCE_UPDATE: Lazy = Lazy::new(|| params::InstanceUpdate { boot_disk: None, @@ -1935,6 +2053,108 @@ pub static VERIFY_ENDPOINTS: Lazy> = Lazy::new(|| { ] }, + /* Affinity Groups */ + + VerifyEndpoint { + url: &DEMO_PROJECT_URL_AFFINITY_GROUPS, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Post( + serde_json::to_value(&*DEMO_AFFINITY_GROUP_CREATE).unwrap() + ), + AllowedMethod::Get, + ], + }, + + VerifyEndpoint { + url: &DEMO_AFFINITY_GROUP_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Delete, + AllowedMethod::Put( + serde_json::to_value(&*DEMO_AFFINITY_GROUP_UPDATE).unwrap() + ), + ], + }, + + VerifyEndpoint { + url: &DEMO_AFFINITY_GROUP_MEMBERS_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + ], + }, + + VerifyEndpoint { + url: &DEMO_AFFINITY_GROUP_INSTANCE_MEMBER_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Delete, + AllowedMethod::Post(serde_json::Value::Null), + ], + }, + + /* Anti-Affinity Groups */ + + VerifyEndpoint { + url: &DEMO_ANTI_AFFINITY_GROUPS_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Post( + serde_json::to_value(&*DEMO_ANTI_AFFINITY_GROUP_CREATE).unwrap() + ), + AllowedMethod::Get, + ], + }, + + VerifyEndpoint { + url: &DEMO_ANTI_AFFINITY_GROUP_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Delete, + AllowedMethod::Put( + serde_json::to_value(&*DEMO_ANTI_AFFINITY_GROUP_UPDATE).unwrap() + ), + ], + }, + + VerifyEndpoint { + url: &DEMO_ANTI_AFFINITY_GROUP_MEMBERS_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + ], + }, + + VerifyEndpoint { + url: &DEMO_ANTI_AFFINITY_GROUP_INSTANCE_MEMBER_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Delete, + AllowedMethod::Post(serde_json::Value::Null) + ], + }, + /* Instances */ VerifyEndpoint { url: &DEMO_PROJECT_URL_INSTANCES, diff --git a/nexus/tests/integration_tests/mod.rs b/nexus/tests/integration_tests/mod.rs index dc404736cd1..6283b51f587 100644 --- a/nexus/tests/integration_tests/mod.rs +++ b/nexus/tests/integration_tests/mod.rs @@ -4,6 +4,7 @@ //! the way it is. mod address_lots; +mod affinity; mod allow_list; mod authn_http; mod authz; diff --git a/nexus/tests/integration_tests/projects.rs b/nexus/tests/integration_tests/projects.rs index d9752b1949f..dcd99de9042 100644 --- a/nexus/tests/integration_tests/projects.rs +++ b/nexus/tests/integration_tests/projects.rs @@ -9,11 +9,17 @@ use http::StatusCode; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::resource_helpers::create_affinity_group; +use nexus_test_utils::resource_helpers::create_anti_affinity_group; +use nexus_test_utils::resource_helpers::create_default_ip_pool; +use nexus_test_utils::resource_helpers::create_disk; use nexus_test_utils::resource_helpers::create_floating_ip; -use nexus_test_utils::resource_helpers::{ - create_default_ip_pool, create_disk, create_project, create_vpc, - object_create, project_get, projects_list, DiskTest, -}; +use nexus_test_utils::resource_helpers::create_project; +use nexus_test_utils::resource_helpers::create_vpc; +use nexus_test_utils::resource_helpers::object_create; +use nexus_test_utils::resource_helpers::project_get; +use nexus_test_utils::resource_helpers::projects_list; +use nexus_test_utils::resource_helpers::DiskTest; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params; use nexus_types::external_api::views; @@ -392,3 +398,67 @@ async fn test_project_deletion_with_vpc(cptestctx: &ControlPlaneTestContext) { .unwrap(); delete_project(&project_url, &client).await; } + +#[nexus_test] +async fn test_project_deletion_with_affinity_group( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + + // Create a project that we'll use for testing. + let name = "springfield-squidport"; + let project_url = format!("/v1/projects/{}", name); + + create_project(&client, &name).await; + delete_project_default_subnet(&name, &client).await; + delete_project_default_vpc(&name, &client).await; + + let group_name = "just-rainsticks"; + create_affinity_group(&client, name, group_name).await; + + assert_eq!( + "project to be deleted contains an affinity group: just-rainsticks", + delete_project_expect_fail(&project_url, &client).await, + ); + + let group_url = + format!("/v1/affinity-groups/{group_name}?project={}", name); + NexusRequest::object_delete(client, &group_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + delete_project(&project_url, &client).await; +} + +#[nexus_test] +async fn test_project_deletion_with_anti_affinity_group( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + + // Create a project that we'll use for testing. + let name = "springfield-squidport"; + let project_url = format!("/v1/projects/{}", name); + + create_project(&client, &name).await; + delete_project_default_subnet(&name, &client).await; + delete_project_default_vpc(&name, &client).await; + + let group_name = "just-rainsticks"; + create_anti_affinity_group(&client, name, group_name).await; + + assert_eq!( + "project to be deleted contains an anti affinity group: just-rainsticks", + delete_project_expect_fail(&project_url, &client).await, + ); + + let group_url = + format!("/v1/anti-affinity-groups/{group_name}?project={}", name); + NexusRequest::object_delete(client, &group_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + delete_project(&project_url, &client).await; +} diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index fa41d17aa96..cbc041f741a 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -17,6 +17,8 @@ use nexus_test_utils::{load_test_config, ControlPlaneTestContextBuilder}; use omicron_common::api::external::SemverVersion; use omicron_common::api::internal::shared::SwitchLocation; use omicron_test_utils::dev::db::{Client, CockroachInstance}; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::SledUuid; use pretty_assertions::{assert_eq, assert_ne}; use similar_asserts; use slog::Logger; @@ -987,7 +989,8 @@ async fn dbinit_equals_sum_of_all_up() { diesel::insert_into(dsl::sled_resource) .values(SledResource { id: Uuid::new_v4(), - sled_id: Uuid::new_v4(), + instance_id: Some(InstanceUuid::new_v4().into()), + sled_id: SledUuid::new_v4().into(), kind: SledResourceKind::Instance, resources: Resources { hardware_threads: 8_u32.into(), diff --git a/nexus/tests/integration_tests/unauthorized.rs b/nexus/tests/integration_tests/unauthorized.rs index 45f87c96ce0..26c0711f90f 100644 --- a/nexus/tests/integration_tests/unauthorized.rs +++ b/nexus/tests/integration_tests/unauthorized.rs @@ -292,6 +292,37 @@ static SETUP_REQUESTS: Lazy> = Lazy::new(|| { body: serde_json::to_value(&*DEMO_INSTANCE_CREATE).unwrap(), id_routes: vec!["/v1/instances/{id}"], }, + // Create a stopped Instance in the Project + SetupReq::Post { + url: &DEMO_PROJECT_URL_INSTANCES, + body: serde_json::to_value(&*DEMO_STOPPED_INSTANCE_CREATE).unwrap(), + id_routes: vec!["/v1/instances/{id}"], + }, + // Create an affinity group in the Project + SetupReq::Post { + url: &DEMO_PROJECT_URL_AFFINITY_GROUPS, + body: serde_json::to_value(&*DEMO_AFFINITY_GROUP_CREATE).unwrap(), + id_routes: vec!["/v1/affinity-groups/{id}"], + }, + // Add a member to the affinity group + SetupReq::Post { + url: &DEMO_AFFINITY_GROUP_INSTANCE_MEMBER_URL, + body: serde_json::Value::Null, + id_routes: vec![], + }, + // Create an anti-affinity group in the Project + SetupReq::Post { + url: &DEMO_PROJECT_URL_ANTI_AFFINITY_GROUPS, + body: serde_json::to_value(&*DEMO_ANTI_AFFINITY_GROUP_CREATE) + .unwrap(), + id_routes: vec!["/v1/anti-affinity-groups/{id}"], + }, + // Add a member to the anti-affinity group + SetupReq::Post { + url: &DEMO_ANTI_AFFINITY_GROUP_INSTANCE_MEMBER_URL, + body: serde_json::Value::Null, + id_routes: vec![], + }, // Lookup the previously created NIC SetupReq::Get { url: &DEMO_INSTANCE_NIC_URL, diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 8294e23e2f2..e87acd4e590 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -10,11 +10,11 @@ use base64::Engine; use chrono::{DateTime, Utc}; use http::Uri; use omicron_common::api::external::{ - AddressLotKind, AllowedSourceIps, BfdMode, BgpPeer, ByteCount, Hostname, - IdentityMetadataCreateParams, IdentityMetadataUpdateParams, - InstanceAutoRestartPolicy, InstanceCpuCount, LinkFec, LinkSpeed, Name, - NameOrId, PaginationOrder, RouteDestination, RouteTarget, SemverVersion, - TxEqConfig, UserId, + AddressLotKind, AffinityPolicy, AllowedSourceIps, BfdMode, BgpPeer, + ByteCount, FailureDomain, Hostname, IdentityMetadataCreateParams, + IdentityMetadataUpdateParams, InstanceAutoRestartPolicy, InstanceCpuCount, + LinkFec, LinkSpeed, Name, NameOrId, PaginationOrder, RouteDestination, + RouteTarget, SemverVersion, TxEqConfig, UserId, }; use omicron_common::disk::DiskVariant; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; @@ -69,6 +69,8 @@ pub struct UninitializedSledId { pub part: String, } +path_param!(AffinityGroupPath, affinity_group, "affinity group"); +path_param!(AntiAffinityGroupPath, anti_affinity_group, "anti affinity group"); path_param!(ProjectPath, project, "project"); path_param!(InstancePath, instance, "instance"); path_param!(NetworkInterfacePath, interface, "network interface"); @@ -806,6 +808,70 @@ where Ok(v) } +// AFFINITY GROUPS + +/// Create-time parameters for an `AffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AffinityGroupCreate { + #[serde(flatten)] + pub identity: IdentityMetadataCreateParams, + + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +/// Updateable properties of an `AffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AffinityGroupUpdate { + #[serde(flatten)] + pub identity: IdentityMetadataUpdateParams, +} + +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct AffinityInstanceGroupMemberPath { + pub affinity_group: NameOrId, + pub instance: NameOrId, +} + +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct AntiAffinityInstanceGroupMemberPath { + pub anti_affinity_group: NameOrId, + pub instance: NameOrId, +} + +/// Create-time parameters for an `AntiAffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AntiAffinityGroupCreate { + #[serde(flatten)] + pub identity: IdentityMetadataCreateParams, + + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +/// Updateable properties of an `AntiAffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AntiAffinityGroupUpdate { + #[serde(flatten)] + pub identity: IdentityMetadataUpdateParams, +} + +#[derive(Deserialize, JsonSchema, Clone)] +pub struct AffinityGroupSelector { + /// Name or ID of the project, only required if `affinity_group` is provided as a `Name` + pub project: Option, + /// Name or ID of the Affinity Group + pub affinity_group: NameOrId, +} + +#[derive(Deserialize, JsonSchema, Clone)] +pub struct AntiAffinityGroupSelector { + /// Name or ID of the project, only required if `anti_affinity_group` is provided as a `Name` + pub project: Option, + /// Name or ID of the Anti Affinity Group + pub anti_affinity_group: NameOrId, +} + // PROJECTS /// Create-time parameters for a `Project` diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 3430d06f724..fe93e4cae60 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -13,9 +13,9 @@ use chrono::DateTime; use chrono::Utc; use diffus::Diffus; use omicron_common::api::external::{ - AllowedSourceIps as ExternalAllowedSourceIps, ByteCount, Digest, Error, - IdentityMetadata, InstanceState, Name, ObjectIdentity, RoleName, - SimpleIdentityOrName, + AffinityPolicy, AllowedSourceIps as ExternalAllowedSourceIps, ByteCount, + Digest, Error, FailureDomain, IdentityMetadata, InstanceState, Name, + ObjectIdentity, RoleName, SimpleIdentityOrName, }; use oxnet::{Ipv4Net, Ipv6Net}; use schemars::JsonSchema; @@ -112,6 +112,24 @@ impl SimpleIdentityOrName for SiloUtilization { } } +// AFFINITY GROUPS + +#[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AffinityGroup { + #[serde(flatten)] + pub identity: IdentityMetadata, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +#[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AntiAffinityGroup { + #[serde(flatten)] + pub identity: IdentityMetadata, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + // IDENTITY PROVIDER #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] diff --git a/openapi/nexus.json b/openapi/nexus.json index 45f2d4787cf..2e4ef3ac597 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -683,6 +683,958 @@ } } }, + "/v1/affinity-groups": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List affinity groups", + "operationId": "affinity_group_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Create an affinity group", + "operationId": "affinity_group_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/affinity-groups/{affinity_group}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an affinity group", + "operationId": "affinity_group_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "affinity" + ], + "summary": "Update an affinity group", + "operationId": "affinity_group_update", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Delete an affinity group", + "operationId": "affinity_group_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/affinity-groups/{affinity_group}/members": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List members of an affinity group", + "operationId": "affinity_group_member_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupMemberResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/affinity-groups/{affinity_group}/members/instance/{instance}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an affinity group member", + "operationId": "affinity_group_member_instance_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Add a member to an affinity group", + "operationId": "affinity_group_member_instance_add", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Remove a member from an affinity group", + "operationId": "affinity_group_member_instance_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/anti-affinity-groups": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List anti-affinity groups", + "operationId": "anti_affinity_group_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Create an anti-affinity group", + "operationId": "anti_affinity_group_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/anti-affinity-groups/{anti_affinity_group}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an anti-affinity group", + "operationId": "anti_affinity_group_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "affinity" + ], + "summary": "Update an anti-affinity group", + "operationId": "anti_affinity_group_update", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Delete an anti-affinity group", + "operationId": "anti_affinity_group_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/anti-affinity-groups/{anti_affinity_group}/members": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List members of an anti-affinity group", + "operationId": "anti_affinity_group_member_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMemberResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an anti-affinity group member", + "operationId": "anti_affinity_group_member_instance_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Add a member to an anti-affinity group", + "operationId": "anti_affinity_group_member_instance_add", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Remove a member from an anti-affinity group", + "operationId": "anti_affinity_group_member_instance_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/certificates": { "get": { "tags": [ @@ -11055,13 +12007,227 @@ "type": "string", "format": "ip" } - }, - "required": [ - "first_address", - "last_address" + }, + "required": [ + "first_address", + "last_address" + ] + }, + "AddressLotBlockResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLotBlock" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AddressLotCreate": { + "description": "Parameters for creating an address lot.", + "type": "object", + "properties": { + "blocks": { + "description": "The blocks to add along with the new address lot.", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLotBlockCreate" + } + }, + "description": { + "type": "string" + }, + "kind": { + "description": "The kind of address lot to create.", + "allOf": [ + { + "$ref": "#/components/schemas/AddressLotKind" + } + ] + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "blocks", + "description", + "kind", + "name" + ] + }, + "AddressLotCreateResponse": { + "description": "An address lot and associated blocks resulting from creating an address lot.", + "type": "object", + "properties": { + "blocks": { + "description": "The address lot blocks that were created.", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLotBlock" + } + }, + "lot": { + "description": "The address lot that was created.", + "allOf": [ + { + "$ref": "#/components/schemas/AddressLot" + } + ] + } + }, + "required": [ + "blocks", + "lot" + ] + }, + "AddressLotKind": { + "description": "The kind associated with an address lot.", + "oneOf": [ + { + "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", + "type": "string", + "enum": [ + "infra" + ] + }, + { + "description": "Pool address lots are used by IP pools.", + "type": "string", + "enum": [ + "pool" + ] + } + ] + }, + "AddressLotResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLot" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AffinityGroup": { + "description": "Identity-related metadata that's included in nearly all public API objects", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "description", + "failure_domain", + "id", + "name", + "policy", + "time_created", + "time_modified" + ] + }, + "AffinityGroupCreate": { + "description": "Create-time parameters for an `AffinityGroup`", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "name": { + "$ref": "#/components/schemas/Name" + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + } + }, + "required": [ + "description", + "failure_domain", + "name", + "policy" + ] + }, + "AffinityGroupMember": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "instance" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + } ] }, - "AddressLotBlockResultsPage": { + "AffinityGroupMemberResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -11069,7 +12235,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/AddressLotBlock" + "$ref": "#/components/schemas/AffinityGroupMember" } }, "next_page": { @@ -11082,104 +12248,63 @@ "items" ] }, - "AddressLotCreate": { - "description": "Parameters for creating an address lot.", + "AffinityGroupResultsPage": { + "description": "A single page of results", "type": "object", "properties": { - "blocks": { - "description": "The blocks to add along with the new address lot.", + "items": { + "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/AddressLotBlockCreate" + "$ref": "#/components/schemas/AffinityGroup" } }, - "description": { + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", "type": "string" - }, - "kind": { - "description": "The kind of address lot to create.", - "allOf": [ - { - "$ref": "#/components/schemas/AddressLotKind" - } - ] - }, - "name": { - "$ref": "#/components/schemas/Name" } }, "required": [ - "blocks", - "description", - "kind", - "name" + "items" ] }, - "AddressLotCreateResponse": { - "description": "An address lot and associated blocks resulting from creating an address lot.", + "AffinityGroupUpdate": { + "description": "Updateable properties of an `AffinityGroup`", "type": "object", "properties": { - "blocks": { - "description": "The address lot blocks that were created.", - "type": "array", - "items": { - "$ref": "#/components/schemas/AddressLotBlock" - } + "description": { + "nullable": true, + "type": "string" }, - "lot": { - "description": "The address lot that was created.", + "name": { + "nullable": true, "allOf": [ { - "$ref": "#/components/schemas/AddressLot" + "$ref": "#/components/schemas/Name" } ] } - }, - "required": [ - "blocks", - "lot" - ] + } }, - "AddressLotKind": { - "description": "The kind associated with an address lot.", + "AffinityPolicy": { "oneOf": [ { - "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", + "description": "If the affinity request cannot be satisfied, allow it anyway.\n\nThis enables a \"best-effort\" attempt to satisfy the affinity policy.", "type": "string", "enum": [ - "infra" + "allow" ] }, { - "description": "Pool address lots are used by IP pools.", + "description": "If the affinity request cannot be satisfied, fail explicitly.", "type": "string", "enum": [ - "pool" + "fail" ] } ] }, - "AddressLotResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/AddressLot" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" - ] - }, "AggregateBgpMessageHistory": { "description": "BGP message history for rack switches.", "type": "object", @@ -11284,6 +12409,161 @@ } ] }, + "AntiAffinityGroup": { + "description": "Identity-related metadata that's included in nearly all public API objects", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "description", + "failure_domain", + "id", + "name", + "policy", + "time_created", + "time_modified" + ] + }, + "AntiAffinityGroupCreate": { + "description": "Create-time parameters for an `AntiAffinityGroup`", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "name": { + "$ref": "#/components/schemas/Name" + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + } + }, + "required": [ + "description", + "failure_domain", + "name", + "policy" + ] + }, + "AntiAffinityGroupMember": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "instance" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "AntiAffinityGroupMemberResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AntiAffinityGroupResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AntiAffinityGroupUpdate": { + "description": "Updateable properties of an `AntiAffinityGroup`", + "type": "object", + "properties": { + "description": { + "nullable": true, + "type": "string" + }, + "name": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + } + } + }, "AuthzScope": { "description": "Authorization scope for a timeseries.\n\nThis describes the level at which a user must be authorized to read data from a timeseries. For example, fleet-scoping means the data is only visible to an operator or fleet reader. Project-scoped, on the other hand, indicates that a user will see data limited to the projects on which they have read permissions.", "oneOf": [ @@ -14635,6 +15915,18 @@ "items" ] }, + "FailureDomain": { + "description": "Describes the scope of affinity for the purposes of co-location.", + "oneOf": [ + { + "description": "Instances are considered co-located if they are on the same sled", + "type": "string", + "enum": [ + "sled" + ] + } + ] + }, "FieldSchema": { "description": "The name and type information for a field of a timeseries schema.", "type": "object", @@ -23264,6 +24556,13 @@ } }, "tags": [ + { + "name": "affinity", + "description": "Affinity groups give control over instance placement.", + "externalDocs": { + "url": "http://docs.oxide.computer/api/affinity" + } + }, { "name": "disks", "description": "Virtual disks are used to store instance-local data which includes the operating system.", diff --git a/schema/crdb/affinity/up01.sql b/schema/crdb/affinity/up01.sql new file mode 100644 index 00000000000..06e5fb06f94 --- /dev/null +++ b/schema/crdb/affinity/up01.sql @@ -0,0 +1,8 @@ +CREATE TYPE IF NOT EXISTS omicron.public.affinity_policy AS ENUM ( + -- If the affinity request cannot be satisfied, fail. + 'fail', + + -- If the affinity request cannot be satisfied, allow it anyway. + 'allow' +); + diff --git a/schema/crdb/affinity/up02.sql b/schema/crdb/affinity/up02.sql new file mode 100644 index 00000000000..9165e6c4006 --- /dev/null +++ b/schema/crdb/affinity/up02.sql @@ -0,0 +1,5 @@ +CREATE TYPE IF NOT EXISTS omicron.public.failure_domain AS ENUM ( + -- Instances are co-located if they are on the same sled. + 'sled' +); + diff --git a/schema/crdb/affinity/up03.sql b/schema/crdb/affinity/up03.sql new file mode 100644 index 00000000000..79238402f8d --- /dev/null +++ b/schema/crdb/affinity/up03.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + diff --git a/schema/crdb/affinity/up04.sql b/schema/crdb/affinity/up04.sql new file mode 100644 index 00000000000..3f455884b98 --- /dev/null +++ b/schema/crdb/affinity/up04.sql @@ -0,0 +1,7 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_affinity_group_by_project ON omicron.public.affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + + diff --git a/schema/crdb/affinity/up05.sql b/schema/crdb/affinity/up05.sql new file mode 100644 index 00000000000..87f5f9e9c11 --- /dev/null +++ b/schema/crdb/affinity/up05.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + diff --git a/schema/crdb/affinity/up06.sql b/schema/crdb/affinity/up06.sql new file mode 100644 index 00000000000..f86037bda3f --- /dev/null +++ b/schema/crdb/affinity/up06.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_affinity_group_instance_membership_by_instance ON omicron.public.affinity_group_instance_membership ( + instance_id +); + diff --git a/schema/crdb/affinity/up07.sql b/schema/crdb/affinity/up07.sql new file mode 100644 index 00000000000..0409c1c4c7b --- /dev/null +++ b/schema/crdb/affinity/up07.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Anti-Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + diff --git a/schema/crdb/affinity/up08.sql b/schema/crdb/affinity/up08.sql new file mode 100644 index 00000000000..536fcf35e37 --- /dev/null +++ b/schema/crdb/affinity/up08.sql @@ -0,0 +1,6 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_anti_affinity_group_by_project ON omicron.public.anti_affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + diff --git a/schema/crdb/affinity/up09.sql b/schema/crdb/affinity/up09.sql new file mode 100644 index 00000000000..9b3333c934e --- /dev/null +++ b/schema/crdb/affinity/up09.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + diff --git a/schema/crdb/affinity/up10.sql b/schema/crdb/affinity/up10.sql new file mode 100644 index 00000000000..c57d98310c5 --- /dev/null +++ b/schema/crdb/affinity/up10.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_instance_membership_by_instance ON omicron.public.anti_affinity_group_instance_membership ( + instance_id +); + diff --git a/schema/crdb/affinity/up11.sql b/schema/crdb/affinity/up11.sql new file mode 100644 index 00000000000..e3b328c326d --- /dev/null +++ b/schema/crdb/affinity/up11.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.sled_resource ADD COLUMN IF NOT EXISTS instance_id UUID; diff --git a/schema/crdb/affinity/up12.sql b/schema/crdb/affinity/up12.sql new file mode 100644 index 00000000000..1b76693bf29 --- /dev/null +++ b/schema/crdb/affinity/up12.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_resource_by_instance ON omicron.public.sled_resource ( + instance_id +); + diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 1ee7b985aff..a2fa3d48e23 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -217,7 +217,7 @@ CREATE INDEX IF NOT EXISTS lookup_sled_by_policy_and_state ON omicron.public.sle ); CREATE TYPE IF NOT EXISTS omicron.public.sled_resource_kind AS ENUM ( - -- omicron.public.instance + -- omicron.public.vmm ; this is called "instance" for historical reasons. 'instance' -- We expect to other resource kinds here in the future; e.g., to track -- resources used by control plane services. For now, we only track @@ -242,7 +242,13 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled_resource ( reservoir_ram INT8 NOT NULL, -- Identifies the type of the resource - kind omicron.public.sled_resource_kind NOT NULL + kind omicron.public.sled_resource_kind NOT NULL, + + -- The UUID of an instance, if this resource belongs to an instance. + instance_id UUID + + -- TODO Add constraint that if kind is instance, instance_id is not NULL? + -- Or will that break backwards compatibility? ); -- Allow looking up all resources which reside on a sled @@ -251,6 +257,10 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_resource_by_sled ON omicron.public.sled id ); +-- Allow looking up all resources by instance +CREATE INDEX IF NOT EXISTS lookup_resource_by_instance ON omicron.public.sled_resource ( + instance_id +); -- Table of all sled subnets allocated for sleds added to an already initialized -- rack. The sleds in this table and their allocated subnets are created before @@ -4094,6 +4104,94 @@ CREATE INDEX IF NOT EXISTS lookup_usable_rendezvous_debug_dataset COMMIT; BEGIN; +-- Describes what happens when +-- (for affinity groups) instance cannot be co-located, or +-- (for anti-affinity groups) instance must be co-located, or +CREATE TYPE IF NOT EXISTS omicron.public.affinity_policy AS ENUM ( + -- If the affinity request cannot be satisfied, fail. + 'fail', + + -- If the affinity request cannot be satisfied, allow it anyway. + 'allow' +); + +-- Determines what "co-location" means for instances within an affinity +-- or anti-affinity group. +CREATE TYPE IF NOT EXISTS omicron.public.failure_domain AS ENUM ( + -- Instances are co-located if they are on the same sled. + 'sled' +); + +-- Describes a grouping of related instances that should be co-located. +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + +-- Names for affinity groups within a project should be unique +CREATE UNIQUE INDEX IF NOT EXISTS lookup_affinity_group_by_project ON omicron.public.affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + +-- Describes an instance's membership within an affinity group +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + +-- We need to look up all memberships of an instance so we can revoke these +-- memberships efficiently when instances are deleted. +CREATE INDEX IF NOT EXISTS lookup_affinity_group_instance_membership_by_instance ON omicron.public.affinity_group_instance_membership ( + instance_id +); + +-- Describes a collection of instances that should not be co-located. +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Anti-Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + +-- Names for anti-affinity groups within a project should be unique +CREATE UNIQUE INDEX IF NOT EXISTS lookup_anti_affinity_group_by_project ON omicron.public.anti_affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + +-- Describes an instance's membership within an anti-affinity group +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + +-- We need to look up all memberships of an instance so we can revoke these +-- memberships efficiently when instances are deleted. +CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_instance_membership_by_instance ON omicron.public.anti_affinity_group_instance_membership ( + instance_id +); + -- Per-VMM state. CREATE TABLE IF NOT EXISTS omicron.public.vmm ( id UUID PRIMARY KEY, @@ -4810,7 +4908,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '122.0.0', NULL) + (TRUE, NOW(), NOW(), '123.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index 1d6dc600226..a27fc43b8e0 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -51,6 +51,8 @@ macro_rules! impl_typed_uuid_kind { // Please keep this list in alphabetical order. impl_typed_uuid_kind! { + AffinityGroup => "affinity_group", + AntiAffinityGroup => "anti_affinity_group", Blueprint => "blueprint", Collection => "collection", Dataset => "dataset",