Expose SNAT IPs in the public API (#8685)

Fixes #8163

Author: bnaecker

common/src/api/external/mod.rs

@@ -2058,7 +2058,6 @@ impl FromStr for L4PortRange {
                     .parse::<NonZeroU16>()
                     .map_err(|e| L4PortRangeError::Value(right.into(), e))?
                     .into();
-
                 Ok(L4PortRange { first, last })
             }
         }

end-to-end-tests/src/instance_launch.rs

@@ -91,7 +91,8 @@ async fn instance_launch() -> Result<()> {
         .send()
         .await?
         .items
-        .first()
+        .iter()
+        .find(|eip| matches!(eip, ExternalIp::Ephemeral { .. }))
         .context("no external IPs")?
         .clone();
 

nexus/db-model/src/external_ip.rs

@@ -533,9 +533,12 @@ impl TryFrom<ExternalIp> for views::ExternalIp {
                 ip: ip.ip.ip(),
                 ip_pool_id: ip.ip_pool_id,
             }),
-            IpKind::SNat => Err(Error::internal_error(
-                "SNAT IP addresses should not be exposed in the API",
-            )),
+            IpKind::SNat => Ok(views::ExternalIp::SNat(views::SNatIp {
+                ip: ip.ip.ip(),
+                first_port: u16::from(ip.first_port),
+                last_port: u16::from(ip.last_port),
+                ip_pool_id: ip.ip_pool_id,
+            })),
         }
     }
 }

nexus/src/app/external_ip.rs

@@ -13,7 +13,6 @@ use nexus_db_lookup::lookup;
 use nexus_db_model::IpAttachState;
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
-use nexus_db_queries::db::model::IpKind;
 use nexus_types::external_api::params;
 use nexus_types::external_api::views;
 use omicron_common::api::external::CreateResult;
@@ -44,9 +43,7 @@ impl super::Nexus {
             .await?
             .into_iter()
             .filter_map(|ip| {
-                if ip.kind == IpKind::SNat
-                    || ip.state != IpAttachState::Attached
-                {
+                if ip.state != IpAttachState::Attached {
                     None
                 } else {
                     Some(ip.try_into().unwrap())

nexus/tests/integration_tests/external_ips.rs

@@ -44,6 +44,7 @@ use nexus_types::external_api::views::FloatingIp;
 use nexus_types::identity::Resource;
 use omicron_common::address::IpRange;
 use omicron_common::address::Ipv4Range;
+use omicron_common::address::NUM_SOURCE_NAT_PORTS;
 use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::IdentityMetadataCreateParams;
 use omicron_common::api::external::IdentityMetadataUpdateParams;
@@ -54,6 +55,8 @@ use omicron_common::api::external::Name;
 use omicron_common::api::external::NameOrId;
 use omicron_uuid_kinds::GenericUuid;
 use omicron_uuid_kinds::InstanceUuid;
+use oxide_client::types::ExternalIpResultsPage;
+use oxide_client::types::IpPoolRangeResultsPage;
 use uuid::Uuid;
 
 type ControlPlaneTestContext =
@@ -778,18 +781,19 @@ async fn test_external_ip_live_attach_detach(
             instance_simulate(nexus, &instance_id).await;
         }
 
-        // Verify that each instance has no external IPs.
+        // Verify that each instance has only an SNAT external IP.
+        let eips = fetch_instance_external_ips(
+            client,
+            INSTANCE_NAMES[i],
+            PROJECT_NAME,
+        )
+        .await;
+        assert_eq!(eips.len(), 1, "Expected exactly 1 SNAT external IP");
         assert_eq!(
-            fetch_instance_external_ips(
-                client,
-                INSTANCE_NAMES[i],
-                PROJECT_NAME
-            )
-            .await
-            .len(),
-            0
+            eips[0].kind(),
+            shared::IpKind::SNat,
+            "Expected exactly 1 SNAT external IP"
         );
-
         instances.push(instance);
     }
 
@@ -816,7 +820,7 @@ async fn test_external_ip_live_attach_detach(
             fetch_instance_external_ips(client, instance_name, PROJECT_NAME)
                 .await;
 
-        assert_eq!(eip_list.len(), 2);
+        assert_eq!(eip_list.len(), 3);
         assert!(eip_list.contains(&eph_resp));
         assert!(
             eip_list
@@ -857,7 +861,8 @@ async fn test_external_ip_live_attach_detach(
             fetch_instance_external_ips(client, instance_name, PROJECT_NAME)
                 .await;
 
-        assert_eq!(eip_list.len(), 0);
+        // We still have 1 SNAT IP
+        assert_eq!(eip_list.len(), 1);
         assert_eq!(fip.ip, fip_resp.ip);
 
         // Check for idempotency: repeat requests should return same values for FIP,
@@ -907,8 +912,21 @@ async fn test_external_ip_live_attach_detach(
     let instance_name = instances[0].identity.name.as_str();
     let eip_list =
         fetch_instance_external_ips(client, instance_name, PROJECT_NAME).await;
-    assert_eq!(eip_list.len(), 1);
-    assert_eq!(eip_list[0].ip(), fips[0].ip);
+    assert_eq!(eip_list.len(), 2, "Expected a Floating and SNAT IP");
+    assert_eq!(
+        eip_list
+            .iter()
+            .find_map(|eip| {
+                if eip.kind() == shared::IpKind::Floating {
+                    Some(eip.ip())
+                } else {
+                    None
+                }
+            })
+            .expect("Should have a Floating IP"),
+        fips[0].ip,
+        "Floating IP object's IP address is not correct",
+    );
 
     // now the other way: floating IP by ID and instance by name
     let floating_ip_id = fips[1].identity.id;
@@ -934,8 +952,21 @@ async fn test_external_ip_live_attach_detach(
 
     let eip_list =
         fetch_instance_external_ips(client, instance_name, PROJECT_NAME).await;
-    assert_eq!(eip_list.len(), 1);
-    assert_eq!(eip_list[0].ip(), fips[1].ip);
+    assert_eq!(eip_list.len(), 2, "Expect a Floating and SNAT IP");
+    assert_eq!(
+        eip_list
+            .iter()
+            .find_map(|eip| {
+                if eip.kind() == shared::IpKind::Floating {
+                    Some(eip.ip())
+                } else {
+                    None
+                }
+            })
+            .expect("Should have a Floating IP"),
+        fips[1].ip,
+        "Floating IP object's IP address is not correct",
+    );
 
     // none of that changed the number of allocated IPs
     assert_ip_pool_utilization(client, "default", 3, 65536, 0, 0).await;
@@ -1223,6 +1254,74 @@ async fn test_external_ip_attach_ephemeral_at_pool_exhaustion(
     assert_eq!(eph_resp_2, eph_resp);
 }
 
+#[nexus_test]
+async fn can_list_instance_snat_ip(cptestctx: &ControlPlaneTestContext) {
+    let client = &cptestctx.external_client;
+
+    let pool = create_default_ip_pool(&client).await;
+    let _project = create_project(client, PROJECT_NAME).await;
+
+    // Get the first address in the pool.
+    let range = NexusRequest::object_get(
+        client,
+        &format!("/v1/system/ip-pools/{}/ranges", pool.identity.id),
+    )
+    .authn_as(AuthnMode::PrivilegedUser)
+    .execute()
+    .await
+    .unwrap_or_else(|e| panic!("failed to get IP pool range: {e}"))
+    .parsed_body::<IpPoolRangeResultsPage>()
+    .unwrap_or_else(|e| panic!("failed to parse IP pool range: {e}"));
+    assert_eq!(range.items.len(), 1, "Should have 1 range in the pool");
+    let oxide_client::types::IpRange::V4(oxide_client::types::Ipv4Range {
+        first,
+        ..
+    }) = &range.items[0].range
+    else {
+        panic!("Expected IPv4 range, found {:?}", &range.items[0]);
+    };
+    let expected_ip = IpAddr::V4(*first);
+
+    // Create a running instance with only an SNAT IP address.
+    let instance_name = INSTANCE_NAMES[0];
+    let instance =
+        instance_for_external_ips(client, instance_name, true, false, &[])
+            .await;
+    let url = format!("/v1/instances/{}/external-ips", instance.identity.id);
+    let page = NexusRequest::object_get(client, &url)
+        .authn_as(AuthnMode::PrivilegedUser)
+        .execute()
+        .await
+        .unwrap_or_else(|e| {
+            panic!("failed to make \"get\" request to {url}: {e}")
+        })
+        .parsed_body::<ExternalIpResultsPage>()
+        .unwrap_or_else(|e| {
+            panic!("failed to make \"get\" request to {url}: {e}")
+        });
+    let ips = page.items;
+    assert_eq!(
+        ips.len(),
+        1,
+        "Instance should have been created with exactly 1 IP"
+    );
+    let oxide_client::types::ExternalIp::Snat {
+        ip,
+        ip_pool_id,
+        first_port,
+        last_port,
+    } = &ips[0]
+    else {
+        panic!("Expected an SNAT external IP, found {:?}", &ips[0]);
+    };
+    assert_eq!(ip_pool_id, &pool.identity.id);
+    assert_eq!(ip, &expected_ip);
+
+    // Port ranges are half-open on the right, e.g., [0, 16384).
+    assert_eq!(*first_port, 0);
+    assert_eq!(*last_port, NUM_SOURCE_NAT_PORTS - 1);
+}
+
 pub async fn floating_ip_get(
     client: &ClientTestContext,
     fip_url: &str,

nexus/tests/integration_tests/instances.rs

@@ -6451,29 +6451,31 @@ async fn test_instance_attach_several_external_ips(
     .await;
 
     // 1 ephemeral + 7 floating + 1 SNAT
-    assert_ip_pool_utilization(client, "default", 9, 10, 0, 0).await;
+    const N_EXPECTED_IPS: u32 = 9;
+    assert_ip_pool_utilization(client, "default", N_EXPECTED_IPS, 10, 0, 0)
+        .await;
 
     // Verify that all external IPs are visible on the instance and have
     // been allocated in order.
     let external_ips =
         fetch_instance_external_ips(&client, instance_name, PROJECT_NAME).await;
-    assert_eq!(external_ips.len(), 8);
-    eprintln!("{external_ips:?}");
+    eprintln!("{external_ips:#?}");
+    assert_eq!(external_ips.len(), N_EXPECTED_IPS as usize);
+
+    // We've created all the FIPs first, before the instance. The instance gets
+    // an SNAT IP automatically, and then _also_ an Ephemeral IP.
     for (i, eip) in external_ips
         .iter()
         .sorted_unstable_by(|a, b| a.ip().cmp(&b.ip()))
         .enumerate()
     {
-        let last_octet = i + if i != external_ips.len() - 1 {
-            assert_eq!(eip.kind(), IpKind::Floating);
-            1
-        } else {
-            // SNAT will occupy 1.0.0.8 here, since it it alloc'd before
-            // the ephemeral.
-            assert_eq!(eip.kind(), IpKind::Ephemeral);
-            2
+        match i {
+            0..7 => assert_eq!(eip.kind(), IpKind::Floating),
+            7 => assert_eq!(eip.kind(), IpKind::SNat),
+            8 => assert_eq!(eip.kind(), IpKind::Ephemeral),
+            _ => unreachable!(),
         };
-        assert_eq!(eip.ip(), Ipv4Addr::new(10, 0, 0, last_octet as u8));
+        assert_eq!(eip.ip(), Ipv4Addr::new(10, 0, 0, (i + 1) as u8));
     }
 
     // Verify that all floating IPs are bound to their parent instance.

nexus/types/src/external_api/shared.rs

@@ -237,6 +237,7 @@ pub enum ServiceUsingCertificate {
 )]
 #[serde(rename_all = "snake_case")]
 pub enum IpKind {
+    SNat,
     Ephemeral,
     Floating,
 }

nexus/types/src/external_api/views.rs

@@ -502,26 +502,48 @@ pub struct IpPoolRange {
 #[derive(Debug, Clone, Deserialize, PartialEq, Serialize, JsonSchema)]
 #[serde(tag = "kind", rename_all = "snake_case")]
 pub enum ExternalIp {
-    Ephemeral { ip: IpAddr, ip_pool_id: Uuid },
+    #[serde(rename = "snat")]
+    SNat(SNatIp),
+    Ephemeral {
+        ip: IpAddr,
+        ip_pool_id: Uuid,
+    },
     Floating(FloatingIp),
 }
 
 impl ExternalIp {
     pub fn ip(&self) -> IpAddr {
         match self {
+            Self::SNat(snat) => snat.ip,
             Self::Ephemeral { ip, .. } => *ip,
             Self::Floating(float) => float.ip,
         }
     }
 
     pub fn kind(&self) -> IpKind {
         match self {
+            Self::SNat(_) => IpKind::SNat,
             Self::Ephemeral { .. } => IpKind::Ephemeral,
             Self::Floating(_) => IpKind::Floating,
         }
     }
 }
 
+/// A source NAT IP address.
+///
+/// SNAT addresses are ephemeral addresses used only for outbound connectivity.
+#[derive(Debug, Clone, Deserialize, PartialEq, Serialize, JsonSchema)]
+pub struct SNatIp {
+    /// The IP address.
+    pub ip: IpAddr,
+    /// The first usable port within the IP address.
+    pub first_port: u16,
+    /// The last usable port within the IP address.
+    pub last_port: u16,
+    /// ID of the IP Pool from which the address is taken.
+    pub ip_pool_id: Uuid,
+}
+
 /// A Floating IP is a well-known IP address which can be attached
 /// and detached from instances.
 #[derive(
@@ -553,9 +575,11 @@ impl TryFrom<ExternalIp> for FloatingIp {
 
     fn try_from(value: ExternalIp) -> Result<Self, Self::Error> {
         match value {
-            ExternalIp::Ephemeral { .. } => Err(Error::internal_error(
-                "tried to convert an ephemeral IP into a floating IP",
-            )),
+            ExternalIp::SNat(_) | ExternalIp::Ephemeral { .. } => {
+                Err(Error::internal_error(
+                    "tried to convert an SNAT or ephemeral IP into a floating IP",
+                ))
+            }
             ExternalIp::Floating(v) => Ok(v),
         }
     }