Merge pull request #181 from dfinity/shiling/engine

feat: add support of new type of subnet (cloud engine) in domain canister matcher

Author: shilingwang

Cargo.toml

@@ -10,7 +10,7 @@ default = []
 
 sev-snp = ["ic-bn-lib/sev-snp"]
 acme = ["ic-bn-lib/acme-dns", "ic-bn-lib/acme-alpn"]
-bench = ["dep:ic-http-certification", "dep:rand_regex", "dep:serde_cbor"]
+bench = ["dep:ic-http-certification", "dep:rand_regex"]
 tokio_console = ["dep:console-subscriber"]
 debug = []
 
@@ -76,7 +76,7 @@ rustls = { version = "0.23.18", default-features = false, features = [
     "brotli",
 ] }
 serde = "1.0.214"
-serde_cbor = { version = "0.11.2", optional = true }
+serde_cbor = "0.11.2"
 serde_json = "1.0.132"
 sha2 = "0.10.8"
 strum = { version = "0.27.1", features = ["derive"] }
@@ -155,3 +155,7 @@ path = "tools/create_acme_account.rs"
 [[bin]]
 name = "cloudflare-check"
 path = "tools/cloudflare_check.rs"
+
+[[bin]]
+name = "gen-testdata"
+path = "tools/gen_testdata.rs"

src/cli.rs

@@ -239,6 +239,14 @@ pub struct Domain {
     #[clap(env, long, requires = "domain_system", value_delimiter = ',')]
     pub domain_app: Vec<FQDN>,
 
+    /// List of domains that serve cloud engines only
+    #[clap(env, long, requires = "domain_app", value_delimiter = ',')]
+    pub domain_engine: Vec<FQDN>,
+
+    /// How frequently to poll the NNS for subnet routing table and type information
+    #[clap(env, long, default_value = "5m", value_parser = parse_duration)]
+    pub subnets_info_poll_interval: Duration,
+
     /// List of canister aliases in format '<alias>:<canister_id>'
     #[clap(env, long, value_delimiter = ',')]
     pub domain_canister_alias: Vec<CanisterAlias>,

src/core.rs

@@ -14,6 +14,7 @@ use ic_bn_lib::{
     vector::client::Vector,
 };
 use ic_bn_lib_common::{
+    principal,
     traits::{custom_domains::ProvidesCustomDomains, tls::ProvidesCertificates},
     types::{
         dns::Options as DnsOptions,
@@ -29,10 +30,13 @@ use tracing_subscriber::{EnvFilter, reload::Handle};
 use crate::{
     cli::Cli,
     metrics,
+    routing::ic::subnets_info::SubnetsInfoFetcher,
     routing::{
         self,
         ic::{
             create_agent,
+            http_service::AgentHttpService,
+            nodes_fetcher::MAINNET_ROOT_SUBNET_ID,
             route_provider::{RouteProviderWrapper, setup_route_provider},
         },
     },
@@ -238,6 +242,29 @@ pub async fn main(
         );
     }
 
+    // Subnet info: periodically fetch the full NNS routing table and subnet
+    // types.  Required for both system-subnet and engine-subnet routing
+    // decisions in DomainCanisterMatcher.
+    let http_service = Arc::new(AgentHttpService::new(
+        http_client_hyper.clone(),
+        cli.ic.ic_request_retry_interval,
+    ));
+    let agent = create_agent(cli, http_service, route_provider.clone())
+        .await
+        .context("unable to create agent for subnets info fetcher")?;
+
+    let root_subnet_id = principal!(MAINNET_ROOT_SUBNET_ID);
+
+    let fetcher = Arc::new(SubnetsInfoFetcher::new(Arc::new(agent), root_subnet_id));
+    let subnets_info = fetcher.info.clone();
+
+    health_manager.add(fetcher.clone());
+    tasks.add_interval(
+        "subnets_info_fetcher",
+        fetcher,
+        cli.domain.subnets_info_poll_interval,
+    );
+
     // Setup WAF
     let waf_layer = if cli.waf.waf_enable {
         let v = WafLayer::new_from_cli(&cli.waf, Some(http_client.clone()))
@@ -265,6 +292,7 @@ pub async fn main(
         vector.clone(),
         waf_layer,
         custom_domains_router,
+        subnets_info,
     )
     .await
     .context("unable to setup Axum router")?;

src/policy/domain_canister.rs

@@ -1,58 +1,44 @@
+use std::sync::Arc;
+
 use ahash::AHashSet;
+use arc_swap::ArcSwapOption;
 use candid::Principal;
 use fqdn::{FQDN, Fqdn};
 
-// System subnets routing table
-pub const SYSTEM_SUBNETS: [(Principal, Principal); 5] = [
-    (
-        Principal::from_slice(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01]),
-        Principal::from_slice(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x01, 0x01]),
-    ),
-    (
-        Principal::from_slice(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x01]),
-        Principal::from_slice(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x01, 0x01]),
-    ),
-    (
-        Principal::from_slice(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x01, 0x01]),
-        Principal::from_slice(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xff, 0xff, 0x01, 0x01]),
-    ),
-    (
-        Principal::from_slice(&[0x00, 0x00, 0x00, 0x00, 0x01, 0xa0, 0x00, 0x00, 0x01, 0x01]),
-        Principal::from_slice(&[0x00, 0x00, 0x00, 0x00, 0x01, 0xaf, 0xff, 0xff, 0x01, 0x01]),
-    ),
-    (
-        Principal::from_slice(&[0x00, 0x00, 0x00, 0x00, 0x02, 0x10, 0x00, 0x00, 0x01, 0x01]),
-        Principal::from_slice(&[0x00, 0x00, 0x00, 0x00, 0x02, 0x1f, 0xff, 0xff, 0x01, 0x01]),
-    ),
-];
-
-/// Checks if given canister id belongs to a system subnet
-pub fn is_system_subnet(canister_id: Principal) -> bool {
-    SYSTEM_SUBNETS
-        .iter()
-        .any(|x| canister_id >= x.0 && canister_id <= x.1)
-}
+use crate::routing::ic::subnets_info::{SubnetType, SubnetsInfo};
 
 /// Things needed to verify domain-canister match
 #[derive(derive_new::new)]
 pub struct DomainCanisterMatcher {
     pre_isolation_canisters: AHashSet<Principal>,
     domains_app: Vec<FQDN>,
     domains_system: Vec<FQDN>,
+    domains_engine: Vec<FQDN>,
+    subnets_info: Arc<ArcSwapOption<SubnetsInfo>>,
 }
 
 impl DomainCanisterMatcher {
-    /// Check if given canister id and host match from policy perspective
+    /// Check if given canister id and host match from policy perspective.
     pub fn check(&self, canister_id: Principal, host: &Fqdn) -> bool {
-        // These are always allowed
-        if self.pre_isolation_canisters.contains(&canister_id) {
+        let guard = self.subnets_info.load();
+        // Compute subnet type once; `None` when no snapshot has been stored yet.
+        let subnet_type = guard.as_deref().and_then(|si| si.subnet_type(canister_id));
+
+        // Pre-isolation canisters are exempt from domain checks, unless they are
+        // on a CloudEngine subnet, where the normal domain policy still applies.
+        if self.pre_isolation_canisters.contains(&canister_id)
+            && subnet_type != Some(SubnetType::CloudEngine)
+        {
             return true;
         }
 
-        let domains = if is_system_subnet(canister_id) {
-            &self.domains_system
-        } else {
-            &self.domains_app
+        let domains = match subnet_type {
+            Some(SubnetType::System) => &self.domains_system,
+            Some(SubnetType::CloudEngine) => &self.domains_engine,
+            Some(SubnetType::Application)
+            | Some(SubnetType::VerifiedApplication)
+            | Some(SubnetType::Unknown)
+            | None => &self.domains_app,
         };
 
         domains.iter().any(|x| host.is_subdomain_of(x))
@@ -61,43 +47,136 @@ impl DomainCanisterMatcher {
 
 #[cfg(test)]
 mod tests {
+    use ahash::AHashMap;
+    use arc_swap::ArcSwapOption;
     use fqdn::fqdn;
     use ic_bn_lib_common::principal;
 
     use super::*;
+    use crate::routing::ic::subnets_info::SubnetType;
+
+    use crate::test::TEST_ROOT_SUBNET_ID;
+
+    // Principals used as subnet IDs in the test snapshot
+    const SUBNET_SYSTEM: &str = TEST_ROOT_SUBNET_ID;
+    const SUBNET_ENGINE: &str = "nl6hn-ja4yw-wvmpy-3z2jx-ymc34-pisx3-3cp5z-3oj4a-qzzny-jbsv3-4qe";
+
+    // Canisters that fall inside the ranges defined below
+    const CANISTER_SYSTEM: &str = "qoctq-giaaa-aaaaa-aaaea-cai"; // NNS
+    const CANISTER_ENGINE: &str = "s6hwe-laaaa-aaaab-qaeba-cai";
+    const CANISTER_APP: &str = "oydqf-haaaa-aaaao-afpsa-cai";
+    const CANISTER_PIC: &str = "2dcn6-oqaaa-aaaai-abvoq-cai"; // pre-isolation
+
+    fn test_snapshot() -> Arc<ArcSwapOption<SubnetsInfo>> {
+        let subnet_system = principal!(SUBNET_SYSTEM);
+        let subnet_engine = principal!(SUBNET_ENGINE);
+
+        let ranges = vec![
+            (
+                principal!(CANISTER_SYSTEM),
+                principal!(CANISTER_SYSTEM),
+                subnet_system,
+            ),
+            (
+                principal!(CANISTER_ENGINE),
+                principal!(CANISTER_ENGINE),
+                subnet_engine,
+            ),
+        ];
+
+        let mut types = AHashMap::new();
+        types.insert(subnet_system, SubnetType::System);
+        types.insert(subnet_engine, SubnetType::CloudEngine);
+
+        Arc::new(ArcSwapOption::new(Some(Arc::new(SubnetsInfo::new(
+            ranges, types,
+        )))))
+    }
+
+    fn matcher() -> DomainCanisterMatcher {
+        let mut pic = AHashSet::new();
+        pic.insert(principal!(CANISTER_PIC));
+
+        DomainCanisterMatcher::new(
+            pic,
+            vec![fqdn!("icp0.io")],   // app
+            vec![fqdn!("ic0.app")],   // system
+            vec![fqdn!("engine.io")], // engine
+            test_snapshot(),
+        )
+    }
 
     #[test]
-    fn test_is_system_subnet() {
-        assert!(is_system_subnet(principal!("qoctq-giaaa-aaaaa-aaaea-cai"),)); // nns
-        assert!(is_system_subnet(principal!("rdmx6-jaaaa-aaaaa-aaadq-cai"))); // identity
-        assert!(!is_system_subnet(principal!("oydqf-haaaa-aaaao-afpsa-cai"))); // something else
+    fn system_canister_allowed_on_system_domain() {
+        assert!(matcher().check(principal!(CANISTER_SYSTEM), &fqdn!("ic0.app")));
     }
 
     #[test]
-    fn test_domain_canister_match() {
-        let mut pic = AHashSet::new();
-        pic.insert(principal!("2dcn6-oqaaa-aaaai-abvoq-cai"));
+    fn system_canister_rejected_on_app_domain() {
+        assert!(!matcher().check(principal!(CANISTER_SYSTEM), &fqdn!("icp0.io")));
+    }
 
-        let dcm = DomainCanisterMatcher::new(pic, vec![fqdn!("icp0.io")], vec![fqdn!("ic0.app")]);
+    #[test]
+    fn engine_canister_allowed_on_engine_domain() {
+        assert!(matcher().check(principal!(CANISTER_ENGINE), &fqdn!("engine.io")));
+    }
 
-        assert!(dcm.check(
-            principal!("qoctq-giaaa-aaaaa-aaaea-cai"), // nns on system domain
-            &fqdn!("ic0.app"),
-        ));
+    #[test]
+    fn engine_canister_rejected_on_app_domain() {
+        assert!(!matcher().check(principal!(CANISTER_ENGINE), &fqdn!("icp0.io")));
+    }
 
-        assert!(!dcm.check(
-            principal!("s6hwe-laaaa-aaaab-qaeba-cai"), // something else on system domain
-            &fqdn!("ic0.app"),
-        ));
+    #[test]
+    fn app_canister_allowed_on_app_domain() {
+        assert!(matcher().check(principal!(CANISTER_APP), &fqdn!("icp0.io")));
+    }
 
-        assert!(dcm.check(
-            principal!("s6hwe-laaaa-aaaab-qaeba-cai"), // something else on app domain
-            &fqdn!("icp0.io"),
-        ));
+    #[test]
+    fn app_canister_rejected_on_system_domain() {
+        assert!(!matcher().check(principal!(CANISTER_APP), &fqdn!("ic0.app")));
+    }
 
-        assert!(dcm.check(
-            principal!("2dcn6-oqaaa-aaaai-abvoq-cai"), // pre-isolation canister on system domain
-            &fqdn!("ic0.app"),
-        ));
+    #[test]
+    fn pre_isolation_canister_allowed_on_non_engine_subnet_domains() {
+        // CANISTER_PIC is not on a CloudEngine subnet, so it bypasses domain checks
+        assert!(matcher().check(principal!(CANISTER_PIC), &fqdn!("ic0.app")));
+        assert!(matcher().check(principal!(CANISTER_PIC), &fqdn!("icp0.io")));
+        assert!(matcher().check(principal!(CANISTER_PIC), &fqdn!("engine.io")));
+    }
+
+    #[test]
+    fn pre_isolation_canister_on_engine_subnet_subject_to_domain_policy() {
+        // Even if CANISTER_ENGINE is in the pre-isolation set, CloudEngine subnet
+        // canisters must still use the engine domain.
+        let mut pic = AHashSet::new();
+        pic.insert(principal!(CANISTER_ENGINE));
+        // Reuse test_snapshot() — CANISTER_ENGINE already maps to CloudEngine there.
+        let m = DomainCanisterMatcher::new(
+            pic,
+            vec![fqdn!("icp0.io")],
+            vec![fqdn!("ic0.app")],
+            vec![fqdn!("engine.io")],
+            test_snapshot(),
+        );
+        assert!(m.check(principal!(CANISTER_ENGINE), &fqdn!("engine.io")));
+        assert!(!m.check(principal!(CANISTER_ENGINE), &fqdn!("icp0.io")));
+        assert!(!m.check(principal!(CANISTER_ENGINE), &fqdn!("ic0.app")));
+    }
+
+    #[test]
+    fn empty_snapshot_falls_through_to_app_domain() {
+        let empty = Arc::new(ArcSwapOption::<SubnetsInfo>::empty());
+        let mut pic = AHashSet::new();
+        pic.insert(principal!(CANISTER_PIC));
+        let m = DomainCanisterMatcher::new(
+            pic,
+            vec![fqdn!("icp0.io")],
+            vec![fqdn!("ic0.app")],
+            vec![fqdn!("engine.io")],
+            empty,
+        );
+        // With no snapshot, subnet type is unknown → app domain for everything
+        assert!(m.check(principal!(CANISTER_APP), &fqdn!("icp0.io")));
+        assert!(!m.check(principal!(CANISTER_APP), &fqdn!("ic0.app")));
     }
 }

src/routing/ic/mod.rs

@@ -6,6 +6,7 @@ pub mod health_check;
 pub mod http_service;
 pub mod nodes_fetcher;
 pub mod route_provider;
+pub mod subnets_info;
 
 use std::{fs, sync::Arc};