diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index 18e67beacf4d..cfe4e1ba2e29 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "1fefbde32ed3b3779e766927393403bb1c21b915c373908eacc9793962c43fa7", + "checksum": "1d88faead9fe3d4ca18cc2287397ac1457604de13e94aded554c8104ad506e69", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -21686,6 +21686,10 @@ "id": "hex-literal 0.4.1", "target": "hex_literal" }, + { + "id": "hickory-resolver 0.25.2", + "target": "hickory_resolver" + }, { "id": "hkdf 0.12.4", "target": "hkdf" @@ -22736,6 +22740,10 @@ "id": "wat 1.244.0", "target": "wat" }, + { + "id": "webpki-roots 1.0.6", + "target": "webpki_roots" + }, { "id": "which 4.4.0", "target": "which" @@ -34021,7 +34029,7 @@ "target": "tower_service" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" } ], @@ -66327,7 +66335,7 @@ "target": "tokio_util" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" } ], @@ -66377,7 +66385,7 @@ "target": "tokio_util" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" } ], @@ -66497,7 +66505,7 @@ "target": "tokio_util" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" } ], @@ -66547,7 +66555,7 @@ "target": "tokio_util" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" } ] @@ -69500,7 +69508,7 @@ "target": "thiserror" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" }, { @@ -91790,14 +91798,14 @@ ], "license_file": "LICENSE" }, - "webpki-roots 1.0.2": { + "webpki-roots 1.0.6": { "name": "webpki-roots", - "version": "1.0.2", + "version": "1.0.6", "package_url": "https://github.com/rustls/webpki-roots", "repository": { "Http": { - "url": "https://static.crates.io/crates/webpki-roots/1.0.2/download", - "sha256": "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" + "url": "https://static.crates.io/crates/webpki-roots/1.0.6/download", + "sha256": "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" } }, "targets": [ @@ -91830,7 +91838,7 @@ "selects": {} }, "edition": "2021", - "version": "1.0.2" + "version": "1.0.6" }, "license": "CDLA-Permissive-2.0", "license_ids": [ @@ -98642,6 +98650,7 @@ "hashlink 0.8.3", "hex 0.4.3", "hex-literal 0.4.1", + "hickory-resolver 0.25.2", "hkdf 0.12.4", "hmac 0.12.1", "hpke 0.12.0", @@ -98912,6 +98921,7 @@ "wasmtime 42.0.1", "wast 244.0.0", "wat 1.244.0", + "webpki-roots 1.0.6", "which 4.4.0", "wirm 2.1.0", "wsl 0.1.0", diff --git a/Cargo.Bazel.toml.lock b/Cargo.Bazel.toml.lock index 00581f21f3c9..a1094b9f98b8 100644 --- a/Cargo.Bazel.toml.lock +++ b/Cargo.Bazel.toml.lock @@ -3713,6 +3713,7 @@ dependencies = [ "hashlink 0.8.3", "hex", "hex-literal 0.4.1", + "hickory-resolver", "hkdf", "hmac", "hpke", @@ -3983,6 +3984,7 @@ dependencies = [ "wasmtime", "wast", "wat", + "webpki-roots 1.0.6", "which", "wirm", "wsl", @@ -5870,7 +5872,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.0", "tower-service", - "webpki-roots 1.0.2", + "webpki-roots 1.0.6", ] [[package]] @@ -11264,7 +11266,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.2", + "webpki-roots 1.0.6", ] [[package]] @@ -11713,7 +11715,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", - "webpki-roots 1.0.2", + "webpki-roots 1.0.6", "x509-parser 0.16.0", ] @@ -15425,9 +15427,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.2" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" dependencies = [ "rustls-pki-types", ] diff --git a/Cargo.lock b/Cargo.lock index bd2349a5bd4e..78584857b739 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12004,6 +12004,7 @@ dependencies = [ "axum-server", "criterion", "futures", + "hickory-resolver", "http-body-util", "hyper 1.8.1", "hyper-util", @@ -12018,10 +12019,12 @@ dependencies = [ "ic-logger", "ic-metrics", "ic-nns-delegation-manager-test-utils", + "ic-protobuf", "ic-registry-client-fake", "ic-registry-client-helpers", "ic-registry-keys", "ic-registry-proto-data-provider", + "ic-registry-subnet-type", "ic-test-utilities-registry", "ic-test-utilities-types", "ic-types", @@ -12037,6 +12040,7 @@ dependencies = [ "tokio-rustls 0.26.4", "tokio-util", "tower 0.5.3", + "webpki-roots 1.0.6", ] [[package]] @@ -18291,6 +18295,7 @@ dependencies = [ "futures", "ic-agent", "ic-base-types", + "ic-canonical-state", "ic-cdk", "ic-certification 0.9.0", "ic-crypto-tree-hash", diff --git a/Cargo.toml b/Cargo.toml index 738037144e9a..a8348181414f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -666,6 +666,7 @@ getrandom = { version = "0.2", features = ["custom"] } goldenfile = "1.8.0" gpt = "4.1" hex = { version = "0.4.3", features = ["serde"] } +hickory-resolver = "0.25.2" hkdf = "^0.12" http = "1.3.1" http-body = "1.0.1" @@ -918,6 +919,7 @@ wasmparser = "0.244.0" wasmprinter = "0.244.0" wast = "244.0.0" wat = "~1.244.0" +webpki-roots = "1.0.6" which = "6.0.3" wirm = { version = "2.1.0", features = ["parallel"] } wsl = "0.1.0" diff --git a/bazel/rust.MODULE.bazel b/bazel/rust.MODULE.bazel index 9eaa412850e8..724c8862ece0 100644 --- a/bazel/rust.MODULE.bazel +++ b/bazel/rust.MODULE.bazel @@ -537,6 +537,10 @@ crate.spec( package = "hex-literal", version = "^0.4.1", ) +crate.spec( + package = "hickory-resolver", + version = "0.25.2", +) crate.spec( package = "hkdf", version = "^0.12", @@ -1899,6 +1903,10 @@ crate.spec( package = "wat", version = "~1.244.0", ) +crate.spec( + package = "webpki-roots", + version = "1.0.6", +) crate.spec( package = "which", version = "^4.2.2", diff --git a/rs/canonical_state/src/lazy_tree_conversion.rs b/rs/canonical_state/src/lazy_tree_conversion.rs index f5d81e240715..e6ee9a048812 100644 --- a/rs/canonical_state/src/lazy_tree_conversion.rs +++ b/rs/canonical_state/src/lazy_tree_conversion.rs @@ -978,7 +978,7 @@ fn canister_metadata_as_tree( /// Helper function to turn a subnet type into a string. /// This is intentionally explicitly implemented here, so that the state tree representation cannot be changed outside this crate, as opposed /// to calling something like `subnet_type.to_string()`. -fn subnet_type_as_string(subnet_type: SubnetType) -> &'static str { +pub fn subnet_type_as_string(subnet_type: SubnetType) -> &'static str { match subnet_type { SubnetType::Application => "application", SubnetType::System => "system", diff --git a/rs/http_endpoints/nns_delegation_manager/BUILD.bazel b/rs/http_endpoints/nns_delegation_manager/BUILD.bazel index 56dfd6757e4b..e22242afe70b 100644 --- a/rs/http_endpoints/nns_delegation_manager/BUILD.bazel +++ b/rs/http_endpoints/nns_delegation_manager/BUILD.bazel @@ -22,15 +22,18 @@ rust_library( "//rs/interfaces/registry", "//rs/monitoring/logger", "//rs/monitoring/metrics", + "//rs/protobuf", "//rs/registry/helpers", "//rs/types/types", "@crate_index//:axum", "@crate_index//:futures", + "@crate_index//:hickory-resolver", "@crate_index//:http-body-util", "@crate_index//:hyper", "@crate_index//:hyper-util", "@crate_index//:prometheus", "@crate_index//:rand", + "@crate_index//:rustls", "@crate_index//:serde", "@crate_index//:serde_cbor", "@crate_index//:slog", @@ -38,6 +41,7 @@ rust_library( "@crate_index//:tokio-rustls", "@crate_index//:tokio-util", "@crate_index//:tower", + "@crate_index//:webpki-roots", ], ) @@ -56,6 +60,7 @@ rust_test( "//rs/registry/fake", "//rs/registry/keys", "//rs/registry/proto_data_provider", + "//rs/registry/subnet_type", "//rs/test_utilities/registry", "//rs/test_utilities/types", "@crate_index//:assert_matches", @@ -64,7 +69,6 @@ rust_test( "@crate_index//:hyper", "@crate_index//:rand", "@crate_index//:rcgen", - "@crate_index//:rustls", "@crate_index//:tokio", ], ) diff --git a/rs/http_endpoints/nns_delegation_manager/Cargo.toml b/rs/http_endpoints/nns_delegation_manager/Cargo.toml index efc9a3179371..4ed31772e014 100644 --- a/rs/http_endpoints/nns_delegation_manager/Cargo.toml +++ b/rs/http_endpoints/nns_delegation_manager/Cargo.toml @@ -13,6 +13,7 @@ harness = false [dependencies] axum = { workspace = true } futures = { workspace = true } +hickory-resolver = { workspace = true } http-body-util = { workspace = true } hyper = { workspace = true } hyper-util = { workspace = true } @@ -24,10 +25,12 @@ ic-crypto-utils-threshold-sig-der = { path = "../../crypto/utils/threshold_sig_d ic-interfaces-registry = { path = "../../interfaces/registry" } ic-logger = { path = "../../monitoring/logger" } ic-metrics = { path = "../../monitoring/metrics" } +ic-protobuf = { path = "../../protobuf" } ic-registry-client-helpers = { path = "../../registry/helpers" } ic-types = { path = "../../types/types" } prometheus = { workspace = true } rand = { workspace = true } +rustls = { workspace = true } serde = { workspace = true } serde_cbor = { workspace = true } slog = { workspace = true } @@ -35,6 +38,7 @@ tokio = { workspace = true } tokio-rustls = { workspace = true } tokio-util = { workspace = true } tower = { workspace = true } +webpki-roots = { workspace = true } [dev-dependencies] assert_matches = { workspace = true } @@ -46,8 +50,8 @@ ic-nns-delegation-manager-test-utils = { path = "test_utils" } ic-registry-client-fake = { path = "../../registry/fake" } ic-registry-keys = { path = "../../registry/keys" } ic-registry-proto-data-provider = { path = "../../registry/proto_data_provider" } +ic-registry-subnet-type = { path = "../../registry/subnet_type" } ic-test-utilities-registry = { path = "../../test_utilities/registry" } ic-test-utilities-types = { path = "../../test_utilities/types" } pprof = { workspace = true } rcgen = { workspace = true } -rustls = { workspace = true } diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 06780762fca7..9edbfb2097c6 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -2,6 +2,7 @@ use std::{convert::TryFrom, net::SocketAddr, sync::Arc, time::Duration}; use axum::body::Body; use futures::FutureExt; +use hickory_resolver::{Resolver, config::LookupIpStrategy}; use http_body_util::{BodyExt, Full, LengthLimitError}; use hyper::{Request, client::conn::http1::SendRequest}; use hyper_util::rt::TokioIo; @@ -11,10 +12,14 @@ use ic_crypto_tls_interfaces::TlsConfig; use ic_crypto_tree_hash::{LabeledTree, Path, lookup_path}; use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; use ic_interfaces_registry::RegistryClient; -use ic_logger::{ReplicaLogger, fatal, info, warn}; +use ic_logger::{ReplicaLogger, info, warn}; use ic_metrics::MetricsRegistry; +use ic_protobuf::registry::node::v1::NodeRewardType; use ic_registry_client_helpers::{ - crypto::CryptoRegistry, node::NodeRegistry, node_operator::ConnectionEndpoint, + api_boundary_node::ApiBoundaryNodeRegistry, + crypto::CryptoRegistry, + node::{NodeRecord, NodeRegistry}, + node_operator::ConnectionEndpoint, subnet::SubnetRegistry, }; use ic_types::{ @@ -26,7 +31,8 @@ use ic_types::{ }, time::expiry_time_from_now, }; -use rand::Rng; +use rand::{Rng, seq::SliceRandom}; +use rustls::{ClientConfig, pki_types::ServerName}; use tokio::{ net::TcpStream, sync::watch, @@ -74,6 +80,7 @@ pub fn start_nns_delegation_manager( config: Config, log: ReplicaLogger, rt_handle: tokio::runtime::Handle, + node_id: NodeId, subnet_id: SubnetId, nns_subnet_id: SubnetId, registry_client: Arc, @@ -84,6 +91,7 @@ pub fn start_nns_delegation_manager( let manager = DelegationManager { config, log, + node_id, subnet_id, nns_subnet_id, registry_client, @@ -107,6 +115,7 @@ pub fn start_nns_delegation_manager( struct DelegationManager { config: Config, log: ReplicaLogger, + node_id: NodeId, subnet_id: SubnetId, nns_subnet_id: SubnetId, registry_client: Arc, @@ -123,6 +132,7 @@ impl DelegationManager { &self.config, &self.log, &self.rt_handle, + self.node_id, self.subnet_id, self.nns_subnet_id, self.registry_client.as_ref(), @@ -172,6 +182,7 @@ async fn load_root_delegation( config: &Config, log: &ReplicaLogger, rt_handle: &tokio::runtime::Handle, + node_id: NodeId, subnet_id: SubnetId, nns_subnet_id: SubnetId, registry_client: &dyn RegistryClient, @@ -190,7 +201,7 @@ async fn load_root_delegation( fetching_root_delagation_attempts += 1; info!( log, - "Fetching delegation from the nns subnet. Attempts: {}.", + "Fetching delegation from the NNS subnet. Attempts: {}.", fetching_root_delagation_attempts ); @@ -202,6 +213,7 @@ async fn load_root_delegation( config, log, rt_handle, + node_id, subnet_id, nns_subnet_id, registry_client, @@ -214,7 +226,7 @@ async fn load_root_delegation( Err(err) => { warn!( log, - "Fetching delegation from nns subnet failed. Retrying again in {} seconds...\ + "Fetching delegation from NNS subnet failed. Retrying again in {} seconds...\ Error received: {}", backoff.as_secs(), err @@ -235,23 +247,13 @@ async fn try_fetch_delegation_from_nns( config: &Config, log: &ReplicaLogger, rt_handle: &tokio::runtime::Handle, + node_id: NodeId, subnet_id: SubnetId, nns_subnet_id: SubnetId, registry_client: &dyn RegistryClient, tls_config: &dyn TlsConfig, metrics: &DelegationManagerMetrics, ) -> Result { - let (peer_id, node) = match get_random_node_from_nns_subnet(registry_client, nns_subnet_id) { - Ok(node_topology) => node_topology, - Err(err) => { - fatal!( - log, - "Could not find a node from the root subnet to talk to. Error :{}", - err - ); - } - }; - let envelope = HttpRequestEnvelope { content: HttpReadStateContent::ReadState { read_state: HttpReadState { @@ -294,22 +296,22 @@ async fn try_fetch_delegation_from_nns( connect( log.clone(), rt_handle, - peer_id, - node, + node_id, + nns_subnet_id, registry_client, tls_config, ), ) .await .map_err(|_| { - format!("Timed out while connecting to the nns node after {CONNECTION_TIMEOUT:?}") + format!("Timed out while connecting to the node after {CONNECTION_TIMEOUT:?}") })??; let uri = format!("/api/v2/subnet/{nns_subnet_id}/read_state"); info!( log, - "Attempt to fetch HTTPS delegation from root subnet node, uri = `{uri}`." + "Attempt to fetch HTTPS delegation from the NNS, uri = `{uri}`." ); let nns_request = Request::builder() @@ -325,8 +327,8 @@ async fn try_fetch_delegation_from_nns( .await .map_err(|_| { format!( - "Timed out while sending request to the nns \ - node after {NNS_DELEGATION_REQUEST_SEND_TIMEOUT:?}", + "Timed out while sending request to the node \ + after {NNS_DELEGATION_REQUEST_SEND_TIMEOUT:?}", ) })??; @@ -427,44 +429,119 @@ async fn try_fetch_delegation_from_nns( async fn connect( log: ReplicaLogger, rt_handle: &tokio::runtime::Handle, - peer_id: NodeId, - node: ConnectionEndpoint, + node_id: NodeId, + nns_subnet_id: SubnetId, registry_client: &dyn RegistryClient, tls_config: &(dyn TlsConfig + Send + Sync), ) -> Result, BoxError> { - let registry_version = registry_client.get_latest_version(); + let node_reward_type = get_node_reward_type(registry_client, node_id).unwrap_or_else(|err| { + warn!( + log, + "Could not determine the reward type: {err}. Connecting to an NNS node directly." + ); + NodeRewardType::Unspecified + }); + + let (peer_id, addr, server_name, tls_client_config) = match node_reward_type { + NodeRewardType::Unspecified + | NodeRewardType::Type0 + | NodeRewardType::Type1 + | NodeRewardType::Type2 + | NodeRewardType::Type3 + | NodeRewardType::Type3dot1 + | NodeRewardType::Type1dot1 => { + let (peer_id, endpoint) = + get_random_node_from_nns_subnet(registry_client, nns_subnet_id).map_err(|err| { + format!("Could not find a node from the NNS to talk to. Error: {err}") + })?; + + let registry_version = registry_client.get_latest_version(); + + let ip_addr = endpoint + .ip_addr + .parse() + .map_err(|err| format!("Failed to parse the ip addr: {err}"))?; + + let addr = SocketAddr::new(ip_addr, endpoint.port as u16); + + let tls_client_config = tls_config + .client_config(peer_id, registry_version) + .map_err(|err| format!("Retrieving TLS client config failed: {err:?}."))?; - let ip_addr = node - .ip_addr - .parse() - .map_err(|err| format!("Failed to parse the ip addr: {err}"))?; + let server_name = ServerName::from(ip_addr); - let addr = SocketAddr::new(ip_addr, node.port as u16); + (peer_id, addr, server_name, tls_client_config) + } + NodeRewardType::Type4 => { + let (api_bn_id, domain) = get_random_api_boundary_node(registry_client) + .map_err(|err| format!("Could not find an API BN to talk to. Error: {err}"))?; + + let mut dns_resolver = Resolver::builder_tokio()?; + dns_resolver.options_mut().ip_strategy = LookupIpStrategy::Ipv6Only; + #[cfg(test)] + { + // In unit tests, the domain does not resolve and we want to fail fast to keep a low + // test execution time. + dns_resolver.options_mut().timeout = Duration::from_millis(100); + dns_resolver.options_mut().attempts = 1; + } + let ip_addr = dns_resolver + .build() + .lookup_ip(domain.as_str()) + .await? + .iter() + .next() + .ok_or_else(|| { + format!("API BN domain {domain} does not resolve to any IPv6 address.",) + })?; + + let addr = SocketAddr::new(ip_addr, 443); + + let root_store = + rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + let tls_client_config = rustls::ClientConfig::builder() + .with_root_certificates(root_store) + .with_no_client_auth(); + + let server_name = ServerName::try_from(domain.clone()) + .map_err(|err| format!("Invalid API BN domain {domain}: {err}"))?; + + (api_bn_id, addr, server_name, tls_client_config) + } + }; - let tls_client_config = tls_config - .client_config(peer_id, registry_version) - .map_err(|err| format!("Retrieving TLS client config failed: {err:?}."))?; + connect_to( + log, + rt_handle, + peer_id, + addr, + server_name, + tls_client_config, + ) + .await +} +async fn connect_to( + log: ReplicaLogger, + rt_handle: &tokio::runtime::Handle, + peer_id: NodeId, + addr: SocketAddr, + server_name: ServerName<'static>, + tls_client_config: ClientConfig, +) -> Result, BoxError> { info!(log, "Establishing TCP connection to {peer_id} @ {addr}"); let tcp_stream: TcpStream = TcpStream::connect(addr) .await .map_err(|err| format!("Could not connect to node {addr}. {err:?}."))?; let tls_connector = TlsConnector::from(Arc::new(tls_client_config)); - let irrelevant_domain = "domain.is-irrelevant-as-hostname-verification-is.disabled"; info!( log, "Establishing TLS stream to {peer_id}. Tcp stream: {tcp_stream:?}" ); let tls_stream = tls_connector - .connect( - irrelevant_domain - .try_into() - // TODO: ideally the expect should run at compile time - .expect("failed to create domain"), - tcp_stream, - ) + .connect(server_name, tcp_stream) .await .map_err(|err| format!("Could not establish TLS stream to node {addr}. {err:?}."))?; @@ -485,30 +562,68 @@ async fn connect( Ok(request_sender) } +fn get_node_reward_type( + registry_client: &dyn RegistryClient, + node_id: NodeId, +) -> Result { + match registry_client.get_node_record(node_id, registry_client.get_latest_version()) { + // node_reward_type() defaults to `Unspecified` if the field is unset or set to an + // invalid enum value + Ok(Some(record)) => Ok(record.node_reward_type()), + Ok(None) => Err(format!("No node record found for node id {node_id}",)), + Err(err) => Err(format!( + "Failed to get node record for node id {node_id}: {err}", + )), + } +} + fn get_random_node_from_nns_subnet( registry_client: &dyn RegistryClient, nns_subnet_id: SubnetId, ) -> Result<(NodeId, ConnectionEndpoint), String> { - use rand::seq::SliceRandom; - let nns_nodes = match registry_client .get_node_ids_on_subnet(nns_subnet_id, registry_client.get_latest_version()) { Ok(Some(nns_nodes)) => Ok(nns_nodes), - Ok(None) => Err("No nns nodes found.".to_string()), - Err(err) => Err(format!("Failed to get nns nodes from registry: {err}")), + Ok(None) => Err("No NNS nodes found.".to_string()), + Err(err) => Err(format!("Failed to get NNS nodes from registry: {err}")), }?; - // Randomly choose a node from the nns subnet. + let (node_id, record) = get_random_node_record_from_ids(registry_client, &nns_nodes)?; + let endpoint = record + .http + .ok_or_else(|| format!("No HTTP endpoint for NNS node {node_id}"))?; + Ok((node_id, endpoint)) +} + +fn get_random_api_boundary_node( + registry_client: &dyn RegistryClient, +) -> Result<(NodeId, String), String> { + let api_bns = registry_client + .get_api_boundary_node_ids(registry_client.get_latest_version()) + .map_err(|err| format!("Failed to get API BNs from registry: {err}"))?; + + let (node_id, record) = get_random_node_record_from_ids(registry_client, &api_bns)?; + let domain = record + .domain + .ok_or_else(|| format!("No domain for API BN {node_id}"))?; + Ok((node_id, domain)) +} + +fn get_random_node_record_from_ids( + registry_client: &dyn RegistryClient, + node_ids: &[NodeId], +) -> Result<(NodeId, NodeRecord), String> { let mut rng = rand::thread_rng(); - let nns_node = nns_nodes.choose(&mut rng).ok_or(format!( - "Failed to choose random nns node. NNS node list: {nns_nodes:?}" + let node_id = node_ids.choose(&mut rng).ok_or(format!( + "Failed to choose a random node. Node list: {node_ids:?}" ))?; - match registry_client.get_node_record(*nns_node, registry_client.get_latest_version()) { - Ok(Some(node)) => Ok((*nns_node, node.http.ok_or("No http endpoint for node")?)), - Ok(None) => Err(format!("No transport info found for nns node. {nns_node}")), + + match registry_client.get_node_record(*node_id, registry_client.get_latest_version()) { + Ok(Some(record)) => Ok((*node_id, record)), + Ok(None) => Err(format!("No node record found for node id {node_id}")), Err(err) => Err(format!( - "failed to get node record for nns node {nns_node}. Err: {err}" + "Failed to get node record for node id {node_id}. Err: {err}" )), } } @@ -542,10 +657,12 @@ mod tests { use ic_crypto_utils_threshold_sig_der::public_key_to_der; use ic_logger::no_op_logger; use ic_metrics::MetricsRegistry; + use ic_protobuf::registry::api_boundary_node::v1::ApiBoundaryNodeRecord; use ic_registry_client_fake::FakeRegistryClient; use ic_registry_client_helpers::node::{ConnectionEndpoint, NodeRecord}; - use ic_registry_keys::make_node_record_key; + use ic_registry_keys::{make_api_boundary_node_record_key, make_node_record_key}; use ic_registry_proto_data_provider::ProtoRegistryDataProvider; + use ic_registry_subnet_type::SubnetType; use ic_test_utilities_registry::{ SubnetRecordBuilder, add_single_subnet_record, add_subnet_key_record, add_subnet_list_record, @@ -574,9 +691,14 @@ mod tests { use super::*; const NNS_SUBNET_ID: SubnetId = ic_test_utilities_types::ids::SUBNET_1; - const NON_NNS_SUBNET_ID: SubnetId = ic_test_utilities_types::ids::SUBNET_2; + const APP_SUBNET_ID: SubnetId = ic_test_utilities_types::ids::SUBNET_2; + const CLOUD_ENGINE_SUBNET_ID: SubnetId = ic_test_utilities_types::ids::SUBNET_3; const NNS_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_1; - const NON_NNS_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_2; + const APP_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_2; + const UNKNOWN_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_3; + const CLOUD_ENGINE_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_4; + const API_BN_ID: NodeId = ic_test_utilities_types::ids::NODE_5; + const API_BN_DOMAIN: &str = "domain.invalid"; // Get a free port on this host to which we can connect transport to. fn get_free_localhost_socket_addr() -> SocketAddr { @@ -629,7 +751,7 @@ mod tests { // None means we will generate a random, valid certificate. override_nns_delegation: Arc>>, delay: Option, - ) -> (Arc, MockTlsConfig) { + ) -> (Arc, Arc) { let registry_version = 1; let data_provider = Arc::new(ProtoRegistryDataProvider::new()); @@ -646,51 +768,122 @@ mod tests { add_single_subnet_record( &data_provider, registry_version, - NON_NNS_SUBNET_ID, + APP_SUBNET_ID, + SubnetRecordBuilder::new() + .with_committee(&[APP_NODE_ID, UNKNOWN_NODE_ID]) + .build(), + ); + + add_single_subnet_record( + &data_provider, + registry_version, + CLOUD_ENGINE_SUBNET_ID, SubnetRecordBuilder::new() - .with_committee(&[NON_NNS_NODE_ID]) + .with_committee(&[CLOUD_ENGINE_NODE_ID]) + .with_subnet_type(SubnetType::CloudEngine) .build(), ); - let (non_nns_public_key, _non_nns_secret_key) = generate_root_of_trust(&mut thread_rng()); let (nns_public_key, nns_secret_key) = generate_root_of_trust(&mut thread_rng()); + let (app_subnet_public_key, _app_subnet_secret_key) = + generate_root_of_trust(&mut thread_rng()); + let (cloud_engine_public_key, _cloud_engine_secret_key) = + generate_root_of_trust(&mut thread_rng()); add_subnet_key_record( &data_provider, registry_version, - NON_NNS_SUBNET_ID, - non_nns_public_key, + NNS_SUBNET_ID, + nns_public_key, ); add_subnet_key_record( &data_provider, registry_version, - NNS_SUBNET_ID, - nns_public_key, + APP_SUBNET_ID, + app_subnet_public_key, + ); + + add_subnet_key_record( + &data_provider, + registry_version, + CLOUD_ENGINE_SUBNET_ID, + cloud_engine_public_key, ); add_subnet_list_record( &data_provider, registry_version, - vec![NNS_SUBNET_ID, NON_NNS_SUBNET_ID], + vec![NNS_SUBNET_ID, APP_SUBNET_ID, CLOUD_ENGINE_SUBNET_ID], ); let addr = get_free_localhost_socket_addr(); let tcp_listener = TcpListener::bind(addr).unwrap(); - let node_record = NodeRecord { - http: Some(ConnectionEndpoint { - ip_addr: addr.ip().to_string(), - port: addr.port() as u32, - }), - ..Default::default() - }; - data_provider .add( &make_node_record_key(NNS_NODE_ID), registry_version.into(), - Some(node_record), + Some(NodeRecord { + http: Some(ConnectionEndpoint { + ip_addr: addr.ip().to_string(), + port: addr.port() as u32, + }), + ..Default::default() + }), + ) + .unwrap(); + + data_provider + .add( + &make_node_record_key(APP_NODE_ID), + registry_version.into(), + Some(NodeRecord { + node_reward_type: Some(NodeRewardType::Type1 as i32), + ..Default::default() + }), + ) + .unwrap(); + + data_provider + .add( + &make_node_record_key(UNKNOWN_NODE_ID), + registry_version.into(), + Some(NodeRecord { + node_reward_type: Some(NodeRewardType::Unspecified as i32), + ..Default::default() + }), + ) + .unwrap(); + + data_provider + .add( + &make_node_record_key(CLOUD_ENGINE_NODE_ID), + registry_version.into(), + Some(NodeRecord { + node_reward_type: Some(NodeRewardType::Type4 as i32), + ..Default::default() + }), + ) + .unwrap(); + + data_provider + .add( + &make_node_record_key(API_BN_ID), + registry_version.into(), + Some(NodeRecord { + domain: Some(API_BN_DOMAIN.to_string()), + ..Default::default() + }), + ) + .unwrap(); + data_provider + .add( + &make_api_boundary_node_record_key(API_BN_ID), + registry_version.into(), + Some(ApiBoundaryNodeRecord { + ..Default::default() + }), ) .unwrap(); @@ -703,10 +896,14 @@ mod tests { let (_certificate, _root_pk, cbor) = CertificateBuilder::new(CertificateData::CustomTree(LabeledTree::SubTree(flatmap![ Label::from("subnet") => LabeledTree::SubTree(flatmap![ - Label::from(NON_NNS_SUBNET_ID.get_ref().to_vec()) => LabeledTree::SubTree(flatmap![ + Label::from(APP_SUBNET_ID.get_ref().to_vec()) => LabeledTree::SubTree(flatmap![ Label::from("canister_ranges") => LabeledTree::Leaf(serialize_to_cbor(&vec![(canister_test_id(0), canister_test_id(10))])), - Label::from("public_key") => LabeledTree::Leaf(public_key_to_der(&non_nns_public_key.into_bytes()).unwrap()), - ]) + Label::from("public_key") => LabeledTree::Leaf(public_key_to_der(&app_subnet_public_key.into_bytes()).unwrap()), + ]), + Label::from(CLOUD_ENGINE_SUBNET_ID.get_ref().to_vec()) => LabeledTree::SubTree(flatmap![ + Label::from("canister_ranges") => LabeledTree::Leaf(serialize_to_cbor(&vec![(canister_test_id(10), canister_test_id(20))])), + Label::from("public_key") => LabeledTree::Leaf(public_key_to_der(&cloud_engine_public_key.into_bytes()).unwrap()), + ]), ]), Label::from("time") => LabeledTree::Leaf(encoded_time(time)) ]))) @@ -805,7 +1002,7 @@ mod tests { .expect_client_config() .returning(move |_, _| Ok(accept_any_config.clone())); - (registry_client, tls_config) + (registry_client, Arc::new(tls_config)) } #[tokio::test] @@ -822,10 +1019,11 @@ mod tests { Config::default(), no_op_logger(), rt_handle, + NNS_NODE_ID, NNS_SUBNET_ID, NNS_SUBNET_ID, registry_client, - Arc::new(tls_config), + tls_config, CancellationToken::new(), ); @@ -835,7 +1033,7 @@ mod tests { } #[tokio::test] - async fn manager_load_root_delegation_on_non_nns_should_return_some_test() { + async fn manager_load_root_delegation_on_app_subnet_should_return_some_test() { let rt_handle = tokio::runtime::Handle::current(); let (registry_client, tls_config) = set_up_nns_delegation_dependencies( rt_handle.clone(), @@ -843,31 +1041,34 @@ mod tests { /*delay=*/ None, ); - let (_, mut reader) = start_nns_delegation_manager( - &MetricsRegistry::new(), - Config::default(), - no_op_logger(), - rt_handle, - NON_NNS_SUBNET_ID, - NNS_SUBNET_ID, - registry_client, - Arc::new(tls_config), - CancellationToken::new(), - ); - - reader.receiver.changed().await.unwrap(); + for node_id in [APP_NODE_ID, UNKNOWN_NODE_ID] { + let (_, mut reader) = start_nns_delegation_manager( + &MetricsRegistry::new(), + Config::default(), + no_op_logger(), + rt_handle.clone(), + node_id, + APP_SUBNET_ID, + NNS_SUBNET_ID, + registry_client.clone(), + tls_config.clone(), + CancellationToken::new(), + ); - let delegation = reader - .get_delegation(CanisterRangesFilter::Flat) - .expect("Should return some delegation on non NNS subnet"); - let parsed_delegation: Certificate = serde_cbor::from_slice(&delegation.certificate) - .expect("Should return a certificate which can be deserialized"); - let tree = LabeledTree::try_from(parsed_delegation.tree) - .expect("Should return a state tree which can be parsed"); - // Verify that the state tree has the a subtree corresponding to the requested subnet - match lookup_path(&tree, &[b"subnet", NON_NNS_SUBNET_ID.get_ref().as_ref()]) { - Some(LabeledTree::SubTree(..)) => (), - _ => panic!("Didn't find the subnet path in the state tree"), + reader.receiver.changed().await.unwrap(); + + let delegation = reader + .get_delegation(CanisterRangesFilter::Flat) + .expect("Should return some delegation on non NNS subnet"); + let parsed_delegation: Certificate = serde_cbor::from_slice(&delegation.certificate) + .expect("Should return a certificate which can be deserialized"); + let tree = LabeledTree::try_from(parsed_delegation.tree) + .expect("Should return a state tree which can be parsed"); + // Verify that the state tree has the a subtree corresponding to the requested subnet + match lookup_path(&tree, &[b"subnet", APP_SUBNET_ID.get_ref().as_ref()]) { + Some(LabeledTree::SubTree(..)) => (), + _ => panic!("Didn't find the subnet path in the state tree"), + } } } @@ -885,10 +1086,11 @@ mod tests { Config::default(), no_op_logger(), rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client, - Arc::new(tls_config), + tls_config, CancellationToken::new(), ); @@ -917,10 +1119,11 @@ mod tests { Config::default(), no_op_logger(), rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client, - Arc::new(tls_config), + tls_config, CancellationToken::new(), ); @@ -951,10 +1154,11 @@ mod tests { Config::default(), no_op_logger(), rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client, - Arc::new(tls_config), + tls_config, CancellationToken::new(), ); @@ -994,10 +1198,11 @@ mod tests { &Config::default(), &no_op_logger(), &rt_handle, + NNS_NODE_ID, NNS_SUBNET_ID, NNS_SUBNET_ID, registry_client.as_ref(), - &tls_config, + tls_config.as_ref(), &DelegationManagerMetrics::new(&MetricsRegistry::new()), ) .await; @@ -1006,7 +1211,47 @@ mod tests { } #[tokio::test] - async fn load_root_delegation_on_non_nns_should_return_some_test() { + async fn load_root_delegation_on_app_subnet_should_return_some_test() { + let rt_handle = tokio::runtime::Handle::current(); + let (registry_client, tls_config) = set_up_nns_delegation_dependencies( + rt_handle.clone(), + Arc::new(RwLock::new(None)), + /*delay=*/ None, + ); + + for node_id in [APP_NODE_ID, UNKNOWN_NODE_ID] { + let builder = load_root_delegation( + &Config::default(), + &no_op_logger(), + &rt_handle, + node_id, + APP_SUBNET_ID, + NNS_SUBNET_ID, + registry_client.as_ref(), + tls_config.as_ref(), + &DelegationManagerMetrics::new(&MetricsRegistry::new()), + ) + .await; + + let builder = builder.expect("Should return Some delegation on non NNS subnet"); + let parsed_delegation: Certificate = serde_cbor::from_slice( + &builder + .build_or_original(CanisterRangesFilter::Flat, &no_op_logger()) + .certificate, + ) + .expect("Should return a certificate which can be deserialized"); + let tree = LabeledTree::try_from(parsed_delegation.tree) + .expect("The deserialized delegation should contain a correct tree"); + // Verify that the state tree has the a subtree corresponding to the requested subnet + match lookup_path(&tree, &[b"subnet", APP_SUBNET_ID.get_ref().as_ref()]) { + Some(LabeledTree::SubTree(..)) => (), + _ => panic!("Didn't find the subnet path in the state tree"), + } + } + } + + #[tokio::test] + async fn load_root_delegation_on_cloud_engine_should_contact_api_bn_test() { let rt_handle = tokio::runtime::Handle::current(); let (registry_client, tls_config) = set_up_nns_delegation_dependencies( rt_handle.clone(), @@ -1014,34 +1259,23 @@ mod tests { /*delay=*/ None, ); - let builder = load_root_delegation( + let response = try_fetch_delegation_from_nns( &Config::default(), &no_op_logger(), &rt_handle, - NON_NNS_SUBNET_ID, + CLOUD_ENGINE_NODE_ID, + CLOUD_ENGINE_SUBNET_ID, NNS_SUBNET_ID, registry_client.as_ref(), - &tls_config, + tls_config.as_ref(), &DelegationManagerMetrics::new(&MetricsRegistry::new()), ) .await; - tokio::time::pause(); - - let builder = builder.expect("Should return Some delegation on non NNS subnet"); - let parsed_delegation: Certificate = serde_cbor::from_slice( - &builder - .build_or_original(CanisterRangesFilter::Flat, &no_op_logger()) - .certificate, - ) - .expect("Should return a certificate which can be deserialized"); - let tree = LabeledTree::try_from(parsed_delegation.tree) - .expect("The deserialized delegation should contain a correct tree"); - // Verify that the state tree has the a subtree corresponding to the requested subnet - match lookup_path(&tree, &[b"subnet", NON_NNS_SUBNET_ID.get_ref().as_ref()]) { - Some(LabeledTree::SubTree(..)) => (), - _ => panic!("Didn't find the subnet path in the state tree"), - } + // Since the API BN is configured with a domain that does not resolve, we expect the + // connection to fail with a name resolution error, which indicates that we indeed tried to + // connect to the API BN instead of an NNS node. + assert_matches!(response, Err(err) if format!("{err:?}").contains("ResolveError")); } #[tokio::test] @@ -1057,10 +1291,11 @@ mod tests { &Config::default(), &no_op_logger(), &rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client.as_ref(), - &tls_config, + tls_config.as_ref(), &DelegationManagerMetrics::new(&MetricsRegistry::new()), ) .await; @@ -1081,10 +1316,11 @@ mod tests { &Config::default(), &no_op_logger(), &rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client.as_ref(), - &tls_config, + tls_config.as_ref(), &DelegationManagerMetrics::new(&MetricsRegistry::new()), ) .await; @@ -1105,10 +1341,11 @@ mod tests { &Config::default(), &no_op_logger(), &rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client.as_ref(), - &tls_config, + tls_config.as_ref(), &DelegationManagerMetrics::new(&MetricsRegistry::new()), ) .await; diff --git a/rs/replica/src/setup_ic_stack.rs b/rs/replica/src/setup_ic_stack.rs index bd1bc1b3b1c5..2f9f1a9a7824 100644 --- a/rs/replica/src/setup_ic_stack.rs +++ b/rs/replica/src/setup_ic_stack.rs @@ -285,6 +285,7 @@ pub fn construct_ic_stack( config.http_handler.clone(), log.clone(), rt_handle_http.clone(), + node_id, subnet_id, root_subnet_id, registry.clone(), diff --git a/rs/tests/driver/src/driver/test_env_api.rs b/rs/tests/driver/src/driver/test_env_api.rs index f3e94086e014..f01752f29723 100644 --- a/rs/tests/driver/src/driver/test_env_api.rs +++ b/rs/tests/driver/src/driver/test_env_api.rs @@ -1057,6 +1057,13 @@ impl IcNodeSnapshot { ) } + pub fn canister_installer_with_raw_wasm<'a>( + &'a self, + raw_wasm: Vec, + ) -> CanisterInstaller<'a> { + CanisterInstaller::new_with_raw_wasm(self, raw_wasm) + } + /// Load wasm binary from the artifacts directory (see [HasArtifacts]) and /// install it on the target node. /// @@ -1200,16 +1207,23 @@ impl IcNodeSnapshot { pub struct CanisterInstaller<'a> { node: &'a IcNodeSnapshot, - wasm_name: String, + wasm: Vec, arg: Option>, cycles_amount: Option, effective_canister_id: PrincipalId, } impl<'a> CanisterInstaller<'a> { - pub fn new(node: &'a IcNodeSnapshot, wasm_name: impl ToString) -> Self { + pub fn new

(node: &'a IcNodeSnapshot, wasm_name: P) -> Self + where + P: AsRef, + { + Self::new_with_raw_wasm(node, load_wasm(wasm_name)) + } + + pub fn new_with_raw_wasm(node: &'a IcNodeSnapshot, wasm: Vec) -> Self { Self { - wasm_name: wasm_name.to_string(), + wasm, arg: None, cycles_amount: None, effective_canister_id: node.effective_canister_id(), @@ -1240,7 +1254,6 @@ impl<'a> CanisterInstaller<'a> { } pub async fn install(self) -> Result { - let canister_bytes = load_wasm(self.wasm_name); let agent = self.node.build_default_agent_async().await; // Create a canister. let mgr = ManagementCanister::create(&agent); @@ -1253,7 +1266,7 @@ impl<'a> CanisterInstaller<'a> { .map_err(|err| anyhow!("Couldn't create canister with provisional API: {err}"))? .0; - let mut install_code = mgr.install_code(&canister_id, &canister_bytes); + let mut install_code = mgr.install_code(&canister_id, &self.wasm); if let Some(arg) = self.arg { install_code = install_code.with_raw_arg(arg) } diff --git a/rs/tests/networking/BUILD.bazel b/rs/tests/networking/BUILD.bazel index c86d62a06c3d..1dc87860b160 100644 --- a/rs/tests/networking/BUILD.bazel +++ b/rs/tests/networking/BUILD.bazel @@ -233,6 +233,7 @@ rust_binary( crate_name = "ic_systest_nns_delegation_test", deps = [ # Keep sorted. + "//rs/canonical_state", "//rs/certification", "//rs/crypto/tree_hash", "//rs/crypto/utils/threshold_sig_der", @@ -243,8 +244,8 @@ rust_binary( "//rs/universal_canister/lib", "@crate_index//:anyhow", "@crate_index//:candid", + "@crate_index//:futures", "@crate_index//:ic-agent", - "@crate_index//:ic-utils", "@crate_index//:leb128", "@crate_index//:reqwest", "@crate_index//:serde", diff --git a/rs/tests/networking/Cargo.toml b/rs/tests/networking/Cargo.toml index 73f95e69af18..de51dbf7e1d8 100644 --- a/rs/tests/networking/Cargo.toml +++ b/rs/tests/networking/Cargo.toml @@ -17,6 +17,7 @@ dfn_candid = { path = "../../rust_canisters/dfn_candid" } futures = { workspace = true } ic-agent = { workspace = true } ic-base-types = { path = "../../types/base_types" } +ic-canonical-state = { path = "../../canonical_state" } ic-cdk = { workspace = true } ic-certification = { path = "../../certification" } ic-crypto-tree-hash = { path = "../../crypto/tree_hash" } diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 63d2692ece2d..72398eddbbea 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -28,6 +28,7 @@ Success:: */ use std::{ borrow::Cow, + collections::BTreeMap, time::{Duration, SystemTime}, }; @@ -37,6 +38,7 @@ use ic_agent::{ Agent, Identity, agent::{Envelope, EnvelopeContent}, }; +use ic_canonical_state::lazy_tree_conversion::subnet_type_as_string; use ic_certification::verify_delegation_certificate; use ic_consensus_system_test_utils::{ rw_message::install_nns_and_check_progress, @@ -49,7 +51,7 @@ use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; use ic_registry_subnet_type::SubnetType; use ic_system_test_driver::{ driver::{ - group::SystemTestGroup, + group::{SystemTestGroup, SystemTestSubGroup}, ic::{InternetComputer, Subnet}, test_env::{HasIcPrepDir, TestEnv}, test_env_api::{ @@ -68,7 +70,6 @@ use ic_types::{ HttpQueryResponseReply, HttpReadStateResponse, NodeSignature, }, }; -use ic_utils::interfaces::ManagementCanister; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use slog::info; @@ -104,61 +105,78 @@ const CERTIFIED_VAR_WAT: &str = r#" ) "#; +const TESTED_SUBNET_TYPES: [SubnetType; 4] = [ + SubnetType::System, + SubnetType::Application, + SubnetType::VerifiedApplication, + SubnetType::CloudEngine, +]; + +const INSTALLED_CANISTER_IDS: &str = "installed_canister_ids"; + /// How long to wait between subsequent nns delegation fetch requests. const RETRY_DELAY: tokio::time::Duration = tokio::time::Duration::from_secs(60); const DKG_LENGTH: Height = Height::new(9); +fn get_installed_canister_id(env: &TestEnv, subnet_type: SubnetType) -> PrincipalId { + *env.read_json_object::, _>(INSTALLED_CANISTER_IDS) + .expect("Could not read installed canister IDs from test environment.") + .get(&subnet_type) + .expect("All subnets should have an installed canister") +} + +fn set_installed_canister_ids(env: &TestEnv, canister_ids: BTreeMap) { + env.write_json_object(INSTALLED_CANISTER_IDS, &canister_ids) + .expect("Could not write installed canister IDs to test environment."); +} + fn setup(env: TestEnv) { - InternetComputer::new() - .add_subnet( - Subnet::fast_single_node(SubnetType::System).with_dkg_interval_length(DKG_LENGTH), - ) - .add_subnet( - Subnet::fast_single_node(SubnetType::Application).with_dkg_interval_length(DKG_LENGTH), - ) - .setup_and_start(&env) + let mut ic = InternetComputer::new().with_api_boundary_nodes_playnet(1); + for subnet_type in TESTED_SUBNET_TYPES { + ic = ic + .add_subnet(Subnet::fast_single_node(subnet_type).with_dkg_interval_length(DKG_LENGTH)); + } + ic.setup_and_start(&env) .expect("Should be able to set up IC under test"); install_nns_and_check_progress(env.topology_snapshot()); info!( env.logger(), - "Installing certified variables canister on the Application subnet" + "Installing certified variables canister on all subnets" ); - let (_subnet, node) = get_subnet_and_node(&env, SubnetType::Application); - let agent = node.build_default_agent(); - let management_canister = ManagementCanister::create(&agent); let wasm = wat::parse_str(CERTIFIED_VAR_WAT).expect("Failed to parse certified variables WAT"); + let canister_ids = block_on(futures::future::join_all( + TESTED_SUBNET_TYPES.iter().copied().map(|subnet_type| { + let env = env.clone(); + let wasm = wasm.clone(); + async move { + let (_subnet, node) = get_subnet_and_node(&env, subnet_type); + let canister_id = node + .canister_installer_with_raw_wasm(wasm) + .install() + .await + .expect("Failed to install the certified variables canister"); + + (subnet_type, PrincipalId::from(canister_id)) + } + }), + )) + .into_iter() + .collect(); - tokio::runtime::Runtime::new().unwrap().block_on(async { - management_canister - .create_canister() - .as_provisional_create_with_amount(None) - .with_effective_canister_id(node.effective_canister_id()) - .call_and_wait() - .await - .expect("Failed to create the certified variables canister"); - - management_canister - .install_code(&node.effective_canister_id().0, &wasm) - .call_and_wait() - .await - .expect("Failed to install the certified variables canister"); - }); - - upgrade_application_subnet_if_necessary(&env); -} + set_installed_canister_ids(&env, canister_ids); -fn nns_delegation_on_nns_test(env: TestEnv) { - block_on(nns_delegation_test(env, SubnetType::System)) + upgrade_non_nns_subnets_if_necessary(&env); } -fn nns_delegation_on_app_subnet_test(env: TestEnv) { - block_on(nns_delegation_test(env, SubnetType::Application)) +/// NNS delegations update over time +fn nns_delegation_updates(env: TestEnv, subnet_type: SubnetType) { + block_on(nns_delegation_updates_async(env, subnet_type)); } -async fn nns_delegation_test(env: TestEnv, subnet_type: SubnetType) { +async fn nns_delegation_updates_async(env: TestEnv, subnet_type: SubnetType) { let (_subnet, node) = get_subnet_and_node(&env, subnet_type); let agent = node.build_default_agent_async().await; @@ -166,18 +184,23 @@ async fn nns_delegation_test(env: TestEnv, subnet_type: SubnetType) { let maybe_initial_delegation_timestamp = get_nns_delegation_timestamp(&agent, node.effective_canister_id()).await; - if subnet_type == SubnetType::System { - assert!( - maybe_initial_delegation_timestamp.is_none(), - "There shouldn't be delegation on the NNS subnet" + let Some(initial_delegation_timestamp) = maybe_initial_delegation_timestamp else { + assert_eq!( + subnet_type, + SubnetType::System, + "Non-NNS subnet should return an NNS delegation with the response" ); // We can return, there is nothing more to be checked. return; - } + }; + + assert_ne!( + subnet_type, + SubnetType::System, + "NNS subnet should not return an NNS delegation with the response" + ); - let initial_delegation_timestamp = maybe_initial_delegation_timestamp - .expect("Non-NNS subnet should return an NNS delegation with the response"); let initial_delegation_time = SystemTime::UNIX_EPOCH .checked_add(std::time::Duration::from_nanos( initial_delegation_timestamp, @@ -248,8 +271,8 @@ async fn get_nns_delegation_timestamp( } /// Responses to `api/v2/subnet/{subnet_id}/read_state` have valid delegations with canister ranges in the flat format. -fn subnet_read_state_v2_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn subnet_read_state_v2_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, @@ -260,18 +283,17 @@ fn subnet_read_state_v2_returns_correct_delegation(env: TestEnv) { validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, None, CertificateDelegationFormat::Flat, ); } /// Responses to `api/v3/subnet/{subnet_id}/read_state` have valid delegations with canister ranges in the flat format. -fn subnet_read_state_v3_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn subnet_read_state_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, @@ -282,68 +304,64 @@ fn subnet_read_state_v3_returns_correct_delegation(env: TestEnv) { validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, None, CertificateDelegationFormat::Pruned, ); } /// Responses to `api/v2/canister/{canister_id}/read_state` have valid delegations with canister ranges in the flat format. -fn canister_read_state_v2_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn canister_read_state_v2_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_id(&env, subnet_type); + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, - format!( - "api/v2/canister/{}/read_state", - node.effective_canister_id() - ), + format!("api/v2/canister/{canister_id}/read_state"), sign_envelope(&read_state_content()), )); let certificate: Certificate = serde_cbor::from_slice(&response.certificate).unwrap(); validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, - Some(node.effective_canister_id()), + subnet_type, + Some(canister_id), CertificateDelegationFormat::Flat, ); } /// Responses to `api/v3/canister/{canister_id}/read_state` have valid delegations with canister ranges in the flat format. -fn canister_read_state_v3_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn canister_read_state_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_id(&env, subnet_type); + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, - format!( - "api/v3/canister/{}/read_state", - node.effective_canister_id() - ), + format!("api/v3/canister/{canister_id}/read_state"), sign_envelope(&read_state_content()), )); let certificate: Certificate = serde_cbor::from_slice(&response.certificate).unwrap(); validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, - Some(node.effective_canister_id()), + subnet_type, + Some(canister_id), CertificateDelegationFormat::Tree, ); } /// Responses to `api/v3/canister/aaaaa-aa/read_state` have valid delegations without canister ranges. -fn canister_read_state_v3_management_canister_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn canister_read_state_v3_management_canister_returns_correct_delegation( + env: TestEnv, + subnet_type: SubnetType, +) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, format!("api/v3/canister/{}/read_state", CanisterId::ic_00()), @@ -353,10 +371,9 @@ fn canister_read_state_v3_management_canister_returns_correct_delegation(env: Te validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, None, CertificateDelegationFormat::Pruned, ); @@ -370,53 +387,54 @@ struct SyncCallResponse { } /// Responses to `api/v3/canister/{canister_id}/call` have valid delegations with canister ranges in the flat format. -fn call_v3_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn call_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_id(&env, subnet_type); + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: SyncCallResponse = block_on(send( &node, - format!("api/v3/canister/{}/call", node.effective_canister_id()), - sign_envelope(&call_content(node.effective_canister_id())), + format!("api/v3/canister/{canister_id}/call"), + sign_envelope(&call_content(canister_id)), )); let certificate: Certificate = serde_cbor::from_slice(&response.certificate).unwrap(); validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, - Some(node.effective_canister_id()), + subnet_type, + Some(canister_id), CertificateDelegationFormat::Flat, ); } /// Responses to `api/v4/canister/{canister_id}/call` have valid delegations with canister ranges in the flat format. -fn call_v4_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn call_v4_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_id(&env, subnet_type); + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: SyncCallResponse = block_on(send( &node, - format!("api/v4/canister/{}/call", node.effective_canister_id()), - sign_envelope(&call_content(node.effective_canister_id())), + format!("api/v4/canister/{canister_id}/call"), + sign_envelope(&call_content(canister_id)), )); let certificate: Certificate = serde_cbor::from_slice(&response.certificate).unwrap(); validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, - Some(node.effective_canister_id()), + subnet_type, + Some(canister_id), CertificateDelegationFormat::Tree, ); } /// Responses to `api/v4/canister/{canister_id}/call` targeting the management canister /// have valid delegations without canister ranges. -fn call_v4_management_canister_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn call_v4_management_canister_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_id(&env, subnet_type); + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let expiration = OffsetDateTime::now_utc() + Duration::from_secs(3 * 60); #[derive(CandidType)] @@ -430,27 +448,23 @@ fn call_v4_management_canister_returns_correct_delegation(env: TestEnv) { sender: get_identity().sender().unwrap(), canister_id: CanisterId::ic_00().into(), method_name: String::from("start_canister"), - arg: Encode!(&Arg { - canister_id: node.effective_canister_id() - }) - .unwrap(), + arg: Encode!(&Arg { canister_id }).unwrap(), nonce: None, }; let response: SyncCallResponse = block_on(send( &node, - format!("api/v4/canister/{}/call", node.effective_canister_id()), + format!("api/v4/canister/{canister_id}/call"), sign_envelope(&call_content), )); let certificate: Certificate = serde_cbor::from_slice(&response.certificate).unwrap(); validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, - Some(node.effective_canister_id()), + subnet_type, + Some(canister_id), CertificateDelegationFormat::Tree, ); } @@ -464,57 +478,57 @@ struct QueryResponse { /// For `api/v2/canister/{canister_id}/query` we pass valid delegations with /// canister ranges in the flat format to the canister. -fn query_v2_passes_correct_delegation_to_canister(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn query_v2_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_id(&env, subnet_type); + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let arg = vec![]; let response: QueryResponse = block_on(send( &node, - format!("api/v2/canister/{}/query", node.effective_canister_id()), - sign_envelope(&query_content(node.effective_canister_id(), arg)), + format!("api/v2/canister/{canister_id}/query"), + sign_envelope(&query_content(canister_id, arg)), )); let certificate: Certificate = serde_cbor::from_slice(&response.reply.arg).unwrap(); validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, - Some(node.effective_canister_id()), + subnet_type, + Some(canister_id), CertificateDelegationFormat::Flat, ); } /// For `api/v3/canister/{canister_id}/query` we pass valid delegations with /// canister ranges in the tree format to the canister. -fn query_v3_passes_correct_delegation_to_canister(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn query_v3_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_id(&env, subnet_type); + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let arg = vec![]; let response: QueryResponse = block_on(send( &node, - format!("api/v3/canister/{}/query", node.effective_canister_id()), - sign_envelope(&query_content(node.effective_canister_id(), arg)), + format!("api/v3/canister/{canister_id}/query"), + sign_envelope(&query_content(canister_id, arg)), )); let certificate: Certificate = serde_cbor::from_slice(&response.reply.arg).unwrap(); validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, - Some(node.effective_canister_id()), + subnet_type, + Some(canister_id), CertificateDelegationFormat::Tree, ); } /// Run query tests several times sequentially to check that we don't return incorrect cached response. -fn interlaced_v2_and_v3_query_requests(env: TestEnv) { +fn interlaced_v2_and_v3_query_requests(env: TestEnv, subnet_type: SubnetType) { for _ in 0..10 { - query_v2_passes_correct_delegation_to_canister(env.clone()); - query_v3_passes_correct_delegation_to_canister(env.clone()); + query_v2_passes_correct_delegation_to_canister(env.clone(), subnet_type); + query_v3_passes_correct_delegation_to_canister(env.clone(), subnet_type); } } @@ -571,11 +585,28 @@ fn sign_envelope(content: &EnvelopeContent) -> Vec { fn validate_delegation( env: &TestEnv, - delegation: &CertificateDelegation, + maybe_delegation: Option<&CertificateDelegation>, subnet_id: SubnetId, + subnet_type: SubnetType, effective_canister_id: Option, expected_delegation_format: CertificateDelegationFormat, ) { + let Some(delegation) = maybe_delegation else { + assert_eq!( + subnet_type, + SubnetType::System, + "Non-NNS subnet should return an NNS delegation with the response" + ); + + // We can return, there is nothing more to be checked. + return; + }; + assert_ne!( + subnet_type, + SubnetType::System, + "NNS subnet should not return an NNS delegation with the response" + ); + let nns_public_key = env.prep_dir("").unwrap().root_public_key().unwrap(); verify_delegation_certificate( &delegation.certificate, @@ -602,7 +633,11 @@ fn validate_delegation( .expect("Every delegation has a '/subnet//type' path") { LabeledTree::Leaf(value) => { - assert_eq!(*value, "application".as_bytes().to_vec()); + assert_eq!( + value, + subnet_type_as_string(subnet_type).as_bytes(), + "Subnet type in the delegation should match the subnet type of the responding node" + ); } LabeledTree::SubTree(_) => panic!("Not a leaf"), }; @@ -710,24 +745,22 @@ where .map_err(|err| format!("Failed to deserialize response: {err:?}. Response: {response:?}",)) } -fn upgrade_application_subnet_if_necessary(env: &TestEnv) { - let (subnet, node) = get_subnet_and_node(env, SubnetType::Application); - let nns_node = get_nns_node(&env.topology_snapshot()); - +fn upgrade_non_nns_subnets_if_necessary(env: &TestEnv) { let initial_version = get_guestos_img_version(); let target_version = get_guestos_update_img_version(); if initial_version == target_version { - info!(env.logger(), "No need to upgrade the application subnet"); + info!(env.logger(), "No need to upgrade the subnets"); return; } info!( env.logger(), - "Upgrade the application subnet from {initial_version:?} to {target_version:?} to test the protocol \ - compatibility between subnets running different replica versions." + "Upgrade the non-NNS subnets from {initial_version:?} to {target_version:?} to test the \ + protocol compatibility between subnets running different replica versions." ); + let nns_node = get_nns_node(&env.topology_snapshot()); let sha256 = get_guestos_update_img_sha256(); let upgrade_url = get_guestos_update_img_url(); @@ -740,38 +773,74 @@ fn upgrade_application_subnet_if_necessary(env: &TestEnv) { vec![upgrade_url.to_string()], )); - block_on(deploy_guestos_to_all_subnet_nodes( - &nns_node, - &target_version, - subnet.subnet_id, + block_on(futures::future::join_all( + TESTED_SUBNET_TYPES + .iter() + .copied() + .filter(|t| *t != SubnetType::System) + // TODO(CON-1696): Remove next line when #9613 reaches mainnet NNS + .filter(|t| *t != SubnetType::CloudEngine) + .map(|subnet_type| { + let env = env.clone(); + let nns_node = nns_node.clone(); + let target_version = target_version.clone(); + async move { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); + + deploy_guestos_to_all_subnet_nodes( + &nns_node, + &target_version, + subnet.subnet_id, + ) + .await; + + assert_assigned_replica_version(&node, &target_version, env.logger()); + } + }), )); +} - assert_assigned_replica_version(&node, &target_version, env.logger()); +macro_rules! systest_all_subnet_types { + ($group: expr, $function_name:path) => { + // Keep up to date with `TESTED_SUBNET_TYPES` constant + $group = $group.add_test(systest!($function_name; SubnetType::System)); + $group = $group.add_test(systest!($function_name; SubnetType::Application)); + $group = $group.add_test(systest!($function_name; SubnetType::VerifiedApplication)); + // TODO(CON-1696): Remove this condition (and always run the test for cloud engines) when + // #9613 reaches mainnet NNS + if get_guestos_img_version() == get_guestos_update_img_version() { + $group = $group.add_test(systest!($function_name; SubnetType::CloudEngine)); + } + }; } fn main() -> Result<()> { + let mut par_group = SystemTestSubGroup::new(); + systest_all_subnet_types!(par_group, nns_delegation_updates); + systest_all_subnet_types!(par_group, canister_read_state_v2_returns_correct_delegation); + systest_all_subnet_types!(par_group, canister_read_state_v3_returns_correct_delegation); + systest_all_subnet_types!( + par_group, + canister_read_state_v3_management_canister_returns_correct_delegation + ); + systest_all_subnet_types!(par_group, subnet_read_state_v2_returns_correct_delegation); + systest_all_subnet_types!(par_group, subnet_read_state_v3_returns_correct_delegation); + // note: the v2 call endpoint doesn't return an NNS delegation, so there is nothing to test + systest_all_subnet_types!(par_group, call_v3_returns_correct_delegation); + systest_all_subnet_types!(par_group, call_v4_returns_correct_delegation); + systest_all_subnet_types!( + par_group, + call_v4_management_canister_returns_correct_delegation + ); + systest_all_subnet_types!(par_group, query_v2_passes_correct_delegation_to_canister); + systest_all_subnet_types!(par_group, query_v3_passes_correct_delegation_to_canister); + systest_all_subnet_types!(par_group, interlaced_v2_and_v3_query_requests); + SystemTestGroup::new() .with_setup(setup) - // We potentially upgrade the app subnet in the setup which could take several minutes - .with_overall_timeout(std::time::Duration::from_secs(25 * 60)) - .with_timeout_per_test(std::time::Duration::from_secs(15 * 60)) - .add_test(systest!(nns_delegation_on_nns_test)) - .add_test(systest!(nns_delegation_on_app_subnet_test)) - .add_test(systest!(canister_read_state_v2_returns_correct_delegation)) - .add_test(systest!(canister_read_state_v3_returns_correct_delegation)) - .add_test(systest!( - canister_read_state_v3_management_canister_returns_correct_delegation - )) - .add_test(systest!(subnet_read_state_v2_returns_correct_delegation)) - .add_test(systest!(subnet_read_state_v3_returns_correct_delegation)) - // note: the v2 call endpoint doesn't return an NNS delegation, so there is nothing to test - .add_test(systest!(call_v3_returns_correct_delegation)) - .add_test(systest!(call_v4_returns_correct_delegation)) - .add_test(systest!( - call_v4_management_canister_returns_correct_delegation - )) - .add_test(systest!(query_v2_passes_correct_delegation_to_canister)) - .add_test(systest!(query_v3_passes_correct_delegation_to_canister)) - .add_test(systest!(interlaced_v2_and_v3_query_requests)) + // We potentially upgrade subnets in the setup which could take several minutes + .with_overall_timeout(std::time::Duration::from_secs(20 * 60)) + .with_timeout_per_test(std::time::Duration::from_secs(10 * 60)) + .add_parallel(par_group) .execute_from_args() }