From 5d788a785cd57a3de6df003b98d135126cc2bf33 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Wed, 25 Mar 2026 10:04:21 +0000 Subject: [PATCH 01/65] feat: make type4 nodes go through API BNs --- Cargo.Bazel.json.lock | 29 +-- Cargo.Bazel.toml.lock | 11 +- Cargo.lock | 2 + Cargo.toml | 1 + bazel/rust.MODULE.bazel | 4 + .../nns_delegation_manager/BUILD.bazel | 4 +- .../nns_delegation_manager/Cargo.toml | 4 +- .../src/nns_delegation_manager.rs | 179 ++++++++++++++---- rs/replica/src/setup_ic_stack.rs | 1 + 9 files changed, 181 insertions(+), 54 deletions(-) diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index d0ca169bf2a7..6ac4786d44f3 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "039c010125fb5a7e9b285985debfc06af3f0beb6e7c0c1ffb027d1431794028a", + "checksum": "8f577132c78faa53399dc593b350de231281a3fe1d5e265b3344c7b797ddd25e", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -22742,6 +22742,10 @@ "id": "wat 1.244.0", "target": "wat" }, + { + "id": "webpki-roots 1.0.6", + "target": "webpki_roots" + }, { "id": "which 4.4.0", "target": "which" @@ -34027,7 +34031,7 @@ "target": "tower_service" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" } ], @@ -66435,7 +66439,7 @@ "target": "tokio_util" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" } ], @@ -66485,7 +66489,7 @@ "target": "tokio_util" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" } ], @@ -66605,7 +66609,7 @@ "target": "tokio_util" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" } ], @@ -66655,7 +66659,7 @@ "target": "tokio_util" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" } ] @@ -69608,7 +69612,7 @@ "target": "thiserror" }, { - "id": "webpki-roots 1.0.2", + "id": "webpki-roots 1.0.6", "target": "webpki_roots" }, { @@ -91843,14 +91847,14 @@ ], "license_file": "LICENSE" }, - "webpki-roots 1.0.2": { + "webpki-roots 1.0.6": { "name": "webpki-roots", - "version": "1.0.2", + "version": "1.0.6", "package_url": "https://github.com/rustls/webpki-roots", "repository": { "Http": { - "url": "https://static.crates.io/crates/webpki-roots/1.0.2/download", - "sha256": "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" + "url": "https://static.crates.io/crates/webpki-roots/1.0.6/download", + "sha256": "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" } }, "targets": [ @@ -91883,7 +91887,7 @@ "selects": {} }, "edition": "2021", - "version": "1.0.2" + "version": "1.0.6" }, "license": "CDLA-Permissive-2.0", "license_ids": [ @@ -98965,6 +98969,7 @@ "wasmtime 42.0.1", "wast 244.0.0", "wat 1.244.0", + "webpki-roots 1.0.6", "which 4.4.0", "wirm 2.1.0", "wsl 0.1.0", diff --git a/Cargo.Bazel.toml.lock b/Cargo.Bazel.toml.lock index eecde9f1b4cb..86a655b0b16a 100644 --- a/Cargo.Bazel.toml.lock +++ b/Cargo.Bazel.toml.lock @@ -3983,6 +3983,7 @@ dependencies = [ "wasmtime", "wast", "wat", + "webpki-roots 1.0.6", "which", "wirm", "wsl", @@ -5870,7 +5871,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.0", "tower-service", - "webpki-roots 1.0.2", + "webpki-roots 1.0.6", ] [[package]] @@ -11276,7 +11277,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.2", + "webpki-roots 1.0.6", ] [[package]] @@ -11725,7 +11726,7 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", - "webpki-roots 1.0.2", + "webpki-roots 1.0.6", "x509-parser 0.16.0", ] @@ -15426,9 +15427,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.2" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" dependencies = [ "rustls-pki-types", ] diff --git a/Cargo.lock b/Cargo.lock index 0e1966a283fd..9b803915d9dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12014,6 +12014,7 @@ dependencies = [ "ic-logger", "ic-metrics", "ic-nns-delegation-manager-test-utils", + "ic-protobuf", "ic-registry-client-fake", "ic-registry-client-helpers", "ic-registry-keys", @@ -12033,6 +12034,7 @@ dependencies = [ "tokio-rustls 0.26.4", "tokio-util", "tower 0.5.3", + "webpki-roots 1.0.6", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 29549529b00a..e227c174b305 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -917,6 +917,7 @@ wasmparser = "0.244.0" wasmprinter = "0.244.0" wast = "244.0.0" wat = "~1.244.0" +webpki-roots = "1.0.6" which = "6.0.3" wirm = { version = "2.1.0", features = ["parallel"] } wsl = "0.1.0" diff --git a/bazel/rust.MODULE.bazel b/bazel/rust.MODULE.bazel index a6542f0286d7..8585dc741931 100644 --- a/bazel/rust.MODULE.bazel +++ b/bazel/rust.MODULE.bazel @@ -1899,6 +1899,10 @@ crate.spec( package = "wat", version = "~1.244.0", ) +crate.spec( + package = "webpki-roots", + version = "1.0.6", +) crate.spec( package = "which", version = "^4.2.2", diff --git a/rs/http_endpoints/nns_delegation_manager/BUILD.bazel b/rs/http_endpoints/nns_delegation_manager/BUILD.bazel index 56dfd6757e4b..7cfec68d777b 100644 --- a/rs/http_endpoints/nns_delegation_manager/BUILD.bazel +++ b/rs/http_endpoints/nns_delegation_manager/BUILD.bazel @@ -22,6 +22,7 @@ rust_library( "//rs/interfaces/registry", "//rs/monitoring/logger", "//rs/monitoring/metrics", + "//rs/protobuf", "//rs/registry/helpers", "//rs/types/types", "@crate_index//:axum", @@ -31,6 +32,7 @@ rust_library( "@crate_index//:hyper-util", "@crate_index//:prometheus", "@crate_index//:rand", + "@crate_index//:rustls", "@crate_index//:serde", "@crate_index//:serde_cbor", "@crate_index//:slog", @@ -38,6 +40,7 @@ rust_library( "@crate_index//:tokio-rustls", "@crate_index//:tokio-util", "@crate_index//:tower", + "@crate_index//:webpki-roots", ], ) @@ -64,7 +67,6 @@ rust_test( "@crate_index//:hyper", "@crate_index//:rand", "@crate_index//:rcgen", - "@crate_index//:rustls", "@crate_index//:tokio", ], ) diff --git a/rs/http_endpoints/nns_delegation_manager/Cargo.toml b/rs/http_endpoints/nns_delegation_manager/Cargo.toml index efc9a3179371..765c3178879d 100644 --- a/rs/http_endpoints/nns_delegation_manager/Cargo.toml +++ b/rs/http_endpoints/nns_delegation_manager/Cargo.toml @@ -24,10 +24,12 @@ ic-crypto-utils-threshold-sig-der = { path = "../../crypto/utils/threshold_sig_d ic-interfaces-registry = { path = "../../interfaces/registry" } ic-logger = { path = "../../monitoring/logger" } ic-metrics = { path = "../../monitoring/metrics" } +ic-protobuf = { path = "../../protobuf" } ic-registry-client-helpers = { path = "../../registry/helpers" } ic-types = { path = "../../types/types" } prometheus = { workspace = true } rand = { workspace = true } +rustls = { workspace = true } serde = { workspace = true } serde_cbor = { workspace = true } slog = { workspace = true } @@ -35,6 +37,7 @@ tokio = { workspace = true } tokio-rustls = { workspace = true } tokio-util = { workspace = true } tower = { workspace = true } +webpki-roots = { workspace = true } [dev-dependencies] assert_matches = { workspace = true } @@ -50,4 +53,3 @@ ic-test-utilities-registry = { path = "../../test_utilities/registry" } ic-test-utilities-types = { path = "../../test_utilities/types" } pprof = { workspace = true } rcgen = { workspace = true } -rustls = { workspace = true } diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 06780762fca7..1882b48f4f06 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -11,11 +11,12 @@ use ic_crypto_tls_interfaces::TlsConfig; use ic_crypto_tree_hash::{LabeledTree, Path, lookup_path}; use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; use ic_interfaces_registry::RegistryClient; -use ic_logger::{ReplicaLogger, fatal, info, warn}; +use ic_logger::{ReplicaLogger, info, warn}; use ic_metrics::MetricsRegistry; +use ic_protobuf::registry::node::v1::NodeRewardType; use ic_registry_client_helpers::{ - crypto::CryptoRegistry, node::NodeRegistry, node_operator::ConnectionEndpoint, - subnet::SubnetRegistry, + api_boundary_node::ApiBoundaryNodeRegistry, crypto::CryptoRegistry, node::NodeRegistry, + node_operator::ConnectionEndpoint, subnet::SubnetRegistry, }; use ic_types::{ NodeId, RegistryVersion, SubnetId, @@ -33,7 +34,7 @@ use tokio::{ task::JoinHandle, time::{sleep, timeout}, }; -use tokio_rustls::TlsConnector; +use tokio_rustls::{TlsConnector, client::TlsStream}; use tokio_util::sync::CancellationToken; use tower::BoxError; @@ -74,6 +75,7 @@ pub fn start_nns_delegation_manager( config: Config, log: ReplicaLogger, rt_handle: tokio::runtime::Handle, + node_id: NodeId, subnet_id: SubnetId, nns_subnet_id: SubnetId, registry_client: Arc, @@ -84,6 +86,7 @@ pub fn start_nns_delegation_manager( let manager = DelegationManager { config, log, + node_id, subnet_id, nns_subnet_id, registry_client, @@ -107,6 +110,7 @@ pub fn start_nns_delegation_manager( struct DelegationManager { config: Config, log: ReplicaLogger, + node_id: NodeId, subnet_id: SubnetId, nns_subnet_id: SubnetId, registry_client: Arc, @@ -123,6 +127,7 @@ impl DelegationManager { &self.config, &self.log, &self.rt_handle, + self.node_id, self.subnet_id, self.nns_subnet_id, self.registry_client.as_ref(), @@ -172,6 +177,7 @@ async fn load_root_delegation( config: &Config, log: &ReplicaLogger, rt_handle: &tokio::runtime::Handle, + node_id: NodeId, subnet_id: SubnetId, nns_subnet_id: SubnetId, registry_client: &dyn RegistryClient, @@ -202,6 +208,7 @@ async fn load_root_delegation( config, log, rt_handle, + node_id, subnet_id, nns_subnet_id, registry_client, @@ -235,23 +242,13 @@ async fn try_fetch_delegation_from_nns( config: &Config, log: &ReplicaLogger, rt_handle: &tokio::runtime::Handle, + node_id: NodeId, subnet_id: SubnetId, nns_subnet_id: SubnetId, registry_client: &dyn RegistryClient, tls_config: &dyn TlsConfig, metrics: &DelegationManagerMetrics, ) -> Result { - let (peer_id, node) = match get_random_node_from_nns_subnet(registry_client, nns_subnet_id) { - Ok(node_topology) => node_topology, - Err(err) => { - fatal!( - log, - "Could not find a node from the root subnet to talk to. Error :{}", - err - ); - } - }; - let envelope = HttpRequestEnvelope { content: HttpReadStateContent::ReadState { read_state: HttpReadState { @@ -292,24 +289,24 @@ async fn try_fetch_delegation_from_nns( let mut request_sender = timeout( CONNECTION_TIMEOUT, connect( - log.clone(), + log, rt_handle, - peer_id, - node, + node_id, + nns_subnet_id, registry_client, tls_config, ), ) .await .map_err(|_| { - format!("Timed out while connecting to the nns node after {CONNECTION_TIMEOUT:?}") + format!("Timed out while connecting to the node after {CONNECTION_TIMEOUT:?}") })??; let uri = format!("/api/v2/subnet/{nns_subnet_id}/read_state"); info!( log, - "Attempt to fetch HTTPS delegation from root subnet node, uri = `{uri}`." + "Attempt to fetch HTTPS delegation from the NNS, uri = `{uri}`." ); let nns_request = Request::builder() @@ -325,8 +322,8 @@ async fn try_fetch_delegation_from_nns( .await .map_err(|_| { format!( - "Timed out while sending request to the nns \ - node after {NNS_DELEGATION_REQUEST_SEND_TIMEOUT:?}", + "Timed out while sending request to the node \ + after {NNS_DELEGATION_REQUEST_SEND_TIMEOUT:?}", ) })??; @@ -425,13 +422,62 @@ async fn try_fetch_delegation_from_nns( } async fn connect( - log: ReplicaLogger, + log: &ReplicaLogger, rt_handle: &tokio::runtime::Handle, + node_id: NodeId, + nns_subnet_id: SubnetId, + registry_client: &dyn RegistryClient, + tls_config: &(dyn TlsConfig + Send + Sync), +) -> Result, BoxError> { + let node_reward_type = get_node_reward_type(registry_client, node_id).unwrap_or_else(|err| { + warn!(log, "Could not determine the reward type: {err}"); + NodeRewardType::Unspecified + }); + + let tls_stream = match node_reward_type { + NodeRewardType::Unspecified + | NodeRewardType::Type0 + | NodeRewardType::Type1 + | NodeRewardType::Type2 + | NodeRewardType::Type3 + | NodeRewardType::Type3dot1 + | NodeRewardType::Type1dot1 => { + let (peer_id, node) = get_random_node_from_nns_subnet(registry_client, nns_subnet_id) + .map_err(|err| { + format!("Could not find a node from the root subnet to talk to: {err}") + })?; + + connect_to_nns_node(log, peer_id, node, registry_client, tls_config).await + } + NodeRewardType::Type4 => { + let (api_bn_id, domain) = get_random_api_boundary_node(registry_client) + .map_err(|err| format!("Could not find an API boundary node to talk to: {err}"))?; + + connect_to_api_bn(log, api_bn_id, domain).await + } + }?; + + let (request_sender, connection) = + hyper::client::conn::http1::handshake(TokioIo::new(tls_stream)).await?; + + // Spawn a task to poll the connection, driving the HTTP state + let log_cl = log.clone(); + rt_handle.spawn(async move { + if let Err(err) = connection.await { + warn!(log_cl, "Polling connection failed: {err:?}."); + } + }); + + Ok(request_sender) +} + +async fn connect_to_nns_node( + log: &ReplicaLogger, peer_id: NodeId, node: ConnectionEndpoint, registry_client: &dyn RegistryClient, tls_config: &(dyn TlsConfig + Send + Sync), -) -> Result, BoxError> { +) -> Result, BoxError> { let registry_version = registry_client.get_latest_version(); let ip_addr = node @@ -468,21 +514,59 @@ async fn connect( .await .map_err(|err| format!("Could not establish TLS stream to node {addr}. {err:?}."))?; + Ok(tls_stream) +} + +async fn connect_to_api_bn( + log: &ReplicaLogger, + api_bn_id: NodeId, + domain: String, +) -> Result, BoxError> { + let mut root_store = rustls::RootCertStore::empty(); + root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + let tls_client_config = rustls::ClientConfig::builder() + .with_root_certificates(root_store) + .with_no_client_auth(); + + let addr = (domain.as_str(), 443 as u16); + info!(log, "Establishing TCP connection to {api_bn_id} @ {addr:?}"); + let tcp_stream: TcpStream = TcpStream::connect(addr) + .await + .map_err(|err| format!("Could not connect to node {api_bn_id}. {err:?}."))?; + + let tls_connector = TlsConnector::from(Arc::new(tls_client_config)); + info!( log, - "Establishing HTTP connection to {peer_id}. Tls stream: {tls_stream:?}" + "Establishing TLS stream to {api_bn_id}. Tcp stream: {tcp_stream:?}" ); - let (request_sender, connection) = - hyper::client::conn::http1::handshake(TokioIo::new(tls_stream)).await?; + let tls_stream = tls_connector + .connect( + domain + .clone() + .try_into() + .map_err(|err| format!("Invalid API BN domain {domain}: {err}"))?, + tcp_stream, + ) + .await + .map_err(|err| format!("Could not establish TLS stream to node {api_bn_id}. {err:?}."))?; - // Spawn a task to poll the connection, driving the HTTP state - rt_handle.spawn(async move { - if let Err(err) = connection.await { - warn!(log, "Polling connection failed: {err:?}."); - } - }); + Ok(tls_stream) +} - Ok(request_sender) +fn get_node_reward_type( + registry_client: &dyn RegistryClient, + node_id: NodeId, +) -> Result { + match registry_client.get_node_record(node_id, registry_client.get_latest_version()) { + // node_reward_type() defaults to `Unspecified` if the field is unset or set to an + // invalid enum value + Ok(Some(record)) => Ok(record.node_reward_type()), + Ok(None) => Err(format!("Node record for node id {node_id} not found",)), + Err(err) => Err(format!( + "Failed to get node record for node id {node_id}: {err}", + )), + } } fn get_random_node_from_nns_subnet( @@ -506,13 +590,38 @@ fn get_random_node_from_nns_subnet( ))?; match registry_client.get_node_record(*nns_node, registry_client.get_latest_version()) { Ok(Some(node)) => Ok((*nns_node, node.http.ok_or("No http endpoint for node")?)), - Ok(None) => Err(format!("No transport info found for nns node. {nns_node}")), + Ok(None) => Err(format!("No node record found for nns node. {nns_node}")), Err(err) => Err(format!( "failed to get node record for nns node {nns_node}. Err: {err}" )), } } +fn get_random_api_boundary_node( + registry_client: &dyn RegistryClient, +) -> Result<(NodeId, String), String> { + use rand::seq::SliceRandom; + + let api_bns = + match registry_client.get_api_boundary_node_ids(registry_client.get_latest_version()) { + Ok(api_bns) => Ok(api_bns), + Err(err) => Err(format!("Failed to get API BNs from registry: {err}")), + }?; + + // Randomly choose a node from the API boundary nodes. + let mut rng = rand::thread_rng(); + let api_bn = api_bns.choose(&mut rng).ok_or(format!( + "Failed to choose random API boundary node. API BN list: {api_bns:?}" + ))?; + match registry_client.get_node_record(*api_bn, registry_client.get_latest_version()) { + Ok(Some(node)) => Ok((*api_bn, node.domain.ok_or("No domain for node")?)), + Ok(None) => Err(format!("No node record found for API BN. {api_bn}")), + Err(err) => Err(format!( + "failed to get node record for nns node {api_bn}. Err: {err}" + )), + } +} + fn get_root_threshold_public_key( registry_client: &dyn RegistryClient, version: RegistryVersion, diff --git a/rs/replica/src/setup_ic_stack.rs b/rs/replica/src/setup_ic_stack.rs index bd1bc1b3b1c5..2f9f1a9a7824 100644 --- a/rs/replica/src/setup_ic_stack.rs +++ b/rs/replica/src/setup_ic_stack.rs @@ -285,6 +285,7 @@ pub fn construct_ic_stack( config.http_handler.clone(), log.clone(), rt_handle_http.clone(), + node_id, subnet_id, root_subnet_id, registry.clone(), From b5663b510ebb6b0afc810a1512d870500c8d4db0 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Wed, 25 Mar 2026 16:02:30 +0000 Subject: [PATCH 02/65] test: extend unit tests --- Cargo.lock | 1 + .../nns_delegation_manager/BUILD.bazel | 1 + .../nns_delegation_manager/Cargo.toml | 1 + .../src/nns_delegation_manager.rs | 291 +++++++++++++----- 4 files changed, 209 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b803915d9dc..f12ddb135fc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12019,6 +12019,7 @@ dependencies = [ "ic-registry-client-helpers", "ic-registry-keys", "ic-registry-proto-data-provider", + "ic-registry-subnet-type", "ic-test-utilities-registry", "ic-test-utilities-types", "ic-types", diff --git a/rs/http_endpoints/nns_delegation_manager/BUILD.bazel b/rs/http_endpoints/nns_delegation_manager/BUILD.bazel index 7cfec68d777b..51febcadeddb 100644 --- a/rs/http_endpoints/nns_delegation_manager/BUILD.bazel +++ b/rs/http_endpoints/nns_delegation_manager/BUILD.bazel @@ -59,6 +59,7 @@ rust_test( "//rs/registry/fake", "//rs/registry/keys", "//rs/registry/proto_data_provider", + "//rs/registry/subnet_type", "//rs/test_utilities/registry", "//rs/test_utilities/types", "@crate_index//:assert_matches", diff --git a/rs/http_endpoints/nns_delegation_manager/Cargo.toml b/rs/http_endpoints/nns_delegation_manager/Cargo.toml index 765c3178879d..0d54aa0040e5 100644 --- a/rs/http_endpoints/nns_delegation_manager/Cargo.toml +++ b/rs/http_endpoints/nns_delegation_manager/Cargo.toml @@ -49,6 +49,7 @@ ic-nns-delegation-manager-test-utils = { path = "test_utils" } ic-registry-client-fake = { path = "../../registry/fake" } ic-registry-keys = { path = "../../registry/keys" } ic-registry-proto-data-provider = { path = "../../registry/proto_data_provider" } +ic-registry-subnet-type = { path = "../../registry/subnet_type" } ic-test-utilities-registry = { path = "../../test_utilities/registry" } ic-test-utilities-types = { path = "../../test_utilities/types" } pprof = { workspace = true } diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 1882b48f4f06..77d2b5cd8723 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -651,10 +651,12 @@ mod tests { use ic_crypto_utils_threshold_sig_der::public_key_to_der; use ic_logger::no_op_logger; use ic_metrics::MetricsRegistry; + use ic_protobuf::registry::api_boundary_node::v1::ApiBoundaryNodeRecord; use ic_registry_client_fake::FakeRegistryClient; use ic_registry_client_helpers::node::{ConnectionEndpoint, NodeRecord}; - use ic_registry_keys::make_node_record_key; + use ic_registry_keys::{make_api_boundary_node_record_key, make_node_record_key}; use ic_registry_proto_data_provider::ProtoRegistryDataProvider; + use ic_registry_subnet_type::SubnetType; use ic_test_utilities_registry::{ SubnetRecordBuilder, add_single_subnet_record, add_subnet_key_record, add_subnet_list_record, @@ -683,9 +685,13 @@ mod tests { use super::*; const NNS_SUBNET_ID: SubnetId = ic_test_utilities_types::ids::SUBNET_1; - const NON_NNS_SUBNET_ID: SubnetId = ic_test_utilities_types::ids::SUBNET_2; + const APP_SUBNET_ID: SubnetId = ic_test_utilities_types::ids::SUBNET_2; + const CLOUD_ENGINE_SUBNET_ID: SubnetId = ic_test_utilities_types::ids::SUBNET_3; const NNS_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_1; - const NON_NNS_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_2; + const APP_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_2; + const UNKNOWN_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_3; + const CLOUD_ENGINE_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_4; + const API_BN_ID: NodeId = ic_test_utilities_types::ids::NODE_5; // Get a free port on this host to which we can connect transport to. fn get_free_localhost_socket_addr() -> SocketAddr { @@ -738,7 +744,7 @@ mod tests { // None means we will generate a random, valid certificate. override_nns_delegation: Arc>>, delay: Option, - ) -> (Arc, MockTlsConfig) { + ) -> (Arc, Arc) { let registry_version = 1; let data_provider = Arc::new(ProtoRegistryDataProvider::new()); @@ -755,51 +761,122 @@ mod tests { add_single_subnet_record( &data_provider, registry_version, - NON_NNS_SUBNET_ID, + APP_SUBNET_ID, SubnetRecordBuilder::new() - .with_committee(&[NON_NNS_NODE_ID]) + .with_committee(&[APP_NODE_ID, UNKNOWN_NODE_ID]) + .build(), + ); + + add_single_subnet_record( + &data_provider, + registry_version, + CLOUD_ENGINE_SUBNET_ID, + SubnetRecordBuilder::new() + .with_committee(&[CLOUD_ENGINE_NODE_ID]) + .with_subnet_type(SubnetType::CloudEngine) .build(), ); - let (non_nns_public_key, _non_nns_secret_key) = generate_root_of_trust(&mut thread_rng()); let (nns_public_key, nns_secret_key) = generate_root_of_trust(&mut thread_rng()); + let (app_subnet_public_key, _app_subnet_secret_key) = + generate_root_of_trust(&mut thread_rng()); + let (cloud_engine_public_key, _cloud_engine_secret_key) = + generate_root_of_trust(&mut thread_rng()); add_subnet_key_record( &data_provider, registry_version, - NON_NNS_SUBNET_ID, - non_nns_public_key, + NNS_SUBNET_ID, + nns_public_key, ); add_subnet_key_record( &data_provider, registry_version, - NNS_SUBNET_ID, - nns_public_key, + APP_SUBNET_ID, + app_subnet_public_key, + ); + + add_subnet_key_record( + &data_provider, + registry_version, + CLOUD_ENGINE_SUBNET_ID, + cloud_engine_public_key, ); add_subnet_list_record( &data_provider, registry_version, - vec![NNS_SUBNET_ID, NON_NNS_SUBNET_ID], + vec![NNS_SUBNET_ID, APP_SUBNET_ID, CLOUD_ENGINE_SUBNET_ID], ); let addr = get_free_localhost_socket_addr(); let tcp_listener = TcpListener::bind(addr).unwrap(); - let node_record = NodeRecord { - http: Some(ConnectionEndpoint { - ip_addr: addr.ip().to_string(), - port: addr.port() as u32, - }), - ..Default::default() - }; - data_provider .add( &make_node_record_key(NNS_NODE_ID), registry_version.into(), - Some(node_record), + Some(NodeRecord { + http: Some(ConnectionEndpoint { + ip_addr: addr.ip().to_string(), + port: addr.port() as u32, + }), + ..Default::default() + }), + ) + .unwrap(); + + data_provider + .add( + &make_node_record_key(APP_NODE_ID), + registry_version.into(), + Some(NodeRecord { + node_reward_type: Some(NodeRewardType::Type1 as i32), + ..Default::default() + }), + ) + .unwrap(); + + data_provider + .add( + &make_node_record_key(UNKNOWN_NODE_ID), + registry_version.into(), + Some(NodeRecord { + node_reward_type: Some(NodeRewardType::Unspecified as i32), + ..Default::default() + }), + ) + .unwrap(); + + data_provider + .add( + &make_node_record_key(CLOUD_ENGINE_NODE_ID), + registry_version.into(), + Some(NodeRecord { + node_reward_type: Some(NodeRewardType::Type4 as i32), + ..Default::default() + }), + ) + .unwrap(); + + data_provider + .add( + &make_node_record_key(API_BN_ID), + registry_version.into(), + Some(NodeRecord { + domain: Some("domain.invalid".to_string()), + ..Default::default() + }), + ) + .unwrap(); + data_provider + .add( + &make_api_boundary_node_record_key(API_BN_ID), + registry_version.into(), + Some(ApiBoundaryNodeRecord { + ..Default::default() + }), ) .unwrap(); @@ -812,10 +889,14 @@ mod tests { let (_certificate, _root_pk, cbor) = CertificateBuilder::new(CertificateData::CustomTree(LabeledTree::SubTree(flatmap![ Label::from("subnet") => LabeledTree::SubTree(flatmap![ - Label::from(NON_NNS_SUBNET_ID.get_ref().to_vec()) => LabeledTree::SubTree(flatmap![ + Label::from(APP_SUBNET_ID.get_ref().to_vec()) => LabeledTree::SubTree(flatmap![ Label::from("canister_ranges") => LabeledTree::Leaf(serialize_to_cbor(&vec![(canister_test_id(0), canister_test_id(10))])), - Label::from("public_key") => LabeledTree::Leaf(public_key_to_der(&non_nns_public_key.into_bytes()).unwrap()), - ]) + Label::from("public_key") => LabeledTree::Leaf(public_key_to_der(&app_subnet_public_key.into_bytes()).unwrap()), + ]), + Label::from(CLOUD_ENGINE_SUBNET_ID.get_ref().to_vec()) => LabeledTree::SubTree(flatmap![ + Label::from("canister_ranges") => LabeledTree::Leaf(serialize_to_cbor(&vec![(canister_test_id(10), canister_test_id(20))])), + Label::from("public_key") => LabeledTree::Leaf(public_key_to_der(&cloud_engine_public_key.into_bytes()).unwrap()), + ]), ]), Label::from("time") => LabeledTree::Leaf(encoded_time(time)) ]))) @@ -914,7 +995,7 @@ mod tests { .expect_client_config() .returning(move |_, _| Ok(accept_any_config.clone())); - (registry_client, tls_config) + (registry_client, Arc::new(tls_config)) } #[tokio::test] @@ -931,10 +1012,11 @@ mod tests { Config::default(), no_op_logger(), rt_handle, + NNS_NODE_ID, NNS_SUBNET_ID, NNS_SUBNET_ID, registry_client, - Arc::new(tls_config), + tls_config, CancellationToken::new(), ); @@ -944,7 +1026,7 @@ mod tests { } #[tokio::test] - async fn manager_load_root_delegation_on_non_nns_should_return_some_test() { + async fn manager_load_root_delegation_on_app_subnet_should_return_some_test() { let rt_handle = tokio::runtime::Handle::current(); let (registry_client, tls_config) = set_up_nns_delegation_dependencies( rt_handle.clone(), @@ -952,31 +1034,34 @@ mod tests { /*delay=*/ None, ); - let (_, mut reader) = start_nns_delegation_manager( - &MetricsRegistry::new(), - Config::default(), - no_op_logger(), - rt_handle, - NON_NNS_SUBNET_ID, - NNS_SUBNET_ID, - registry_client, - Arc::new(tls_config), - CancellationToken::new(), - ); - - reader.receiver.changed().await.unwrap(); - - let delegation = reader - .get_delegation(CanisterRangesFilter::Flat) - .expect("Should return some delegation on non NNS subnet"); - let parsed_delegation: Certificate = serde_cbor::from_slice(&delegation.certificate) - .expect("Should return a certificate which can be deserialized"); - let tree = LabeledTree::try_from(parsed_delegation.tree) - .expect("Should return a state tree which can be parsed"); - // Verify that the state tree has the a subtree corresponding to the requested subnet - match lookup_path(&tree, &[b"subnet", NON_NNS_SUBNET_ID.get_ref().as_ref()]) { - Some(LabeledTree::SubTree(..)) => (), - _ => panic!("Didn't find the subnet path in the state tree"), + for node_id in [APP_NODE_ID, UNKNOWN_NODE_ID] { + let (_, mut reader) = start_nns_delegation_manager( + &MetricsRegistry::new(), + Config::default(), + no_op_logger(), + rt_handle.clone(), + node_id, + APP_SUBNET_ID, + NNS_SUBNET_ID, + registry_client.clone(), + tls_config.clone(), + CancellationToken::new(), + ); + + reader.receiver.changed().await.unwrap(); + + let delegation = reader + .get_delegation(CanisterRangesFilter::Flat) + .expect("Should return some delegation on non NNS subnet"); + let parsed_delegation: Certificate = serde_cbor::from_slice(&delegation.certificate) + .expect("Should return a certificate which can be deserialized"); + let tree = LabeledTree::try_from(parsed_delegation.tree) + .expect("Should return a state tree which can be parsed"); + // Verify that the state tree has the a subtree corresponding to the requested subnet + match lookup_path(&tree, &[b"subnet", APP_SUBNET_ID.get_ref().as_ref()]) { + Some(LabeledTree::SubTree(..)) => (), + _ => panic!("Didn't find the subnet path in the state tree"), + } } } @@ -994,10 +1079,11 @@ mod tests { Config::default(), no_op_logger(), rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client, - Arc::new(tls_config), + tls_config, CancellationToken::new(), ); @@ -1026,10 +1112,11 @@ mod tests { Config::default(), no_op_logger(), rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client, - Arc::new(tls_config), + tls_config, CancellationToken::new(), ); @@ -1060,10 +1147,11 @@ mod tests { Config::default(), no_op_logger(), rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client, - Arc::new(tls_config), + tls_config, CancellationToken::new(), ); @@ -1103,10 +1191,11 @@ mod tests { &Config::default(), &no_op_logger(), &rt_handle, + NNS_NODE_ID, NNS_SUBNET_ID, NNS_SUBNET_ID, registry_client.as_ref(), - &tls_config, + tls_config.as_ref(), &DelegationManagerMetrics::new(&MetricsRegistry::new()), ) .await; @@ -1115,7 +1204,7 @@ mod tests { } #[tokio::test] - async fn load_root_delegation_on_non_nns_should_return_some_test() { + async fn load_root_delegation_on_app_subnet_should_return_some_test() { let rt_handle = tokio::runtime::Handle::current(); let (registry_client, tls_config) = set_up_nns_delegation_dependencies( rt_handle.clone(), @@ -1123,34 +1212,63 @@ mod tests { /*delay=*/ None, ); - let builder = load_root_delegation( + for node_id in [APP_NODE_ID, UNKNOWN_NODE_ID] { + let builder = load_root_delegation( + &Config::default(), + &no_op_logger(), + &rt_handle, + node_id, + APP_SUBNET_ID, + NNS_SUBNET_ID, + registry_client.as_ref(), + tls_config.as_ref(), + &DelegationManagerMetrics::new(&MetricsRegistry::new()), + ) + .await; + + let builder = builder.expect("Should return Some delegation on non NNS subnet"); + let parsed_delegation: Certificate = serde_cbor::from_slice( + &builder + .build_or_original(CanisterRangesFilter::Flat, &no_op_logger()) + .certificate, + ) + .expect("Should return a certificate which can be deserialized"); + let tree = LabeledTree::try_from(parsed_delegation.tree) + .expect("The deserialized delegation should contain a correct tree"); + // Verify that the state tree has the a subtree corresponding to the requested subnet + match lookup_path(&tree, &[b"subnet", APP_SUBNET_ID.get_ref().as_ref()]) { + Some(LabeledTree::SubTree(..)) => (), + _ => panic!("Didn't find the subnet path in the state tree"), + } + } + } + + #[tokio::test] + async fn load_root_delegation_on_cloud_engine_should_contact_api_bn_test() { + let rt_handle = tokio::runtime::Handle::current(); + let (registry_client, tls_config) = set_up_nns_delegation_dependencies( + rt_handle.clone(), + Arc::new(RwLock::new(None)), + /*delay=*/ None, + ); + + let response = try_fetch_delegation_from_nns( &Config::default(), &no_op_logger(), &rt_handle, - NON_NNS_SUBNET_ID, + CLOUD_ENGINE_NODE_ID, + CLOUD_ENGINE_SUBNET_ID, NNS_SUBNET_ID, registry_client.as_ref(), - &tls_config, + tls_config.as_ref(), &DelegationManagerMetrics::new(&MetricsRegistry::new()), ) .await; - tokio::time::pause(); - - let builder = builder.expect("Should return Some delegation on non NNS subnet"); - let parsed_delegation: Certificate = serde_cbor::from_slice( - &builder - .build_or_original(CanisterRangesFilter::Flat, &no_op_logger()) - .certificate, - ) - .expect("Should return a certificate which can be deserialized"); - let tree = LabeledTree::try_from(parsed_delegation.tree) - .expect("The deserialized delegation should contain a correct tree"); - // Verify that the state tree has the a subtree corresponding to the requested subnet - match lookup_path(&tree, &[b"subnet", NON_NNS_SUBNET_ID.get_ref().as_ref()]) { - Some(LabeledTree::SubTree(..)) => (), - _ => panic!("Didn't find the subnet path in the state tree"), - } + // Since the API BN is configured with a domain that does not resolve, we expect the + // connection to fail with a name resolution error, which indicates that we indeed tried to + // connect to the API BN instead of an NNS node. + assert_matches!(response, Err(err) if err.to_string().contains("Temporary failure in name resolution")); } #[tokio::test] @@ -1166,10 +1284,11 @@ mod tests { &Config::default(), &no_op_logger(), &rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client.as_ref(), - &tls_config, + tls_config.as_ref(), &DelegationManagerMetrics::new(&MetricsRegistry::new()), ) .await; @@ -1190,10 +1309,11 @@ mod tests { &Config::default(), &no_op_logger(), &rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client.as_ref(), - &tls_config, + tls_config.as_ref(), &DelegationManagerMetrics::new(&MetricsRegistry::new()), ) .await; @@ -1214,10 +1334,11 @@ mod tests { &Config::default(), &no_op_logger(), &rt_handle, - NON_NNS_SUBNET_ID, + APP_NODE_ID, + APP_SUBNET_ID, NNS_SUBNET_ID, registry_client.as_ref(), - &tls_config, + tls_config.as_ref(), &DelegationManagerMetrics::new(&MetricsRegistry::new()), ) .await; From 2a905f84c9ce3ba5baa329d1e9f2b0b8216424f4 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 09:27:34 +0000 Subject: [PATCH 03/65] test: extend system test --- rs/tests/driver/src/driver/bootstrap.rs | 2 +- rs/tests/driver/src/driver/ic.rs | 7 + rs/tests/networking/BUILD.bazel | 1 + rs/tests/networking/nns_delegation_test.rs | 240 ++++++++++++--------- 4 files changed, 152 insertions(+), 98 deletions(-) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 40b037609529..8227128c5bc1 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -613,7 +613,7 @@ fn node_to_config(node: &Node) -> NodeConfiguration { node_operator_principal_id: None, secret_key_store: node.secret_key_store.clone(), domain: node.domain.clone(), - node_reward_type: None, + node_reward_type: node.node_reward_type.map(String::from), } } diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index 2cdf69277b80..b093cd0c9e8b 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -12,6 +12,7 @@ use crate::driver::{ use anyhow::Result; use ic_prep_lib::prep_state_directory::IcPrepStateDir; use ic_prep_lib::{node::NodeSecretKeyStore, subnet_configuration::SubnetRunningState}; +use ic_protobuf::registry::node::v1::NodeRewardType; use ic_regedit; use ic_registry_canister_api::IPv4Config; use ic_registry_subnet_features::{ChainKeyConfig, SubnetFeatures}; @@ -851,6 +852,7 @@ pub struct Node { pub malicious_behavior: Option, pub ipv4: Option, pub domain: Option, + pub node_reward_type: Option, pub recovery_hash: Option, pub boot_image: BootImage, } @@ -905,6 +907,11 @@ impl Node { self } + pub fn with_node_reward_type(mut self, node_reward_type: NodeRewardType) -> Self { + self.node_reward_type = Some(node_reward_type); + self + } + pub fn with_recovery_hash(mut self, recovery_hash: String) -> Self { self.recovery_hash = Some(recovery_hash); self diff --git a/rs/tests/networking/BUILD.bazel b/rs/tests/networking/BUILD.bazel index c86d62a06c3d..14794f8f378b 100644 --- a/rs/tests/networking/BUILD.bazel +++ b/rs/tests/networking/BUILD.bazel @@ -236,6 +236,7 @@ rust_binary( "//rs/certification", "//rs/crypto/tree_hash", "//rs/crypto/utils/threshold_sig_der", + "//rs/protobuf", "//rs/registry/subnet_type", "//rs/tests/consensus/utils", "//rs/tests/driver:ic-system-test-driver", diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 63d2692ece2d..430e9ae8ff26 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -46,11 +46,12 @@ use ic_consensus_system_test_utils::{ }; use ic_crypto_tree_hash::{LabeledTree, lookup_path}; use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; +use ic_protobuf::registry::node::v1::NodeRewardType; use ic_registry_subnet_type::SubnetType; use ic_system_test_driver::{ driver::{ group::SystemTestGroup, - ic::{InternetComputer, Subnet}, + ic::{InternetComputer, Node, Subnet}, test_env::{HasIcPrepDir, TestEnv}, test_env_api::{ HasPublicApiUrl, HasTopologySnapshot, IcNodeContainer, IcNodeSnapshot, SubnetSnapshot, @@ -111,12 +112,20 @@ const DKG_LENGTH: Height = Height::new(9); fn setup(env: TestEnv) { InternetComputer::new() + .with_api_boundary_nodes(1) .add_subnet( Subnet::fast_single_node(SubnetType::System).with_dkg_interval_length(DKG_LENGTH), ) .add_subnet( Subnet::fast_single_node(SubnetType::Application).with_dkg_interval_length(DKG_LENGTH), ) + .add_subnet( + Subnet::new(SubnetType::CloudEngine) + .with_unit_delay(Duration::from_millis(200)) + .with_initial_notary_delay(Duration::from_millis(200)) + .add_node(Node::new().with_node_reward_type(NodeRewardType::Type4)) + .with_dkg_interval_length(DKG_LENGTH), + ) .setup_and_start(&env) .expect("Should be able to set up IC under test"); @@ -147,18 +156,15 @@ fn setup(env: TestEnv) { .expect("Failed to install the certified variables canister"); }); - upgrade_application_subnet_if_necessary(&env); + upgrade_non_nns_subnets_if_necessary(&env); } -fn nns_delegation_on_nns_test(env: TestEnv) { - block_on(nns_delegation_test(env, SubnetType::System)) +/// NNS delegations update over time +fn nns_delegation_updates(env: TestEnv, subnet_type: SubnetType) { + block_on(nns_delegation_updates_async(env, subnet_type)); } -fn nns_delegation_on_app_subnet_test(env: TestEnv) { - block_on(nns_delegation_test(env, SubnetType::Application)) -} - -async fn nns_delegation_test(env: TestEnv, subnet_type: SubnetType) { +async fn nns_delegation_updates_async(env: TestEnv, subnet_type: SubnetType) { let (_subnet, node) = get_subnet_and_node(&env, subnet_type); let agent = node.build_default_agent_async().await; @@ -166,18 +172,21 @@ async fn nns_delegation_test(env: TestEnv, subnet_type: SubnetType) { let maybe_initial_delegation_timestamp = get_nns_delegation_timestamp(&agent, node.effective_canister_id()).await; - if subnet_type == SubnetType::System { + let Some(initial_delegation_timestamp) = maybe_initial_delegation_timestamp else { assert!( - maybe_initial_delegation_timestamp.is_none(), - "There shouldn't be delegation on the NNS subnet" + subnet_type == SubnetType::System, + "Non-NNS subnet should return an NNS delegation with the response" ); // We can return, there is nothing more to be checked. return; - } + }; + + assert!( + subnet_type != SubnetType::System, + "NNS subnet should not return an NNS delegation with the response" + ); - let initial_delegation_timestamp = maybe_initial_delegation_timestamp - .expect("Non-NNS subnet should return an NNS delegation with the response"); let initial_delegation_time = SystemTime::UNIX_EPOCH .checked_add(std::time::Duration::from_nanos( initial_delegation_timestamp, @@ -248,8 +257,8 @@ async fn get_nns_delegation_timestamp( } /// Responses to `api/v2/subnet/{subnet_id}/read_state` have valid delegations with canister ranges in the flat format. -fn subnet_read_state_v2_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn subnet_read_state_v2_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, @@ -260,18 +269,17 @@ fn subnet_read_state_v2_returns_correct_delegation(env: TestEnv) { validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, None, CertificateDelegationFormat::Flat, ); } /// Responses to `api/v3/subnet/{subnet_id}/read_state` have valid delegations with canister ranges in the flat format. -fn subnet_read_state_v3_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn subnet_read_state_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, @@ -282,18 +290,17 @@ fn subnet_read_state_v3_returns_correct_delegation(env: TestEnv) { validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, None, CertificateDelegationFormat::Pruned, ); } /// Responses to `api/v2/canister/{canister_id}/read_state` have valid delegations with canister ranges in the flat format. -fn canister_read_state_v2_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn canister_read_state_v2_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, @@ -307,18 +314,17 @@ fn canister_read_state_v2_returns_correct_delegation(env: TestEnv) { validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, Some(node.effective_canister_id()), CertificateDelegationFormat::Flat, ); } /// Responses to `api/v3/canister/{canister_id}/read_state` have valid delegations with canister ranges in the flat format. -fn canister_read_state_v3_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn canister_read_state_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, @@ -332,18 +338,20 @@ fn canister_read_state_v3_returns_correct_delegation(env: TestEnv) { validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, Some(node.effective_canister_id()), CertificateDelegationFormat::Tree, ); } /// Responses to `api/v3/canister/aaaaa-aa/read_state` have valid delegations without canister ranges. -fn canister_read_state_v3_management_canister_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn canister_read_state_v3_management_canister_returns_correct_delegation( + env: TestEnv, + subnet_type: SubnetType, +) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, format!("api/v3/canister/{}/read_state", CanisterId::ic_00()), @@ -353,10 +361,9 @@ fn canister_read_state_v3_management_canister_returns_correct_delegation(env: Te validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, None, CertificateDelegationFormat::Pruned, ); @@ -370,8 +377,8 @@ struct SyncCallResponse { } /// Responses to `api/v3/canister/{canister_id}/call` have valid delegations with canister ranges in the flat format. -fn call_v3_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn call_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: SyncCallResponse = block_on(send( &node, @@ -382,18 +389,17 @@ fn call_v3_returns_correct_delegation(env: TestEnv) { validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, Some(node.effective_canister_id()), CertificateDelegationFormat::Flat, ); } /// Responses to `api/v4/canister/{canister_id}/call` have valid delegations with canister ranges in the flat format. -fn call_v4_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn call_v4_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: SyncCallResponse = block_on(send( &node, @@ -404,10 +410,9 @@ fn call_v4_returns_correct_delegation(env: TestEnv) { validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, Some(node.effective_canister_id()), CertificateDelegationFormat::Tree, ); @@ -415,8 +420,8 @@ fn call_v4_returns_correct_delegation(env: TestEnv) { /// Responses to `api/v4/canister/{canister_id}/call` targeting the management canister /// have valid delegations without canister ranges. -fn call_v4_management_canister_returns_correct_delegation(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn call_v4_management_canister_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let expiration = OffsetDateTime::now_utc() + Duration::from_secs(3 * 60); #[derive(CandidType)] @@ -446,10 +451,9 @@ fn call_v4_management_canister_returns_correct_delegation(env: TestEnv) { validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, Some(node.effective_canister_id()), CertificateDelegationFormat::Tree, ); @@ -464,8 +468,8 @@ struct QueryResponse { /// For `api/v2/canister/{canister_id}/query` we pass valid delegations with /// canister ranges in the flat format to the canister. -fn query_v2_passes_correct_delegation_to_canister(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn query_v2_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: SubnetType) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let arg = vec![]; let response: QueryResponse = block_on(send( @@ -477,10 +481,9 @@ fn query_v2_passes_correct_delegation_to_canister(env: TestEnv) { validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, Some(node.effective_canister_id()), CertificateDelegationFormat::Flat, ); @@ -488,8 +491,8 @@ fn query_v2_passes_correct_delegation_to_canister(env: TestEnv) { /// For `api/v3/canister/{canister_id}/query` we pass valid delegations with /// canister ranges in the tree format to the canister. -fn query_v3_passes_correct_delegation_to_canister(env: TestEnv) { - let (subnet, node) = get_subnet_and_node(&env, SubnetType::Application); +fn query_v3_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: SubnetType) { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); let arg = vec![]; let response: QueryResponse = block_on(send( @@ -501,20 +504,19 @@ fn query_v3_passes_correct_delegation_to_canister(env: TestEnv) { validate_delegation( &env, - &certificate - .delegation - .expect("Should have an NNS delegation attached"), + certificate.delegation.as_ref(), subnet.subnet_id, + subnet_type, Some(node.effective_canister_id()), CertificateDelegationFormat::Tree, ); } /// Run query tests several times sequentially to check that we don't return incorrect cached response. -fn interlaced_v2_and_v3_query_requests(env: TestEnv) { +fn interlaced_v2_and_v3_query_requests(env: TestEnv, subnet_type: SubnetType) { for _ in 0..10 { - query_v2_passes_correct_delegation_to_canister(env.clone()); - query_v3_passes_correct_delegation_to_canister(env.clone()); + query_v2_passes_correct_delegation_to_canister(env.clone(), subnet_type); + query_v3_passes_correct_delegation_to_canister(env.clone(), subnet_type); } } @@ -571,11 +573,26 @@ fn sign_envelope(content: &EnvelopeContent) -> Vec { fn validate_delegation( env: &TestEnv, - delegation: &CertificateDelegation, + maybe_delegation: Option<&CertificateDelegation>, subnet_id: SubnetId, + subnet_type: SubnetType, effective_canister_id: Option, expected_delegation_format: CertificateDelegationFormat, ) { + let Some(delegation) = maybe_delegation else { + assert!( + subnet_type == SubnetType::System, + "Every non-NNS subnet should have a delegation attached to the response" + ); + + // We can return, there is nothing more to be checked. + return; + }; + assert!( + subnet_type != SubnetType::System, + "NNS subnet should not have a delegation attached to the response" + ); + let nns_public_key = env.prep_dir("").unwrap().root_public_key().unwrap(); verify_delegation_certificate( &delegation.certificate, @@ -602,7 +619,11 @@ fn validate_delegation( .expect("Every delegation has a '/subnet//type' path") { LabeledTree::Leaf(value) => { - assert_eq!(*value, "application".as_bytes().to_vec()); + assert_eq!( + value, + subnet_type.as_ref().as_bytes(), + "Subnet type in the delegation should match the subnet type of the responding node" + ); } LabeledTree::SubTree(_) => panic!("Not a leaf"), }; @@ -710,8 +731,9 @@ where .map_err(|err| format!("Failed to deserialize response: {err:?}. Response: {response:?}",)) } -fn upgrade_application_subnet_if_necessary(env: &TestEnv) { - let (subnet, node) = get_subnet_and_node(env, SubnetType::Application); +fn upgrade_non_nns_subnets_if_necessary(env: &TestEnv) { + let (app_subnet, app_node) = get_subnet_and_node(env, SubnetType::Application); + let (cloud_engine, cloud_engine_node) = get_subnet_and_node(env, SubnetType::CloudEngine); let nns_node = get_nns_node(&env.topology_snapshot()); let initial_version = get_guestos_img_version(); @@ -724,8 +746,8 @@ fn upgrade_application_subnet_if_necessary(env: &TestEnv) { info!( env.logger(), - "Upgrade the application subnet from {initial_version:?} to {target_version:?} to test the protocol \ - compatibility between subnets running different replica versions." + "Upgrade the application subnet and cloud engine from {initial_version:?} to {target_version:?} to + test the protocol compatibility between subnets running different replica versions." ); let sha256 = get_guestos_update_img_sha256(); @@ -743,35 +765,59 @@ fn upgrade_application_subnet_if_necessary(env: &TestEnv) { block_on(deploy_guestos_to_all_subnet_nodes( &nns_node, &target_version, - subnet.subnet_id, + app_subnet.subnet_id, )); - assert_assigned_replica_version(&node, &target_version, env.logger()); + block_on(deploy_guestos_to_all_subnet_nodes( + &nns_node, + &target_version, + cloud_engine.subnet_id, + )); + + assert_assigned_replica_version(&app_node, &target_version, env.logger()); + assert_assigned_replica_version(&cloud_engine_node, &target_version, env.logger()); +} + +macro_rules! systest_all_subnet_types { + ($group: expr, $function_name:path) => { + $group = $group.add_test(systest!($function_name; SubnetType::System)); + $group = $group.add_test(systest!($function_name; SubnetType::Application)); + $group = $group.add_test(systest!($function_name; SubnetType::CloudEngine)); + }; } fn main() -> Result<()> { - SystemTestGroup::new() + let mut test_group = SystemTestGroup::new() .with_setup(setup) // We potentially upgrade the app subnet in the setup which could take several minutes .with_overall_timeout(std::time::Duration::from_secs(25 * 60)) - .with_timeout_per_test(std::time::Duration::from_secs(15 * 60)) - .add_test(systest!(nns_delegation_on_nns_test)) - .add_test(systest!(nns_delegation_on_app_subnet_test)) - .add_test(systest!(canister_read_state_v2_returns_correct_delegation)) - .add_test(systest!(canister_read_state_v3_returns_correct_delegation)) - .add_test(systest!( - canister_read_state_v3_management_canister_returns_correct_delegation - )) - .add_test(systest!(subnet_read_state_v2_returns_correct_delegation)) - .add_test(systest!(subnet_read_state_v3_returns_correct_delegation)) - // note: the v2 call endpoint doesn't return an NNS delegation, so there is nothing to test - .add_test(systest!(call_v3_returns_correct_delegation)) - .add_test(systest!(call_v4_returns_correct_delegation)) - .add_test(systest!( - call_v4_management_canister_returns_correct_delegation - )) - .add_test(systest!(query_v2_passes_correct_delegation_to_canister)) - .add_test(systest!(query_v3_passes_correct_delegation_to_canister)) - .add_test(systest!(interlaced_v2_and_v3_query_requests)) - .execute_from_args() + .with_timeout_per_test(std::time::Duration::from_secs(15 * 60)); + + systest_all_subnet_types!(test_group, nns_delegation_updates); + systest_all_subnet_types!( + test_group, + canister_read_state_v2_returns_correct_delegation + ); + systest_all_subnet_types!( + test_group, + canister_read_state_v3_returns_correct_delegation + ); + systest_all_subnet_types!( + test_group, + canister_read_state_v3_management_canister_returns_correct_delegation + ); + systest_all_subnet_types!(test_group, subnet_read_state_v2_returns_correct_delegation); + systest_all_subnet_types!(test_group, subnet_read_state_v3_returns_correct_delegation); + // note: the v2 call endpoint doesn't return an NNS delegation, so there is nothing to test + systest_all_subnet_types!(test_group, call_v3_returns_correct_delegation); + systest_all_subnet_types!(test_group, call_v4_returns_correct_delegation); + systest_all_subnet_types!( + test_group, + call_v4_management_canister_returns_correct_delegation + ); + systest_all_subnet_types!(test_group, query_v2_passes_correct_delegation_to_canister); + systest_all_subnet_types!(test_group, query_v3_passes_correct_delegation_to_canister); + systest_all_subnet_types!(test_group, interlaced_v2_and_v3_query_requests); + + test_group.execute_from_args() } From 69a2a2b8aff1688d572d1059936ccf821ccc294f Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 10:11:32 +0000 Subject: [PATCH 04/65] style: consistent styling --- .../src/nns_delegation_manager.rs | 55 +++++++++++-------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 77d2b5cd8723..68482a26363f 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -196,7 +196,7 @@ async fn load_root_delegation( fetching_root_delagation_attempts += 1; info!( log, - "Fetching delegation from the nns subnet. Attempts: {}.", + "Fetching delegation from the NNS subnet. Attempts: {}.", fetching_root_delagation_attempts ); @@ -221,7 +221,7 @@ async fn load_root_delegation( Err(err) => { warn!( log, - "Fetching delegation from nns subnet failed. Retrying again in {} seconds...\ + "Fetching delegation from NNS subnet failed. Retrying again in {} seconds...\ Error received: {}", backoff.as_secs(), err @@ -442,21 +442,25 @@ async fn connect( | NodeRewardType::Type3 | NodeRewardType::Type3dot1 | NodeRewardType::Type1dot1 => { - let (peer_id, node) = get_random_node_from_nns_subnet(registry_client, nns_subnet_id) - .map_err(|err| { - format!("Could not find a node from the root subnet to talk to: {err}") - })?; + let (peer_id, endpoint) = + get_random_node_from_nns_subnet(registry_client, nns_subnet_id).map_err(|err| { + format!("Could not find a node from the NNS to talk to. Error: {err}") + })?; - connect_to_nns_node(log, peer_id, node, registry_client, tls_config).await + connect_to_nns_node(log, peer_id, endpoint, registry_client, tls_config).await } NodeRewardType::Type4 => { let (api_bn_id, domain) = get_random_api_boundary_node(registry_client) - .map_err(|err| format!("Could not find an API boundary node to talk to: {err}"))?; + .map_err(|err| format!("Could not find an API BN to talk to. Error: {err}"))?; connect_to_api_bn(log, api_bn_id, domain).await } }?; + info!( + log, + "Establishing HTTP connection. Tls stream: {tls_stream:?}" + ); let (request_sender, connection) = hyper::client::conn::http1::handshake(TokioIo::new(tls_stream)).await?; @@ -474,18 +478,18 @@ async fn connect( async fn connect_to_nns_node( log: &ReplicaLogger, peer_id: NodeId, - node: ConnectionEndpoint, + endpoint: ConnectionEndpoint, registry_client: &dyn RegistryClient, tls_config: &(dyn TlsConfig + Send + Sync), ) -> Result, BoxError> { let registry_version = registry_client.get_latest_version(); - let ip_addr = node + let ip_addr = endpoint .ip_addr .parse() .map_err(|err| format!("Failed to parse the ip addr: {err}"))?; - let addr = SocketAddr::new(ip_addr, node.port as u16); + let addr = SocketAddr::new(ip_addr, endpoint.port as u16); let tls_client_config = tls_config .client_config(peer_id, registry_version) @@ -494,7 +498,7 @@ async fn connect_to_nns_node( info!(log, "Establishing TCP connection to {peer_id} @ {addr}"); let tcp_stream: TcpStream = TcpStream::connect(addr) .await - .map_err(|err| format!("Could not connect to node {addr}. {err:?}."))?; + .map_err(|err| format!("Could not connect to node {peer_id}. {err:?}."))?; let tls_connector = TlsConnector::from(Arc::new(tls_client_config)); let irrelevant_domain = "domain.is-irrelevant-as-hostname-verification-is.disabled"; @@ -512,7 +516,7 @@ async fn connect_to_nns_node( tcp_stream, ) .await - .map_err(|err| format!("Could not establish TLS stream to node {addr}. {err:?}."))?; + .map_err(|err| format!("Could not establish TLS stream to node {peer_id}. {err:?}."))?; Ok(tls_stream) } @@ -562,7 +566,7 @@ fn get_node_reward_type( // node_reward_type() defaults to `Unspecified` if the field is unset or set to an // invalid enum value Ok(Some(record)) => Ok(record.node_reward_type()), - Ok(None) => Err(format!("Node record for node id {node_id} not found",)), + Ok(None) => Err(format!("No node record found for node id {node_id}",)), Err(err) => Err(format!( "Failed to get node record for node id {node_id}: {err}", )), @@ -579,20 +583,23 @@ fn get_random_node_from_nns_subnet( .get_node_ids_on_subnet(nns_subnet_id, registry_client.get_latest_version()) { Ok(Some(nns_nodes)) => Ok(nns_nodes), - Ok(None) => Err("No nns nodes found.".to_string()), - Err(err) => Err(format!("Failed to get nns nodes from registry: {err}")), + Ok(None) => Err("No NNS nodes found.".to_string()), + Err(err) => Err(format!("Failed to get NNS nodes from registry: {err}")), }?; // Randomly choose a node from the nns subnet. let mut rng = rand::thread_rng(); let nns_node = nns_nodes.choose(&mut rng).ok_or(format!( - "Failed to choose random nns node. NNS node list: {nns_nodes:?}" + "Failed to choose a random NNS node. NNS node list: {nns_nodes:?}" ))?; match registry_client.get_node_record(*nns_node, registry_client.get_latest_version()) { - Ok(Some(node)) => Ok((*nns_node, node.http.ok_or("No http endpoint for node")?)), - Ok(None) => Err(format!("No node record found for nns node. {nns_node}")), + Ok(Some(node)) => Ok(( + *nns_node, + node.http.ok_or("No HTTP endpoint for node {nns_node}")?, + )), + Ok(None) => Err(format!("No node record found for NNS node {nns_node}")), Err(err) => Err(format!( - "failed to get node record for nns node {nns_node}. Err: {err}" + "Failed to get node record for NNS node {nns_node}. Err: {err}" )), } } @@ -611,13 +618,13 @@ fn get_random_api_boundary_node( // Randomly choose a node from the API boundary nodes. let mut rng = rand::thread_rng(); let api_bn = api_bns.choose(&mut rng).ok_or(format!( - "Failed to choose random API boundary node. API BN list: {api_bns:?}" + "Failed to choose a random API boundary node. API BN list: {api_bns:?}" ))?; match registry_client.get_node_record(*api_bn, registry_client.get_latest_version()) { - Ok(Some(node)) => Ok((*api_bn, node.domain.ok_or("No domain for node")?)), - Ok(None) => Err(format!("No node record found for API BN. {api_bn}")), + Ok(Some(node)) => Ok((*api_bn, node.domain.ok_or("No domain for node {api_bn}")?)), + Ok(None) => Err(format!("No node record found for API BN {api_bn}")), Err(err) => Err(format!( - "failed to get node record for nns node {api_bn}. Err: {err}" + "Failed to get node record for API BN {api_bn}. Err: {err}" )), } } From 53fd4e66a284866602564ab921e2ab42c6977781 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 10:34:46 +0000 Subject: [PATCH 05/65] trying out Daniel's new lint --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 68482a26363f..d8c4641baddd 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -532,7 +532,7 @@ async fn connect_to_api_bn( .with_root_certificates(root_store) .with_no_client_auth(); - let addr = (domain.as_str(), 443 as u16); + let addr = (domain.as_str(), 443u16); info!(log, "Establishing TCP connection to {api_bn_id} @ {addr:?}"); let tcp_stream: TcpStream = TcpStream::connect(addr) .await From 43469e29a114bf56858a863b9c9ea9e0baf038ab Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 10:49:14 +0000 Subject: [PATCH 06/65] it works! --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index d8c4641baddd..6c1f5c62a6dd 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -532,7 +532,7 @@ async fn connect_to_api_bn( .with_root_certificates(root_store) .with_no_client_auth(); - let addr = (domain.as_str(), 443u16); + let addr = (domain.as_str(), 443_u16); info!(log, "Establishing TCP connection to {api_bn_id} @ {addr:?}"); let tcp_stream: TcpStream = TcpStream::connect(addr) .await From a1bfec0d9fdfcec00c6f192533ec9bd7c4d7989c Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 10:50:21 +0000 Subject: [PATCH 07/65] refactor: simplify cloud engine creation --- rs/tests/driver/src/driver/ic.rs | 7 ++++++- rs/tests/networking/nns_delegation_test.rs | 8 +------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index b093cd0c9e8b..80678fc9ae8f 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -615,7 +615,12 @@ impl Subnet { }) } - pub fn add_node(mut self, node: Node) -> Self { + pub fn add_node(mut self, mut node: Node) -> Self { + // If the subnet is a cloud engine, ensure that all nodes have reward type 4 + if self.subnet_type == SubnetType::CloudEngine { + node = node.with_node_reward_type(NodeRewardType::Type4) + } + self.nodes.push(node); self } diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 430e9ae8ff26..4febb531a330 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -119,13 +119,7 @@ fn setup(env: TestEnv) { .add_subnet( Subnet::fast_single_node(SubnetType::Application).with_dkg_interval_length(DKG_LENGTH), ) - .add_subnet( - Subnet::new(SubnetType::CloudEngine) - .with_unit_delay(Duration::from_millis(200)) - .with_initial_notary_delay(Duration::from_millis(200)) - .add_node(Node::new().with_node_reward_type(NodeRewardType::Type4)) - .with_dkg_interval_length(DKG_LENGTH), - ) + .add_subnet(Subnet::new(SubnetType::CloudEngine).with_dkg_interval_length(DKG_LENGTH)) .setup_and_start(&env) .expect("Should be able to set up IC under test"); From 61c3624cf8ca8f7cba8380cfc0dda94bcc397b29 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 12:11:30 +0000 Subject: [PATCH 08/65] fix: clippy --- rs/tests/networking/BUILD.bazel | 1 - rs/tests/networking/nns_delegation_test.rs | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/rs/tests/networking/BUILD.bazel b/rs/tests/networking/BUILD.bazel index 14794f8f378b..c86d62a06c3d 100644 --- a/rs/tests/networking/BUILD.bazel +++ b/rs/tests/networking/BUILD.bazel @@ -236,7 +236,6 @@ rust_binary( "//rs/certification", "//rs/crypto/tree_hash", "//rs/crypto/utils/threshold_sig_der", - "//rs/protobuf", "//rs/registry/subnet_type", "//rs/tests/consensus/utils", "//rs/tests/driver:ic-system-test-driver", diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 4febb531a330..2c2f2b0129c0 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -46,12 +46,11 @@ use ic_consensus_system_test_utils::{ }; use ic_crypto_tree_hash::{LabeledTree, lookup_path}; use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; -use ic_protobuf::registry::node::v1::NodeRewardType; use ic_registry_subnet_type::SubnetType; use ic_system_test_driver::{ driver::{ group::SystemTestGroup, - ic::{InternetComputer, Node, Subnet}, + ic::{InternetComputer, Subnet}, test_env::{HasIcPrepDir, TestEnv}, test_env_api::{ HasPublicApiUrl, HasTopologySnapshot, IcNodeContainer, IcNodeSnapshot, SubnetSnapshot, From d359111c4d2f11876613103dcad5b56101618a02 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 12:27:39 +0000 Subject: [PATCH 09/65] fix: non-empty subnet --- rs/tests/networking/nns_delegation_test.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 2c2f2b0129c0..064fc3d72112 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -118,7 +118,9 @@ fn setup(env: TestEnv) { .add_subnet( Subnet::fast_single_node(SubnetType::Application).with_dkg_interval_length(DKG_LENGTH), ) - .add_subnet(Subnet::new(SubnetType::CloudEngine).with_dkg_interval_length(DKG_LENGTH)) + .add_subnet( + Subnet::fast_single_node(SubnetType::CloudEngine).with_dkg_interval_length(DKG_LENGTH), + ) .setup_and_start(&env) .expect("Should be able to set up IC under test"); From b9eb752b44bad46fc31c8b5d54ca157c0d29daea Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 13:26:49 +0000 Subject: [PATCH 10/65] fix: satisfy cost schedule registry invariant --- rs/prep/src/bin/prep.rs | 1 + rs/prep/src/prep_state_directory.rs | 1 + rs/prep/src/subnet_configuration.rs | 7 ++++++- rs/registry/regedit/src/tests.rs | 1 + rs/replica_tests/src/lib.rs | 1 + rs/tests/driver/src/driver/bootstrap.rs | 1 + rs/tests/driver/src/driver/ic.rs | 16 ++++++++++++++++ 7 files changed, 27 insertions(+), 1 deletion(-) diff --git a/rs/prep/src/bin/prep.rs b/rs/prep/src/bin/prep.rs index 6c8d92acb544..311f6d08ab49 100644 --- a/rs/prep/src/bin/prep.rs +++ b/rs/prep/src/bin/prep.rs @@ -208,6 +208,7 @@ fn main() -> Result<()> { /*max_instructions_per_install_code=*/ None, /*features=*/ None, /*chain_key_config=*/ None, + /*canister_cycles_cost_schedule*/ None, /*max_number_of_canisters=*/ None, valid_args.ssh_readonly_access.clone(), valid_args.ssh_backup_access.clone(), diff --git a/rs/prep/src/prep_state_directory.rs b/rs/prep/src/prep_state_directory.rs index d492ce9b39a1..8a13d8730118 100755 --- a/rs/prep/src/prep_state_directory.rs +++ b/rs/prep/src/prep_state_directory.rs @@ -128,6 +128,7 @@ mod tests { None, None, None, + None, vec![], vec![], SubnetRunningState::Active, diff --git a/rs/prep/src/subnet_configuration.rs b/rs/prep/src/subnet_configuration.rs index 476663a7c858..7d412caf3224 100644 --- a/rs/prep/src/subnet_configuration.rs +++ b/rs/prep/src/subnet_configuration.rs @@ -112,6 +112,9 @@ pub struct SubnetConfig { /// Optional chain key configuration for this subnet. pub chain_key_config: Option, + /// The cost schedule for canister execution on this subnet. + pub canister_cycles_cost_schedule: CanisterCyclesCostSchedule, + /// The number of canisters allowed to be created on this subnet. pub max_number_of_canisters: u64, @@ -248,6 +251,7 @@ impl SubnetConfig { max_instructions_per_install_code: Option, features: Option, chain_key_config: Option, + canister_cycles_cost_schedule: Option, max_number_of_canisters: Option, ssh_readonly_access: Vec, ssh_backup_access: Vec, @@ -283,6 +287,7 @@ impl SubnetConfig { .unwrap_or_else(|| scheduler_config.max_instructions_per_install_code.get()), features: features.unwrap_or_default(), chain_key_config, + canister_cycles_cost_schedule: canister_cycles_cost_schedule.unwrap_or_default(), max_number_of_canisters: max_number_of_canisters.unwrap_or(0), ssh_readonly_access, ssh_backup_access, @@ -340,7 +345,7 @@ impl SubnetConfig { ssh_readonly_access: self.ssh_readonly_access, ssh_backup_access: self.ssh_backup_access, chain_key_config: self.chain_key_config, - canister_cycles_cost_schedule: CanisterCyclesCostSchedule::Normal as i32, + canister_cycles_cost_schedule: self.canister_cycles_cost_schedule.into(), subnet_admins: vec![], resource_limits: Default::default(), recalled_replica_version_ids: vec![], diff --git a/rs/registry/regedit/src/tests.rs b/rs/registry/regedit/src/tests.rs index 647674650803..d0abda88eaa9 100644 --- a/rs/registry/regedit/src/tests.rs +++ b/rs/registry/regedit/src/tests.rs @@ -161,6 +161,7 @@ pub fn run_ic_prep() -> (TempDir, IcPrepStateDir) { None, None, None, + None, vec![], vec![], SubnetRunningState::Active, diff --git a/rs/replica_tests/src/lib.rs b/rs/replica_tests/src/lib.rs index 9ff3b47b60f0..44ab497823e4 100644 --- a/rs/replica_tests/src/lib.rs +++ b/rs/replica_tests/src/lib.rs @@ -243,6 +243,7 @@ pub fn get_ic_config() -> IcConfig { /*max_instructions_per_install_code=*/ None, /*features=*/ None, /*chain_key_config=*/ None, + /*canister_cycles_cost_schedule*/ None, /*max_number_of_canisters=*/ None, /*ssh_readonly_access=*/ vec![], /*ssh_backup_access=*/ vec![], diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 8227128c5bc1..ab2c08aaed29 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -168,6 +168,7 @@ pub fn init_ic( subnet.max_instructions_per_install_code, subnet.features, subnet.chain_key_config.clone().map(|c| c.into()), + subnet.canister_cycles_cost_schedule.map(|s| s.into()), subnet.max_number_of_canisters, subnet.ssh_readonly_access.clone(), subnet.ssh_backup_access.clone(), diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index 80678fc9ae8f..d5ff603cc4e4 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -19,6 +19,7 @@ use ic_registry_subnet_features::{ChainKeyConfig, SubnetFeatures}; use ic_registry_subnet_type::SubnetType; use ic_types::malicious_behavior::MaliciousBehavior; use ic_types::{Height, NodeId, PrincipalId}; +use ic_types_cycles::CanisterCyclesCostSchedule; use phantom_newtype::AmountOf; use serde::{Deserialize, Serialize}; use slog::info; @@ -479,6 +480,7 @@ pub struct Subnet { pub max_instructions_per_round: Option, pub max_instructions_per_install_code: Option, pub features: Option, + pub canister_cycles_cost_schedule: Option, pub max_number_of_canisters: Option, pub ssh_readonly_access: Vec, pub ssh_backup_access: Vec, @@ -490,6 +492,10 @@ pub struct Subnet { impl Subnet { pub fn new(subnet_type: SubnetType) -> Self { + let canister_cycles_cost_schedule = match subnet_type { + SubnetType::CloudEngine => Some(CanisterCyclesCostSchedule::Free), + _ => None, + }; Self { vm_resource_overrides: Default::default(), vm_allocation: Default::default(), @@ -508,6 +514,7 @@ impl Subnet { max_instructions_per_round: None, max_instructions_per_install_code: None, features: None, + canister_cycles_cost_schedule, max_number_of_canisters: None, subnet_type, ssh_readonly_access: vec![], @@ -679,6 +686,14 @@ impl Subnet { self } + pub fn with_canister_cycles_cost_schedule( + mut self, + schedule: CanisterCyclesCostSchedule, + ) -> Self { + self.canister_cycles_cost_schedule = Some(schedule); + self + } + pub fn with_max_number_of_canisters(mut self, max_number_of_canisters: u64) -> Self { self.max_number_of_canisters = Some(max_number_of_canisters); self @@ -774,6 +789,7 @@ impl Default for Subnet { max_instructions_per_round: None, max_instructions_per_install_code: None, features: None, + canister_cycles_cost_schedule: None, max_number_of_canisters: None, ssh_readonly_access: vec![], ssh_backup_access: vec![], From 91281b6ef90b8adafa953a86483bbae50a045009 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 13:44:04 +0000 Subject: [PATCH 11/65] fix: install canister on cloud engine --- rs/tests/networking/nns_delegation_test.rs | 39 +++++++++++----------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 064fc3d72112..b8e3f38f06ed 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -128,28 +128,29 @@ fn setup(env: TestEnv) { info!( env.logger(), - "Installing certified variables canister on the Application subnet" + "Installing certified variables canister on the application subnet and cloud engine" ); - let (_subnet, node) = get_subnet_and_node(&env, SubnetType::Application); - let agent = node.build_default_agent(); - let management_canister = ManagementCanister::create(&agent); let wasm = wat::parse_str(CERTIFIED_VAR_WAT).expect("Failed to parse certified variables WAT"); + for subnet_type in [SubnetType::Application, SubnetType::CloudEngine] { + let (_subnet, node) = get_subnet_and_node(&env, subnet_type); + let agent = node.build_default_agent(); + let management_canister = ManagementCanister::create(&agent); + block_on(async { + management_canister + .create_canister() + .as_provisional_create_with_amount(None) + .with_effective_canister_id(node.effective_canister_id()) + .call_and_wait() + .await + .expect("Failed to create the certified variables canister"); - tokio::runtime::Runtime::new().unwrap().block_on(async { - management_canister - .create_canister() - .as_provisional_create_with_amount(None) - .with_effective_canister_id(node.effective_canister_id()) - .call_and_wait() - .await - .expect("Failed to create the certified variables canister"); - - management_canister - .install_code(&node.effective_canister_id().0, &wasm) - .call_and_wait() - .await - .expect("Failed to install the certified variables canister"); - }); + management_canister + .install_code(&node.effective_canister_id().0, &wasm) + .call_and_wait() + .await + .expect("Failed to install the certified variables canister"); + }); + } upgrade_non_nns_subnets_if_necessary(&env); } From bcb57b482ad086180df4109370121255bbd41be3 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 13:44:43 +0000 Subject: [PATCH 12/65] fix: install canister on NNS --- rs/tests/networking/BUILD.bazel | 1 + rs/tests/networking/nns_delegation_test.rs | 123 +++++++++++++-------- 2 files changed, 76 insertions(+), 48 deletions(-) diff --git a/rs/tests/networking/BUILD.bazel b/rs/tests/networking/BUILD.bazel index c86d62a06c3d..119c1e90a6ec 100644 --- a/rs/tests/networking/BUILD.bazel +++ b/rs/tests/networking/BUILD.bazel @@ -243,6 +243,7 @@ rust_binary( "//rs/universal_canister/lib", "@crate_index//:anyhow", "@crate_index//:candid", + "@crate_index//:futures", "@crate_index//:ic-agent", "@crate_index//:ic-utils", "@crate_index//:leb128", diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index b8e3f38f06ed..9bd099c7b6fb 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -28,6 +28,7 @@ Success:: */ use std::{ borrow::Cow, + collections::BTreeMap, time::{Duration, SystemTime}, }; @@ -104,11 +105,23 @@ const CERTIFIED_VAR_WAT: &str = r#" ) "#; +const INSTALLED_CANISTER_IDS: &str = "installed_canister_ids"; + /// How long to wait between subsequent nns delegation fetch requests. const RETRY_DELAY: tokio::time::Duration = tokio::time::Duration::from_secs(60); const DKG_LENGTH: Height = Height::new(9); +fn get_installed_canister_ids(env: &TestEnv) -> BTreeMap { + env.read_json_object(INSTALLED_CANISTER_IDS) + .expect("Could not read installed canister IDs from test environment.") +} + +fn set_installed_canister_ids(env: &TestEnv, canister_ids: BTreeMap) { + env.write_json_object(INSTALLED_CANISTER_IDS, &canister_ids) + .expect("Could not write installed canister IDs to test environment."); +} + fn setup(env: TestEnv) { InternetComputer::new() .with_api_boundary_nodes(1) @@ -128,29 +141,45 @@ fn setup(env: TestEnv) { info!( env.logger(), - "Installing certified variables canister on the application subnet and cloud engine" + "Installing certified variables canister on all subnets" ); let wasm = wat::parse_str(CERTIFIED_VAR_WAT).expect("Failed to parse certified variables WAT"); - for subnet_type in [SubnetType::Application, SubnetType::CloudEngine] { - let (_subnet, node) = get_subnet_and_node(&env, subnet_type); - let agent = node.build_default_agent(); - let management_canister = ManagementCanister::create(&agent); - block_on(async { - management_canister - .create_canister() - .as_provisional_create_with_amount(None) - .with_effective_canister_id(node.effective_canister_id()) - .call_and_wait() - .await - .expect("Failed to create the certified variables canister"); + let canister_ids = block_on(futures::future::join_all( + [ + SubnetType::System, + SubnetType::Application, + SubnetType::CloudEngine, + ] + .map(|subnet_type| { + let env = env.clone(); + let wasm = wasm.clone(); + async move { + let (_subnet, node) = get_subnet_and_node(&env, subnet_type); + let agent = node.build_default_agent_async().await; + let management_canister = ManagementCanister::create(&agent); + let canister_id = management_canister + .create_canister() + .as_provisional_create_with_amount(None) + .with_effective_canister_id(node.effective_canister_id()) + .call_and_wait() + .await + .expect("Failed to create the certified variables canister") + .0; + + management_canister + .install_code(&canister_id, &wasm) + .call_and_wait() + .await + .expect("Failed to install the certified variables canister"); + + (subnet_type, PrincipalId::from(canister_id)) + } + }), + )) + .into_iter() + .collect(); - management_canister - .install_code(&node.effective_canister_id().0, &wasm) - .call_and_wait() - .await - .expect("Failed to install the certified variables canister"); - }); - } + set_installed_canister_ids(&env, canister_ids); upgrade_non_nns_subnets_if_necessary(&env); } @@ -296,14 +325,12 @@ fn subnet_read_state_v3_returns_correct_delegation(env: TestEnv, subnet_type: Su /// Responses to `api/v2/canister/{canister_id}/read_state` have valid delegations with canister ranges in the flat format. fn canister_read_state_v2_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_ids(&env)[&subnet_type]; let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, - format!( - "api/v2/canister/{}/read_state", - node.effective_canister_id() - ), + format!("api/v2/canister/{canister_id}/read_state"), sign_envelope(&read_state_content()), )); let certificate: Certificate = serde_cbor::from_slice(&response.certificate).unwrap(); @@ -313,21 +340,19 @@ fn canister_read_state_v2_returns_correct_delegation(env: TestEnv, subnet_type: certificate.delegation.as_ref(), subnet.subnet_id, subnet_type, - Some(node.effective_canister_id()), + Some(canister_id), CertificateDelegationFormat::Flat, ); } /// Responses to `api/v3/canister/{canister_id}/read_state` have valid delegations with canister ranges in the flat format. fn canister_read_state_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_ids(&env)[&subnet_type]; let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( &node, - format!( - "api/v3/canister/{}/read_state", - node.effective_canister_id() - ), + format!("api/v3/canister/{canister_id}/read_state"), sign_envelope(&read_state_content()), )); let certificate: Certificate = serde_cbor::from_slice(&response.certificate).unwrap(); @@ -337,7 +362,7 @@ fn canister_read_state_v3_returns_correct_delegation(env: TestEnv, subnet_type: certificate.delegation.as_ref(), subnet.subnet_id, subnet_type, - Some(node.effective_canister_id()), + Some(canister_id), CertificateDelegationFormat::Tree, ); } @@ -374,12 +399,13 @@ struct SyncCallResponse { /// Responses to `api/v3/canister/{canister_id}/call` have valid delegations with canister ranges in the flat format. fn call_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_ids(&env)[&subnet_type]; let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: SyncCallResponse = block_on(send( &node, - format!("api/v3/canister/{}/call", node.effective_canister_id()), - sign_envelope(&call_content(node.effective_canister_id())), + format!("api/v3/canister/{canister_id}/call"), + sign_envelope(&call_content(canister_id)), )); let certificate: Certificate = serde_cbor::from_slice(&response.certificate).unwrap(); @@ -388,19 +414,20 @@ fn call_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { certificate.delegation.as_ref(), subnet.subnet_id, subnet_type, - Some(node.effective_canister_id()), + Some(canister_id), CertificateDelegationFormat::Flat, ); } /// Responses to `api/v4/canister/{canister_id}/call` have valid delegations with canister ranges in the flat format. fn call_v4_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_ids(&env)[&subnet_type]; let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: SyncCallResponse = block_on(send( &node, - format!("api/v4/canister/{}/call", node.effective_canister_id()), - sign_envelope(&call_content(node.effective_canister_id())), + format!("api/v4/canister/{canister_id}/call"), + sign_envelope(&call_content(canister_id)), )); let certificate: Certificate = serde_cbor::from_slice(&response.certificate).unwrap(); @@ -409,7 +436,7 @@ fn call_v4_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { certificate.delegation.as_ref(), subnet.subnet_id, subnet_type, - Some(node.effective_canister_id()), + Some(canister_id), CertificateDelegationFormat::Tree, ); } @@ -417,6 +444,7 @@ fn call_v4_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { /// Responses to `api/v4/canister/{canister_id}/call` targeting the management canister /// have valid delegations without canister ranges. fn call_v4_management_canister_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_ids(&env)[&subnet_type]; let (subnet, node) = get_subnet_and_node(&env, subnet_type); let expiration = OffsetDateTime::now_utc() + Duration::from_secs(3 * 60); @@ -431,16 +459,13 @@ fn call_v4_management_canister_returns_correct_delegation(env: TestEnv, subnet_t sender: get_identity().sender().unwrap(), canister_id: CanisterId::ic_00().into(), method_name: String::from("start_canister"), - arg: Encode!(&Arg { - canister_id: node.effective_canister_id() - }) - .unwrap(), + arg: Encode!(&Arg { canister_id }).unwrap(), nonce: None, }; let response: SyncCallResponse = block_on(send( &node, - format!("api/v4/canister/{}/call", node.effective_canister_id()), + format!("api/v4/canister/{canister_id}/call"), sign_envelope(&call_content), )); let certificate: Certificate = serde_cbor::from_slice(&response.certificate).unwrap(); @@ -450,7 +475,7 @@ fn call_v4_management_canister_returns_correct_delegation(env: TestEnv, subnet_t certificate.delegation.as_ref(), subnet.subnet_id, subnet_type, - Some(node.effective_canister_id()), + Some(canister_id), CertificateDelegationFormat::Tree, ); } @@ -465,13 +490,14 @@ struct QueryResponse { /// For `api/v2/canister/{canister_id}/query` we pass valid delegations with /// canister ranges in the flat format to the canister. fn query_v2_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_ids(&env)[&subnet_type]; let (subnet, node) = get_subnet_and_node(&env, subnet_type); let arg = vec![]; let response: QueryResponse = block_on(send( &node, - format!("api/v2/canister/{}/query", node.effective_canister_id()), - sign_envelope(&query_content(node.effective_canister_id(), arg)), + format!("api/v2/canister/{canister_id}/query"), + sign_envelope(&query_content(canister_id, arg)), )); let certificate: Certificate = serde_cbor::from_slice(&response.reply.arg).unwrap(); @@ -480,7 +506,7 @@ fn query_v2_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: Sub certificate.delegation.as_ref(), subnet.subnet_id, subnet_type, - Some(node.effective_canister_id()), + Some(canister_id), CertificateDelegationFormat::Flat, ); } @@ -488,13 +514,14 @@ fn query_v2_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: Sub /// For `api/v3/canister/{canister_id}/query` we pass valid delegations with /// canister ranges in the tree format to the canister. fn query_v3_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: SubnetType) { + let canister_id = get_installed_canister_ids(&env)[&subnet_type]; let (subnet, node) = get_subnet_and_node(&env, subnet_type); let arg = vec![]; let response: QueryResponse = block_on(send( &node, - format!("api/v3/canister/{}/query", node.effective_canister_id()), - sign_envelope(&query_content(node.effective_canister_id(), arg)), + format!("api/v3/canister/{canister_id}/query"), + sign_envelope(&query_content(canister_id, arg)), )); let certificate: Certificate = serde_cbor::from_slice(&response.reply.arg).unwrap(); @@ -503,7 +530,7 @@ fn query_v3_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: Sub certificate.delegation.as_ref(), subnet.subnet_id, subnet_type, - Some(node.effective_canister_id()), + Some(canister_id), CertificateDelegationFormat::Tree, ); } From 0a4de55725e58ced8c245af35b7ffc9eabbfc221 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 16:17:16 +0000 Subject: [PATCH 13/65] feat: works when DNS and TLS certs are setup --- .../src/nns_delegation_manager.rs | 50 ++++++++++++++++- rs/tests/networking/nns_delegation_test.rs | 55 +++++++++++++++++-- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 6c1f5c62a6dd..5505092b1e0a 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -28,6 +28,11 @@ use ic_types::{ time::expiry_time_from_now, }; use rand::Rng; +use rustls::{ + DigitallySignedStruct, SignatureScheme, + client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, + pki_types::{CertificateDer, ServerName, UnixTime}, +}; use tokio::{ net::TcpStream, sync::watch, @@ -521,15 +526,54 @@ async fn connect_to_nns_node( Ok(tls_stream) } +#[derive(Debug)] +struct NoVerify; +impl ServerCertVerifier for NoVerify { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer, + _intermediates: &[CertificateDer], + _server_name: &ServerName, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + fn verify_tls12_signature( + &self, + _: &[u8], + _: &CertificateDer<'_>, + _: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + fn verify_tls13_signature( + &self, + _: &[u8], + _: &CertificateDer<'_>, + _: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} + async fn connect_to_api_bn( log: &ReplicaLogger, api_bn_id: NodeId, domain: String, ) -> Result, BoxError> { - let mut root_store = rustls::RootCertStore::empty(); - root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + // TODO: Uncomment me + // let mut root_store = rustls::RootCertStore::empty(); + // root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); let tls_client_config = rustls::ClientConfig::builder() - .with_root_certificates(root_store) + // .with_root_certificates(root_store) + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoVerify)) .with_no_client_auth(); let addr = (domain.as_str(), 443_u16); diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 9bd099c7b6fb..bc11613659b4 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -54,12 +54,12 @@ use ic_system_test_driver::{ ic::{InternetComputer, Subnet}, test_env::{HasIcPrepDir, TestEnv}, test_env_api::{ - HasPublicApiUrl, HasTopologySnapshot, IcNodeContainer, IcNodeSnapshot, SubnetSnapshot, - get_guestos_img_version, get_guestos_update_img_sha256, get_guestos_update_img_url, - get_guestos_update_img_version, + HasPublicApiUrl, HasTopologySnapshot, IcNodeContainer, IcNodeSnapshot, SshSession, + SubnetSnapshot, get_guestos_img_version, get_guestos_update_img_sha256, + get_guestos_update_img_url, get_guestos_update_img_version, }, }, - systest, + retry_with_msg, systest, util::{block_on, get_identity, get_nns_node}, }; use ic_types::{ @@ -137,6 +137,53 @@ fn setup(env: TestEnv) { .setup_and_start(&env) .expect("Should be able to set up IC under test"); + // HACK: Overwrite the DNS of all nodes to point the API BNs' domains to their IPs + for node in env + .topology_snapshot() + .subnets() + .flat_map(|subnet| subnet.nodes()) + { + let mappings = env + .topology_snapshot() + .api_boundary_nodes() + .map(|api_bn| (api_bn.get_domain().unwrap(), api_bn.get_ip_addr())) + .collect::>(); + + // File-system is read-only, so we modify /etc/hosts in a temporary file and replace the + // original with a bind mount. + let mut command = String::from( + r#" + set -euo pipefail + sudo cp /etc/hosts /tmp/hosts + "#, + ); + for (domain, ip) in mappings { + command.push_str(&format!( + r#" + echo "{ip} {domain}" | sudo tee -a /tmp/hosts > /dev/null + "# + )); + } + // Match the original /etc/hosts file permissions. + command.push_str( + r#" + sudo chown --reference=/etc/hosts /tmp/hosts + sudo chmod --reference=/etc/hosts /tmp/hosts + + sudo mount --bind /tmp/hosts /etc/hosts + "#, + ); + + retry_with_msg!( + format!("Overwrite /etc/hosts on node {}", node.node_id), + env.logger(), + Duration::from_secs(60), + Duration::from_secs(5), + || node.block_on_bash_script(&command) + ) + .expect("Should be able to overwrite /etc/hosts on the node"); + } + install_nns_and_check_progress(env.topology_snapshot()); info!( From d93c932feb01c98c44b79606b266f772cb54d1a0 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 16:44:29 +0000 Subject: [PATCH 14/65] docs --- rs/tests/driver/src/driver/ic.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index 5c322fc01ccb..9d39a2a30a60 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -492,6 +492,7 @@ pub struct Subnet { impl Subnet { pub fn new(subnet_type: SubnetType) -> Self { + // An invariant in the registry ensures that cloud engines have a free cost schedule let canister_cycles_cost_schedule = match subnet_type { SubnetType::CloudEngine => CanisterCyclesCostSchedule::Free, _ => CanisterCyclesCostSchedule::Normal, From d013ae7fcb7aac25c2ac9c75bc223c488e0899b4 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 16:48:37 +0000 Subject: [PATCH 15/65] fix: clippy --- rs/tests/nns/BUILD.bazel | 1 - rs/tests/nns/delete_subnet_test.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/rs/tests/nns/BUILD.bazel b/rs/tests/nns/BUILD.bazel index 7465b47766d3..025bf8da128a 100644 --- a/rs/tests/nns/BUILD.bazel +++ b/rs/tests/nns/BUILD.bazel @@ -75,7 +75,6 @@ system_test( "//rs/tests/consensus/utils", "//rs/tests/driver:ic-system-test-driver", "//rs/types/base_types", - "//rs/types/cycles", "//rs/types/types", "//rs/universal_canister/lib", "@crate_index//:anyhow", diff --git a/rs/tests/nns/delete_subnet_test.rs b/rs/tests/nns/delete_subnet_test.rs index 9f3bd02606ae..f84639fad68d 100644 --- a/rs/tests/nns/delete_subnet_test.rs +++ b/rs/tests/nns/delete_subnet_test.rs @@ -22,7 +22,6 @@ use ic_system_test_driver::nns::get_subnet_list_from_registry; use ic_system_test_driver::systest; use ic_system_test_driver::util::{UniversalCanister, assert_create_agent, block_on}; use ic_types::{Height, RegistryVersion, SubnetId}; -use ic_types_cycles::CanisterCyclesCostSchedule; use registry_canister::init::RegistryCanisterInitPayloadBuilder; use registry_canister::mutations::do_delete_subnet::DeleteSubnetPayload; use std::collections::BTreeSet; From 613fb55a13fd5fd9cd4277788eb1f72cd5a9caa2 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 16:55:16 +0000 Subject: [PATCH 16/65] test: ensure free cost schedule --- rs/tests/driver/src/driver/ic.rs | 7 ++++++- rs/tests/nns/BUILD.bazel | 1 - rs/tests/nns/delete_subnet_test.rs | 4 +--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index d89ce5a82e72..8205876e8f24 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -491,6 +491,11 @@ pub struct Subnet { impl Subnet { pub fn new(subnet_type: SubnetType) -> Self { + // An invariant in the registry ensures that cloud engines have a free cost schedule + let canister_cycles_cost_schedule = match subnet_type { + SubnetType::CloudEngine => CanisterCyclesCostSchedule::Free, + _ => CanisterCyclesCostSchedule::Normal, + }; Self { vm_resource_overrides: Default::default(), vm_allocation: Default::default(), @@ -511,7 +516,7 @@ impl Subnet { features: None, max_number_of_canisters: None, subnet_type, - canister_cycles_cost_schedule: CanisterCyclesCostSchedule::Normal, + canister_cycles_cost_schedule, ssh_readonly_access: vec![], ssh_backup_access: vec![], chain_key_config: None, diff --git a/rs/tests/nns/BUILD.bazel b/rs/tests/nns/BUILD.bazel index 7465b47766d3..025bf8da128a 100644 --- a/rs/tests/nns/BUILD.bazel +++ b/rs/tests/nns/BUILD.bazel @@ -75,7 +75,6 @@ system_test( "//rs/tests/consensus/utils", "//rs/tests/driver:ic-system-test-driver", "//rs/types/base_types", - "//rs/types/cycles", "//rs/types/types", "//rs/universal_canister/lib", "@crate_index//:anyhow", diff --git a/rs/tests/nns/delete_subnet_test.rs b/rs/tests/nns/delete_subnet_test.rs index f497cd57e1d7..f84639fad68d 100644 --- a/rs/tests/nns/delete_subnet_test.rs +++ b/rs/tests/nns/delete_subnet_test.rs @@ -22,7 +22,6 @@ use ic_system_test_driver::nns::get_subnet_list_from_registry; use ic_system_test_driver::systest; use ic_system_test_driver::util::{UniversalCanister, assert_create_agent, block_on}; use ic_types::{Height, RegistryVersion, SubnetId}; -use ic_types_cycles::CanisterCyclesCostSchedule; use registry_canister::init::RegistryCanisterInitPayloadBuilder; use registry_canister::mutations::do_delete_subnet::DeleteSubnetPayload; use std::collections::BTreeSet; @@ -56,8 +55,7 @@ pub fn setup(env: TestEnv) { ) .add_subnet( Subnet::fast(SubnetType::CloudEngine, NUM_ENGINE_NODES) - .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)) - .with_cost_schedule(CanisterCyclesCostSchedule::Free), + .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), ) .setup_and_start(&env) .expect("failed to setup IC under test"); From 81030b7526743a8241926b96eed826ffe7012f9e Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 16:55:30 +0000 Subject: [PATCH 17/65] test: ensure type4 node reward types --- rs/tests/driver/src/driver/bootstrap.rs | 2 +- rs/tests/driver/src/driver/ic.rs | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index c3ad48185c71..1d102ba0fd4b 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -615,7 +615,7 @@ fn node_to_config(node: &Node) -> NodeConfiguration { node_operator_principal_id: None, secret_key_store: node.secret_key_store.clone(), domain: node.domain.clone(), - node_reward_type: None, + node_reward_type: node.node_reward_type.map(String::from), } } diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index 8205876e8f24..531863d22d77 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -12,6 +12,7 @@ use crate::driver::{ use anyhow::Result; use ic_prep_lib::prep_state_directory::IcPrepStateDir; use ic_prep_lib::{node::NodeSecretKeyStore, subnet_configuration::SubnetRunningState}; +use ic_protobuf::registry::node::v1::NodeRewardType; use ic_regedit; use ic_registry_canister_api::IPv4Config; use ic_registry_subnet_features::{ChainKeyConfig, SubnetFeatures}; @@ -622,7 +623,12 @@ impl Subnet { }) } - pub fn add_node(mut self, node: Node) -> Self { + pub fn add_node(mut self, mut node: Node) -> Self { + // If the subnet is a cloud engine, ensure that all nodes have reward type 4 + if self.subnet_type == SubnetType::CloudEngine { + node = node.with_node_reward_type(NodeRewardType::Type4); + } + self.nodes.push(node); self } @@ -865,6 +871,7 @@ pub struct Node { pub malicious_behavior: Option, pub ipv4: Option, pub domain: Option, + pub node_reward_type: Option, pub recovery_hash: Option, pub boot_image: BootImage, } @@ -919,6 +926,11 @@ impl Node { self } + pub fn with_node_reward_type(mut self, node_reward_type: NodeRewardType) -> Self { + self.node_reward_type = Some(node_reward_type); + self + } + pub fn with_recovery_hash(mut self, recovery_hash: String) -> Self { self.recovery_hash = Some(recovery_hash); self From 38c5891ad6f72a919550a346a652ecb3eee94b40 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 18:36:40 +0000 Subject: [PATCH 18/65] test: add API BNs --- rs/tests/driver/src/driver/ic.rs | 13 +++++++++++++ .../xnet/xnet_cloud_engine_isolation_test.rs | 1 + rs/tests/nns/delete_subnet_test.rs | 1 + 3 files changed, 15 insertions(+) diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index 531863d22d77..b4cd4d5c50eb 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -226,6 +226,19 @@ impl InternetComputer { &mut self, env: &TestEnv, ) -> Result> { + if self + .subnets + .iter() + .any(|s| s.subnet_type == SubnetType::CloudEngine) + { + // Cloud engines use API boundary nodes to replicate their registry and to fetch + // delegations since the firewall on the NNS blocks them. + assert!( + !self.api_boundary_nodes.is_empty(), + "At least one API boundary node is required when using a cloud engine subnet" + ); + } + // propagate required host features and resource settings to all vms let farm = Farm::from_test_env(env, "Internet Computer"); for node in self diff --git a/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs b/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs index 78f2a3db608f..65c451bb95c0 100644 --- a/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs +++ b/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs @@ -62,6 +62,7 @@ fn main() -> Result<()> { fn setup(env: TestEnv) { InternetComputer::new() + .with_api_boundary_nodes(1) .add_subnet(Subnet::fast_single_node(SubnetType::System)) .add_subnet(Subnet::fast_single_node(SubnetType::Application)) .add_subnet(Subnet::fast_single_node(SubnetType::CloudEngine)) diff --git a/rs/tests/nns/delete_subnet_test.rs b/rs/tests/nns/delete_subnet_test.rs index f84639fad68d..35c9d4fbc158 100644 --- a/rs/tests/nns/delete_subnet_test.rs +++ b/rs/tests/nns/delete_subnet_test.rs @@ -41,6 +41,7 @@ fn main() -> Result<()> { pub fn setup(env: TestEnv) { InternetComputer::new() + .with_api_boundary_nodes(1) .add_subnet( Subnet::fast(SubnetType::System, NUM_NODES) .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), From 63af301b266e6506815c1c9435a6bf21a3cb2d33 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Thu, 26 Mar 2026 18:50:29 +0000 Subject: [PATCH 19/65] fix: fix mainnet variant --- rs/tests/networking/nns_delegation_test.rs | 106 ++++++++++++--------- 1 file changed, 59 insertions(+), 47 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index bc11613659b4..fc4633b7f156 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -60,7 +60,7 @@ use ic_system_test_driver::{ }, }, retry_with_msg, systest, - util::{block_on, get_identity, get_nns_node}, + util::{EndpointsStatus, assert_nodes_health_statuses, block_on, get_identity, get_nns_node}, }; use ic_types::{ CanisterId, Height, PrincipalId, SubnetId, @@ -137,52 +137,7 @@ fn setup(env: TestEnv) { .setup_and_start(&env) .expect("Should be able to set up IC under test"); - // HACK: Overwrite the DNS of all nodes to point the API BNs' domains to their IPs - for node in env - .topology_snapshot() - .subnets() - .flat_map(|subnet| subnet.nodes()) - { - let mappings = env - .topology_snapshot() - .api_boundary_nodes() - .map(|api_bn| (api_bn.get_domain().unwrap(), api_bn.get_ip_addr())) - .collect::>(); - - // File-system is read-only, so we modify /etc/hosts in a temporary file and replace the - // original with a bind mount. - let mut command = String::from( - r#" - set -euo pipefail - sudo cp /etc/hosts /tmp/hosts - "#, - ); - for (domain, ip) in mappings { - command.push_str(&format!( - r#" - echo "{ip} {domain}" | sudo tee -a /tmp/hosts > /dev/null - "# - )); - } - // Match the original /etc/hosts file permissions. - command.push_str( - r#" - sudo chown --reference=/etc/hosts /tmp/hosts - sudo chmod --reference=/etc/hosts /tmp/hosts - - sudo mount --bind /tmp/hosts /etc/hosts - "#, - ); - - retry_with_msg!( - format!("Overwrite /etc/hosts on node {}", node.node_id), - env.logger(), - Duration::from_secs(60), - Duration::from_secs(5), - || node.block_on_bash_script(&command) - ) - .expect("Should be able to overwrite /etc/hosts on the node"); - } + hack(&env); install_nns_and_check_progress(env.topology_snapshot()); @@ -844,6 +799,14 @@ fn upgrade_non_nns_subnets_if_necessary(env: &TestEnv) { cloud_engine.subnet_id, )); + std::thread::sleep(Duration::from_secs(90)); + assert_nodes_health_statuses( + env.logger(), + &cloud_engine.nodes().collect::>(), + EndpointsStatus::AllUnhealthy, + ); + hack(env); + assert_assigned_replica_version(&app_node, &target_version, env.logger()); assert_assigned_replica_version(&cloud_engine_node, &target_version, env.logger()); } @@ -891,3 +854,52 @@ fn main() -> Result<()> { test_group.execute_from_args() } + +// HACK: Overwrite the DNS of all nodes to point the API BNs' domains to their IPs +fn hack(env: &TestEnv) { + for node in env + .topology_snapshot() + .subnets() + .flat_map(|subnet| subnet.nodes()) + { + let mappings = env + .topology_snapshot() + .api_boundary_nodes() + .map(|api_bn| (api_bn.get_domain().unwrap(), api_bn.get_ip_addr())) + .collect::>(); + + // File-system is read-only, so we modify /etc/hosts in a temporary file and replace the + // original with a bind mount. + let mut command = String::from( + r#" + set -euo pipefail + sudo cp /etc/hosts /tmp/hosts + "#, + ); + for (domain, ip) in mappings { + command.push_str(&format!( + r#" + echo "{ip} {domain}" | sudo tee -a /tmp/hosts > /dev/null + "# + )); + } + // Match the original /etc/hosts file permissions. + command.push_str( + r#" + sudo chown --reference=/etc/hosts /tmp/hosts + sudo chmod --reference=/etc/hosts /tmp/hosts + + sudo mount --bind /tmp/hosts /etc/hosts + "#, + ); + + retry_with_msg!( + format!("Overwrite /etc/hosts on node {}", node.node_id), + env.logger(), + Duration::from_secs(60), + Duration::from_secs(5), + || node.block_on_bash_script(&command) + ) + .expect("Should be able to overwrite /etc/hosts on the node"); + } +} From ae279033a5fe5d1db30915d0ec2670445ca127a1 Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 26 Mar 2026 19:37:16 +0000 Subject: [PATCH 20/65] Create IcBoundaryTlsCert config object and update generate_ic_config to write tls --- .../tool/src/guestos/generate_ic_config.rs | 36 ++++++++----- .../fixtures/guestos_v1.15.0.json | 52 +++++++++++++++++++ .../fixtures/hostos_v1.15.0.json | 52 +++++++++++++++++++ rs/ic_os/config/types/src/lib.rs | 16 +++++- rs/tests/driver/src/driver/bootstrap.rs | 1 + 5 files changed, 143 insertions(+), 14 deletions(-) create mode 100644 rs/ic_os/config/types/compatibility_tests/fixtures/guestos_v1.15.0.json create mode 100644 rs/ic_os/config/types/compatibility_tests/fixtures/hostos_v1.15.0.json diff --git a/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs b/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs index 08e4f21cb4de..ea7c4aed5b58 100644 --- a/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs +++ b/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs @@ -44,12 +44,12 @@ pub fn generate_ic_config(guestos_config: &GuestOSConfig, output_path: &Path) -> perms.set_mode(0o644); std::fs::set_permissions(output_path, perms)?; - // Generate and inject a self-signed TLS certificate and key for ic-boundary - // for the given domain name. To be used in system tests only. - if let Some(domain_name) = &guestos_config - .guestos_settings - .guestos_dev_settings - .generate_ic_boundary_tls_cert + // Set up TLS certificate and key for ic-boundary (system tests only). + // A pre-generated cert (e.g. from a playnet) takes priority over self-signed generation. + let dev_settings = &guestos_config.guestos_settings.guestos_dev_settings; + if let Some(tls_cert) = &dev_settings.ic_boundary_tls_cert { + write_tls_certificate(&tls_cert.cert_pem, &tls_cert.key_pem)?; + } else if let Some(domain_name) = &dev_settings.generate_ic_boundary_tls_cert && !domain_name.is_empty() { generate_tls_certificate(domain_name)?; @@ -235,10 +235,16 @@ pub fn get_best_interface_ipv6_address() -> Result { bail!("Cannot determine an IPv6 address, aborting"); } -fn generate_tls_certificate(domain_name: &str) -> Result<()> { - let tls_key_path = "/var/lib/ic/data/ic-boundary-tls.key"; - let tls_cert_path = "/var/lib/ic/data/ic-boundary-tls.crt"; +const TLS_KEY_PATH: &str = "/var/lib/ic/data/ic-boundary-tls.key"; +const TLS_CERT_PATH: &str = "/var/lib/ic/data/ic-boundary-tls.crt"; + +fn write_tls_certificate(cert_pem: &str, key_pem: &str) -> Result<()> { + write(TLS_CERT_PATH, cert_pem).context("Failed to write TLS certificate")?; + write(TLS_KEY_PATH, key_pem).context("Failed to write TLS key")?; + set_tls_file_ownership_and_permissions() +} +fn generate_tls_certificate(domain_name: &str) -> Result<()> { let status = Command::new("openssl") .args([ "req", @@ -246,9 +252,9 @@ fn generate_tls_certificate(domain_name: &str) -> Result<()> { "-newkey", "rsa:2048", "-keyout", - tls_key_path, + TLS_KEY_PATH, "-out", - tls_cert_path, + TLS_CERT_PATH, "-sha256", "-days", "3650", @@ -265,8 +271,12 @@ fn generate_tls_certificate(domain_name: &str) -> Result<()> { bail!("openssl command failed with status: {}", status); } + set_tls_file_ownership_and_permissions() +} + +fn set_tls_file_ownership_and_permissions() -> Result<()> { let status = Command::new("chown") - .args(["ic-replica:nogroup", tls_key_path, tls_cert_path]) + .args(["ic-replica:nogroup", TLS_KEY_PATH, TLS_CERT_PATH]) .status() .context("Failed to set ownership of TLS files")?; @@ -275,7 +285,7 @@ fn generate_tls_certificate(domain_name: &str) -> Result<()> { } let status = Command::new("chmod") - .args(["644", tls_key_path, tls_cert_path]) + .args(["644", TLS_KEY_PATH, TLS_CERT_PATH]) .status() .context("Failed to set permissions of TLS files")?; diff --git a/rs/ic_os/config/types/compatibility_tests/fixtures/guestos_v1.15.0.json b/rs/ic_os/config/types/compatibility_tests/fixtures/guestos_v1.15.0.json new file mode 100644 index 000000000000..dfba380fd5d1 --- /dev/null +++ b/rs/ic_os/config/types/compatibility_tests/fixtures/guestos_v1.15.0.json @@ -0,0 +1,52 @@ +{ + "config_version": "1.15.0", + "network_settings": { + "ipv6_config": { + "Fixed": { + "address": "2a00:fb01:400:200::1/64", + "gateway": "2a00:fb01:400:200::1" + } + }, + "ipv4_config": { + "address": "192.168.1.1", + "gateway": "192.168.1.254", + "prefix_length": 24 + }, + "domain_name": "ic.test" + }, + "icos_settings": { + "node_reward_type": "type3.1", + "mgmt_mac": "00:00:00:00:00:01", + "deployment_environment": "mainnet", + "nns_urls": [ + "https://icp-api.io,https//icp0.io,https://ic0.app" + ], + "node_operator_private_key": null, + "enable_trusted_execution_environment": false, + "use_ssh_authorized_keys": false, + "icos_dev_settings": {} + }, + "guestos_settings": { + "guestos_dev_settings": { + "backup_spool": null, + "malicious_behavior": null, + "query_stats_epoch_length": null, + "bitcoind_addr": null, + "dogecoind_addr": null, + "jaeger_addr": null, + "socks_proxy": null, + "hostname": null, + "generate_ic_boundary_tls_cert": null, + "ic_boundary_tls_cert": null, + "nns_pub_key_override": null + } + }, + "guest_vm_type": "default", + "upgrade_config": { + "peer_guest_vm_address": "2a00:fb01:400:200:6801:95ff:fed7:d475" + }, + "trusted_execution_environment_config": { + "sev_cert_chain_pem": "-----BEGIN CERTIFICATE----------END CERTIFICATE-----" + }, + "recovery_config": null +} \ No newline at end of file diff --git a/rs/ic_os/config/types/compatibility_tests/fixtures/hostos_v1.15.0.json b/rs/ic_os/config/types/compatibility_tests/fixtures/hostos_v1.15.0.json new file mode 100644 index 000000000000..5dda6713f7f8 --- /dev/null +++ b/rs/ic_os/config/types/compatibility_tests/fixtures/hostos_v1.15.0.json @@ -0,0 +1,52 @@ +{ + "config_version": "1.15.0", + "network_settings": { + "ipv6_config": { + "Fixed": { + "address": "2a00:fb01:400:200::1/64", + "gateway": "2a00:fb01:400:200::1" + } + }, + "ipv4_config": { + "address": "192.168.1.1", + "gateway": "192.168.1.254", + "prefix_length": 24 + }, + "domain_name": "ic.test" + }, + "icos_settings": { + "node_reward_type": "type3.1", + "mgmt_mac": "00:00:00:00:00:01", + "deployment_environment": "mainnet", + "nns_urls": [ + "https://icp-api.io,https//icp0.io,https://ic0.app" + ], + "node_operator_private_key": null, + "enable_trusted_execution_environment": false, + "use_ssh_authorized_keys": false, + "icos_dev_settings": {} + }, + "hostos_settings": { + "hostos_dev_settings": { + "vm_memory": 16, + "vm_cpu": "kvm", + "vm_nr_of_vcpus": 64 + }, + "verbose": false + }, + "guestos_settings": { + "guestos_dev_settings": { + "backup_spool": null, + "malicious_behavior": null, + "query_stats_epoch_length": null, + "bitcoind_addr": null, + "dogecoind_addr": null, + "jaeger_addr": null, + "socks_proxy": null, + "hostname": null, + "generate_ic_boundary_tls_cert": null, + "ic_boundary_tls_cert": null, + "nns_pub_key_override": null + } + } +} \ No newline at end of file diff --git a/rs/ic_os/config/types/src/lib.rs b/rs/ic_os/config/types/src/lib.rs index c1570f484464..6c6e51ee081f 100644 --- a/rs/ic_os/config/types/src/lib.rs +++ b/rs/ic_os/config/types/src/lib.rs @@ -40,7 +40,7 @@ use std::str::FromStr; use strum::{Display, EnumString}; use url::Url; -pub const CONFIG_VERSION: &str = "1.14.0"; +pub const CONFIG_VERSION: &str = "1.15.0"; /// List of field paths that have been removed and should not be reused. pub static RESERVED_FIELD_PATHS: &[&str] = &[ @@ -219,6 +219,16 @@ pub struct GuestOSSettings { pub guestos_dev_settings: GuestOSDevSettings, } +/// Pre-generated TLS certificate and key for ic-boundary. +/// Used to inject a real (e.g. playnet) certificate instead of a self-signed one. +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] +pub struct IcBoundaryTlsCert { + /// PEM-encoded certificate (leaf + chain concatenated). + pub cert_pem: String, + /// PEM-encoded private key. + pub key_pem: String, +} + /// GuestOS development configuration. These settings are strictly used for development images. #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Default, Clone)] pub struct GuestOSDevSettings { @@ -234,6 +244,10 @@ pub struct GuestOSDevSettings { /// Generate and inject a self-signed TLS certificate and key for ic-boundary /// for the given domain name. To be used in system tests only. pub generate_ic_boundary_tls_cert: Option, + /// Pre-generated TLS certificate and key for ic-boundary. + /// When set, takes priority over `generate_ic_boundary_tls_cert`. + #[serde(default)] + pub ic_boundary_tls_cert: Option, /// PEM-encoded NNS public key. /// Overrides the hardcoded NNS public key on the rootfs. pub nns_pub_key_override: Option, diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index c3ad48185c71..05b88dde507b 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -582,6 +582,7 @@ fn create_guestos_config_for_node( .ok(), hostname: Some(node.node_id.to_string()), generate_ic_boundary_tls_cert: node.node_config.domain.clone(), + ic_boundary_tls_cert: None, nns_pub_key_override, }; From a67c01ee28af383d5ce6a9dbc61c867b81b6f57f Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 26 Mar 2026 20:25:17 +0000 Subject: [PATCH 21/65] Assign real domains to API BNs --- rs/tests/driver/src/driver/bootstrap.rs | 34 +++++++++++-- rs/tests/driver/src/driver/ic.rs | 67 ++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 7 deletions(-) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 05b88dde507b..b824edd4cc5a 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -1,5 +1,4 @@ -use crate::driver::ic_gateway_vm::HasIcGatewayVm; -use crate::driver::ic_gateway_vm::IC_GATEWAY_VM_NAME; +use crate::driver::ic_gateway_vm::{HasIcGatewayVm, IC_GATEWAY_VM_NAME, Playnet}; use crate::driver::ic_images::try_get_setupos_img_version; use crate::driver::nested::NestedVm; use crate::driver::resource::BootImage; @@ -36,8 +35,8 @@ use config_tool::setupos::{ }; use config_types::{ CONFIG_VERSION, DeploymentEnvironment, GuestOSConfig, GuestOSDevSettings, GuestOSSettings, - GuestOSUpgradeConfig, GuestVMType, ICOSDevSettings, ICOSSettings, Ipv4Config, Ipv6Config, - NetworkSettings, RecoveryConfig, + GuestOSUpgradeConfig, GuestVMType, ICOSDevSettings, ICOSSettings, IcBoundaryTlsCert, + Ipv4Config, Ipv6Config, NetworkSettings, RecoveryConfig, }; use ic_base_types::NodeId; use ic_prep_lib::{ @@ -247,6 +246,22 @@ pub fn setup_and_start_vms( for node in initialized_ic.api_boundary_nodes.values() { nodes.push(node.clone()); } + let api_bn_tls_cert: Option = if ic.api_bn_use_playnet { + let playnet = Playnet::read_attribute(env); + let cert = &playnet.playnet_cert.cert; + Some(IcBoundaryTlsCert { + cert_pem: format!("{}{}", cert.cert_pem, cert.chain_pem), + key_pem: cert.priv_key_pem.clone(), + }) + } else { + None + }; + let api_bn_node_ids: std::collections::HashSet = initialized_ic + .api_boundary_nodes + .values() + .map(|n| n.node_id) + .collect(); + let mut join_handles: Vec>> = vec![]; let mut nodes_info = NodesInfo::new(); for node in nodes { @@ -260,6 +275,11 @@ pub fn setup_and_start_vms( let ipv4_config = ic.get_ipv4_config_of_node(node.node_id); let domain = ic.get_domain_of_node(node.node_id); let recovery_hash: Option = ic.get_recovery_hash_of_node(node.node_id); + let ic_boundary_tls_cert = if api_bn_node_ids.contains(&node.node_id) { + api_bn_tls_cert.clone() + } else { + None + }; nodes_info.insert(node.node_id, malicious_behavior.clone()); join_handles.push(thread::spawn(move || { create_config_disk_image( @@ -270,6 +290,7 @@ pub fn setup_and_start_vms( ipv4_config, domain, recovery_hash, + ic_boundary_tls_cert, &t_env, )?; @@ -444,6 +465,7 @@ fn create_config_disk_image( ipv4_config: Option, domain_name: Option, recovery_hash: Option, + ic_boundary_tls_cert: Option, test_env: &TestEnv, ) -> anyhow::Result<()> { let mut bootstrap_options = BootstrapOptions { @@ -465,6 +487,7 @@ fn create_config_disk_image( ipv4_config, domain_name, recovery_hash, + ic_boundary_tls_cert, test_env, ic_name, )?; @@ -508,6 +531,7 @@ fn create_guestos_config_for_node( ipv4_config: Option, domain_name: Option, recovery_hash: Option, + ic_boundary_tls_cert: Option, test_env: &TestEnv, ic_name: &str, ) -> anyhow::Result { @@ -582,7 +606,7 @@ fn create_guestos_config_for_node( .ok(), hostname: Some(node.node_id.to_string()), generate_ic_boundary_tls_cert: node.node_config.domain.clone(), - ic_boundary_tls_cert: None, + ic_boundary_tls_cert, nns_pub_key_override, }; diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index d89ce5a82e72..6c4cf803343d 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -1,12 +1,16 @@ use crate::driver::resource::BootImage; use crate::driver::{ bootstrap::{init_ic, setup_and_start_vms}, - farm::{Farm, HostFeature}, + farm::{DnsRecord, DnsRecordType, Farm, HostFeature}, + ic_gateway_vm::Playnet, nested::UnassignedRecordConfig, node_software_version::NodeSoftwareVersion, resource::{AllocatedVm, ResourceGroup, allocate_resources, get_resource_request}, test_env::{TestEnv, TestEnvAttribute}, - test_env_api::{HasRegistryLocalStore, HasTopologySnapshot}, + test_env_api::{ + AcquirePlaynetCertificate, CreatePlaynetDnsRecords, HasRegistryLocalStore, + HasTopologySnapshot, + }, test_setup::GroupSetup, }; use anyhow::Result; @@ -50,6 +54,7 @@ pub struct InternetComputer { use_specified_ids_allocation_range: bool, pub unassigned_record_config: Option, pub api_boundary_nodes: Vec, + pub api_bn_use_playnet: bool, } #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Debug, Deserialize, Serialize)] @@ -153,6 +158,23 @@ impl InternetComputer { self } + /// Add the given number of API boundary nodes with playnet support. + /// Unlike `with_api_boundary_nodes`, nodes are created without domains here — + /// real domains (e.g. `apibn-0.ic50.farm.dfinity.systems`) are assigned during + /// `setup_and_start` after a playnet certificate is acquired from Farm. + pub fn with_api_boundary_nodes_playnet(mut self, no_of_nodes: usize) -> Self { + self.api_bn_use_playnet = true; + for _ in 0..no_of_nodes { + self.api_boundary_nodes.push( + Node::new() + .with_vm_allocation(self.vm_allocation.clone()) + .with_boot_image(BootImage::GroupDefault) + .with_required_host_features(self.required_host_features.clone()), + ); + } + self + } + /// Add an API boundary node with custom settings (domain name) pub fn with_api_boundary_node(mut self, node: Node) -> Self { self.api_boundary_nodes.push(node); @@ -254,6 +276,11 @@ impl InternetComputer { let res_group = allocate_resources(&farm, &res_request, env)?; self.propagate_ip_addrs(&res_group); + + if self.api_bn_use_playnet { + self.setup_api_bn_playnet(env); + } + let init_ic = init_ic( self, env, @@ -320,6 +347,42 @@ impl InternetComputer { } } + /// Acquires a playnet certificate from Farm, assigns real domains to API BN + /// nodes, creates DNS records, and writes the Playnet attribute for reuse + /// by the IC gateway. + fn setup_api_bn_playnet(&mut self, env: &TestEnv) { + let playnet_cert = env.acquire_playnet_certificate(); + let fqdn = &playnet_cert.playnet; + info!(env.logger(), "Acquired playnet for API BNs: {}", fqdn); + + for (idx, node) in self.api_boundary_nodes.iter_mut().enumerate() { + node.domain = Some(format!("apibn-{idx}.{fqdn}")); + } + + let dns_records: Vec = self + .api_boundary_nodes + .iter() + .enumerate() + .map(|(idx, node)| DnsRecord { + name: format!("apibn-{idx}"), + record_type: DnsRecordType::AAAA, + records: vec![node.ipv6.expect("API BN missing IPv6").to_string()], + }) + .collect(); + let suffix = env.create_playnet_dns_records(dns_records); + info!( + env.logger(), + "Created playnet DNS records for API BNs under {}", suffix + ); + + let playnet = Playnet { + playnet_cert, + aaaa_records: vec![], + a_records: vec![], + }; + playnet.write_attribute(env); + } + pub fn has_malicious_behaviors(&self) -> bool { let has_malicious_nodes: bool = self .subnets From dffb8cefac53916a331e2053b6f79832bfd0dc5a Mon Sep 17 00:00:00 2001 From: Andrew Battat Date: Thu, 26 Mar 2026 22:00:49 +0000 Subject: [PATCH 22/65] Minor touch-ups --- rs/ic_os/config/tool/src/guestos/generate_ic_config.rs | 2 +- rs/ic_os/config/types/src/lib.rs | 3 +-- rs/tests/driver/src/driver/bootstrap.rs | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs b/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs index ea7c4aed5b58..0e72e19a7f41 100644 --- a/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs +++ b/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs @@ -44,7 +44,7 @@ pub fn generate_ic_config(guestos_config: &GuestOSConfig, output_path: &Path) -> perms.set_mode(0o644); std::fs::set_permissions(output_path, perms)?; - // Set up TLS certificate and key for ic-boundary (system tests only). + // Set up TLS certificate and key for ic-boundary (system test use only). // A pre-generated cert (e.g. from a playnet) takes priority over self-signed generation. let dev_settings = &guestos_config.guestos_settings.guestos_dev_settings; if let Some(tls_cert) = &dev_settings.ic_boundary_tls_cert { diff --git a/rs/ic_os/config/types/src/lib.rs b/rs/ic_os/config/types/src/lib.rs index 6c6e51ee081f..0469f01987e4 100644 --- a/rs/ic_os/config/types/src/lib.rs +++ b/rs/ic_os/config/types/src/lib.rs @@ -220,7 +220,7 @@ pub struct GuestOSSettings { } /// Pre-generated TLS certificate and key for ic-boundary. -/// Used to inject a real (e.g. playnet) certificate instead of a self-signed one. +/// Used in system tests to inject a Farm-issued certificate. #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct IcBoundaryTlsCert { /// PEM-encoded certificate (leaf + chain concatenated). @@ -245,7 +245,6 @@ pub struct GuestOSDevSettings { /// for the given domain name. To be used in system tests only. pub generate_ic_boundary_tls_cert: Option, /// Pre-generated TLS certificate and key for ic-boundary. - /// When set, takes priority over `generate_ic_boundary_tls_cert`. #[serde(default)] pub ic_boundary_tls_cert: Option, /// PEM-encoded NNS public key. diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index b824edd4cc5a..e09f4a1cb1cd 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -256,7 +256,7 @@ pub fn setup_and_start_vms( } else { None }; - let api_bn_node_ids: std::collections::HashSet = initialized_ic + let api_bn_node_ids: Vec = initialized_ic .api_boundary_nodes .values() .map(|n| n.node_id) From 837f40dc36a7fb602dcca8b82f5c385d2f398a42 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Fri, 27 Mar 2026 10:11:19 +0000 Subject: [PATCH 23/65] refactor: allow overwriting --- rs/tests/driver/src/driver/ic.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index b4cd4d5c50eb..44a5fb650456 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -475,6 +475,7 @@ pub struct Subnet { pub vm_allocation: Option, pub boot_image: BootImage, pub required_host_features: Vec, + pub default_node_reward_type: Option, pub nodes: Vec, pub max_ingress_bytes_per_message: Option, pub max_ingress_messages_per_block: Option, @@ -505,16 +506,24 @@ pub struct Subnet { impl Subnet { pub fn new(subnet_type: SubnetType) -> Self { - // An invariant in the registry ensures that cloud engines have a free cost schedule - let canister_cycles_cost_schedule = match subnet_type { - SubnetType::CloudEngine => CanisterCyclesCostSchedule::Free, - _ => CanisterCyclesCostSchedule::Normal, + // Invariants in the registry ensures that + // - Cloud engines have a free cost schedule + // - Cloud engines only have nodes with reward type 4 + let (canister_cycles_cost_schedule, default_node_reward_type) = match subnet_type { + SubnetType::Application | SubnetType::System | SubnetType::VerifiedApplication => { + (CanisterCyclesCostSchedule::Normal, None) + } + SubnetType::CloudEngine => ( + CanisterCyclesCostSchedule::Free, + Some(NodeRewardType::Type4), + ), }; Self { vm_resource_overrides: Default::default(), vm_allocation: Default::default(), boot_image: Default::default(), required_host_features: vec![], + default_node_reward_type, nodes: vec![], max_ingress_bytes_per_message: None, max_ingress_bytes_per_block: None, @@ -637,9 +646,8 @@ impl Subnet { } pub fn add_node(mut self, mut node: Node) -> Self { - // If the subnet is a cloud engine, ensure that all nodes have reward type 4 - if self.subnet_type == SubnetType::CloudEngine { - node = node.with_node_reward_type(NodeRewardType::Type4); + if node.node_reward_type.is_none() { + node = node.with_node_reward_type(self.default_node_reward_type); } self.nodes.push(node); @@ -786,6 +794,7 @@ impl Default for Subnet { vm_allocation: Default::default(), boot_image: BootImage::GroupDefault, required_host_features: vec![], + default_node_reward_type: None, nodes: vec![], max_ingress_bytes_per_message: None, max_ingress_bytes_per_block: None, From 62a38584f37f666c80758de432817f7ffe6913f5 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Fri, 27 Mar 2026 10:22:49 +0000 Subject: [PATCH 24/65] docs: typo --- rs/tests/driver/src/driver/ic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index 44a5fb650456..81d0a213dda5 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -506,7 +506,7 @@ pub struct Subnet { impl Subnet { pub fn new(subnet_type: SubnetType) -> Self { - // Invariants in the registry ensures that + // Invariants in the registry ensure that // - Cloud engines have a free cost schedule // - Cloud engines only have nodes with reward type 4 let (canister_cycles_cost_schedule, default_node_reward_type) = match subnet_type { From 06bd0533ae3d8510e910a74e8a423f1830bc237c Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Fri, 27 Mar 2026 10:57:52 +0000 Subject: [PATCH 25/65] fix: clippy --- rs/tests/driver/src/driver/ic.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index 81d0a213dda5..41f9189424f4 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -646,8 +646,10 @@ impl Subnet { } pub fn add_node(mut self, mut node: Node) -> Self { - if node.node_reward_type.is_none() { - node = node.with_node_reward_type(self.default_node_reward_type); + if node.node_reward_type.is_none() + && let Some(reward_type) = self.default_node_reward_type + { + node = node.with_node_reward_type(reward_type); } self.nodes.push(node); From 41bf948e2d7012201c0eb0cf4bfede5897c5362a Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Fri, 27 Mar 2026 13:50:35 +0000 Subject: [PATCH 26/65] feat: disable DNS and TLS hacks (thank you Andrew!) --- .../src/nns_delegation_manager.rs | 50 +------------- rs/tests/networking/nns_delegation_test.rs | 69 ++----------------- 2 files changed, 8 insertions(+), 111 deletions(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 5505092b1e0a..6c1f5c62a6dd 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -28,11 +28,6 @@ use ic_types::{ time::expiry_time_from_now, }; use rand::Rng; -use rustls::{ - DigitallySignedStruct, SignatureScheme, - client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, - pki_types::{CertificateDer, ServerName, UnixTime}, -}; use tokio::{ net::TcpStream, sync::watch, @@ -526,54 +521,15 @@ async fn connect_to_nns_node( Ok(tls_stream) } -#[derive(Debug)] -struct NoVerify; -impl ServerCertVerifier for NoVerify { - fn verify_server_cert( - &self, - _end_entity: &CertificateDer, - _intermediates: &[CertificateDer], - _server_name: &ServerName, - _ocsp_response: &[u8], - _now: UnixTime, - ) -> Result { - Ok(ServerCertVerified::assertion()) - } - fn verify_tls12_signature( - &self, - _: &[u8], - _: &CertificateDer<'_>, - _: &DigitallySignedStruct, - ) -> Result { - Ok(HandshakeSignatureValid::assertion()) - } - fn verify_tls13_signature( - &self, - _: &[u8], - _: &CertificateDer<'_>, - _: &DigitallySignedStruct, - ) -> Result { - Ok(HandshakeSignatureValid::assertion()) - } - fn supported_verify_schemes(&self) -> Vec { - rustls::crypto::ring::default_provider() - .signature_verification_algorithms - .supported_schemes() - } -} - async fn connect_to_api_bn( log: &ReplicaLogger, api_bn_id: NodeId, domain: String, ) -> Result, BoxError> { - // TODO: Uncomment me - // let mut root_store = rustls::RootCertStore::empty(); - // root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + let mut root_store = rustls::RootCertStore::empty(); + root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); let tls_client_config = rustls::ClientConfig::builder() - // .with_root_certificates(root_store) - .dangerous() - .with_custom_certificate_verifier(Arc::new(NoVerify)) + .with_root_certificates(root_store) .with_no_client_auth(); let addr = (domain.as_str(), 443_u16); diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index fc4633b7f156..9bd099c7b6fb 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -54,13 +54,13 @@ use ic_system_test_driver::{ ic::{InternetComputer, Subnet}, test_env::{HasIcPrepDir, TestEnv}, test_env_api::{ - HasPublicApiUrl, HasTopologySnapshot, IcNodeContainer, IcNodeSnapshot, SshSession, - SubnetSnapshot, get_guestos_img_version, get_guestos_update_img_sha256, - get_guestos_update_img_url, get_guestos_update_img_version, + HasPublicApiUrl, HasTopologySnapshot, IcNodeContainer, IcNodeSnapshot, SubnetSnapshot, + get_guestos_img_version, get_guestos_update_img_sha256, get_guestos_update_img_url, + get_guestos_update_img_version, }, }, - retry_with_msg, systest, - util::{EndpointsStatus, assert_nodes_health_statuses, block_on, get_identity, get_nns_node}, + systest, + util::{block_on, get_identity, get_nns_node}, }; use ic_types::{ CanisterId, Height, PrincipalId, SubnetId, @@ -137,8 +137,6 @@ fn setup(env: TestEnv) { .setup_and_start(&env) .expect("Should be able to set up IC under test"); - hack(&env); - install_nns_and_check_progress(env.topology_snapshot()); info!( @@ -799,14 +797,6 @@ fn upgrade_non_nns_subnets_if_necessary(env: &TestEnv) { cloud_engine.subnet_id, )); - std::thread::sleep(Duration::from_secs(90)); - assert_nodes_health_statuses( - env.logger(), - &cloud_engine.nodes().collect::>(), - EndpointsStatus::AllUnhealthy, - ); - hack(env); - assert_assigned_replica_version(&app_node, &target_version, env.logger()); assert_assigned_replica_version(&cloud_engine_node, &target_version, env.logger()); } @@ -854,52 +844,3 @@ fn main() -> Result<()> { test_group.execute_from_args() } - -// HACK: Overwrite the DNS of all nodes to point the API BNs' domains to their IPs -fn hack(env: &TestEnv) { - for node in env - .topology_snapshot() - .subnets() - .flat_map(|subnet| subnet.nodes()) - { - let mappings = env - .topology_snapshot() - .api_boundary_nodes() - .map(|api_bn| (api_bn.get_domain().unwrap(), api_bn.get_ip_addr())) - .collect::>(); - - // File-system is read-only, so we modify /etc/hosts in a temporary file and replace the - // original with a bind mount. - let mut command = String::from( - r#" - set -euo pipefail - sudo cp /etc/hosts /tmp/hosts - "#, - ); - for (domain, ip) in mappings { - command.push_str(&format!( - r#" - echo "{ip} {domain}" | sudo tee -a /tmp/hosts > /dev/null - "# - )); - } - // Match the original /etc/hosts file permissions. - command.push_str( - r#" - sudo chown --reference=/etc/hosts /tmp/hosts - sudo chmod --reference=/etc/hosts /tmp/hosts - - sudo mount --bind /tmp/hosts /etc/hosts - "#, - ); - - retry_with_msg!( - format!("Overwrite /etc/hosts on node {}", node.node_id), - env.logger(), - Duration::from_secs(60), - Duration::from_secs(5), - || node.block_on_bash_script(&command) - ) - .expect("Should be able to overwrite /etc/hosts on the node"); - } -} From ae3d4e83ffc4e6d26e1179555a68378cda02a356 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Fri, 27 Mar 2026 13:59:41 +0000 Subject: [PATCH 27/65] feat: disable test driver changes (done in separate PR) --- rs/tests/driver/src/driver/bootstrap.rs | 2 +- rs/tests/driver/src/driver/ic.rs | 19 +------------------ rs/tests/nns/BUILD.bazel | 1 + rs/tests/nns/delete_subnet_test.rs | 4 +++- 4 files changed, 6 insertions(+), 20 deletions(-) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 1d102ba0fd4b..c3ad48185c71 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -615,7 +615,7 @@ fn node_to_config(node: &Node) -> NodeConfiguration { node_operator_principal_id: None, secret_key_store: node.secret_key_store.clone(), domain: node.domain.clone(), - node_reward_type: node.node_reward_type.map(String::from), + node_reward_type: None, } } diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index 9d39a2a30a60..eb42253a51d4 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -12,7 +12,6 @@ use crate::driver::{ use anyhow::Result; use ic_prep_lib::prep_state_directory::IcPrepStateDir; use ic_prep_lib::{node::NodeSecretKeyStore, subnet_configuration::SubnetRunningState}; -use ic_protobuf::registry::node::v1::NodeRewardType; use ic_regedit; use ic_registry_canister_api::IPv4Config; use ic_registry_subnet_features::{ChainKeyConfig, SubnetFeatures}; @@ -492,11 +491,6 @@ pub struct Subnet { impl Subnet { pub fn new(subnet_type: SubnetType) -> Self { - // An invariant in the registry ensures that cloud engines have a free cost schedule - let canister_cycles_cost_schedule = match subnet_type { - SubnetType::CloudEngine => CanisterCyclesCostSchedule::Free, - _ => CanisterCyclesCostSchedule::Normal, - }; Self { vm_resource_overrides: Default::default(), vm_allocation: Default::default(), @@ -517,7 +511,7 @@ impl Subnet { features: None, max_number_of_canisters: None, subnet_type, - canister_cycles_cost_schedule, + canister_cycles_cost_schedule: CanisterCyclesCostSchedule::Normal, ssh_readonly_access: vec![], ssh_backup_access: vec![], chain_key_config: None, @@ -624,11 +618,6 @@ impl Subnet { } pub fn add_node(mut self, mut node: Node) -> Self { - // If the subnet is a cloud engine, ensure that all nodes have reward type 4 - if self.subnet_type == SubnetType::CloudEngine { - node = node.with_node_reward_type(NodeRewardType::Type4) - } - self.nodes.push(node); self } @@ -871,7 +860,6 @@ pub struct Node { pub malicious_behavior: Option, pub ipv4: Option, pub domain: Option, - pub node_reward_type: Option, pub recovery_hash: Option, pub boot_image: BootImage, } @@ -926,11 +914,6 @@ impl Node { self } - pub fn with_node_reward_type(mut self, node_reward_type: NodeRewardType) -> Self { - self.node_reward_type = Some(node_reward_type); - self - } - pub fn with_recovery_hash(mut self, recovery_hash: String) -> Self { self.recovery_hash = Some(recovery_hash); self diff --git a/rs/tests/nns/BUILD.bazel b/rs/tests/nns/BUILD.bazel index 025bf8da128a..7465b47766d3 100644 --- a/rs/tests/nns/BUILD.bazel +++ b/rs/tests/nns/BUILD.bazel @@ -75,6 +75,7 @@ system_test( "//rs/tests/consensus/utils", "//rs/tests/driver:ic-system-test-driver", "//rs/types/base_types", + "//rs/types/cycles", "//rs/types/types", "//rs/universal_canister/lib", "@crate_index//:anyhow", diff --git a/rs/tests/nns/delete_subnet_test.rs b/rs/tests/nns/delete_subnet_test.rs index f84639fad68d..f497cd57e1d7 100644 --- a/rs/tests/nns/delete_subnet_test.rs +++ b/rs/tests/nns/delete_subnet_test.rs @@ -22,6 +22,7 @@ use ic_system_test_driver::nns::get_subnet_list_from_registry; use ic_system_test_driver::systest; use ic_system_test_driver::util::{UniversalCanister, assert_create_agent, block_on}; use ic_types::{Height, RegistryVersion, SubnetId}; +use ic_types_cycles::CanisterCyclesCostSchedule; use registry_canister::init::RegistryCanisterInitPayloadBuilder; use registry_canister::mutations::do_delete_subnet::DeleteSubnetPayload; use std::collections::BTreeSet; @@ -55,7 +56,8 @@ pub fn setup(env: TestEnv) { ) .add_subnet( Subnet::fast(SubnetType::CloudEngine, NUM_ENGINE_NODES) - .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), + .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)) + .with_cost_schedule(CanisterCyclesCostSchedule::Free), ) .setup_and_start(&env) .expect("failed to setup IC under test"); From c3121dface8f75e8729e3dde436e33a221000d90 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Fri, 27 Mar 2026 17:05:39 +0000 Subject: [PATCH 28/65] feat: use Andrew's DNS + TLS impl --- .../message_routing/xnet/xnet_cloud_engine_isolation_test.rs | 2 +- rs/tests/networking/nns_delegation_test.rs | 2 +- rs/tests/nns/delete_subnet_test.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs b/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs index 65c451bb95c0..56bf7fd973dd 100644 --- a/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs +++ b/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs @@ -62,7 +62,7 @@ fn main() -> Result<()> { fn setup(env: TestEnv) { InternetComputer::new() - .with_api_boundary_nodes(1) + .with_api_boundary_nodes_playnet(1) .add_subnet(Subnet::fast_single_node(SubnetType::System)) .add_subnet(Subnet::fast_single_node(SubnetType::Application)) .add_subnet(Subnet::fast_single_node(SubnetType::CloudEngine)) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 9bd099c7b6fb..f4acbc8eb373 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -124,7 +124,7 @@ fn set_installed_canister_ids(env: &TestEnv, canister_ids: BTreeMap Result<()> { pub fn setup(env: TestEnv) { InternetComputer::new() - .with_api_boundary_nodes(1) + .with_api_boundary_nodes_playnet(1) .add_subnet( Subnet::fast(SubnetType::System, NUM_NODES) .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), From b7a5dcceef76a12b23c65d701ae7e6314e501a45 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Fri, 27 Mar 2026 17:09:46 +0000 Subject: [PATCH 29/65] fix: temporarily disable mainnet version on cloud engines --- rs/tests/networking/nns_delegation_test.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index f4acbc8eb373..ac71dd5ba509 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -791,14 +791,16 @@ fn upgrade_non_nns_subnets_if_necessary(env: &TestEnv) { app_subnet.subnet_id, )); - block_on(deploy_guestos_to_all_subnet_nodes( - &nns_node, - &target_version, - cloud_engine.subnet_id, - )); + // TODO(CON-1696): Uncomment me when #9613 reaches mainnet NNS + // block_on(deploy_guestos_to_all_subnet_nodes( + // &nns_node, + // &target_version, + // cloud_engine.subnet_id, + // )); assert_assigned_replica_version(&app_node, &target_version, env.logger()); - assert_assigned_replica_version(&cloud_engine_node, &target_version, env.logger()); + // TODO(CON-1696): Uncomment me when #9613 reaches mainnet NNS + // assert_assigned_replica_version(&cloud_engine_node, &target_version, env.logger()); } macro_rules! systest_all_subnet_types { From 3bad8dba83ec375d441a0eb466516f8cfdd97523 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Fri, 27 Mar 2026 17:22:25 +0000 Subject: [PATCH 30/65] fix: clippy --- rs/tests/networking/nns_delegation_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index ac71dd5ba509..248788ba4b65 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -756,7 +756,7 @@ where fn upgrade_non_nns_subnets_if_necessary(env: &TestEnv) { let (app_subnet, app_node) = get_subnet_and_node(env, SubnetType::Application); - let (cloud_engine, cloud_engine_node) = get_subnet_and_node(env, SubnetType::CloudEngine); + let (_cloud_engine, _cloud_engine_node) = get_subnet_and_node(env, SubnetType::CloudEngine); let nns_node = get_nns_node(&env.topology_snapshot()); let initial_version = get_guestos_img_version(); From 3d3383a3bb1f95c161752dadc6da9d45a62959e9 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 08:03:58 +0000 Subject: [PATCH 31/65] fix: temporarily disable mainnet version on cloud engines (continued) --- rs/tests/networking/nns_delegation_test.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 248788ba4b65..397cd9be5d0f 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -807,7 +807,11 @@ macro_rules! systest_all_subnet_types { ($group: expr, $function_name:path) => { $group = $group.add_test(systest!($function_name; SubnetType::System)); $group = $group.add_test(systest!($function_name; SubnetType::Application)); - $group = $group.add_test(systest!($function_name; SubnetType::CloudEngine)); + // TODO(CON-1696): Remove this condition (and always run the test for cloud engines) when + // #9613 reaches mainnet NNS + if get_guestos_img_version() == get_guestos_update_img_version() { + $group = $group.add_test(systest!($function_name; SubnetType::CloudEngine)); + } }; } From 8be0cf5c0067432076e5f960324532f8d20fcb32 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 08:12:52 +0000 Subject: [PATCH 32/65] style: subnet_type as argument --- rs/tests/networking/nns_delegation_test.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 397cd9be5d0f..c63043703dbd 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -112,9 +112,9 @@ const RETRY_DELAY: tokio::time::Duration = tokio::time::Duration::from_secs(60); const DKG_LENGTH: Height = Height::new(9); -fn get_installed_canister_ids(env: &TestEnv) -> BTreeMap { - env.read_json_object(INSTALLED_CANISTER_IDS) - .expect("Could not read installed canister IDs from test environment.") +fn get_installed_canister_id(env: &TestEnv, subnet_type: SubnetType) -> PrincipalId { + env.read_json_object::, _>(INSTALLED_CANISTER_IDS) + .expect("Could not read installed canister IDs from test environment.")[&subnet_type] } fn set_installed_canister_ids(env: &TestEnv, canister_ids: BTreeMap) { @@ -325,7 +325,7 @@ fn subnet_read_state_v3_returns_correct_delegation(env: TestEnv, subnet_type: Su /// Responses to `api/v2/canister/{canister_id}/read_state` have valid delegations with canister ranges in the flat format. fn canister_read_state_v2_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { - let canister_id = get_installed_canister_ids(&env)[&subnet_type]; + let canister_id = get_installed_canister_id(&env, subnet_type); let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( @@ -347,7 +347,7 @@ fn canister_read_state_v2_returns_correct_delegation(env: TestEnv, subnet_type: /// Responses to `api/v3/canister/{canister_id}/read_state` have valid delegations with canister ranges in the flat format. fn canister_read_state_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { - let canister_id = get_installed_canister_ids(&env)[&subnet_type]; + let canister_id = get_installed_canister_id(&env, subnet_type); let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: HttpReadStateResponse = block_on(send( @@ -399,7 +399,7 @@ struct SyncCallResponse { /// Responses to `api/v3/canister/{canister_id}/call` have valid delegations with canister ranges in the flat format. fn call_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { - let canister_id = get_installed_canister_ids(&env)[&subnet_type]; + let canister_id = get_installed_canister_id(&env, subnet_type); let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: SyncCallResponse = block_on(send( @@ -421,7 +421,7 @@ fn call_v3_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { /// Responses to `api/v4/canister/{canister_id}/call` have valid delegations with canister ranges in the flat format. fn call_v4_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { - let canister_id = get_installed_canister_ids(&env)[&subnet_type]; + let canister_id = get_installed_canister_id(&env, subnet_type); let (subnet, node) = get_subnet_and_node(&env, subnet_type); let response: SyncCallResponse = block_on(send( @@ -444,7 +444,7 @@ fn call_v4_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { /// Responses to `api/v4/canister/{canister_id}/call` targeting the management canister /// have valid delegations without canister ranges. fn call_v4_management_canister_returns_correct_delegation(env: TestEnv, subnet_type: SubnetType) { - let canister_id = get_installed_canister_ids(&env)[&subnet_type]; + let canister_id = get_installed_canister_id(&env, subnet_type); let (subnet, node) = get_subnet_and_node(&env, subnet_type); let expiration = OffsetDateTime::now_utc() + Duration::from_secs(3 * 60); @@ -490,7 +490,7 @@ struct QueryResponse { /// For `api/v2/canister/{canister_id}/query` we pass valid delegations with /// canister ranges in the flat format to the canister. fn query_v2_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: SubnetType) { - let canister_id = get_installed_canister_ids(&env)[&subnet_type]; + let canister_id = get_installed_canister_id(&env, subnet_type); let (subnet, node) = get_subnet_and_node(&env, subnet_type); let arg = vec![]; @@ -514,7 +514,7 @@ fn query_v2_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: Sub /// For `api/v3/canister/{canister_id}/query` we pass valid delegations with /// canister ranges in the tree format to the canister. fn query_v3_passes_correct_delegation_to_canister(env: TestEnv, subnet_type: SubnetType) { - let canister_id = get_installed_canister_ids(&env)[&subnet_type]; + let canister_id = get_installed_canister_id(&env, subnet_type); let (subnet, node) = get_subnet_and_node(&env, subnet_type); let arg = vec![]; From 24eeaf43fdeba618dd5dfcbee65564380dbf2510 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 08:34:26 +0000 Subject: [PATCH 33/65] Revert "feat: use Andrew's DNS + TLS impl" This reverts commit c3121dface8f75e8729e3dde436e33a221000d90. --- .../message_routing/xnet/xnet_cloud_engine_isolation_test.rs | 2 +- rs/tests/networking/nns_delegation_test.rs | 2 +- rs/tests/nns/delete_subnet_test.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs b/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs index 56bf7fd973dd..65c451bb95c0 100644 --- a/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs +++ b/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs @@ -62,7 +62,7 @@ fn main() -> Result<()> { fn setup(env: TestEnv) { InternetComputer::new() - .with_api_boundary_nodes_playnet(1) + .with_api_boundary_nodes(1) .add_subnet(Subnet::fast_single_node(SubnetType::System)) .add_subnet(Subnet::fast_single_node(SubnetType::Application)) .add_subnet(Subnet::fast_single_node(SubnetType::CloudEngine)) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index c63043703dbd..0d159eb83180 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -124,7 +124,7 @@ fn set_installed_canister_ids(env: &TestEnv, canister_ids: BTreeMap Result<()> { pub fn setup(env: TestEnv) { InternetComputer::new() - .with_api_boundary_nodes_playnet(1) + .with_api_boundary_nodes(1) .add_subnet( Subnet::fast(SubnetType::System, NUM_NODES) .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), From 54dd02a3c47498fee790ec7c090218bba556b95a Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 09:45:28 +0000 Subject: [PATCH 34/65] Revert "Minor touch-ups" This reverts commit dffb8cefac53916a331e2053b6f79832bfd0dc5a. Revert "Assign real domains to API BNs" This reverts commit a67c01ee28af383d5ce6a9dbc61c867b81b6f57f. Revert "Create IcBoundaryTlsCert config object and update generate_ic_config to write tls" This reverts commit ae279033a5fe5d1db30915d0ec2670445ca127a1. --- .../tool/src/guestos/generate_ic_config.rs | 36 ++++------ .../fixtures/guestos_v1.15.0.json | 52 -------------- .../fixtures/hostos_v1.15.0.json | 52 -------------- rs/ic_os/config/types/src/lib.rs | 15 +---- rs/tests/driver/src/driver/bootstrap.rs | 33 ++------- rs/tests/driver/src/driver/ic.rs | 67 +------------------ 6 files changed, 20 insertions(+), 235 deletions(-) delete mode 100644 rs/ic_os/config/types/compatibility_tests/fixtures/guestos_v1.15.0.json delete mode 100644 rs/ic_os/config/types/compatibility_tests/fixtures/hostos_v1.15.0.json diff --git a/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs b/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs index 0e72e19a7f41..08e4f21cb4de 100644 --- a/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs +++ b/rs/ic_os/config/tool/src/guestos/generate_ic_config.rs @@ -44,12 +44,12 @@ pub fn generate_ic_config(guestos_config: &GuestOSConfig, output_path: &Path) -> perms.set_mode(0o644); std::fs::set_permissions(output_path, perms)?; - // Set up TLS certificate and key for ic-boundary (system test use only). - // A pre-generated cert (e.g. from a playnet) takes priority over self-signed generation. - let dev_settings = &guestos_config.guestos_settings.guestos_dev_settings; - if let Some(tls_cert) = &dev_settings.ic_boundary_tls_cert { - write_tls_certificate(&tls_cert.cert_pem, &tls_cert.key_pem)?; - } else if let Some(domain_name) = &dev_settings.generate_ic_boundary_tls_cert + // Generate and inject a self-signed TLS certificate and key for ic-boundary + // for the given domain name. To be used in system tests only. + if let Some(domain_name) = &guestos_config + .guestos_settings + .guestos_dev_settings + .generate_ic_boundary_tls_cert && !domain_name.is_empty() { generate_tls_certificate(domain_name)?; @@ -235,16 +235,10 @@ pub fn get_best_interface_ipv6_address() -> Result { bail!("Cannot determine an IPv6 address, aborting"); } -const TLS_KEY_PATH: &str = "/var/lib/ic/data/ic-boundary-tls.key"; -const TLS_CERT_PATH: &str = "/var/lib/ic/data/ic-boundary-tls.crt"; - -fn write_tls_certificate(cert_pem: &str, key_pem: &str) -> Result<()> { - write(TLS_CERT_PATH, cert_pem).context("Failed to write TLS certificate")?; - write(TLS_KEY_PATH, key_pem).context("Failed to write TLS key")?; - set_tls_file_ownership_and_permissions() -} - fn generate_tls_certificate(domain_name: &str) -> Result<()> { + let tls_key_path = "/var/lib/ic/data/ic-boundary-tls.key"; + let tls_cert_path = "/var/lib/ic/data/ic-boundary-tls.crt"; + let status = Command::new("openssl") .args([ "req", @@ -252,9 +246,9 @@ fn generate_tls_certificate(domain_name: &str) -> Result<()> { "-newkey", "rsa:2048", "-keyout", - TLS_KEY_PATH, + tls_key_path, "-out", - TLS_CERT_PATH, + tls_cert_path, "-sha256", "-days", "3650", @@ -271,12 +265,8 @@ fn generate_tls_certificate(domain_name: &str) -> Result<()> { bail!("openssl command failed with status: {}", status); } - set_tls_file_ownership_and_permissions() -} - -fn set_tls_file_ownership_and_permissions() -> Result<()> { let status = Command::new("chown") - .args(["ic-replica:nogroup", TLS_KEY_PATH, TLS_CERT_PATH]) + .args(["ic-replica:nogroup", tls_key_path, tls_cert_path]) .status() .context("Failed to set ownership of TLS files")?; @@ -285,7 +275,7 @@ fn set_tls_file_ownership_and_permissions() -> Result<()> { } let status = Command::new("chmod") - .args(["644", TLS_KEY_PATH, TLS_CERT_PATH]) + .args(["644", tls_key_path, tls_cert_path]) .status() .context("Failed to set permissions of TLS files")?; diff --git a/rs/ic_os/config/types/compatibility_tests/fixtures/guestos_v1.15.0.json b/rs/ic_os/config/types/compatibility_tests/fixtures/guestos_v1.15.0.json deleted file mode 100644 index dfba380fd5d1..000000000000 --- a/rs/ic_os/config/types/compatibility_tests/fixtures/guestos_v1.15.0.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "config_version": "1.15.0", - "network_settings": { - "ipv6_config": { - "Fixed": { - "address": "2a00:fb01:400:200::1/64", - "gateway": "2a00:fb01:400:200::1" - } - }, - "ipv4_config": { - "address": "192.168.1.1", - "gateway": "192.168.1.254", - "prefix_length": 24 - }, - "domain_name": "ic.test" - }, - "icos_settings": { - "node_reward_type": "type3.1", - "mgmt_mac": "00:00:00:00:00:01", - "deployment_environment": "mainnet", - "nns_urls": [ - "https://icp-api.io,https//icp0.io,https://ic0.app" - ], - "node_operator_private_key": null, - "enable_trusted_execution_environment": false, - "use_ssh_authorized_keys": false, - "icos_dev_settings": {} - }, - "guestos_settings": { - "guestos_dev_settings": { - "backup_spool": null, - "malicious_behavior": null, - "query_stats_epoch_length": null, - "bitcoind_addr": null, - "dogecoind_addr": null, - "jaeger_addr": null, - "socks_proxy": null, - "hostname": null, - "generate_ic_boundary_tls_cert": null, - "ic_boundary_tls_cert": null, - "nns_pub_key_override": null - } - }, - "guest_vm_type": "default", - "upgrade_config": { - "peer_guest_vm_address": "2a00:fb01:400:200:6801:95ff:fed7:d475" - }, - "trusted_execution_environment_config": { - "sev_cert_chain_pem": "-----BEGIN CERTIFICATE----------END CERTIFICATE-----" - }, - "recovery_config": null -} \ No newline at end of file diff --git a/rs/ic_os/config/types/compatibility_tests/fixtures/hostos_v1.15.0.json b/rs/ic_os/config/types/compatibility_tests/fixtures/hostos_v1.15.0.json deleted file mode 100644 index 5dda6713f7f8..000000000000 --- a/rs/ic_os/config/types/compatibility_tests/fixtures/hostos_v1.15.0.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "config_version": "1.15.0", - "network_settings": { - "ipv6_config": { - "Fixed": { - "address": "2a00:fb01:400:200::1/64", - "gateway": "2a00:fb01:400:200::1" - } - }, - "ipv4_config": { - "address": "192.168.1.1", - "gateway": "192.168.1.254", - "prefix_length": 24 - }, - "domain_name": "ic.test" - }, - "icos_settings": { - "node_reward_type": "type3.1", - "mgmt_mac": "00:00:00:00:00:01", - "deployment_environment": "mainnet", - "nns_urls": [ - "https://icp-api.io,https//icp0.io,https://ic0.app" - ], - "node_operator_private_key": null, - "enable_trusted_execution_environment": false, - "use_ssh_authorized_keys": false, - "icos_dev_settings": {} - }, - "hostos_settings": { - "hostos_dev_settings": { - "vm_memory": 16, - "vm_cpu": "kvm", - "vm_nr_of_vcpus": 64 - }, - "verbose": false - }, - "guestos_settings": { - "guestos_dev_settings": { - "backup_spool": null, - "malicious_behavior": null, - "query_stats_epoch_length": null, - "bitcoind_addr": null, - "dogecoind_addr": null, - "jaeger_addr": null, - "socks_proxy": null, - "hostname": null, - "generate_ic_boundary_tls_cert": null, - "ic_boundary_tls_cert": null, - "nns_pub_key_override": null - } - } -} \ No newline at end of file diff --git a/rs/ic_os/config/types/src/lib.rs b/rs/ic_os/config/types/src/lib.rs index 0469f01987e4..c1570f484464 100644 --- a/rs/ic_os/config/types/src/lib.rs +++ b/rs/ic_os/config/types/src/lib.rs @@ -40,7 +40,7 @@ use std::str::FromStr; use strum::{Display, EnumString}; use url::Url; -pub const CONFIG_VERSION: &str = "1.15.0"; +pub const CONFIG_VERSION: &str = "1.14.0"; /// List of field paths that have been removed and should not be reused. pub static RESERVED_FIELD_PATHS: &[&str] = &[ @@ -219,16 +219,6 @@ pub struct GuestOSSettings { pub guestos_dev_settings: GuestOSDevSettings, } -/// Pre-generated TLS certificate and key for ic-boundary. -/// Used in system tests to inject a Farm-issued certificate. -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] -pub struct IcBoundaryTlsCert { - /// PEM-encoded certificate (leaf + chain concatenated). - pub cert_pem: String, - /// PEM-encoded private key. - pub key_pem: String, -} - /// GuestOS development configuration. These settings are strictly used for development images. #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Default, Clone)] pub struct GuestOSDevSettings { @@ -244,9 +234,6 @@ pub struct GuestOSDevSettings { /// Generate and inject a self-signed TLS certificate and key for ic-boundary /// for the given domain name. To be used in system tests only. pub generate_ic_boundary_tls_cert: Option, - /// Pre-generated TLS certificate and key for ic-boundary. - #[serde(default)] - pub ic_boundary_tls_cert: Option, /// PEM-encoded NNS public key. /// Overrides the hardcoded NNS public key on the rootfs. pub nns_pub_key_override: Option, diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 5252ec3aedfc..1d102ba0fd4b 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -1,4 +1,5 @@ -use crate::driver::ic_gateway_vm::{HasIcGatewayVm, IC_GATEWAY_VM_NAME, Playnet}; +use crate::driver::ic_gateway_vm::HasIcGatewayVm; +use crate::driver::ic_gateway_vm::IC_GATEWAY_VM_NAME; use crate::driver::ic_images::try_get_setupos_img_version; use crate::driver::nested::NestedVm; use crate::driver::resource::BootImage; @@ -35,8 +36,8 @@ use config_tool::setupos::{ }; use config_types::{ CONFIG_VERSION, DeploymentEnvironment, GuestOSConfig, GuestOSDevSettings, GuestOSSettings, - GuestOSUpgradeConfig, GuestVMType, ICOSDevSettings, ICOSSettings, IcBoundaryTlsCert, - Ipv4Config, Ipv6Config, NetworkSettings, RecoveryConfig, + GuestOSUpgradeConfig, GuestVMType, ICOSDevSettings, ICOSSettings, Ipv4Config, Ipv6Config, + NetworkSettings, RecoveryConfig, }; use ic_base_types::NodeId; use ic_prep_lib::{ @@ -246,22 +247,6 @@ pub fn setup_and_start_vms( for node in initialized_ic.api_boundary_nodes.values() { nodes.push(node.clone()); } - let api_bn_tls_cert: Option = if ic.api_bn_use_playnet { - let playnet = Playnet::read_attribute(env); - let cert = &playnet.playnet_cert.cert; - Some(IcBoundaryTlsCert { - cert_pem: format!("{}{}", cert.cert_pem, cert.chain_pem), - key_pem: cert.priv_key_pem.clone(), - }) - } else { - None - }; - let api_bn_node_ids: Vec = initialized_ic - .api_boundary_nodes - .values() - .map(|n| n.node_id) - .collect(); - let mut join_handles: Vec>> = vec![]; let mut nodes_info = NodesInfo::new(); for node in nodes { @@ -275,11 +260,6 @@ pub fn setup_and_start_vms( let ipv4_config = ic.get_ipv4_config_of_node(node.node_id); let domain = ic.get_domain_of_node(node.node_id); let recovery_hash: Option = ic.get_recovery_hash_of_node(node.node_id); - let ic_boundary_tls_cert = if api_bn_node_ids.contains(&node.node_id) { - api_bn_tls_cert.clone() - } else { - None - }; nodes_info.insert(node.node_id, malicious_behavior.clone()); join_handles.push(thread::spawn(move || { create_config_disk_image( @@ -290,7 +270,6 @@ pub fn setup_and_start_vms( ipv4_config, domain, recovery_hash, - ic_boundary_tls_cert, &t_env, )?; @@ -465,7 +444,6 @@ fn create_config_disk_image( ipv4_config: Option, domain_name: Option, recovery_hash: Option, - ic_boundary_tls_cert: Option, test_env: &TestEnv, ) -> anyhow::Result<()> { let mut bootstrap_options = BootstrapOptions { @@ -487,7 +465,6 @@ fn create_config_disk_image( ipv4_config, domain_name, recovery_hash, - ic_boundary_tls_cert, test_env, ic_name, )?; @@ -531,7 +508,6 @@ fn create_guestos_config_for_node( ipv4_config: Option, domain_name: Option, recovery_hash: Option, - ic_boundary_tls_cert: Option, test_env: &TestEnv, ic_name: &str, ) -> anyhow::Result { @@ -606,7 +582,6 @@ fn create_guestos_config_for_node( .ok(), hostname: Some(node.node_id.to_string()), generate_ic_boundary_tls_cert: node.node_config.domain.clone(), - ic_boundary_tls_cert, nns_pub_key_override, }; diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index b14388243ed8..41f9189424f4 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -1,16 +1,12 @@ use crate::driver::resource::BootImage; use crate::driver::{ bootstrap::{init_ic, setup_and_start_vms}, - farm::{DnsRecord, DnsRecordType, Farm, HostFeature}, - ic_gateway_vm::Playnet, + farm::{Farm, HostFeature}, nested::UnassignedRecordConfig, node_software_version::NodeSoftwareVersion, resource::{AllocatedVm, ResourceGroup, allocate_resources, get_resource_request}, test_env::{TestEnv, TestEnvAttribute}, - test_env_api::{ - AcquirePlaynetCertificate, CreatePlaynetDnsRecords, HasRegistryLocalStore, - HasTopologySnapshot, - }, + test_env_api::{HasRegistryLocalStore, HasTopologySnapshot}, test_setup::GroupSetup, }; use anyhow::Result; @@ -55,7 +51,6 @@ pub struct InternetComputer { use_specified_ids_allocation_range: bool, pub unassigned_record_config: Option, pub api_boundary_nodes: Vec, - pub api_bn_use_playnet: bool, } #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Debug, Deserialize, Serialize)] @@ -159,23 +154,6 @@ impl InternetComputer { self } - /// Add the given number of API boundary nodes with playnet support. - /// Unlike `with_api_boundary_nodes`, nodes are created without domains here — - /// real domains (e.g. `apibn-0.ic50.farm.dfinity.systems`) are assigned during - /// `setup_and_start` after a playnet certificate is acquired from Farm. - pub fn with_api_boundary_nodes_playnet(mut self, no_of_nodes: usize) -> Self { - self.api_bn_use_playnet = true; - for _ in 0..no_of_nodes { - self.api_boundary_nodes.push( - Node::new() - .with_vm_allocation(self.vm_allocation.clone()) - .with_boot_image(BootImage::GroupDefault) - .with_required_host_features(self.required_host_features.clone()), - ); - } - self - } - /// Add an API boundary node with custom settings (domain name) pub fn with_api_boundary_node(mut self, node: Node) -> Self { self.api_boundary_nodes.push(node); @@ -290,11 +268,6 @@ impl InternetComputer { let res_group = allocate_resources(&farm, &res_request, env)?; self.propagate_ip_addrs(&res_group); - - if self.api_bn_use_playnet { - self.setup_api_bn_playnet(env); - } - let init_ic = init_ic( self, env, @@ -361,42 +334,6 @@ impl InternetComputer { } } - /// Acquires a playnet certificate from Farm, assigns real domains to API BN - /// nodes, creates DNS records, and writes the Playnet attribute for reuse - /// by the IC gateway. - fn setup_api_bn_playnet(&mut self, env: &TestEnv) { - let playnet_cert = env.acquire_playnet_certificate(); - let fqdn = &playnet_cert.playnet; - info!(env.logger(), "Acquired playnet for API BNs: {}", fqdn); - - for (idx, node) in self.api_boundary_nodes.iter_mut().enumerate() { - node.domain = Some(format!("apibn-{idx}.{fqdn}")); - } - - let dns_records: Vec = self - .api_boundary_nodes - .iter() - .enumerate() - .map(|(idx, node)| DnsRecord { - name: format!("apibn-{idx}"), - record_type: DnsRecordType::AAAA, - records: vec![node.ipv6.expect("API BN missing IPv6").to_string()], - }) - .collect(); - let suffix = env.create_playnet_dns_records(dns_records); - info!( - env.logger(), - "Created playnet DNS records for API BNs under {}", suffix - ); - - let playnet = Playnet { - playnet_cert, - aaaa_records: vec![], - a_records: vec![], - }; - playnet.write_attribute(env); - } - pub fn has_malicious_behaviors(&self) -> bool { let has_malicious_nodes: bool = self .subnets From 04409be3efc0a1b1809cb60d3a7b1b5eb59fcc9e Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 09:46:14 +0000 Subject: [PATCH 35/65] Revert "fix: clippy" This reverts commit 06bd0533ae3d8510e910a74e8a423f1830bc237c. Revert "docs: typo" This reverts commit 62a38584f37f666c80758de432817f7ffe6913f5. Revert "refactor: allow overwriting" This reverts commit 837f40dc36a7fb602dcca8b82f5c385d2f398a42. Revert "test: add API BNs" This reverts commit 38c5891ad6f72a919550a346a652ecb3eee94b40. Revert "test: ensure type4 node reward types" This reverts commit 81030b7526743a8241926b96eed826ffe7012f9e. Revert "test: ensure free cost schedule" This reverts commit 613fb55a13fd5fd9cd4277788eb1f72cd5a9caa2. --- rs/tests/driver/src/driver/bootstrap.rs | 2 +- rs/tests/driver/src/driver/ic.rs | 45 +------------------ .../xnet/xnet_cloud_engine_isolation_test.rs | 1 - rs/tests/nns/BUILD.bazel | 1 + rs/tests/nns/delete_subnet_test.rs | 5 ++- 5 files changed, 7 insertions(+), 47 deletions(-) diff --git a/rs/tests/driver/src/driver/bootstrap.rs b/rs/tests/driver/src/driver/bootstrap.rs index 1d102ba0fd4b..c3ad48185c71 100644 --- a/rs/tests/driver/src/driver/bootstrap.rs +++ b/rs/tests/driver/src/driver/bootstrap.rs @@ -615,7 +615,7 @@ fn node_to_config(node: &Node) -> NodeConfiguration { node_operator_principal_id: None, secret_key_store: node.secret_key_store.clone(), domain: node.domain.clone(), - node_reward_type: node.node_reward_type.map(String::from), + node_reward_type: None, } } diff --git a/rs/tests/driver/src/driver/ic.rs b/rs/tests/driver/src/driver/ic.rs index 41f9189424f4..d89ce5a82e72 100644 --- a/rs/tests/driver/src/driver/ic.rs +++ b/rs/tests/driver/src/driver/ic.rs @@ -12,7 +12,6 @@ use crate::driver::{ use anyhow::Result; use ic_prep_lib::prep_state_directory::IcPrepStateDir; use ic_prep_lib::{node::NodeSecretKeyStore, subnet_configuration::SubnetRunningState}; -use ic_protobuf::registry::node::v1::NodeRewardType; use ic_regedit; use ic_registry_canister_api::IPv4Config; use ic_registry_subnet_features::{ChainKeyConfig, SubnetFeatures}; @@ -226,19 +225,6 @@ impl InternetComputer { &mut self, env: &TestEnv, ) -> Result> { - if self - .subnets - .iter() - .any(|s| s.subnet_type == SubnetType::CloudEngine) - { - // Cloud engines use API boundary nodes to replicate their registry and to fetch - // delegations since the firewall on the NNS blocks them. - assert!( - !self.api_boundary_nodes.is_empty(), - "At least one API boundary node is required when using a cloud engine subnet" - ); - } - // propagate required host features and resource settings to all vms let farm = Farm::from_test_env(env, "Internet Computer"); for node in self @@ -475,7 +461,6 @@ pub struct Subnet { pub vm_allocation: Option, pub boot_image: BootImage, pub required_host_features: Vec, - pub default_node_reward_type: Option, pub nodes: Vec, pub max_ingress_bytes_per_message: Option, pub max_ingress_messages_per_block: Option, @@ -506,24 +491,11 @@ pub struct Subnet { impl Subnet { pub fn new(subnet_type: SubnetType) -> Self { - // Invariants in the registry ensure that - // - Cloud engines have a free cost schedule - // - Cloud engines only have nodes with reward type 4 - let (canister_cycles_cost_schedule, default_node_reward_type) = match subnet_type { - SubnetType::Application | SubnetType::System | SubnetType::VerifiedApplication => { - (CanisterCyclesCostSchedule::Normal, None) - } - SubnetType::CloudEngine => ( - CanisterCyclesCostSchedule::Free, - Some(NodeRewardType::Type4), - ), - }; Self { vm_resource_overrides: Default::default(), vm_allocation: Default::default(), boot_image: Default::default(), required_host_features: vec![], - default_node_reward_type, nodes: vec![], max_ingress_bytes_per_message: None, max_ingress_bytes_per_block: None, @@ -539,7 +511,7 @@ impl Subnet { features: None, max_number_of_canisters: None, subnet_type, - canister_cycles_cost_schedule, + canister_cycles_cost_schedule: CanisterCyclesCostSchedule::Normal, ssh_readonly_access: vec![], ssh_backup_access: vec![], chain_key_config: None, @@ -645,13 +617,7 @@ impl Subnet { }) } - pub fn add_node(mut self, mut node: Node) -> Self { - if node.node_reward_type.is_none() - && let Some(reward_type) = self.default_node_reward_type - { - node = node.with_node_reward_type(reward_type); - } - + pub fn add_node(mut self, node: Node) -> Self { self.nodes.push(node); self } @@ -796,7 +762,6 @@ impl Default for Subnet { vm_allocation: Default::default(), boot_image: BootImage::GroupDefault, required_host_features: vec![], - default_node_reward_type: None, nodes: vec![], max_ingress_bytes_per_message: None, max_ingress_bytes_per_block: None, @@ -895,7 +860,6 @@ pub struct Node { pub malicious_behavior: Option, pub ipv4: Option, pub domain: Option, - pub node_reward_type: Option, pub recovery_hash: Option, pub boot_image: BootImage, } @@ -950,11 +914,6 @@ impl Node { self } - pub fn with_node_reward_type(mut self, node_reward_type: NodeRewardType) -> Self { - self.node_reward_type = Some(node_reward_type); - self - } - pub fn with_recovery_hash(mut self, recovery_hash: String) -> Self { self.recovery_hash = Some(recovery_hash); self diff --git a/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs b/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs index 65c451bb95c0..78f2a3db608f 100644 --- a/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs +++ b/rs/tests/message_routing/xnet/xnet_cloud_engine_isolation_test.rs @@ -62,7 +62,6 @@ fn main() -> Result<()> { fn setup(env: TestEnv) { InternetComputer::new() - .with_api_boundary_nodes(1) .add_subnet(Subnet::fast_single_node(SubnetType::System)) .add_subnet(Subnet::fast_single_node(SubnetType::Application)) .add_subnet(Subnet::fast_single_node(SubnetType::CloudEngine)) diff --git a/rs/tests/nns/BUILD.bazel b/rs/tests/nns/BUILD.bazel index 025bf8da128a..7465b47766d3 100644 --- a/rs/tests/nns/BUILD.bazel +++ b/rs/tests/nns/BUILD.bazel @@ -75,6 +75,7 @@ system_test( "//rs/tests/consensus/utils", "//rs/tests/driver:ic-system-test-driver", "//rs/types/base_types", + "//rs/types/cycles", "//rs/types/types", "//rs/universal_canister/lib", "@crate_index//:anyhow", diff --git a/rs/tests/nns/delete_subnet_test.rs b/rs/tests/nns/delete_subnet_test.rs index 35c9d4fbc158..f497cd57e1d7 100644 --- a/rs/tests/nns/delete_subnet_test.rs +++ b/rs/tests/nns/delete_subnet_test.rs @@ -22,6 +22,7 @@ use ic_system_test_driver::nns::get_subnet_list_from_registry; use ic_system_test_driver::systest; use ic_system_test_driver::util::{UniversalCanister, assert_create_agent, block_on}; use ic_types::{Height, RegistryVersion, SubnetId}; +use ic_types_cycles::CanisterCyclesCostSchedule; use registry_canister::init::RegistryCanisterInitPayloadBuilder; use registry_canister::mutations::do_delete_subnet::DeleteSubnetPayload; use std::collections::BTreeSet; @@ -41,7 +42,6 @@ fn main() -> Result<()> { pub fn setup(env: TestEnv) { InternetComputer::new() - .with_api_boundary_nodes(1) .add_subnet( Subnet::fast(SubnetType::System, NUM_NODES) .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), @@ -56,7 +56,8 @@ pub fn setup(env: TestEnv) { ) .add_subnet( Subnet::fast(SubnetType::CloudEngine, NUM_ENGINE_NODES) - .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)), + .with_dkg_interval_length(Height::from(DKG_INTERVAL_LENGTH)) + .with_cost_schedule(CanisterCyclesCostSchedule::Free), ) .setup_and_start(&env) .expect("failed to setup IC under test"); From 9317ca8973a2ea30d3f28e8c75b7fb75c296ac9e Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 10:01:32 +0000 Subject: [PATCH 36/65] style --- rs/tests/networking/nns_delegation_test.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 0d159eb83180..32a0ffc54ac9 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -114,7 +114,9 @@ const DKG_LENGTH: Height = Height::new(9); fn get_installed_canister_id(env: &TestEnv, subnet_type: SubnetType) -> PrincipalId { env.read_json_object::, _>(INSTALLED_CANISTER_IDS) - .expect("Could not read installed canister IDs from test environment.")[&subnet_type] + .expect("Could not read installed canister IDs from test environment.") + .get(&subnet_type) + .expect("All subnets should have an installed canister") } fn set_installed_canister_ids(env: &TestEnv, canister_ids: BTreeMap) { From 6a4c2098855beb89c28a3ede2f3fe6d4d387223f Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 11:02:11 +0000 Subject: [PATCH 37/65] style: consistency --- rs/tests/networking/nns_delegation_test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 32a0ffc54ac9..1b5f4857ad2b 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -607,7 +607,7 @@ fn validate_delegation( let Some(delegation) = maybe_delegation else { assert!( subnet_type == SubnetType::System, - "Every non-NNS subnet should have a delegation attached to the response" + "Non-NNS subnet should return an NNS delegation with the response" ); // We can return, there is nothing more to be checked. @@ -615,7 +615,7 @@ fn validate_delegation( }; assert!( subnet_type != SubnetType::System, - "NNS subnet should not have a delegation attached to the response" + "NNS subnet should not return an NNS delegation with the response" ); let nns_public_key = env.prep_dir("").unwrap().root_public_key().unwrap(); From 78a19c3eb6f9339cf30b771c75d85465bf8d8f33 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 11:05:56 +0000 Subject: [PATCH 38/65] style: use const --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 6c1f5c62a6dd..0a9a73341d66 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -699,6 +699,7 @@ mod tests { const UNKNOWN_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_3; const CLOUD_ENGINE_NODE_ID: NodeId = ic_test_utilities_types::ids::NODE_4; const API_BN_ID: NodeId = ic_test_utilities_types::ids::NODE_5; + const API_BN_DOMAIN: &str = "domain.invalid"; // Get a free port on this host to which we can connect transport to. fn get_free_localhost_socket_addr() -> SocketAddr { @@ -872,7 +873,7 @@ mod tests { &make_node_record_key(API_BN_ID), registry_version.into(), Some(NodeRecord { - domain: Some("domain.invalid".to_string()), + domain: Some(API_BN_DOMAIN.to_string()), ..Default::default() }), ) From 0eef91d2a92f43bf5147b3a082b287a28a9b78bc Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 11:47:26 +0000 Subject: [PATCH 39/65] refactor: deduplicate code --- .../src/nns_delegation_manager.rs | 142 ++++++++---------- 1 file changed, 62 insertions(+), 80 deletions(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 0a9a73341d66..1d2a257e4bc9 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -28,13 +28,14 @@ use ic_types::{ time::expiry_time_from_now, }; use rand::Rng; +use rustls::{ClientConfig, pki_types::ServerName}; use tokio::{ - net::TcpStream, + net::{TcpStream, lookup_host}, sync::watch, task::JoinHandle, time::{sleep, timeout}, }; -use tokio_rustls::{TlsConnector, client::TlsStream}; +use tokio_rustls::TlsConnector; use tokio_util::sync::CancellationToken; use tower::BoxError; @@ -434,7 +435,7 @@ async fn connect( NodeRewardType::Unspecified }); - let tls_stream = match node_reward_type { + let (peer_id, addr, domain, tls_client_config) = match node_reward_type { NodeRewardType::Unspecified | NodeRewardType::Type0 | NodeRewardType::Type1 @@ -447,115 +448,96 @@ async fn connect( format!("Could not find a node from the NNS to talk to. Error: {err}") })?; - connect_to_nns_node(log, peer_id, endpoint, registry_client, tls_config).await + let registry_version = registry_client.get_latest_version(); + + let ip_addr = endpoint + .ip_addr + .parse() + .map_err(|err| format!("Failed to parse the ip addr: {err}"))?; + + let addr = SocketAddr::new(ip_addr, endpoint.port as u16); + + let tls_client_config = tls_config + .client_config(peer_id, registry_version) + .map_err(|err| format!("Retrieving TLS client config failed: {err:?}."))?; + + let irrelevant_domain = "domain.is-irrelevant-as-hostname-verification-is.disabled" + .try_into() + // TODO: ideally the expect should run at compile time + .expect("failed to create domain"); + + (peer_id, addr, irrelevant_domain, tls_client_config) } NodeRewardType::Type4 => { let (api_bn_id, domain) = get_random_api_boundary_node(registry_client) .map_err(|err| format!("Could not find an API BN to talk to. Error: {err}"))?; - connect_to_api_bn(log, api_bn_id, domain).await - } - }?; + let addr = lookup_host((domain.as_str(), 443_u16)) + .await? + .next() + .ok_or_else(|| { + format!("API BN domain {domain} does not resolve to any IP address.",) + })?; - info!( - log, - "Establishing HTTP connection. Tls stream: {tls_stream:?}" - ); - let (request_sender, connection) = - hyper::client::conn::http1::handshake(TokioIo::new(tls_stream)).await?; + let mut root_store = rustls::RootCertStore::empty(); + root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + let tls_client_config = rustls::ClientConfig::builder() + .with_root_certificates(root_store) + .with_no_client_auth(); - // Spawn a task to poll the connection, driving the HTTP state - let log_cl = log.clone(); - rt_handle.spawn(async move { - if let Err(err) = connection.await { - warn!(log_cl, "Polling connection failed: {err:?}."); + let domain = domain + .clone() + .try_into() + .map_err(|err| format!("Invalid API BN domain {domain}: {err}"))?; + + (api_bn_id, addr, domain, tls_client_config) } - }); + }; - Ok(request_sender) + connect_to(log, rt_handle, peer_id, addr, domain, tls_client_config).await } -async fn connect_to_nns_node( +async fn connect_to( log: &ReplicaLogger, + rt_handle: &tokio::runtime::Handle, peer_id: NodeId, - endpoint: ConnectionEndpoint, - registry_client: &dyn RegistryClient, - tls_config: &(dyn TlsConfig + Send + Sync), -) -> Result, BoxError> { - let registry_version = registry_client.get_latest_version(); - - let ip_addr = endpoint - .ip_addr - .parse() - .map_err(|err| format!("Failed to parse the ip addr: {err}"))?; - - let addr = SocketAddr::new(ip_addr, endpoint.port as u16); - - let tls_client_config = tls_config - .client_config(peer_id, registry_version) - .map_err(|err| format!("Retrieving TLS client config failed: {err:?}."))?; - + addr: SocketAddr, + domain: ServerName<'static>, + tls_client_config: ClientConfig, +) -> Result, BoxError> { info!(log, "Establishing TCP connection to {peer_id} @ {addr}"); let tcp_stream: TcpStream = TcpStream::connect(addr) .await .map_err(|err| format!("Could not connect to node {peer_id}. {err:?}."))?; let tls_connector = TlsConnector::from(Arc::new(tls_client_config)); - let irrelevant_domain = "domain.is-irrelevant-as-hostname-verification-is.disabled"; info!( log, "Establishing TLS stream to {peer_id}. Tcp stream: {tcp_stream:?}" ); + let tls_stream = tls_connector - .connect( - irrelevant_domain - .try_into() - // TODO: ideally the expect should run at compile time - .expect("failed to create domain"), - tcp_stream, - ) + .connect(domain, tcp_stream) .await .map_err(|err| format!("Could not establish TLS stream to node {peer_id}. {err:?}."))?; - Ok(tls_stream) -} - -async fn connect_to_api_bn( - log: &ReplicaLogger, - api_bn_id: NodeId, - domain: String, -) -> Result, BoxError> { - let mut root_store = rustls::RootCertStore::empty(); - root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); - let tls_client_config = rustls::ClientConfig::builder() - .with_root_certificates(root_store) - .with_no_client_auth(); - - let addr = (domain.as_str(), 443_u16); - info!(log, "Establishing TCP connection to {api_bn_id} @ {addr:?}"); - let tcp_stream: TcpStream = TcpStream::connect(addr) - .await - .map_err(|err| format!("Could not connect to node {api_bn_id}. {err:?}."))?; - - let tls_connector = TlsConnector::from(Arc::new(tls_client_config)); - info!( log, - "Establishing TLS stream to {api_bn_id}. Tcp stream: {tcp_stream:?}" + "Establishing HTTP connection to {peer_id}. Tls stream: {tls_stream:?}" ); - let tls_stream = tls_connector - .connect( - domain - .clone() - .try_into() - .map_err(|err| format!("Invalid API BN domain {domain}: {err}"))?, - tcp_stream, - ) - .await - .map_err(|err| format!("Could not establish TLS stream to node {api_bn_id}. {err:?}."))?; + let (request_sender, connection) = + hyper::client::conn::http1::handshake(TokioIo::new(tls_stream)).await?; - Ok(tls_stream) + // Spawn a task to poll the connection, driving the HTTP state + let log_cl = log.clone(); + rt_handle.spawn(async move { + if let Err(err) = connection.await { + warn!(log_cl, "Polling connection failed: {err:?}."); + } + }); + + Ok(request_sender) } fn get_node_reward_type( From 561370080e5268e0d6d31e3f51f82cadbcd8b817 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 12:19:39 +0000 Subject: [PATCH 40/65] fix --- rs/tests/networking/nns_delegation_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 1b5f4857ad2b..32b6e413941e 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -113,7 +113,7 @@ const RETRY_DELAY: tokio::time::Duration = tokio::time::Duration::from_secs(60); const DKG_LENGTH: Height = Height::new(9); fn get_installed_canister_id(env: &TestEnv, subnet_type: SubnetType) -> PrincipalId { - env.read_json_object::, _>(INSTALLED_CANISTER_IDS) + *env.read_json_object::, _>(INSTALLED_CANISTER_IDS) .expect("Could not read installed canister IDs from test environment.") .get(&subnet_type) .expect("All subnets should have an installed canister") From e8c1f219d081537052175b264c458d6e6dc4cbce Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 12:28:42 +0000 Subject: [PATCH 41/65] style: revert unrelated changes --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 1d2a257e4bc9..1ed82592c0d9 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -508,7 +508,7 @@ async fn connect_to( info!(log, "Establishing TCP connection to {peer_id} @ {addr}"); let tcp_stream: TcpStream = TcpStream::connect(addr) .await - .map_err(|err| format!("Could not connect to node {peer_id}. {err:?}."))?; + .map_err(|err| format!("Could not connect to node {addr}. {err:?}."))?; let tls_connector = TlsConnector::from(Arc::new(tls_client_config)); @@ -516,11 +516,10 @@ async fn connect_to( log, "Establishing TLS stream to {peer_id}. Tcp stream: {tcp_stream:?}" ); - let tls_stream = tls_connector .connect(domain, tcp_stream) .await - .map_err(|err| format!("Could not establish TLS stream to node {peer_id}. {err:?}."))?; + .map_err(|err| format!("Could not establish TLS stream to node {addr}. {err:?}."))?; info!( log, From b3c86092223bb18cdbe3c0eca5dcc04a0a856954 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 12:33:45 +0000 Subject: [PATCH 42/65] re-trigger CI From f094abb848d7b902390c3a91f313d7a0fe50e980 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Mon, 30 Mar 2026 13:47:17 +0000 Subject: [PATCH 43/65] style --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 1ed82592c0d9..f07a1a8143be 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -479,8 +479,8 @@ async fn connect( format!("API BN domain {domain} does not resolve to any IP address.",) })?; - let mut root_store = rustls::RootCertStore::empty(); - root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + let root_store = + rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); let tls_client_config = rustls::ClientConfig::builder() .with_root_certificates(root_store) .with_no_client_auth(); From f7abb645d365f2f996824b7359a08e8f9e360fe8 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 07:54:37 +0000 Subject: [PATCH 44/65] refactor: reuse canister_installer --- rs/tests/driver/src/driver/test_env_api.rs | 30 +++++++++++++++++++--- rs/tests/networking/BUILD.bazel | 1 - rs/tests/networking/nns_delegation_test.rs | 18 +++---------- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/rs/tests/driver/src/driver/test_env_api.rs b/rs/tests/driver/src/driver/test_env_api.rs index 99cea435e849..2cd1657fdb89 100644 --- a/rs/tests/driver/src/driver/test_env_api.rs +++ b/rs/tests/driver/src/driver/test_env_api.rs @@ -1057,6 +1057,13 @@ impl IcNodeSnapshot { ) } + pub fn canister_installer_with_raw_wasm<'a>( + &'a self, + raw_wasm: Vec, + ) -> CanisterInstaller<'a> { + CanisterInstaller::new_with_raw_wasm(self, raw_wasm) + } + /// Load wasm binary from the artifacts directory (see [HasArtifacts]) and /// install it on the target node. /// @@ -1198,9 +1205,13 @@ impl IcNodeSnapshot { } } +enum WasmSource { + Path(String), + Raw(Vec), +} pub struct CanisterInstaller<'a> { node: &'a IcNodeSnapshot, - wasm_name: String, + wasm: WasmSource, arg: Option>, cycles_amount: Option, effective_canister_id: PrincipalId, @@ -1209,7 +1220,17 @@ pub struct CanisterInstaller<'a> { impl<'a> CanisterInstaller<'a> { pub fn new(node: &'a IcNodeSnapshot, wasm_name: impl ToString) -> Self { Self { - wasm_name: wasm_name.to_string(), + wasm: WasmSource::Path(wasm_name.to_string()), + arg: None, + cycles_amount: None, + effective_canister_id: node.effective_canister_id(), + node, + } + } + + pub fn new_with_raw_wasm(node: &'a IcNodeSnapshot, raw_wasm: Vec) -> Self { + Self { + wasm: WasmSource::Raw(raw_wasm), arg: None, cycles_amount: None, effective_canister_id: node.effective_canister_id(), @@ -1240,7 +1261,10 @@ impl<'a> CanisterInstaller<'a> { } pub async fn install(self) -> Result { - let canister_bytes = load_wasm(self.wasm_name); + let canister_bytes = match self.wasm { + WasmSource::Path(path) => load_wasm(path), + WasmSource::Raw(bytes) => bytes, + }; let agent = self.node.build_default_agent_async().await; // Create a canister. let mgr = ManagementCanister::create(&agent); diff --git a/rs/tests/networking/BUILD.bazel b/rs/tests/networking/BUILD.bazel index 119c1e90a6ec..cc461610d48a 100644 --- a/rs/tests/networking/BUILD.bazel +++ b/rs/tests/networking/BUILD.bazel @@ -245,7 +245,6 @@ rust_binary( "@crate_index//:candid", "@crate_index//:futures", "@crate_index//:ic-agent", - "@crate_index//:ic-utils", "@crate_index//:leb128", "@crate_index//:reqwest", "@crate_index//:serde", diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 32b6e413941e..ac40fee7880c 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -69,7 +69,6 @@ use ic_types::{ HttpQueryResponseReply, HttpReadStateResponse, NodeSignature, }, }; -use ic_utils::interfaces::ManagementCanister; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use slog::info; @@ -157,20 +156,9 @@ fn setup(env: TestEnv) { let wasm = wasm.clone(); async move { let (_subnet, node) = get_subnet_and_node(&env, subnet_type); - let agent = node.build_default_agent_async().await; - let management_canister = ManagementCanister::create(&agent); - let canister_id = management_canister - .create_canister() - .as_provisional_create_with_amount(None) - .with_effective_canister_id(node.effective_canister_id()) - .call_and_wait() - .await - .expect("Failed to create the certified variables canister") - .0; - - management_canister - .install_code(&canister_id, &wasm) - .call_and_wait() + let canister_id = node + .canister_installer_with_raw_wasm(wasm) + .install() .await .expect("Failed to install the certified variables canister"); From afd4641f1f5d139b586468160f207a557c0b2c08 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 08:35:36 +0000 Subject: [PATCH 45/65] refactor: test VerifiedApplication and use const --- rs/tests/networking/nns_delegation_test.rs | 77 +++++++++++----------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index ac40fee7880c..b97d85b9bd5b 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -104,6 +104,13 @@ const CERTIFIED_VAR_WAT: &str = r#" ) "#; +const TESTED_SUBNET_TYPES: [SubnetType; 4] = [ + SubnetType::System, + SubnetType::Application, + SubnetType::VerifiedApplication, + SubnetType::CloudEngine, +]; + const INSTALLED_CANISTER_IDS: &str = "installed_canister_ids"; /// How long to wait between subsequent nns delegation fetch requests. @@ -124,18 +131,12 @@ fn set_installed_canister_ids(env: &TestEnv, canister_ids: BTreeMap { + // Keep up to date with `TESTED_SUBNET_TYPES` constant $group = $group.add_test(systest!($function_name; SubnetType::System)); $group = $group.add_test(systest!($function_name; SubnetType::Application)); + $group = $group.add_test(systest!($function_name; SubnetType::VerifiedApplication)); // TODO(CON-1696): Remove this condition (and always run the test for cloud engines) when // #9613 reaches mainnet NNS if get_guestos_img_version() == get_guestos_update_img_version() { From 508e5af308b1493e520b2518b861a6bdcb7f1b9e Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 08:41:11 +0000 Subject: [PATCH 46/65] refactor: take ownership of logger --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index f07a1a8143be..5d8d1bcee891 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -290,7 +290,7 @@ async fn try_fetch_delegation_from_nns( let mut request_sender = timeout( CONNECTION_TIMEOUT, connect( - log, + log.clone(), rt_handle, node_id, nns_subnet_id, @@ -423,7 +423,7 @@ async fn try_fetch_delegation_from_nns( } async fn connect( - log: &ReplicaLogger, + log: ReplicaLogger, rt_handle: &tokio::runtime::Handle, node_id: NodeId, nns_subnet_id: SubnetId, @@ -498,7 +498,7 @@ async fn connect( } async fn connect_to( - log: &ReplicaLogger, + log: ReplicaLogger, rt_handle: &tokio::runtime::Handle, peer_id: NodeId, addr: SocketAddr, @@ -529,10 +529,9 @@ async fn connect_to( hyper::client::conn::http1::handshake(TokioIo::new(tls_stream)).await?; // Spawn a task to poll the connection, driving the HTTP state - let log_cl = log.clone(); rt_handle.spawn(async move { if let Err(err) = connection.await { - warn!(log_cl, "Polling connection failed: {err:?}."); + warn!(log, "Polling connection failed: {err:?}."); } }); From 19dd0d9306ba06e855b26cf2d454d757996bc39b Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 08:51:30 +0000 Subject: [PATCH 47/65] refactor: extract common logic to function --- .../src/nns_delegation_manager.rs | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 5d8d1bcee891..c47d237719ca 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -15,8 +15,11 @@ use ic_logger::{ReplicaLogger, info, warn}; use ic_metrics::MetricsRegistry; use ic_protobuf::registry::node::v1::NodeRewardType; use ic_registry_client_helpers::{ - api_boundary_node::ApiBoundaryNodeRegistry, crypto::CryptoRegistry, node::NodeRegistry, - node_operator::ConnectionEndpoint, subnet::SubnetRegistry, + api_boundary_node::ApiBoundaryNodeRegistry, + crypto::CryptoRegistry, + node::{NodeRecord, NodeRegistry}, + node_operator::ConnectionEndpoint, + subnet::SubnetRegistry, }; use ic_types::{ NodeId, RegistryVersion, SubnetId, @@ -27,7 +30,7 @@ use ic_types::{ }, time::expiry_time_from_now, }; -use rand::Rng; +use rand::{Rng, seq::SliceRandom}; use rustls::{ClientConfig, pki_types::ServerName}; use tokio::{ net::{TcpStream, lookup_host}, @@ -557,8 +560,6 @@ fn get_random_node_from_nns_subnet( registry_client: &dyn RegistryClient, nns_subnet_id: SubnetId, ) -> Result<(NodeId, ConnectionEndpoint), String> { - use rand::seq::SliceRandom; - let nns_nodes = match registry_client .get_node_ids_on_subnet(nns_subnet_id, registry_client.get_latest_version()) { @@ -567,44 +568,43 @@ fn get_random_node_from_nns_subnet( Err(err) => Err(format!("Failed to get NNS nodes from registry: {err}")), }?; - // Randomly choose a node from the nns subnet. - let mut rng = rand::thread_rng(); - let nns_node = nns_nodes.choose(&mut rng).ok_or(format!( - "Failed to choose a random NNS node. NNS node list: {nns_nodes:?}" - ))?; - match registry_client.get_node_record(*nns_node, registry_client.get_latest_version()) { - Ok(Some(node)) => Ok(( - *nns_node, - node.http.ok_or("No HTTP endpoint for node {nns_node}")?, - )), - Ok(None) => Err(format!("No node record found for NNS node {nns_node}")), - Err(err) => Err(format!( - "Failed to get node record for NNS node {nns_node}. Err: {err}" - )), - } + let (node_id, record) = get_random_node_record_from_ids(registry_client, &nns_nodes)?; + let endpoint = record + .http + .ok_or_else(|| format!("No HTTP endpoint for NNS node {node_id}"))?; + Ok((node_id, endpoint)) } fn get_random_api_boundary_node( registry_client: &dyn RegistryClient, ) -> Result<(NodeId, String), String> { - use rand::seq::SliceRandom; - let api_bns = match registry_client.get_api_boundary_node_ids(registry_client.get_latest_version()) { Ok(api_bns) => Ok(api_bns), Err(err) => Err(format!("Failed to get API BNs from registry: {err}")), }?; - // Randomly choose a node from the API boundary nodes. + let (node_id, record) = get_random_node_record_from_ids(registry_client, &api_bns)?; + let domain = record + .domain + .ok_or_else(|| format!("No domain for API BN {node_id}"))?; + Ok((node_id, domain)) +} + +fn get_random_node_record_from_ids( + registry_client: &dyn RegistryClient, + node_ids: &[NodeId], +) -> Result<(NodeId, NodeRecord), String> { let mut rng = rand::thread_rng(); - let api_bn = api_bns.choose(&mut rng).ok_or(format!( - "Failed to choose a random API boundary node. API BN list: {api_bns:?}" + let node_id = node_ids.choose(&mut rng).ok_or(format!( + "Failed to choose a random node. Node list: {node_ids:?}" ))?; - match registry_client.get_node_record(*api_bn, registry_client.get_latest_version()) { - Ok(Some(node)) => Ok((*api_bn, node.domain.ok_or("No domain for node {api_bn}")?)), - Ok(None) => Err(format!("No node record found for API BN {api_bn}")), + + match registry_client.get_node_record(*node_id, registry_client.get_latest_version()) { + Ok(Some(record)) => Ok((*node_id, record)), + Ok(None) => Err(format!("No node record found for node id {node_id}")), Err(err) => Err(format!( - "Failed to get node record for API BN {api_bn}. Err: {err}" + "Failed to get node record for node id {node_id}. Err: {err}" )), } } From 8347a2580fccf65c2eac1515600959d6edd67e36 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 08:56:53 +0000 Subject: [PATCH 48/65] style: remove type suffix --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index c47d237719ca..ffe4bc8b4dde 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -475,7 +475,7 @@ async fn connect( let (api_bn_id, domain) = get_random_api_boundary_node(registry_client) .map_err(|err| format!("Could not find an API BN to talk to. Error: {err}"))?; - let addr = lookup_host((domain.as_str(), 443_u16)) + let addr = lookup_host((domain.as_str(), 443)) .await? .next() .ok_or_else(|| { From 7d76486ea530a47f933f5d2d9c5c07e8acda20de Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 08:57:11 +0000 Subject: [PATCH 49/65] style: assert_eq/assert_ne --- rs/tests/networking/nns_delegation_test.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index b97d85b9bd5b..5b3f7537b764 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -184,8 +184,9 @@ async fn nns_delegation_updates_async(env: TestEnv, subnet_type: SubnetType) { get_nns_delegation_timestamp(&agent, node.effective_canister_id()).await; let Some(initial_delegation_timestamp) = maybe_initial_delegation_timestamp else { - assert!( - subnet_type == SubnetType::System, + assert_eq!( + subnet_type, + SubnetType::System, "Non-NNS subnet should return an NNS delegation with the response" ); @@ -193,8 +194,9 @@ async fn nns_delegation_updates_async(env: TestEnv, subnet_type: SubnetType) { return; }; - assert!( - subnet_type != SubnetType::System, + assert_ne!( + subnet_type, + SubnetType::System, "NNS subnet should not return an NNS delegation with the response" ); @@ -589,16 +591,18 @@ fn validate_delegation( expected_delegation_format: CertificateDelegationFormat, ) { let Some(delegation) = maybe_delegation else { - assert!( - subnet_type == SubnetType::System, + assert_eq!( + subnet_type, + SubnetType::System, "Non-NNS subnet should return an NNS delegation with the response" ); // We can return, there is nothing more to be checked. return; }; - assert!( - subnet_type != SubnetType::System, + assert_ne!( + subnet_type, + SubnetType::System, "NNS subnet should not return an NNS delegation with the response" ); From 70efb941bd2711f04e939941fee50f598003c817 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 09:13:39 +0000 Subject: [PATCH 50/65] refactor: use subnet_type_as_string --- Cargo.lock | 1 + rs/canonical_state/src/lazy_tree_conversion.rs | 2 +- rs/tests/networking/BUILD.bazel | 1 + rs/tests/networking/Cargo.toml | 1 + rs/tests/networking/nns_delegation_test.rs | 3 ++- 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b278d71d16e..b19bd0382674 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18290,6 +18290,7 @@ dependencies = [ "futures", "ic-agent", "ic-base-types", + "ic-canonical-state", "ic-cdk", "ic-certification 0.9.0", "ic-crypto-tree-hash", diff --git a/rs/canonical_state/src/lazy_tree_conversion.rs b/rs/canonical_state/src/lazy_tree_conversion.rs index f5d81e240715..e6ee9a048812 100644 --- a/rs/canonical_state/src/lazy_tree_conversion.rs +++ b/rs/canonical_state/src/lazy_tree_conversion.rs @@ -978,7 +978,7 @@ fn canister_metadata_as_tree( /// Helper function to turn a subnet type into a string. /// This is intentionally explicitly implemented here, so that the state tree representation cannot be changed outside this crate, as opposed /// to calling something like `subnet_type.to_string()`. -fn subnet_type_as_string(subnet_type: SubnetType) -> &'static str { +pub fn subnet_type_as_string(subnet_type: SubnetType) -> &'static str { match subnet_type { SubnetType::Application => "application", SubnetType::System => "system", diff --git a/rs/tests/networking/BUILD.bazel b/rs/tests/networking/BUILD.bazel index cc461610d48a..1dc87860b160 100644 --- a/rs/tests/networking/BUILD.bazel +++ b/rs/tests/networking/BUILD.bazel @@ -233,6 +233,7 @@ rust_binary( crate_name = "ic_systest_nns_delegation_test", deps = [ # Keep sorted. + "//rs/canonical_state", "//rs/certification", "//rs/crypto/tree_hash", "//rs/crypto/utils/threshold_sig_der", diff --git a/rs/tests/networking/Cargo.toml b/rs/tests/networking/Cargo.toml index 73f95e69af18..de51dbf7e1d8 100644 --- a/rs/tests/networking/Cargo.toml +++ b/rs/tests/networking/Cargo.toml @@ -17,6 +17,7 @@ dfn_candid = { path = "../../rust_canisters/dfn_candid" } futures = { workspace = true } ic-agent = { workspace = true } ic-base-types = { path = "../../types/base_types" } +ic-canonical-state = { path = "../../canonical_state" } ic-cdk = { workspace = true } ic-certification = { path = "../../certification" } ic-crypto-tree-hash = { path = "../../crypto/tree_hash" } diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 5b3f7537b764..65270d02b19a 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -38,6 +38,7 @@ use ic_agent::{ Agent, Identity, agent::{Envelope, EnvelopeContent}, }; +use ic_canonical_state::lazy_tree_conversion::subnet_type_as_string; use ic_certification::verify_delegation_certificate; use ic_consensus_system_test_utils::{ rw_message::install_nns_and_check_progress, @@ -634,7 +635,7 @@ fn validate_delegation( LabeledTree::Leaf(value) => { assert_eq!( value, - subnet_type.as_ref().as_bytes(), + subnet_type_as_string(subnet_type).as_bytes(), "Subnet type in the delegation should match the subnet type of the responding node" ); } From 9cb58a6ff59d40552322d579e007e1670a4ff957 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 09:38:28 +0000 Subject: [PATCH 51/65] feat: log connecting to an NNS node directly --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index ffe4bc8b4dde..12402015ceb5 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -434,7 +434,10 @@ async fn connect( tls_config: &(dyn TlsConfig + Send + Sync), ) -> Result, BoxError> { let node_reward_type = get_node_reward_type(registry_client, node_id).unwrap_or_else(|err| { - warn!(log, "Could not determine the reward type: {err}"); + warn!( + log, + "Could not determine the reward type: {err}. Connecting to an NNS node directly." + ); NodeRewardType::Unspecified }); From 0cdc46cfd8a6d52fcd34b9abd5084df3daeb6554 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 10:19:33 +0000 Subject: [PATCH 52/65] refactor: simplify CanisterInstaller --- rs/tests/driver/src/driver/test_env_api.rs | 29 +++++++--------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/rs/tests/driver/src/driver/test_env_api.rs b/rs/tests/driver/src/driver/test_env_api.rs index 2cd1657fdb89..9963b937413f 100644 --- a/rs/tests/driver/src/driver/test_env_api.rs +++ b/rs/tests/driver/src/driver/test_env_api.rs @@ -1205,32 +1205,25 @@ impl IcNodeSnapshot { } } -enum WasmSource { - Path(String), - Raw(Vec), -} pub struct CanisterInstaller<'a> { node: &'a IcNodeSnapshot, - wasm: WasmSource, + wasm: Vec, arg: Option>, cycles_amount: Option, effective_canister_id: PrincipalId, } impl<'a> CanisterInstaller<'a> { - pub fn new(node: &'a IcNodeSnapshot, wasm_name: impl ToString) -> Self { - Self { - wasm: WasmSource::Path(wasm_name.to_string()), - arg: None, - cycles_amount: None, - effective_canister_id: node.effective_canister_id(), - node, - } + pub fn new

(node: &'a IcNodeSnapshot, wasm_name: P) -> Self + where + P: AsRef, + { + Self::new_with_raw_wasm(node, load_wasm(wasm_name)) } - pub fn new_with_raw_wasm(node: &'a IcNodeSnapshot, raw_wasm: Vec) -> Self { + pub fn new_with_raw_wasm(node: &'a IcNodeSnapshot, wasm: Vec) -> Self { Self { - wasm: WasmSource::Raw(raw_wasm), + wasm, arg: None, cycles_amount: None, effective_canister_id: node.effective_canister_id(), @@ -1261,10 +1254,6 @@ impl<'a> CanisterInstaller<'a> { } pub async fn install(self) -> Result { - let canister_bytes = match self.wasm { - WasmSource::Path(path) => load_wasm(path), - WasmSource::Raw(bytes) => bytes, - }; let agent = self.node.build_default_agent_async().await; // Create a canister. let mgr = ManagementCanister::create(&agent); @@ -1277,7 +1266,7 @@ impl<'a> CanisterInstaller<'a> { .map_err(|err| anyhow!("Couldn't create canister with provisional API: {err}"))? .0; - let mut install_code = mgr.install_code(&canister_id, &canister_bytes); + let mut install_code = mgr.install_code(&canister_id, &self.wasm); if let Some(arg) = self.arg { install_code = install_code.with_raw_arg(arg) } From 3f7a1cd94fe5b8d76cbd3a9f4904c4086fc091c3 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 11:55:49 +0000 Subject: [PATCH 53/65] feat: use IP as server name for regular nodes --- .../src/nns_delegation_manager.rs | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 12402015ceb5..19618110f5fc 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -441,7 +441,7 @@ async fn connect( NodeRewardType::Unspecified }); - let (peer_id, addr, domain, tls_client_config) = match node_reward_type { + let (peer_id, addr, server_name, tls_client_config) = match node_reward_type { NodeRewardType::Unspecified | NodeRewardType::Type0 | NodeRewardType::Type1 @@ -467,12 +467,9 @@ async fn connect( .client_config(peer_id, registry_version) .map_err(|err| format!("Retrieving TLS client config failed: {err:?}."))?; - let irrelevant_domain = "domain.is-irrelevant-as-hostname-verification-is.disabled" - .try_into() - // TODO: ideally the expect should run at compile time - .expect("failed to create domain"); + let server_name = ServerName::from(ip_addr); - (peer_id, addr, irrelevant_domain, tls_client_config) + (peer_id, addr, server_name, tls_client_config) } NodeRewardType::Type4 => { let (api_bn_id, domain) = get_random_api_boundary_node(registry_client) @@ -491,16 +488,22 @@ async fn connect( .with_root_certificates(root_store) .with_no_client_auth(); - let domain = domain - .clone() - .try_into() + let server_name = ServerName::try_from(domain.clone()) .map_err(|err| format!("Invalid API BN domain {domain}: {err}"))?; - (api_bn_id, addr, domain, tls_client_config) + (api_bn_id, addr, server_name, tls_client_config) } }; - connect_to(log, rt_handle, peer_id, addr, domain, tls_client_config).await + connect_to( + log, + rt_handle, + peer_id, + addr, + server_name, + tls_client_config, + ) + .await } async fn connect_to( @@ -508,7 +511,7 @@ async fn connect_to( rt_handle: &tokio::runtime::Handle, peer_id: NodeId, addr: SocketAddr, - domain: ServerName<'static>, + server_name: ServerName<'static>, tls_client_config: ClientConfig, ) -> Result, BoxError> { info!(log, "Establishing TCP connection to {peer_id} @ {addr}"); @@ -523,7 +526,7 @@ async fn connect_to( "Establishing TLS stream to {peer_id}. Tcp stream: {tcp_stream:?}" ); let tls_stream = tls_connector - .connect(domain, tcp_stream) + .connect(server_name, tcp_stream) .await .map_err(|err| format!("Could not establish TLS stream to node {addr}. {err:?}."))?; From 48c60bccac2798361df7aeb6803b6f96b1f8e133 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 12:42:13 +0000 Subject: [PATCH 54/65] feat: resolve to IPv6 --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 19618110f5fc..b7d5f3e05fee 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -477,9 +477,10 @@ async fn connect( let addr = lookup_host((domain.as_str(), 443)) .await? + .filter(|addr| addr.is_ipv6()) .next() .ok_or_else(|| { - format!("API BN domain {domain} does not resolve to any IP address.",) + format!("API BN domain {domain} does not resolve to any IPv6 address.",) })?; let root_store = From c5ada4c37338331d29713c07eeaf6b46f279c829 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 12:54:47 +0000 Subject: [PATCH 55/65] style: clippy --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index b7d5f3e05fee..291a6e14704d 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -477,8 +477,7 @@ async fn connect( let addr = lookup_host((domain.as_str(), 443)) .await? - .filter(|addr| addr.is_ipv6()) - .next() + .find(|addr| addr.is_ipv6()) .ok_or_else(|| { format!("API BN domain {domain} does not resolve to any IPv6 address.",) })?; From 9adf1ed566097138c16249b68e289c1a386899b9 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 15:04:28 +0000 Subject: [PATCH 56/65] feat: use hickory-resolver --- Cargo.Bazel.json.lock | 7 ++++++- Cargo.Bazel.toml.lock | 1 + Cargo.lock | 1 + Cargo.toml | 1 + bazel/rust.MODULE.bazel | 4 ++++ .../nns_delegation_manager/BUILD.bazel | 1 + .../nns_delegation_manager/Cargo.toml | 1 + .../src/nns_delegation_manager.rs | 18 +++++++++++++----- 8 files changed, 28 insertions(+), 6 deletions(-) diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index f9bc51e5b0cf..cfe4e1ba2e29 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "781fc130dc0a872413d66d369cacf838edebb741dae07ba64c9d0040276d0d3a", + "checksum": "1d88faead9fe3d4ca18cc2287397ac1457604de13e94aded554c8104ad506e69", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -21686,6 +21686,10 @@ "id": "hex-literal 0.4.1", "target": "hex_literal" }, + { + "id": "hickory-resolver 0.25.2", + "target": "hickory_resolver" + }, { "id": "hkdf 0.12.4", "target": "hkdf" @@ -98646,6 +98650,7 @@ "hashlink 0.8.3", "hex 0.4.3", "hex-literal 0.4.1", + "hickory-resolver 0.25.2", "hkdf 0.12.4", "hmac 0.12.1", "hpke 0.12.0", diff --git a/Cargo.Bazel.toml.lock b/Cargo.Bazel.toml.lock index 69e05a60dc1b..a1094b9f98b8 100644 --- a/Cargo.Bazel.toml.lock +++ b/Cargo.Bazel.toml.lock @@ -3713,6 +3713,7 @@ dependencies = [ "hashlink 0.8.3", "hex", "hex-literal 0.4.1", + "hickory-resolver", "hkdf", "hmac", "hpke", diff --git a/Cargo.lock b/Cargo.lock index c5906719e164..aa13c2d7b565 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12003,6 +12003,7 @@ dependencies = [ "axum-server", "criterion", "futures", + "hickory-resolver", "http-body-util", "hyper 1.8.1", "hyper-util", diff --git a/Cargo.toml b/Cargo.toml index 8de1f9475b11..a8348181414f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -666,6 +666,7 @@ getrandom = { version = "0.2", features = ["custom"] } goldenfile = "1.8.0" gpt = "4.1" hex = { version = "0.4.3", features = ["serde"] } +hickory-resolver = "0.25.2" hkdf = "^0.12" http = "1.3.1" http-body = "1.0.1" diff --git a/bazel/rust.MODULE.bazel b/bazel/rust.MODULE.bazel index 343bc3b24ded..7a46503ef063 100644 --- a/bazel/rust.MODULE.bazel +++ b/bazel/rust.MODULE.bazel @@ -537,6 +537,10 @@ crate.spec( package = "hex-literal", version = "^0.4.1", ) +crate.spec( + package = "hickory-resolver", + version = "0.25.2" +) crate.spec( package = "hkdf", version = "^0.12", diff --git a/rs/http_endpoints/nns_delegation_manager/BUILD.bazel b/rs/http_endpoints/nns_delegation_manager/BUILD.bazel index 51febcadeddb..e22242afe70b 100644 --- a/rs/http_endpoints/nns_delegation_manager/BUILD.bazel +++ b/rs/http_endpoints/nns_delegation_manager/BUILD.bazel @@ -27,6 +27,7 @@ rust_library( "//rs/types/types", "@crate_index//:axum", "@crate_index//:futures", + "@crate_index//:hickory-resolver", "@crate_index//:http-body-util", "@crate_index//:hyper", "@crate_index//:hyper-util", diff --git a/rs/http_endpoints/nns_delegation_manager/Cargo.toml b/rs/http_endpoints/nns_delegation_manager/Cargo.toml index 0d54aa0040e5..4ed31772e014 100644 --- a/rs/http_endpoints/nns_delegation_manager/Cargo.toml +++ b/rs/http_endpoints/nns_delegation_manager/Cargo.toml @@ -13,6 +13,7 @@ harness = false [dependencies] axum = { workspace = true } futures = { workspace = true } +hickory-resolver = { workspace = true } http-body-util = { workspace = true } hyper = { workspace = true } hyper-util = { workspace = true } diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 291a6e14704d..3f65d7c0656a 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -2,6 +2,7 @@ use std::{convert::TryFrom, net::SocketAddr, sync::Arc, time::Duration}; use axum::body::Body; use futures::FutureExt; +use hickory_resolver::{Resolver, config::LookupIpStrategy}; use http_body_util::{BodyExt, Full, LengthLimitError}; use hyper::{Request, client::conn::http1::SendRequest}; use hyper_util::rt::TokioIo; @@ -33,7 +34,7 @@ use ic_types::{ use rand::{Rng, seq::SliceRandom}; use rustls::{ClientConfig, pki_types::ServerName}; use tokio::{ - net::{TcpStream, lookup_host}, + net::TcpStream, sync::watch, task::JoinHandle, time::{sleep, timeout}, @@ -61,7 +62,7 @@ const DELEGATION_RETRY_MAX_BACKOFF_SECONDS: u64 = 15; #[cfg(not(test))] const CONNECTION_TIMEOUT: Duration = Duration::from_secs(10); #[cfg(test)] -const CONNECTION_TIMEOUT: Duration = Duration::from_secs(1); +const CONNECTION_TIMEOUT: Duration = Duration::from_secs(60); #[cfg(not(test))] const NNS_DELEGATION_BODY_RECEIVE_TIMEOUT: Duration = Duration::from_secs(300); @@ -475,13 +476,20 @@ async fn connect( let (api_bn_id, domain) = get_random_api_boundary_node(registry_client) .map_err(|err| format!("Could not find an API BN to talk to. Error: {err}"))?; - let addr = lookup_host((domain.as_str(), 443)) + let mut dns_resolver = Resolver::builder_tokio()?; + dns_resolver.options_mut().ip_strategy = LookupIpStrategy::Ipv6Only; + let ip_addr = dns_resolver + .build() + .lookup_ip(domain.as_str()) .await? - .find(|addr| addr.is_ipv6()) + .iter() + .next() .ok_or_else(|| { format!("API BN domain {domain} does not resolve to any IPv6 address.",) })?; + let addr = SocketAddr::new(ip_addr, 443); + let root_store = rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); let tls_client_config = rustls::ClientConfig::builder() @@ -1262,7 +1270,7 @@ mod tests { // Since the API BN is configured with a domain that does not resolve, we expect the // connection to fail with a name resolution error, which indicates that we indeed tried to // connect to the API BN instead of an NNS node. - assert_matches!(response, Err(err) if err.to_string().contains("Temporary failure in name resolution")); + assert_matches!(response, Err(err) if format!("{err:?}").contains("ResolveError")); } #[tokio::test] From 62768631b756dd56067b33549a151c7fb8fb8666 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Tue, 31 Mar 2026 15:32:08 +0000 Subject: [PATCH 57/65] Automatically fixing code for linting and formatting issues --- bazel/rust.MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/rust.MODULE.bazel b/bazel/rust.MODULE.bazel index 7a46503ef063..724c8862ece0 100644 --- a/bazel/rust.MODULE.bazel +++ b/bazel/rust.MODULE.bazel @@ -539,7 +539,7 @@ crate.spec( ) crate.spec( package = "hickory-resolver", - version = "0.25.2" + version = "0.25.2", ) crate.spec( package = "hkdf", From 4501d32a17c7f8c6cfabdd8a6399fe33ec361f0b Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Tue, 31 Mar 2026 17:30:00 +0000 Subject: [PATCH 58/65] fix: reduce timeout and attempts in tests --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 3f65d7c0656a..76ca7888f000 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -62,7 +62,7 @@ const DELEGATION_RETRY_MAX_BACKOFF_SECONDS: u64 = 15; #[cfg(not(test))] const CONNECTION_TIMEOUT: Duration = Duration::from_secs(10); #[cfg(test)] -const CONNECTION_TIMEOUT: Duration = Duration::from_secs(60); +const CONNECTION_TIMEOUT: Duration = Duration::from_secs(1); #[cfg(not(test))] const NNS_DELEGATION_BODY_RECEIVE_TIMEOUT: Duration = Duration::from_secs(300); @@ -478,6 +478,13 @@ async fn connect( let mut dns_resolver = Resolver::builder_tokio()?; dns_resolver.options_mut().ip_strategy = LookupIpStrategy::Ipv6Only; + #[cfg(test)] + { + // In unit tests, the domain does not resolve and we want to fail fast to keep a low + // test execution time. + dns_resolver.options_mut().timeout = Duration::from_millis(100); + dns_resolver.options_mut().attempts = 1; + } let ip_addr = dns_resolver .build() .lookup_ip(domain.as_str()) From c9ad435ea3c73e2660098df64940251efc5da455 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Wed, 1 Apr 2026 07:46:29 +0000 Subject: [PATCH 59/65] style: match -> map_err --- .../nns_delegation_manager/src/nns_delegation_manager.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs index 76ca7888f000..9edbfb2097c6 100644 --- a/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs +++ b/rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs @@ -599,11 +599,9 @@ fn get_random_node_from_nns_subnet( fn get_random_api_boundary_node( registry_client: &dyn RegistryClient, ) -> Result<(NodeId, String), String> { - let api_bns = - match registry_client.get_api_boundary_node_ids(registry_client.get_latest_version()) { - Ok(api_bns) => Ok(api_bns), - Err(err) => Err(format!("Failed to get API BNs from registry: {err}")), - }?; + let api_bns = registry_client + .get_api_boundary_node_ids(registry_client.get_latest_version()) + .map_err(|err| format!("Failed to get API BNs from registry: {err}"))?; let (node_id, record) = get_random_node_record_from_ids(registry_client, &api_bns)?; let domain = record From d919185b6d94087ecd792fa1ec328f8144140b7d Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Wed, 1 Apr 2026 08:00:04 +0000 Subject: [PATCH 60/65] perf: upgrade subnets in parallel --- rs/tests/networking/nns_delegation_test.rs | 45 ++++++++++++---------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 65270d02b19a..e6e5be0f6223 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -773,26 +773,31 @@ fn upgrade_non_nns_subnets_if_necessary(env: &TestEnv) { vec![upgrade_url.to_string()], )); - for subnet_type in TESTED_SUBNET_TYPES - .iter() - .copied() - .filter(|t| *t != SubnetType::System) - { - // TODO(CON-1696): Remove me when #9613 reaches mainnet NNS - if subnet_type == SubnetType::CloudEngine { - continue; - } - - let (subnet, node) = get_subnet_and_node(env, subnet_type); - - block_on(deploy_guestos_to_all_subnet_nodes( - &nns_node, - &target_version, - subnet.subnet_id, - )); - - assert_assigned_replica_version(&node, &target_version, env.logger()); - } + block_on(futures::future::join_all( + TESTED_SUBNET_TYPES + .iter() + .copied() + .filter(|t| *t != SubnetType::System) + // TODO(CON-1696): Remove next line when #9613 reaches mainnet NNS + .filter(|t| *t != SubnetType::CloudEngine) + .map(|subnet_type| { + let env = env.clone(); + let nns_node = nns_node.clone(); + let target_version = target_version.clone(); + async move { + let (subnet, node) = get_subnet_and_node(&env, subnet_type); + + deploy_guestos_to_all_subnet_nodes( + &nns_node, + &target_version, + subnet.subnet_id, + ) + .await; + + assert_assigned_replica_version(&node, &target_version, env.logger()); + } + }), + )); } macro_rules! systest_all_subnet_types { From 1e3efdb885bd3a1aa62edafe31da474ef844a358 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Wed, 1 Apr 2026 08:03:18 +0000 Subject: [PATCH 61/65] chore: tighten test timeout by 5 minutes --- rs/tests/networking/nns_delegation_test.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index e6e5be0f6223..90367dca02ff 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -817,9 +817,9 @@ macro_rules! systest_all_subnet_types { fn main() -> Result<()> { let mut test_group = SystemTestGroup::new() .with_setup(setup) - // We potentially upgrade the app subnet in the setup which could take several minutes - .with_overall_timeout(std::time::Duration::from_secs(25 * 60)) - .with_timeout_per_test(std::time::Duration::from_secs(15 * 60)); + // We potentially upgrade subnets in the setup which could take several minutes + .with_overall_timeout(std::time::Duration::from_secs(20 * 60)) + .with_timeout_per_test(std::time::Duration::from_secs(10 * 60)); systest_all_subnet_types!(test_group, nns_delegation_updates); systest_all_subnet_types!( From 6a7e2e06a9bc2ee8d38b8647dcb22d35545b1385 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Wed, 1 Apr 2026 10:13:57 +0000 Subject: [PATCH 62/65] perf: sequential cases but parallel across different subnets --- rs/tests/networking/nns_delegation_test.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 90367dca02ff..d0f8819cbf42 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -51,7 +51,7 @@ use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; use ic_registry_subnet_type::SubnetType; use ic_system_test_driver::{ driver::{ - group::SystemTestGroup, + group::{SystemTestGroup, SystemTestSubGroup}, ic::{InternetComputer, Subnet}, test_env::{HasIcPrepDir, TestEnv}, test_env_api::{ @@ -802,15 +802,18 @@ fn upgrade_non_nns_subnets_if_necessary(env: &TestEnv) { macro_rules! systest_all_subnet_types { ($group: expr, $function_name:path) => { + let mut par_group = SystemTestSubGroup::new(); // Keep up to date with `TESTED_SUBNET_TYPES` constant - $group = $group.add_test(systest!($function_name; SubnetType::System)); - $group = $group.add_test(systest!($function_name; SubnetType::Application)); - $group = $group.add_test(systest!($function_name; SubnetType::VerifiedApplication)); + par_group = par_group.add_test(systest!($function_name; SubnetType::System)); + par_group = par_group.add_test(systest!($function_name; SubnetType::Application)); + par_group = par_group.add_test(systest!($function_name; SubnetType::VerifiedApplication)); // TODO(CON-1696): Remove this condition (and always run the test for cloud engines) when // #9613 reaches mainnet NNS if get_guestos_img_version() == get_guestos_update_img_version() { - $group = $group.add_test(systest!($function_name; SubnetType::CloudEngine)); + par_group = par_group.add_test(systest!($function_name; SubnetType::CloudEngine)); } + + $group = $group.add_parallel(par_group); }; } From acf5a9316fb52456983d57617096971939174468 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Wed, 1 Apr 2026 10:16:24 +0000 Subject: [PATCH 63/65] Revert "perf: sequential cases but parallel across different subnets" This reverts commit 6a7e2e06a9bc2ee8d38b8647dcb22d35545b1385. --- rs/tests/networking/nns_delegation_test.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index d0f8819cbf42..90367dca02ff 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -51,7 +51,7 @@ use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; use ic_registry_subnet_type::SubnetType; use ic_system_test_driver::{ driver::{ - group::{SystemTestGroup, SystemTestSubGroup}, + group::SystemTestGroup, ic::{InternetComputer, Subnet}, test_env::{HasIcPrepDir, TestEnv}, test_env_api::{ @@ -802,18 +802,15 @@ fn upgrade_non_nns_subnets_if_necessary(env: &TestEnv) { macro_rules! systest_all_subnet_types { ($group: expr, $function_name:path) => { - let mut par_group = SystemTestSubGroup::new(); // Keep up to date with `TESTED_SUBNET_TYPES` constant - par_group = par_group.add_test(systest!($function_name; SubnetType::System)); - par_group = par_group.add_test(systest!($function_name; SubnetType::Application)); - par_group = par_group.add_test(systest!($function_name; SubnetType::VerifiedApplication)); + $group = $group.add_test(systest!($function_name; SubnetType::System)); + $group = $group.add_test(systest!($function_name; SubnetType::Application)); + $group = $group.add_test(systest!($function_name; SubnetType::VerifiedApplication)); // TODO(CON-1696): Remove this condition (and always run the test for cloud engines) when // #9613 reaches mainnet NNS if get_guestos_img_version() == get_guestos_update_img_version() { - par_group = par_group.add_test(systest!($function_name; SubnetType::CloudEngine)); + $group = $group.add_test(systest!($function_name; SubnetType::CloudEngine)); } - - $group = $group.add_parallel(par_group); }; } From bf20059b25a84141fed6575a369948589e778aee Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Wed, 1 Apr 2026 10:16:37 +0000 Subject: [PATCH 64/65] perf: all cases in parallel --- rs/tests/networking/nns_delegation_test.rs | 47 ++++++++++------------ 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 90367dca02ff..63dbd8939f6a 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -51,7 +51,7 @@ use ic_crypto_utils_threshold_sig_der::parse_threshold_sig_key_from_der; use ic_registry_subnet_type::SubnetType; use ic_system_test_driver::{ driver::{ - group::SystemTestGroup, + group::{SystemTestGroup, SystemTestSubGroup}, ic::{InternetComputer, Subnet}, test_env::{HasIcPrepDir, TestEnv}, test_env_api::{ @@ -815,37 +815,32 @@ macro_rules! systest_all_subnet_types { } fn main() -> Result<()> { - let mut test_group = SystemTestGroup::new() - .with_setup(setup) - // We potentially upgrade subnets in the setup which could take several minutes - .with_overall_timeout(std::time::Duration::from_secs(20 * 60)) - .with_timeout_per_test(std::time::Duration::from_secs(10 * 60)); - - systest_all_subnet_types!(test_group, nns_delegation_updates); - systest_all_subnet_types!( - test_group, - canister_read_state_v2_returns_correct_delegation - ); + let mut par_group = SystemTestSubGroup::new(); + systest_all_subnet_types!(par_group, nns_delegation_updates); + systest_all_subnet_types!(par_group, canister_read_state_v2_returns_correct_delegation); + systest_all_subnet_types!(par_group, canister_read_state_v3_returns_correct_delegation); systest_all_subnet_types!( - test_group, - canister_read_state_v3_returns_correct_delegation - ); - systest_all_subnet_types!( - test_group, + par_group, canister_read_state_v3_management_canister_returns_correct_delegation ); - systest_all_subnet_types!(test_group, subnet_read_state_v2_returns_correct_delegation); - systest_all_subnet_types!(test_group, subnet_read_state_v3_returns_correct_delegation); + systest_all_subnet_types!(par_group, subnet_read_state_v2_returns_correct_delegation); + systest_all_subnet_types!(par_group, subnet_read_state_v3_returns_correct_delegation); // note: the v2 call endpoint doesn't return an NNS delegation, so there is nothing to test - systest_all_subnet_types!(test_group, call_v3_returns_correct_delegation); - systest_all_subnet_types!(test_group, call_v4_returns_correct_delegation); + systest_all_subnet_types!(par_group, call_v3_returns_correct_delegation); + systest_all_subnet_types!(par_group, call_v4_returns_correct_delegation); systest_all_subnet_types!( - test_group, + par_group, call_v4_management_canister_returns_correct_delegation ); - systest_all_subnet_types!(test_group, query_v2_passes_correct_delegation_to_canister); - systest_all_subnet_types!(test_group, query_v3_passes_correct_delegation_to_canister); - systest_all_subnet_types!(test_group, interlaced_v2_and_v3_query_requests); + systest_all_subnet_types!(par_group, query_v2_passes_correct_delegation_to_canister); + systest_all_subnet_types!(par_group, query_v3_passes_correct_delegation_to_canister); + systest_all_subnet_types!(par_group, interlaced_v2_and_v3_query_requests); - test_group.execute_from_args() + SystemTestGroup::new() + .with_setup(setup) + // We potentially upgrade subnets in the setup which could take several minutes + .with_overall_timeout(std::time::Duration::from_secs(20 * 60)) + .with_timeout_per_test(std::time::Duration::from_secs(10 * 60)) + .add_parallel(par_group) + .execute_from_args() } From 6ae2dbaf52d536f165f101ef76c83a0d37fb3065 Mon Sep 17 00:00:00 2001 From: Pierugo Pace Date: Wed, 1 Apr 2026 10:27:15 +0000 Subject: [PATCH 65/65] feat: use API BN with valid DNS and TLS --- rs/tests/networking/nns_delegation_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/tests/networking/nns_delegation_test.rs b/rs/tests/networking/nns_delegation_test.rs index 63dbd8939f6a..72398eddbbea 100644 --- a/rs/tests/networking/nns_delegation_test.rs +++ b/rs/tests/networking/nns_delegation_test.rs @@ -132,7 +132,7 @@ fn set_installed_canister_ids(env: &TestEnv, canister_ids: BTreeMap